可以检测certificate pinning和mutual authentication。

common/include/tfe_utils.h

@@ -5,6 +5,7 @@
 
 #pragma once
 #include <MESA/MESA_handle_logger.h>
+#include <MESA/MESA_htable.h>
 #include <time.h>
 #include <dirent.h> //scan_dir
 
@@ -164,3 +165,6 @@ int tfe_scandir(const char *dir, struct dirent ***namelist,
               int(*compar)(const void *, const void *));
 
 const char * tfe_version();
+int __wrapper_MESA_htable_set_opt(MESA_htable_handle table, enum MESA_htable_opt opt_type, unsigned value);
+int __wrapper_MESA_htable_set_opt(MESA_htable_handle table, enum MESA_htable_opt opt_type, void * val, size_t len);
+

common/src/tfe_utils.cpp

@@ -1,9 +1,11 @@
 
+#include <MESA/MESA_htable.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <tfe_utils.h>
 #include <time.h>
+#include <assert.h>
 
 int addr_sock_to_layer(struct sockaddr * sock_addr, int sockaddrlen, struct layer_addr * layer_addr)
 {

platform/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_executable(tfe src/key_keeper.cpp src/kni_acceptor.cpp src/ssl_stream.cpp
-				   src/ssl_sess_cache.cpp src/ssl_trusted_cert_storage.cpp src/ev_root_ca_metadata.cpp
+				   src/ssl_sess_cache.cpp src/ssl_service_cache.cpp
+				   src/ssl_trusted_cert_storage.cpp src/ev_root_ca_metadata.cpp
 		       src/ssl_utils.cpp src/tcp_stream.cpp src/main.cpp src/proxy.cpp)
 
 target_include_directories(tfe PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include/external)

platform/include/internal/ssl_service_cache.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <tfe_types.h>
+#include <ssl_utils.h>
+#include <event2/event.h>
+
+#define PINNING_ST_NOT_PINNING		0
+#define PINNING_ST_PINNING			1
+#define PINNING_ST_MAYBE_PINNING	2
+struct ssl_service_status
+{
+	char pinning_status;
+	char is_ev;
+	char is_ct;
+	char is_mutual_auth;
+};
+
+struct ssl_service_cache;
+struct ssl_service_cache* ssl_service_cache_create(unsigned int slot_size, unsigned int expire_seconds);
+void ssl_service_cache_destroy(struct ssl_service_cache* cache);
+
+int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, struct ssl_service_status* result);
+void ssl_service_cache_write(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct ssl_service_status* status);
+

platform/include/internal/ssl_stream.h

@@ -15,7 +15,7 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
 	void * logger);
 //, ssl_stream_new_cb* new_func, ssl_stream_free_cb* free_func, void* u_para);
 void ssl_manager_destroy(struct ssl_mgr * mgr);
-void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, struct ssl_mgr* mgr);
+unsigned long ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, struct ssl_mgr* mgr);
 
 enum ssl_stream_action
 {

platform/src/ssl_service_cache.cpp

@@ -0,0 +1,247 @@
+#include <ssl_service_cache.h>
+#include <tfe_utils.h>
+#include <MESA/MESA_htable.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define	FAIL_AS_PINNING_COUNT	4
+#define	FAIL_AS_PINNING_TIME	30
+struct ssl_service_client_st
+{
+	time_t last_update_time;
+	unsigned int fail_count;
+	char is_mutual_auth;
+};
+struct ssl_service_server_st
+{
+	char is_ev;
+	char is_ct;
+	long long ev_st_switched;
+	long long ct_st_switched;
+};
+struct ssl_service_cache
+{
+	MESA_htable_handle cli_st_hash;
+	MESA_htable_handle srv_st_hash;
+	long long pinning_cli_cnt, mutual_auth_cli_cnt, ev_srv_cnt, ct_srv_cnt;
+};
+struct ssl_service_write_args
+{
+	struct ssl_service_cache* cache;
+	const struct ssl_service_status* status;
+};
+static size_t ssl_service_client_st_mk_key(const struct ssl_chello* chello, char* key_buff, size_t sz)
+{
+	size_t key_sz=0;
+	key_sz=snprintf(key_buff, sz, "%d.%d-%d.%d:%s:%s:%s:%s", chello->min_version.major, chello->min_version.minor,
+													chello->max_version.major, chello->max_version.minor,
+													chello->sni, chello->alpn, chello->cipher_suites, chello->cipher_suites_tls13);
+	return key_sz;
+}
+static long cli_st_read_cb(void * data, const uchar * key, uint size, void * user_arg)
+{
+	struct ssl_service_client_st* cli_st=(struct ssl_service_client_st*)data;
+	struct ssl_service_status* result=(struct ssl_service_status*)user_arg;
+	
+	if (cli_st == NULL)
+	{
+		return 0;
+	}
+	if(cli_st->fail_count==0)
+	{		
+		result->pinning_status=PINNING_ST_NOT_PINNING;
+	}
+	else if(cli_st->fail_count<FAIL_AS_PINNING_COUNT)
+	{
+		result->pinning_status=PINNING_ST_MAYBE_PINNING;
+	}
+	else
+	{
+		result->pinning_status=PINNING_ST_PINNING;
+	}
+	result->is_mutual_auth=cli_st->is_mutual_auth;
+	return 1;
+}
+static long cli_st_write_cb(void * data, const uchar * key, uint size, void * user_arg)
+{
+	struct ssl_service_client_st* cli_st=(struct ssl_service_client_st*)data;
+	struct ssl_service_write_args* args=(struct ssl_service_write_args*)user_arg;
+	const struct ssl_service_status* status=args->status;
+	struct ssl_service_cache* cache=args->cache;
+	UNUSED int ret = 0;
+	time_t now=time(NULL);
+	if(cli_st==NULL)
+	{
+		cli_st=ALLOC(struct ssl_service_client_st, 1);
+		ret = MESA_htable_add(cache->cli_st_hash, key, size, cli_st);
+		assert(ret >= 0);
+	}
+	if(cli_st->fail_count<FAIL_AS_PINNING_COUNT && cli_st->last_update_time-now>FAIL_AS_PINNING_TIME)
+	{
+		cli_st->fail_count=0;
+	}
+	if(status->pinning_status!=PINNING_ST_NOT_PINNING && cli_st->fail_count<FAIL_AS_PINNING_COUNT)
+	{
+		if(status->pinning_status==PINNING_ST_PINNING)
+		{
+			cli_st->fail_count=FAIL_AS_PINNING_COUNT;
+		}
+		else
+		{
+			cli_st->fail_count++;
+		}
+		cli_st->last_update_time=now;
+		if(cli_st->fail_count==FAIL_AS_PINNING_COUNT)
+		{
+			cache->pinning_cli_cnt++;
+		}
+	}
+	else if(status->pinning_status==PINNING_ST_PINNING)
+	{
+		cli_st->fail_count=FAIL_AS_PINNING_COUNT;
+		cli_st->last_update_time=now;
+	}
+
+	if(status->is_mutual_auth==1&&cli_st->is_mutual_auth==0)
+	{
+		cache->mutual_auth_cli_cnt++;	
+		cli_st->is_mutual_auth=1;
+	}
+	return 1;
+}
+
+static long srv_st_read_cb(void * data, const uchar * key, uint size, void * user_arg)
+{
+	struct ssl_service_server_st* srv_st=(struct ssl_service_server_st*)data;
+	struct ssl_service_status* result=(struct ssl_service_status*)user_arg;
+	if (srv_st == NULL)
+	{
+		return 0;
+	}
+	result->is_ct=srv_st->is_ct;
+	result->is_ev=srv_st->is_ev;
+	return 1;
+}
+static long srv_st_write_cb(void * data, const uchar * key, uint size, void * user_arg)
+{
+	struct ssl_service_server_st* srv_st=(struct ssl_service_server_st*)data;
+	struct ssl_service_write_args* args=(struct ssl_service_write_args*)user_arg;
+	const struct ssl_service_status* status=args->status;
+	struct ssl_service_cache* cache=args->cache;
+	UNUSED int ret = 0;
+	if(srv_st==NULL)
+	{
+		srv_st=ALLOC(struct ssl_service_server_st, 1);
+		ret = MESA_htable_add(cache->srv_st_hash, key, size, srv_st);
+		assert(ret >= 0);
+	}
+	if(status->is_ev==1&&srv_st->is_ev==0)
+	{
+		srv_st->is_ev=1;
+		cache->ev_srv_cnt++;
+	}
+	if(status->is_ev!=srv_st->is_ev)
+	{
+		srv_st->ev_st_switched++;
+	}
+	if(status->is_ct==1&&srv_st->is_ct==0)
+	{
+		srv_st->is_ct=1;
+		cache->ct_srv_cnt++;
+	}
+	if(status->is_ct!=srv_st->is_ct)
+	{
+		srv_st->ct_st_switched++;
+	}
+	assert(srv_st->ev_st_switched<2&&srv_st->ct_st_switched<2);
+	return 1;
+}
+
+int ssl_service_cache_read(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, struct ssl_service_status* result)
+{
+	long cli_st_cb_ret=0, svr_st_cb_ret=0;
+	char cli_st_key[2048];
+	size_t cli_st_key_sz=0;
+	if(chello->sni==NULL)
+	{
+		return 0;
+	}
+	cli_st_key_sz=ssl_service_client_st_mk_key(chello, cli_st_key, sizeof(cli_st_key));
+	MESA_htable_search_cb(svc_cache->cli_st_hash, (unsigned char*) cli_st_key, (unsigned int) cli_st_key_sz, cli_st_read_cb, result, &cli_st_cb_ret);	
+	MESA_htable_search_cb(svc_cache->srv_st_hash, (unsigned char*) chello->sni, (unsigned int) strlen(chello->sni), srv_st_read_cb, result, &svr_st_cb_ret);
+	if(cli_st_cb_ret||svr_st_cb_ret)
+	{
+		return 1;
+	}
+	else
+	{
+		return 0;
+	}
+}
+
+void ssl_service_cache_write(struct ssl_service_cache* svc_cache, const struct ssl_chello* chello, const struct ssl_service_status* status)
+{
+	long cli_st_cb_ret=0, svr_st_cb_ret=0;
+	char cli_st_key[2048];
+	size_t cli_st_key_sz=0;
+	if(chello->sni==NULL)
+	{
+		return;
+	}
+	struct ssl_service_write_args write_args={svc_cache, status};
+	if(status->is_mutual_auth||status->pinning_status!=PINNING_ST_NOT_PINNING)
+	{
+		cli_st_key_sz=ssl_service_client_st_mk_key(chello, cli_st_key, sizeof(cli_st_key));
+		MESA_htable_search_cb(svc_cache->cli_st_hash, (unsigned char*)cli_st_key, (unsigned int) cli_st_key_sz, cli_st_write_cb, &write_args, &cli_st_cb_ret);	
+	}
+	if(status->is_ct||status->is_ev)
+	{
+		MESA_htable_search_cb(svc_cache->srv_st_hash, (unsigned char*)chello->sni, (unsigned int) strlen(chello->sni), srv_st_write_cb, &write_args, &svr_st_cb_ret);
+	}
+}
+struct ssl_service_cache* ssl_service_cache_create(unsigned int slot_size, unsigned int expire_seconds)
+{
+	struct ssl_service_cache * cache = ALLOC(struct ssl_service_cache, 1);
+	unsigned max_num = slot_size * 4;
+	UNUSED int ret = 0;
+	MESA_htable_handle htable=NULL, saved[2];
+	int i=0, opt_val=0;
+	for(i=0; i<2; i++)
+	{
+		htable = MESA_htable_born();
+		opt_val=0;
+		ret = MESA_htable_set_opt(htable, MHO_SCREEN_PRINT_CTRL, &opt_val, sizeof(opt_val));
+		opt_val=1;
+		ret = MESA_htable_set_opt(htable, MHO_THREAD_SAFE, &opt_val, sizeof(opt_val));
+		opt_val=16;
+		ret = MESA_htable_set_opt(htable, MHO_MUTEX_NUM, &opt_val, sizeof(opt_val));
+		ret = MESA_htable_set_opt(htable, MHO_HASH_SLOT_SIZE, &slot_size, sizeof(slot_size));
+		ret = MESA_htable_set_opt(htable, MHO_HASH_MAX_ELEMENT_NUM, &max_num, sizeof(max_num));
+		ret = MESA_htable_set_opt(htable, MHO_EXPIRE_TIME, &expire_seconds, sizeof(expire_seconds));
+		
+		opt_val=HASH_ELIMINATE_ALGO_LRU;
+		ret = MESA_htable_set_opt(htable, MHO_ELIMIMINATE_TYPE,
+			&opt_val, sizeof(int));
+		ret = MESA_htable_set_opt(htable, MHO_CBFUN_DATA_FREE,
+			(void *)free, sizeof(&free));
+		
+		ret = MESA_htable_mature(htable);
+		assert(ret == 0);
+		saved[i]=htable;
+	}
+	cache->cli_st_hash=saved[0];
+	cache->srv_st_hash=saved[1];
+	
+	return cache;
+}
+void ssl_service_cache_destroy(struct ssl_service_cache* cache)
+{
+	MESA_htable_destroy(cache->cli_st_hash, NULL);
+	cache->cli_st_hash=NULL;
+	MESA_htable_destroy(cache->srv_st_hash, NULL);
+	cache->srv_st_hash=NULL;
+	free(cache);
+	return;
+}
+
+

platform/src/ssl_service_cache.h

@@ -1,22 +0,0 @@
-#pragma once
-
-#include <tfe_types.h>
-#include <ssl_utils.h>
-#include <event2/event.h>
-
-#define PINNING_ST_NOT_PINNING		0
-#define PINNING_ST_PINNING			1
-#define PINNING_ST_MAYBE_PINNING	2
-struct ssl_service_desc
-{
-	char pinning_status;
-	char is_ev;
-	char is_ct;
-};
-
-struct ssl_service_cache;
-struct ssl_service_cache* ssl_service_cache_init();
-
-int ssl_service_cache_read(struct ssl_chello* chello,  struct ssl_service_desc* result);
-int ssl_service_cache_log_fail(struct ssl_chello* chello);
-

platform/src/ssl_sess_cache.cpp

@@ -281,41 +281,30 @@ void down_session_del(struct sess_cache * cache, const SSL_SESSION * sess)
 	}
 	return;
 }
-
-int __wrapper_MESA_htable_set_opt(MESA_htable_handle table, enum MESA_htable_opt opt_type, unsigned value)
-{
-	int ret = MESA_htable_set_opt(table, opt_type, &value, (int)(sizeof(value)));
-	assert(ret == 0);
-	return ret;
-}
-
-int __wrapper_MESA_htable_set_opt(MESA_htable_handle table, enum MESA_htable_opt opt_type, void * val, size_t len)
-{
-	int ret = MESA_htable_set_opt(table, opt_type, val, (int)len);
-	assert(ret == 0);
-	return ret;
-}
-
 struct sess_cache * ssl_sess_cache_create(unsigned int slot_size, unsigned int expire_seconds, enum tfe_conn_dir served)
 {
 	struct sess_cache * cache = ALLOC(struct sess_cache, 1);
 	unsigned max_num = slot_size * 4;
 	UNUSED int ret = 0;
+	int opt_val=0;
 	
 	MESA_htable_handle htable = MESA_htable_born();
-	ret = __wrapper_MESA_htable_set_opt(htable, MHO_SCREEN_PRINT_CTRL, 0);
+	opt_val=0;
-	ret = __wrapper_MESA_htable_set_opt(htable, MHO_THREAD_SAFE, 1);
+	ret = MESA_htable_set_opt(htable, MHO_SCREEN_PRINT_CTRL, &opt_val, sizeof(opt_val));
+	opt_val=1;
+	ret = MESA_htable_set_opt(htable, MHO_THREAD_SAFE, &opt_val, sizeof(opt_val));
+	opt_val=16;
+	ret = MESA_htable_set_opt(htable, MHO_MUTEX_NUM, &opt_val, sizeof(opt_val));
+	ret = MESA_htable_set_opt(htable, MHO_HASH_SLOT_SIZE, &slot_size, sizeof(slot_size));
+	ret = MESA_htable_set_opt(htable, MHO_HASH_MAX_ELEMENT_NUM, &max_num, sizeof(max_num));
+	ret = MESA_htable_set_opt(htable, MHO_EXPIRE_TIME, &expire_seconds, sizeof(expire_seconds));
 
-	ret = __wrapper_MESA_htable_set_opt(htable, MHO_MUTEX_NUM, 16);
+	opt_val=HASH_ELIMINATE_ALGO_FIFO;
-	ret = __wrapper_MESA_htable_set_opt(htable, MHO_HASH_SLOT_SIZE, slot_size);
+	ret = MESA_htable_set_opt(htable, MHO_ELIMIMINATE_TYPE,
-	ret = __wrapper_MESA_htable_set_opt(htable, MHO_HASH_MAX_ELEMENT_NUM, max_num);
+		&opt_val, sizeof(int));
-	ret = __wrapper_MESA_htable_set_opt(htable, MHO_EXPIRE_TIME, expire_seconds);
+	ret = MESA_htable_set_opt(htable, MHO_CBFUN_DATA_FREE,
-
-	ret = __wrapper_MESA_htable_set_opt(htable, MHO_ELIMIMINATE_TYPE,
-		HASH_ELIMINATE_ALGO_FIFO);
-	ret = __wrapper_MESA_htable_set_opt(htable, MHO_CBFUN_DATA_FREE,
 		(void *)ssl_sess_free_serialized, sizeof(&ssl_sess_free_serialized));
-	ret = __wrapper_MESA_htable_set_opt(htable, MHO_CBFUN_DATA_EXPIRE_NOTIFY,
+	ret = MESA_htable_set_opt(htable, MHO_CBFUN_DATA_EXPIRE_NOTIFY,
 		(void *)ssl_sess_verify_cb, sizeof(&ssl_sess_verify_cb));
 
 	ret = MESA_htable_mature(htable);

platform/src/ssl_stream.cpp

@@ -39,9 +39,11 @@
 #include <ssl_sess_cache.h>
 #include <ssl_trusted_cert_storage.h>
 #include <ssl_utils.h>
+#include <ssl_service_cache.h>
 #include <platform.h>
 
-static int SSL_EX_DATA_IDX_SSLMGR;
+static int SSL_CTX_EX_DATA_IDX_SSLMGR;
+static int SSL_EX_DATA_IDX_SSLSTREAM;
 
 #define MAX_NET_RETRIES        50
 #define	LATENCY_WARNING_THRESHOLD_MS		1000
@@ -131,6 +133,7 @@ struct ssl_mgr
 
 	struct sess_cache * down_sess_cache;
 	struct sess_cache * up_sess_cache;
+	struct ssl_service_cache* svc_cache;
 	struct session_ticket_key ticket_key;
 
 	char default_ciphers[TFE_SYMBOL_MAX];
@@ -175,6 +178,7 @@ struct ssl_upstream_parts
 	struct cert_verify_param verify_param;
 	struct cert_verify_result verify_result;
 	char verify_failed_action;
+	struct ssl_service_status svc_status;
 	
 	struct ssl_bypass bypass_condition;
 	struct ssl_chello * client_hello;
@@ -195,6 +199,7 @@ struct ssl_stream
 		struct ssl_downstream_parts down_parts;
 	};
 	const unsigned char* alpn_selected;	//reference to SSL_ALPN_HTTP_2/SSL_ALPN_HTTP_1_1
+	struct ssl_stream* peer;
 	struct __ssl_stream_debug _do_not_use;
 };
 
@@ -588,7 +593,8 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
 	}
 	//tfe2a uses SSLv23_method, it was been deprecated and replaced with the TLS_method() in openssl 1.1.0.
 	mgr->sslmethod = TLS_method;
-    SSL_EX_DATA_IDX_SSLMGR   = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL);
+    SSL_CTX_EX_DATA_IDX_SSLMGR = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL);
+	SSL_EX_DATA_IDX_SSLSTREAM  = SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL);
 	MESA_load_profile_uint_def(ini_profile, section, "ssl_compression", &(mgr->sslcomp), 1);
 	MESA_load_profile_uint_def(ini_profile, section, "no_ssl2", &(mgr->no_ssl2), 1);
 	MESA_load_profile_uint_def(ini_profile, section, "no_ssl3", &(mgr->no_ssl3), 1);
@@ -615,6 +621,7 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
 		mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM);
 		mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM);
 	}
+	mgr->svc_cache=ssl_service_cache_create(mgr->cache_slots, mgr->sess_expire_seconds);
 
 	//Reference to NGINX: http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_ticket_key
 	//Support key rotation in futher.
@@ -780,6 +787,14 @@ static void ssl_async_peek_client_hello(struct future * f, evutil_socket_t fd, i
 	event_add(ctx->ev, NULL);
 	return;
 }
+int ossl_client_cert_cb(SSL *ssl, X509 **x509, EVP_PKEY **pkey)
+{
+	struct ssl_stream* s_upstream=NULL;
+	s_upstream=(struct ssl_stream*)SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM);
+	s_upstream->up_parts.svc_status.is_mutual_auth=1;
+	ssl_service_cache_write(s_upstream->mgr->svc_cache, s_upstream->up_parts.client_hello, &s_upstream->up_parts.svc_status);
+	return 0;
+}
 
 /*
  * Create new SSL context for outgoing connections to the original destination.
@@ -818,13 +833,14 @@ static SSL * upstream_ssl_create(struct ssl_mgr * mgr, struct ssl_stream* s_stre
 	}
 	
 	SSL_CTX_set_verify(sslctx, SSL_VERIFY_NONE, NULL);
+	SSL_CTX_set_client_cert_cb(sslctx, ossl_client_cert_cb);
 	ssl = SSL_new(sslctx);
 	SSL_CTX_free(sslctx);  /* SSL_new() increments refcount */
-
 	if (!ssl)
 	{
 		return NULL;
 	}
+	SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream);
 
 	if (chello->sni)
 	{
@@ -910,9 +926,9 @@ const char* ssl_stream_dump_info(struct ssl_stream *stream, char* buffer, size_t
 								stream->dir==CONN_DIR_UPSTREAM ? stream->up_parts.client_hello->sni:NULL);
 	return buffer;
 }
-void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, struct ssl_mgr* mgr)
+unsigned long ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, struct ssl_mgr* mgr)
 {
-	unsigned long sslerr=0;
+	unsigned long sslerr=0, ret_sslerr=0;
 	int fd=bufferevent_getfd(bev);
 	char* addr_string=tfe_string_addr_create_by_fd(fd, dir);
 	void* logger=mgr->logger;
@@ -937,6 +953,7 @@ void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, struc
 		default:
 			fs_id=-1;
 	}
+	ret_sslerr=ERR_GET_REASON(sslerr);
 	if(fs_id>=0)
 	{
 		mgr->stat_val[fs_id]++;
@@ -1015,7 +1032,7 @@ void ssl_stream_log_error(struct bufferevent * bev, enum tfe_conn_dir dir, struc
 		}
 	}
 	free(addr_string);
-
+	return ret_sslerr;
 }
 
 /*
@@ -1080,6 +1097,9 @@ static void ssl_server_connected_eventcb(struct bufferevent * bev, short events,
 				s_stream->up_parts.is_server_cert_verify_passed = ssl_trusted_cert_storage_verify_conn(s_stream->mgr->trust_CA_store, 
 													s_stream->ssl, s_stream->up_parts.client_hello->sni, &(s_stream->up_parts.verify_param),
 													error_str, sizeof(error_str), &(s_stream->up_parts.verify_result));
+				s_stream->up_parts.svc_status.is_ct=s_stream->up_parts.verify_result.is_ct;
+				s_stream->up_parts.svc_status.is_ev=s_stream->up_parts.verify_result.is_ev;
+				ssl_service_cache_write(mgr->svc_cache, s_stream->up_parts.client_hello, &(s_stream->up_parts.svc_status));
 				TFE_LOG_DEBUG(mgr->logger, "SNI: %s hostmatch:%d, ct:%d, ev:%d",
 											s_upstream->client_hello->sni,
 											s_stream->up_parts.verify_result.is_hostmatched,
@@ -1149,8 +1169,21 @@ static void peek_chello_on_succ(future_result_t * result, void * user)
 	{
 		ATOMIC_INC(&(ctx->mgr->stat_val[SSL_NO_SNI]));
 	}
+	int ret=0;
+	struct ssl_service_status* svc_status=NULL;
 	clock_gettime(CLOCK_MONOTONIC, &(ctx->start));
 	ctx->s_stream = ssl_stream_new(ctx->mgr, ctx->fd_upstream, CONN_DIR_UPSTREAM, chello, NULL, NULL);
+	svc_status=&ctx->s_stream->up_parts.svc_status;
+	ret=ssl_service_cache_read(ctx->mgr->svc_cache, chello, svc_status);
+	if(ret==1)
+	{
+		TFE_LOG_DEBUG(ctx->mgr->logger, "SNI: %s service status pinning:%d, mauth:%d, ct:%d, ev:%d",
+									chello->sni,
+									svc_status->pinning_status,
+									svc_status->is_mutual_auth,
+									svc_status->is_ct,
+									svc_status->is_ev);
+	}
 	ctx->bev = bufferevent_openssl_socket_new(evbase, ctx->fd_upstream,
 		ctx->s_stream->ssl, BUFFEREVENT_SSL_CONNECTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE );
 
@@ -1207,7 +1240,7 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn,
     const EVP_CIPHER *cipher=EVP_aes_256_cbc();
 	size_t size=32;
 	
-	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl_conn, SSL_EX_DATA_IDX_SSLMGR);
+	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl_conn, SSL_CTX_EX_DATA_IDX_SSLMGR);
 	assert(mgr!=NULL);
 	
 	struct session_ticket_key* key=&(mgr->ticket_key);
@@ -1285,7 +1318,7 @@ static int ossl_session_ticket_key_callback(SSL *ssl_conn,
  */
 static int ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess)
 {
-	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR);
+	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_CTX_EX_DATA_IDX_SSLMGR);
 
 #ifdef HAVE_SSLV2
 
@@ -1312,7 +1345,7 @@ static int ossl_sessnew_cb(SSL * ssl, SSL_SESSION * sess)
  */
 static void ossl_sessremove_cb(SSL_CTX * sslctx, SSL_SESSION * sess)
 {
-	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_CTX_get_ex_data(sslctx, SSL_EX_DATA_IDX_SSLMGR);
+	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_CTX_get_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR);
 	assert(mgr != NULL);
 
 	if (sess && !mgr->no_sesscache)
@@ -1329,7 +1362,7 @@ static void ossl_sessremove_cb(SSL_CTX * sslctx, SSL_SESSION * sess)
  */
 static SSL_SESSION * ossl_sessget_cb(SSL * ssl, const unsigned char * id, int idlen, int * copy)
 {
-	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR);
+	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_CTX_EX_DATA_IDX_SSLMGR);
 	SSL_SESSION * sess=NULL;
 	if(!mgr->no_sesscache)
 	{
@@ -1447,7 +1480,7 @@ static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt, c
 	SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL);
 	SSL_CTX_set_session_id_context(sslctx, (const unsigned char *) mgr->ssl_session_context,
 		sizeof(mgr->ssl_session_context));
-	ret = SSL_CTX_set_ex_data(sslctx, SSL_EX_DATA_IDX_SSLMGR, mgr);
+	ret = SSL_CTX_set_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR, mgr);
 	assert(ret == 1);
 	if (mgr->dh)
 	{
@@ -1488,7 +1521,7 @@ static SSL * downstream_ssl_create(struct ssl_mgr * mgr, struct keyring * crt, c
 	SSL_CTX_free(sslctx);            // SSL_new() increments refcount
 	sslctx = NULL;
 
-	ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLMGR, mgr);
+	ret = SSL_set_ex_data(ssl, SSL_CTX_EX_DATA_IDX_SSLMGR, mgr);
 	assert(ret == 1);
 
 	if (mgr->ssl_mode_release_buffers == 1)
@@ -1555,16 +1588,24 @@ static void ssl_client_connected_eventcb(struct bufferevent * bev, short events,
 	const char* sni=s_upstream->client_hello->sni?s_upstream->client_hello->sni:"null";
 	char error_str[TFE_STRING_MAX]={0};
 	long jiffies_ms=0;	
+	unsigned long sslerr=0;
 	if (events & BEV_EVENT_ERROR)
 	{
 		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));
-		ssl_stream_log_error(bev, CONN_DIR_DOWNSTREAM, mgr);
+		sslerr=ssl_stream_log_error(bev, CONN_DIR_DOWNSTREAM, mgr);
+		if(sslerr==SSL_R_SSLV3_ALERT_CERTIFICATE_UNKNOWN)
+		{
+			s_upstream->svc_status.pinning_status=PINNING_ST_PINNING;
+			ssl_service_cache_write(mgr->svc_cache, s_upstream->client_hello, &s_upstream->svc_status);
+		}
 		snprintf(error_str, sizeof(error_str), "connect to client failed : sni=%s", sni);
 		promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str);
 	}
 	else if(events & BEV_EVENT_EOF)
 	{		
 		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_ERR]));	
+		s_stream->peer->up_parts.svc_status.pinning_status=PINNING_ST_MAYBE_PINNING;
+		ssl_service_cache_write(mgr->svc_cache, s_stream->peer->up_parts.client_hello, &(s_stream->peer->up_parts.svc_status));
 		snprintf(error_str, sizeof(error_str), "client side closed : sni=%s", sni);
 		promise_failed(p, FUTURE_ERROR_EXCEPTION, error_str);
 	}
@@ -1613,6 +1654,7 @@ void ask_keyring_on_succ(void * result, void * user)
 	clock_gettime(CLOCK_MONOTONIC, &(ctx->start));
 	ctx->downstream = ssl_stream_new(mgr, ctx->fd_downstream, CONN_DIR_DOWNSTREAM, NULL, kyr,
 										ctx->origin_ssl?ctx->origin_ssl->alpn_selected:NULL);
+	ctx->downstream->peer=ctx->origin_ssl;
 	ctx->bev_down = bufferevent_openssl_socket_new(evbase, ctx->fd_downstream, ctx->downstream->ssl,
 		BUFFEREVENT_SSL_ACCEPTING, BEV_OPT_DEFER_CALLBACKS | BEV_OPT_THREADSAFE);
 	bufferevent_openssl_set_allow_dirty_shutdown(ctx->bev_down, 1);