手工合并李杰stek-roation分支，支持session ticket rotation，详见 #123

platform/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_executable(tfe src/key_keeper.cpp src/kni_acceptor.cpp src/ssl_stream.cpp
-				   src/ssl_sess_cache.cpp src/ssl_service_cache.cpp
+				   src/ssl_sess_cache.cpp src/ssl_sess_ticket.cpp src/ssl_service_cache.cpp
 				   src/ssl_trusted_cert_storage.cpp src/ev_root_ca_metadata.cpp src/ssl_utils.cpp
 		      src/tcp_stream.cpp src/main.cpp src/proxy.cpp)
 

platform/include/internal/ssl_sess_ticket.h

@@ -0,0 +1,22 @@
+#pragma once
+#include <event2/event.h>
+
+struct sess_ticket_key
+{
+    size_t size;
+    unsigned char name[16];
+    unsigned char hmac_key[32];
+    unsigned char aes_key[32];
+};
+enum STEK_GET_RET
+{
+	STEK_ERROR=-1,
+	STEK_NOT_FOUND=0,
+	STEK_FOUND_FRESH=1,
+	STEK_FOUND_STALED=2
+};
+struct sess_ticket_box;
+struct sess_ticket_box * sess_ticket_box_create(struct event_base * ev_base, unsigned int stek_group_num, unsigned int round_time, void * logger);
+void sess_ticket_box_get_key_for_enc(struct sess_ticket_box * box, const char* sni, struct sess_ticket_key *result);
+enum STEK_GET_RET sess_ticket_box_get_key_for_dec(struct sess_ticket_box * box, const char* sni, const unsigned char* key_name, struct sess_ticket_key *result);
+

platform/src/ssl_sess_ticket.cpp

@@ -0,0 +1,194 @@
+/*
+ *	SSL session ticket rotation implementation.
+ *	Author: lijie2@iie.ac.cn (graduate student of 2019), zhengchao@iie.ac.cn
+ *	Create: 2019-5-24
+ */
+
+#include <time.h>
+#include <tfe_utils.h>
+#include <ssl_sess_ticket.h>
+#include <openssl/ssl.h>
+#include <pthread.h>
+
+
+#define STEK_NUM 2
+#define STEK_SIZE 80
+#define SEED_MAX_LEN 100
+#define RAND_MAX_VALUE 256
+
+static unsigned int murmurhash2(const void * key, int len, const unsigned int seed)
+{
+	// 'm' and 'r' are mixing constants generated offline.
+	// They're not really 'magic', they just happen to work well.
+
+	const unsigned int m = 0x5bd1e995;
+	const int r = 24;
+
+	// Initialize the hash to a 'random' value
+
+	unsigned int h = seed ^ len;
+
+	// Mix 4 bytes at a time into the hash
+
+	const unsigned char * data = (const unsigned char *)key;
+
+	while(len >= 4)
+	{
+		unsigned int k = *(unsigned int *)data;
+
+		k *= m;
+		k ^= k >> r;
+		k *= m;
+
+		h *= m;
+		h ^= k;
+
+		data += 4;
+		len -= 4;
+	}
+
+	// Handle the last few bytes of the input array
+
+	switch(len)
+	{
+	case 3: h ^= data[2] << 16;
+	case 2: h ^= data[1] << 8;
+	case 1: h ^= data[0];
+	        h *= m;
+	};
+
+	// Do a few final mixes of the hash to ensure the last few
+	// bytes are well-incorporated.
+
+	h ^= h >> 13;
+	h *= m;
+	h ^= h >> 15;
+
+	return h;
+}
+
+struct sess_ticket_box
+{
+    unsigned int ticket_group_num;             // Number of STEK group used simultaneously
+    unsigned int stek_rotation_seconds;
+	
+    struct sess_ticket_key **ticket_keys;
+    pthread_rwlock_t stek_rwlock;
+    struct event * stek_rotation_cb;
+    struct event_base * ref_ev_base;
+};
+static void set_stek_rand_seed(unsigned int stek_rotation_time)
+{
+    time_t seed;
+    time(&seed);
+    seed = (seed/stek_rotation_time);
+    srand(seed);
+}
+
+static void stek_key_reset(struct sess_ticket_key * stek)
+{
+    int i = 0;
+    unsigned char buf[STEK_SIZE];
+    memset(buf,0,sizeof(buf));
+    for(i = 0;i < STEK_SIZE; i++)
+    {
+        buf[i]=rand()%RAND_MAX_VALUE;
+    }
+    stek->size = STEK_SIZE;
+    memcpy(stek->name, buf, 16);
+    memcpy(stek->hmac_key, buf+16, 32);
+    memcpy(stek->aes_key, buf+48, 32);
+}
+
+static void ssl_stek_rotation_cb(evutil_socket_t fd, short what, void * arg)
+{
+    unsigned int i = 0;
+    struct sess_ticket_box * ticket = (struct sess_ticket_box *) arg;
+    set_stek_rand_seed(ticket->stek_rotation_seconds);
+    pthread_rwlock_wrlock(&(ticket->stek_rwlock));
+    for(i = 0; i < ticket->ticket_group_num; i ++)
+    {
+        memcpy(&(ticket->ticket_keys[i][1]), &(ticket->ticket_keys[i][0]), STEK_SIZE);
+        stek_key_reset(&(ticket->ticket_keys[i][0]));
+     }
+     pthread_rwlock_unlock(&(ticket->stek_rwlock));
+    //update the stek rotation time
+    struct timeval stek_time = {ticket->stek_rotation_seconds, 0};
+    evtimer_add(ticket->stek_rotation_cb, &stek_time);
+}
+
+struct sess_ticket_box * sess_ticket_box_create(struct event_base * ev_base, unsigned int stek_group_num, unsigned int round_time, void * logger)
+{
+    UNUSED int ret = 0;
+    unsigned int i = 0;
+    struct sess_ticket_box *ticket = ALLOC(struct sess_ticket_box,1);
+    if(pthread_rwlock_init(&(ticket->stek_rwlock),NULL) != 0)
+    {
+        return NULL;
+    }
+    ticket->ticket_group_num = stek_group_num ;
+    ticket->stek_rotation_seconds = round_time;
+    ticket->ticket_keys = ALLOC(struct sess_ticket_key*,stek_group_num);    
+    set_stek_rand_seed(round_time);
+    pthread_rwlock_wrlock(&(ticket->stek_rwlock));
+    for(i = 0; i < stek_group_num; i++)
+    {
+        ticket->ticket_keys[i] = ALLOC(struct sess_ticket_key, STEK_NUM);
+        stek_key_reset(&(ticket->ticket_keys[i][0]));
+    }
+    pthread_rwlock_unlock(&(ticket->stek_rwlock));
+   
+    time_t now;
+    time(&now);
+	unsigned int till_first_rotation = round_time - now%round_time;
+    struct timeval stek_first_rotate_time = {till_first_rotation, 0};
+    ticket->stek_rotation_cb = event_new(ev_base, -1, 0, ssl_stek_rotation_cb, ticket);   
+    evtimer_add(ticket->stek_rotation_cb, &stek_first_rotate_time);
+    return ticket;
+}
+static int stek_get_idx_by_sni(const char* sni, int group_num)
+{
+	unsigned int stek_index = 0;
+    if(sni != NULL)
+    {
+        stek_index = murmurhash2(sni, strlen(sni), 1021);
+        stek_index = stek_index % group_num; 
+    }
+	return stek_index;
+
+}
+void sess_ticket_box_get_key_for_enc(struct sess_ticket_box * box, const char* sni, struct sess_ticket_key *result)
+{
+  
+    unsigned int stek_index = 0;
+	
+    pthread_rwlock_rdlock(&(box->stek_rwlock));
+	stek_index=stek_get_idx_by_sni(sni, box->ticket_group_num);	
+	*result=box->ticket_keys[stek_index][0];
+    pthread_rwlock_unlock(&(box->stek_rwlock)); 
+	return;
+}
+enum STEK_GET_RET sess_ticket_box_get_key_for_dec(struct sess_ticket_box * box, const char* sni, const unsigned char* key_name, struct sess_ticket_key *result)
+{
+    unsigned int stek_index = 0;
+	enum STEK_GET_RET ret=STEK_NOT_FOUND;
+	struct sess_ticket_key** steks=NULL;
+    pthread_rwlock_rdlock(&(box->stek_rwlock));
+	steks=box->ticket_keys;
+	stek_index=stek_get_idx_by_sni(sni, box->ticket_group_num);
+	
+	if (memcmp(key_name, steks[stek_index][0].name, 16) == 0)
+	{
+		*result=steks[stek_index][0];
+		ret = STEK_FOUND_FRESH;
+	}
+	else if(memcmp(key_name, steks[stek_index][1].name, 16) == 0)
+	{
+		*result=steks[stek_index][1];
+		ret = STEK_FOUND_STALED;
+	}
+	
+    pthread_rwlock_unlock(&(box->stek_rwlock));
+	return ret;
+}
+

platform/src/ssl_stream.cpp

@@ -37,6 +37,7 @@
 #include <tfe_proxy.h>
 #include <key_keeper.h>
 #include <ssl_sess_cache.h>
+#include <ssl_sess_ticket.h>
 #include <ssl_trusted_cert_storage.h>
 #include <ssl_utils.h>
 #include <ssl_service_cache.h>
@@ -106,13 +107,6 @@ enum ssl_stream_stat
 	
 	SSL_STAT_MAX
 };
-struct session_ticket_key
-{
-	size_t size;
-	unsigned char name[16];
-	unsigned char hmac_key[32];
-	unsigned char aes_key[32];
-};
 
 struct ssl_mgr
 {
@@ -137,10 +131,12 @@ struct ssl_mgr
 
 	struct sess_cache * down_sess_cache;
 	struct sess_cache * up_sess_cache;
+	struct sess_ticket_box * down_stek_box;
+	
 	struct ssl_service_cache* svc_cache;
 	ssl_stream_new_hook* on_new_upstream_cb;
 	void* upstream_cb_param;
-	struct session_ticket_key ticket_key;
+	struct sess_ticket_key ticket_key;
 
 	char default_ciphers[TFE_SYMBOL_MAX];
 	DH * dh;
@@ -589,11 +585,9 @@ void ssl_manager_destroy(struct ssl_mgr * mgr)
 struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section, 
 	struct event_base * ev_base_gc, void * logger)
 {
-	unsigned char key_name[]="!mesalab-tfe3a~&";
+    unsigned int stek_group_num = 0;
-	unsigned char aes_key_def[]={0xC5,0xAC,0xC1,0xA6,0xB2,0xBB,0xCA,0xC7,0xE3,0xBE,0xE3,0xB2,0xC6,0xA3,0xB1,0xB9
+    unsigned int stek_rotation_time = 0;
-						,0xA3,0xAC,0xB6,0xF8,0xCA,0xC7,0xD1,0xDB,0xBE,0xA6,0xC0,0xEF,0xD3,0xD0,0xB9,0x84};
+
-	unsigned char hmac_key_def[]={0xD6,0xC8,0xD0,0xF2,0xC8,0xCB,0xBF,0xDA,0xBA,0xCD,0xBF,0xC6,0xBC,0xBC,0xA3,0xAC
-						,0xC8,0xCB,0xC0,0xE0,0xC9,0xE7,0xBB,0xE1,0xB5,0xC4,0xBB,0xF9,0xCA,0xAF,0x19,0x84};
 
 	struct ssl_mgr * mgr = ALLOC(struct ssl_mgr, 1);
 	int ret = 0;
@@ -649,20 +643,24 @@ struct ssl_mgr * ssl_manager_init(const char * ini_profile, const char * section
 		&(mgr->cache_slots), 4 * 1024 * 1024);
 	MESA_load_profile_uint_def(ini_profile, section, "session_cache_expire_seconds",
 		&(mgr->sess_expire_seconds), 30 * 60);
+	
 
 	if(!mgr->no_sesscache)
 	{
 		mgr->up_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_UPSTREAM);
 		mgr->down_sess_cache = ssl_sess_cache_create(mgr->cache_slots, mgr->sess_expire_seconds, CONN_DIR_DOWNSTREAM);
 	}
+	
+    MESA_load_profile_uint_def(ini_profile, section, "stek_group_num", &stek_group_num, 1);
+    MESA_load_profile_uint_def(ini_profile, section, "stek_rotation_time", &stek_rotation_time, 3600);
+
+	if(!mgr->no_sessticket)
+    {
+        mgr->down_stek_box = sess_ticket_box_create(ev_base_gc, stek_group_num, stek_rotation_time, logger);
+    }
+
 	mgr->svc_cache=ssl_service_cache_create(mgr->cache_slots, mgr->sess_expire_seconds);
 
-	//Reference to NGINX: http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_session_ticket_key
-	//Support key rotation in futher.
-	
-	memcpy(mgr->ticket_key.name,key_name, sizeof(mgr->ticket_key.name));
-	memcpy(mgr->ticket_key.aes_key, aes_key_def, sizeof(mgr->ticket_key.aes_key));
-	memcpy(mgr->ticket_key.hmac_key, hmac_key_def, sizeof(mgr->ticket_key.hmac_key));
 	mgr->key_keeper = key_keeper_init(ini_profile, "key_keeper", logger);
 	if (mgr->key_keeper == NULL)
 	{
@@ -1310,80 +1308,84 @@ void ssl_async_upstream_create(struct future * f, struct ssl_mgr * mgr, evutil_s
 	ctx->f_peek_chello = future_create("peek_sni", peek_chello_on_succ, peek_chello_on_fail, p);
 	ssl_async_peek_client_hello(ctx->f_peek_chello, fd_downstream, !mgr->no_mirror_client_cipher_suite, evbase, mgr->logger);
 }
+	
 
 static int ossl_session_ticket_key_callback(SSL *ssl_conn,
     unsigned char *name, unsigned char *iv, EVP_CIPHER_CTX *ectx,
     HMAC_CTX *hctx, int enc)
 {
+    enum STEK_GET_RET ret=STEK_NOT_FOUND;
     const EVP_MD  *digest=EVP_sha256();
     const EVP_CIPHER *cipher=EVP_aes_256_cbc();
-	size_t size=32;
+    size_t size=32;
+	struct ssl_stream* s_stream = (struct ssl_stream*)  SSL_get_ex_data(ssl_conn, SSL_EX_DATA_IDX_SSLSTREAM);
+	struct ssl_mgr* mgr= s_stream->mgr;
+	void* logger = s_stream->mgr->logger;
+	const char * sni = s_stream->peer->up_parts.client_hello->sni;
+    struct sess_ticket_box * stek_box = s_stream->mgr->down_stek_box;
+	struct sess_ticket_key cur_stek;
 	
-	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl_conn, SSL_CTX_EX_DATA_IDX_SSLMGR);
+    unsigned char buf[33]={0};  
-	assert(mgr!=NULL);
+    ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TIKCET_QUERY]));
-	
-	struct session_ticket_key* key=&(mgr->ticket_key);
-	
-	unsigned char buf[33]={0};
-	
-	ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TIKCET_QUERY]));
     if (enc == 1)
-	{
+    {
-        /* encrypt session ticket */
+    	
- 
+		sess_ticket_box_get_key_for_enc(stek_box, sni, &cur_stek);
-
+        /* encrypt session stek */                   
-		if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1)
+        if (RAND_bytes(iv, EVP_CIPHER_iv_length(cipher)) != 1)
-		{
+        {
-			TFE_LOG_ERROR(mgr->logger, "Session Ticket RAND_bytes() failed");
+            TFE_LOG_ERROR(mgr->logger, "Session Ticket RAND_bytes() failed");
-			return -1;
+			ret=STEK_ERROR;
-		}
+			goto leave;
-
-        if (EVP_EncryptInit_ex(ectx, cipher, NULL, key[0].aes_key, iv) != 1)
-		{
-			TFE_LOG_ERROR(mgr->logger, "EVP_EncryptInit_ex() failed");
-            return -1;
         }
-
+        if (EVP_EncryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1)
-        if (HMAC_Init_ex(hctx, key[0].hmac_key, size, digest, NULL) != 1) 
+        {
-		{
+            TFE_LOG_ERROR(mgr->logger, "EVP_EncryptInit_ex() failed");
-			TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed");
+			ret=STEK_ERROR;
-			return -1;
+			goto leave;
         }
-        memcpy(name, key[0].name, 16);
+        if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1) 
-		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NEW]));
+        {
-        return 1;
-
-    } 
-	else 
-	{
-        /* decrypt session ticket */
-		if(0!=memcmp(name, key[0].name, 16))
-		{
-			TFE_LOG_INFO(mgr->logger, "ssl session ticket decrypt, key: \"%*s\" not found"
-						,(int)(tfe_hexdump(buf, name ,16)-buf), buf);
-			ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NOTFOUND]));
-			return 0;
-		}
-		
-        
-
-        if (HMAC_Init_ex(hctx, key[0].hmac_key, size, digest, NULL) != 1)
-		{
             TFE_LOG_ERROR(mgr->logger, "HMAC_Init_ex() failed");
-            return -1;
+			ret=STEK_ERROR;
+			goto leave;
         }
+        memcpy(name, cur_stek.name, sizeof(cur_stek.name));
+        ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NEW]));
+		ret=STEK_FOUND_FRESH;
+    } 
+    else 
+    {
+       /* decrypt session stek */          	
+		ret=sess_ticket_box_get_key_for_dec(stek_box, sni, name, &cur_stek);
 
-
+		if(ret==STEK_FOUND_FRESH||ret==STEK_FOUND_STALED)
-        if (EVP_DecryptInit_ex(ectx, cipher, NULL, key[0].aes_key, iv) != 1) 
 		{
-            TFE_LOG_ERROR(mgr->logger, "EVP_DecryptInit_ex() failed");
+			if (HMAC_Init_ex(hctx, cur_stek.hmac_key, size, digest, NULL) != 1)
-            return -1;
+			{
-        }
+				TFE_LOG_ERROR(logger, "HMAC_Init_ex() failed");
-		ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_REUSE]));
+				ret= STEK_ERROR;
-		//Todo: Implement key expire.
+				goto leave;
-        return 1; //Return 1 indicates that the ctx and hctx have been set and the session can continue on those parameters.
+			}													
-        
+			if (EVP_DecryptInit_ex(ectx, cipher, NULL, cur_stek.aes_key, iv) != 1) 
+			{
+				TFE_LOG_ERROR(logger, "EVP_DecryptInit_ex() failed");
+				ret= STEK_ERROR;
+				goto leave;
+			}
+			ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_REUSE]));
+		}
+		else
+		{
+			// full handshake and update the session stek_box
+			TFE_LOG_DEBUG(logger, "full handshake and update the stek_box, STEK is overdue STEK."); 
+			TFE_LOG_INFO(logger, "ssl session stek_box decrypt, key: \"%*s\" not found",(int)(tfe_hexdump(buf, name ,16)-buf), buf);
+			ATOMIC_INC(&(mgr->stat_val[SSL_DOWN_TICKET_NOTFOUND]));
+		}
     }
+	
+leave:
+	return (int)ret;
 }
 
 
@@ -1441,7 +1443,8 @@ static void ossl_downsess_remove_cb(SSL_CTX * sslctx, SSL_SESSION * sess)
  */
 static SSL_SESSION * ossl_downsess_get_cb(SSL * ssl, const unsigned char * id, int idlen, int * copy)
 {
-	struct ssl_mgr * mgr = (struct ssl_mgr *) SSL_get_ex_data(ssl, SSL_CTX_EX_DATA_IDX_SSLMGR);
+	struct ssl_stream* s_stream = (struct ssl_stream*) SSL_get_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM);
+	struct ssl_mgr * mgr = s_stream->mgr;
 	SSL_SESSION * sess=NULL;
 	if(!mgr->no_sesscache)
 	{
@@ -1566,7 +1569,7 @@ void downstream_ossl_init(struct ssl_stream *s_stream)
 	SSL_CTX_set_session_cache_mode(sslctx, SSL_SESS_CACHE_SERVER | SSL_SESS_CACHE_NO_INTERNAL);
 	SSL_CTX_set_session_id_context(sslctx, (const unsigned char *) mgr->ssl_session_context,
 		sizeof(mgr->ssl_session_context));
-	ret = SSL_CTX_set_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR, mgr);
+	ret = SSL_CTX_set_ex_data(sslctx, SSL_CTX_EX_DATA_IDX_SSLMGR, mgr);//used by down session cache get
 	assert(ret == 1);
 	if (mgr->dh)
 	{
@@ -1607,9 +1610,9 @@ void downstream_ossl_init(struct ssl_stream *s_stream)
 	SSL_CTX_free(sslctx);            // SSL_new() increments refcount
 	sslctx = NULL;
 
-	ret = SSL_set_ex_data(ssl, SSL_CTX_EX_DATA_IDX_SSLMGR, mgr);
+	ret = SSL_set_ex_data(ssl, SSL_EX_DATA_IDX_SSLSTREAM, s_stream);
 	assert(ret == 1);
-
+	
 	if (mgr->ssl_mode_release_buffers == 1)
 	{
 		/* lower memory footprint for idle connections */