From 6e8bf26282de2006ef00cd86fc30e21188662a84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=B4=94=E4=B8=80=E9=B8=A3?= <cuiyiming@iie.ac.cn>
Date: Fri, 13 Dec 2019 11:41:42 +0800
Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=AF=81=E4=B9=A6=E8=A7=A3?=
 =?UTF-8?q?=E6=9E=90=E5=8A=9F=E8=83=BD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitignore                                    |   1 +
 DataSet/DataTag/chelloFeature.py              |  58 +++++
 DataSet/FeatureExtract/entry/CMakeLists.txt   |   9 +-
 DataSet/FeatureExtract/entry/include/ssl.h    | 230 ++++++++++++++++++
 .../entry/src/sslstat_entry.cpp               | 131 ++++++++++
 .../src/{tcp_entry.cpp => stmstat_entry.cpp}  |  10 +-
 6 files changed, 434 insertions(+), 5 deletions(-)
 create mode 100644 DataSet/DataTag/chelloFeature.py
 create mode 100644 DataSet/FeatureExtract/entry/include/ssl.h
 create mode 100644 DataSet/FeatureExtract/entry/src/sslstat_entry.cpp
 rename DataSet/FeatureExtract/entry/src/{tcp_entry.cpp => stmstat_entry.cpp} (96%)

diff --git a/.gitignore b/.gitignore
index 87de3eb..b936f68 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 .vscode/*
 DataSet/FeatureExtract/pcap/*
+DataSet/FeatureExtract/.vscode/*
diff --git a/DataSet/DataTag/chelloFeature.py b/DataSet/DataTag/chelloFeature.py
new file mode 100644
index 0000000..3253d8a
--- /dev/null
+++ b/DataSet/DataTag/chelloFeature.py
@@ -0,0 +1,58 @@
+
+
+import json
+import sys
+import traceback
+
+streamTagDict = dict()
+appCipherDict = dict()
+
+def streamTagDictBuild():
+    filename = "../result/2019-12-06-0/stream_tag.txt"
+    with open(filename) as f:
+        logs = f.readlines()
+        for log in logs:
+            log = log.split(":")
+            stream = log[0].split(" ")
+            streamStr = ""
+            for item in stream:
+                streamStr += item
+                streamStr += ','
+            tag = log[1]
+            streamTagDict[streamStr] = tag
+
+def appCipherDictBuild():
+    filename = "../result/2019-12-06-0/stream_feature.txt"
+    tagFailCount = 0
+    tagSuccCount = 0
+    with open(filename) as f:
+        logs = f.readlines()
+        for log in logs:
+            try:
+                log = json.loads(log)
+                streamStr = log["sip"] + "," + str(log["sport"]) + ',' + log["dip"] + ',' + str(log["dport"]) + ',' 
+                appName = streamTagDict[streamStr]
+                cipherSuites = log['tls']['cipher_suites']
+                cipherSuitesStr = ""
+                for cipherSuite in cipherSuites:
+                    cipherSuitesStr += cipherSuite
+                if appName not in appCipherDict.keys():
+                    appCipherDict[appName] = set()
+                appCipherDict[appName].add(cipherSuitesStr)
+                tagSuccCount += 1
+            except:
+                tagFailCount += 1
+                #traceback.print_exc()
+                continue
+    print("tagFailCount = " + str(tagFailCount))
+    print("tagSuccCount = " + str(tagSuccCount))
+
+def main():
+    streamTagDictBuild()
+    appCipherDictBuild()
+    for appName, cipherList in appCipherDict.items():
+        print(appName)
+        print(cipherList)
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/DataSet/FeatureExtract/entry/CMakeLists.txt b/DataSet/FeatureExtract/entry/CMakeLists.txt
index 2e95212..99b511a 100644
--- a/DataSet/FeatureExtract/entry/CMakeLists.txt
+++ b/DataSet/FeatureExtract/entry/CMakeLists.txt
@@ -1,3 +1,8 @@
-add_library(stmstat SHARED src/tcp_entry.cpp src/ssl_utils.cpp)
+add_library(stmstat SHARED src/sslstat_entry.cpp src/ssl_utils.cpp)
 target_include_directories(stmstat PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
-target_link_libraries(stmstat MESA_prof_load MESA_field_stat cjson)
\ No newline at end of file
+target_link_libraries(stmstat MESA_prof_load MESA_field_stat cjson)
+
+
+add_library(sslstat SHARED src/sslstat_entry.cpp)
+target_include_directories(sslstat PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
+target_link_libraries(sslstat MESA_prof_load MESA_field_stat cjson)
\ No newline at end of file
diff --git a/DataSet/FeatureExtract/entry/include/ssl.h b/DataSet/FeatureExtract/entry/include/ssl.h
new file mode 100644
index 0000000..5981228
--- /dev/null
+++ b/DataSet/FeatureExtract/entry/include/ssl.h
@@ -0,0 +1,230 @@
+
+#ifndef H_SSL_H
+#define H_SSL_H
+
+#include <stdio.h>
+#include <string.h>
+
+#define SSH_H_VERSION_20160910_ADD_CERT				0
+
+#define SSL_KEY 									3
+#define SSL_TRUE 								1
+#define SSL_FLASE 								0
+
+
+#define SSL_INTEREST_KEY		(1<<SSL_INTEREST_KEY_MASK)
+#define SSL_CERTIFICATE			(1<<SSL_CERTIFICATE_MASK)
+#define SSL_CERTIFICATE_DETAIL	(1<<SSL_CERTIFICATE_DETAIL_MASK)
+#define SSL_APPLICATION_DATA	(1<<SSL_APPLICATION_DATA_MASK)
+#define SSL_CLIENT_HELLO		(1<<SSL_CLIENT_HELLO_MASK)
+#define SSL_SERVER_HELLO		(1<<SSL_SERVER_HELLO_MASK)
+#define SSL_VERSION				(1<<SSL_VERSION_MASK)
+
+typedef enum
+{
+	/*1*/
+	SSL_INTEREST_KEY_MASK = 0,
+	SSL_CERTIFICATE_DETAIL_MASK = 1,
+	SSL_CLIENT_HELLO_MASK = 2,	
+	SSL_SERVER_HELLO_MASK= 3,	
+	SSL_CERTIFICATE_MASK,	
+	SSL_APPLICATION_DATA_MASK,
+	SSL_VERSION_MASK,
+}ssl_interested_region;
+
+typedef struct cdata_buf
+{
+	char*					p_data;
+	unsigned int 			data_size;		
+}cdata_buf;
+
+typedef struct _st_random_t
+{
+	unsigned int 			gmt_time;	//4 
+	unsigned char 			random_bytes[28];	//28 byte random_bytes
+}st_random_t;
+
+typedef struct _st_session_t
+{
+	unsigned char 			session_len;	//4 
+	unsigned char*			session_value;	
+}st_session_t;
+
+typedef struct _st_suites_t
+{
+	unsigned short  		suite_len;	//4 
+	unsigned char*			suite_value;	
+}st_suites_t;
+
+typedef struct _st_compress_methods_t
+{
+	unsigned char 			methlen;	
+	unsigned char*			methods;//default 0:null
+}st_compress_methods_t;
+
+//#############################################client hello
+#define CLIENT_HELLO_HDRLEN 		4
+#define MAX_EXTENSION_NUM			16
+#define MAX_EXT_DATA_LEN			256
+#define SERVER_NAME_EXT_TYPE		0x0000
+#define SERVER_NAME_HOST_TYPE 	0x0000
+#define SERVER_NAME_OTHER_TYPE 	0x0008
+
+
+typedef struct _st_client_ext_t
+{
+	unsigned short 			type;
+	unsigned short 			len;
+	unsigned char 			data[MAX_EXT_DATA_LEN];//if longer,cut off
+}__attribute__((packed))st_client_ext_t;
+
+typedef struct _st_client_server_name_t
+{
+	short 					server_name_list_len;
+	unsigned short 			server_name_type;
+	unsigned char 			server_name_len;
+	unsigned char* 			server_name_data;
+}__attribute__((packed))st_client_server_name_t;
+
+
+//client hello info
+typedef struct _st_client_hello_t
+{
+	int 						totallen;	//3 
+	unsigned short 				client_ver;	
+	st_random_t 				random;	//32 byte random,not used currently
+	st_session_t 				session;	
+	st_suites_t 				ciphersuits;
+	st_compress_methods_t 		com_method;	//compress method
+	unsigned short 				extlen;	
+	unsigned short 				ext_num;	//number of extensions
+	st_client_ext_t 			exts[MAX_EXTENSION_NUM];	//extensions content:1 or more extentions
+	unsigned char 				server_name[512];  	// server_name = host_name+...
+}st_client_hello_t;
+
+//#############################################client hello end
+
+//#############################################server hello
+#define  SERVER_HELLO_HDRLEN 4
+
+//client hello info
+typedef struct _st_server_hello_t
+{
+	int 					totallen;	//3 
+	unsigned short 			client_ver;	
+	st_random_t 			random;	//32 byte random,not used currently
+	st_session_t 			session;	
+	st_suites_t 			ciphersuits;
+	st_compress_methods_t 	com_method;	//compress method
+}st_server_hello_t;
+
+//#############################################server hello end
+
+//#############################################certificate
+#define CERTIFICATE_HDRLEN		7
+#define SSL_CERTIFICATE_HDRLEN  3
+//#define SAN_MAXNUM  			128
+
+typedef struct _san_t
+{
+	char 					san[64]; 
+}san_t;
+
+typedef struct _st_san_t
+{
+	int 					count;
+	san_t* 					san_array; 					//指针数组
+}st_san_t;
+
+typedef struct _st_cert_t
+{
+	int 					totallen;
+	int 					certlen;
+	char 					SSLVersion[10];   
+	char 					SSLSerialNum[128];
+	char 					SSLAgID [64];     
+	char 					SSLIssuer[512];  	
+	char 					SSLSub[512];    
+	char 					SSLFrom[80];     
+	char 					SSLTo[80];     
+	char 					SSLFPAg[32];  	 	
+	char 					SSLIssuerC[64]; //country  	
+	char 					SSLIssuerO[64]; //organize
+	char 					SSLIssuerCN[64];//cname	
+	char 					SSLSubC[64]; //country  	
+	char 					SSLSubO[64]; //organize
+	char 					SSLSubCN[64];//cname		
+	st_san_t* 				SSLSubAltName;  
+	uint8_t 				cert_type;
+}st_cert_t;
+
+//#############################################certificate end
+
+
+typedef struct _business_infor_t
+{
+	void*						param;
+	unsigned char				return_value;
+}business_infor_t;
+
+typedef struct _ssl_stream_t
+{
+	unsigned long long 			output_region_flag;
+	unsigned char 				link_state;
+	unsigned char 				over_flag;
+	unsigned char 				ucContType;
+	unsigned char 				is_ssl_stream;	
+	unsigned int 				uiSslVersion;
+
+	int 						uiAllMsgLen; //hand shake msg length  
+	int 						uiMsgProcLen;
+	unsigned int 				uiMsgState;   	
+	int 						uiMaxBuffLen;
+
+	
+	cdata_buf*					p_output_buffer;
+	st_client_hello_t*			stClientHello;	
+	st_server_hello_t*			stServerHello;
+	st_cert_t*					stSSLCert;
+
+	business_infor_t*			business;
+
+	char*						pcSslBuffer;	 
+	ssl_interested_region 		output_region_mask;	
+	int 						uiCurBuffLen; 
+}ssl_stream;
+
+/*ssl_read_all_cert中的结构体*/
+typedef struct cert_chain_s
+{
+     char* 						cert;
+	 uint32_t					cert_len;
+}cert_chain_t;
+
+/*ssl_read_specific_cert中cert_type的参数*/
+#define CERT_TYPE_INDIVIDUAL		0  //个人证书
+#define CERT_TYPE_ROOT				1  //根证书
+#define CERT_TYPE_MIDDLE			2  //中间证书，个人证书的上级证书
+#define CERT_TYPE_CHAIN				3  //其他: 格式[len(3bytes)+cert+len(3bytes)+certlen(3bytes)+cert......]
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*return : chain 个数, 按照从个人证书到根证书的顺序存储*/
+int ssl_read_all_cert(const char* conj_cert_buf, uint32_t conj_buflen, cert_chain_t* cert_unit, uint32_t unit_size);
+
+/*return :  1 存在，0 不存在*/
+int ssl_read_specific_cert(const char* conj_cert_buf, uint32_t conj_buflen, uint8_t cert_type, char** cert, uint32_t* cert_len);
+
+const char* ssl_get_suite(st_suites_t* ciphersuits);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+
+
diff --git a/DataSet/FeatureExtract/entry/src/sslstat_entry.cpp b/DataSet/FeatureExtract/entry/src/sslstat_entry.cpp
new file mode 100644
index 0000000..350a98a
--- /dev/null
+++ b/DataSet/FeatureExtract/entry/src/sslstat_entry.cpp
@@ -0,0 +1,131 @@
+
+
+
+#include "base_utils.h"
+#include "stream_inc/stream_base.h"
+#include "stream_inc/stream_entry.h"
+#include "ssl.h"
+#include "cjson/cJSON.h"
+
+#define CERT_COUNT_MAX 16
+
+struct ssl_context{
+    char sip[INET_ADDRSTRLEN];
+    int sport;
+    char dip[INET_ADDRSTRLEN];
+    int dport;
+    unsigned char *sni;
+    char *san;
+    int cert_count;
+    cert_chain_t certs[CERT_COUNT_MAX];
+};
+
+FILE *g_fp = NULL;
+
+static char *ssl_assemble_san(st_cert_t *cert){
+	int tmp_buflen = 0, total_buflen = 0;
+	char *san_buf = NULL;
+	for (int i = 0; i < cert->SSLSubAltName->count; i++){
+		tmp_buflen = strlen(cert->SSLSubAltName->san_array[i].san);
+		san_buf = (char *)realloc(san_buf, total_buflen + tmp_buflen + 1);
+		san_buf[total_buflen + tmp_buflen] = ';';
+		memcpy(san_buf + total_buflen, cert->SSLSubAltName->san_array[i].san, tmp_buflen);
+		total_buflen += tmp_buflen + 1;
+	}
+	san_buf[total_buflen - 1] = '\0';
+	return san_buf;
+}
+
+void ssl_ctx_close(struct ssl_context *ctx, struct streaminfo *stream, ssl_stream *a_ssl){
+	if (ctx != NULL){
+        cJSON *log_obj = cJSON_CreateObject();
+        cJSON_AddStringToObject(log_obj, "sip", ctx->sip);
+        cJSON_AddNumberToObject(log_obj, "sport", ctx->sport);
+        cJSON_AddStringToObject(log_obj, "dip", ctx->dip);
+        cJSON_AddNumberToObject(log_obj, "dport", ctx->dport);
+        cJSON_AddStringToObject(log_obj, "proto", "tcp");
+        cJSON_AddStringToObject(log_obj, "sni", (const char*)ctx->sni);
+        cJSON_AddStringToObject(log_obj, "san", ctx->san);
+        //cert
+        cJSON *Cert = cJSON_CreateObject();
+        cJSON_AddNumberToObject(Cert, "cert_count", ctx->cert_count);
+        cJSON *cert_info_list = cJSON_CreateArray();
+        for(int i = 0; i < ctx->cert_count; i++){
+            cJSON *cert_info = cJSON_CreateObject();
+            cJSON_AddNumberToObject(cert_info, "length", ctx->certs[i].cert_len);
+            if(i == 0){
+                cJSON_AddStringToObject(cert_info, "type", "individual");
+            }
+            else{
+                cJSON_AddStringToObject(cert_info, "type", "no-individual");
+            }
+            cJSON_AddItemToArray(cert_info_list, cert_info);
+        }
+        cJSON_AddItemToObject(Cert, "cert_list", cert_info_list);
+        cJSON_AddItemToObject(log_obj, "Cert", Cert);
+        char *log_msg = cJSON_PrintUnformatted(log_obj);
+        fputs(log_msg, g_fp);
+        fputs("\n", g_fp);
+        cJSON_Delete(log_obj);
+        cJSON_free(log_msg);
+        FREE(&(ctx->san))
+		FREE(&ctx);
+	}
+	return;
+}
+
+extern "C" unsigned char sslstat_entry(stSessionInfo *session_info, void **param, int thread_seq, struct streaminfo *stream, void *a_packet){
+	ssl_stream *a_ssl = (ssl_stream *)(session_info->app_info);
+    struct ssl_context *ctx = (ssl_context *)*param;
+	if ((session_info->session_state & SESSION_STATE_PENDING) == SESSION_STATE_PENDING){
+        ctx = ALLOC(struct ssl_context, 1);
+        *param = ctx;
+        struct stream_tuple4_v4 *tuple4 = stream->addr.tuple4_v4;
+        inet_ntop(AF_INET, &(tuple4->saddr), ctx->sip, INET_ADDRSTRLEN);
+        inet_ntop(AF_INET, &(tuple4->daddr), ctx->dip, INET_ADDRSTRLEN);
+        ctx->sport = ntohs(tuple4->source);
+        ctx->dport = ntohs(tuple4->dest);
+	}
+	switch (session_info->prot_flag){
+		case SSL_CLIENT_HELLO:
+			if (a_ssl != NULL && a_ssl->stClientHello != NULL){
+                    ctx->sni = a_ssl->stClientHello->server_name;
+			}
+			break;
+        case SSL_CERTIFICATE:
+            ctx->cert_count = ssl_read_all_cert((const char*)session_info->buf, session_info->buflen, ctx->certs, CERT_COUNT_MAX);
+		case SSL_CERTIFICATE_DETAIL:
+			if (a_ssl != NULL && a_ssl->stSSLCert != NULL && stream->curdir == DIR_S2C){
+				st_cert_t *cert = a_ssl->stSSLCert;
+				if (cert->cert_type == CERT_TYPE_INDIVIDUAL){
+					if (cert->SSLSubAltName != NULL && cert->SSLSubAltName->count > 0){
+						char *san_buf = ssl_assemble_san(cert);
+                        ctx->san = san_buf;
+					}
+				}
+			}
+			break;
+		case SSL_APPLICATION_DATA:
+			break;
+		default:
+			break;
+	}
+
+	if ((session_info->session_state & SESSION_STATE_CLOSE) == SESSION_STATE_CLOSE){
+
+//close_ssl:
+		ssl_ctx_close(ctx, stream, (ssl_stream *)session_info->app_info);
+		return PROT_STATE_DROPME;
+	}
+	return PROT_STATE_GIVEME;
+}
+
+
+extern "C" int sslstat_init(){
+    g_fp = fopen("./ssl_stat.txt", "w+");
+    return 0;
+}
+
+extern "C" void sslstat_destroy(void){
+	return;
+}
\ No newline at end of file
diff --git a/DataSet/FeatureExtract/entry/src/tcp_entry.cpp b/DataSet/FeatureExtract/entry/src/stmstat_entry.cpp
similarity index 96%
rename from DataSet/FeatureExtract/entry/src/tcp_entry.cpp
rename to DataSet/FeatureExtract/entry/src/stmstat_entry.cpp
index 1733dcf..3485068 100644
--- a/DataSet/FeatureExtract/entry/src/tcp_entry.cpp
+++ b/DataSet/FeatureExtract/entry/src/stmstat_entry.cpp
@@ -4,7 +4,7 @@
 #include "MESA/stream_inc/stream_base.h"
 #include "MESA/stream_inc/stream_rawpkt.h"
 #include "cjson/cJSON.h"
-#define STREAM_PACKET_COUNT_MAX 200
+#define STREAM_PACKET_COUNT_MAX 10000
 
 /*
 {
@@ -42,6 +42,7 @@ int g_stream_count = 0;
 int g_stream_succ_count = 0;
 int g_stream_fail_count = 0;
 int g_log_succ_count = 0;
+int g_exceed_max_pkts_count = 0;
 
 struct pkt_stat_info{
     struct timeval pkt_time;
@@ -107,6 +108,8 @@ int ipv4_header_parse(const void *a_packet, struct pkt_parsed_info* pktinfo){
 
 int packet_stat(struct streaminfo *stream, struct pme_info *pmeinfo, struct pkt_parsed_info* pktinfo){
     if(pmeinfo->total_pkts == STREAM_PACKET_COUNT_MAX){
+        printf("packet nums > STREAM_PACKET_COUNT_MAX\n");
+        g_exceed_max_pkts_count++;
         return -1;
     }
     pmeinfo->pkt_info_list[pmeinfo->total_pkts].bytes = pktinfo->data_len;
@@ -258,9 +261,10 @@ void pme_info_destroy(struct pme_info *pmeinfo){
 }
 
 extern "C" char stmstat_entry(struct streaminfo *stream, void** pme, int thread_seq, const void* a_packet){
-    if(g_count % 100 == 5){
+    if(g_count % 10 == 5){
         printf("handle %d packets\n", g_count);
-        printf("stream_count: %d\nsucc_count: %d\nfail_count: %d\ng_log_succ_count: %d\n", g_stream_count, g_stream_succ_count, g_stream_fail_count, g_log_succ_count);
+        printf("stream_count: %d\nsucc_count: %d\nfail_count: %d\ng_log_succ_count: %d, g_exceed_max_pkts_count: %d\n", 
+            g_stream_count, g_stream_succ_count, g_stream_fail_count, g_log_succ_count, g_exceed_max_pkts_count);
     }
     g_count++;
 	char ret;