gfwleak/cuiyiming-gradproj
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加证书解析功能

Browse Source
This commit is contained in:
崔一鸣
2019-12-13 11:41:42 +08:00
parent 113201f203
commit 6e8bf26282
6 changed files with 434 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
.vscode/*
DataSet/FeatureExtract/pcap/*
DataSet/FeatureExtract/.vscode/*

58
DataSet/DataTag/chelloFeature.py Normal file
View File

@@ -0,0 +1,58 @@
import json
import sys
import traceback
streamTagDict = dict()
appCipherDict = dict()
def streamTagDictBuild():
filename = "../result/2019-12-06-0/stream_tag.txt"
with open(filename) as f:
logs = f.readlines()
for log in logs:
log = log.split(":")
stream = log[0].split(" ")
streamStr = ""
for item in stream:
streamStr += item
streamStr += ','
tag = log[1]
streamTagDict[streamStr] = tag
def appCipherDictBuild():
filename = "../result/2019-12-06-0/stream_feature.txt"
tagFailCount = 0
tagSuccCount = 0
with open(filename) as f:
logs = f.readlines()
for log in logs:
try:
log = json.loads(log)
streamStr = log["sip"] + "," + str(log["sport"]) + ',' + log["dip"] + ',' + str(log["dport"]) + ','
appName = streamTagDict[streamStr]
cipherSuites = log['tls']['cipher_suites']
cipherSuitesStr = ""
for cipherSuite in cipherSuites:
cipherSuitesStr += cipherSuite
if appName not in appCipherDict.keys():
appCipherDict[appName] = set()
appCipherDict[appName].add(cipherSuitesStr)
tagSuccCount += 1
except:
tagFailCount += 1
#traceback.print_exc()
continue
print("tagFailCount = " + str(tagFailCount))
print("tagSuccCount = " + str(tagSuccCount))
def main():
streamTagDictBuild()
appCipherDictBuild()
for appName, cipherList in appCipherDict.items():
print(appName)
print(cipherList)
if __name__ == '__main__':
main()

9
DataSet/FeatureExtract/entry/CMakeLists.txt
View File

@@ -1,3 +1,8 @@
add_library(stmstat SHARED src/tcp_entry.cpp src/ssl_utils.cpp)
add_library(stmstat SHARED src/sslstat_entry.cpp src/ssl_utils.cpp)
target_include_directories(stmstat PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
target_link_libraries(stmstat MESA_prof_load MESA_field_stat cjson)
target_link_libraries(stmstat MESA_prof_load MESA_field_stat cjson)
add_library(sslstat SHARED src/sslstat_entry.cpp)
target_include_directories(sslstat PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
target_link_libraries(sslstat MESA_prof_load MESA_field_stat cjson)

230
DataSet/FeatureExtract/entry/include/ssl.h Normal file
View File

@@ -0,0 +1,230 @@
#ifndef H_SSL_H
#define H_SSL_H
#include <stdio.h>
#include <string.h>
#define SSH_H_VERSION_20160910_ADD_CERT 0
#define SSL_KEY 3
#define SSL_TRUE 1
#define SSL_FLASE 0
#define SSL_INTEREST_KEY (1<<SSL_INTEREST_KEY_MASK)
#define SSL_CERTIFICATE (1<<SSL_CERTIFICATE_MASK)
#define SSL_CERTIFICATE_DETAIL (1<<SSL_CERTIFICATE_DETAIL_MASK)
#define SSL_APPLICATION_DATA (1<<SSL_APPLICATION_DATA_MASK)
#define SSL_CLIENT_HELLO (1<<SSL_CLIENT_HELLO_MASK)
#define SSL_SERVER_HELLO (1<<SSL_SERVER_HELLO_MASK)
#define SSL_VERSION (1<<SSL_VERSION_MASK)
typedef enum
{
/*1*/
SSL_INTEREST_KEY_MASK = 0,
SSL_CERTIFICATE_DETAIL_MASK = 1,
SSL_CLIENT_HELLO_MASK = 2,
SSL_SERVER_HELLO_MASK= 3,
SSL_CERTIFICATE_MASK,
SSL_APPLICATION_DATA_MASK,
SSL_VERSION_MASK,
}ssl_interested_region;
typedef struct cdata_buf
{
char* p_data;
unsigned int data_size;
}cdata_buf;
typedef struct _st_random_t
{
unsigned int gmt_time; //4
unsigned char random_bytes[28]; //28 byte random_bytes
}st_random_t;
typedef struct _st_session_t
{
unsigned char session_len; //4
unsigned char* session_value;
}st_session_t;
typedef struct _st_suites_t
{
unsigned short suite_len; //4
unsigned char* suite_value;
}st_suites_t;
typedef struct _st_compress_methods_t
{
unsigned char methlen;
unsigned char* methods;//default 0:null
}st_compress_methods_t;
//#############################################client hello
#define CLIENT_HELLO_HDRLEN 4
#define MAX_EXTENSION_NUM 16
#define MAX_EXT_DATA_LEN 256
#define SERVER_NAME_EXT_TYPE 0x0000
#define SERVER_NAME_HOST_TYPE 0x0000
#define SERVER_NAME_OTHER_TYPE 0x0008
typedef struct _st_client_ext_t
{
unsigned short type;
unsigned short len;
unsigned char data[MAX_EXT_DATA_LEN];//if longer,cut off
}__attribute__((packed))st_client_ext_t;
typedef struct _st_client_server_name_t
{
short server_name_list_len;
unsigned short server_name_type;
unsigned char server_name_len;
unsigned char* server_name_data;
}__attribute__((packed))st_client_server_name_t;
//client hello info
typedef struct _st_client_hello_t
{
int totallen; //3
unsigned short client_ver;
st_random_t random; //32 byte random,not used currently
st_session_t session;
st_suites_t ciphersuits;
st_compress_methods_t com_method; //compress method
unsigned short extlen;
unsigned short ext_num; //number of extensions
st_client_ext_t exts[MAX_EXTENSION_NUM]; //extensions content:1 or more extentions
unsigned char server_name[512]; // server_name = host_name+...
}st_client_hello_t;
//#############################################client hello end
//#############################################server hello
#define SERVER_HELLO_HDRLEN 4
//client hello info
typedef struct _st_server_hello_t
{
int totallen; //3
unsigned short client_ver;
st_random_t random; //32 byte random,not used currently
st_session_t session;
st_suites_t ciphersuits;
st_compress_methods_t com_method; //compress method
}st_server_hello_t;
//#############################################server hello end
//#############################################certificate
#define CERTIFICATE_HDRLEN 7
#define SSL_CERTIFICATE_HDRLEN 3
//#define SAN_MAXNUM 128
typedef struct _san_t
{
char san[64];
}san_t;
typedef struct _st_san_t
{
int count;
san_t* san_array; //ָ<><D6B8><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
}st_san_t;
typedef struct _st_cert_t
{
int totallen;
int certlen;
char SSLVersion[10];
char SSLSerialNum[128];
char SSLAgID [64];
char SSLIssuer[512];
char SSLSub[512];
char SSLFrom[80];
char SSLTo[80];
char SSLFPAg[32];
char SSLIssuerC[64]; //country
char SSLIssuerO[64]; //organize
char SSLIssuerCN[64];//cname
char SSLSubC[64]; //country
char SSLSubO[64]; //organize
char SSLSubCN[64];//cname
st_san_t* SSLSubAltName;
uint8_t cert_type;
}st_cert_t;
//#############################################certificate end
typedef struct _business_infor_t
{
void* param;
unsigned char return_value;
}business_infor_t;
typedef struct _ssl_stream_t
{
unsigned long long output_region_flag;
unsigned char link_state;
unsigned char over_flag;
unsigned char ucContType;
unsigned char is_ssl_stream;
unsigned int uiSslVersion;
int uiAllMsgLen; //hand shake msg length
int uiMsgProcLen;
unsigned int uiMsgState;
int uiMaxBuffLen;
cdata_buf* p_output_buffer;
st_client_hello_t* stClientHello;
st_server_hello_t* stServerHello;
st_cert_t* stSSLCert;
business_infor_t* business;
char* pcSslBuffer;
ssl_interested_region output_region_mask;
int uiCurBuffLen;
}ssl_stream;
/*ssl_read_all_cert<72>еĽṹ<C4BD><E1B9B9>*/
typedef struct cert_chain_s
{
char* cert;
uint32_t cert_len;
}cert_chain_t;
/*ssl_read_specific_cert<72><74>cert_type<70>IJ<EFBFBD><C4B2><EFBFBD>*/
#define CERT_TYPE_INDIVIDUAL 0 //<2F><><EFBFBD><EFBFBD>֤<EFBFBD><D6A4>
#define CERT_TYPE_ROOT 1 //<2F><>֤<EFBFBD><D6A4>
#define CERT_TYPE_MIDDLE 2 //<2F>м<EFBFBD>֤<EFBFBD>飬<EFBFBD><E9A3AC><EFBFBD><EFBFBD>֤<EFBFBD><D6A4><EFBFBD><EFBFBD><EFBFBD>ϼ<EFBFBD>֤<EFBFBD><D6A4>
#define CERT_TYPE_CHAIN 3 //<2F><><EFBFBD><EFBFBD>: <20><>ʽ[len(3bytes)+cert+len(3bytes)+certlen(3bytes)+cert......]
#ifdef __cplusplus
extern "C" {
#endif
/*return : chain <20><><EFBFBD><EFBFBD>, <20><><EFBFBD>մӸ<D5B4><D3B8><EFBFBD>֤<EFBFBD>鵽<EFBFBD><E9B5BD>֤<EFBFBD><D6A4><EFBFBD><EFBFBD>˳<EFBFBD><CBB3><EFBFBD>洢*/
int ssl_read_all_cert(const char* conj_cert_buf, uint32_t conj_buflen, cert_chain_t* cert_unit, uint32_t unit_size);
/*return : 1 <20><><EFBFBD>ڣ<EFBFBD>0 <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>*/
int ssl_read_specific_cert(const char* conj_cert_buf, uint32_t conj_buflen, uint8_t cert_type, char** cert, uint32_t* cert_len);
const char* ssl_get_suite(st_suites_t* ciphersuits);
#ifdef __cplusplus
}
#endif
#endif

131
DataSet/FeatureExtract/entry/src/sslstat_entry.cpp Normal file
View File

@@ -0,0 +1,131 @@
#include "base_utils.h"
#include "stream_inc/stream_base.h"
#include "stream_inc/stream_entry.h"
#include "ssl.h"
#include "cjson/cJSON.h"
#define CERT_COUNT_MAX 16
struct ssl_context{
char sip[INET_ADDRSTRLEN];
int sport;
char dip[INET_ADDRSTRLEN];
int dport;
unsigned char *sni;
char *san;
int cert_count;
cert_chain_t certs[CERT_COUNT_MAX];
};
FILE *g_fp = NULL;
static char *ssl_assemble_san(st_cert_t *cert){
int tmp_buflen = 0, total_buflen = 0;
char *san_buf = NULL;
for (int i = 0; i < cert->SSLSubAltName->count; i++){
tmp_buflen = strlen(cert->SSLSubAltName->san_array[i].san);
san_buf = (char *)realloc(san_buf, total_buflen + tmp_buflen + 1);
san_buf[total_buflen + tmp_buflen] = ';';
memcpy(san_buf + total_buflen, cert->SSLSubAltName->san_array[i].san, tmp_buflen);
total_buflen += tmp_buflen + 1;
}
san_buf[total_buflen - 1] = '\0';
return san_buf;
}
void ssl_ctx_close(struct ssl_context *ctx, struct streaminfo *stream, ssl_stream *a_ssl){
if (ctx != NULL){
cJSON *log_obj = cJSON_CreateObject();
cJSON_AddStringToObject(log_obj, "sip", ctx->sip);
cJSON_AddNumberToObject(log_obj, "sport", ctx->sport);
cJSON_AddStringToObject(log_obj, "dip", ctx->dip);
cJSON_AddNumberToObject(log_obj, "dport", ctx->dport);
cJSON_AddStringToObject(log_obj, "proto", "tcp");
cJSON_AddStringToObject(log_obj, "sni", (const char*)ctx->sni);
cJSON_AddStringToObject(log_obj, "san", ctx->san);
//cert
cJSON *Cert = cJSON_CreateObject();
cJSON_AddNumberToObject(Cert, "cert_count", ctx->cert_count);
cJSON *cert_info_list = cJSON_CreateArray();
for(int i = 0; i < ctx->cert_count; i++){
cJSON *cert_info = cJSON_CreateObject();
cJSON_AddNumberToObject(cert_info, "length", ctx->certs[i].cert_len);
if(i == 0){
cJSON_AddStringToObject(cert_info, "type", "individual");
}
else{
cJSON_AddStringToObject(cert_info, "type", "no-individual");
}
cJSON_AddItemToArray(cert_info_list, cert_info);
}
cJSON_AddItemToObject(Cert, "cert_list", cert_info_list);
cJSON_AddItemToObject(log_obj, "Cert", Cert);
char *log_msg = cJSON_PrintUnformatted(log_obj);
fputs(log_msg, g_fp);
fputs("\n", g_fp);
cJSON_Delete(log_obj);
cJSON_free(log_msg);
FREE(&(ctx->san))
FREE(&ctx);
}
return;
}
extern "C" unsigned char sslstat_entry(stSessionInfo *session_info, void **param, int thread_seq, struct streaminfo *stream, void *a_packet){
ssl_stream *a_ssl = (ssl_stream *)(session_info->app_info);
struct ssl_context *ctx = (ssl_context *)*param;
if ((session_info->session_state & SESSION_STATE_PENDING) == SESSION_STATE_PENDING){
ctx = ALLOC(struct ssl_context, 1);
*param = ctx;
struct stream_tuple4_v4 *tuple4 = stream->addr.tuple4_v4;
inet_ntop(AF_INET, &(tuple4->saddr), ctx->sip, INET_ADDRSTRLEN);
inet_ntop(AF_INET, &(tuple4->daddr), ctx->dip, INET_ADDRSTRLEN);
ctx->sport = ntohs(tuple4->source);
ctx->dport = ntohs(tuple4->dest);
}
switch (session_info->prot_flag){
case SSL_CLIENT_HELLO:
if (a_ssl != NULL && a_ssl->stClientHello != NULL){
ctx->sni = a_ssl->stClientHello->server_name;
}
break;
case SSL_CERTIFICATE:
ctx->cert_count = ssl_read_all_cert((const char*)session_info->buf, session_info->buflen, ctx->certs, CERT_COUNT_MAX);
case SSL_CERTIFICATE_DETAIL:
if (a_ssl != NULL && a_ssl->stSSLCert != NULL && stream->curdir == DIR_S2C){
st_cert_t *cert = a_ssl->stSSLCert;
if (cert->cert_type == CERT_TYPE_INDIVIDUAL){
if (cert->SSLSubAltName != NULL && cert->SSLSubAltName->count > 0){
char *san_buf = ssl_assemble_san(cert);
ctx->san = san_buf;
}
}
}
break;
case SSL_APPLICATION_DATA:
break;
default:
break;
}
if ((session_info->session_state & SESSION_STATE_CLOSE) == SESSION_STATE_CLOSE){
//close_ssl:
ssl_ctx_close(ctx, stream, (ssl_stream *)session_info->app_info);
return PROT_STATE_DROPME;
}
return PROT_STATE_GIVEME;
}
extern "C" int sslstat_init(){
g_fp = fopen("./ssl_stat.txt", "w+");
return 0;
}
extern "C" void sslstat_destroy(void){
return;
}

10
DataSet/FeatureExtract/entry/src/tcp_entry.cpp → DataSet/FeatureExtract/entry/src/stmstat_entry.cpp
View File

@@ -4,7 +4,7 @@
#include "MESA/stream_inc/stream_base.h"
#include "MESA/stream_inc/stream_rawpkt.h"
#include "cjson/cJSON.h"
#define STREAM_PACKET_COUNT_MAX 200
#define STREAM_PACKET_COUNT_MAX 10000
/*
{
@@ -42,6 +42,7 @@ int g_stream_count = 0;
int g_stream_succ_count = 0;
int g_stream_fail_count = 0;
int g_log_succ_count = 0;
int g_exceed_max_pkts_count = 0;
struct pkt_stat_info{
struct timeval pkt_time;
@@ -107,6 +108,8 @@ int ipv4_header_parse(const void *a_packet, struct pkt_parsed_info* pktinfo){
int packet_stat(struct streaminfo *stream, struct pme_info *pmeinfo, struct pkt_parsed_info* pktinfo){
if(pmeinfo->total_pkts == STREAM_PACKET_COUNT_MAX){
printf("packet nums > STREAM_PACKET_COUNT_MAX\n");
g_exceed_max_pkts_count++;
return -1;
}
pmeinfo->pkt_info_list[pmeinfo->total_pkts].bytes = pktinfo->data_len;
@@ -258,9 +261,10 @@ void pme_info_destroy(struct pme_info *pmeinfo){
}
extern "C" char stmstat_entry(struct streaminfo *stream, void** pme, int thread_seq, const void* a_packet){
if(g_count % 100 == 5){
if(g_count % 10 == 5){
printf("handle %d packets\n", g_count);
printf("stream_count: %d\nsucc_count: %d\nfail_count: %d\ng_log_succ_count: %d\n", g_stream_count, g_stream_succ_count, g_stream_fail_count, g_log_succ_count);
printf("stream_count: %d\nsucc_count: %d\nfail_count: %d\ng_log_succ_count: %d, g_exceed_max_pkts_count: %d\n",
g_stream_count, g_stream_succ_count, g_stream_fail_count, g_log_succ_count, g_exceed_max_pkts_count);
}
g_count++;
char ret;