gfwleak/cuiyiming-gradproj
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完成流特征统计插件

Browse Source
This commit is contained in:
崔一鸣
2019-12-06 19:47:13 +08:00
parent 0ee4877328
commit b66d14ce67
18 changed files with 5843 additions and 9028 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
DataSet/FeatureExtract/.vscode/ftp-kr.json vendored Normal file
View File

@@ -0,0 +1,17 @@
{
"host": "192.168.40.133",
"username": "root",
"password": "111111",
"remotePath": "/root/cuiyiming/FeatureExtract",
"protocol": "sftp",
"port": 22,
"fileNameEncoding": "utf8",
"autoUpload": true,
"autoDelete": false,
"autoDownload": false,
"ignore": [
".git",
"/.vscode",
"*.pcap"
]
}

80
DataSet/FeatureExtract/.vscode/ftp-kr.sync.cache.json vendored Normal file
View File

@@ -0,0 +1,80 @@
{
"sftp://192.168.40.133:22@root": {
"root": {
"cuiyiming": {
"FeatureExtract": {
".DS_Store": {
"type": "-",
"size": 6148,
"lmtime": 0,
"modified": false
},
".gitignore": {
"type": "-",
"size": 40,
"lmtime": 0,
"modified": false
},
"CMakeLists.txt": {
"type": "-",
"size": 551,
"lmtime": 1575560079518,
"modified": false
},
"entry": {
"CMakeLists.txt": {
"type": "-",
"size": 208,
"lmtime": 1575560105179,
"modified": false
},
"include": {
"base_utils.h": {
"type": "-",
"size": 1158,
"lmtime": 1575606465908,
"modified": false
},
"ssl_utils.h": {
"type": "-",
"size": 952,
"lmtime": 1575626701193,
"modified": false
}
},
"src": {
"ssl_utils.cpp": {
"type": "-",
"size": 13870,
"lmtime": 1575627774070,
"modified": false
},
"tcp_entry.cpp": {
"type": "-",
"size": 11448,
"lmtime": 1575631855171,
"modified": false
}
}
},
"pcap": {},
"vendor": {
"CMakeLists.txt": {
"type": "-",
"size": 2740,
"lmtime": 1575559328943,
"modified": false
},
"cJSON-1.7.7.tar.gz": {
"type": "-",
"size": 345681,
"lmtime": 1575559219369,
"modified": false
}
}
}
}
}
},
"$version": 1
}

21
DataSet/FeatureExtract/CMakeLists.txt Normal file
View File

@@ -0,0 +1,21 @@
cmake_minimum_required(VERSION 3.5)
project(kni)
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_C_STANDARD 11)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
set (CMAKE_CXX_FLAGS "-Wall")
#set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -lasan -fsanitize-recover=address -fsanitize=address -fno-omit-frame-pointer")
add_definitions(-D_GNU_SOURCE)
if (CMAKE_BUILD_TYPE STREQUAL Debug)
add_definitions(-DDEBUG)
endif()
add_subdirectory(vendor)
add_subdirectory(entry)
# cmake -DCMAKE_BUILD_TYPE=Debug/RelWithDebInfo

3
DataSet/FeatureExtract/entry/CMakeLists.txt Normal file
View File

@@ -0,0 +1,3 @@
add_library(stmstat SHARED src/tcp_entry.cpp src/ssl_utils.cpp)
target_include_directories(stmstat PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
target_link_libraries(stmstat MESA_prof_load MESA_field_stat cjson)

44
DataSet/FeatureExtract/entry/include/base_utils.h Normal file
View File

@@ -0,0 +1,44 @@
#pragma once
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <errno.h>
#include <unistd.h>
#include <inttypes.h>
#include <arpa/inet.h>
#include <netinet/ip6.h>
#include <net/if.h>
#include <string.h>
#include <pthread.h>
#include "MESA/MESA_handle_logger.h"
#ifndef MAX
#define MAX(a, b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
#define STRING_MAX 128
#define likely(expr) __builtin_expect((expr), 1)
#define unlikely(expr) __builtin_expect((expr), 0)
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
#define FREE(p) {free(*p);*p=NULL;}
#define LOG_ERROR(handler, fmt, ...) \
do { \
MESA_handle_runtime_log(handler, RLOG_LV_FATAL, "kni", fmt, ##__VA_ARGS__); } while(0)
#define LOG_INFO(handler, fmt, ...) \
do { \
MESA_handle_runtime_log(handler, RLOG_LV_INFO, "kni", fmt, ##__VA_ARGS__); } while(0)
#define LOG_DEBUG(handler, fmt, ...) \
do { \
MESA_handle_runtime_log(handler, RLOG_LV_DEBUG, "kni", fmt, ##__VA_ARGS__); } while(0)

50
DataSet/FeatureExtract/entry/include/ssl_utils.h Normal file
View File

@@ -0,0 +1,50 @@
#pragma once
#define EXTENSION_COUNT_MAX 128
#define CIPHER_SUITE_COUNT_MAX 256
struct cipher_suite
{
uint16_t value;
const char* name;
};
struct tls_extension{
int value;
const char* name;
};
enum chello_parse_result
{
CHELLO_PARSE_SUCCESS = 0,
CHELLO_PARSE_INVALID_FORMAT = -1,
CHELLO_PARSE_NOT_ENOUGH_BUFF = -2
};
struct ssl_version
{
uint8_t minor;
uint8_t major;
uint16_t ossl_format;
char str_format[STRING_MAX];
};
struct ssl_chello
{
struct ssl_version min_version;
struct ssl_version max_version;
int cipher_suites_count;
int extension_count;
int cipher_suite_list[CIPHER_SUITE_COUNT_MAX];
int extension_list[EXTENSION_COUNT_MAX];
char sni[STRING_MAX];
char alpn[STRING_MAX];
};
struct ssl_version_map{
int value;
const char *name;
};
void ssl_chello_parse(struct ssl_chello* _chello, const unsigned char* buff, size_t buff_len, enum chello_parse_result* result);
void ssl_chello_free(struct ssl_chello* chello);

468
DataSet/FeatureExtract/entry/src/ssl_utils.cpp Normal file
View File

@@ -0,0 +1,468 @@
#include "base_utils.h"
#include "ssl_utils.h"
struct ssl_version_map g_ssl_veriosn_map_list[] = {
{0x0301, "tls1.0"},
{0x0302, "tls1.1"},
{0x0303, "tls1.2"},
{0x0304, "tls1.3"},
};
struct cipher_suite g_cipher_suite_list[] = {
{0xC030, "ECDHE-RSA-AES256-GCM-SHA384"},
{0xC02C, "ECDHE-ECDSA-AES256-GCM-SHA384"},
{0xC028, "ECDHE-RSA-AES256-SHA384"},
{0xC024, "ECDHE-ECDSA-AES256-SHA384"},
{0xC014, "ECDHE-RSA-AES256-SHA"},
{0xC00A, "ECDHE-ECDSA-AES256-SHA"},
{0x00A5, "DH-DSS-AES256-GCM-SHA384"},
{0x00A3, "DHE-DSS-AES256-GCM-SHA384"},
{0x00A1, "DH-RSA-AES256-GCM-SHA384"},
{0x009F, "DHE-RSA-AES256-GCM-SHA384"},
{0x006B, "DHE-RSA-AES256-SHA256"},
{0x006A, "DHE-DSS-AES256-SHA256"},
{0x0069, "DH-RSA-AES256-SHA256"},
{0x0068, "DH-DSS-AES256-SHA256"},
{0x0039, "DHE-RSA-AES256-SHA"},
{0x0038, "DHE-DSS-AES256-SHA"},
{0x0037, "DH-RSA-AES256-SHA"},
{0x0036, "DH-DSS-AES256-SHA"},
{0x0088, "DHE-RSA-CAMELLIA256-SHA"},
{0x0087, "DHE-DSS-CAMELLIA256-SHA"},
{0x0086, "DH-RSA-CAMELLIA256-SHA"},
{0x0085, "DH-DSS-CAMELLIA256-SHA"},
{0xC019, "AECDH-AES256-SHA"},
{0x00A7, "ADH-AES256-GCM-SHA384"},
{0x006D, "ADH-AES256-SHA256"},
{0x003A, "ADH-AES256-SHA"},
{0x0089, "ADH-CAMELLIA256-SHA"},
{0xC032, "ECDH-RSA-AES256-GCM-SHA384"},
{0xC02E, "ECDH-ECDSA-AES256-GCM-SHA384"},
{0xC02A, "ECDH-RSA-AES256-SHA384"},
{0xC026, "ECDH-ECDSA-AES256-SHA384"},
{0xC00F, "ECDH-RSA-AES256-SHA"},
{0xC005, "ECDH-ECDSA-AES256-SHA"},
{0x009D, "AES256-GCM-SHA384"},
{0x003D, "AES256-SHA256"},
{0x0035, "AES256-SHA"},
{0x0084, "CAMELLIA256-SHA"},
{0x008D, "PSK-AES256-CBC-SHA"},
{0xC02F, "ECDHE-RSA-AES128-GCM-SHA256"},
{0xC02B, "ECDHE-ECDSA-AES128-GCM-SHA256"},
{0xC027, "ECDHE-RSA-AES128-SHA256"},
{0xC023, "ECDHE-ECDSA-AES128-SHA256"},
{0xC013, "ECDHE-RSA-AES128-SHA"},
{0xC009, "ECDHE-ECDSA-AES128-SHA"},
{0x00A4, "DH-DSS-AES128-GCM-SHA256"},
{0x00A2, "DHE-DSS-AES128-GCM-SHA256"},
{0x00A0, "DH-RSA-AES128-GCM-SHA256"},
{0x009E, "DHE-RSA-AES128-GCM-SHA256"},
{0x0067, "DHE-RSA-AES128-SHA256"},
{0x0040, "DHE-DSS-AES128-SHA256"},
{0x003F, "DH-RSA-AES128-SHA256"},
{0x003E, "DH-DSS-AES128-SHA256"},
{0x0033, "DHE-RSA-AES128-SHA"},
{0x0032, "DHE-DSS-AES128-SHA"},
{0x0031, "DH-RSA-AES128-SHA"},
{0x0030, "DH-DSS-AES128-SHA"},
{0x009A, "DHE-RSA-SEED-SHA"},
{0x0099, "DHE-DSS-SEED-SHA"},
{0x0098, "DH-RSA-SEED-SHA"},
{0x0097, "DH-DSS-SEED-SHA"},
{0x0045, "DHE-RSA-CAMELLIA128-SHA"},
{0x0044, "DHE-DSS-CAMELLIA128-SHA"},
{0x0043, "DH-RSA-CAMELLIA128-SHA"},
{0x0042, "DH-DSS-CAMELLIA128-SHA"},
{0xC018, "AECDH-AES128-SHA"},
{0x00A6, "ADH-AES128-GCM-SHA256"},
{0x006C, "ADH-AES128-SHA256"},
{0x0034, "ADH-AES128-SHA"},
{0x009B, "ADH-SEED-SHA"},
{0x0046, "ADH-CAMELLIA128-SHA"},
{0xC031, "ECDH-RSA-AES128-GCM-SHA256"},
{0xC02D, "ECDH-ECDSA-AES128-GCM-SHA256"},
{0xC029, "ECDH-RSA-AES128-SHA256"},
{0xC025, "ECDH-ECDSA-AES128-SHA256"},
{0xC00E, "ECDH-RSA-AES128-SHA"},
{0xC004, "ECDH-ECDSA-AES128-SHA"},
{0x009C, "AES128-GCM-SHA256"},
{0x003C, "AES128-SHA256"},
{0x002F, "AES128-SHA"},
{0x0096, "SEED-SHA"},
{0x0041, "CAMELLIA128-SHA"},
{0x008C, "PSK-AES128-CBC-SHA"},
{0xC012, "ECDHE-RSA-DES-CBC3-SHA"},
{0xC008, "ECDHE-ECDSA-DES-CBC3-SHA"},
{0x0016, "EDH-RSA-DES-CBC3-SHA"},
{0x0013, "EDH-DSS-DES-CBC3-SHA"},
{0x0010, "DH-RSA-DES-CBC3-SHA"},
{0x000D, "DH-DSS-DES-CBC3-SHA"},
{0xC017, "AECDH-DES-CBC3-SHA"},
{0x001B, "ADH-DES-CBC3-SHA"},
{0xC00D, "ECDH-RSA-DES-CBC3-SHA"},
{0xC003, "ECDH-ECDSA-DES-CBC3-SHA"},
{0x000A, "DES-CBC3-SHA"},
{0x0007, "IDEA-CBC-SHA"},
{0x008B, "PSK-3DES-EDE-CBC-SHA"},
{0x0021, "KRB5-IDEA-CBC-SHA"},
{0x001F, "KRB5-DES-CBC3-SHA"},
{0x0025, "KRB5-IDEA-CBC-MD5"},
{0x0023, "KRB5-DES-CBC3-MD5"},
{0xC011, "ECDHE-RSA-RC4-SHA"},
{0xC007, "ECDHE-ECDSA-RC4-SHA"},
{0xC016, "AECDH-RC4-SHA"},
{0x0018, "ADH-RC4-MD5"},
{0xC00C, "ECDH-RSA-RC4-SHA"},
{0xC002, "ECDH-ECDSA-RC4-SHA"},
{0x0005, "RC4-SHA"},
{0x0004, "RC4-MD5"},
{0x008A, "PSK-RC4-SHA"},
{0x0020, "KRB5-RC4-SHA"},
{0x0024, "KRB5-RC4-MD5"},
{0xC010, "ECDHE-RSA-NULL-SHA"},
{0xC006, "ECDHE-ECDSA-NULL-SHA"},
{0xC015, "AECDH-NULL-SHA"},
{0xC00B, "ECDH-RSA-NULL-SHA"},
{0xC001, "ECDH-ECDSA-NULL-SHA"},
{0x003B, "NULL-SHA256"},
{0x0002, "NULL-SHA"},
{0x0001, "NULL-MD5"},
{0x1301, "TLS_AES_128_GCM_SHA256"},
{0x1302, "TLS_AES_256_GCM_SHA384"},
{0x1303, "TLS_CHACHA20_POLY1305_SHA256"},
{0x1304, "TLS_AES_128_CCM_SHA256"},
{0x1305, "TLS_AES_128_CCM_8_SHA256"},
};
struct tls_extension g_tls_extension_list[] = {
{0, "server_name"},
{1, "max_fragment_length"},
{2, "client_certificate_url"},
{3, "trusted_ca_keys"},
{4, "truncated_hmac"},
{5, "status_request"},
{6, "user_mapping"},
{7, "client_authz"},
{8, "server_authz"},
{9, "cert_type"},
{10, "supported_groups"},
{11, "ec_point_formats"},
{12, "srp"},
{13, "signature_algorithms"},
{14, "use_srtp"},
{15, "heartbeat"},
{16, "application_layer_protocol_negotiation"},
{17, "status_request_v2"},
{18, "signed_certificate_timestamp"},
{19, "client_certificate_type"},
{20, "server_certificate_type"},
{21, "padding"},
{22, "encrypt_then_mac"},
{23, "extended_master_secret"},
{24, "token_binding"},
{25, "cached_info"},
{26, "tls_lts"},
{27, "compress_certificate"},
{28, "record_size_limit"},
{29, "pwd_protect"},
{30, "pwd_clear"},
{31, "password_salt"},
{32, "ticket_pinning"},
{35, "session_ticket"},
{41, "pre_shared_key"},
{42, "early_data"},
{43, "supported_versions"},
{44, "cookie"},
{45, "psk_key_exchange_modes"},
{47, "certificate_authorities"},
{48, "oid_filters"},
{49, "post_handshake_auth"},
{50, "signature_algorithms_cert"},
{51, "key_share"},
{52, "transparency_info"},
{53, "connection_id"},
{55, "external_id_hash"},
{56, "external_session_id"},
{65281, "renegotiation_info"},
};
void ssl_chello_free(struct ssl_chello* chello){
if(chello==NULL){
return;
}
free(chello);
}
static void parse_alpn_extension(const unsigned char* buff, size_t buff_len, enum chello_parse_result* result, char *alpn){
size_t pos = 0;
size_t len = ((size_t)buff[pos] << 8) + (size_t)buff[pos + 1];
if(2 + len != buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
strncpy((char*)alpn, (const char*)buff + 2, len);
alpn[len] = '\0';
*result = CHELLO_PARSE_SUCCESS;
return;
}
static void parse_server_name_extension(const unsigned char* buff, size_t buff_len, enum chello_parse_result* result, char *sni){
size_t pos = 2; /* skip server name list length */
size_t len;
while (pos + 3 < buff_len){
len = ((size_t)buff[pos + 1] << 8) + (size_t)buff[pos + 2];
if (pos + 3 + len > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
switch (buff[pos]){
case 0x00: /* host_name */
strncpy(sni, (const char*)buff + pos + 3, len);
sni[len] = '\0';
*result = CHELLO_PARSE_SUCCESS;
}
pos += 3 + len;
}
if (pos != buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
}
return;
}
int is_extension_existed(int value){
int n = sizeof(g_tls_extension_list) / sizeof(tls_extension);
for(int i = 0; i < n; i++){
if(g_tls_extension_list[i].value == value){
return 1;
}
}
return 0;
}
static enum chello_parse_result parse_extensions(const unsigned char* buff, size_t buff_len, struct ssl_chello* chello) {
size_t pos = 0;
/* Parse each 4 bytes for the extension header */
while (pos + 4 <= buff_len){
size_t type = ((size_t)buff[pos] << 8) + (size_t)buff[pos + 1];
size_t len = ((size_t)buff[pos + 2] << 8) + (size_t)buff[pos + 3];
int ret = is_extension_existed(type);
if(ret == 0){
pos += (4 + len);
continue;
}
chello->extension_list[chello->extension_count] = type;
chello->extension_count++;
/* Check if it's a server name extension */
if (buff[pos] == 0x00 && buff[pos + 1] == 0x00){
if (pos + 4 + len > buff_len){
return CHELLO_PARSE_INVALID_FORMAT;
}
enum chello_parse_result result = CHELLO_PARSE_SUCCESS;
parse_server_name_extension(buff + pos + 4, len, &result, chello->sni);
if(result != CHELLO_PARSE_SUCCESS){
return result;
}
}
/* Check if it's a alpn extension */
if (buff[pos] == 0x00 && buff[pos + 1] == 0x10){
if (pos + 4 + len > buff_len){
return CHELLO_PARSE_INVALID_FORMAT;
}
enum chello_parse_result result = CHELLO_PARSE_SUCCESS;
parse_alpn_extension(buff + pos + 4, len, &result, chello->alpn);
if(result != CHELLO_PARSE_SUCCESS){
return result;
}
}
pos += (4 + len);
}
/* Check we ended where we expected to */
if (pos != buff_len){
return CHELLO_PARSE_INVALID_FORMAT;
}
return CHELLO_PARSE_SUCCESS;
}
static void parse_cipher_suites(struct ssl_chello* chello, const unsigned char* buff, size_t buff_len, enum chello_parse_result* result){
int n = sizeof(g_cipher_suite_list) / sizeof(struct cipher_suite);
size_t pos = 0;
int flag = 0;
while(pos < buff_len){
int i = 0;
for(i = 0;i < n; i++){
int val = (buff[pos] << 8) + buff[pos + 1];
if(g_cipher_suite_list[i].value == val){
chello->cipher_suite_list[chello->cipher_suites_count] = val;
chello->cipher_suites_count++;
}
}
pos += 2;
if(flag == 1){
break;
}
}
if(pos != buff_len && flag == 0){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
*result = CHELLO_PARSE_SUCCESS;
return;
}
void get_ssl_version_str_format(uint16_t value, char *name){
int n = sizeof(g_ssl_veriosn_map_list) / sizeof(struct ssl_version_map);
for(int i = 0; i < n; i++){
if(g_ssl_veriosn_map_list[i].value == value){
strcpy(name, g_ssl_veriosn_map_list[i].name);
return;
}
}
printf("Failed at get_ssl_version_str_format(), value is %02x\n", value);
}
void ssl_chello_parse(struct ssl_chello* _chello, const unsigned char* buff, size_t buff_len, enum chello_parse_result* result){
if(buff == NULL){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
if(buff_len < 1){
*result = CHELLO_PARSE_NOT_ENOUGH_BUFF;
return;
}
if(buff[0] != 0x80 && buff[0] != 0x16){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
/* SSL 2.0 compatible Client Hello
* High bit of first byte (length) and content type is Client Hello
* See RFC5246 Appendix E.2
* if it is SSL 2.0, only parse version
*/
if(buff[0] == 0x80){
_chello->min_version.major = 0x02;
if(buff_len < 2){
*result = CHELLO_PARSE_NOT_ENOUGH_BUFF;
return;
}
size_t len = (size_t)buff[1];
if (buff_len < len + 2){
*result = CHELLO_PARSE_NOT_ENOUGH_BUFF;
return;
}
buff_len = len + 2;
size_t pos = 2;
/* Handshark Message Type: Client Hello */
if (pos + 1 > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
if (buff[pos] != 0x01){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
pos += 1;
/* Version */
if(pos + 2 > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
_chello->max_version.major = buff[pos];
_chello->max_version.minor = buff[pos + 1];
_chello->max_version.ossl_format=(uint16_t)_chello->max_version.major<<8|_chello->max_version.minor;
get_ssl_version_str_format(_chello->max_version.ossl_format, _chello->max_version.str_format);
*result = CHELLO_PARSE_SUCCESS;
return;
}
else{
if (buff_len < 5){
*result = CHELLO_PARSE_NOT_ENOUGH_BUFF;
return;
}
if(buff[1] != 3 || buff[2] > 4 || buff[2] < 0){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
_chello->min_version.major = buff[1];
_chello->min_version.minor = buff[2];
_chello->min_version.ossl_format=(uint16_t)_chello->min_version.major<<8|_chello->min_version.minor;
get_ssl_version_str_format(_chello->min_version.ossl_format, _chello->min_version.str_format);
_chello->max_version.major = -1;
_chello->max_version.minor = -1;
/* TLS record length */
size_t len = ((size_t)buff[3] << 8) + (size_t)buff[4] + 5;
if (buff_len < len){
*result = CHELLO_PARSE_NOT_ENOUGH_BUFF;
return;
}
buff_len = len;
size_t pos = 5;
if (pos + 1 > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
if (buff[pos] != 0x01){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
pos += 4;
if(pos + 2 > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
_chello->max_version.major = buff[pos];
_chello->max_version.minor = buff[pos+1];
_chello->max_version.ossl_format=(uint16_t)_chello->max_version.major<<8|_chello->max_version.minor;
get_ssl_version_str_format(_chello->max_version.ossl_format, _chello->max_version.str_format);
pos += 34;
/* Session ID */
if (pos + 1 > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
len = (size_t)buff[pos];
pos += 1 + len;
/* Cipher Suites */
if (pos + 2 > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
len = ((size_t)buff[pos] << 8) + (size_t)buff[pos + 1];
pos += 2;
if(pos + len > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
parse_cipher_suites(_chello, buff + pos, len, result);
if(*result != CHELLO_PARSE_SUCCESS){
return;
}
pos += len;
/* Compression Methods */
if (pos >= buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
len = (size_t)buff[pos];
pos += 1 + len;
/* no extensions */
if(pos == buff_len){
*result = CHELLO_PARSE_SUCCESS;
return;
}
/* Extensions */
if (pos + 2 > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
len = ((size_t)buff[pos] << 8) + (size_t)buff[pos + 1];
pos += 2;
if (pos + len > buff_len){
*result = CHELLO_PARSE_INVALID_FORMAT;
return;
}
enum chello_parse_result ret = parse_extensions(buff + pos, len, _chello);
*result = ret;
return;
}
}

317
DataSet/FeatureExtract/entry/src/tcp_entry.cpp Normal file
View File

@@ -0,0 +1,317 @@
#include "base_utils.h"
#include "ssl_utils.h"
#include "MESA/stream_inc/stream_base.h"
#include "MESA/stream_inc/stream_rawpkt.h"
#include "cjson/cJSON.h"
#define STREAM_PACKET_COUNT_MAX 200
/*
{
"sip": "0.0.0.0", // IP source address
"dip": "255.255.255.255", // IP destination address
"proto": 17, // IP protocol number (17 = UDP)
"sport": 68, // UDP source port "dp":
"dport": 67, // UDP destination port
"c2s_bytes": 900, // c to s bytes
"c2s__pkts": 3, // packets sent from c to s
"s2c_bytes": 900, // s to c bytes
"s2c__pkts": 3, // packets sent from s to c
"duration": 456, // 456 ms
"packets": [
{
"byte": 300, // bytes in UDP Data field
"dir": "c2s", // direction: sa -> da
"interval": 0 // inter-packet time: 0 ms since time_start
}, ...
],
"tls":{
"ssl_version": "tls1.3",
"cipher_suite": [...],
"sni" : "www.baidu.com",
"alpn" : "http2",
"extensions": [...]
}
}
*/
void *g_logger = NULL;
FILE *g_fp = NULL;
int g_count = 0;
int g_stream_count = 0;
int g_stream_succ_count = 0;
int g_stream_fail_count = 0;
int g_log_succ_count = 0;
struct pkt_stat_info{
struct timeval pkt_time;
int bytes;
int dir;
int interval;
};
struct pkt_parsed_info{
addr_type_t addr_type;
union{
struct iphdr *v4;
struct ip6_hdr *v6;
}iphdr;
uint16_t iphdr_len;
uint16_t ip_totlen;
struct tcphdr *tcphdr;
uint16_t tcphdr_len;
char *data;
uint16_t data_len;
};
struct pme_info{
int _errno;
char sip[INET_ADDRSTRLEN];
int sport;
char dip[INET_ADDRSTRLEN];
int dport;
int c2s_bytes;
int s2c_bytes;
int c2s_pkts;
int s2c_pkts;
int total_pkts;
int duration;
struct timeval start_time;
struct timeval end_time;
int last_c2s_pkt_index;
int last_s2c_pkt_index;
struct pkt_stat_info pkt_info_list[STREAM_PACKET_COUNT_MAX];
struct ssl_chello chello;
};
int ipv4_header_parse(const void *a_packet, struct pkt_parsed_info* pktinfo){
if(a_packet == NULL){
return -1;
}
pktinfo->addr_type = ADDR_TYPE_IPV4;
pktinfo->iphdr.v4 = (struct iphdr*)a_packet;
pktinfo->iphdr_len = pktinfo->iphdr.v4->ihl * 4;
pktinfo->ip_totlen = ntohs(pktinfo->iphdr.v4->tot_len);
pktinfo->tcphdr = (struct tcphdr*)((char*)pktinfo->iphdr.v4 + pktinfo->iphdr_len);
pktinfo->tcphdr_len = pktinfo->tcphdr->doff * 4;
pktinfo->data = (char*)pktinfo->tcphdr + pktinfo->tcphdr_len;
pktinfo->data_len = pktinfo->ip_totlen - pktinfo->iphdr_len - pktinfo->tcphdr_len;
/*
struct iphdr *_iphdr = pktinfo->iphdr.v4;
int ttl = _iphdr->ttl;
int ipid = ntohs(_iphdr->id);
printf("ipv4: ipid = %02x, ttl = %d, data_len = %d\n", ipid, ttl, pktinfo->data_len);
*/
return 0;
}
int packet_stat(struct streaminfo *stream, struct pme_info *pmeinfo, struct pkt_parsed_info* pktinfo){
if(pmeinfo->total_pkts == STREAM_PACKET_COUNT_MAX){
return -1;
}
pmeinfo->pkt_info_list[pmeinfo->total_pkts].bytes = pktinfo->data_len;
pmeinfo->pkt_info_list[pmeinfo->total_pkts].dir = stream->curdir;
get_rawpkt_opt_from_streaminfo(stream, RAW_PKT_GET_TIMESTAMP, &(pmeinfo->pkt_info_list[pmeinfo->total_pkts].pkt_time));
int last_pkt_index = -1;
if(stream->curdir == 1){
pmeinfo->c2s_pkts++;
pmeinfo->c2s_bytes += pktinfo->data_len;
last_pkt_index = pmeinfo->last_c2s_pkt_index;
pmeinfo->last_c2s_pkt_index = pmeinfo->total_pkts;
}
if(stream->curdir == 2){
pmeinfo->s2c_pkts++;
pmeinfo->s2c_bytes += pktinfo->data_len;
last_pkt_index = pmeinfo->last_s2c_pkt_index;
pmeinfo->last_s2c_pkt_index = pmeinfo->total_pkts;
}
if(last_pkt_index >= 0){
pmeinfo->pkt_info_list[pmeinfo->total_pkts].interval =
(pmeinfo->pkt_info_list[pmeinfo->total_pkts].pkt_time.tv_sec - pmeinfo->pkt_info_list[last_pkt_index].pkt_time.tv_sec) * 1000 +
(pmeinfo->pkt_info_list[pmeinfo->total_pkts].pkt_time.tv_usec - pmeinfo->pkt_info_list[last_pkt_index].pkt_time.tv_usec) / 1000;
}
pmeinfo->total_pkts++;
return 0;
}
int chello_packet_parse(struct pme_info *pmeinfo, struct pkt_parsed_info *pktinfo){
enum chello_parse_result chello_status = CHELLO_PARSE_INVALID_FORMAT;
char *buff = pktinfo->data;
int len = pktinfo->data_len;
ssl_chello_parse(&(pmeinfo->chello), (const unsigned char*)buff, len, &chello_status);
if(chello_status != CHELLO_PARSE_SUCCESS){
printf("Error: chello parse failed\n");
return -1;
}
return 0;
}
int packet_need_filter(struct pkt_parsed_info *pktinfo){
struct iphdr *_iphdr = pktinfo->iphdr.v4;
int ttl = _iphdr->ttl;
if(ttl == 70 || ttl == 75){
//printf("packet_need_filter: ret = 1, ttl = %d\n", ttl);
return 1;
}
int data_len = pktinfo->data_len;
if(data_len == 0){
//printf("packet_need_filter: ret 1, data_len = %d\n", data_len);
return 1;
}
return 0;
}
char pending_opstate(struct streaminfo *stream, struct pme_info *pmeinfo, struct pkt_parsed_info *pktinfo){
pmeinfo->last_c2s_pkt_index = -1;
pmeinfo->last_s2c_pkt_index = -1;
get_rawpkt_opt_from_streaminfo(stream, RAW_PKT_GET_TIMESTAMP, &(pmeinfo->start_time));
struct stream_tuple4_v4 *tuple4 = stream->addr.tuple4_v4;
inet_ntop(AF_INET, &(tuple4->saddr), pmeinfo->sip, INET_ADDRSTRLEN);
inet_ntop(AF_INET, &(tuple4->daddr), pmeinfo->dip, INET_ADDRSTRLEN);
pmeinfo->sport = ntohs(tuple4->source);
pmeinfo->dport = ntohs(tuple4->dest);
if(packet_need_filter(pktinfo) == 1){
return APP_STATE_FAWPKT | APP_STATE_GIVEME;
}
int ret = chello_packet_parse(pmeinfo, pktinfo);
if(ret < 0){
pmeinfo->_errno = -1;
return APP_STATE_FAWPKT | APP_STATE_DROPME;
}
packet_stat(stream, pmeinfo, pktinfo);
return APP_STATE_FAWPKT | APP_STATE_GIVEME;
}
char data_opstate(struct streaminfo *stream, struct pme_info *pmeinfo, struct pkt_parsed_info *pktinfo){
if(packet_need_filter(pktinfo) == 0){
int ret = packet_stat(stream, pmeinfo, pktinfo);
if(ret == -1){
return APP_STATE_FAWPKT | APP_STATE_DROPME;
}
}
return APP_STATE_FAWPKT | APP_STATE_GIVEME;
}
void output_result(struct pme_info *pmeinfo){
cJSON *log_obj = cJSON_CreateObject();
cJSON_AddStringToObject(log_obj, "sip", pmeinfo->sip);
cJSON_AddNumberToObject(log_obj, "sport", pmeinfo->sport);
cJSON_AddStringToObject(log_obj, "dip", pmeinfo->dip);
cJSON_AddNumberToObject(log_obj, "dport", pmeinfo->dport);
cJSON_AddStringToObject(log_obj, "proto", "tcp");
cJSON_AddNumberToObject(log_obj, "c2s_bytes", pmeinfo->c2s_bytes);
cJSON_AddNumberToObject(log_obj, "s2c_bytes", pmeinfo->s2c_bytes);
cJSON_AddNumberToObject(log_obj, "c2s_pkts", pmeinfo->c2s_pkts);
cJSON_AddNumberToObject(log_obj, "s2c_pkts", pmeinfo->s2c_pkts);
cJSON_AddNumberToObject(log_obj, "total_pkts", pmeinfo->total_pkts);
cJSON_AddNumberToObject(log_obj, "duration", pmeinfo->duration);
cJSON *pkt_info_list = cJSON_CreateArray();
for(int i = 0; i < pmeinfo->total_pkts; i++){
cJSON *pkt_info = cJSON_CreateObject();
cJSON_AddNumberToObject(pkt_info, "bytes", pmeinfo->pkt_info_list[i].bytes);
cJSON_AddNumberToObject(pkt_info, "dir", pmeinfo->pkt_info_list[i].dir);
cJSON_AddNumberToObject(pkt_info, "interval", pmeinfo->pkt_info_list[i].interval);
cJSON_AddItemToArray(pkt_info_list, pkt_info);
}
cJSON_AddItemToObject(log_obj, "packets", pkt_info_list);
cJSON *tls_info = cJSON_CreateObject();
cJSON_AddStringToObject(tls_info, "version", pmeinfo->chello.max_version.str_format);
cJSON_AddStringToObject(tls_info, "sni", pmeinfo->chello.sni);
cJSON_AddStringToObject(tls_info, "alpn", pmeinfo->chello.alpn);
cJSON *cipher_suite_list = cJSON_CreateArray();
for(int i = 0; i < pmeinfo->chello.cipher_suites_count; i++){
char cipher_suite_str[4] = {0};
sprintf(cipher_suite_str, "%04x", pmeinfo->chello.cipher_suite_list[i]);
cJSON_AddItemToArray(cipher_suite_list, cJSON_CreateString(cipher_suite_str));
}
cJSON_AddItemToObject(tls_info, "cipher_suites", cipher_suite_list);
cJSON *extensions_list = cJSON_CreateArray();
for(int i = 0; i < pmeinfo->chello.extension_count; i++){
cJSON_AddItemToArray(extensions_list, cJSON_CreateNumber(pmeinfo->chello.extension_list[i]));
}
cJSON_AddItemToObject(tls_info, "extensions_list", extensions_list);
cJSON_AddItemToObject(log_obj, "tls", tls_info);
char *log_msg = cJSON_PrintUnformatted(log_obj);
//printf("%s\n\n", log_msg);
LOG_INFO(g_logger, log_msg);
fputs(log_msg, g_fp);
fputs("\n", g_fp);
g_log_succ_count++;
cJSON_Delete(log_obj);
cJSON_free(log_msg);
}
char close_opstate(struct streaminfo *stream, struct pme_info *pmeinfo, struct pkt_parsed_info *pktinfo, const void *a_packet){
if(a_packet != NULL){
if(packet_need_filter(pktinfo) == 0){
packet_stat(stream, pmeinfo, pktinfo);
}
}
get_rawpkt_opt_from_streaminfo(stream, RAW_PKT_GET_TIMESTAMP, &(pmeinfo->end_time));
pmeinfo->duration = (pmeinfo->end_time.tv_sec - pmeinfo->start_time.tv_sec) * 1000 + (pmeinfo->end_time.tv_usec - pmeinfo->start_time.tv_usec) / 1000;
return APP_STATE_FAWPKT | APP_STATE_DROPME;
}
void pme_info_destroy(struct pme_info *pmeinfo){
FREE(&pmeinfo);
pmeinfo = NULL;
}
extern "C" char stmstat_entry(struct streaminfo *stream, void** pme, int thread_seq, const void* a_packet){
if(g_count % 100 == 5){
printf("handle %d packets\n", g_count);
printf("stream_count: %d\nsucc_count: %d\nfail_count: %d\ng_log_succ_count: %d\n", g_stream_count, g_stream_succ_count, g_stream_fail_count, g_log_succ_count);
}
g_count++;
char ret;
struct pme_info *pmeinfo = *(struct pme_info **)pme;
//printf("pmeinfo = %p, opstate = %d, a_packet = %p\n", pmeinfo, stream->opstate, a_packet);
if(a_packet == NULL && stream->opstate != OP_STATE_CLOSE){
return APP_STATE_FAWPKT | APP_STATE_GIVEME;
}
struct pkt_parsed_info pktinfo;
memset(&pktinfo, 0, sizeof(pktinfo));
ipv4_header_parse(a_packet, &pktinfo);
switch(stream->opstate){
case OP_STATE_PENDING:
//printf("call pending\n");
pmeinfo = ALLOC(struct pme_info, 1);
*pme = pmeinfo;
g_stream_count++;
ret = pending_opstate(stream, pmeinfo, &pktinfo);
break;
case OP_STATE_DATA:
//printf("call data\n");
ret = data_opstate(stream, pmeinfo, &pktinfo);
break;
case OP_STATE_CLOSE:
//printf("call close\n");
ret = close_opstate(stream, pmeinfo, &pktinfo, a_packet);
break;
default:
break;
}
if((ret & APP_STATE_DROPME)){
if(pmeinfo->_errno >= 0){
g_stream_succ_count++;
output_result(pmeinfo);
}
else{
g_stream_fail_count++;
}
pme_info_destroy(pmeinfo);
}
return ret;
}
extern "C" int stmstat_init(){
char *log_path = (char*)"./stream_stat.log";
int log_level = 10;
g_logger = MESA_create_runtime_log_handle(log_path, log_level);
g_fp = fopen("./stream_stat.txt", "w+");
return 0;
}
extern "C" void stmstat_destroy(){
printf("stream_count: %d\nsucc_count: %d\nfail_count: %d\n", g_stream_count, g_stream_succ_count, g_stream_fail_count);
}

53
DataSet/FeatureExtract/vendor/CMakeLists.txt vendored Normal file
View File

@@ -0,0 +1,53 @@
# CMakeFiles for 3rd vendor library
include(ExternalProject)
### cJSON: 注意: -DCMAKE_POSITION_INDEPENDENT_CODE=ON
ExternalProject_Add(cJSON PREFIX cJSON
URL ${CMAKE_CURRENT_SOURCE_DIR}/cJSON-1.7.7.tar.gz
URL_MD5 715009c99728bf81d6c97352718650ff
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DBUILD_SHARED_AND_STATIC_LIBS=1)
ExternalProject_Get_Property(cJSON INSTALL_DIR)
file(MAKE_DIRECTORY ${INSTALL_DIR}/include)
add_library(cjson SHARED IMPORTED GLOBAL)
add_dependencies(cjson cJSON)
set_property(TARGET cjson PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib64/libcjson.a)
set_property(TARGET cjson PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)
### MESA Framework
set(MESA_FRAMEWORK_LIB_DIR /opt/MESA/lib)
set(MESA_FRAMEWORK_INCLUDE_DIR /opt/MESA/include)
add_library(MESA_handle_logger SHARED IMPORTED GLOBAL)
set_property(TARGET MESA_handle_logger PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/libMESA_handle_logger.so)
set_property(TARGET MESA_handle_logger PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR})
add_library(MESA_prof_load SHARED IMPORTED GLOBAL)
set_property(TARGET MESA_prof_load PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/libMESA_prof_load.so)
set_property(TARGET MESA_prof_load PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR})
add_library(wiredcfg SHARED IMPORTED GLOBAL)
set_property(TARGET wiredcfg PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/libwiredcfg.so)
set_property(TARGET wiredcfg PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR})
add_library(MESA_htable SHARED IMPORTED GLOBAL)
set_property(TARGET MESA_htable PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/libMESA_htable.so)
set_property(TARGET MESA_htable PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR})
add_library(maatframe SHARED IMPORTED GLOBAL)
set_property(TARGET maatframe PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/libmaatframe.so)
set_property(TARGET maatframe PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR})
add_library(MESA_field_stat SHARED IMPORTED GLOBAL)
set_property(TARGET MESA_field_stat PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/libMESA_field_stat2.so)
set_property(TARGET MESA_field_stat PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR})
add_library(rdkafka SHARED IMPORTED GLOBAL)
set_property(TARGET rdkafka PROPERTY IMPORTED_LOCATION ${MESA_FRAMEWORK_LIB_DIR}/librdkafka.so)
set_property(TARGET rdkafka PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${MESA_FRAMEWORK_INCLUDE_DIR})

BIN
DataSet/FeatureExtract/vendor/cJSON-1.7.7.tar.gz vendored Normal file
View File

Binary file not shown.