From 3abe8a2fc5a3acacb29ddfa1b3be8ad9771d4324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E4=B8=80=E9=B8=A3?= Date: Wed, 13 Nov 2019 16:11:38 +0800 Subject: [PATCH 01/32] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kni=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E6=89=AB=E6=8F=8F=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + inc/ssl_utils.h | 35 ++++ inc/tsg_rule.h | 1 - src/CMakeLists.txt | 2 +- src/ssl_utils.cpp | 473 +++++++++++++++++++++++++++++++++++++++++++++ src/tsg_rule.cpp | 95 ++++++++- 6 files changed, 595 insertions(+), 12 deletions(-) create mode 100644 .gitignore create mode 100644 inc/ssl_utils.h create mode 100644 src/ssl_utils.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a3062be --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode/* diff --git a/inc/ssl_utils.h b/inc/ssl_utils.h new file mode 100644 index 0000000..5da3bcf --- /dev/null +++ b/inc/ssl_utils.h @@ -0,0 +1,35 @@ +#pragma once + +struct cipher_suite +{ + int value; + const char* name; +}; + +enum chello_parse_result +{ + CHELLO_PARSE_SUCCESS = 0, + CHELLO_PARSE_INVALID_FORMAT = -1, + CHELLO_PARSE_NOT_ENOUGH_BUFF = -2 +}; + +struct ssl_version +{ + uint8_t minor; + uint8_t major; + uint16_t ossl_format; +}; + +struct ssl_chello +{ + struct ssl_version min_version; + struct ssl_version max_version; + + char* sni; + char* alpn; + char* cipher_suites; + char* cipher_suites_tls13; +}; +struct ssl_chello* ssl_chello_parse(const unsigned char* buff, size_t buff_len, enum chello_parse_result* result); + +void ssl_chello_free(struct ssl_chello* chello); \ No newline at end of file diff --git a/inc/tsg_rule.h b/inc/tsg_rule.h index d1fac1a..c8d085e 100644 --- a/inc/tsg_rule.h +++ b/inc/tsg_rule.h @@ -30,7 +30,6 @@ int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, voi //return -1 if failed, return 0 on success; int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, scan_status_t *mid, void *logger, int thread_seq); - #ifdef __cplusplus } #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3361c84..2d19c2b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8) add_definitions(-fPIC) -set(SRC tsg_entry.cpp tsg_rule.cpp tsg_send_log.cpp) +set(SRC tsg_entry.cpp tsg_rule.cpp ssl_utils.cpp tsg_send_log.cpp) include_directories(${CMAKE_SOURCE_DIR}/inc) include_directories(/opt/MESA/include/MESA/) diff --git a/src/ssl_utils.cpp b/src/ssl_utils.cpp new file mode 100644 index 0000000..fa297b5 --- /dev/null +++ b/src/ssl_utils.cpp @@ -0,0 +1,473 @@ +#include +#include +#include +#include +#include "ssl_utils.h" + +#define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) +#define FREE(p) {free(*p);*p=NULL;} +#define KNI_STRING_MAX 2048 + +struct cipher_suite cipher_suite_list[] = +{ + {0xC030, "ECDHE-RSA-AES256-GCM-SHA384"}, + {0xC02C, "ECDHE-ECDSA-AES256-GCM-SHA384"}, + {0xC028, "ECDHE-RSA-AES256-SHA384"}, + {0xC024, "ECDHE-ECDSA-AES256-SHA384"}, + {0xC014, "ECDHE-RSA-AES256-SHA"}, + {0xC00A, "ECDHE-ECDSA-AES256-SHA"}, + {0x00A5, "DH-DSS-AES256-GCM-SHA384"}, + {0x00A3, "DHE-DSS-AES256-GCM-SHA384"}, + {0x00A1, "DH-RSA-AES256-GCM-SHA384"}, + {0x009F, "DHE-RSA-AES256-GCM-SHA384"}, + {0x006B, "DHE-RSA-AES256-SHA256"}, + {0x006A, "DHE-DSS-AES256-SHA256"}, + {0x0069, "DH-RSA-AES256-SHA256"}, + {0x0068, "DH-DSS-AES256-SHA256"}, + {0x0039, "DHE-RSA-AES256-SHA"}, + {0x0038, "DHE-DSS-AES256-SHA"}, + {0x0037, "DH-RSA-AES256-SHA"}, + {0x0036, "DH-DSS-AES256-SHA"}, + {0x0088, "DHE-RSA-CAMELLIA256-SHA"}, + {0x0087, "DHE-DSS-CAMELLIA256-SHA"}, + {0x0086, "DH-RSA-CAMELLIA256-SHA"}, + {0x0085, "DH-DSS-CAMELLIA256-SHA"}, + {0xC019, "AECDH-AES256-SHA"}, + {0x00A7, "ADH-AES256-GCM-SHA384"}, + {0x006D, "ADH-AES256-SHA256"}, + {0x003A, "ADH-AES256-SHA"}, + {0x0089, "ADH-CAMELLIA256-SHA"}, + {0xC032, "ECDH-RSA-AES256-GCM-SHA384"}, + {0xC02E, "ECDH-ECDSA-AES256-GCM-SHA384"}, + {0xC02A, "ECDH-RSA-AES256-SHA384"}, + {0xC026, "ECDH-ECDSA-AES256-SHA384"}, + {0xC00F, "ECDH-RSA-AES256-SHA"}, + {0xC005, "ECDH-ECDSA-AES256-SHA"}, + {0x009D, "AES256-GCM-SHA384"}, + {0x003D, "AES256-SHA256"}, + {0x0035, "AES256-SHA"}, + {0x0084, "CAMELLIA256-SHA"}, + {0x008D, "PSK-AES256-CBC-SHA"}, + {0xC02F, "ECDHE-RSA-AES128-GCM-SHA256"}, + {0xC02B, "ECDHE-ECDSA-AES128-GCM-SHA256"}, + {0xC027, "ECDHE-RSA-AES128-SHA256"}, + {0xC023, "ECDHE-ECDSA-AES128-SHA256"}, + {0xC013, "ECDHE-RSA-AES128-SHA"}, + {0xC009, "ECDHE-ECDSA-AES128-SHA"}, + {0x00A4, "DH-DSS-AES128-GCM-SHA256"}, + {0x00A2, "DHE-DSS-AES128-GCM-SHA256"}, + {0x00A0, "DH-RSA-AES128-GCM-SHA256"}, + {0x009E, "DHE-RSA-AES128-GCM-SHA256"}, + {0x0067, "DHE-RSA-AES128-SHA256"}, + {0x0040, "DHE-DSS-AES128-SHA256"}, + {0x003F, "DH-RSA-AES128-SHA256"}, + {0x003E, "DH-DSS-AES128-SHA256"}, + {0x0033, "DHE-RSA-AES128-SHA"}, + {0x0032, "DHE-DSS-AES128-SHA"}, + {0x0031, "DH-RSA-AES128-SHA"}, + {0x0030, "DH-DSS-AES128-SHA"}, + {0x009A, "DHE-RSA-SEED-SHA"}, + {0x0099, "DHE-DSS-SEED-SHA"}, + {0x0098, "DH-RSA-SEED-SHA"}, + {0x0097, "DH-DSS-SEED-SHA"}, + {0x0045, "DHE-RSA-CAMELLIA128-SHA"}, + {0x0044, "DHE-DSS-CAMELLIA128-SHA"}, + {0x0043, "DH-RSA-CAMELLIA128-SHA"}, + {0x0042, "DH-DSS-CAMELLIA128-SHA"}, + {0xC018, "AECDH-AES128-SHA"}, + {0x00A6, "ADH-AES128-GCM-SHA256"}, + {0x006C, "ADH-AES128-SHA256"}, + {0x0034, "ADH-AES128-SHA"}, + {0x009B, "ADH-SEED-SHA"}, + {0x0046, "ADH-CAMELLIA128-SHA"}, + {0xC031, "ECDH-RSA-AES128-GCM-SHA256"}, + {0xC02D, "ECDH-ECDSA-AES128-GCM-SHA256"}, + {0xC029, "ECDH-RSA-AES128-SHA256"}, + {0xC025, "ECDH-ECDSA-AES128-SHA256"}, + {0xC00E, "ECDH-RSA-AES128-SHA"}, + {0xC004, "ECDH-ECDSA-AES128-SHA"}, + {0x009C, "AES128-GCM-SHA256"}, + {0x003C, "AES128-SHA256"}, + {0x002F, "AES128-SHA"}, + {0x0096, "SEED-SHA"}, + {0x0041, "CAMELLIA128-SHA"}, + {0x008C, "PSK-AES128-CBC-SHA"}, + {0xC012, "ECDHE-RSA-DES-CBC3-SHA"}, + {0xC008, "ECDHE-ECDSA-DES-CBC3-SHA"}, + {0x0016, "EDH-RSA-DES-CBC3-SHA"}, + {0x0013, "EDH-DSS-DES-CBC3-SHA"}, + {0x0010, "DH-RSA-DES-CBC3-SHA"}, + {0x000D, "DH-DSS-DES-CBC3-SHA"}, + {0xC017, "AECDH-DES-CBC3-SHA"}, + {0x001B, "ADH-DES-CBC3-SHA"}, + {0xC00D, "ECDH-RSA-DES-CBC3-SHA"}, + {0xC003, "ECDH-ECDSA-DES-CBC3-SHA"}, + {0x000A, "DES-CBC3-SHA"}, + {0x0007, "IDEA-CBC-SHA"}, + {0x008B, "PSK-3DES-EDE-CBC-SHA"}, + {0x0021, "KRB5-IDEA-CBC-SHA"}, + {0x001F, "KRB5-DES-CBC3-SHA"}, + {0x0025, "KRB5-IDEA-CBC-MD5"}, + {0x0023, "KRB5-DES-CBC3-MD5"}, + {0xC011, "ECDHE-RSA-RC4-SHA"}, + {0xC007, "ECDHE-ECDSA-RC4-SHA"}, + {0xC016, "AECDH-RC4-SHA"}, + {0x0018, "ADH-RC4-MD5"}, + {0xC00C, "ECDH-RSA-RC4-SHA"}, + {0xC002, "ECDH-ECDSA-RC4-SHA"}, + {0x0005, "RC4-SHA"}, + {0x0004, "RC4-MD5"}, + {0x008A, "PSK-RC4-SHA"}, + {0x0020, "KRB5-RC4-SHA"}, + {0x0024, "KRB5-RC4-MD5"}, + {0xC010, "ECDHE-RSA-NULL-SHA"}, + {0xC006, "ECDHE-ECDSA-NULL-SHA"}, + {0xC015, "AECDH-NULL-SHA"}, + {0xC00B, "ECDH-RSA-NULL-SHA"}, + {0xC001, "ECDH-ECDSA-NULL-SHA"}, + {0x003B, "NULL-SHA256"}, + {0x0002, "NULL-SHA"}, + {0x0001, "NULL-MD5"} +}; + +struct cipher_suite cipher_suite_list_tls13[] = +{ + {0x1301, "TLS_AES_128_GCM_SHA256"}, + {0x1302, "TLS_AES_256_GCM_SHA384"}, + {0x1303, "TLS_CHACHA20_POLY1305_SHA256"}, + {0x1304, "TLS_AES_128_CCM_SHA256"}, + {0x1305, "TLS_AES_128_CCM_8_SHA256"} +}; + +void ssl_chello_free(struct ssl_chello* chello) +{ + if(chello==NULL) + { + return; + } + free(chello->sni); + chello->sni = NULL; + free(chello->alpn); + chello->alpn = NULL; + free(chello->cipher_suites); + chello->cipher_suites = NULL; + free(chello->cipher_suites_tls13); + chello->cipher_suites_tls13 = NULL; + free(chello); +} + +static char* parse_alpn_extension(const unsigned char* buff, size_t buff_len, enum chello_parse_result* result) +{ + size_t pos = 0; + size_t len = ((size_t)buff[pos] << 8) + (size_t)buff[pos + 1]; + if(2 + len != buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return NULL; + } + char* alpn = ALLOC(char, len + 1); + strncpy((char*)alpn, (const char*)buff + 2, len); + alpn[len] = '\0'; + *result = CHELLO_PARSE_SUCCESS; + return alpn; +} + +static char* parse_server_name_extension(const unsigned char* buff, size_t buff_len, enum chello_parse_result* result) +{ + size_t pos = 2; /* skip server name list length */ + size_t len; + char* sni = NULL; + while (pos + 3 < buff_len) + { + len = ((size_t)buff[pos + 1] << 8) + (size_t)buff[pos + 2]; + if (pos + 3 + len > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return NULL; + } + switch (buff[pos]) + { + case 0x00: /* host_name */ + sni = (char*)malloc(len + 1); + strncpy(sni, (const char*)buff + pos + 3, len); + sni[len] = '\0'; + *result = CHELLO_PARSE_SUCCESS; + } + pos += 3 + len; + } + if (pos != buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + } + return sni; +} + +static enum chello_parse_result parse_extensions(const unsigned char* buff, size_t buff_len, struct ssl_chello* chello) { + size_t pos = 0; + /* Parse each 4 bytes for the extension header */ + while (pos + 4 <= buff_len) + { + size_t len = ((size_t)buff[pos + 2] << 8) + (size_t)buff[pos + 3]; + /* Check if it's a server name extension */ + if (buff[pos] == 0x00 && buff[pos + 1] == 0x00) + { + if (pos + 4 + len > buff_len) + { + return CHELLO_PARSE_INVALID_FORMAT; + } + enum chello_parse_result result = CHELLO_PARSE_SUCCESS; + chello->sni = parse_server_name_extension(buff + pos + 4, len, &result); + if(result != CHELLO_PARSE_SUCCESS) + { + return result; + } + } + /* Check if it's a alpn extension */ + if (buff[pos] == 0x00 && buff[pos + 1] == 0x10) + { + if (pos + 4 + len > buff_len) + { + return CHELLO_PARSE_INVALID_FORMAT; + } + enum chello_parse_result result = CHELLO_PARSE_SUCCESS; + chello->alpn = parse_alpn_extension(buff + pos + 4, len, &result); + if(result != CHELLO_PARSE_SUCCESS) + { + return result; + } + } + pos += (4 + len); + } + /* Check we ended where we expected to */ + if (pos != buff_len) + { + return CHELLO_PARSE_INVALID_FORMAT; + } + return CHELLO_PARSE_SUCCESS; +} + +static char* parse_cipher_suites(struct cipher_suite* _cipher_suite_list, int n, const unsigned char* buff, size_t buff_len, enum chello_parse_result* result) +{ + char* cipher_suites_str = (char* )malloc(KNI_STRING_MAX); + cipher_suites_str[0] = '\0'; + size_t pos = 0; + int flag = 0; + while(pos < buff_len) + { + int i = 0; + for(i = 0;i < n; i++) + { + int val = (buff[pos] << 8) + buff[pos + 1]; + if(_cipher_suite_list[i].value == val) + { + if(strnlen(_cipher_suite_list[i].name, KNI_STRING_MAX) + strnlen(cipher_suites_str, KNI_STRING_MAX) + 1 > KNI_STRING_MAX) + { + flag = 1; + break; + } + strncat(cipher_suites_str, _cipher_suite_list[i].name, KNI_STRING_MAX); + strncat(cipher_suites_str, ":", KNI_STRING_MAX); + } + } + pos += 2; + if(flag == 1) + { + break; + } + } + int len = strnlen(cipher_suites_str, KNI_STRING_MAX); + if(len > 0) + { + cipher_suites_str[len-1] = '\0'; + } + if(pos != buff_len && flag == 0) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + free(cipher_suites_str); + return NULL; + } + *result = CHELLO_PARSE_SUCCESS; + return cipher_suites_str; +} + +struct ssl_chello* ssl_chello_parse(const unsigned char* buff, size_t buff_len, enum chello_parse_result* result) +{ + if(buff == NULL) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return NULL; + } + if(buff_len < 1) + { + *result = CHELLO_PARSE_NOT_ENOUGH_BUFF; + return NULL; + } + if(buff[0] != 0x80 && buff[0] != 0x16) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return NULL; + } + /* SSL 2.0 compatible Client Hello + * High bit of first byte (length) and content type is Client Hello + * See RFC5246 Appendix E.2 + * if it is SSL 2.0, only parse version + */ + if(buff[0] == 0x80) + { + struct ssl_chello* _chello = (struct ssl_chello*)ALLOC(struct ssl_chello, 1); + _chello->min_version.major = 0x02; + if(buff_len < 2) + { + *result = CHELLO_PARSE_NOT_ENOUGH_BUFF; + return _chello; + } + size_t len = (size_t)buff[1]; + if (buff_len < len + 2) + { + *result = CHELLO_PARSE_NOT_ENOUGH_BUFF; + return _chello; + } + buff_len = len + 2; + size_t pos = 2; + /* Handshark Message Type: Client Hello */ + if (pos + 1 > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + if (buff[pos] != 0x01) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + pos += 1; + /* Version */ + if(pos + 2 > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + _chello->max_version.major = buff[pos]; + _chello->max_version.minor = buff[pos + 1]; + _chello->max_version.ossl_format=(uint16_t)_chello->max_version.major<<8|_chello->max_version.minor; + *result = CHELLO_PARSE_SUCCESS; + return _chello; + } + else + { + if (buff_len < 5) + { + *result = CHELLO_PARSE_NOT_ENOUGH_BUFF; + return NULL; + } + if(buff[1] != 3 || buff[2] > 4 || buff[2] < 0) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return NULL; + } + struct ssl_chello* _chello = (struct ssl_chello*)ALLOC(struct ssl_chello, 1); + _chello->min_version.major = buff[1]; + _chello->min_version.minor = buff[2]; + _chello->min_version.ossl_format=(uint16_t)_chello->min_version.major<<8|_chello->min_version.minor; + _chello->max_version.major = -1; + _chello->max_version.minor = -1; + _chello->sni = NULL; + _chello->alpn = NULL; + _chello->cipher_suites = NULL; + _chello->cipher_suites_tls13 = NULL; + /* TLS record length */ + size_t len = ((size_t)buff[3] << 8) + (size_t)buff[4] + 5; + if (buff_len < len) + { + *result = CHELLO_PARSE_NOT_ENOUGH_BUFF; + return _chello; + } + buff_len = len; + size_t pos = 5; + if (pos + 1 > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + if (buff[pos] != 0x01) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + pos += 4; + if(pos + 2 > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + _chello->max_version.major = buff[pos]; + _chello->max_version.minor = buff[pos+1]; + _chello->max_version.ossl_format=(uint16_t)_chello->max_version.major<<8|_chello->max_version.minor; + + pos += 34; + /* Session ID */ + if (pos + 1 > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + len = (size_t)buff[pos]; + pos += 1 + len; + /* Cipher Suites */ + if (pos + 2 > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + len = ((size_t)buff[pos] << 8) + (size_t)buff[pos + 1]; + pos += 2; + if(pos + len > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + int n = sizeof(cipher_suite_list) / sizeof(struct cipher_suite); + _chello->cipher_suites = parse_cipher_suites(cipher_suite_list, n, buff + pos, len, result); + if(*result != CHELLO_PARSE_SUCCESS) + { + return _chello; + } + n = sizeof(cipher_suite_list_tls13) / sizeof(struct cipher_suite); + _chello->cipher_suites_tls13 = parse_cipher_suites(cipher_suite_list_tls13, n, buff + pos, len, result); + if(*result != CHELLO_PARSE_SUCCESS) + { + return _chello; + } + pos += len; + /* Compression Methods */ + if (pos >= buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + len = (size_t)buff[pos]; + pos += 1 + len; + /* no extensions */ + if(pos == buff_len) + { + *result = CHELLO_PARSE_SUCCESS; + return _chello; + } + /* Extensions */ + if (pos + 2 > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + len = ((size_t)buff[pos] << 8) + (size_t)buff[pos + 1]; + pos += 2; + if (pos + len > buff_len) + { + *result = CHELLO_PARSE_INVALID_FORMAT; + return _chello; + } + enum chello_parse_result ret = parse_extensions(buff + pos, len, _chello); + *result = ret; + return _chello; + } +} diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 639285e..4c7e843 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -1,18 +1,93 @@ -#include -#include -#include -#include "tsg_rule.h" +#include "MESA/MESA_handle_logger.h" +#include "Maat_rule.h" +#include "Maat_command.h" +#include "ssl_utils.h" -extern Maat_feather_t g_tsg_maat_feather; +#define KNI_DOMAIN_MAX 256 -int tsg_pull_policy_result(PULL_RESULT_TYPE pull_result_type, Maat_rule_t*result, int result_num) +enum kni_protocol{ + KNI_PROTOCOL_UNKNOWN = 0, + KNI_PROTOCOL_SSL, + KNI_PROTOCOL_HTTP, +}; + +enum kni_scan_table { - return 0; + TSG_FIELD_SSL_SNI, + TSG_FIELD_HTTP_HOST, + SCAN_TABLE_MAX +}; + +struct kni_protocol_identify_result{ + int protocol; + char domain[KNI_DOMAIN_MAX]; + int domain_len; +}; + +Maat_feather_t g_kni_maat_feather; +const char *g_kni_scan_table_name[SCAN_TABLE_MAX]; +int g_kni_scan_tableid[SCAN_TABLE_MAX] = {0}; + +static void protocol_identify(char *buff, int buff_len, struct kni_protocol_identify_result *result){ + result->protocol = KNI_PROTOCOL_UNKNOWN; + //TODO: http: get from http protocol plugin + + //ssl + enum chello_parse_result chello_status = CHELLO_PARSE_INVALID_FORMAT; + struct ssl_chello *chello = NULL; + chello = ssl_chello_parse((const unsigned char*)buff, buff_len, &chello_status); + if(chello_status == CHELLO_PARSE_SUCCESS){ + result->protocol = KNI_PROTOCOL_SSL; + if(chello->sni == NULL){ + result->domain_len = 0; + } + else{ + result->domain_len = strnlen(chello->sni, KNI_DOMAIN_MAX); + strncpy(result->domain, chello->sni, strnlen(chello->sni, sizeof(result->domain) - 1)); + } + } + ssl_chello_free(chello); + return; } -int tsg_scan_nesting_addr(Maat_feather_t maat_feather, struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num) -{ - return 0; +//return -1 if failed, return 0 on success; +int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, void *logger){ + g_kni_maat_feather = maat_feather; + g_kni_scan_table_name[TSG_FIELD_HTTP_HOST] = "TSG_FIELD_HTTP_HOST"; + g_kni_scan_table_name[TSG_FIELD_SSL_SNI] = "TSG_FIELD_SSL_SNI"; + int i; + for(i = 0; i < SCAN_TABLE_MAX; i++){ + g_kni_scan_tableid[i] = Maat_table_register(maat_feather, g_kni_scan_table_name[i]); + if(g_kni_scan_tableid[i] < 0){ + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Failed at Maat_table_register, tablename = %s, ret = %d", + g_kni_scan_table_name[i], g_kni_scan_tableid[i]); + return -1; + } + } + return 0; } +//return -1 if failed, return 0 on success +int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, scan_status_t *mid, void *logger, int thread_seq){ + struct kni_protocol_identify_result protocol_identify_res; + memset(&protocol_identify_res, 0, sizeof(protocol_identify_res)); + protocol_identify((char*)pkt, pkt_len, &protocol_identify_res); + switch(protocol_identify_res.protocol){ + case KNI_PROTOCOL_UNKNOWN: + //TODO: field_stat refactor + //FS_operate(g_kni_fs_handle->handle, g_kni_fs_handle->fields[KNI_FIELD_STMERR_UNSUPPORTED_PROTOCOL], 0, FS_OP_ADD, 1); + //pmeinfo->error = STREAM_ERROR_UNSUPPORTED_PROTOCOL; + return -1; + case KNI_PROTOCOL_SSL: + return Maat_full_scan_string(g_kni_maat_feather, g_kni_scan_tableid[TSG_FIELD_SSL_SNI], CHARSET_UTF8, + protocol_identify_res.domain, protocol_identify_res.domain_len, + result, NULL, result_num, mid, thread_seq); + case KNI_PROTOCOL_HTTP: + return Maat_full_scan_string(g_kni_maat_feather, g_kni_scan_tableid[TSG_FIELD_HTTP_HOST], CHARSET_UTF8, + protocol_identify_res.domain, protocol_identify_res.domain_len, + result, NULL, result_num, mid, thread_seq); + default: + return -1; + } +} \ No newline at end of file From f9aee05bcc1b87e8c9dfd9f6deeedd3570a9905f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E4=B8=80=E9=B8=A3?= Date: Wed, 13 Nov 2019 16:17:59 +0800 Subject: [PATCH 02/32] =?UTF-8?q?=E4=BF=AE=E6=94=B9ssl=5Futils=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/{ssl_utils.h => tsg_ssl_utils.h} | 0 src/{ssl_utils.cpp => tsg_ssl_utils.cpp} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename inc/{ssl_utils.h => tsg_ssl_utils.h} (100%) rename src/{ssl_utils.cpp => tsg_ssl_utils.cpp} (100%) diff --git a/inc/ssl_utils.h b/inc/tsg_ssl_utils.h similarity index 100% rename from inc/ssl_utils.h rename to inc/tsg_ssl_utils.h diff --git a/src/ssl_utils.cpp b/src/tsg_ssl_utils.cpp similarity index 100% rename from src/ssl_utils.cpp rename to src/tsg_ssl_utils.cpp From db7282dab7f157bb3fd82f1843f7d953d2dbc1ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E4=B8=80=E9=B8=A3?= Date: Wed, 13 Nov 2019 19:24:22 +0800 Subject: [PATCH 03/32] =?UTF-8?q?=E6=89=AB=E6=8F=8F=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0domain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_rule.cpp | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 4c7e843..646b4ea 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -43,8 +43,8 @@ static void protocol_identify(char *buff, int buff_len, struct kni_protocol_iden result->domain_len = 0; } else{ - result->domain_len = strnlen(chello->sni, KNI_DOMAIN_MAX); strncpy(result->domain, chello->sni, strnlen(chello->sni, sizeof(result->domain) - 1)); + result->domain_len = strlen(result->domain); } } ssl_chello_free(chello); @@ -69,25 +69,23 @@ int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, voi } //return -1 if failed, return 0 on success -int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, scan_status_t *mid, void *logger, int thread_seq){ +int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, char *domain, int *domain_len, + scan_status_t *mid, void *logger, int thread_seq){ struct kni_protocol_identify_result protocol_identify_res; memset(&protocol_identify_res, 0, sizeof(protocol_identify_res)); protocol_identify((char*)pkt, pkt_len, &protocol_identify_res); - switch(protocol_identify_res.protocol){ - case KNI_PROTOCOL_UNKNOWN: - //TODO: field_stat refactor - //FS_operate(g_kni_fs_handle->handle, g_kni_fs_handle->fields[KNI_FIELD_STMERR_UNSUPPORTED_PROTOCOL], 0, FS_OP_ADD, 1); - //pmeinfo->error = STREAM_ERROR_UNSUPPORTED_PROTOCOL; - return -1; - case KNI_PROTOCOL_SSL: - return Maat_full_scan_string(g_kni_maat_feather, g_kni_scan_tableid[TSG_FIELD_SSL_SNI], CHARSET_UTF8, - protocol_identify_res.domain, protocol_identify_res.domain_len, - result, NULL, result_num, mid, thread_seq); - case KNI_PROTOCOL_HTTP: - return Maat_full_scan_string(g_kni_maat_feather, g_kni_scan_tableid[TSG_FIELD_HTTP_HOST], CHARSET_UTF8, - protocol_identify_res.domain, protocol_identify_res.domain_len, - result, NULL, result_num, mid, thread_seq); - default: - return -1; + if(protocol_identify_res.protocol == KNI_PROTOCOL_UNKNOWN){ + return -1; } + *domain_len = protocol_identify_res.domain_len; + strncpy(domain, protocol_identify_res.domain, *domain_len); + int tableid; + if(protocol_identify_res.protocol == KNI_PROTOCOL_SSL){ + tableid = g_kni_scan_tableid[TSG_FIELD_SSL_SNI]; + } + if(protocol_identify_res.protocol == KNI_PROTOCOL_HTTP){ + tableid = g_kni_scan_tableid[TSG_FIELD_HTTP_HOST]; + } + return Maat_full_scan_string(g_kni_maat_feather, tableid, CHARSET_UTF8, domain, *domain_len, + result, NULL, result_num, mid, thread_seq); } \ No newline at end of file From c76bc5534dcc4128b1e9b73e8fc25d981056411c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E4=B8=80=E9=B8=A3?= Date: Wed, 13 Nov 2019 19:56:19 +0800 Subject: [PATCH 04/32] =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=A2=9E=E5=8A=A0proto?= =?UTF-8?q?col=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/tsg_rule.h | 9 ++++++++- src/tsg_rule.cpp | 29 +++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/inc/tsg_rule.h b/inc/tsg_rule.h index dba5407..778c19c 100644 --- a/inc/tsg_rule.h +++ b/inc/tsg_rule.h @@ -16,6 +16,13 @@ typedef enum _PULL_RESULT_TYPE PULL_FW_RESULT }PULL_RESULT_TYPE; +enum tsg_protocol{ + TSG_PROTOCOL_UNKNOWN = 0, + TSG_PROTOCOL_SSL, + TSG_PROTOCOL_HTTP +}; +#define TSG_DOMAIN_MAX 256 + extern Maat_feather_t g_tsg_maat_feather; int tsg_rule_init(const char *conffile); @@ -23,7 +30,7 @@ int tsg_rule_init(const char *conffile); int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num); //return 0 if failed, return >0 on success; -int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t *result, int result_num, char *domain, int *domain_len); +int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t *result, int result_num, enum tsg_protocol *protocol, char *domain, int *domain_len); //return -1 if failed, return 0 on success; int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, void *logger); diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 646b4ea..db1c67d 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -3,25 +3,17 @@ #include "Maat_rule.h" #include "Maat_command.h" #include "ssl_utils.h" +#include "tsg_rule.h" -#define KNI_DOMAIN_MAX 256 - -enum kni_protocol{ - KNI_PROTOCOL_UNKNOWN = 0, - KNI_PROTOCOL_SSL, - KNI_PROTOCOL_HTTP, -}; - -enum kni_scan_table -{ +enum kni_scan_table{ TSG_FIELD_SSL_SNI, TSG_FIELD_HTTP_HOST, SCAN_TABLE_MAX }; struct kni_protocol_identify_result{ - int protocol; - char domain[KNI_DOMAIN_MAX]; + enum tsg_protocol protocol; + char domain[TSG_DOMAIN_MAX]; int domain_len; }; @@ -30,7 +22,7 @@ const char *g_kni_scan_table_name[SCAN_TABLE_MAX]; int g_kni_scan_tableid[SCAN_TABLE_MAX] = {0}; static void protocol_identify(char *buff, int buff_len, struct kni_protocol_identify_result *result){ - result->protocol = KNI_PROTOCOL_UNKNOWN; + result->protocol = TSG_PROTOCOL_UNKNOWN; //TODO: http: get from http protocol plugin //ssl @@ -38,7 +30,7 @@ static void protocol_identify(char *buff, int buff_len, struct kni_protocol_iden struct ssl_chello *chello = NULL; chello = ssl_chello_parse((const unsigned char*)buff, buff_len, &chello_status); if(chello_status == CHELLO_PARSE_SUCCESS){ - result->protocol = KNI_PROTOCOL_SSL; + result->protocol = TSG_PROTOCOL_SSL; if(chello->sni == NULL){ result->domain_len = 0; } @@ -69,21 +61,22 @@ int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, voi } //return -1 if failed, return 0 on success -int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, char *domain, int *domain_len, +int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, enum tsg_protocol *protocol, char *domain, int *domain_len, scan_status_t *mid, void *logger, int thread_seq){ struct kni_protocol_identify_result protocol_identify_res; memset(&protocol_identify_res, 0, sizeof(protocol_identify_res)); protocol_identify((char*)pkt, pkt_len, &protocol_identify_res); - if(protocol_identify_res.protocol == KNI_PROTOCOL_UNKNOWN){ + if(protocol_identify_res.protocol == TSG_PROTOCOL_UNKNOWN){ return -1; } + *protocol = protocol_identify_res.protocol; *domain_len = protocol_identify_res.domain_len; strncpy(domain, protocol_identify_res.domain, *domain_len); int tableid; - if(protocol_identify_res.protocol == KNI_PROTOCOL_SSL){ + if(protocol_identify_res.protocol == TSG_PROTOCOL_SSL){ tableid = g_kni_scan_tableid[TSG_FIELD_SSL_SNI]; } - if(protocol_identify_res.protocol == KNI_PROTOCOL_HTTP){ + if(protocol_identify_res.protocol == TSG_PROTOCOL_HTTP){ tableid = g_kni_scan_tableid[TSG_FIELD_HTTP_HOST]; } return Maat_full_scan_string(g_kni_maat_feather, tableid, CHARSET_UTF8, domain, *domain_len, From 8913a7d22eacccc639fe6ae2fd21f21b269b9b18 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Fri, 15 Nov 2019 19:29:54 +0800 Subject: [PATCH 05/32] =?UTF-8?q?=E8=A7=84=E5=88=99=E6=8E=92=E5=BA=8F?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=20=E7=BB=9F=E4=B8=80=E5=8F=91=E9=80=81?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/tsg_entry.h | 45 ++++ inc/tsg_log_id.h | 41 ---- inc/tsg_send_log.h | 4 +- inc/tsg_types.h | 127 ++++------- src/tsg_entry.cpp | 279 ++++++++++++++++++++++- src/tsg_rule.cpp | 245 +++++++++++++++++++- src/tsg_send_log.cpp | 437 ++++++++++++++++++++++++++++++++---- src/tsg_send_log_internal.h | 56 ++++- 8 files changed, 1050 insertions(+), 184 deletions(-) diff --git a/inc/tsg_entry.h b/inc/tsg_entry.h index 8b13789..7d56227 100644 --- a/inc/tsg_entry.h +++ b/inc/tsg_entry.h @@ -1 +1,46 @@ +#ifndef __TSG_ENTRY_H__ +#define __TSG_ENTRY_H__ +#include +#include "tsg_types.h" + +#define FW_ACTION_UNKNOWN 0x00 +#define FW_ACTION_DROP 0x01 +#define FW_ACTION_RESET 0x02 +#define FW_ACTION_BLOCK 0x04 + +#define TSG_ACTION_NONE 0x00 +#define TSG_ACTION_MONITOR 0x01 +#define TSG_ACTION_INTERCEPT 0x02 +#define TSG_ACTION_DENY 0x10 +#define TSG_ACTION_MANIPULATE 0x30 +#define TSG_ACTION_BYPASS 0x80 + + +#define MAX_RESULT_NUM 8 +#define MAX_DOAMIN_LEN 2048 + +typedef struct _policy_priority_label +{ + tsg_protocol_t proto; //enum _tsg_protocol (tsg_types.h) + int domain_len; + int result_type; //enum _PULL_RESULT_TYPE (tsg_rule.h) + int result_num; + char domain[MAX_DOAMIN_LEN]; + Maat_rule_t result[MAX_RESULT_NUM]; +}policy_priority_label_t; + + +typedef struct _tsg_para +{ + int ip_addr_table_id; + int subscribe_id_table_id; + int priority_project_id; + void *logger; +}g_tsg_para_t; + + + +extern g_tsg_para_t g_tsg_para; + +#endif \ No newline at end of file diff --git a/inc/tsg_log_id.h b/inc/tsg_log_id.h index 0f5f2f5..177ebf8 100644 --- a/inc/tsg_log_id.h +++ b/inc/tsg_log_id.h @@ -85,47 +85,6 @@ #define CABOT_LOG_OPT_DNS_RR 78 -#define CABOT_LOG_OPT_SSL_VERSION 81 -#define CABOT_LOG_OPT_SSL_SNI 82 -#define CABOT_LOG_OPT_SSL_SAN 83 -#define CABOT_LOG_OPT_SSL_CN 84 -#define CABOT_LOG_OPT_FTP_URL 88 -#define CABOT_LOG_OPT_FTP_CONTENT 89 - -#define CABOT_LOG_OPT_VOIP_CALLING_ACCOUNT 90 -#define CABOT_LOG_OPT_VOIP_CALLED_ACCOUNT 91 -#define CABOT_LOG_OPT_VOIP_CALLING_NUMBER 92 -#define CABOT_LOG_OPT_VOIP_CALLED_NUMBER 93 - -#define CABOT_LOG_OPT_BGP_PACKET_TYPE 100 -#define CABOT_LOG_OPT_BGP_AS_NUM 101 -#define CABOT_LOG_OPT_BGP_ROUTE 102 - -#define CABOT_LOG_OPT_RADIUS_PACKET_TYPE 112 -#define CABOT_LOG_OPT_RADIUS_ACCOUNT 113 -#define CABOT_LOG_OPT_RADIUS_CALLBACK_NUMBER 114 -#define CABOT_LOG_OPT_RADIUS_CALLBACK_ID 115 -#define CABOT_LOG_OPT_RADIUS_CALLED_STATION_ID 116 -#define CABOT_LOG_OPT_RADIUS_CALLING_STATION_ID 117 -#define CABOT_LOG_OPT_RADIUS_ACCT_SESSION_ID 118 -#define CABOT_LOG_OPT_RADIUS_ACCT_MULTI_SESSION_ID 119 -#define CABOT_LOG_OPT_RADIUS_NAS_IP_ADDRESS 120 -#define CABOT_LOG_OPT_RADIUS_FRAMED_IP_ADDRESS 121 -#define CABOT_LOG_OPT_RADIUS_FRAMED_IP_NETMASK 122 -#define CABOT_LOG_OPT_RADIUS_SERVICE_TYPE 123 -#define CABOT_LOG_OPT_RADIUS_FRAMED_MTU 124 -#define CABOT_LOG_OPT_RADIUS_SESSION_TIMEOUT 125 -#define CABOT_LOG_OPT_RADIUS_IDLE_TIMEOUT 126 -#define CABOT_LOG_OPT_RADIUS_TERMINATION_CATION 127 -#define CABOT_LOG_OPT_RADIUS_PROXY_STATE 128 -#define CABOT_LOG_OPT_RADIUS_ACCT_STATUS_TYPE 129 -#define CABOT_LOG_OPT_RADIUS_ACCT_INPUT_OCTETS 130 -#define CABOT_LOG_OPT_RADIUS_ACCT_INPUT_PACKETS 131 -#define CABOT_LOG_OPT_RADIUS_ACCT_OUTPUT_OCTETS 132 -#define CABOT_LOG_OPT_RADIUS_ACCT_OUTPUT_PACKETS 133 -#define CABOT_LOG_OPT_RADIUS_ACCT_TERMINATE_CAUSE 134 -#define CABOT_LOG_OPT_RADIUS_ACCT_LINK_COUNT 135 -#define CABOT_LOG_OPT_RADIUS_ACCT_INTERIM_INTERVAL 136 #endif diff --git a/inc/tsg_send_log.h b/inc/tsg_send_log.h index 6758046..cb0f5d7 100644 --- a/inc/tsg_send_log.h +++ b/inc/tsg_send_log.h @@ -17,9 +17,11 @@ typedef struct _tsg_log typedef enum _tld_type { + TLD_TYPE_UNKNOWN=0, TLD_TYPE_LONG=1, TLD_TYPE_STRING, - TLD_TYPE_FILE + TLD_TYPE_FILE, + TLD_TYPE_MAX }TLD_TYPE; diff --git a/inc/tsg_types.h b/inc/tsg_types.h index 0967ef3..da32e5f 100644 --- a/inc/tsg_types.h +++ b/inc/tsg_types.h @@ -1,97 +1,52 @@ #ifndef __TSG_TYPES_H__ #define __TSG_TYPES_H__ -typedef enum _tsg_opt +typedef enum _tsg_log_field_id { - LOG_OPT_HTTP_URL=1, - LOG_OPT_HTTP_HOST, - LOG_OPT_HTTP_REQUEST_LINE, - LOG_OPT_HTTP_RESPONSE_LINE, - LOG_OPT_HTTP_REQUEST_HEADER, - LOG_OPT_HTTP_RESPONSE_HEADER, - LOG_OPT_HTTP_REQUEST_BODY, - LOG_OPT_HTTP_RESPONSE_BODY, - LOG_OPT_HTTP_PROXY_FLAG, - LOG_OPT_HTTP_SEQUENCE, - LOG_OPT_HTTP_SNAPSHOT, - LOG_OPT_HTTP_COOKIE, - LOG_OPT_HTTP_REFERER, - LOG_OPT_HTTP_USER_AGENT, - LOG_OPT_HTTP_CONTENT_LENGTH, - LOG_OPT_HTTP_CONTENT_TYPE, - LOG_OPT_HTTP_SET_COOKIE, - LOG_OPT_HTTP_VERSION, + LOG_COMMON_SERVICE, + LOG_COMMON_DIRECTION, + LOG_COMMON_L4_PROTOCOL, + LOG_COMMON_ADDRESS_TYPE, + LOG_COMMON_SCHAME_TYPE, + LOG_COMMON_POLICY_ID, + LOG_COMMON_USER_TAG, + LOG_COMMON_ACTION, + LOG_COMMON_USER_REGION, + LOG_COMMON_CLIENT_IP, + LOG_COMMON_CLIENT_PORT, + LOG_COMMON_ENTRANCE_ID, + LOG_COMMON_DEVICE_ID, + LOG_COMMON_LINK_ID, + LOG_COMMON_ISP, + LOG_COMMON_ENCAPSULATION, + LOG_COMMON_SLED_IP, + LOG_COMMON_SERVER_IP, + LOG_COMMON_SERVER_PORT, + LOG_COMMON_APP_LABEL, + LOG_COMMON_APP_ID, + LOG_COMMON_PROTOCOL_ID, + LOG_COMMON_C2S_PKT_NUM, + LOG_COMMON_S2C_PKT_NUM, + LOG_COMMON_C2S_BYTE_NUM, + LOG_COMMON_S2C_BYTE_NUM, + LOG_COMMON_START_TIME, + LOG_COMMON_END_TIME, + LOG_COMMON_CON_DURATION_MS, + LOG_COMMON_STREAM_DIR, + LOG_COMMON_ADDRESS_LIST, + LOG_COMMON_HAS_DUP_TRAFFIC, + LOG_COMMON_STREAM_ERROR, + LOG_COMMON_STREAM_TRACE_ID, + LOG_HTTP_HOST, + LOG_SSL_SNI, + LOG_COMMON_MAX +}tsg_log_field_id_t; - LOG_OPT_MAIL_PROTOCOL_TYPE, - LOG_OPT_MAIL_SENDER, - LOG_OPT_MAIL_RECEIVER, - LOG_OPT_MAIL_SUBJECT, - LOG_OPT_MAIL_CONTENT, - LOG_OPT_MAIL_ATTACHMENT_NAME, - LOG_OPT_MAIL_ATTACHMENT_CONTENT, - LOG_OPT_MAIL_EML_FILE, - LOG_OPT_MAIL_SNAPSHOT, - LOG_OPT_MAIL_SUBJECT_CHARSET, - - LOG_OPT_DNS_MESSAGE_ID, - LOG_OPT_DNS_QR, - LOG_OPT_DNS_OPCODE, - LOG_OPT_DNS_AA, - LOG_OPT_DNS_TC, - LOG_OPT_DNS_RD, - LOG_OPT_DNS_RA, - LOG_OPT_DNS_RCODE, - LOG_OPT_DNS_QDCOUNT, - LOG_OPT_DNS_ANCOUNT, - LOG_OPT_DNS_NSCOUNT, - LOG_OPT_DNS_ARCOUNT, - LOG_OPT_DNS_QNAME, - LOG_OPT_DNS_QTYPE, - LOG_OPT_DNS_QCLASS, - LOG_OPT_DNS_CNAME, - LOG_OPT_DNS_SUB, - LOG_OPT_DNS_RR, - - LOG_OPT_SSL_VERSION, - LOG_OPT_SSL_SNI, - LOG_OPT_SSL_SAN, - LOG_OPT_SSL_CN, - LOG_OPT_SSL_PINNINGST, - LOG_OPT_SSL_INTERCEPT_STATE, - LOG_OPT_SSL_SERVER_SIDE_LATENCY, - LOG_OPT_SSL_CLINET_SIDE_LATENCY, - LOG_OPT_SSL_SERVER_SIDE_VERSION, - LOG_OPT_SSL_CLIENT_SIDE_VERSION, - LOG_OPT_SSL_CERT_VERIFY, - LOG_OPT_SSL_ERROR, - LOG_OPT_SSL_CON_LATENCY_MS, - - LOG_OPT_FTP_URL, - LOG_OPT_FTP_CONTENT, - - LOG_OPT_BGP_TYPE, - LOG_OPT_BGP_AS_NUM, - LOG_OPT_BGP_ROUTE, - - LOG_OPT_VOIP_CALLING_ACCOUNT, - LOG_OPT_VOIP_CALLED_ACCOUNT, - LOG_OPT_VOIP_CALLING_NUMBER, - LOG_OPT_VOIP_CALLED_NUMBER, - - LOG_OPT_RADIUS_PACKET_TYPE, - LOG_OPT_RADIUS_NAS_IP, - LOG_OPT_RADIUS_FRAMED_IP, - LOG_OPT_RADIUS_ACCOUNT, - LOG_OPT_RADIUS_SEESION_TIMEOUT, - LOG_OPT_RADIUS_IDLE_TIMEOUT, - LOG_OPT_RADIUS_ACCT_STATUS_TYPE, - LOG_OPT_RADIUS_ACCT_TERMINATE_CAUSE, - LOG_OPT_MAX -}tsg_opt_t; typedef enum _tsg_protocol { - PROTO_IPv4, + PROTO_UNKONWN=0, + PROTO_IPv4=1, PROTO_IPv6, PROTO_TCP, PROTO_UDP, diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 8b69083..ec658ab 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -1,6 +1,283 @@ #include #include #include +#include -char TSG_MASTER_VERSION_20191112=0; +#include +#include +#include + +#include "tsg_rule.h" +#include "tsg_entry.h" +#include "tsg_send_log.h" +#include "tsg_send_log_internal.h" + + + +char TSG_MASTER_VERSION_20191115=0; +const char *tsg_conffile="tsgconf/main.conf"; +g_tsg_para_t g_tsg_para; + +static void free_policy_label(int thread_seq, void *project_req_value) +{ + dictator_free(thread_seq, project_req_value); + project_req_value=NULL; +} + +#if 0 +static int is_ip_policy(Maat_rule_t *p_result, char *protocol, int len, int thread_seq) +{ + int ret=0; + cJSON *item=NULL; + char *service_defined=NULL; + cJSON *user_define_object=NULL; + + if(p_result->serv_def_len>MAX_SERVICE_DEFINE_LEN) + { + service_defined=dictator_malloc(thread_seq, p_result->serv_def_len+1); + ret=Maat_read_rule(g_tsg_maat_feather, p_result, MAAT_RULE_SERV_DEFINE, service_defined, p_result->serv_def_len+1); + assert(ret==p_result->serv_def_len+1); + + user_define_object=cJSON_Parse(service_defined); + } + else + { + user_define_object=cJSON_Parse(p_result->service_defined); + } + + if(user_define_object!=NULL) + { + item=cJSON_GetObjectItem(user_define_object, "protocol"); + if(item!=NULL && item->valuestring!=NULL) + { + memcpy(protocol, item->valuestring, (len>strlen(item->valuestring)) ? strlen(item->valuestring): len); + } + + item=cJSON_GetObjectItem(user_define_object, "method"); + if((item==NULL) || ((strncasecmp(item->valuestring, "http", strlen(item->valuestring)))!=0 && (strncasecmp(item->valuestring, "ssl", strlen(item->valuestring)))!=0)) + { + ret=1; + } + + cJSON_Delete(user_define_object); + user_define_object=NULL; + } + + if(service_defined!=NULL) + { + dictator_free(thread_seq, service_defined); + service_defined=NULL; + } + + return ret; +} +#endif +static Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result_num) +{ + int i=0; + Maat_rule_t *p_result=NULL; + if(result==NULL || result_num<=0) + { + return NULL; + } + + p_result=&result[0]; + + for(i=1; ip_result->action) + { + p_result=&result[i]; + continue; + } + + if(result[i].action==p_result->action) + { + if(result[i].config_idconfig_id) + { + p_result=&result[i]; + } + } + } + + return p_result; +} + +extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet) +{ + int send_log=0; + int ret=0,hit_num=0,ip_policy=0; + int state=APP_STATE_DROPME; + scan_status_t mid=NULL; + char *domain_field_name=NULL; + char *schema_field_name=NULL; + Maat_rule_t *p_result=NULL; + Maat_rule_t *q_result=NULL; + tsg_log_t log_msg; + TLD_handle_t TLD_handle=NULL; + struct _identify_info identify_info; + Maat_rule_t all_result[MAX_RESULT_NUM]; + policy_priority_label_t *priority_label=NULL; + + switch(a_tcp->pktstate) + { + case OP_STATE_PENDING: + if((a_tcp->ptcpdetail->pdata==NULL) || (a_tcp->ptcpdetail->datalen<=0) || (a_tcp->dir==DIR_DOUBLE && a_tcp->curdir==DIR_S2C)) + { + return APP_STATE_GIVEME; + } + + ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, PROTO_MAX, &mid, all_result, MAX_RESULT_NUM-hit_num); + if(ret>0) + { + hit_num+=ret; + q_result=tsg_policy_decision_criteria(all_result, hit_num); + } + + + if(a_tcp->curdir==DIR_C2S) + { + memset(&identify_info, 0, sizeof(identify_info)); + + ret=tsg_scan_shared_policy(g_tsg_maat_feather, + a_tcp->ptcpdetail->pdata, + a_tcp->ptcpdetail->datalen, + all_result, + MAX_RESULT_NUM-hit_num, + &identify_info, + &mid, + g_tsg_para.logger, + thread_seq); + if(ret>0) + { + hit_num+=ret; + } + } + + p_result=tsg_policy_decision_criteria(all_result, hit_num); + + if(p_result!=NULL) + { + if(q_result!=NULL && (p_result==q_result)) + { + ip_policy=1; + send_log=1; + } + + switch(p_result->action) + { + case TSG_ACTION_DENY: + if(ip_policy==1) + { + MESA_kill_tcp(a_tcp, a_packet); + state|=APP_STATE_DROPPKT; + } + break; + case TSG_ACTION_MONITOR: + break; + case TSG_ACTION_BYPASS: + send_log=1; + state|=APP_STATE_DROPPKT; //TODO + break; + case TSG_ACTION_INTERCEPT: + priority_label=(policy_priority_label_t *)dictator_malloc(thread_seq, sizeof(policy_priority_label_t)); + + priority_label->result_num=1; + priority_label->domain_len=identify_info.domain_len; + memcpy(priority_label->domain, identify_info.domain, identify_info.domain_len); + memcpy(priority_label->result, p_result, sizeof(struct Maat_rule_t)); + + ret=project_req_add_struct(a_tcp, g_tsg_para.priority_project_id, (void *)priority_label); + if(ret<0) + { + free_policy_label(thread_seq, (void *)priority_label); + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_ADD", "Add policy_priority_label failed ..."); + } + break; + case TSG_ACTION_NONE: + default: + assert(0); + break; + } + + if(send_log==1 && p_result->do_log>0) + { + TLD_handle=TLD_create(thread_seq); + if(priority_label->proto!=PROTO_UNKONWN) + { + schema_field_name=log_field_id2name(g_tsg_log_instance, LOG_COMMON_SCHAME_TYPE); + TLD_append(TLD_handle, schema_field_name, (void *)((priority_label->proto==PROTO_HTTP) ? "HTTP" : "SSL"), TLD_TYPE_LONG); + + domain_field_name=log_field_id2name(g_tsg_log_instance, ((priority_label->proto==PROTO_HTTP) ? LOG_HTTP_HOST : LOG_SSL_SNI)); + TLD_append(TLD_handle, domain_field_name, (void *)priority_label->domain, TLD_TYPE_STRING); + } + + log_msg.a_stream=a_tcp; + log_msg.result=p_result; + log_msg.result_num=1; + tsg_send_log(g_tsg_log_instance, TLD_handle, &log_msg, thread_seq); + } + } + break; + case OP_STATE_DATA: + case OP_STATE_CLOSE: + default: + break; + } + + return state; +} + + + +extern "C" int TSG_MASTER_INIT() +{ + int ret=0,level=30; + char log_path[128]={0}; + char label_buff[128]={0}; + + memset(&g_tsg_para, 0, sizeof(g_tsg_para)); + + MESA_load_profile_int_def(tsg_conffile, "SYSTEM","LOG_LEVEL", &level, 30); + MESA_load_profile_string_def(tsg_conffile, "SYSTEM","LOG_PATH", log_path, sizeof(log_path), NULL); + + g_tsg_para.logger=MESA_create_runtime_log_handle(log_path, level); + if(g_tsg_para.logger==NULL) + { + printf("MESA_create_runtime_log_handle failed ...\n"); + return -1; + } + + + MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "POLICY_PRIORITY_LABEL", label_buff, sizeof(label_buff), "POLICY_PRIORITY"); + g_tsg_para.priority_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_STRUCT, free_policy_label); + if(g_tsg_para.priority_project_id<0) + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_REGISTER", "Register %s failed ...", label_buff); + return -1; + } + + ret=tsg_rule_init(tsg_conffile, g_tsg_para.logger); + if(ret<0) + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_MAAT", "tsg_rule_init failed ..."); + return -1; + } + + g_tsg_log_instance=tsg_sendlog_init(tsg_conffile); + if(g_tsg_log_instance==NULL) + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_SENDLOG", "tsg_sendlog_init failed ..."); + return -1; + } + + return 0; +} + + + +extern "C" int T2_MASTER_UNLOAD() +{ + return 0; +} diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 639285e..7a1c296 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -1,18 +1,257 @@ #include #include #include +#include +#include + +#include +#include +#include #include "tsg_rule.h" +#include "tsg_entry.h" -extern Maat_feather_t g_tsg_maat_feather; +Maat_feather_t g_tsg_maat_feather; -int tsg_pull_policy_result(PULL_RESULT_TYPE pull_result_type, Maat_rule_t*result, int result_num) +#define MAX_PATH_LEN 1024 + +int tsg_rule_init(const char* conffile, void *logger) { + unsigned short redis_port = 0; + int ret=0,scan_detail=0,effect_interval=60; + const char* instance_name=NULL,*module="MAAT"; + int factor=0, redis_port_num=0,redis_index=0; + char redis_ip[16]={0}, effective_flag[1024]={0}; + int maat_mode=0,maat_stat_on=0,maat_perf_on=0,thread_max=0; + char ip_addr_table[32]={0},subscriber_id_table[32]={0}; + char json_cfg_file[MAX_PATH_LEN]={0},maat_stat_file[MAX_PATH_LEN]={0}; + char table_info[MAX_PATH_LEN]={0},inc_cfg_dir[MAX_PATH_LEN]={0},ful_cfg_dir[MAX_PATH_LEN]={0}; + + memset(effective_flag, 0, sizeof(effective_flag)); + MESA_load_profile_string_def(conffile,module,"EFFECTIVE_FLAG",effective_flag, sizeof(effective_flag),""); + + MESA_load_profile_int_def(conffile, module,"MAAT_MODE", &(maat_mode),0); + MESA_load_profile_int_def(conffile, module,"STAT_SWITCH", &(maat_stat_on),1); + MESA_load_profile_int_def(conffile, module,"PERF_SWITCH", &(maat_perf_on),1); + + MESA_load_profile_string_def(conffile,module,"TABLE_INFO",table_info, sizeof(table_info), ""); + MESA_load_profile_string_def(conffile,module,"STAT_FILE",maat_stat_file, sizeof(maat_stat_file), ""); + MESA_load_profile_int_def(conffile, module,"EFFECT_INTERVAL_S", &(effect_interval), 60); + effect_interval*=1000;//convert s to ms + + thread_max=get_thread_count(); + g_tsg_maat_feather=Maat_feather(thread_max, table_info, logger); + if(maat_mode==2) + { + MESA_load_profile_string_def(conffile,module,"REDIS_IP", redis_ip, sizeof(redis_ip),""); + MESA_load_profile_int_def(conffile, module,"REDIS_PORT_NUM", &(redis_port_num), 1); + MESA_load_profile_short_def(conffile, module,"REDIS_PORT", (short*)&(redis_port), 6379); + MESA_load_profile_int_def(conffile, module,"REDIS_INDEX", &redis_index, 0); + + if(strlen(effective_flag)!=0) + { + Maat_set_feather_opt(g_tsg_maat_feather,MAAT_OPT_ACCEPT_TAGS,effective_flag, strlen(effective_flag)+1); + } + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_EFFECT_INVERVAL_MS, &effect_interval, sizeof(effect_interval)); + + srand((unsigned int)time(NULL)); + factor = rand()%redis_port_num; + redis_port = redis_port+factor; + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_REDIS_IP, redis_ip, strlen(redis_ip)+1); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_REDIS_PORT, (void *)&redis_port, sizeof(redis_port)); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_STAT_FILE_PATH, maat_stat_file, strlen(maat_stat_file)+1); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_STAT_ON, NULL, 0); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_PERF_ON, NULL, 0); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_REDIS_INDEX, &redis_index, sizeof(redis_index)); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail)); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_DEFERRED_LOAD, NULL,0); + } + else + { + if(strlen(effective_flag)!=0) + { + ret=Maat_set_feather_opt(g_tsg_maat_feather,MAAT_OPT_ACCEPT_TAGS,effective_flag, strlen(effective_flag)+1); + assert(ret>=0); + } + Maat_set_feather_opt(g_tsg_maat_feather,MAAT_OPT_INSTANCE_NAME,instance_name, strlen(instance_name)+1); + if(maat_mode==1) + { + MESA_load_profile_string_def(conffile,module,"JSON_CFG_FILE",json_cfg_file, sizeof(json_cfg_file),""); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_JSON_FILE_PATH, json_cfg_file, strlen(json_cfg_file)+1); + } + else + { + MESA_load_profile_string_def(conffile,module,"INC_CFG_DIR",inc_cfg_dir, sizeof(inc_cfg_dir),""); + MESA_load_profile_string_def(conffile,module,"FULL_CFG_DIR",ful_cfg_dir, sizeof(ful_cfg_dir),""); + assert(strlen(inc_cfg_dir)!=0&&strlen(ful_cfg_dir)!=0); + + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_FULL_CFG_DIR, ful_cfg_dir, strlen(ful_cfg_dir)+1); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_INC_CFG_DIR, inc_cfg_dir, strlen(inc_cfg_dir)+1); + } + if(maat_stat_on) + { + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_STAT_FILE_PATH, maat_stat_file, strlen(maat_stat_file)+1); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_STAT_ON, NULL, 0); + if(maat_perf_on) + { + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_PERF_ON, NULL, 0); + } + } + + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_EFFECT_INVERVAL_MS, &effect_interval, sizeof(effect_interval)); + Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail)); + } + + ret=Maat_initiate_feather(g_tsg_maat_feather); + if(ret<0) + { + return -1; + } + + MESA_load_profile_string_def(conffile, module, "IP_ADDR_TABLE", ip_addr_table, sizeof(ip_addr_table), "TSG_OBJ_IP_ADDR"); + MESA_load_profile_string_def(conffile, module, "SUBSCRIBER_ID_TABLE", subscriber_id_table, sizeof(subscriber_id_table), "TSG_OBJ_SUBSCRIBER_ID"); + + g_tsg_para.ip_addr_table_id=Maat_table_register(g_tsg_maat_feather, ip_addr_table); + if(g_tsg_para.ip_addr_table_id<0) + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_MAAT", "Register %s failed ...", ip_addr_table); + return -1; + } + + g_tsg_para.subscribe_id_table_id=Maat_table_register(g_tsg_maat_feather, subscriber_id_table); + if(g_tsg_para.subscribe_id_table_id<0) + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_MAAT", "Register %s failed ...", subscriber_id_table); + return -1; + } + + ret=tsg_shared_table_init(conffile, g_tsg_maat_feather, logger); + if(ret<0) + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_MAAT", "tsg_shared_table_init failed ..."); + return -1; + } + + return 0; +} + + +int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t*result, int result_num, struct _identify_info *identify_info) +{ + int num=0; + policy_priority_label_t *label=NULL; + + label=(policy_priority_label_t *)project_req_get_struct(a_stream, g_tsg_para.priority_project_id); + if(label!=NULL && result!=NULL && result_num>0 && identify_info!=NULL) + { + if(label->result_type==pull_result_type) + { + num=(label->result_num>result_num) ? result_num : label->result_num; + memcpy(result, label->result, num*sizeof(Maat_rule_t)); + + memcpy(identify_info->domain, label->domain, label->domain_len); + identify_info->domain_len=label->domain_len; + + identify_info->proto = label->proto; + + return num; + } + } + return 0; } int tsg_scan_nesting_addr(Maat_feather_t maat_feather, struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num) { - return 0; + struct ipaddr t_addr; + struct ipaddr* p_addr = NULL; + char subscribe_id[64]={0}; + int hit_num=0,tans_proto=0; + int is_scan_addr=1, maat_ret=0,found_pos=0; + const struct streaminfo *cur_stream = a_stream; + + if(result == NULL || result_num <= 0 || a_stream == NULL || maat_feather == NULL) + { + return -1; + } + + do + { + if(cur_stream->addr.addrtype == __ADDR_TYPE_IP_PAIR_V4 || cur_stream->addr.addrtype == ADDR_TYPE_IPV4 || cur_stream->addr.addrtype == __ADDR_TYPE_IP_PAIR_V6 || cur_stream->addr.addrtype == ADDR_TYPE_IPV6) + { + is_scan_addr = 1; + if(cur_stream->addr.addrtype == __ADDR_TYPE_IP_PAIR_V4 || cur_stream->addr.addrtype == __ADDR_TYPE_IP_PAIR_V6) + { + memcpy(&t_addr, &cur_stream->addr, sizeof(t_addr)); + if(cur_stream->addr.addrtype == __ADDR_TYPE_IP_PAIR_V4) + t_addr.addrtype = ADDR_TYPE_IPV4; + else + t_addr.addrtype = ADDR_TYPE_IPV6; + p_addr = &t_addr; + } + else + { + p_addr = (struct ipaddr *)&cur_stream->addr; + } + } + else + { + is_scan_addr = 0; + p_addr = NULL; + } + + if(is_scan_addr==1 && p_addr!=NULL) + { + switch(cur_stream->type) + { + case STREAM_TYPE_TCP: + tans_proto=6; + break; + case STREAM_TYPE_UDP: + tans_proto=17; + break; + default: + tans_proto=255; + break; + } + + maat_ret=Maat_scan_proto_addr(maat_feather, + g_tsg_para.ip_addr_table_id, + p_addr, + tans_proto, + result+hit_num, + result_num-hit_num, + mid, + cur_stream->threadnum); + if(maat_ret > 0) + { + hit_num+=maat_ret; + } + } + + cur_stream = cur_stream->pfather; + + }while(cur_stream != NULL && hit_num < result_num); + + + if(hit_num < result_num) + { + maat_ret=Maat_full_scan_string(maat_feather, + g_tsg_para.subscribe_id_table_id, + CHARSET_GBK, + subscribe_id, + strlen(subscribe_id), + result+hit_num, + &found_pos, + result_num-hit_num, + mid, + a_stream->threadnum); + if(maat_ret > 0) + { + hit_num+=maat_ret; + } + } + + return hit_num; } diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 7ae1681..636fb5d 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -11,7 +12,6 @@ #include #include -#include #include #include @@ -19,7 +19,161 @@ #include "tsg_send_log.h" #include "tsg_send_log_internal.h" -tsg_logger_info_t tsg_logger_info; +char TSG_SEND_LOG_VERSION_20191115=0; +tsg_log_instance_t g_tsg_log_instance; + +const id2field_t tld_type[TLD_TYPE_MAX]={{TLD_TYPE_UNKNOWN, TLD_TYPE_UNKNOWN, "UNKOWN"}, + {TLD_TYPE_LONG, TLD_TYPE_LONG, "LONG"}, + {TLD_TYPE_STRING, TLD_TYPE_STRING, "STRING"}, + {TLD_TYPE_FILE, TLD_TYPE_FILE, "FILE"} + }; + +int TLD_cancel(TLD_handle_t handle) +{ + int thread_id=0; + struct _tld_handle *_handle=NULL; + if(handle!=NULL) + { + _handle=(struct _tld_handle *)handle; + cJSON_Delete(_handle->object); + _handle->object=NULL; + + thread_id=_handle->thread_id; + dictator_free(thread_id, handle); + + handle=NULL; + } + + return 0; +} + +int TLD_delete(TLD_handle_t handle, char *key) +{ + struct _tld_handle *_handle=(struct _tld_handle *)handle; + + if(_handle!=NULL && key!=NULL) + { + cJSON_DeleteItemFromObject(_handle->object, key); + } + + return 0; +} + +int TLD_append(TLD_handle_t handle, char *key, void *value, TLD_TYPE type) +{ + char buff[128]={0}; + struct _tld_handle *_handle=(struct _tld_handle *)handle; + + if(_handle==NULL || key==NULL || value==NULL) + { + return -1; + } + + switch(type) + { + case TLD_TYPE_LONG: + snprintf(buff, sizeof(buff), "%ld", *(long *)value); + cJSON_AddStringToObject(_handle->object, key, buff); + break; + case TLD_TYPE_FILE: + break; + case TLD_TYPE_STRING: + cJSON_AddStringToObject(_handle->object, key, (char *)value); + break; + default: + return -1; + break; + } + + + return 0; +} + +TLD_handle_t TLD_create(int thread_id) +{ + struct _tld_handle *_handle=(struct _tld_handle *)dictator_malloc(thread_id, sizeof(struct _tld_handle)); + _handle->thread_id = thread_id; + _handle->object = cJSON_CreateObject(); + + return (TLD_handle_t)_handle; +} + + +char *log_field_id2name(tsg_log_instance_t instance, tsg_log_field_id_t id) +{ + struct _tsg_log_instance *log_instance = (struct _tsg_log_instance *)instance; + if(log_instance!=NULL) + { + return log_instance->id2field[id].name; + } + + return NULL; +} + +int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handle *_handle, struct streaminfo *a_stream) +{ + int ret=0,addr_type=0; + unsigned short tunnel_type=0; + char nest_addr_buf[1024]; + char *addr_proto=NULL; + int tunnel_type_size=sizeof(tunnel_type); + struct layer_addr_ipv4 *ipv4=NULL; + struct layer_addr_ipv6 *ipv6=NULL; + char server_ip[MAX_IPV4_LEN*8]={0}; + char client_ip[MAX_IPV4_LEN*8]={0}; + + switch(a_stream->addr.addrtype) + { + case ADDR_TYPE_IPV4: + case __ADDR_TYPE_IP_PAIR_V4: + ipv4=a_stream->addr.ipv4; + inet_ntop(AF_INET, (void *)&ipv4->saddr, client_ip, sizeof(client_ip)); + inet_ntop(AF_INET, (void *)&ipv4->daddr, server_ip, sizeof(server_ip)); + + addr_type=4; + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_IP].name, (void *)server_ip, TLD_TYPE_STRING); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_IP].name, (void *)client_ip, TLD_TYPE_STRING); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_PORT].name, (void *)&ipv4->dest, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_PORT].name, (void *)&ipv4->source, TLD_TYPE_LONG); + break; + case ADDR_TYPE_IPV6: + case __ADDR_TYPE_IP_PAIR_V6: + ipv6=a_stream->addr.ipv6; + inet_ntop(AF_INET6, (void *)ipv6->saddr, client_ip, sizeof(client_ip)); + inet_ntop(AF_INET6, (void *)ipv6->daddr, server_ip, sizeof(server_ip)); + + addr_type=6; + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_IP].name, (void *)server_ip, TLD_TYPE_STRING); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_IP].name, (void *)client_ip, TLD_TYPE_STRING); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_PORT].name, (void *)&ipv6->dest, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_PORT].name, (void *)&ipv6->source, TLD_TYPE_LONG); + break; + default: + break; + } + + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)&a_stream->dir, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)&addr_type, TLD_TYPE_LONG); + + + addr_proto=(char *)layer_addr_prefix_ntop(a_stream); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_L4_PROTOCOL].name, (void *)addr_proto, TLD_TYPE_STRING); + + + ret=MESA_get_stream_opt(a_stream, MSO_STREAM_TUNNEL_TYPE, &tunnel_type, &tunnel_type_size); + assert(ret==0); + if(tunnel_type==STREAM_TUNNLE_NON) + { + layer_addr_ntop_r(a_stream,nest_addr_buf, sizeof(nest_addr_buf)); + } + else + { + stream_addr_list_ntop(a_stream,nest_addr_buf, sizeof(nest_addr_buf)); + } + + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ADDRESS_LIST].name, (void *)nest_addr_buf, TLD_TYPE_STRING); + return 0; +} static unsigned int get_ip_by_eth_name(const char *ifname) { @@ -46,65 +200,260 @@ error: return INADDR_NONE; } -int steaminfo2opt() +static int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t *service2topic) { - return 0; -} - - -int tsg_sendlog_init(char *filename) -{ - int ret=0; - unsigned int local_ip_nr=0; - char nic_name[32]; - - memset(&tsg_logger_info, 0, sizeof(tsg_logger_info)); - - MESA_load_profile_int_def(filename, "TSG_LOG", "MODE",&(tsg_logger_info.mode), 0); - MESA_load_profile_string_def(filename, "TSG_LOG", "FIELD_FILE", tsg_logger_info.field_file, sizeof(tsg_logger_info.field_file), NULL); - MESA_load_profile_string_def(filename, "TSG_LOG", "BROKER_LIST", tsg_logger_info.broker_list, sizeof(tsg_logger_info.broker_list), NULL); - - MESA_load_profile_int_def(filename, "TSG_LOG", "LEVEL",&(tsg_logger_info.level), 30); - MESA_load_profile_string_def(filename, "TSG_LOG", "LOG_PATH", tsg_logger_info.log_path, sizeof(tsg_logger_info.log_path), NULL); - - tsg_logger_info.logger=MESA_create_runtime_log_handle(tsg_logger_info.log_path, tsg_logger_info.level); - if(tsg_logger_info.logger==NULL) + int i=0; + int ret=0,id=0; + FILE *fp=NULL; + char line[1024]={0}; + char field_name[64]={0}; + char type_name[32]={0}; + + fp=fopen(filename, "r"); + if(fp==NULL) { - printf("MESA_create_runtime_log_handle failed ..., path: %s level: %d", tsg_logger_info.log_path, tsg_logger_info.level); + printf("Open %s failed ...", filename); return -1; } - tsg_logger_info.cabot_handle=cabot_sendlog_create(); - cabot_sendlog_set(tsg_logger_info.cabot_handle, SENDLOG_MODE,(void *)&(tsg_logger_info.mode)); - cabot_sendlog_set(tsg_logger_info.cabot_handle, CONFIG_FILE,(void *)tsg_logger_info.field_file); - cabot_sendlog_set(tsg_logger_info.cabot_handle, BROKER_LIST,(void *)&(tsg_logger_info.mode)); + memset(line, 0, sizeof(line)); - ret=cabot_sendlog_init(tsg_logger_info.cabot_handle, tsg_logger_info.logger); - if(ret<0) + while((fgets(line, sizeof(line), fp))!=NULL) { - MESA_handle_runtime_log(tsg_logger_info.logger, RLOG_LV_FATAL, "CABOT_INIT", "cabot_sendlog_init failed ..."); - return -2; + if(line[0]=='#' || line[0]=='\n' || line[0]=='\r' ||line[0]=='\0') + { + continue; + } + memset(type_name, 0, sizeof(type_name)); + ret=sscanf(line, "%s %s %d", type_name, field_name, &id); + assert(ret==3 && idmode), 0); + MESA_load_profile_string_def(conffile, "TSG_LOG", "COMMON_FIELD_FILE", _instance->common_field_file, sizeof(_instance->common_field_file), NULL); + MESA_load_profile_string_def(conffile, "TSG_LOG", "BROKER_LIST", _instance->broker_list, sizeof(_instance->broker_list), NULL); + + MESA_load_profile_int_def(conffile, "TSG_LOG", "LEVEL",&(level), 30); + MESA_load_profile_string_def(conffile, "TSG_LOG", "LOG_PATH", log_path, sizeof(log_path), NULL); + + _instance->logger=MESA_create_runtime_log_handle(log_path, level); + if(_instance->logger==NULL) + { + printf("MESA_create_runtime_log_handle failed ..., path: %s level: %d", log_path, level); + return NULL; + } + + MESA_load_profile_string_def(conffile, "TSG_LOG", "NIC_NAME", nic_name, sizeof(nic_name), "eth0"); local_ip_nr=get_ip_by_eth_name(nic_name); if(local_ip_nr==INADDR_NONE) { - MESA_handle_runtime_log(tsg_logger_info.logger,RLOG_LV_FATAL, "GET_LOCAL_IP","get NIC_NAME: %s error.", nic_name); - return -3; + MESA_handle_runtime_log(_instance->logger,RLOG_LV_FATAL, "GET_LOCAL_IP","get NIC_NAME: %s error.", nic_name); + return NULL; } - inet_ntop(AF_INET,&(local_ip_nr),tsg_logger_info.local_ip_str,sizeof(tsg_logger_info.local_ip_str)); + inet_ntop(AF_INET,&(local_ip_nr),_instance->local_ip_str,sizeof(_instance->local_ip_str)); - //maat - + rdkafka_conf = rd_kafka_conf_new(); + rd_kafka_conf_set(rdkafka_conf, "queue.buffering.max.messages", "1000000", kafka_errstr, sizeof(kafka_errstr)); + rd_kafka_conf_set(rdkafka_conf, "topic.metadata.refresh.interval.ms", "600000",kafka_errstr, sizeof(kafka_errstr)); + rd_kafka_conf_set(rdkafka_conf, "request.required.acks", "1", kafka_errstr, sizeof(kafka_errstr)); + if(!(kafka_handle=rd_kafka_new(RD_KAFKA_PRODUCER, rdkafka_conf, kafka_errstr, sizeof(kafka_errstr)))) + { + MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "CABOT_INIT", "rd_kafka_new is error"); + return NULL; + } + + if(rd_kafka_brokers_add(kafka_handle, _instance->broker_list) == 0) + { + MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "KAFKA_INIT", "rd_kafka_brokers_add is error, broker_list: %s", _instance->broker_list); + return NULL; + } + + MESA_load_profile_int_def(conffile, "TSG_LOG", "MAX_SERVICE",&(_instance->max_service), 0); + *(_instance->topic_rkt)=(rd_kafka_topic_t *)calloc(1, (1+_instance->max_service)*sizeof(rd_kafka_topic_t*)); + _instance->service2topic=(id2field_t *)calloc(1, (1+_instance->max_service)*sizeof(id2field_t)); + load_log_common_field(_instance->common_field_file, _instance->id2field, _instance->service2topic); + + for(i=0; i<_instance->max_service+1; i++) + { + if(_instance->service2topic[i].type==TLD_TYPE_MAX) + { + topic_conf=rd_kafka_topic_conf_new(); + _instance->topic_rkt[_instance->service2topic[i].id]=rd_kafka_topic_new(kafka_handle, _instance->service2topic[i].name, topic_conf); + } + } + + return (tsg_log_instance_t)_instance; +} + + +int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *log_msg, int thread_id) +{ + int i=0,status=0; + char *payload=NULL; + time_t cur_time; + struct vxlan_info vinfo; + int opt_val_len = sizeof(vinfo); + struct _tld_handle *_handle = (struct _tld_handle *)handle; + struct _tsg_log_instance *_instance = (struct _tsg_log_instance *)instance; + + if(_instance==NULL || _handle==NULL || log_msg==NULL) + { + MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "TSG_SEND_LOG", " instance==NULL || TLD_handle==NULL || log_msg==NULL "); + return -1; + } + + if(_instance->mode==CLOSE) + { + MESA_handle_runtime_log(_instance->logger, RLOG_LV_INFO, "TSG_SEND_LOG", "Disable tsg_send_log."); + return 0; + } + + //TODO + //common_user_tags + //common_user_region + //common_isp + //common_app_label + //common_app_id + //common_protocol_id + //common_has_dup_traffic + //common_stream_error + //common_stream_trace_id + + TLD_append_streaminfo(_instance, _handle, log_msg->a_stream); + + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SLED_IP].name, (void *)(_instance->local_ip_str), TLD_TYPE_STRING); + + if(log_msg->a_stream!=NULL && log_msg->a_stream->ptcpdetail!=NULL) + { + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(&log_msg->a_stream->ptcpdetail->createtime), TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(&log_msg->a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG); + } + else + { + cur_time=time(NULL); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)&cur_time, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)&cur_time, TLD_TYPE_LONG); + } + + + status=MESA_get_stream_opt(log_msg->a_stream, MSO_STREAM_VXLAN_INFO, &vinfo, &opt_val_len); + if(status < 0) + { + MESA_handle_runtime_log(_instance->logger, RLOG_LV_DEBUG, "TSG_SEND_LOG", "tsg log: get vxlan info error, tuple4: %s", printaddr(&log_msg->a_stream->addr, thread_id)); + } + else + { + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_LINK_ID].name, (void *)&vinfo.link_id, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_DIRECTION].name, (void *)&vinfo.link_dir, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_DEVICE_ID].name, (void *)&vinfo.dev_id, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENTRANCE_ID].name, (void *)&vinfo.entrance_id, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENCAPSULATION].name, (void *)&vinfo.encap_type, TLD_TYPE_LONG); + } + + for(i=0;iresult_num; i++) + { + switch(log_msg->result[i].do_log) + { + case LOG_ABORT: + MESA_handle_runtime_log(_instance->logger, RLOG_LV_INFO, + "TSG_SEND_LOG", + "tsg abort log:cfg_id=%d service=%d addr=%s", + log_msg->result[i].config_id, + log_msg->result[i].service_id, + printaddr(&(log_msg->a_stream->addr), thread_id)); + + continue; + break; + case LOG_ALL: + break; + case LOG_NOFILE: + break; + default: + break; + } + + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(&log_msg->result[i].config_id), TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(&log_msg->result[i].service_id), TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(&log_msg->result[i].action), TLD_TYPE_LONG); + + payload = cJSON_PrintUnformatted(_handle->object); + + status = rd_kafka_produce(_instance->topic_rkt[log_msg->result[i].service_id], RD_KAFKA_PARTITION_UA, RD_KAFKA_MSG_F_COPY, payload, strlen(payload), NULL, 0, NULL); + + if(status < 0) + { + MESA_handle_runtime_log(_instance->logger, RLOG_LV_INFO, "TSG_SEND_LOG", + "tsg_send_log to kafka is error, status: %d, topic: %s payload: %s", + status, _instance->service2topic[log_msg->result[i].service_id].name, payload); + } + else + { + MESA_handle_runtime_log(_instance->logger,RLOG_LV_INFO, "TSG_SEND_LOG", + "log send successfully %s: %s", _instance->service2topic[log_msg->result[i].service_id].name, payload); + } + + free(payload); + payload=NULL; + + TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name); + TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVICE].name); + TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ACTION].name); + } + + + cJSON_Delete(_handle->object); + dictator_free(thread_id, handle); + handle=NULL; + return 0; } -void tsg_send_log(const tsg_log_t* log_msg, struct _opt_unit_t* log_opt, int opt_num, int thread_id) -{ - - -} diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index 02b5104..e18bd5d 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -1,17 +1,57 @@ #ifndef __TSG_SEND_LOG_INTERNAL_H__ #define __TSG_SEND_LOG_INTERNAL_H__ -typedef struct _tsg_logger_info +#include +#include + +#include "tsg_types.h" + + +#define MAX_IPV4_LEN 16 +#define MAX_STRING_LEN 32 + +enum _SEND_MODE +{ + CLOSE=0, + KAFKA=1, +}; + +enum _DO_LOG +{ + LOG_ABORT=0, + LOG_NOFILE=1, + LOG_ALL=2 +}; + +typedef struct _id2field +{ + TLD_TYPE type; + int id; + char name[MAX_STRING_LEN]; +}id2field_t; + +struct _tld_handle +{ + int thread_id; + cJSON *object; +}; + +struct _tsg_log_instance { int mode; - int level; + int max_service; void *logger; - void *cabot_handle; - char field_file[128]; - char broker_list[128]; - char log_path[128]; - char local_ip_str[16]; -}tsg_logger_info_t; + char common_field_file[MAX_STRING_LEN*4]; + char broker_list[MAX_STRING_LEN*4]; + char local_ip_str[MAX_IPV4_LEN]; + id2field_t id2field[LOG_COMMON_MAX]; + rd_kafka_topic_t **topic_rkt; + id2field_t *service2topic; +}; + + +char *log_field_id2name(tsg_log_instance_t instance, tsg_log_field_id_t id); +tsg_log_instance_t tsg_sendlog_init(const char *filename); #endif From 39f82ed3f70718b975ca215c486fbe736f6a06b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E4=B8=80=E9=B8=A3?= Date: Fri, 15 Nov 2019 21:22:39 +0800 Subject: [PATCH 06/32] =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_rule.cpp | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index db1c67d..cbe4876 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -11,33 +11,33 @@ enum kni_scan_table{ SCAN_TABLE_MAX }; -struct kni_protocol_identify_result{ - enum tsg_protocol protocol; - char domain[TSG_DOMAIN_MAX]; - int domain_len; -}; - Maat_feather_t g_kni_maat_feather; const char *g_kni_scan_table_name[SCAN_TABLE_MAX]; int g_kni_scan_tableid[SCAN_TABLE_MAX] = {0}; -static void protocol_identify(char *buff, int buff_len, struct kni_protocol_identify_result *result){ - result->protocol = TSG_PROTOCOL_UNKNOWN; +static void protocol_identify(char *buff, int buff_len, struct _identify_info *result){ + result->protocol = -1; //TODO: http: get from http protocol plugin - + /* + if(is_http){ + result->protocol = PROTO_HTTP; + return; + } + */ //ssl enum chello_parse_result chello_status = CHELLO_PARSE_INVALID_FORMAT; struct ssl_chello *chello = NULL; chello = ssl_chello_parse((const unsigned char*)buff, buff_len, &chello_status); if(chello_status == CHELLO_PARSE_SUCCESS){ - result->protocol = TSG_PROTOCOL_SSL; + result->protocol = PROTO_SSL; if(chello->sni == NULL){ result->domain_len = 0; } else{ - strncpy(result->domain, chello->sni, strnlen(chello->sni, sizeof(result->domain) - 1)); - result->domain_len = strlen(result->domain); + result->domain_len = strnlen(chello->sni, sizeof(result->domain) - 1); + strncpy(result->domain, chello->sni, result->domain_len); } + result->domain[result->domain_len] = '\0'; } ssl_chello_free(chello); return; @@ -60,25 +60,22 @@ int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, voi return 0; } -//return -1 if failed, return 0 on success -int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, enum tsg_protocol *protocol, char *domain, int *domain_len, - scan_status_t *mid, void *logger, int thread_seq){ - struct kni_protocol_identify_result protocol_identify_res; - memset(&protocol_identify_res, 0, sizeof(protocol_identify_res)); - protocol_identify((char*)pkt, pkt_len, &protocol_identify_res); - if(protocol_identify_res.protocol == TSG_PROTOCOL_UNKNOWN){ + +//return value: -1: failed, 0: not hit, >0: hit count +int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, + struct _identify_info *identify_info, scan_status_t *mid, void *logger, int thread_seq){ + memset(identify_info, 0, sizeof(*identify_info)); + protocol_identify((char*)pkt, pkt_len, identify_info); + if(identify_info->protocol != TSG_PROTOCOL_SSL && identify_info->protocol != TSG_PROTOCOL_HTTP){ return -1; } - *protocol = protocol_identify_res.protocol; - *domain_len = protocol_identify_res.domain_len; - strncpy(domain, protocol_identify_res.domain, *domain_len); int tableid; - if(protocol_identify_res.protocol == TSG_PROTOCOL_SSL){ + if(identify_info->protocol == TSG_PROTOCOL_SSL){ tableid = g_kni_scan_tableid[TSG_FIELD_SSL_SNI]; } - if(protocol_identify_res.protocol == TSG_PROTOCOL_HTTP){ + else{ tableid = g_kni_scan_tableid[TSG_FIELD_HTTP_HOST]; } - return Maat_full_scan_string(g_kni_maat_feather, tableid, CHARSET_UTF8, domain, *domain_len, + return Maat_full_scan_string(g_kni_maat_feather, tableid, CHARSET_UTF8, identify_info->domain, identify_info->domain_len, result, NULL, result_num, mid, thread_seq); } \ No newline at end of file From 3b13075701f1459f4bf24cb173ceb5c20a5857a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AD=A6=E5=88=A9?= Date: Mon, 18 Nov 2019 13:20:57 +0800 Subject: [PATCH 07/32] Delete tsg_entry.h --- inc/tsg_entry.h | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 inc/tsg_entry.h diff --git a/inc/tsg_entry.h b/inc/tsg_entry.h deleted file mode 100644 index 7d56227..0000000 --- a/inc/tsg_entry.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef __TSG_ENTRY_H__ -#define __TSG_ENTRY_H__ - -#include -#include "tsg_types.h" - -#define FW_ACTION_UNKNOWN 0x00 -#define FW_ACTION_DROP 0x01 -#define FW_ACTION_RESET 0x02 -#define FW_ACTION_BLOCK 0x04 - -#define TSG_ACTION_NONE 0x00 -#define TSG_ACTION_MONITOR 0x01 -#define TSG_ACTION_INTERCEPT 0x02 -#define TSG_ACTION_DENY 0x10 -#define TSG_ACTION_MANIPULATE 0x30 -#define TSG_ACTION_BYPASS 0x80 - - -#define MAX_RESULT_NUM 8 -#define MAX_DOAMIN_LEN 2048 - -typedef struct _policy_priority_label -{ - tsg_protocol_t proto; //enum _tsg_protocol (tsg_types.h) - int domain_len; - int result_type; //enum _PULL_RESULT_TYPE (tsg_rule.h) - int result_num; - char domain[MAX_DOAMIN_LEN]; - Maat_rule_t result[MAX_RESULT_NUM]; -}policy_priority_label_t; - - -typedef struct _tsg_para -{ - int ip_addr_table_id; - int subscribe_id_table_id; - int priority_project_id; - void *logger; -}g_tsg_para_t; - - - -extern g_tsg_para_t g_tsg_para; - -#endif \ No newline at end of file From ab58f918e5ff3f6c22c1403a303c06653c3b3b3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AD=A6=E5=88=A9?= Date: Mon, 18 Nov 2019 13:21:05 +0800 Subject: [PATCH 08/32] Delete tsg_log_id.h --- inc/tsg_log_id.h | 90 ------------------------------------------------ 1 file changed, 90 deletions(-) delete mode 100644 inc/tsg_log_id.h diff --git a/inc/tsg_log_id.h b/inc/tsg_log_id.h deleted file mode 100644 index 177ebf8..0000000 --- a/inc/tsg_log_id.h +++ /dev/null @@ -1,90 +0,0 @@ -#ifndef __TSG_CABOT_LOG_ID_H__ -#define __TSG_CABOT_LOG_ID_H__ - -#define CABOT_LOG_OPT_POLICY_ID 1 -#define CABOT_LOG_OPT_POLICY_SERVICE 2 -#define CABOT_LOG_OPT_POLICY_ACTION 3 -#define CABOT_LOG_OPT_START_TIME 4 -#define CABOT_LOG_OPT_END_TIME 5 -#define CABOT_LOG_OPT_L4_PROTOCL 6 -#define CABOT_LOG_OPT_ADDRESS_TYPE 7 -#define CABOT_LOG_OPT_SERVER_IP 8 -#define CABOT_LOG_OPT_CLIENT_IP 9 -#define CABOT_LOG_OPT_SERVER_PORT 10 -#define CABOT_LOG_OPT_CLIENT_PORT 11 -#define CABOT_LOG_OPT_STREAM_DIR 12 -#define CABOT_LOG_OPT_ADDRESS_LIST 13 - -#define CABOT_LOG_OPT_ENTRANCE_ID 14 -#define CABOT_LOG_OPT_DEVICE_ID 15 -#define CABOT_LOG_OPT_LINK_ID 16 -#define CABOT_LOG_OPT_ISP 17 -#define CABOT_LOG_OPT_ENCAPSULATION 18 -#define CABOT_LOG_OPT_DIRECTION 19 -#define CABOT_LOG_OPT_SLED_IP 20 - -#define CABOT_LOG_OPT_USER_TAGS 21 -#define CABOT_LOG_OPT_USER_REGION 22 -#define CABOT_LOG_OPT_APP_ID 23 -#define CABOT_LOG_OPT_PROTOCIL_ID 24 -#define CABOT_LOG_OPT_C2S_PKT_NUM 25 -#define CABOT_LOG_OPT_S2C_PKT_NUM 26 -#define CABOT_LOG_OPT_C2S_BYTE_NUM 27 -#define CABOT_LOG_OPT_S2C_BYTE_NUM 28 -#define CABOT_LOG_OPT_CON_DURATION_MS 29 -#define CABOT_LOG_OPT_CON_LATENCY_MS 30 -#define CABOT_LOG_OPT_STREAM_TRACE_ID 31 - -#define CABOT_LOG_OPT_HTTP_URL 32 -#define CABOT_LOG_OPT_HTTP_REQUEST_LINE 33 -#define CABOT_LOG_OPT_HTTP_RESPONSE_LINE 34 -#define CABOT_LOG_OPT_HTTP_REQUEST_HEADER 35 -#define CABOT_LOG_OPT_HTTP_RESPONSE_HEADER 36 -#define CABOT_LOG_OPT_HTTP_REQUEST_BODY 37 -#define CABOT_LOG_OPT_HTTP_RESPONSE_BODY 38 -#define CABOT_LOG_OPT_HTTP_C2S_ISN 39 -#define CABOT_LOG_OPT_HTTP_PROXY_FLAG 40 -#define CABOT_LOG_OPT_HTTP_SEQUENCE 41 -#define CABOT_LOG_OPT_HTTP_SNAPSHOT 42 -#define CABOT_LOG_OPT_HTTP_COOKIE 43 -#define CABOT_LOG_OPT_HTTP_REFERER 44 -#define CABOT_LOG_OPT_HTTP_USER_AGENT 45 -#define CABOT_LOG_OPT_HTTP_CONTENT_LENGTH 46 -#define CABOT_LOG_OPT_HTTP_CONTENT_TYPE 47 -#define CABOT_LOG_OPT_HTTP_SET_COOKIE 48 - - -#define CABOT_LOG_OPT_MAIL_PROTOCOL 51 -#define CABOT_LOG_OPT_MAIL_SENDER 52 -#define CABOT_LOG_OPT_MAIL_RECEIVER 53 -#define CABOT_LOG_OPT_MAIL_SUBJECT 54 -#define CABOT_LOG_OPT_MAIL_CONTENT 55 -#define CABOT_LOG_OPT_MAIL_ATTACHMENT_NAME 56 -#define CABOT_LOG_OPT_MAIL_ATTACHMENT_CONTENT 57 -#define CABOT_LOG_OPT_MAIL_EML_FILE 58 -#define CABOT_LOG_OPT_MAIL_SNAPSHOT 59 - - -#define CABOT_LOG_OPT_DNS_MESSAGE_ID 61 -#define CABOT_LOG_OPT_DNS_QR 62 -#define CABOT_LOG_OPT_DNS_OPCODE 63 -#define CABOT_LOG_OPT_DNS_AA 64 -#define CABOT_LOG_OPT_DNS_TC 65 -#define CABOT_LOG_OPT_DNS_RD 66 -#define CABOT_LOG_OPT_DNS_RA 67 -#define CABOT_LOG_OPT_DNS_RCODE 68 -#define CABOT_LOG_OPT_DNS_QDCOUNT 69 -#define CABOT_LOG_OPT_DNS_ANCOUNT 70 -#define CABOT_LOG_OPT_DNS_NSCOUNT 71 -#define CABOT_LOG_OPT_DNS_ARCOUNT 72 -#define CABOT_LOG_OPT_DNS_QNAME 73 -#define CABOT_LOG_OPT_DNS_QTYPE 74 -#define CABOT_LOG_OPT_DNS_QCLASS 75 -#define CABOT_LOG_OPT_DNS_CNAME 76 -#define CABOT_LOG_OPT_DNS_SUB 77 -#define CABOT_LOG_OPT_DNS_RR 78 - - - - -#endif From 382ca0a7932bbda8bbb735cd9d71e4984af0c145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AD=A6=E5=88=A9?= Date: Mon, 18 Nov 2019 13:21:25 +0800 Subject: [PATCH 09/32] Delete tsg_types.h --- inc/tsg_types.h | 64 ------------------------------------------------- 1 file changed, 64 deletions(-) delete mode 100644 inc/tsg_types.h diff --git a/inc/tsg_types.h b/inc/tsg_types.h deleted file mode 100644 index da32e5f..0000000 --- a/inc/tsg_types.h +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef __TSG_TYPES_H__ -#define __TSG_TYPES_H__ - -typedef enum _tsg_log_field_id -{ - LOG_COMMON_SERVICE, - LOG_COMMON_DIRECTION, - LOG_COMMON_L4_PROTOCOL, - LOG_COMMON_ADDRESS_TYPE, - LOG_COMMON_SCHAME_TYPE, - LOG_COMMON_POLICY_ID, - LOG_COMMON_USER_TAG, - LOG_COMMON_ACTION, - LOG_COMMON_USER_REGION, - LOG_COMMON_CLIENT_IP, - LOG_COMMON_CLIENT_PORT, - LOG_COMMON_ENTRANCE_ID, - LOG_COMMON_DEVICE_ID, - LOG_COMMON_LINK_ID, - LOG_COMMON_ISP, - LOG_COMMON_ENCAPSULATION, - LOG_COMMON_SLED_IP, - LOG_COMMON_SERVER_IP, - LOG_COMMON_SERVER_PORT, - LOG_COMMON_APP_LABEL, - LOG_COMMON_APP_ID, - LOG_COMMON_PROTOCOL_ID, - LOG_COMMON_C2S_PKT_NUM, - LOG_COMMON_S2C_PKT_NUM, - LOG_COMMON_C2S_BYTE_NUM, - LOG_COMMON_S2C_BYTE_NUM, - LOG_COMMON_START_TIME, - LOG_COMMON_END_TIME, - LOG_COMMON_CON_DURATION_MS, - LOG_COMMON_STREAM_DIR, - LOG_COMMON_ADDRESS_LIST, - LOG_COMMON_HAS_DUP_TRAFFIC, - LOG_COMMON_STREAM_ERROR, - LOG_COMMON_STREAM_TRACE_ID, - LOG_HTTP_HOST, - LOG_SSL_SNI, - LOG_COMMON_MAX -}tsg_log_field_id_t; - - -typedef enum _tsg_protocol -{ - PROTO_UNKONWN=0, - PROTO_IPv4=1, - PROTO_IPv6, - PROTO_TCP, - PROTO_UDP, - PROTO_HTTP, - PROTO_MAIL, - PROTO_DNS, - PROTO_FTP, - PROTO_SSL, - PROTO_SIP, - PROTO_BGP, - PROTO_STREAMING_MEDIA, - PROTO_MAX -}tsg_protocol_t; - -#endif From 88d9368cee4afac762f5be1f786a74a259c09e85 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Mon, 18 Nov 2019 13:25:38 +0800 Subject: [PATCH 10/32] =?UTF-8?q?=E5=90=88=E5=B9=B6=E5=A4=B4=E6=96=87?= =?UTF-8?q?=E4=BB=B6=EF=BC=8C=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8=E5=A4=B4?= =?UTF-8?q?=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 21 ++++++++++++++----- inc/tsg_rule.h | 19 +++++++++++++++++ src/tsg_entry.cpp | 2 +- src/tsg_rule.cpp | 3 ++- src/tsg_send_log.cpp | 4 ++-- src/tsg_send_log_internal.h | 41 +++++++++++++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 879fe12..e7b90cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,7 @@ include(Version) set(CMAKE_MACOSX_RPATH 0) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") include_directories(${PROJECT_SOURCE_DIR}/inc/) include_directories(/opt/MESA/include/) @@ -28,6 +29,8 @@ elseif(ENABLE_SANITIZE_THREAD) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan") endif() +set(CMAKE_INSTALL_PREFIX /home/mesasoft/sapp_run) + add_subdirectory (src) set(CONFLIST /plug/platform/conflist_platform.inf) @@ -35,20 +38,28 @@ file(WRITE ${PROJECT_SOURCE_DIR}/install.sh "#!/bin/sh\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "DST=\${RPM_INSTALL_PREFIX}\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "mkdir -p \${DST}/plug/platform/\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "touch \${DST}${CONFLIST}\r\n") + +file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "if [[ -z `grep -rn 'POLICY_PRIORITY' ${DST}/conf/project_list.conf` ]];then\r\n") +file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "echo 'POLICY_PRIORITY struct' >> \${DST}/conf/project_list.conf\r\nfi\r\n") + file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "if [[ -z `grep -rn 'tsg_master.inf' \${DST}${CONFLIST}` ]];then\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "echo './plug/platform/tsg_master/tsg_master.inf' >> \${DST}/plug/platform/conflist_platform.inf\r\nfi\r\n") + SET(CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/install.sh") file(WRITE ${PROJECT_SOURCE_DIR}/uninstall.sh "#!/bin/sh\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "DST=\${RPM_INSTALL_PREFIX}\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "mkdir -p \${DST}/plug/platform/\r\n") + file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "sed -i '/tsg_master/d' \${DST}/plug/platform/conflist_platform.inf\r\n") +file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "sed -i '/POLICY_PRIORITY/d' \${DST}/conf/project_list.conf\r\n") + SET(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/uninstall.sh") -install(FILES inc/soq_sendlog.h DESTINATION /opt/MESA/include/soq) -install(FILES inc/soq_types.h DESTINATION /opt/MESA/include/soq) -install(FILES inc/t1_public.h DESTINATION /opt/MESA/include/soq) -install(FILES inc/t2_public.h DESTINATION /opt/MESA/include/soq) -install(FILES ddp/inc/ddp.h DESTINATION /opt/MESA/include/soq) +install(FILES bin/main.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf) +install(FILES bin/tsg_tableinfo.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf) +install(FILES inc/tsg_send_log.h DESTINATION /opt/MESA/include/tsg) +install(FILES inc/tsg_types.h DESTINATION /opt/MESA/include/tsg) +install(FILES inc/tsg_rule.h DESTINATION /opt/MESA/include/tsg) include(Package) diff --git a/inc/tsg_rule.h b/inc/tsg_rule.h index bbd3667..7f2ff3f 100644 --- a/inc/tsg_rule.h +++ b/inc/tsg_rule.h @@ -9,6 +9,25 @@ extern "C" { #endif +typedef enum _tsg_protocol +{ + PROTO_UNKONWN=0, + PROTO_IPv4=1, + PROTO_IPv6, + PROTO_TCP, + PROTO_UDP, + PROTO_HTTP, + PROTO_MAIL, + PROTO_DNS, + PROTO_FTP, + PROTO_SSL, + PROTO_SIP, + PROTO_BGP, + PROTO_STREAMING_MEDIA, + PROTO_MAX +}tsg_protocol_t; + + #define MAX_DOAMIN_LEN 2048 struct _identify_info diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index ec658ab..d104cb6 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -276,7 +276,7 @@ extern "C" int TSG_MASTER_INIT() -extern "C" int T2_MASTER_UNLOAD() +extern "C" int TSG_MASTER_UNLOAD() { return 0; } diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 7a1c296..8f71b7c 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -19,7 +19,7 @@ int tsg_rule_init(const char* conffile, void *logger) { unsigned short redis_port = 0; int ret=0,scan_detail=0,effect_interval=60; - const char* instance_name=NULL,*module="MAAT"; + const char* instance_name="TSG",*module="MAAT"; int factor=0, redis_port_num=0,redis_index=0; char redis_ip[16]={0}, effective_flag[1024]={0}; int maat_mode=0,maat_stat_on=0,maat_perf_on=0,thread_max=0; @@ -41,6 +41,7 @@ int tsg_rule_init(const char* conffile, void *logger) thread_max=get_thread_count(); g_tsg_maat_feather=Maat_feather(thread_max, table_info, logger); + if(maat_mode==2) { MESA_load_profile_string_def(conffile,module,"REDIS_IP", redis_ip, sizeof(redis_ip),""); diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 636fb5d..c6f210b 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -279,7 +279,7 @@ tsg_log_instance_t tsg_sendlog_init(const char *conffile) MESA_load_profile_string_def(conffile, "TSG_LOG", "COMMON_FIELD_FILE", _instance->common_field_file, sizeof(_instance->common_field_file), NULL); MESA_load_profile_string_def(conffile, "TSG_LOG", "BROKER_LIST", _instance->broker_list, sizeof(_instance->broker_list), NULL); - MESA_load_profile_int_def(conffile, "TSG_LOG", "LEVEL",&(level), 30); + MESA_load_profile_int_def(conffile, "TSG_LOG", "LOG_LEVEL",&(level), 30); MESA_load_profile_string_def(conffile, "TSG_LOG", "LOG_PATH", log_path, sizeof(log_path), NULL); _instance->logger=MESA_create_runtime_log_handle(log_path, level); @@ -306,7 +306,7 @@ tsg_log_instance_t tsg_sendlog_init(const char *conffile) if(!(kafka_handle=rd_kafka_new(RD_KAFKA_PRODUCER, rdkafka_conf, kafka_errstr, sizeof(kafka_errstr)))) { - MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "CABOT_INIT", "rd_kafka_new is error"); + MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "KAFKA_INIT", "rd_kafka_new is error"); return NULL; } diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index e18bd5d..7162b42 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -49,6 +49,47 @@ struct _tsg_log_instance id2field_t *service2topic; }; +typedef enum _tsg_log_field_id +{ + LOG_COMMON_SERVICE, + LOG_COMMON_DIRECTION, + LOG_COMMON_L4_PROTOCOL, + LOG_COMMON_ADDRESS_TYPE, + LOG_COMMON_SCHAME_TYPE, + LOG_COMMON_POLICY_ID, + LOG_COMMON_USER_TAG, + LOG_COMMON_ACTION, + LOG_COMMON_USER_REGION, + LOG_COMMON_CLIENT_IP, + LOG_COMMON_CLIENT_PORT, + LOG_COMMON_ENTRANCE_ID, + LOG_COMMON_DEVICE_ID, + LOG_COMMON_LINK_ID, + LOG_COMMON_ISP, + LOG_COMMON_ENCAPSULATION, + LOG_COMMON_SLED_IP, + LOG_COMMON_SERVER_IP, + LOG_COMMON_SERVER_PORT, + LOG_COMMON_APP_LABEL, + LOG_COMMON_APP_ID, + LOG_COMMON_PROTOCOL_ID, + LOG_COMMON_C2S_PKT_NUM, + LOG_COMMON_S2C_PKT_NUM, + LOG_COMMON_C2S_BYTE_NUM, + LOG_COMMON_S2C_BYTE_NUM, + LOG_COMMON_START_TIME, + LOG_COMMON_END_TIME, + LOG_COMMON_CON_DURATION_MS, + LOG_COMMON_STREAM_DIR, + LOG_COMMON_ADDRESS_LIST, + LOG_COMMON_HAS_DUP_TRAFFIC, + LOG_COMMON_STREAM_ERROR, + LOG_COMMON_STREAM_TRACE_ID, + LOG_HTTP_HOST, + LOG_SSL_SNI, + LOG_COMMON_MAX +}tsg_log_field_id_t; + char *log_field_id2name(tsg_log_instance_t instance, tsg_log_field_id_t id); tsg_log_instance_t tsg_sendlog_init(const char *filename); From f33d11c50b01db96f7df652cd0f1dd314a71939e Mon Sep 17 00:00:00 2001 From: liuxueli Date: Mon, 18 Nov 2019 13:29:28 +0800 Subject: [PATCH 11/32] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/tsg_rule.h | 1 - src/tsg_send_log.cpp | 1 - src/tsg_send_log_internal.h | 54 +++++++++++++++++-------------------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/inc/tsg_rule.h b/inc/tsg_rule.h index 7f2ff3f..ffd5682 100644 --- a/inc/tsg_rule.h +++ b/inc/tsg_rule.h @@ -2,7 +2,6 @@ #define __TSG_RULE_H__ #include -#include "tsg_types.h" #ifdef __cplusplus extern "C" diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index c6f210b..d163bde 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -15,7 +15,6 @@ #include #include -#include "tsg_types.h" #include "tsg_send_log.h" #include "tsg_send_log_internal.h" diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index 7162b42..b041aa2 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -4,9 +4,6 @@ #include #include -#include "tsg_types.h" - - #define MAX_IPV4_LEN 16 #define MAX_STRING_LEN 32 @@ -23,32 +20,6 @@ enum _DO_LOG LOG_ALL=2 }; -typedef struct _id2field -{ - TLD_TYPE type; - int id; - char name[MAX_STRING_LEN]; -}id2field_t; - -struct _tld_handle -{ - int thread_id; - cJSON *object; -}; - -struct _tsg_log_instance -{ - int mode; - int max_service; - void *logger; - char common_field_file[MAX_STRING_LEN*4]; - char broker_list[MAX_STRING_LEN*4]; - char local_ip_str[MAX_IPV4_LEN]; - id2field_t id2field[LOG_COMMON_MAX]; - rd_kafka_topic_t **topic_rkt; - id2field_t *service2topic; -}; - typedef enum _tsg_log_field_id { LOG_COMMON_SERVICE, @@ -90,6 +61,31 @@ typedef enum _tsg_log_field_id LOG_COMMON_MAX }tsg_log_field_id_t; +typedef struct _id2field +{ + TLD_TYPE type; + int id; + char name[MAX_STRING_LEN]; +}id2field_t; + +struct _tld_handle +{ + int thread_id; + cJSON *object; +}; + +struct _tsg_log_instance +{ + int mode; + int max_service; + void *logger; + char common_field_file[MAX_STRING_LEN*4]; + char broker_list[MAX_STRING_LEN*4]; + char local_ip_str[MAX_IPV4_LEN]; + id2field_t id2field[LOG_COMMON_MAX]; + rd_kafka_topic_t **topic_rkt; + id2field_t *service2topic; +}; char *log_field_id2name(tsg_log_instance_t instance, tsg_log_field_id_t id); tsg_log_instance_t tsg_sendlog_init(const char *filename); From e7532d6280c71ec79dc535eacb5e151ee25dfd45 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Mon, 18 Nov 2019 13:33:29 +0800 Subject: [PATCH 12/32] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=81=97=E6=BC=8F?= =?UTF-8?q?=E5=A4=B4=E6=96=87=E4=BB=B6tsg=5Fentry.h?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_entry.h | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 src/tsg_entry.h diff --git a/src/tsg_entry.h b/src/tsg_entry.h new file mode 100644 index 0000000..f3c9734 --- /dev/null +++ b/src/tsg_entry.h @@ -0,0 +1,45 @@ +#ifndef __TSG_ENTRY_H__ +#define __TSG_ENTRY_H__ + +#include + +#define FW_ACTION_UNKNOWN 0x00 +#define FW_ACTION_DROP 0x01 +#define FW_ACTION_RESET 0x02 +#define FW_ACTION_BLOCK 0x04 + +#define TSG_ACTION_NONE 0x00 +#define TSG_ACTION_MONITOR 0x01 +#define TSG_ACTION_INTERCEPT 0x02 +#define TSG_ACTION_DENY 0x10 +#define TSG_ACTION_MANIPULATE 0x30 +#define TSG_ACTION_BYPASS 0x80 + + +#define MAX_RESULT_NUM 8 +#define MAX_DOAMIN_LEN 2048 + +typedef struct _policy_priority_label +{ + tsg_protocol_t proto; //enum _tsg_protocol (tsg_types.h) + int domain_len; + int result_type; //enum _PULL_RESULT_TYPE (tsg_rule.h) + int result_num; + char domain[MAX_DOAMIN_LEN]; + Maat_rule_t result[MAX_RESULT_NUM]; +}policy_priority_label_t; + + +typedef struct _tsg_para +{ + int ip_addr_table_id; + int subscribe_id_table_id; + int priority_project_id; + void *logger; +}g_tsg_para_t; + + + +extern g_tsg_para_t g_tsg_para; + +#endif \ No newline at end of file From 6b6cbeffa1469becf9cbeea927102067eb9fad08 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Mon, 18 Nov 2019 14:50:32 +0800 Subject: [PATCH 13/32] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_send_log.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index d163bde..803b0bb 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -316,7 +316,9 @@ tsg_log_instance_t tsg_sendlog_init(const char *conffile) } MESA_load_profile_int_def(conffile, "TSG_LOG", "MAX_SERVICE",&(_instance->max_service), 0); + (_instance->topic_rkt)=(rd_kafka_topic_t **)calloc(1, sizeof(rd_kafka_topic_t*)); *(_instance->topic_rkt)=(rd_kafka_topic_t *)calloc(1, (1+_instance->max_service)*sizeof(rd_kafka_topic_t*)); + _instance->service2topic=(id2field_t *)calloc(1, (1+_instance->max_service)*sizeof(id2field_t)); load_log_common_field(_instance->common_field_file, _instance->id2field, _instance->service2topic); From 484feaed94006226f610ef68015c4a38ac150054 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Tue, 19 Nov 2019 13:39:37 +0800 Subject: [PATCH 14/32] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=8E=B7=E5=8F=96deny?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E8=A7=84=E5=88=99=E7=9A=84=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=20=E4=BF=AE=E5=A4=8D=E8=87=AA=E6=B5=8B=E8=AF=95=E8=BF=87?= =?UTF-8?q?=E7=A8=8B=E4=B8=AD=E9=81=87=E5=88=B0=E7=9A=84BUG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/tsg_rule.h | 11 ++--- inc/tsg_send_log.h | 8 ---- src/CMakeLists.txt | 2 +- src/tsg_entry.cpp | 23 ++++----- src/tsg_entry.h | 13 +---- src/tsg_rule.cpp | 57 ++++++++++++++++++++-- src/tsg_send_log.cpp | 96 ++++++++++++++----------------------- src/tsg_send_log_internal.h | 27 ++++++----- src/version.map | 3 +- 9 files changed, 125 insertions(+), 115 deletions(-) diff --git a/inc/tsg_rule.h b/inc/tsg_rule.h index 4e0e198..7fd7a93 100644 --- a/inc/tsg_rule.h +++ b/inc/tsg_rule.h @@ -3,11 +3,6 @@ #include -#ifdef __cplusplus -extern "C" -{ -#endif - typedef enum _tsg_protocol { PROTO_UNKONWN=0, @@ -27,6 +22,7 @@ typedef enum _tsg_protocol }tsg_protocol_t; +#define MAX_RESULT_NUM 8 #define MAX_DOAMIN_LEN 2048 struct _identify_info @@ -59,8 +55,7 @@ int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, voi //return value: -1: failed, 0: not hit, >0: hit count int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, struct _identify_info *identify_info, scan_status_t *mid, void *logger, int thread_seq); -#ifdef __cplusplus -} -#endif +//return NULL if none exists, otherwise return one deny rule; +Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num); #endif diff --git a/inc/tsg_send_log.h b/inc/tsg_send_log.h index cb0f5d7..0115f46 100644 --- a/inc/tsg_send_log.h +++ b/inc/tsg_send_log.h @@ -3,10 +3,6 @@ #include -#ifdef __cplusplus -extern "C" -{ -#endif typedef struct _tsg_log { @@ -37,8 +33,4 @@ int TLD_cancel(TLD_handle_t handle); int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *log_msg, int thread_id); -#ifdef __cplusplus -} -#endif - #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dd58965..b41393f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,7 @@ set(SRC tsg_entry.cpp tsg_rule.cpp tsg_ssl_utils.cpp tsg_send_log.cpp) include_directories(${CMAKE_SOURCE_DIR}/inc) include_directories(/opt/MESA/include/MESA/) -set(TSG_MASTER_DEPEND_DYN_LIB MESA_handle_logger MESA_prof_load maatframe pthread MESA_field_stat2) +set(TSG_MASTER_DEPEND_DYN_LIB MESA_handle_logger MESA_prof_load maatframe pthread MESA_field_stat2 rdkafka cjson) set(CMAKE_INSTALL_PREFIX /home/mesasoft/sapp_run) diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index d104cb6..439d246 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -14,7 +14,7 @@ -char TSG_MASTER_VERSION_20191115=0; +char TSG_MASTER_VERSION_20191119=0; const char *tsg_conffile="tsgconf/main.conf"; g_tsg_para_t g_tsg_para; @@ -85,7 +85,7 @@ static Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result for(i=1; ip_result->action) + if((unsigned char)result[i].action>(unsigned char)p_result->action) { p_result=&result[i]; continue; @@ -105,7 +105,7 @@ static Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet) { - int send_log=0; + int send_log=0,identify_flag=0; int ret=0,hit_num=0,ip_policy=0; int state=APP_STATE_DROPME; scan_status_t mid=NULL; @@ -122,12 +122,13 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in switch(a_tcp->pktstate) { case OP_STATE_PENDING: + case OP_STATE_DATA: if((a_tcp->ptcpdetail->pdata==NULL) || (a_tcp->ptcpdetail->datalen<=0) || (a_tcp->dir==DIR_DOUBLE && a_tcp->curdir==DIR_S2C)) { return APP_STATE_GIVEME; } - ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, PROTO_MAX, &mid, all_result, MAX_RESULT_NUM-hit_num); + ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, PROTO_MAX, &mid, all_result+hit_num, MAX_RESULT_NUM-hit_num); if(ret>0) { hit_num+=ret; @@ -142,7 +143,7 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in ret=tsg_scan_shared_policy(g_tsg_maat_feather, a_tcp->ptcpdetail->pdata, a_tcp->ptcpdetail->datalen, - all_result, + all_result+hit_num, MAX_RESULT_NUM-hit_num, &identify_info, &mid, @@ -151,6 +152,7 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in if(ret>0) { hit_num+=ret; + identify_flag=1; } } @@ -164,7 +166,7 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in send_log=1; } - switch(p_result->action) + switch((unsigned char)p_result->action) { case TSG_ACTION_DENY: if(ip_policy==1) @@ -203,13 +205,13 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in if(send_log==1 && p_result->do_log>0) { TLD_handle=TLD_create(thread_seq); - if(priority_label->proto!=PROTO_UNKONWN) + if(identify_flag==1) { schema_field_name=log_field_id2name(g_tsg_log_instance, LOG_COMMON_SCHAME_TYPE); - TLD_append(TLD_handle, schema_field_name, (void *)((priority_label->proto==PROTO_HTTP) ? "HTTP" : "SSL"), TLD_TYPE_LONG); + TLD_append(TLD_handle, schema_field_name, (void *)((identify_info.proto==PROTO_HTTP) ? "HTTP" : "SSL"), TLD_TYPE_STRING); - domain_field_name=log_field_id2name(g_tsg_log_instance, ((priority_label->proto==PROTO_HTTP) ? LOG_HTTP_HOST : LOG_SSL_SNI)); - TLD_append(TLD_handle, domain_field_name, (void *)priority_label->domain, TLD_TYPE_STRING); + domain_field_name=log_field_id2name(g_tsg_log_instance, ((identify_info.proto==PROTO_HTTP) ? LOG_HTTP_HOST : LOG_SSL_SNI)); + TLD_append(TLD_handle, domain_field_name, (void *)identify_info.domain, TLD_TYPE_STRING); } log_msg.a_stream=a_tcp; @@ -219,7 +221,6 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in } } break; - case OP_STATE_DATA: case OP_STATE_CLOSE: default: break; diff --git a/src/tsg_entry.h b/src/tsg_entry.h index f3c9734..d4aa7b1 100644 --- a/src/tsg_entry.h +++ b/src/tsg_entry.h @@ -2,11 +2,7 @@ #define __TSG_ENTRY_H__ #include - -#define FW_ACTION_UNKNOWN 0x00 -#define FW_ACTION_DROP 0x01 -#define FW_ACTION_RESET 0x02 -#define FW_ACTION_BLOCK 0x04 +#include "tsg_rule.h" #define TSG_ACTION_NONE 0x00 #define TSG_ACTION_MONITOR 0x01 @@ -15,10 +11,6 @@ #define TSG_ACTION_MANIPULATE 0x30 #define TSG_ACTION_BYPASS 0x80 - -#define MAX_RESULT_NUM 8 -#define MAX_DOAMIN_LEN 2048 - typedef struct _policy_priority_label { tsg_protocol_t proto; //enum _tsg_protocol (tsg_types.h) @@ -38,8 +30,7 @@ typedef struct _tsg_para void *logger; }g_tsg_para_t; - - extern g_tsg_para_t g_tsg_para; + #endif \ No newline at end of file diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 3d9642a..9685c4f 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -122,6 +122,29 @@ int tsg_rule_init(const char* conffile, void *logger) MESA_load_profile_string_def(conffile, module, "IP_ADDR_TABLE", ip_addr_table, sizeof(ip_addr_table), "TSG_OBJ_IP_ADDR"); MESA_load_profile_string_def(conffile, module, "SUBSCRIBER_ID_TABLE", subscriber_id_table, sizeof(subscriber_id_table), "TSG_OBJ_SUBSCRIBER_ID"); + + g_tsg_para.ip_addr_table_id=Maat_table_register(g_tsg_maat_feather, ip_addr_table); + if(g_tsg_para.ip_addr_table_id<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Maat_table_register %s failed", ip_addr_table); + return -1; + } + + g_tsg_para.subscribe_id_table_id=Maat_table_register(g_tsg_maat_feather, subscriber_id_table); + if(g_tsg_para.subscribe_id_table_id<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "RULE_INIT", "Maat_table_register %s failed", subscriber_id_table); + return -1; + } + + ret=tsg_shared_table_init(conffile, g_tsg_maat_feather, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "RULE_INIT", "tsg_shared_table_init %s failed"); + return -1; + } + + return 0; } static void protocol_identify(char *buff, int buff_len, struct _identify_info *result){ @@ -195,7 +218,7 @@ int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_re return 0; } -int tsg_scan_nesting_addr(Maat_feather_t maat_feather, struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num) +int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num) { struct ipaddr t_addr; struct ipaddr* p_addr = NULL; @@ -292,7 +315,8 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, struct streaminfo *a_stre //return value: -1: failed, 0: not hit, >0: hit count int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, - struct _identify_info *identify_info, scan_status_t *mid, void *logger, int thread_seq){ + struct _identify_info *identify_info, scan_status_t *mid, void *logger, int thread_seq) +{ memset(identify_info, 0, sizeof(*identify_info)); protocol_identify((char*)pkt, pkt_len, identify_info); if(identify_info->proto != PROTO_SSL && identify_info->proto != PROTO_HTTP){ @@ -307,4 +331,31 @@ int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, } return Maat_full_scan_string(g_tsg_maat_feather, tableid, CHARSET_UTF8, identify_info->domain, identify_info->domain_len, result, NULL, result_num, mid, thread_seq); -} \ No newline at end of file +} + + +Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num) +{ + int i=0; + Maat_rule_t *p_result=NULL; + + for(i=0; i< result_num; i++) + { + if(result[i].action==TSG_ACTION_DENY) + { + if(p_result==NULL) + { + p_result=&result[i]; + continue; + } + + if(result[i].action > p_result->action) + { + p_result=&result[i]; + } + } + } + + return p_result; +} + diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 803b0bb..08732f0 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -18,7 +18,7 @@ #include "tsg_send_log.h" #include "tsg_send_log_internal.h" -char TSG_SEND_LOG_VERSION_20191115=0; +char TSG_SEND_LOG_VERSION_20191119=0; tsg_log_instance_t g_tsg_log_instance; const id2field_t tld_type[TLD_TYPE_MAX]={{TLD_TYPE_UNKNOWN, TLD_TYPE_UNKNOWN, "UNKOWN"}, @@ -27,6 +27,8 @@ const id2field_t tld_type[TLD_TYPE_MAX]={{TLD_TYPE_UNKNOWN, TLD_TYPE_UNKNOWN, " {TLD_TYPE_FILE, TLD_TYPE_FILE, "FILE"} }; +extern "C" int MESA_get_dev_ipv4(const char *device, int *ip_add); + int TLD_cancel(TLD_handle_t handle) { int thread_id=0; @@ -60,7 +62,6 @@ int TLD_delete(TLD_handle_t handle, char *key) int TLD_append(TLD_handle_t handle, char *key, void *value, TLD_TYPE type) { - char buff[128]={0}; struct _tld_handle *_handle=(struct _tld_handle *)handle; if(_handle==NULL || key==NULL || value==NULL) @@ -71,8 +72,7 @@ int TLD_append(TLD_handle_t handle, char *key, void *value, TLD_TYPE type) switch(type) { case TLD_TYPE_LONG: - snprintf(buff, sizeof(buff), "%ld", *(long *)value); - cJSON_AddStringToObject(_handle->object, key, buff); + cJSON_AddNumberToObject(_handle->object, key, (long)value); break; case TLD_TYPE_FILE: break; @@ -115,6 +115,7 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl unsigned short tunnel_type=0; char nest_addr_buf[1024]; char *addr_proto=NULL; + unsigned short c_port=0, s_port=0; int tunnel_type_size=sizeof(tunnel_type); struct layer_addr_ipv4 *ipv4=NULL; struct layer_addr_ipv6 *ipv6=NULL; @@ -126,33 +127,35 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl case ADDR_TYPE_IPV4: case __ADDR_TYPE_IP_PAIR_V4: ipv4=a_stream->addr.ipv4; + addr_type=4; + c_port=ntohs(ipv4->source); + s_port=ntohs(ipv4->dest); + inet_ntop(AF_INET, (void *)&ipv4->saddr, client_ip, sizeof(client_ip)); inet_ntop(AF_INET, (void *)&ipv4->daddr, server_ip, sizeof(server_ip)); - - addr_type=4; - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_IP].name, (void *)server_ip, TLD_TYPE_STRING); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_IP].name, (void *)client_ip, TLD_TYPE_STRING); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_PORT].name, (void *)&ipv4->dest, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_PORT].name, (void *)&ipv4->source, TLD_TYPE_LONG); break; case ADDR_TYPE_IPV6: case __ADDR_TYPE_IP_PAIR_V6: ipv6=a_stream->addr.ipv6; + addr_type=6; + c_port=ntohs(ipv6->source); + s_port=ntohs(ipv6->dest); + inet_ntop(AF_INET6, (void *)ipv6->saddr, client_ip, sizeof(client_ip)); inet_ntop(AF_INET6, (void *)ipv6->daddr, server_ip, sizeof(server_ip)); - - addr_type=6; - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_IP].name, (void *)server_ip, TLD_TYPE_STRING); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_IP].name, (void *)client_ip, TLD_TYPE_STRING); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_PORT].name, (void *)&ipv6->dest, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_PORT].name, (void *)&ipv6->source, TLD_TYPE_LONG); break; default: break; } - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)&a_stream->dir, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)&addr_type, TLD_TYPE_LONG); + + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_IP].name, (void *)server_ip, TLD_TYPE_STRING); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_IP].name, (void *)client_ip, TLD_TYPE_STRING); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_PORT].name, (void *)(long)s_port, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_PORT].name, (void *)(long)c_port, TLD_TYPE_LONG); + + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)(long)a_stream->dir, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)(long)addr_type, TLD_TYPE_LONG); addr_proto=(char *)layer_addr_prefix_ntop(a_stream); @@ -174,31 +177,6 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl return 0; } -static unsigned int get_ip_by_eth_name(const char *ifname) -{ - int sockfd; - struct ifreq ifr; - unsigned int ip; - - sockfd = socket(AF_INET, SOCK_DGRAM, 0); - if (-1 == sockfd) { - goto error; - } - - strcpy(ifr.ifr_name,ifname); - if (ioctl(sockfd, SIOCGIFADDR, &ifr) < 0) { - goto error; - } - - ip = ((struct sockaddr_in*)&(ifr.ifr_addr))->sin_addr.s_addr; - close(sockfd); - return ip; - -error: - close(sockfd); - return INADDR_NONE; -} - static int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t *service2topic) { int i=0; @@ -262,7 +240,7 @@ static int load_log_common_field(const char *filename, id2field_t *id2field, id2 tsg_log_instance_t tsg_sendlog_init(const char *conffile) { - int i=0,level=30; + int i=0,ret=0,level=30; char nic_name[32]={0}; char kafka_errstr[1024]={0}; unsigned int local_ip_nr=0; @@ -289,10 +267,10 @@ tsg_log_instance_t tsg_sendlog_init(const char *conffile) } MESA_load_profile_string_def(conffile, "TSG_LOG", "NIC_NAME", nic_name, sizeof(nic_name), "eth0"); - local_ip_nr=get_ip_by_eth_name(nic_name); - if(local_ip_nr==INADDR_NONE) + ret=MESA_get_dev_ipv4(nic_name, (int *)&local_ip_nr); + if(ret<0) { - MESA_handle_runtime_log(_instance->logger,RLOG_LV_FATAL, "GET_LOCAL_IP","get NIC_NAME: %s error.", nic_name); + MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "GET_LOCAL_IP", "MESA_get_dev_ipv4 is error, nic_name: %s", nic_name); return NULL; } inet_ntop(AF_INET,&(local_ip_nr),_instance->local_ip_str,sizeof(_instance->local_ip_str)); @@ -375,14 +353,14 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo if(log_msg->a_stream!=NULL && log_msg->a_stream->ptcpdetail!=NULL) { - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(&log_msg->a_stream->ptcpdetail->createtime), TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(&log_msg->a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->createtime), TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG); } else { cur_time=time(NULL); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)&cur_time, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)&cur_time, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)cur_time, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)cur_time, TLD_TYPE_LONG); } @@ -393,11 +371,11 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo } else { - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_LINK_ID].name, (void *)&vinfo.link_id, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_DIRECTION].name, (void *)&vinfo.link_dir, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_DEVICE_ID].name, (void *)&vinfo.dev_id, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENTRANCE_ID].name, (void *)&vinfo.entrance_id, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENCAPSULATION].name, (void *)&vinfo.encap_type, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_LINK_ID].name, (void *)(long)vinfo.link_id, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_DIRECTION].name, (void *)(long)vinfo.link_dir, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_DEVICE_ID].name, (void *)(long)vinfo.dev_id, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENTRANCE_ID].name, (void *)(long)vinfo.entrance_id, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENCAPSULATION].name, (void *)(long)vinfo.encap_type, TLD_TYPE_LONG); } for(i=0;iresult_num; i++) @@ -422,9 +400,9 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo break; } - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(&log_msg->result[i].config_id), TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(&log_msg->result[i].service_id), TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(&log_msg->result[i].action), TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(long)(log_msg->result[i].config_id), TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG); payload = cJSON_PrintUnformatted(_handle->object); diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index b041aa2..1c1d937 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -4,6 +4,7 @@ #include #include + #define MAX_IPV4_LEN 16 #define MAX_STRING_LEN 32 @@ -22,25 +23,28 @@ enum _DO_LOG typedef enum _tsg_log_field_id { + LOG_COMMON_POLICY_ID=1, LOG_COMMON_SERVICE, - LOG_COMMON_DIRECTION, + LOG_COMMON_ACTION, + LOG_COMMON_START_TIME, + LOG_COMMON_END_TIME, LOG_COMMON_L4_PROTOCOL, LOG_COMMON_ADDRESS_TYPE, - LOG_COMMON_SCHAME_TYPE, - LOG_COMMON_POLICY_ID, - LOG_COMMON_USER_TAG, - LOG_COMMON_ACTION, - LOG_COMMON_USER_REGION, + LOG_COMMON_SERVER_IP, LOG_COMMON_CLIENT_IP, + LOG_COMMON_SERVER_PORT, LOG_COMMON_CLIENT_PORT, + LOG_COMMON_STREAM_DIR, + LOG_COMMON_ADDRESS_LIST, LOG_COMMON_ENTRANCE_ID, LOG_COMMON_DEVICE_ID, LOG_COMMON_LINK_ID, LOG_COMMON_ISP, LOG_COMMON_ENCAPSULATION, - LOG_COMMON_SLED_IP, - LOG_COMMON_SERVER_IP, - LOG_COMMON_SERVER_PORT, + LOG_COMMON_DIRECTION, + LOG_COMMON_SLED_IP, + LOG_COMMON_USER_TAG, + LOG_COMMON_USER_REGION, LOG_COMMON_APP_LABEL, LOG_COMMON_APP_ID, LOG_COMMON_PROTOCOL_ID, @@ -48,14 +52,11 @@ typedef enum _tsg_log_field_id LOG_COMMON_S2C_PKT_NUM, LOG_COMMON_C2S_BYTE_NUM, LOG_COMMON_S2C_BYTE_NUM, - LOG_COMMON_START_TIME, - LOG_COMMON_END_TIME, LOG_COMMON_CON_DURATION_MS, - LOG_COMMON_STREAM_DIR, - LOG_COMMON_ADDRESS_LIST, LOG_COMMON_HAS_DUP_TRAFFIC, LOG_COMMON_STREAM_ERROR, LOG_COMMON_STREAM_TRACE_ID, + LOG_COMMON_SCHAME_TYPE, LOG_HTTP_HOST, LOG_SSL_SNI, LOG_COMMON_MAX diff --git a/src/version.map b/src/version.map index bbd4de5..3e1b353 100644 --- a/src/version.map +++ b/src/version.map @@ -3,11 +3,12 @@ global: extern "C++" { g_*; *TSG_MASTER_INIT*; - *TSG_BWLIST_TCP_ENTRY*; + *TSG_MASTER_TCPALL_ENTRY*; *TSG_MASTER_UNLOAD*; *tsg_send_log*; *tsg_scan_nesting_addr*; *tsg_pull_policy_result*; + *tsg_send_log*; }; local: *; }; From 7912ec54b2669ca7aa84a1885dcca025135c7d0e Mon Sep 17 00:00:00 2001 From: liuxueli Date: Tue, 19 Nov 2019 18:41:31 +0800 Subject: [PATCH 15/32] =?UTF-8?q?=E4=BF=AE=E5=A4=8DBUG=EF=BC=8C=E5=91=BD?= =?UTF-8?q?=E4=B8=AD=E6=8B=A6=E6=88=AA=E7=AD=96=E7=95=A5=E6=97=B6=E6=9C=AA?= =?UTF-8?q?=E5=AF=B9=E7=B1=BB=E5=9E=8B=E8=BF=9B=E8=A1=8C=E8=B5=8B=E5=80=BC?= =?UTF-8?q?=EF=BC=8C=E5=AF=BC=E8=87=B4KNI=E8=8E=B7=E5=8F=96=E4=B8=8D?= =?UTF-8?q?=E5=88=B0=E5=91=BD=E4=B8=AD=E7=BB=93=E6=9E=9C=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E6=80=BB=E6=8E=A7=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= =?UTF-8?q?=20=E6=B7=BB=E5=8A=A0FS2=E7=BB=9F=E8=AE=A1=E6=97=A5=E5=BF=97=20?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=94=99=E8=AF=AF=E6=97=A5=E5=BF=97=20?= =?UTF-8?q?=E6=9E=84=E9=80=A0=E9=80=82=E7=94=A8sapp4.0=E7=9A=84rpm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CMakeLists.txt | 22 +- src/tsg_entry.cpp | 621 +++++++++++++++++++++++-------------------- src/tsg_entry.h | 13 + src/tsg_rule.cpp | 14 + src/tsg_send_log.cpp | 11 +- src/version.map | 4 +- 6 files changed, 389 insertions(+), 296 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e7b90cc..3190ebb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,17 +33,23 @@ set(CMAKE_INSTALL_PREFIX /home/mesasoft/sapp_run) add_subdirectory (src) -set(CONFLIST /plug/platform/conflist_platform.inf) +set(CONFLIST /plug/conflist.inf) +set(MASTER_INF "./plug/platform/tsg_master/tsg_master.inf") +set(PROJECT_LIST /etc/project_list.conf) + file(WRITE ${PROJECT_SOURCE_DIR}/install.sh "#!/bin/sh\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "DST=\${RPM_INSTALL_PREFIX}\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "mkdir -p \${DST}/plug/platform/\r\n") +file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "mkdir -p \${DST}/etc/\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "touch \${DST}${CONFLIST}\r\n") +file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "touch \${DST}${PROJECT_LIST}\r\n") -file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "if [[ -z `grep -rn 'POLICY_PRIORITY' ${DST}/conf/project_list.conf` ]];then\r\n") -file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "echo 'POLICY_PRIORITY struct' >> \${DST}/conf/project_list.conf\r\nfi\r\n") +file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "if [[ -z `grep -rn 'POLICY_PRIORITY' \${DST}${PROJECT_LIST}` ]];then\r\n") +file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "echo 'POLICY_PRIORITY struct' >> \${DST}${PROJECT_LIST}\r\nfi\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "if [[ -z `grep -rn 'tsg_master.inf' \${DST}${CONFLIST}` ]];then\r\n") -file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "echo './plug/platform/tsg_master/tsg_master.inf' >> \${DST}/plug/platform/conflist_platform.inf\r\nfi\r\n") +file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "sed -i '/\\[platform\\]/a\\${MASTER_INF}' \${DST}${CONFLIST}\r\nfi\r\n") +#file(APPEND ${PROJECT_SOURCE_DIR}/install.sh "echo './plug/platform/tsg_master/tsg_master.inf' >> \${DST}\${CONFLIST}\r\nfi\r\n") SET(CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/install.sh") @@ -51,15 +57,17 @@ file(WRITE ${PROJECT_SOURCE_DIR}/uninstall.sh "#!/bin/sh\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "DST=\${RPM_INSTALL_PREFIX}\r\n") file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "mkdir -p \${DST}/plug/platform/\r\n") -file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "sed -i '/tsg_master/d' \${DST}/plug/platform/conflist_platform.inf\r\n") -file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "sed -i '/POLICY_PRIORITY/d' \${DST}/conf/project_list.conf\r\n") +file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "sed -i '/tsg_master.inf/d' \${DST}${CONFLIST}\r\n") +file(APPEND ${PROJECT_SOURCE_DIR}/uninstall.sh "sed -i '/POLICY_PRIORITY/d' \${DST}${PROJECT_LIST}\r\n") SET(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/uninstall.sh") install(FILES bin/main.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf) install(FILES bin/tsg_tableinfo.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf) +install(FILES bin/tsg_log_field.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf) +install(FILES bin/tsg_maat.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf) +install(FILES bin/tsg_maat_ip_deny.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf) install(FILES inc/tsg_send_log.h DESTINATION /opt/MESA/include/tsg) -install(FILES inc/tsg_types.h DESTINATION /opt/MESA/include/tsg) install(FILES inc/tsg_rule.h DESTINATION /opt/MESA/include/tsg) include(Package) diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 439d246..174693d 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -1,284 +1,337 @@ -#include -#include -#include -#include - -#include -#include -#include - -#include "tsg_rule.h" -#include "tsg_entry.h" -#include "tsg_send_log.h" -#include "tsg_send_log_internal.h" - - - -char TSG_MASTER_VERSION_20191119=0; -const char *tsg_conffile="tsgconf/main.conf"; -g_tsg_para_t g_tsg_para; - -static void free_policy_label(int thread_seq, void *project_req_value) -{ - dictator_free(thread_seq, project_req_value); - project_req_value=NULL; -} - -#if 0 -static int is_ip_policy(Maat_rule_t *p_result, char *protocol, int len, int thread_seq) -{ - int ret=0; - cJSON *item=NULL; - char *service_defined=NULL; - cJSON *user_define_object=NULL; - - if(p_result->serv_def_len>MAX_SERVICE_DEFINE_LEN) - { - service_defined=dictator_malloc(thread_seq, p_result->serv_def_len+1); - ret=Maat_read_rule(g_tsg_maat_feather, p_result, MAAT_RULE_SERV_DEFINE, service_defined, p_result->serv_def_len+1); - assert(ret==p_result->serv_def_len+1); - - user_define_object=cJSON_Parse(service_defined); - } - else - { - user_define_object=cJSON_Parse(p_result->service_defined); - } - - if(user_define_object!=NULL) - { - item=cJSON_GetObjectItem(user_define_object, "protocol"); - if(item!=NULL && item->valuestring!=NULL) - { - memcpy(protocol, item->valuestring, (len>strlen(item->valuestring)) ? strlen(item->valuestring): len); - } - - item=cJSON_GetObjectItem(user_define_object, "method"); - if((item==NULL) || ((strncasecmp(item->valuestring, "http", strlen(item->valuestring)))!=0 && (strncasecmp(item->valuestring, "ssl", strlen(item->valuestring)))!=0)) - { - ret=1; - } - - cJSON_Delete(user_define_object); - user_define_object=NULL; - } - - if(service_defined!=NULL) - { - dictator_free(thread_seq, service_defined); - service_defined=NULL; - } - - return ret; -} -#endif -static Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result_num) -{ - int i=0; - Maat_rule_t *p_result=NULL; - if(result==NULL || result_num<=0) - { - return NULL; - } - - p_result=&result[0]; - - for(i=1; i(unsigned char)p_result->action) - { - p_result=&result[i]; - continue; - } - - if(result[i].action==p_result->action) - { - if(result[i].config_idconfig_id) - { - p_result=&result[i]; - } - } - } - - return p_result; -} - -extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet) -{ - int send_log=0,identify_flag=0; - int ret=0,hit_num=0,ip_policy=0; - int state=APP_STATE_DROPME; - scan_status_t mid=NULL; - char *domain_field_name=NULL; - char *schema_field_name=NULL; - Maat_rule_t *p_result=NULL; - Maat_rule_t *q_result=NULL; - tsg_log_t log_msg; - TLD_handle_t TLD_handle=NULL; - struct _identify_info identify_info; - Maat_rule_t all_result[MAX_RESULT_NUM]; - policy_priority_label_t *priority_label=NULL; - - switch(a_tcp->pktstate) - { - case OP_STATE_PENDING: - case OP_STATE_DATA: - if((a_tcp->ptcpdetail->pdata==NULL) || (a_tcp->ptcpdetail->datalen<=0) || (a_tcp->dir==DIR_DOUBLE && a_tcp->curdir==DIR_S2C)) - { - return APP_STATE_GIVEME; - } - - ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, PROTO_MAX, &mid, all_result+hit_num, MAX_RESULT_NUM-hit_num); - if(ret>0) - { - hit_num+=ret; - q_result=tsg_policy_decision_criteria(all_result, hit_num); - } - - - if(a_tcp->curdir==DIR_C2S) - { - memset(&identify_info, 0, sizeof(identify_info)); - - ret=tsg_scan_shared_policy(g_tsg_maat_feather, - a_tcp->ptcpdetail->pdata, - a_tcp->ptcpdetail->datalen, - all_result+hit_num, - MAX_RESULT_NUM-hit_num, - &identify_info, - &mid, - g_tsg_para.logger, - thread_seq); - if(ret>0) - { - hit_num+=ret; - identify_flag=1; - } - } - - p_result=tsg_policy_decision_criteria(all_result, hit_num); - - if(p_result!=NULL) - { - if(q_result!=NULL && (p_result==q_result)) - { - ip_policy=1; - send_log=1; - } - - switch((unsigned char)p_result->action) - { - case TSG_ACTION_DENY: - if(ip_policy==1) - { - MESA_kill_tcp(a_tcp, a_packet); - state|=APP_STATE_DROPPKT; - } - break; - case TSG_ACTION_MONITOR: - break; - case TSG_ACTION_BYPASS: - send_log=1; - state|=APP_STATE_DROPPKT; //TODO - break; - case TSG_ACTION_INTERCEPT: - priority_label=(policy_priority_label_t *)dictator_malloc(thread_seq, sizeof(policy_priority_label_t)); - - priority_label->result_num=1; - priority_label->domain_len=identify_info.domain_len; - memcpy(priority_label->domain, identify_info.domain, identify_info.domain_len); - memcpy(priority_label->result, p_result, sizeof(struct Maat_rule_t)); - - ret=project_req_add_struct(a_tcp, g_tsg_para.priority_project_id, (void *)priority_label); - if(ret<0) - { - free_policy_label(thread_seq, (void *)priority_label); - MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_ADD", "Add policy_priority_label failed ..."); - } - break; - case TSG_ACTION_NONE: - default: - assert(0); - break; - } - - if(send_log==1 && p_result->do_log>0) - { - TLD_handle=TLD_create(thread_seq); - if(identify_flag==1) - { - schema_field_name=log_field_id2name(g_tsg_log_instance, LOG_COMMON_SCHAME_TYPE); - TLD_append(TLD_handle, schema_field_name, (void *)((identify_info.proto==PROTO_HTTP) ? "HTTP" : "SSL"), TLD_TYPE_STRING); - - domain_field_name=log_field_id2name(g_tsg_log_instance, ((identify_info.proto==PROTO_HTTP) ? LOG_HTTP_HOST : LOG_SSL_SNI)); - TLD_append(TLD_handle, domain_field_name, (void *)identify_info.domain, TLD_TYPE_STRING); - } - - log_msg.a_stream=a_tcp; - log_msg.result=p_result; - log_msg.result_num=1; - tsg_send_log(g_tsg_log_instance, TLD_handle, &log_msg, thread_seq); - } - } - break; - case OP_STATE_CLOSE: - default: - break; - } - - return state; -} - - - -extern "C" int TSG_MASTER_INIT() -{ - int ret=0,level=30; - char log_path[128]={0}; - char label_buff[128]={0}; - - memset(&g_tsg_para, 0, sizeof(g_tsg_para)); - - MESA_load_profile_int_def(tsg_conffile, "SYSTEM","LOG_LEVEL", &level, 30); - MESA_load_profile_string_def(tsg_conffile, "SYSTEM","LOG_PATH", log_path, sizeof(log_path), NULL); - - g_tsg_para.logger=MESA_create_runtime_log_handle(log_path, level); - if(g_tsg_para.logger==NULL) - { - printf("MESA_create_runtime_log_handle failed ...\n"); - return -1; - } - - - MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "POLICY_PRIORITY_LABEL", label_buff, sizeof(label_buff), "POLICY_PRIORITY"); - g_tsg_para.priority_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_STRUCT, free_policy_label); - if(g_tsg_para.priority_project_id<0) - { - MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_REGISTER", "Register %s failed ...", label_buff); - return -1; - } - - ret=tsg_rule_init(tsg_conffile, g_tsg_para.logger); - if(ret<0) - { - MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_MAAT", "tsg_rule_init failed ..."); - return -1; - } - - g_tsg_log_instance=tsg_sendlog_init(tsg_conffile); - if(g_tsg_log_instance==NULL) - { - MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_SENDLOG", "tsg_sendlog_init failed ..."); - return -1; - } - - return 0; -} - - - -extern "C" int TSG_MASTER_UNLOAD() -{ - return 0; -} - +#include +#include +#include +#include + +#include +#include +#include + +#include "tsg_rule.h" +#include "tsg_entry.h" +#include "tsg_send_log.h" +#include "tsg_send_log_internal.h" + + + +char TSG_MASTER_VERSION_20191119=0; +const char *tsg_conffile="tsgconf/main.conf"; +g_tsg_para_t g_tsg_para; + +id2field_t g_tsg_fs2_field[TSG_FS2_MAX]={{TLD_TYPE_UNKNOWN, TSG_FS2_LINKS, "links"}, + {TLD_TYPE_UNKNOWN, TSG_FS2_BYPASS, "bypass"}, + {TLD_TYPE_UNKNOWN, TSG_FS2_HIT_ADDR, "hit_addr"}, + {TLD_TYPE_UNKNOWN, TSG_FS2_HIT_SHARE, "hit_share"}, + {TLD_TYPE_UNKNOWN, TSG_FS2_INTERCEPT, "intercept"}, + {TLD_TYPE_UNKNOWN, TSG_FS2_LOG, "log"} + }; +static void free_policy_label(int thread_seq, void *project_req_value) +{ + dictator_free(thread_seq, project_req_value); + project_req_value=NULL; +} + +#if 0 +static int is_ip_policy(Maat_rule_t *p_result, char *protocol, int len, int thread_seq) +{ + int ret=0; + cJSON *item=NULL; + char *service_defined=NULL; + cJSON *user_define_object=NULL; + + if(p_result->serv_def_len>MAX_SERVICE_DEFINE_LEN) + { + service_defined=dictator_malloc(thread_seq, p_result->serv_def_len+1); + ret=Maat_read_rule(g_tsg_maat_feather, p_result, MAAT_RULE_SERV_DEFINE, service_defined, p_result->serv_def_len+1); + assert(ret==p_result->serv_def_len+1); + + user_define_object=cJSON_Parse(service_defined); + } + else + { + user_define_object=cJSON_Parse(p_result->service_defined); + } + + if(user_define_object!=NULL) + { + item=cJSON_GetObjectItem(user_define_object, "protocol"); + if(item!=NULL && item->valuestring!=NULL) + { + memcpy(protocol, item->valuestring, (len>strlen(item->valuestring)) ? strlen(item->valuestring): len); + } + + item=cJSON_GetObjectItem(user_define_object, "method"); + if((item==NULL) || ((strncasecmp(item->valuestring, "http", strlen(item->valuestring)))!=0 && (strncasecmp(item->valuestring, "ssl", strlen(item->valuestring)))!=0)) + { + ret=1; + } + + cJSON_Delete(user_define_object); + user_define_object=NULL; + } + + if(service_defined!=NULL) + { + dictator_free(thread_seq, service_defined); + service_defined=NULL; + } + + return ret; +} +#endif +static Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result_num) +{ + int i=0; + Maat_rule_t *p_result=NULL; + if(result==NULL || result_num<=0) + { + return NULL; + } + + p_result=&result[0]; + + for(i=1; i(unsigned char)p_result->action) + { + p_result=&result[i]; + continue; + } + + if(result[i].action==p_result->action) + { + if(result[i].config_idconfig_id) + { + p_result=&result[i]; + } + } + } + + return p_result; +} + +extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet) +{ + int send_log=0,identify_flag=0; + int ret=0,hit_num=0,ip_policy=0; + int state=APP_STATE_DROPME; + scan_status_t mid=NULL; + char *domain_field_name=NULL; + char *schema_field_name=NULL; + Maat_rule_t *p_result=NULL; + Maat_rule_t *q_result=NULL; + tsg_log_t log_msg; + TLD_handle_t TLD_handle=NULL; + struct _identify_info identify_info; + Maat_rule_t all_result[MAX_RESULT_NUM]; + policy_priority_label_t *priority_label=NULL; + + switch(a_tcp->pktstate) + { + case OP_STATE_PENDING: + FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LINKS], 0, FS_OP_ADD, 1); + case OP_STATE_DATA: + if((a_tcp->ptcpdetail->pdata==NULL) || (a_tcp->ptcpdetail->datalen<=0) || (a_tcp->dir==DIR_DOUBLE && a_tcp->curdir==DIR_S2C)) + { + return APP_STATE_GIVEME; + } + + ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, PROTO_MAX, &mid, all_result+hit_num, MAX_RESULT_NUM-hit_num); + if(ret>0) + { + hit_num+=ret; + q_result=tsg_policy_decision_criteria(all_result, hit_num); + FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1); + } + + + if(a_tcp->curdir==DIR_C2S) + { + memset(&identify_info, 0, sizeof(identify_info)); + + ret=tsg_scan_shared_policy(g_tsg_maat_feather, + a_tcp->ptcpdetail->pdata, + a_tcp->ptcpdetail->datalen, + all_result+hit_num, + MAX_RESULT_NUM-hit_num, + &identify_info, + &mid, + g_tsg_para.logger, + thread_seq); + if(ret>0) + { + hit_num+=ret; + identify_flag=1; + FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1); + } + } + + p_result=tsg_policy_decision_criteria(all_result, hit_num); + + if(p_result!=NULL) + { + if(q_result!=NULL && (p_result==q_result)) + { + ip_policy=1; + send_log=1; + } + + switch((unsigned char)p_result->action) + { + case TSG_ACTION_DENY: + if(ip_policy==1) + { + MESA_kill_tcp(a_tcp, a_packet); + state|=APP_STATE_DROPPKT; + } + break; + case TSG_ACTION_MONITOR: + break; + case TSG_ACTION_BYPASS: + send_log=1; + state|=APP_STATE_DROPPKT; //TODO + FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_BYPASS], 0, FS_OP_ADD, 1); + break; + case TSG_ACTION_INTERCEPT: + send_log=0; + FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_INTERCEPT], 0, FS_OP_ADD, 1); + priority_label=(policy_priority_label_t *)dictator_malloc(thread_seq, sizeof(policy_priority_label_t)); + + priority_label->result_num=1; + priority_label->result_type=PULL_KNI_RESULT; + priority_label->domain_len=identify_info.domain_len; + memcpy(priority_label->domain, identify_info.domain, identify_info.domain_len); + memcpy(priority_label->result, p_result, sizeof(struct Maat_rule_t)); + + ret=project_req_add_struct(a_tcp, g_tsg_para.priority_project_id, (void *)priority_label); + if(ret<0) + { + free_policy_label(thread_seq, (void *)priority_label); + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_ADD", "Add policy_priority_label failed ..."); + } + + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "INTERCEPT", "Hit intercept policy, policy_id: %d action: %d addr: %s", + priority_label->result[0].config_id, priority_label->result[0].action, printaddr(&a_tcp->addr, thread_seq)); + break; + case TSG_ACTION_NONE: + default: + assert(0); + break; + } + + if(send_log==1 && p_result->do_log>0) + { + TLD_handle=TLD_create(thread_seq); + if(identify_flag==1) + { + schema_field_name=log_field_id2name(g_tsg_log_instance, LOG_COMMON_SCHAME_TYPE); + TLD_append(TLD_handle, schema_field_name, (void *)((identify_info.proto==PROTO_HTTP) ? "HTTP" : "SSL"), TLD_TYPE_STRING); + + domain_field_name=log_field_id2name(g_tsg_log_instance, ((identify_info.proto==PROTO_HTTP) ? LOG_HTTP_HOST : LOG_SSL_SNI)); + TLD_append(TLD_handle, domain_field_name, (void *)identify_info.domain, TLD_TYPE_STRING); + } + + log_msg.a_stream=a_tcp; + log_msg.result=p_result; + log_msg.result_num=1; + tsg_send_log(g_tsg_log_instance, TLD_handle, &log_msg, thread_seq); + } + } + break; + case OP_STATE_CLOSE: + default: + break; + } + + return state; +} + + + +extern "C" int TSG_MASTER_INIT() +{ + int i=0,ret=0,cycle=0; + int value=0,level=30; + unsigned short fs_server_port=0; + char app_name[MAX_STRING_LEN]={0}; + char log_path[MAX_STRING_LEN*4]={0}; + char label_buff[MAX_STRING_LEN*4]={0}; + char fs_server_ip[MAX_IPV4_LEN]={0}; + char fs_output_path[MAX_STRING_LEN*4]={0}; + + memset(&g_tsg_para, 0, sizeof(g_tsg_para)); + + MESA_load_profile_int_def(tsg_conffile, "SYSTEM","LOG_LEVEL", &level, 30); + MESA_load_profile_string_def(tsg_conffile, "SYSTEM","LOG_PATH", log_path, sizeof(log_path), NULL); + + g_tsg_para.logger=MESA_create_runtime_log_handle(log_path, level); + if(g_tsg_para.logger==NULL) + { + printf("MESA_create_runtime_log_handle failed ...\n"); + return -1; + } + + + MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "POLICY_PRIORITY_LABEL", label_buff, sizeof(label_buff), "POLICY_PRIORITY"); + g_tsg_para.priority_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_STRUCT, free_policy_label); + if(g_tsg_para.priority_project_id<0) + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_REGISTER", "Register %s failed ...", label_buff); + return -1; + } + + ret=tsg_rule_init(tsg_conffile, g_tsg_para.logger); + if(ret<0) + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_MAAT", "tsg_rule_init failed ..."); + return -1; + } + + g_tsg_log_instance=tsg_sendlog_init(tsg_conffile); + if(g_tsg_log_instance==NULL) + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_SENDLOG", "tsg_sendlog_init failed ..."); + return -1; + } + + MESA_load_profile_int_def(tsg_conffile, "FIELD_STAT", "CYCLE", &cycle, 30); + MESA_load_profile_short_nodef(tsg_conffile, "FIELD_STAT","TELEGRAF_PORT", (short *)&(fs_server_port)); + MESA_load_profile_string_nodef(tsg_conffile,"FIELD_STAT","TELEGRAF_IP",fs_server_ip, sizeof(fs_server_ip)); + MESA_load_profile_string_def(tsg_conffile,"FIELD_STAT","OUTPUT_PATH",fs_output_path, sizeof(fs_output_path), "tsg_stat.log"); + MESA_load_profile_string_def(tsg_conffile,"FIELD_STAT","APP_NAME", app_name, sizeof(app_name), "tsg_master"); + + g_tsg_para.fs2_handle=FS_create_handle(); + + value=1;//Rewrite + FS_set_para(g_tsg_para.fs2_handle, PRINT_MODE, &value, sizeof(value)); + value=1;//Do not create stat thread + FS_set_para(g_tsg_para.fs2_handle, CREATE_THREAD, &value, sizeof(value)); + + FS_set_para(g_tsg_para.fs2_handle, STAT_CYCLE, &cycle, sizeof(cycle)); + FS_set_para(g_tsg_para.fs2_handle, APP_NAME, app_name, strlen(app_name)+1); + FS_set_para(g_tsg_para.fs2_handle, OUTPUT_DEVICE, fs_output_path, strlen(fs_output_path)+1); + + if(fs_server_port > 0 && strlen(fs_server_ip) > 0) + { + FS_set_para(g_tsg_para.fs2_handle, STATS_SERVER_IP,fs_server_ip, strlen(fs_server_ip)+1); + FS_set_para(g_tsg_para.fs2_handle, STATS_SERVER_PORT,&(fs_server_port), sizeof(fs_server_port)); + } + + for(i=0; i +#include #include "tsg_rule.h" #define TSG_ACTION_NONE 0x00 @@ -11,6 +12,16 @@ #define TSG_ACTION_MANIPULATE 0x30 #define TSG_ACTION_BYPASS 0x80 +enum TSG_FS2_TYPE{ + TSG_FS2_LINKS=0, + TSG_FS2_BYPASS, + TSG_FS2_HIT_ADDR, + TSG_FS2_HIT_SHARE, + TSG_FS2_INTERCEPT, + TSG_FS2_LOG, + TSG_FS2_MAX +}; + typedef struct _policy_priority_label { tsg_protocol_t proto; //enum _tsg_protocol (tsg_types.h) @@ -27,7 +38,9 @@ typedef struct _tsg_para int ip_addr_table_id; int subscribe_id_table_id; int priority_project_id; + int fs2_field_id[TSG_FS2_MAX]; void *logger; + screen_stat_handle_t fs2_handle; }g_tsg_para_t; extern g_tsg_para_t g_tsg_para; diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 9685c4f..f2dc007 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -213,6 +213,20 @@ int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_re return num; } + else + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "PULL_RESULT", + "pull policy failed, result_type: %s, addr: %s", + (label->result_type==PULL_KNI_RESULT) ? "KNI" : "FW", + printaddr(&a_stream->addr, a_stream->threadnum)); + } + } + else + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PULL_RESULT", + "pull policy failed, result_type=%d, addr: %s", + (label==NULL) ? -1 : label->result_type, + printaddr(&a_stream->addr, a_stream->threadnum)); } return 0; diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 08732f0..20744cd 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -15,6 +15,7 @@ #include #include +#include "tsg_entry.h" #include "tsg_send_log.h" #include "tsg_send_log_internal.h" @@ -36,9 +37,11 @@ int TLD_cancel(TLD_handle_t handle) if(handle!=NULL) { _handle=(struct _tld_handle *)handle; - cJSON_Delete(_handle->object); - _handle->object=NULL; - + if(_handle->object!=NULL) + { + cJSON_Delete(_handle->object); + _handle->object=NULL; + } thread_id=_handle->thread_id; dictator_free(thread_id, handle); @@ -426,6 +429,8 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name); TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVICE].name); TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ACTION].name); + + FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LOG], 0, FS_OP_ADD, 1); } diff --git a/src/version.map b/src/version.map index 3e1b353..5b10f26 100644 --- a/src/version.map +++ b/src/version.map @@ -5,10 +5,10 @@ global: *TSG_MASTER_INIT*; *TSG_MASTER_TCPALL_ENTRY*; *TSG_MASTER_UNLOAD*; - *tsg_send_log*; *tsg_scan_nesting_addr*; *tsg_pull_policy_result*; - *tsg_send_log*; + *tsg_*; + *TLD_*; }; local: *; }; From 28da97b53b7777a3a024a0df191fe3239c0e981a Mon Sep 17 00:00:00 2001 From: liuxueli Date: Wed, 20 Nov 2019 14:45:41 +0800 Subject: [PATCH 16/32] =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=B9=B3=E5=8F=B0?= =?UTF-8?q?=E6=8F=90=E4=BE=9B=E7=9A=84APP=5FSTATE=5FKILL=5FOTHER=20?= =?UTF-8?q?=E4=BF=AE=E5=A4=8DBUG=EF=BC=8C=E5=A4=A9=E5=89=91=E6=B5=81?= =?UTF-8?q?=E6=A0=87=E7=AD=BE=E6=97=B6=E6=9C=AA=E5=AF=B9=E5=8D=8F=E8=AE=AE?= =?UTF-8?q?=E8=BF=9B=E8=A1=8C=E8=B5=8B=E5=80=BC=EF=BC=8C=E5=AF=BC=E8=87=B4?= =?UTF-8?q?tfe=E9=A2=81=E5=8F=91=E8=AF=81=E4=B9=A6=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_entry.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 174693d..b985fd5 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -14,7 +14,7 @@ -char TSG_MASTER_VERSION_20191119=0; +char TSG_MASTER_VERSION_20191120=0; const char *tsg_conffile="tsgconf/main.conf"; g_tsg_para_t g_tsg_para; @@ -182,14 +182,14 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in if(ip_policy==1) { MESA_kill_tcp(a_tcp, a_packet); - state|=APP_STATE_DROPPKT; + state|=APP_STATE_DROPPKT|APP_STATE_KILL_OTHER; } break; case TSG_ACTION_MONITOR: break; case TSG_ACTION_BYPASS: send_log=1; - state|=APP_STATE_DROPPKT; //TODO + state|=APP_STATE_KILL_OTHER; //TODO FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_BYPASS], 0, FS_OP_ADD, 1); break; case TSG_ACTION_INTERCEPT: @@ -199,6 +199,7 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in priority_label->result_num=1; priority_label->result_type=PULL_KNI_RESULT; + priority_label->proto=identify_info.proto; priority_label->domain_len=identify_info.domain_len; memcpy(priority_label->domain, identify_info.domain, identify_info.domain_len); memcpy(priority_label->result, p_result, sizeof(struct Maat_rule_t)); From 291c05ea79fe3eda8fdef68079808e321cf2d1f5 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Wed, 20 Nov 2019 16:40:19 +0800 Subject: [PATCH 17/32] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E5=85=A8=E5=B1=80=E6=B5=81ID=E7=9A=84=E6=8E=A5=E5=8F=A3=20?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=80=BB=E6=8E=A7=E6=B3=A8=E5=86=8C=E4=B8=BA?= =?UTF-8?q?TCP=E5=85=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/tsg_send_log.h | 2 ++ src/tsg_entry.cpp | 45 +++++++++++++++++++------------------------- src/tsg_entry.h | 1 + src/tsg_send_log.cpp | 23 +++++++++++++++++++++- 4 files changed, 44 insertions(+), 27 deletions(-) diff --git a/inc/tsg_send_log.h b/inc/tsg_send_log.h index 0115f46..b3a4b5c 100644 --- a/inc/tsg_send_log.h +++ b/inc/tsg_send_log.h @@ -32,5 +32,7 @@ int TLD_cancel(TLD_handle_t handle); int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *log_msg, int thread_id); +unsigned long long tsg_get_stream_id(struct streaminfo *a_stream); + #endif diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index b985fd5..51c6592 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -110,7 +110,7 @@ static Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result return p_result; } -extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet) +extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet) { int send_log=0,identify_flag=0; int ret=0,hit_num=0,ip_policy=0; @@ -126,15 +126,10 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in Maat_rule_t all_result[MAX_RESULT_NUM]; policy_priority_label_t *priority_label=NULL; - switch(a_tcp->pktstate) + switch(a_tcp->opstate) { case OP_STATE_PENDING: FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LINKS], 0, FS_OP_ADD, 1); - case OP_STATE_DATA: - if((a_tcp->ptcpdetail->pdata==NULL) || (a_tcp->ptcpdetail->datalen<=0) || (a_tcp->dir==DIR_DOUBLE && a_tcp->curdir==DIR_S2C)) - { - return APP_STATE_GIVEME; - } ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, PROTO_MAX, &mid, all_result+hit_num, MAX_RESULT_NUM-hit_num); if(ret>0) @@ -144,26 +139,22 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1); } - - if(a_tcp->curdir==DIR_C2S) + memset(&identify_info, 0, sizeof(identify_info)); + + ret=tsg_scan_shared_policy(g_tsg_maat_feather, + a_tcp->ptcpdetail->pdata, + a_tcp->ptcpdetail->datalen, + all_result+hit_num, + MAX_RESULT_NUM-hit_num, + &identify_info, + &mid, + g_tsg_para.logger, + thread_seq); + if(ret>0) { - memset(&identify_info, 0, sizeof(identify_info)); - - ret=tsg_scan_shared_policy(g_tsg_maat_feather, - a_tcp->ptcpdetail->pdata, - a_tcp->ptcpdetail->datalen, - all_result+hit_num, - MAX_RESULT_NUM-hit_num, - &identify_info, - &mid, - g_tsg_para.logger, - thread_seq); - if(ret>0) - { - hit_num+=ret; - identify_flag=1; - FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1); - } + hit_num+=ret; + identify_flag=1; + FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1); } p_result=tsg_policy_decision_criteria(all_result, hit_num); @@ -239,6 +230,7 @@ extern "C" char TSG_MASTER_TCPALL_ENTRY(struct streaminfo *a_tcp, void **pme, in } } break; + case OP_STATE_DATA: case OP_STATE_CLOSE: default: break; @@ -272,6 +264,7 @@ extern "C" int TSG_MASTER_INIT() return -1; } + MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "DEVICE_ID", &g_tsg_para.device_id, 0); MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "POLICY_PRIORITY_LABEL", label_buff, sizeof(label_buff), "POLICY_PRIORITY"); g_tsg_para.priority_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_STRUCT, free_policy_label); diff --git a/src/tsg_entry.h b/src/tsg_entry.h index fdd873b..5f5b5c1 100644 --- a/src/tsg_entry.h +++ b/src/tsg_entry.h @@ -35,6 +35,7 @@ typedef struct _policy_priority_label typedef struct _tsg_para { + int device_id; int ip_addr_table_id; int subscribe_id_table_id; int priority_project_id; diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 20744cd..4f8cbe3 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -19,7 +19,7 @@ #include "tsg_send_log.h" #include "tsg_send_log_internal.h" -char TSG_SEND_LOG_VERSION_20191119=0; +char TSG_SEND_LOG_VERSION_20191120=0; tsg_log_instance_t g_tsg_log_instance; const id2field_t tld_type[TLD_TYPE_MAX]={{TLD_TYPE_UNKNOWN, TLD_TYPE_UNKNOWN, "UNKOWN"}, @@ -30,6 +30,22 @@ const id2field_t tld_type[TLD_TYPE_MAX]={{TLD_TYPE_UNKNOWN, TLD_TYPE_UNKNOWN, " extern "C" int MESA_get_dev_ipv4(const char *device, int *ip_add); + +unsigned long long tsg_get_stream_id(struct streaminfo * a_stream) +{ + int ret=0; + int device_id_size=sizeof(unsigned long long); + unsigned long long device_id=(unsigned long long)g_tsg_para.device_id; + + ret=MESA_get_stream_opt(a_stream, MSO_GLOBAL_STREAM_ID, (void *)device_id, &device_id_size); + if(ret==0) + { + return device_id; + } + + return -1; +} + int TLD_cancel(TLD_handle_t handle) { int thread_id=0; @@ -159,6 +175,11 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)(long)a_stream->dir, TLD_TYPE_LONG); TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)(long)addr_type, TLD_TYPE_LONG); + + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->clientpktnum, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->clientbytes, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->serverpktnum, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->serverbytes, TLD_TYPE_LONG); addr_proto=(char *)layer_addr_prefix_ntop(a_stream); From 5c04ba23aa312b174e6707d2cf62b4b7e48f7a03 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Wed, 20 Nov 2019 18:00:35 +0800 Subject: [PATCH 18/32] =?UTF-8?q?SSL=5FSNI=E5=92=8CHTTP=5FHOST=E7=9A=84den?= =?UTF-8?q?y=E5=8A=A8=E4=BD=9C=E7=94=B1=E6=80=BB=E6=8E=A7=E8=B4=9F?= =?UTF-8?q?=E8=B4=A3=E5=A4=84=E7=90=86=20=E4=BF=AE=E5=A4=8D=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E5=85=A8=E5=B1=80=E6=B5=81ID=E6=97=B6=E7=9A=84?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_entry.cpp | 23 +++++++++-------------- src/tsg_send_log.cpp | 7 +++++-- src/version.map | 2 +- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 51c6592..37a3030 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -113,7 +113,7 @@ static Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet) { int send_log=0,identify_flag=0; - int ret=0,hit_num=0,ip_policy=0; + int ret=0,hit_num=0; int state=APP_STATE_DROPME; scan_status_t mid=NULL; char *domain_field_name=NULL; @@ -160,23 +160,19 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t p_result=tsg_policy_decision_criteria(all_result, hit_num); if(p_result!=NULL) - { - if(q_result!=NULL && (p_result==q_result)) - { - ip_policy=1; - send_log=1; - } - + { switch((unsigned char)p_result->action) { case TSG_ACTION_DENY: - if(ip_policy==1) - { - MESA_kill_tcp(a_tcp, a_packet); - state|=APP_STATE_DROPPKT|APP_STATE_KILL_OTHER; - } + send_log=1; + MESA_kill_tcp(a_tcp, a_packet); + state|=APP_STATE_DROPPKT|APP_STATE_KILL_OTHER; break; case TSG_ACTION_MONITOR: + if(q_result!=NULL && (p_result==q_result)) + { + send_log=1; + } break; case TSG_ACTION_BYPASS: send_log=1; @@ -184,7 +180,6 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_BYPASS], 0, FS_OP_ADD, 1); break; case TSG_ACTION_INTERCEPT: - send_log=0; FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_INTERCEPT], 0, FS_OP_ADD, 1); priority_label=(policy_priority_label_t *)dictator_malloc(thread_seq, sizeof(policy_priority_label_t)); diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 4f8cbe3..7a83299 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -37,7 +37,7 @@ unsigned long long tsg_get_stream_id(struct streaminfo * a_stream) int device_id_size=sizeof(unsigned long long); unsigned long long device_id=(unsigned long long)g_tsg_para.device_id; - ret=MESA_get_stream_opt(a_stream, MSO_GLOBAL_STREAM_ID, (void *)device_id, &device_id_size); + ret=MESA_get_stream_opt(a_stream, MSO_GLOBAL_STREAM_ID, (void *)&device_id, &device_id_size); if(ret==0) { return device_id; @@ -134,6 +134,7 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl unsigned short tunnel_type=0; char nest_addr_buf[1024]; char *addr_proto=NULL; + unsigned long long stream_id=0; unsigned short c_port=0, s_port=0; int tunnel_type_size=sizeof(tunnel_type); struct layer_addr_ipv4 *ipv4=NULL; @@ -181,7 +182,9 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->serverpktnum, TLD_TYPE_LONG); TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->serverbytes, TLD_TYPE_LONG); - + stream_id=tsg_get_stream_id(a_stream); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_STREAM_TRACE_ID].name, (void *)(long)stream_id, TLD_TYPE_LONG); + addr_proto=(char *)layer_addr_prefix_ntop(a_stream); TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_L4_PROTOCOL].name, (void *)addr_proto, TLD_TYPE_STRING); diff --git a/src/version.map b/src/version.map index 5b10f26..ceb1542 100644 --- a/src/version.map +++ b/src/version.map @@ -3,7 +3,7 @@ global: extern "C++" { g_*; *TSG_MASTER_INIT*; - *TSG_MASTER_TCPALL_ENTRY*; + *TSG_MASTER_TCP_ENTRY*; *TSG_MASTER_UNLOAD*; *tsg_scan_nesting_addr*; *tsg_pull_policy_result*; From 0a11db6d2eee3e187a4242abce9f2e66b02b30ce Mon Sep 17 00:00:00 2001 From: liuxueli Date: Wed, 20 Nov 2019 18:02:54 +0800 Subject: [PATCH 19/32] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=80=BB=E6=8E=A7?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/main.conf | 39 ++++++++++++++++++ bin/tsg_log_field.conf | 41 +++++++++++++++++++ bin/tsg_maat.json | 84 +++++++++++++++++++++++++++++++++++++++ bin/tsg_maat_ip_deny.json | 40 +++++++++++++++++++ bin/tsg_master.inf | 9 +++++ bin/tsg_tableinfo.conf | 38 ++++++++++++++++++ 6 files changed, 251 insertions(+) create mode 100644 bin/main.conf create mode 100644 bin/tsg_log_field.conf create mode 100644 bin/tsg_maat.json create mode 100644 bin/tsg_maat_ip_deny.json create mode 100644 bin/tsg_master.inf create mode 100644 bin/tsg_tableinfo.conf diff --git a/bin/main.conf b/bin/main.conf new file mode 100644 index 0000000..d79582c --- /dev/null +++ b/bin/main.conf @@ -0,0 +1,39 @@ +[MAAT] +MAAT_MODE=1 +#EFFECTIVE_FLAG= +STAT_SWITCH=1 +PERF_SWITCH=1 +TABLE_INFO=tsgconf/tsg_tableinfo.conf +STAT_FILE=tsg_maat.status +EFFECT_INTERVAL_S=1 +REDIS_IP=127.0.0.1 +REDIS_PORT_NUM=10 +REDIS_PORT=6380 +REDIS_INDEX=2 +JSON_CFG_FILE=tsgconf/tsg_maat.json +INC_CFG_DIR=tsgrule/inc/index/ +FULL_CFG_DIR=tsgrule/full/index/ + +IP_ADDR_TABLE=TSG_OBJ_IP_ADDR +SUBSCRIBER_ID_TABLE=TSG_OBJ_SUBSCRIBER_ID + +[TSG_LOG] +MODE=1 +NIC_NAME=eth1 +MAX_SERVICE=0 +LOG_LEVEL=10 +LOG_PATH=./tsglog/tsglog +BROKER_LIST=127.0.0.1:9092 +COMMON_FIELD_FILE=tsgconf/tsg_log_field.conf + +[FIELD_STAT] +CYCLE=3 +TELEGRAF_PORT=8125 +TELEGRAF_IP=127.0.0.1 +OUTPUT_PATH=./tsg_stat.log +APP_NAME=tsg_master + +[SYSTEM] +LOG_LEVEL=10 +LOG_PATH=./tsglog/tsg_master +POLICY_PRIORITY_LABEL=POLICY_PRIORITY diff --git a/bin/tsg_log_field.conf b/bin/tsg_log_field.conf new file mode 100644 index 0000000..17e8c11 --- /dev/null +++ b/bin/tsg_log_field.conf @@ -0,0 +1,41 @@ +#TYPE：1:UCHAR,2:USHORT,3:ULONG,4:ULOG,5:USTRING,6:FILE,7:UBASE64,8:PACKET +#TYPE TOPIC SERVICE +TOPIC SECURITY-EVENT-LOG 0 + +#TYPE FIELD VALUE +LONG common_policy_id 1 +LONG common_service 2 +LONG common_action 3 +LONG common_start_time 4 +LONG common_end_time 5 +STRING common_l4_protocol 6 +LONG common_address_type 7 +STRING common_server_ip 8 +STRING common_client_ip 9 +LONG common_server_port 10 +LONG common_client_port 11 +LONG common_stream_dir 12 +STRING common_address_list 13 +LONG common_entrance_id 14 +LONG common_device_id 15 +LONG common_link_id 16 +STRING common_isp 17 +LONG common_encapsulation 18 +LONG common_direction 19 +STRING common_sled_ip 20 +STRING common_user_tags 21 +STRING common_user_region 22 +STRING common_app_label 23 +LONG common_app_id 24 +LONG common_protocol_id 25 +LONG common_c2s_pkt_num 26 +LONG common_s2c_pkt_num 27 +LONG common_c2s_byte_num 28 +LONG common_s2c_byte_num 29 +LONG common_con_duration_ms 30 +LONG common_has_dup_traffic 31 +STRING common_stream_error 32 +STRING common_stream_trace_id 33 +STRING common_schema_type 34 +STRING http_host 35 +STRING ssl_sni 36 diff --git a/bin/tsg_maat.json b/bin/tsg_maat.json new file mode 100644 index 0000000..a592c25 --- /dev/null +++ b/bin/tsg_maat.json @@ -0,0 +1,84 @@ +{ + "compile_table": "TSG_SECURITY_COMPILE", + "group_table": "POLICY_OBJECT", + "rules": [ + { + "compile_id": 1, + "service": 0, + "action": 16, + "do_blacklist": 0, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_1", + "regions": [ + { + "table_name": "TSG_OBJ_IP_ADDR", + "table_type": "ip", + "table_content": { + "addr_type": "ipv4", + "src_ip": "61.135.169.125", + "mask_src_ip": "255.255.255.255", + "src_port": "80", + "mask_src_port": "65535", + "dst_ip": "192.168.41.228", + "mask_dst_ip": "255.255.255.255", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + } + ] + } + ] + }, + { + "compile_id": 2, + "service": 0, + "action": 128, + "do_blacklist": 0, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "FQDN_SNI", + "regions": [ + { + "table_name": "TSG_OBJ_FQDN", + "table_type": "expr", + "table_content": { + "keywords": "baidu.com", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 3, + "service": 0, + "action": 128, + "do_blacklist": 0, + "do_log": 1, + "effective_rage": 0, + "user_region": "Virtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"FQDN_SNI", + "virtual_table":"TSG_FIELD_SSL_SNI", + "not_flag" : 0 + } + ] + } + ] +} diff --git a/bin/tsg_maat_ip_deny.json b/bin/tsg_maat_ip_deny.json new file mode 100644 index 0000000..9e0a2e0 --- /dev/null +++ b/bin/tsg_maat_ip_deny.json @@ -0,0 +1,40 @@ +{ + "compile_table": "TSG_SECURITY_COMPILE", + "group_table": "POLICY_OBJECT", + "rules": [ + { + "compile_id": 1, + "service": 0, + "action": 16, + "do_blacklist": 0, + "do_log": 1, + "effective_rage": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "group_1", + "regions": [ + { + "table_name": "TSG_OBJ_IP_ADDR", + "table_type": "ip", + "table_content": { + "addr_type": "ipv4", + "src_ip": "117.18.237.29", + "mask_src_ip": "255.255.255.255", + "src_port": "80", + "mask_src_port": "65535", + "dst_ip": "192.168.41.228", + "mask_dst_ip": "255.255.255.255", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/bin/tsg_master.inf b/bin/tsg_master.inf new file mode 100644 index 0000000..e18e8dd --- /dev/null +++ b/bin/tsg_master.inf @@ -0,0 +1,9 @@ +[PLUGINFO] +PLUGNAME=TSG_MASTER +SO_PATH=./plug/platform/tsg_master/tsg_master.so +INIT_FUNC=TSG_MASTER_INIT +DESTROY_FUNC=TSG_MASTER_UNLOAD + +[TCP] +FUNC_FLAG=ALL +FUNC_NAME=TSG_MASTER_TCP_ENTRY diff --git a/bin/tsg_tableinfo.conf b/bin/tsg_tableinfo.conf new file mode 100644 index 0000000..9eb7726 --- /dev/null +++ b/bin/tsg_tableinfo.conf @@ -0,0 +1,38 @@ +#each collumn seperate with '\t' +#id (0~65535) +#name string +#type one of ip,expr,expr_plus,digest,intval,compile or plugin +#src_charset one of GBK,BIG5,UNICODE,UTF8 +#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/' +#do_merege yes or no +#cross cache 0~max +#quickswitch quickon or quick off +#id name type src_charset dst_charset do_merge cross_cache quickswitch +0 TSG_SECURITY_COMPILE compile escape -- +1 POLICY_OBJECT group UTF8 UTF8 no 0 +2 TSG_OBJ_IP_ADDR ip UTF8 UTF8 no 0 +3 TSG_OBJ_SUBSCRIBER_ID expr UTF8 UTF8 no 0 +4 TSG_OBJ_ACCOUNT expr UTF8 UTF8 no 0 +5 TSG_OBJ_URL expr UTF8 UTF8 no 0 +6 TSG_OBJ_FQDN expr UTF8 UTF8 no 0 +7 TSG_OBJ_KEYWORDS expr UTF8 UTF8 no 0 +8 TSG_FIELD_HTTP_HOST virtual TSG_OBJ_FQDN -- +9 TSG_FIELD_HTTP_URL virtual TSG_OBJ_URL -- +10 TSG_FIELD_HTTP_REQ_HDR virtual TSG_OBJ_HTTP_SIGNATURE -- +11 TSG_FIELD_HTTP_RES_HDR virtual TSG_OBJ_HTTP_SIGNATURE -- +12 TSG_FIELD_HTTP_REQ_CONTENT virtual TSG_OBJ_KEYWORDS -- +13 TSG_FIELD_HTTP_RES_CONTENT virtual TSG_OBJ_KEYWORDS -- +14 TSG_FIELD_SSL_SNI virtual TSG_OBJ_FQDN -- +15 TSG_FIELD_SSL_CN virtual TSG_OBJ_FQDN -- +16 TSG_FIELD_SSL_SAN virtual TSG_OBJ_FQDN -- +17 TSG_FIELD_DNS_QNAME virtual TSG_OBJ_FQDN -- +18 TSG_FIELD_MAIL_ACCOUNT virtual TSG_OBJ_ACCOUNT -- +19 TSG_FIELD_MAIL_FROM virtual TSG_OBJ_ACCOUNT -- +20 TSG_FIELD_MAIL_TO virtual TSG_OBJ_ACCOUNT -- +21 TSG_FIELD_MAIL_SUBJECT virtual TSG_OBJ_KEYWORDS -- +22 TSG_FIELD_MAIL_CONTENT virtual TSG_OBJ_KEYWORDS -- +23 TSG_FIELD_MAIL_ATT_NAME virtual TSG_OBJ_KEYWORDS -- +24 TSG_FIELD_MAIL_ATT_CONTENT virtual TSG_OBJ_KEYWORDS -- +25 TSG_FIELD_FTP_URI virtual TSG_OBJ_URL -- +26 TSG_FIELD_FTP_CONTENT virtual TSG_OBJ_KEYWORDS -- +27 TSG_FIELD_FTP_ACCOUNT virtual TSG_OBJ_ACCOUNT -- From 8d954e4cc58c9e10ef917eb6cad480a9a0a599e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E4=B8=80=E9=B8=A3?= Date: Wed, 20 Nov 2019 19:55:59 +0800 Subject: [PATCH 20/32] =?UTF-8?q?=E4=BF=AE=E6=94=B9proto=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=80=BC=E4=B8=BAPROTO=5FUNKONWN?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_rule.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index f2dc007..245ae79 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -148,7 +148,7 @@ int tsg_rule_init(const char* conffile, void *logger) } static void protocol_identify(char *buff, int buff_len, struct _identify_info *result){ - result->proto = PROTO_MAX; + result->proto = PROTO_UNKONWN; //TODO: http: get from http protocol plugin /* if(is_http){ @@ -169,7 +169,6 @@ static void protocol_identify(char *buff, int buff_len, struct _identify_info *r result->domain_len = strnlen(chello->sni, sizeof(result->domain) - 1); strncpy(result->domain, chello->sni, result->domain_len); } - result->domain[result->domain_len] = '\0'; } ssl_chello_free(chello); return; From 46a233b78242f6515963c8235d4ac0702492f32b Mon Sep 17 00:00:00 2001 From: liuxueli Date: Thu, 21 Nov 2019 10:18:08 +0800 Subject: [PATCH 21/32] =?UTF-8?q?KNI=E5=8F=91=E9=80=81=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9A=84=E7=BA=BF=E7=A8=8B=E4=B8=8D=E6=98=AFsapp=E7=94=B3?= =?UTF-8?q?=E8=AF=B7=E7=9A=84=EF=BC=8C=E6=89=80=E4=BB=A5=E4=B8=8D=E8=83=BD?= =?UTF-8?q?=E4=BD=BF=E7=94=A8dictator=5Fmalloc=E7=94=B3=E8=AF=B7=E5=86=85?= =?UTF-8?q?=E5=AD=98=20=E5=A2=9E=E5=8A=A0user=5Fregion=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_send_log.cpp | 47 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 7a83299..434a7ff 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -19,7 +19,7 @@ #include "tsg_send_log.h" #include "tsg_send_log_internal.h" -char TSG_SEND_LOG_VERSION_20191120=0; +char TSG_SEND_LOG_VERSION_20191121=0; tsg_log_instance_t g_tsg_log_instance; const id2field_t tld_type[TLD_TYPE_MAX]={{TLD_TYPE_UNKNOWN, TLD_TYPE_UNKNOWN, "UNKOWN"}, @@ -48,7 +48,6 @@ unsigned long long tsg_get_stream_id(struct streaminfo * a_stream) int TLD_cancel(TLD_handle_t handle) { - int thread_id=0; struct _tld_handle *_handle=NULL; if(handle!=NULL) { @@ -58,9 +57,8 @@ int TLD_cancel(TLD_handle_t handle) cJSON_Delete(_handle->object); _handle->object=NULL; } - thread_id=_handle->thread_id; - dictator_free(thread_id, handle); - + + free(handle); handle=NULL; } @@ -109,7 +107,9 @@ int TLD_append(TLD_handle_t handle, char *key, void *value, TLD_TYPE type) TLD_handle_t TLD_create(int thread_id) { - struct _tld_handle *_handle=(struct _tld_handle *)dictator_malloc(thread_id, sizeof(struct _tld_handle)); + //struct _tld_handle *_handle=(struct _tld_handle *)dictator_malloc(thread_id, sizeof(struct _tld_handle)); + + struct _tld_handle *_handle=(struct _tld_handle *)calloc(1, sizeof(struct _tld_handle)); _handle->thread_id = thread_id; _handle->object = cJSON_CreateObject(); @@ -343,7 +343,7 @@ tsg_log_instance_t tsg_sendlog_init(const char *conffile) int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *log_msg, int thread_id) { - int i=0,status=0; + int i=0,ret=0,status=0; char *payload=NULL; time_t cur_time; struct vxlan_info vinfo; @@ -431,6 +431,34 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG); TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG); + if(log_msg->result[i].serv_def_len<128) + { + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)(log_msg->result[i].service_defined), TLD_TYPE_STRING); + } + else + { + char *service_defined=(char *)dictator_malloc(thread_id, log_msg->result[i].serv_def_len+1); + ret=Maat_read_rule(g_tsg_maat_feather, &log_msg->result[i], MAAT_RULE_SERV_DEFINE, service_defined, log_msg->result[i].serv_def_len); + if(ret==log_msg->result[i].serv_def_len) + { + service_defined[log_msg->result[i].serv_def_len]='\0'; + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)(log_msg->result[i].service_defined), TLD_TYPE_STRING); + } + else + { + MESA_handle_runtime_log(_instance->logger, + RLOG_LV_FATAL, + "TSG_SEND_LOG", + "Fetch service_defined failed, policy_id: %d service: %d action: %d addr: %s", + log_msg->result[i].config_id, + log_msg->result[i].service_id, + log_msg->result[i].action, + printaddr(&log_msg->a_stream->addr, thread_id)); + } + dictator_free(thread_id, (void *)service_defined); + service_defined=NULL; + } + payload = cJSON_PrintUnformatted(_handle->object); status = rd_kafka_produce(_instance->topic_rkt[log_msg->result[i].service_id], RD_KAFKA_PARTITION_UA, RD_KAFKA_MSG_F_COPY, payload, strlen(payload), NULL, 0, NULL); @@ -453,13 +481,16 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name); TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVICE].name); TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ACTION].name); + TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_USER_REGION].name); FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LOG], 0, FS_OP_ADD, 1); } cJSON_Delete(_handle->object); - dictator_free(thread_id, handle); + _handle->object=NULL; + + free(handle); handle=NULL; return 0; From 27cb2ad3b2da0d9ce7307513f0e119b181ad13b6 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Thu, 21 Nov 2019 19:21:08 +0800 Subject: [PATCH 22/32] =?UTF-8?q?=E8=8E=B7=E5=8F=96=E5=8C=85=E6=95=B0?= =?UTF-8?q?=E3=80=81=E5=AD=97=E8=8A=82=E6=95=B0=E4=B8=8Esapp=E5=A4=B4?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=B8=8D=E4=B8=80=E8=87=B4=EF=BC=8C=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=BB=9F=E8=AE=A1=E4=BF=A1=E6=81=AF=E9=A2=A0=E5=80=92?= =?UTF-8?q?=20=E5=88=A0=E9=99=A4=E4=BB=8Evxlan=E9=87=8C=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_send_log.cpp | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 434a7ff..9025ea8 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -81,7 +81,7 @@ int TLD_append(TLD_handle_t handle, char *key, void *value, TLD_TYPE type) { struct _tld_handle *_handle=(struct _tld_handle *)handle; - if(_handle==NULL || key==NULL || value==NULL) + if(_handle==NULL || key==NULL || (value==NULL && type!=TLD_TYPE_LONG)) { return -1; } @@ -177,10 +177,10 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)(long)a_stream->dir, TLD_TYPE_LONG); TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)(long)addr_type, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->clientpktnum, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->clientbytes, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->serverpktnum, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->serverbytes, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->clientpktnum, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->clientbytes, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->serverpktnum, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->serverbytes, TLD_TYPE_LONG); stream_id=tsg_get_stream_id(a_stream); TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_STREAM_TRACE_ID].name, (void *)(long)stream_id, TLD_TYPE_LONG); @@ -346,8 +346,7 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo int i=0,ret=0,status=0; char *payload=NULL; time_t cur_time; - struct vxlan_info vinfo; - int opt_val_len = sizeof(vinfo); + long common_con_duration_ms=0; struct _tld_handle *_handle = (struct _tld_handle *)handle; struct _tsg_log_instance *_instance = (struct _tsg_log_instance *)instance; @@ -365,14 +364,12 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo //TODO //common_user_tags - //common_user_region //common_isp //common_app_label //common_app_id //common_protocol_id //common_has_dup_traffic //common_stream_error - //common_stream_trace_id TLD_append_streaminfo(_instance, _handle, log_msg->a_stream); @@ -381,16 +378,22 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo if(log_msg->a_stream!=NULL && log_msg->a_stream->ptcpdetail!=NULL) { TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->createtime), TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG); + + common_con_duration_ms=log_msg->a_stream->ptcpdetail->lastmtime-log_msg->a_stream->ptcpdetail->createtime; + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); } else { cur_time=time(NULL); TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)cur_time, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)cur_time, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)cur_time, TLD_TYPE_LONG); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); } - +#if 0 + struct vxlan_info vinfo; + int opt_val_len = sizeof(vinfo); status=MESA_get_stream_opt(log_msg->a_stream, MSO_STREAM_VXLAN_INFO, &vinfo, &opt_val_len); if(status < 0) { @@ -404,6 +407,7 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENTRANCE_ID].name, (void *)(long)vinfo.entrance_id, TLD_TYPE_LONG); TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENCAPSULATION].name, (void *)(long)vinfo.encap_type, TLD_TYPE_LONG); } +#endif for(i=0;iresult_num; i++) { @@ -437,12 +441,11 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo } else { - char *service_defined=(char *)dictator_malloc(thread_id, log_msg->result[i].serv_def_len+1); + char *service_defined=(char *)calloc(1, log_msg->result[i].serv_def_len+1); ret=Maat_read_rule(g_tsg_maat_feather, &log_msg->result[i], MAAT_RULE_SERV_DEFINE, service_defined, log_msg->result[i].serv_def_len); if(ret==log_msg->result[i].serv_def_len) { - service_defined[log_msg->result[i].serv_def_len]='\0'; - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)(log_msg->result[i].service_defined), TLD_TYPE_STRING); + TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)service_defined, TLD_TYPE_STRING); } else { @@ -455,7 +458,7 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo log_msg->result[i].action, printaddr(&log_msg->a_stream->addr, thread_id)); } - dictator_free(thread_id, (void *)service_defined); + free((void *)service_defined); service_defined=NULL; } From 7c9dabdab58db3405cbb86d0e98694ad38e8b26c Mon Sep 17 00:00:00 2001 From: liuxueli Date: Mon, 25 Nov 2019 10:51:37 +0800 Subject: [PATCH 23/32] =?UTF-8?q?=E7=9F=AB=E6=AD=A3tableinfo=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/tsg_tableinfo.conf | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/bin/tsg_tableinfo.conf b/bin/tsg_tableinfo.conf index 9eb7726..d9ac4a1 100644 --- a/bin/tsg_tableinfo.conf +++ b/bin/tsg_tableinfo.conf @@ -16,23 +16,25 @@ 5 TSG_OBJ_URL expr UTF8 UTF8 no 0 6 TSG_OBJ_FQDN expr UTF8 UTF8 no 0 7 TSG_OBJ_KEYWORDS expr UTF8 UTF8 no 0 -8 TSG_FIELD_HTTP_HOST virtual TSG_OBJ_FQDN -- -9 TSG_FIELD_HTTP_URL virtual TSG_OBJ_URL -- -10 TSG_FIELD_HTTP_REQ_HDR virtual TSG_OBJ_HTTP_SIGNATURE -- -11 TSG_FIELD_HTTP_RES_HDR virtual TSG_OBJ_HTTP_SIGNATURE -- -12 TSG_FIELD_HTTP_REQ_CONTENT virtual TSG_OBJ_KEYWORDS -- -13 TSG_FIELD_HTTP_RES_CONTENT virtual TSG_OBJ_KEYWORDS -- -14 TSG_FIELD_SSL_SNI virtual TSG_OBJ_FQDN -- -15 TSG_FIELD_SSL_CN virtual TSG_OBJ_FQDN -- -16 TSG_FIELD_SSL_SAN virtual TSG_OBJ_FQDN -- -17 TSG_FIELD_DNS_QNAME virtual TSG_OBJ_FQDN -- -18 TSG_FIELD_MAIL_ACCOUNT virtual TSG_OBJ_ACCOUNT -- -19 TSG_FIELD_MAIL_FROM virtual TSG_OBJ_ACCOUNT -- -20 TSG_FIELD_MAIL_TO virtual TSG_OBJ_ACCOUNT -- -21 TSG_FIELD_MAIL_SUBJECT virtual TSG_OBJ_KEYWORDS -- -22 TSG_FIELD_MAIL_CONTENT virtual TSG_OBJ_KEYWORDS -- -23 TSG_FIELD_MAIL_ATT_NAME virtual TSG_OBJ_KEYWORDS -- -24 TSG_FIELD_MAIL_ATT_CONTENT virtual TSG_OBJ_KEYWORDS -- -25 TSG_FIELD_FTP_URI virtual TSG_OBJ_URL -- -26 TSG_FIELD_FTP_CONTENT virtual TSG_OBJ_KEYWORDS -- -27 TSG_FIELD_FTP_ACCOUNT virtual TSG_OBJ_ACCOUNT -- +8 TSG_OBJ_HTTP_SIGNATURE expr_plus UTF8 UTF8 no 0 +9 TSG_FIELD_HTTP_HOST virtual TSG_OBJ_FQDN -- +10 TSG_FIELD_HTTP_URL virtual TSG_OBJ_URL -- +11 TSG_FIELD_HTTP_REQ_HDR virtual TSG_OBJ_HTTP_SIGNATURE -- +12 TSG_FIELD_HTTP_RES_HDR virtual TSG_OBJ_HTTP_SIGNATURE -- +13 TSG_FIELD_HTTP_REQ_CONTENT virtual TSG_OBJ_KEYWORDS -- +14 TSG_FIELD_HTTP_RES_CONTENT virtual TSG_OBJ_KEYWORDS -- +15 TSG_FIELD_SSL_SNI virtual TSG_OBJ_FQDN -- +16 TSG_FIELD_SSL_CN virtual TSG_OBJ_FQDN -- +17 TSG_FIELD_SSL_SAN virtual TSG_OBJ_FQDN -- +18 TSG_FIELD_DNS_QNAME virtual TSG_OBJ_FQDN -- +19 TSG_FIELD_MAIL_ACCOUNT virtual TSG_OBJ_ACCOUNT -- +20 TSG_FIELD_MAIL_FROM virtual TSG_OBJ_ACCOUNT -- +21 TSG_FIELD_MAIL_TO virtual TSG_OBJ_ACCOUNT -- +22 TSG_FIELD_MAIL_SUBJECT virtual TSG_OBJ_KEYWORDS -- +23 TSG_FIELD_MAIL_CONTENT virtual TSG_OBJ_KEYWORDS -- +24 TSG_FIELD_MAIL_ATT_NAME virtual TSG_OBJ_KEYWORDS -- +25 TSG_FIELD_MAIL_ATT_CONTENT virtual TSG_OBJ_KEYWORDS -- +26 TSG_FIELD_FTP_URI virtual TSG_OBJ_URL -- +27 TSG_FIELD_FTP_CONTENT virtual TSG_OBJ_KEYWORDS -- +28 TSG_FIELD_FTP_ACCOUNT virtual TSG_OBJ_ACCOUNT -- + From 4aef0628e541ea2616f24d62cc2748474c2d8edc Mon Sep 17 00:00:00 2001 From: liuxueli Date: Mon, 25 Nov 2019 18:36:48 +0800 Subject: [PATCH 24/32] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_entry.cpp | 22 +++++++++++++++++++++- src/tsg_rule.cpp | 4 ++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 37a3030..7af7549 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -14,7 +14,7 @@ -char TSG_MASTER_VERSION_20191120=0; +char TSG_MASTER_VERSION_20191125=0; const char *tsg_conffile="tsgconf/main.conf"; g_tsg_para_t g_tsg_para; @@ -155,6 +155,26 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t hit_num+=ret; identify_flag=1; FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1); + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "SCAN_IP_AND_DOAMIN", + "Hit %s: %s addr: %s", + (identify_info.proto==PROTO_HTTP) ? "host" : "sni", + identify_info.domain, + printaddr(&a_tcp->addr, thread_seq) + ); + + } + else + { + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "SCAN_IP_AND_DOAMIN", + "Not hit %s: %s addr: %s", + (ret==0) ? ((identify_info.proto==PROTO_HTTP) ? "host" : "sni") : "NULL", + (ret==0) ? identify_info.domain : "NULL", + printaddr(&a_tcp->addr, thread_seq) + ); } p_result=tsg_policy_decision_criteria(all_result, hit_num); diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 245ae79..e812eb9 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -76,7 +76,7 @@ int tsg_rule_init(const char* conffile, void *logger) Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_PERF_ON, NULL, 0); Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_REDIS_INDEX, &redis_index, sizeof(redis_index)); Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail)); - Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_DEFERRED_LOAD, NULL,0); + //Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_DEFERRED_LOAD, NULL,0); } else { @@ -304,7 +304,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo * }while(cur_stream != NULL && hit_num < result_num); - if(hit_num < result_num) + if(hit_num0) { maat_ret=Maat_full_scan_string(maat_feather, g_tsg_para.subscribe_id_table_id, From 8cfc6b32fd40a21062a3f2e9cf23d34bb4a66f3a Mon Sep 17 00:00:00 2001 From: liuxueli Date: Mon, 25 Nov 2019 19:30:08 +0800 Subject: [PATCH 25/32] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_entry.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 7af7549..e2fa348 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -171,8 +171,8 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t RLOG_LV_DEBUG, "SCAN_IP_AND_DOAMIN", "Not hit %s: %s addr: %s", - (ret==0) ? ((identify_info.proto==PROTO_HTTP) ? "host" : "sni") : "NULL", - (ret==0) ? identify_info.domain : "NULL", + (ret==-1) ? "NULL" : ((identify_info.proto==PROTO_HTTP) ? "host" : "sni"), + (ret==-1) ? "NULL" : identify_info.domain, printaddr(&a_tcp->addr, thread_seq) ); } From 132d807af17882081510ce8af05bdd9706eac52e Mon Sep 17 00:00:00 2001 From: liuxueli Date: Tue, 26 Nov 2019 16:58:54 +0800 Subject: [PATCH 26/32] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_entry.cpp | 8 ++++---- src/tsg_rule.cpp | 13 +++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index e2fa348..c755b04 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -14,7 +14,7 @@ -char TSG_MASTER_VERSION_20191125=0; +char TSG_MASTER_VERSION_20191126=0; const char *tsg_conffile="tsgconf/main.conf"; g_tsg_para_t g_tsg_para; @@ -100,7 +100,7 @@ static Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result if(result[i].action==p_result->action) { - if(result[i].config_idconfig_id) + if(result[i].config_id>p_result->config_id) { p_result=&result[i]; } @@ -157,7 +157,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1); MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, - "SCAN_IP_AND_DOAMIN", + "SCAN_IP_AND_FQDN", "Hit %s: %s addr: %s", (identify_info.proto==PROTO_HTTP) ? "host" : "sni", identify_info.domain, @@ -169,7 +169,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t { MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, - "SCAN_IP_AND_DOAMIN", + "SCAN_IP_AND_FQDN", "Not hit %s: %s addr: %s", (ret==-1) ? "NULL" : ((identify_info.proto==PROTO_HTTP) ? "host" : "sni"), (ret==-1) ? "NULL" : identify_info.domain, diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index e812eb9..dcdd430 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -215,16 +215,21 @@ int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_re else { MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "PULL_RESULT", - "pull policy failed, result_type: %s, addr: %s", + "pull policy failed, hit: %s %s: %s policy_id: %d service: %d action: %d addr: %s", (label->result_type==PULL_KNI_RESULT) ? "KNI" : "FW", + label->proto==PROTO_HTTP ? "host" : "sni", + label->domain, + label->result->config_id, + label->result->service_id, + label->result->action, printaddr(&a_stream->addr, a_stream->threadnum)); } } else { - MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PULL_RESULT", - "pull policy failed, result_type=%d, addr: %s", - (label==NULL) ? -1 : label->result_type, + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "PULL_RESULT", + "pull policy failed, Not hit, label is %s addr: %s", + (label==NULL) ? "NULL" : label->domain, printaddr(&a_stream->addr, a_stream->threadnum)); } From 04963fe1bfb56ce4712817d2683796d8833362e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B4=94=E4=B8=80=E9=B8=A3?= Date: Tue, 26 Nov 2019 17:41:43 +0800 Subject: [PATCH 27/32] =?UTF-8?q?=E5=A2=9E=E5=8A=A0http=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_rule.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index dcdd430..460110b 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -9,6 +9,7 @@ #include "MESA/MESA_handle_logger.h" #include "Maat_rule.h" #include "Maat_command.h" +#include "MESA/http.h" #include "tsg_ssl_utils.h" #include "tsg_rule.h" #include "tsg_entry.h" @@ -17,6 +18,10 @@ Maat_feather_t g_tsg_maat_feather; #define MAX_PATH_LEN 1024 +#ifndef MIN +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#endif + enum kni_scan_table{ TSG_FIELD_SSL_SNI, TSG_FIELD_HTTP_HOST, @@ -149,13 +154,21 @@ int tsg_rule_init(const char* conffile, void *logger) static void protocol_identify(char *buff, int buff_len, struct _identify_info *result){ result->proto = PROTO_UNKONWN; - //TODO: http: get from http protocol plugin - /* - if(is_http){ - result->protocol = PROTO_HTTP; + //http + char *host = NULL; + int ret = http_host_parser(buff, (uint32_t)buff_len, DIR_C2S, &host); + //printf("http_host_parse: ret = %d, buff_len = %d, buff = %s\n", ret, buff_len, buff); + if(ret >= 0){ + result->proto = PROTO_HTTP; + if(ret == 0){ + result->domain_len = 0; + } + else{ + result->domain_len = MIN(ret, (int)sizeof(result->domain) - 1); + strncpy(result->domain, host, result->domain_len); + } return; } - */ //ssl enum chello_parse_result chello_status = CHELLO_PARSE_INVALID_FORMAT; struct ssl_chello *chello = NULL; From 27f0cfc91a6d85c0460d87e3e3ee05e4df02dc3c Mon Sep 17 00:00:00 2001 From: liuxueli Date: Wed, 27 Nov 2019 10:33:48 +0800 Subject: [PATCH 28/32] =?UTF-8?q?=E6=8F=90=E4=BE=9BTLD=5Fappend=5Fstreamin?= =?UTF-8?q?fo=E5=87=BD=E6=95=B0=E6=8E=A5=E5=8F=A3=E4=BE=9BKNI=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=20=E4=BF=AE=E6=94=B9=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/tsg_tableinfo.conf | 8 ++++---- inc/tsg_send_log.h | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/tsg_tableinfo.conf b/bin/tsg_tableinfo.conf index d9ac4a1..72ef02f 100644 --- a/bin/tsg_tableinfo.conf +++ b/bin/tsg_tableinfo.conf @@ -9,14 +9,14 @@ #quickswitch quickon or quick off #id name type src_charset dst_charset do_merge cross_cache quickswitch 0 TSG_SECURITY_COMPILE compile escape -- -1 POLICY_OBJECT group UTF8 UTF8 no 0 -2 TSG_OBJ_IP_ADDR ip UTF8 UTF8 no 0 +1 GROUP_COMPILE_RELATION group UTF8 UTF8 no 0 +2 TSG_OBJ_IP_ADDR ip_plus UTF8 UTF8 no 0 3 TSG_OBJ_SUBSCRIBER_ID expr UTF8 UTF8 no 0 4 TSG_OBJ_ACCOUNT expr UTF8 UTF8 no 0 -5 TSG_OBJ_URL expr UTF8 UTF8 no 0 +5 TSG_OBJ_URL expr UTF8 UTF8/GBK no 0 6 TSG_OBJ_FQDN expr UTF8 UTF8 no 0 7 TSG_OBJ_KEYWORDS expr UTF8 UTF8 no 0 -8 TSG_OBJ_HTTP_SIGNATURE expr_plus UTF8 UTF8 no 0 +8 TSG_OBJ_HTTP_SIGNATURE expr_plus UTF8 UTF8/GBK no 0 9 TSG_FIELD_HTTP_HOST virtual TSG_OBJ_FQDN -- 10 TSG_FIELD_HTTP_URL virtual TSG_OBJ_URL -- 11 TSG_FIELD_HTTP_REQ_HDR virtual TSG_OBJ_HTTP_SIGNATURE -- diff --git a/inc/tsg_send_log.h b/inc/tsg_send_log.h index b3a4b5c..a399b43 100644 --- a/inc/tsg_send_log.h +++ b/inc/tsg_send_log.h @@ -28,6 +28,7 @@ extern tsg_log_instance_t g_tsg_log_instance; TLD_handle_t TLD_create(int thread_id); int TLD_append(TLD_handle_t handle, char *key, void *value, TLD_TYPE type); +int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handle *_handle, struct streaminfo *a_stream); int TLD_cancel(TLD_handle_t handle); int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *log_msg, int thread_id); From 2d1a530165be653f5ee3ee41bc89521e1be830f4 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Wed, 27 Nov 2019 11:27:06 +0800 Subject: [PATCH 29/32] =?UTF-8?q?=E4=BD=BF=E7=94=A8=E5=89=8D=E7=BD=AE?= =?UTF-8?q?=E5=A3=B0=E6=98=8E=E7=BB=93=E6=9E=84=E4=BD=93=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E6=8E=A5=E5=8F=A3=E5=87=BD=E6=95=B0=E5=8E=9F=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/tsg_rule.h | 2 +- inc/tsg_send_log.h | 16 ++--- src/tsg_entry.cpp | 2 +- src/tsg_rule.cpp | 2 +- src/tsg_send_log.cpp | 127 +++++++++++++++++++----------------- src/tsg_send_log_internal.h | 8 +-- 6 files changed, 82 insertions(+), 75 deletions(-) diff --git a/inc/tsg_rule.h b/inc/tsg_rule.h index 7fd7a93..54f83ba 100644 --- a/inc/tsg_rule.h +++ b/inc/tsg_rule.h @@ -56,6 +56,6 @@ int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, voi int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, struct _identify_info *identify_info, scan_status_t *mid, void *logger, int thread_seq); //return NULL if none exists, otherwise return one deny rule; -Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num); +struct Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num); #endif diff --git a/inc/tsg_send_log.h b/inc/tsg_send_log.h index a399b43..3223f36 100644 --- a/inc/tsg_send_log.h +++ b/inc/tsg_send_log.h @@ -21,17 +21,17 @@ typedef enum _tld_type }TLD_TYPE; -typedef void* TLD_handle_t; -typedef void* tsg_log_instance_t; +struct TLD_handle_t; +struct tsg_log_instance_t; -extern tsg_log_instance_t g_tsg_log_instance; +extern struct tsg_log_instance_t *g_tsg_log_instance; -TLD_handle_t TLD_create(int thread_id); -int TLD_append(TLD_handle_t handle, char *key, void *value, TLD_TYPE type); -int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handle *_handle, struct streaminfo *a_stream); -int TLD_cancel(TLD_handle_t handle); +struct TLD_handle_t *TLD_create(int thread_id); +int TLD_append(struct TLD_handle_t *handle, char *key, void *value, TLD_TYPE type); +int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, struct streaminfo *a_stream); +int TLD_cancel(struct TLD_handle_t *handle); -int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *log_msg, int thread_id); +int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, tsg_log_t *log_msg, int thread_id); unsigned long long tsg_get_stream_id(struct streaminfo *a_stream); diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index c755b04..2972bb5 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -121,7 +121,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t Maat_rule_t *p_result=NULL; Maat_rule_t *q_result=NULL; tsg_log_t log_msg; - TLD_handle_t TLD_handle=NULL; + struct TLD_handle_t *TLD_handle=NULL; struct _identify_info identify_info; Maat_rule_t all_result[MAX_RESULT_NUM]; policy_priority_label_t *priority_label=NULL; diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 460110b..454f70a 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -365,7 +365,7 @@ int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, } -Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num) +struct Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num) { int i=0; Maat_rule_t *p_result=NULL; diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 9025ea8..217da6a 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -20,7 +20,8 @@ #include "tsg_send_log_internal.h" char TSG_SEND_LOG_VERSION_20191121=0; -tsg_log_instance_t g_tsg_log_instance; +struct tsg_log_instance_t *g_tsg_log_instance; + const id2field_t tld_type[TLD_TYPE_MAX]={{TLD_TYPE_UNKNOWN, TLD_TYPE_UNKNOWN, "UNKOWN"}, {TLD_TYPE_LONG, TLD_TYPE_LONG, "LONG"}, @@ -46,12 +47,12 @@ unsigned long long tsg_get_stream_id(struct streaminfo * a_stream) return -1; } -int TLD_cancel(TLD_handle_t handle) +int TLD_cancel(struct TLD_handle_t *handle) { - struct _tld_handle *_handle=NULL; - if(handle!=NULL) + struct TLD_handle_t *_handle=handle; + + if(_handle!=NULL) { - _handle=(struct _tld_handle *)handle; if(_handle->object!=NULL) { cJSON_Delete(_handle->object); @@ -65,9 +66,9 @@ int TLD_cancel(TLD_handle_t handle) return 0; } -int TLD_delete(TLD_handle_t handle, char *key) +int TLD_delete(struct TLD_handle_t *handle, char *key) { - struct _tld_handle *_handle=(struct _tld_handle *)handle; + struct TLD_handle_t *_handle=handle; if(_handle!=NULL && key!=NULL) { @@ -77,9 +78,9 @@ int TLD_delete(TLD_handle_t handle, char *key) return 0; } -int TLD_append(TLD_handle_t handle, char *key, void *value, TLD_TYPE type) +int TLD_append(struct TLD_handle_t *handle, char *key, void *value, TLD_TYPE type) { - struct _tld_handle *_handle=(struct _tld_handle *)handle; + struct TLD_handle_t *_handle=handle; if(_handle==NULL || key==NULL || (value==NULL && type!=TLD_TYPE_LONG)) { @@ -105,30 +106,30 @@ int TLD_append(TLD_handle_t handle, char *key, void *value, TLD_TYPE type) return 0; } -TLD_handle_t TLD_create(int thread_id) +struct TLD_handle_t *TLD_create(int thread_id) { //struct _tld_handle *_handle=(struct _tld_handle *)dictator_malloc(thread_id, sizeof(struct _tld_handle)); - struct _tld_handle *_handle=(struct _tld_handle *)calloc(1, sizeof(struct _tld_handle)); + struct TLD_handle_t *_handle=(struct TLD_handle_t *)calloc(1, sizeof(struct TLD_handle_t)); _handle->thread_id = thread_id; _handle->object = cJSON_CreateObject(); - return (TLD_handle_t)_handle; + return _handle; } -char *log_field_id2name(tsg_log_instance_t instance, tsg_log_field_id_t id) +char *log_field_id2name(struct tsg_log_instance_t *instance, tsg_log_field_id_t id) { - struct _tsg_log_instance *log_instance = (struct _tsg_log_instance *)instance; - if(log_instance!=NULL) + struct tsg_log_instance_t *_instance=instance; + if(_instance!=NULL) { - return log_instance->id2field[id].name; + return _instance->id2field[id].name; } return NULL; } -int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handle *_handle, struct streaminfo *a_stream) +int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, struct streaminfo *a_stream) { int ret=0,addr_type=0; unsigned short tunnel_type=0; @@ -141,6 +142,14 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl struct layer_addr_ipv6 *ipv6=NULL; char server_ip[MAX_IPV4_LEN*8]={0}; char client_ip[MAX_IPV4_LEN*8]={0}; + + struct TLD_handle_t *_handle=handle; + struct tsg_log_instance_t *_instance=instance; + + if(_instance==NULL || _handle==NULL || a_stream==NULL) + { + return -1; + } switch(a_stream->addr.addrtype) { @@ -169,24 +178,24 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl } - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_IP].name, (void *)server_ip, TLD_TYPE_STRING); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_IP].name, (void *)client_ip, TLD_TYPE_STRING); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVER_PORT].name, (void *)(long)s_port, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CLIENT_PORT].name, (void *)(long)c_port, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_IP].name, (void *)server_ip, TLD_TYPE_STRING); + TLD_append(_handle, _instance->id2field[LOG_COMMON_CLIENT_IP].name, (void *)client_ip, TLD_TYPE_STRING); + TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_PORT].name, (void *)(long)s_port, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_CLIENT_PORT].name, (void *)(long)c_port, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)(long)a_stream->dir, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)(long)addr_type, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)(long)a_stream->dir, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)(long)addr_type, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->clientpktnum, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->clientbytes, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->serverpktnum, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->serverbytes, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->clientpktnum, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->clientbytes, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->serverpktnum, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->serverbytes, TLD_TYPE_LONG); stream_id=tsg_get_stream_id(a_stream); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_STREAM_TRACE_ID].name, (void *)(long)stream_id, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_STREAM_TRACE_ID].name, (void *)(long)stream_id, TLD_TYPE_LONG); addr_proto=(char *)layer_addr_prefix_ntop(a_stream); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_L4_PROTOCOL].name, (void *)addr_proto, TLD_TYPE_STRING); + TLD_append(_handle, _instance->id2field[LOG_COMMON_L4_PROTOCOL].name, (void *)addr_proto, TLD_TYPE_STRING); ret=MESA_get_stream_opt(a_stream, MSO_STREAM_TUNNEL_TYPE, &tunnel_type, &tunnel_type_size); @@ -200,7 +209,8 @@ int TLD_append_streaminfo(struct _tsg_log_instance *_instance, struct _tld_handl stream_addr_list_ntop(a_stream,nest_addr_buf, sizeof(nest_addr_buf)); } - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ADDRESS_LIST].name, (void *)nest_addr_buf, TLD_TYPE_STRING); + TLD_append(_handle, _instance->id2field[LOG_COMMON_ADDRESS_LIST].name, (void *)nest_addr_buf, TLD_TYPE_STRING); + return 0; } @@ -265,7 +275,7 @@ static int load_log_common_field(const char *filename, id2field_t *id2field, id2 return 0; } -tsg_log_instance_t tsg_sendlog_init(const char *conffile) +struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) { int i=0,ret=0,level=30; char nic_name[32]={0}; @@ -275,9 +285,9 @@ tsg_log_instance_t tsg_sendlog_init(const char *conffile) rd_kafka_t *kafka_handle = NULL; rd_kafka_conf_t *rdkafka_conf = NULL; rd_kafka_topic_conf_t *topic_conf; - struct _tsg_log_instance *_instance=NULL; + struct tsg_log_instance_t *_instance=NULL; - _instance=(struct _tsg_log_instance *)calloc(1, sizeof(struct _tsg_log_instance)); + _instance=(struct tsg_log_instance_t *)calloc(1, sizeof(struct tsg_log_instance_t)); MESA_load_profile_int_def(conffile, "TSG_LOG", "MODE",&(_instance->mode), 0); MESA_load_profile_string_def(conffile, "TSG_LOG", "COMMON_FIELD_FILE", _instance->common_field_file, sizeof(_instance->common_field_file), NULL); @@ -337,27 +347,29 @@ tsg_log_instance_t tsg_sendlog_init(const char *conffile) } } - return (tsg_log_instance_t)_instance; + return _instance; } -int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *log_msg, int thread_id) +int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, tsg_log_t *log_msg, int thread_id) { int i=0,ret=0,status=0; char *payload=NULL; time_t cur_time; long common_con_duration_ms=0; - struct _tld_handle *_handle = (struct _tld_handle *)handle; - struct _tsg_log_instance *_instance = (struct _tsg_log_instance *)instance; + struct TLD_handle_t *_handle=handle; + struct tsg_log_instance_t *_instance=instance; if(_instance==NULL || _handle==NULL || log_msg==NULL) { + TLD_cancel(handle); MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "TSG_SEND_LOG", " instance==NULL || TLD_handle==NULL || log_msg==NULL "); return -1; } if(_instance->mode==CLOSE) { + TLD_cancel(handle); MESA_handle_runtime_log(_instance->logger, RLOG_LV_INFO, "TSG_SEND_LOG", "Disable tsg_send_log."); return 0; } @@ -371,24 +383,24 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo //common_has_dup_traffic //common_stream_error - TLD_append_streaminfo(_instance, _handle, log_msg->a_stream); + TLD_append_streaminfo(instance, handle, log_msg->a_stream); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SLED_IP].name, (void *)(_instance->local_ip_str), TLD_TYPE_STRING); + TLD_append(_handle, _instance->id2field[LOG_COMMON_SLED_IP].name, (void *)(_instance->local_ip_str), TLD_TYPE_STRING); if(log_msg->a_stream!=NULL && log_msg->a_stream->ptcpdetail!=NULL) { - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->createtime), TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->createtime), TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG); common_con_duration_ms=log_msg->a_stream->ptcpdetail->lastmtime-log_msg->a_stream->ptcpdetail->createtime; - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); } else { cur_time=time(NULL); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)cur_time, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)cur_time, TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)cur_time, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)cur_time, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); } #if 0 @@ -431,13 +443,13 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo break; } - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(long)(log_msg->result[i].config_id), TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG); - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(long)(log_msg->result[i].config_id), TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG); if(log_msg->result[i].serv_def_len<128) { - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)(log_msg->result[i].service_defined), TLD_TYPE_STRING); + TLD_append(_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)(log_msg->result[i].service_defined), TLD_TYPE_STRING); } else { @@ -445,7 +457,7 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo ret=Maat_read_rule(g_tsg_maat_feather, &log_msg->result[i], MAAT_RULE_SERV_DEFINE, service_defined, log_msg->result[i].serv_def_len); if(ret==log_msg->result[i].serv_def_len) { - TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)service_defined, TLD_TYPE_STRING); + TLD_append(_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)service_defined, TLD_TYPE_STRING); } else { @@ -481,20 +493,15 @@ int tsg_send_log(tsg_log_instance_t instance, TLD_handle_t handle, tsg_log_t *lo free(payload); payload=NULL; - TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name); - TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_SERVICE].name); - TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ACTION].name); - TLD_delete((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_USER_REGION].name); + TLD_delete(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name); + TLD_delete(_handle, _instance->id2field[LOG_COMMON_SERVICE].name); + TLD_delete(_handle, _instance->id2field[LOG_COMMON_ACTION].name); + TLD_delete(_handle, _instance->id2field[LOG_COMMON_USER_REGION].name); FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LOG], 0, FS_OP_ADD, 1); } - - cJSON_Delete(_handle->object); - _handle->object=NULL; - - free(handle); - handle=NULL; + TLD_cancel(handle); return 0; } diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index 1c1d937..7f05dd5 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -69,13 +69,13 @@ typedef struct _id2field char name[MAX_STRING_LEN]; }id2field_t; -struct _tld_handle +struct TLD_handle_t { int thread_id; cJSON *object; }; -struct _tsg_log_instance +struct tsg_log_instance_t { int mode; int max_service; @@ -88,8 +88,8 @@ struct _tsg_log_instance id2field_t *service2topic; }; -char *log_field_id2name(tsg_log_instance_t instance, tsg_log_field_id_t id); -tsg_log_instance_t tsg_sendlog_init(const char *filename); +char *log_field_id2name(struct tsg_log_instance_t *instance, tsg_log_field_id_t id); +struct tsg_log_instance_t *tsg_sendlog_init(const char *filename); #endif From ff053e3e2cad06804f852e8b46275865f4e85b0c Mon Sep 17 00:00:00 2001 From: liuxueli Date: Wed, 27 Nov 2019 11:38:35 +0800 Subject: [PATCH 30/32] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=B0=83=E8=AF=95?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_send_log.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 217da6a..44b8360 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -147,7 +147,13 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle struct tsg_log_instance_t *_instance=instance; if(_instance==NULL || _handle==NULL || a_stream==NULL) - { + { + MESA_handle_runtime_log(_instance->logger, + ((a_stream==NULL) ? RLOG_LV_DEBUG: RLOG_LV_FATAL), + "TLD_APPEND_STREAM", + "instance==NULL || TLD_handle==NULL || addr: %s", + ((a_stream==NULL) ? "NULL": (printaddr(&a_stream->addr, a_stream->threadnum))) + ); return -1; } From 91574444c3950c37b0147ad923f8eef08e029ca8 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Tue, 3 Dec 2019 11:44:16 +0800 Subject: [PATCH 31/32] =?UTF-8?q?=E5=85=AC=E5=85=B1=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E9=80=9A=E4=BF=A1=E5=B8=82=E6=97=B6=E9=95=BF?= =?UTF-8?q?=E5=8D=95=E4=BD=8D=E6=98=AF=EF=BC=9Ams=EF=BC=9B=E8=BD=AC?= =?UTF-8?q?=E6=8D=A2=E5=8D=95=E4=BD=8D=20KNI=E5=8F=91=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E9=9C=80=E7=AD=89=E5=BE=85TFE=E7=BB=93=E6=9D=9F=EF=BC=8C?= =?UTF-8?q?=E9=9C=80=E6=8F=90=E5=89=8D=E6=9E=84=E9=80=A0streaminfo?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=AD=97=E6=AE=B5=EF=BC=8C=E8=B0=83=E6=95=B4?= =?UTF-8?q?=E5=8C=85=E6=95=B0=E5=AD=97=E8=8A=82=E6=95=B0=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=20=E7=BB=86=E5=8C=96=E8=B0=83=E6=95=B4?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/tsg_entry.cpp | 40 +++++++++++++++++++++++++++++++--------- src/tsg_entry.h | 2 ++ src/tsg_rule.cpp | 8 +++++++- src/tsg_send_log.cpp | 39 +++++++++++++++++++-------------------- 4 files changed, 59 insertions(+), 30 deletions(-) diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 2972bb5..b5434d2 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -14,7 +14,7 @@ -char TSG_MASTER_VERSION_20191126=0; +char TSG_MASTER_VERSION_20191129=0; const char *tsg_conffile="tsgconf/main.conf"; g_tsg_para_t g_tsg_para; @@ -23,7 +23,8 @@ id2field_t g_tsg_fs2_field[TSG_FS2_MAX]={{TLD_TYPE_UNKNOWN, TSG_FS2_LINKS, "link {TLD_TYPE_UNKNOWN, TSG_FS2_HIT_ADDR, "hit_addr"}, {TLD_TYPE_UNKNOWN, TSG_FS2_HIT_SHARE, "hit_share"}, {TLD_TYPE_UNKNOWN, TSG_FS2_INTERCEPT, "intercept"}, - {TLD_TYPE_UNKNOWN, TSG_FS2_LOG, "log"} + {TLD_TYPE_UNKNOWN, TSG_FS2_LOG, "log"}, + {TLD_TYPE_UNKNOWN, TSG_FS2_DENY, "deny"} }; static void free_policy_label(int thread_seq, void *project_req_value) { @@ -79,7 +80,7 @@ static int is_ip_policy(Maat_rule_t *p_result, char *protocol, int len, int thre return ret; } #endif -static Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result_num) +static struct Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result_num) { int i=0; Maat_rule_t *p_result=NULL; @@ -136,8 +137,22 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t { hit_num+=ret; q_result=tsg_policy_decision_criteria(all_result, hit_num); - FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1); + FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1); + MESA_handle_runtime_log(g_tsg_para.logger, + RLOG_LV_DEBUG, + "SCAN_IP", + "Hit addr: %s scan ret: %d policy_id: %d service: %d action: %d", + printaddr(&a_tcp->addr, thread_seq), + ret, + q_result->config_id, + q_result->service_id, + q_result->action); } + else + { + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "SCAN_IP", "Not hit %s scan ret: %d", printaddr(&a_tcp->addr, thread_seq), ret); + } + memset(&identify_info, 0, sizeof(identify_info)); @@ -152,24 +167,28 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t thread_seq); if(ret>0) { - hit_num+=ret; - identify_flag=1; FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1); MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, - "SCAN_IP_AND_FQDN", - "Hit %s: %s addr: %s", + "SCAN_FQDN", + "Hit %s: %s policy_id: %d service: %d action: %d addr: %s", (identify_info.proto==PROTO_HTTP) ? "host" : "sni", identify_info.domain, + all_result[hit_num].config_id, + all_result[hit_num].service_id, + all_result[hit_num].action, printaddr(&a_tcp->addr, thread_seq) ); + + hit_num+=ret; + identify_flag=1; } else { MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, - "SCAN_IP_AND_FQDN", + "SCAN_FQDN", "Not hit %s: %s addr: %s", (ret==-1) ? "NULL" : ((identify_info.proto==PROTO_HTTP) ? "host" : "sni"), (ret==-1) ? "NULL" : identify_info.domain, @@ -187,6 +206,9 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t send_log=1; MESA_kill_tcp(a_tcp, a_packet); state|=APP_STATE_DROPPKT|APP_STATE_KILL_OTHER; + FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_DENY], 0, FS_OP_ADD, 1); + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "DENY", "Hit deny policy, policy_id: %d action: %d addr: %s", + p_result[0].config_id, p_result[0].action, printaddr(&a_tcp->addr, thread_seq)); break; case TSG_ACTION_MONITOR: if(q_result!=NULL && (p_result==q_result)) diff --git a/src/tsg_entry.h b/src/tsg_entry.h index 5f5b5c1..20d15e8 100644 --- a/src/tsg_entry.h +++ b/src/tsg_entry.h @@ -12,6 +12,7 @@ #define TSG_ACTION_MANIPULATE 0x30 #define TSG_ACTION_BYPASS 0x80 + enum TSG_FS2_TYPE{ TSG_FS2_LINKS=0, TSG_FS2_BYPASS, @@ -19,6 +20,7 @@ enum TSG_FS2_TYPE{ TSG_FS2_HIT_SHARE, TSG_FS2_INTERCEPT, TSG_FS2_LOG, + TSG_FS2_DENY, TSG_FS2_MAX }; diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 454f70a..78feef4 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -372,7 +372,7 @@ struct Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num) for(i=0; i< result_num; i++) { - if(result[i].action==TSG_ACTION_DENY) + if(result[i].action==TSG_ACTION_DENY || result[i].action==TSG_ACTION_BYPASS) { if(p_result==NULL) { @@ -381,6 +381,12 @@ struct Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num) } if(result[i].action > p_result->action) + { + p_result=&result[i]; + continue; + } + + if((result[i].action==p_result->action) && (result[i].config_id > p_result->config_id)) { p_result=&result[i]; } diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 44b8360..3f03a1d 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -19,7 +19,7 @@ #include "tsg_send_log.h" #include "tsg_send_log_internal.h" -char TSG_SEND_LOG_VERSION_20191121=0; +char TSG_SEND_LOG_VERSION_20191129=0; struct tsg_log_instance_t *g_tsg_log_instance; @@ -135,6 +135,8 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle unsigned short tunnel_type=0; char nest_addr_buf[1024]; char *addr_proto=NULL; + time_t cur_time; + long common_con_duration_ms=0; unsigned long long stream_id=0; unsigned short c_port=0, s_port=0; int tunnel_type_size=sizeof(tunnel_type); @@ -197,6 +199,22 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)a_stream->ptcpdetail->serverpktnum, TLD_TYPE_LONG); TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)a_stream->ptcpdetail->serverbytes, TLD_TYPE_LONG); + if(a_stream!=NULL && a_stream->ptcpdetail!=NULL) + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(a_stream->ptcpdetail->createtime), TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG); + + common_con_duration_ms=(a_stream->ptcpdetail->lastmtime-a_stream->ptcpdetail->createtime)*1000; + TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); + } + else + { + cur_time=time(NULL); + TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)cur_time, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)cur_time, TLD_TYPE_LONG); + TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); + } + stream_id=tsg_get_stream_id(a_stream); TLD_append(_handle, _instance->id2field[LOG_COMMON_STREAM_TRACE_ID].name, (void *)(long)stream_id, TLD_TYPE_LONG); @@ -361,8 +379,6 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl { int i=0,ret=0,status=0; char *payload=NULL; - time_t cur_time; - long common_con_duration_ms=0; struct TLD_handle_t *_handle=handle; struct tsg_log_instance_t *_instance=instance; @@ -390,25 +406,8 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl //common_stream_error TLD_append_streaminfo(instance, handle, log_msg->a_stream); - TLD_append(_handle, _instance->id2field[LOG_COMMON_SLED_IP].name, (void *)(_instance->local_ip_str), TLD_TYPE_STRING); - if(log_msg->a_stream!=NULL && log_msg->a_stream->ptcpdetail!=NULL) - { - TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->createtime), TLD_TYPE_LONG); - TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(log_msg->a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG); - - common_con_duration_ms=log_msg->a_stream->ptcpdetail->lastmtime-log_msg->a_stream->ptcpdetail->createtime; - TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); - } - else - { - cur_time=time(NULL); - TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)cur_time, TLD_TYPE_LONG); - TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)cur_time, TLD_TYPE_LONG); - TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG); - } - #if 0 struct vxlan_info vinfo; int opt_val_len = sizeof(vinfo); From 8cbf0f29a17554ef0cd05f6b224051b4757e9c10 Mon Sep 17 00:00:00 2001 From: liuxueli Date: Tue, 10 Dec 2019 18:36:53 +0800 Subject: [PATCH 32/32] =?UTF-8?q?=E5=A2=9E=E5=8A=A0CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitlab-ci.yml | 119 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..0b90ff4 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,119 @@ +image: "docker:stable" + +services: + - docker:dind + +variables: + GIT_STRATEGY: "clone" + IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + + +stages: + - build + - package + - release + - docker + +.build_before_script: + before_script: + - mkdir -p /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/$CI_PROJECT_NAMESPACE/ + - ln -s $CI_PROJECT_DIR /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/$CI_PROJECT_PATH + - cd /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/$CI_PROJECT_PATH + +.build_master: + extends: .build_before_script + image: git.mesalab.cn:7443/mesa_platform/mesa_protocol:master + script: + - source /etc/profile.d/MESA.sh + - mkdir -p build + - cd build + - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE + - make + - make package + +.package_master: + extends: .build_before_script + image: git.mesalab.cn:7443/mesa_platform/mesa_protocol:master + script: + - source /etc/profile.d/MESA.sh + - mkdir -p build + - cd build + - cmake .. -DCMAKE_BUILD_TYPE=$BUILD_TYPE + - make package + +build: + stage: build + extends: .build_master + tags: + - share + variables: + BUILD_TYPE: "Debug" + except: + - tags + +tsg-master-release: + stage: package + extends: .package_master + tags: + - share + variables: + BUILD_TYPE: "Release" + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" + paths: + - ./build/*.rpm + only: + - tags + +tsg-master-debug: + stage: package + extends: .package_master + tags: + - share + variables: + BUILD_TYPE: "Debug" + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME" + paths: + - ./build/*.rpm + only: + - tags + +release-tsg-master-debug: + stage: release + image: git.mesalab.cn:7443/mesa_platform/mesa_protocol:master + tags: + - share + variables: + ARTIFACTS_JOB: "tsg-master-debug" + PROJECT_NAME: "tsg-master" + USER_DEFINE: "debug" + only: + - tags + script: + - /bin/bash -x ./autorelease.sh $CI_API_V4_URL $CI_PROJECT_URL $CI_PROJECT_ID $CI_TOKEN $CI_COMMIT_TAG $ARTIFACTS_JOB $PROJECT_NAME $USER_DEFINE + +release-tsg-master-release: + stage: release + image: git.mesalab.cn:7443/mesa_platform/mesa_protocol:master + tags: + - share + variables: + ARTIFACTS_JOB: "tsg-master-release" + PROJECT_NAME: "tsg-master" + USER_DEFINE: "release" + only: + - tags + script: + - /bin/bash -x ./autorelease.sh $CI_API_V4_URL $CI_PROJECT_URL $CI_PROJECT_ID $CI_TOKEN $CI_COMMIT_TAG $ARTIFACTS_JOB $PROJECT_NAME $USER_DEFINE + +build-docker: + stage: docker + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker build -t $IMAGE_TAG . + - docker push $IMAGE_TAG + tags: + - share + only: + - master \ No newline at end of file