gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持同步接收APP的识别结果

Browse Source 
支持一个流具有多个APP属性
适配新的APP ID
This commit is contained in:
刘学利
2021-04-27 09:49:15 +00:00
parent 1a4d35dec8
commit d639394077
12 changed files with 2346 additions and 1172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitlab-ci.yml
View File

@@ -3,7 +3,7 @@ variables:
GIT_STRATEGY: "clone" GIT_STRATEGY: "clone"
BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/ BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/
INSTALL_PREFIX: "/home/mesasoft/sapp_run/" INSTALL_PREFIX: "/home/mesasoft/sapp_run/"
INSTALL_DEPENDENCY_LIBRARY: libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaatframe-devel quic-devel libasan INSTALL_DEPENDENCY_LIBRARY: libMESA_handle_logger-devel libcjson-devel libMESA_field_stat2-devel sapp sapp-devel framework_env libMESA_prof_load-devel http-devel dns-devel ftp-devel mail-devel ssl-devel librdkafka-devel libmaatframe-devel quic-devel mesa_sip-devel libasan
stages: stages:
- build - build

3
bin/app_l7_proto_id.conf
View File

@@ -49,3 +49,6 @@ STRING BJNP 147
STRING LDAP 148 STRING LDAP 148
STRING RTMP 149 STRING RTMP 149
STRING RTSP 150 STRING RTSP 150
STRING POP3 116
STRING SMTP 122
STRING IMAP 151

1
bin/main.conf
View File

@@ -31,4 +31,5 @@ ENTRANCE_ID=18
LOG_LEVEL=10 LOG_LEVEL=10
LOG_PATH=./tsglog/tsg_master LOG_PATH=./tsglog/tsg_master
POLICY_PRIORITY_LABEL=POLICY_PRIORITY POLICY_PRIORITY_LABEL=POLICY_PRIORITY
L7_RPTOCOL_FILE="./tsgconf/tsg_l7_protocol.conf"
DEVICE_ID_COMMAND=hostname | awk -F'-' '{print $3}'| awk -F'ADC' '{print $2}' DEVICE_ID_COMMAND=hostname | awk -F'-' '{print $3}'| awk -F'ADC' '{print $2}'

55
bin/tsg_l7_protocol.conf Normal file
View File

@@ -0,0 +1,55 @@
#TYPE：1:UCHAR,2:USHORT,3:USTRING,4:ULOG,5:USTRING,6:FILE,7:UBASE64,8:PACKET
#TYPE FIELD VALUE
STRING UNCATEGORIZED 15001
STRING UNCATEGORIZED 15002
STRING UNKNOWN_OTHER 15003
STRING DNS 32
STRING FTP 45
STRING FTPS 751
STRING HTTP 67
STRING HTTPS 68
STRING ICMP 70
STRING IKE 15004
STRING MAIL 15005
STRING IMAP 75
STRING IMAPS 76
STRING IPSEC 85
STRING XMPP 94
STRING L2TP 98
STRING NTP 137
STRING POP3 147
STRING POP3S 148
STRING PPTP 153
STRING QUIC 2521
STRING SIP 182
STRING SMB 185
STRING SMTP 186
STRING SMTPS 187
STRING SPDY 1469
STRING SSH 198
STRING SSL 199
STRING SOCKS 15006
STRING TELNET 209
STRING DHCP 29
STRING RADIUS 158
STRING OPENVPN 336
STRING STUN 201
STRING TEREDO 555
STRING DTLS 1291
STRING DoH 15007
STRING ISAKMP 92
STRING MDNS 3835
STRING NETBIOS 129
STRING NETFLOW 130
STRING RDP 150
STRING RTCP 174
STRING RTP 175
STRING SLP 15008
STRING SNMP 190
STRING SSDP 197
STRING TFTP 211
STRING BJNP 2481
STRING LDAP 100
STRING RTMP 337
STRING RTSP 176
STRING ESNI 15009

32
inc/app_label.h
View File

@@ -1,17 +1,39 @@
#ifndef __APP_LABEL_H__ #ifndef __APP_LABEL_H__
#define __APP_LABEL_H__ #define __APP_LABEL_H__
#define MAX_APP_ID_NUM 8
struct app_id_label struct app_id_label
{ {
int surrogate_id; int app_id_num;
int app_id; unsigned int app_id[MAX_APP_ID_NUM];
unsigned int surrogate_id[MAX_APP_ID_NUM];
}; };
struct basic_proto_label struct basic_proto_label
{ {
unsigned char continue_scan_flag; //0: stop; 1: continue int continue_scan_flag; //0: stop; 1: continue
unsigned char pad; int protocol_id_num;
unsigned short proto_id; unsigned short protocol_id[MAX_APP_ID_NUM];
};
enum APP_IDENTIFY_ORIGIN
{
ORIGIN_BASIC_PROTOCOL,
ORIGIN_USER_DEFINE,
ORIGIN_DKPT,
ORIGIN_QM_ENGINE,
ORIGIN_MAX
};
struct app_identify_result
{
enum APP_IDENTIFY_ORIGIN origin;
int app_id_num;
unsigned int app_id[MAX_APP_ID_NUM];
unsigned int surrogate_id[MAX_APP_ID_NUM];
}; };
enum _ATTRIBUTE_TYPE enum _ATTRIBUTE_TYPE

1
inc/tsg_send_log.h
View File

@@ -36,6 +36,5 @@ int TLD_cancel(struct TLD_handle_t *handle);
int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, tsg_log_t *log_msg, int thread_id); int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, tsg_log_t *log_msg, int thread_id);
unsigned long long tsg_get_stream_id(struct streaminfo *a_stream); unsigned long long tsg_get_stream_id(struct streaminfo *a_stream);
char *tsg_l7_protocol_id2name(struct tsg_log_instance_t *instance, unsigned short id);
#endif #endif

1131
src/tsg_entry.cpp
View File

File diff suppressed because it is too large Load Diff

53
src/tsg_entry.h
View File

@@ -3,7 +3,10 @@
#include <MESA/Maat_rule.h> #include <MESA/Maat_rule.h>
#include <MESA/field_stat2.h> #include <MESA/field_stat2.h>
#include "uthash.h"
#include "tsg_rule.h" #include "tsg_rule.h"
#include "app_label.h"
#include "tsg_label.h" #include "tsg_label.h"
#include "tsg_statistic.h" #include "tsg_statistic.h"
@@ -28,15 +31,6 @@ typedef int atomic_t;
#define PRINTADDR(a, b) ((b)<RLOG_LV_FATAL ? printaddr(&(a->addr), a->threadnum) : "") #define PRINTADDR(a, b) ((b)<RLOG_LV_FATAL ? printaddr(&(a->addr), a->threadnum) : "")
#endif #endif
#define DNS_PROTO_ID 103
#define FTP_PROTO_ID 104
#define HTTP_PROTO_ID 106
#define MAIL_PROTO_ID 110
#define QUIC_PROTO_ID 119
#define SIP_PROTO_ID 120
#define SSL_PROTO_ID 126
#define RTP_PROTO_ID 142
#define APP_SCAN_FLAG_STOP 0 #define APP_SCAN_FLAG_STOP 0
#define APP_SCAN_FLAG_CONTINUE 1 #define APP_SCAN_FLAG_CONTINUE 1
@@ -101,6 +95,20 @@ struct _str2index
char *type; char *type;
}; };
struct gather_app_result
{
struct app_identify_result result[ORIGIN_MAX];
};
struct l7_protocol
{
int id; /* first key */
char name[32]; /* second key */
UT_hash_handle hh1; /* handle for first hash table */
UT_hash_handle hh2; /* handle for second hash table */
};
struct _fqdn_category_t struct _fqdn_category_t
{ {
int ref_cnt; int ref_cnt;
@@ -114,14 +122,13 @@ struct master_context
tsg_protocol_t proto; tsg_protocol_t proto;
int hit_cnt; int hit_cnt;
int app_id; int app_id;
int is_default_policy; int is_esni;
char continue_scan_app_id;
char continue_scan_proto_id; char continue_scan_proto_id;
unsigned short basic_proto_id; unsigned short basic_proto_id;
int domain_len; char *domain;
char domain[MAX_DOAMIN_LEN];
struct Maat_rule_t *result; struct Maat_rule_t *result;
scan_status_t mid; scan_status_t mid;
struct timespec last_scan_time;
}; };
#define _MAX_TABLE_NAME_LEN 64 #define _MAX_TABLE_NAME_LEN 64
@@ -130,18 +137,19 @@ typedef struct _tsg_para
int level; int level;
int mail_proto_id; int mail_proto_id;
unsigned short timeout; unsigned short timeout;
unsigned short depolyment_mode;
int app_id_table_type; int app_id_table_type;
int device_id; int device_id;
int entrance_id; int entrance_id;
int scan_time_interval;
int default_compile_switch; int default_compile_switch;
int default_compile_id; int default_compile_id;
int table_id[TABLE_MAX]; int table_id[TABLE_MAX];
int dyn_subscribe_ip_table_id; //TSG_DYN_SUBSCRIBER_IP int dyn_subscribe_ip_table_id; //TSG_DYN_SUBSCRIBER_IP
int priority_project_id; int priority_project_id;
int internal_project_id; int internal_project_id;
int l7_proto_project_id; int context_project_id;
int app_id_project_id; int gather_app_project_id;
int app_bridge_id;
int proto_flag; //tsg_protocol_t int proto_flag; //tsg_protocol_t
int fs2_field_id[TSG_FS2_MAX]; int fs2_field_id[TSG_FS2_MAX];
char device_sn[MAX_DOAMIN_LEN/8]; char device_sn[MAX_DOAMIN_LEN/8];
@@ -151,6 +159,8 @@ typedef struct _tsg_para
char table_name[TABLE_MAX][_MAX_TABLE_NAME_LEN]; char table_name[TABLE_MAX][_MAX_TABLE_NAME_LEN];
void *logger; void *logger;
screen_stat_handle_t fs2_handle; screen_stat_handle_t fs2_handle;
struct l7_protocol *name_by_id;
struct l7_protocol *id_by_name;
}g_tsg_para_t; }g_tsg_para_t;
extern g_tsg_para_t g_tsg_para; extern g_tsg_para_t g_tsg_para;
@@ -244,11 +254,14 @@ void location_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void*
void ASN_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp); void ASN_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp);
void subscribe_id_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp); void subscribe_id_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp);
void security_compile_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp); void security_compile_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp);
char *tsg_schema_index2string(tsg_protocol_t proto); struct Maat_rule_t *tsg_policy_decision_criteria(struct streaminfo *a_stream, Maat_rule_t *result, int result_num, int thread_seq);
struct Maat_rule_t *tsg_policy_decision_criteria(struct streaminfo *a_stream, Maat_rule_t *result, int result_num, struct _identify_info *identify_info, int thread_seq); int tsg_scan_shared_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, char *domain, Maat_rule_t *result, int result_num, scan_status_t *mid, int table_id, int thread_seq);
int tsg_scan_shared_policy(Maat_feather_t maat_feather, struct streaminfo *a_stream, struct identify_info *identify_info, Maat_rule_t *result, int result_num, scan_status_t *mid, int thread_seq);
int tsg_scan_app_id_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *name, unsigned int id, int thread_seq); int tsg_scan_app_id_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *name, unsigned int id, int thread_seq);
int tsg_scan_app_properties_policy(Maat_feather_t maat_feather, struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *property, char *district, int thread_seq); int tsg_scan_app_properties_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *property, char *district, int thread_seq);
int tsg_scan_subscribe_id_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, struct _subscribe_id_info_t *user_info, int thread_seq); int tsg_scan_subscribe_id_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, struct _subscribe_id_info_t *user_info, int thread_seq);
int tsg_scan_fqdn_category_id(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, int table_id, unsigned int *category_id, int category_id_num, int thread_seq);
unsigned int tsg_l7_protocol_name2id(const char *l7_protocol_name);
char *tsg_l7_protocol_id2name(unsigned int l7_protocol_id);
#endif #endif

213
src/tsg_rule.cpp
View File

@@ -12,6 +12,7 @@
#include "Maat_command.h" #include "Maat_command.h"
#include "MESA/http.h" #include "MESA/http.h"
#include "tsg_rule.h" #include "tsg_rule.h"
#include "tsg_label.h"
#include "tsg_entry.h" #include "tsg_entry.h"
#include "tsg_send_log.h" #include "tsg_send_log.h"
#include "tsg_send_log_internal.h" #include "tsg_send_log_internal.h"
@@ -76,35 +77,6 @@ static char* str_unescape(char* s)
return s; return s;
} }
static int proto_str2id(tsg_protocol_t proto)
{
switch(proto)
{
case PROTO_TCP: return 100;
case PROTO_UDP: return 101;
case PROTO_HTTP: return 106;
case PROTO_MAIL: return 110;
case PROTO_IMAP: return 151;
case PROTO_POP3: return 116;//116
case PROTO_SMTP: return 122;//122
case PROTO_DNS: return 103;
case PROTO_FTP: return 104;
case PROTO_SSL: return 126;
case PROTO_SIP: return 120;
case PROTO_QUIC: return 119;
case PROTO_SSH: return 125;
case PROTO_RTP: return 142;
case PROTO_IPv6:
case PROTO_IPv4:
case PROTO_STREAMING_MEDIA:
case PROTO_BGP:
default:
break;
}
return 0;
}
static int get_data_center(char *accept_tag, char *effective_tag_key, char *data_center, int data_center_len) static int get_data_center(char *accept_tag, char *effective_tag_key, char *data_center, int data_center_len)
{ {
int i=0,len; int i=0,len;
@@ -954,18 +926,20 @@ int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_re
{ {
if(label->result_type==pull_result_type) if(label->result_type==pull_result_type)
{ {
num=(label->result_num>result_num) ? result_num : label->result_num; num=MIN(label->result_num, result_num);
memcpy(result, label->result, num*sizeof(Maat_rule_t)); memcpy(result, label->result, num*sizeof(Maat_rule_t));
if(label->domain_len>0)
{
memcpy(identify_info->domain, label->domain, label->domain_len); memcpy(identify_info->domain, label->domain, label->domain_len);
identify_info->domain_len=label->domain_len; identify_info->domain_len=label->domain_len;
}
identify_info->proto = label->proto; identify_info->proto = label->proto;
return num; return num;
} }
else
{
MESA_handle_runtime_log(g_tsg_para.logger, MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG, RLOG_LV_DEBUG,
"PULL_RESULT", "PULL_RESULT",
@@ -979,17 +953,6 @@ int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_re
PRINTADDR(a_stream, g_tsg_para.level) PRINTADDR(a_stream, g_tsg_para.level)
); );
} }
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"PULL_RESULT",
"pull policy failed, Not hit, label is %s addr: %s",
(label==NULL) ? "NULL" : label->domain,
PRINTADDR(a_stream, g_tsg_para.level)
);
}
return 0; return 0;
} }
@@ -1192,8 +1155,8 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num) int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num)
{ {
int ret=0; int ret=0;
struct ipaddr t_addr;
unsigned int proto_id=0; unsigned int proto_id=0;
struct ipaddr t_addr;
struct ipaddr* p_addr=NULL; struct ipaddr* p_addr=NULL;
int hit_num=0,tans_proto=0; int hit_num=0,tans_proto=0;
int is_scan_addr=1, maat_ret=0; int is_scan_addr=1, maat_ret=0;
@@ -1274,11 +1237,11 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
if(hit_num<result_num && proto>PROTO_UNKONWN && proto<PROTO_MAX) if(hit_num<result_num && proto>PROTO_UNKONWN && proto<PROTO_MAX)
{ {
proto_id=proto_str2id(proto); proto_id=tsg_l7_protocol_name2id(g_tsg_proto_name2id[proto].name);
hit_num+=tsg_scan_app_id_policy(maat_feather, a_stream, result+hit_num, result_num-hit_num, mid, g_tsg_proto_name2id[proto].name, proto_id, (int)a_stream->threadnum); hit_num+=tsg_scan_app_id_policy(maat_feather, a_stream, result+hit_num, result_num-hit_num, mid, g_tsg_proto_name2id[proto].name, proto_id, (int)a_stream->threadnum);
if(proto==PROTO_SMTP || proto==PROTO_IMAP || proto==PROTO_POP3) if(proto==PROTO_SMTP || proto==PROTO_IMAP || proto==PROTO_POP3)
{ {
proto_id=proto_str2id(PROTO_MAIL); proto_id=tsg_l7_protocol_name2id(g_tsg_proto_name2id[PROTO_MAIL].name);
hit_num+=tsg_scan_app_id_policy(maat_feather, a_stream, result+hit_num, result_num-hit_num, mid, g_tsg_proto_name2id[PROTO_MAIL].name, proto_id, (int)a_stream->threadnum); hit_num+=tsg_scan_app_id_policy(maat_feather, a_stream, result+hit_num, result_num-hit_num, mid, g_tsg_proto_name2id[PROTO_MAIL].name, proto_id, (int)a_stream->threadnum);
} }
} }
@@ -1332,133 +1295,36 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
//return value: -1: failed, 0: not hit, >0: hit count //return value: -1: failed, 0: not hit, >0: hit count
int tsg_scan_shared_policy(Maat_feather_t maat_feather, struct streaminfo *a_stream, struct identify_info *identify_info, Maat_rule_t *result, int result_num, scan_status_t *mid, int thread_seq) int tsg_scan_shared_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, char *domain, Maat_rule_t *result, int result_num, scan_status_t *mid, int table_id, int thread_seq)
{ {
char *field_name=NULL; int ret=0;
int i=0,ret=0,idx=0,hit_num=0;
struct _session_attribute_label_t *attribute_label=NULL;
if(identify_info->proto!=PROTO_UNKONWN && strlen(identify_info->domain)>0 && identify_info->domain_len>0) if(table_id<0 || domain==NULL)
{ {
switch(identify_info->proto)
{
case PROTO_HTTP:
idx=TABLE_HTTP_HOST;
field_name=(char *)"http_host";
break;
case PROTO_SSL:
idx=TABLE_SSL_SNI;
field_name=(char *)"ssl_sni";
break;
case PROTO_QUIC:
idx=TABLE_QUIC_SNI;
field_name=(char *)"quic_sni";
break;
default:
return 0; return 0;
break;
} }
ret=Maat_full_scan_string(g_tsg_maat_feather, ret=Maat_full_scan_string(g_tsg_maat_feather, table_id, CHARSET_UTF8, domain, strlen(domain), result, NULL, result_num, mid, thread_seq);
g_tsg_para.table_id[idx],
CHARSET_UTF8,
identify_info->domain,
identify_info->domain_len,
result,
NULL,
result_num,
mid,
thread_seq
);
if(ret>0) if(ret>0)
{ {
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1); FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1);
MESA_handle_runtime_log(g_tsg_para.logger, MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG, RLOG_LV_DEBUG,
"SCAN_FQDN", "SCAN_FQDN",
"Hit %s: %s policy_id: %d service: %d action: %d addr: %s", "Hit %s policy_id: %d service: %d action: %d addr: %s",
field_name, domain,
identify_info->domain, result[0].config_id,
result[hit_num].config_id, result[0].service_id,
result[hit_num].service_id, (unsigned char)result[0].action,
(unsigned char)result[hit_num].action,
PRINTADDR(a_stream, g_tsg_para.level) PRINTADDR(a_stream, g_tsg_para.level)
); );
hit_num+=ret; return ret;
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_FQDN",
"Not hit %s: %s ret: %d stream_dir: %d addr: %s",
(ret==-1) ? "NULL" : ((identify_info->proto==PROTO_HTTP) ? "host" : "sni"),
(ret==-1) ? "NULL" : identify_info->domain,
ret,
a_stream->dir,
PRINTADDR(a_stream, g_tsg_para.level)
);
} }
attribute_label=(struct _session_attribute_label_t *)project_req_get_struct(a_stream, g_tsg_para.internal_project_id); MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "SCAN_FQDN", "Not hit %s ret: %d stream_dir: %d addr: %s", domain, ret, a_stream->dir, PRINTADDR(a_stream, g_tsg_para.level));
if(attribute_label!=NULL)
{
attribute_label->fqdn_category_id_num=tsg_get_fqdn_category_id(g_tsg_maat_feather,
identify_info->domain,
attribute_label->fqdn_category_id,
MAX_CATEGORY_ID_NUM,
g_tsg_para.logger,
thread_seq
);
return 0;
for(i=0; i<attribute_label->fqdn_category_id_num; i++)
{
int idx=identify_info->proto==PROTO_HTTP ? TABLE_HTTP_HOST : TABLE_SSL_SNI;
ret=Maat_scan_intval(g_tsg_maat_feather,
g_tsg_para.table_id[idx],
(unsigned int)attribute_label->fqdn_category_id[i],
result+hit_num,
result_num-hit_num,
mid,
thread_seq
);
if(ret>0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_FQDN_CAT",
"Hit %s: %s category_id: %d policy_id: %d service: %d action: %d addr: %s",
(identify_info->proto==PROTO_HTTP) ? "host" : "sni",
identify_info->domain,
attribute_label->fqdn_category_id[i],
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
PRINTADDR(a_stream, g_tsg_para.level)
);
hit_num+=ret;
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_FQDN_CAT",
"Not hit %s: %s category_id: %d ret: %d stream_dir: %d addr: %s",
(ret==-1) ? "NULL" : ((identify_info->proto==PROTO_HTTP) ? "host" : "sni"),
(ret==-1) ? "NULL" : identify_info->domain,
attribute_label->fqdn_category_id[i],
ret,
a_stream->dir,
PRINTADDR(a_stream, g_tsg_para.level)
);
}
}
}
}
return hit_num;
} }
@@ -1551,6 +1417,41 @@ int tsg_get_fqdn_category_id(Maat_feather_t maat_feather, char *fqdn, unsigned i
return 0; return 0;
} }
int tsg_scan_fqdn_category_id(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, int table_id, unsigned int *category_id, int category_id_num, int thread_seq)
{
int i=0,ret=0,hit_num=0;
if(table_id<0 || result_num<=0 || category_id==NULL)
{
return 0;
}
for(i=0; i<category_id_num; i++)
{
ret=Maat_scan_intval(g_tsg_maat_feather, table_id, (unsigned int)category_id[i], result+hit_num, result_num-hit_num, mid, thread_seq);
if(ret>0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_FQDN_CAT",
"Hit category_id: %d policy_id: %d service: %d action: %d addr: %s",
category_id[i],
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
PRINTADDR(a_stream, g_tsg_para.level)
);
hit_num+=ret;
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "SCAN_FQDN_CAT", "Not hit category_id: %d ret: %d addr: %s", category_id[i], ret, PRINTADDR(a_stream, g_tsg_para.level));
}
}
return hit_num;
}
int tsg_scan_app_id_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *name, unsigned int id, int thread_seq) int tsg_scan_app_id_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *name, unsigned int id, int thread_seq)
{ {
@@ -1580,7 +1481,7 @@ int tsg_scan_app_id_policy(Maat_feather_t maat_feather, const struct streaminfo
return 0; return 0;
} }
int tsg_scan_app_properties_policy(Maat_feather_t maat_feather, struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *property, char *district, int thread_seq) int tsg_scan_app_properties_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *property, char *district, int thread_seq)
{ {
int i=0,ret=0; int i=0,ret=0;
int ret2=0, hit_num=0; int ret2=0, hit_num=0;

750
src/tsg_send_log.cpp
View File

@@ -73,7 +73,346 @@ static int is_tunnels(struct streaminfo *a_stream)
return is_tunnel; return is_tunnel;
} }
static int convert_mac_to_string(unsigned char *mac, char *buff) static int set_isn(struct TLD_handle_t *_handle, struct streaminfo *a_stream, char *field_name, enum MESA_stream_opt type)
{
int ret=0;
unsigned int isn=0;
int size=sizeof(unsigned long long);
size=sizeof(unsigned int);
ret=MESA_get_stream_opt(a_stream, type, &isn, &size);
if(ret==0)
{
TLD_append(_handle, field_name, (void *)(long)isn, TLD_TYPE_LONG);
}
return 1;
}
static int set_tcp_isn(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{
if(a_stream->type==STREAM_TYPE_TCP)
{
switch(a_stream->dir)
{
case DIR_C2S:
set_isn(_handle, a_stream, _instance->id2field[LOG_COMMON_TCP_CLIENT_ISN].name, MSO_TCP_ISN_C2S);
break;
case DIR_S2C:
set_isn(_handle, a_stream, _instance->id2field[LOG_COMMON_TCP_SERVER_ISN].name, MSO_TCP_ISN_S2C);
break;
case DIR_DOUBLE:
set_isn(_handle, a_stream, _instance->id2field[LOG_COMMON_TCP_CLIENT_ISN].name, MSO_TCP_ISN_C2S);
set_isn(_handle, a_stream, _instance->id2field[LOG_COMMON_TCP_SERVER_ISN].name, MSO_TCP_ISN_S2C);
break;
default:
break;
}
}
return 1;
}
static int set_direction(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{
int direction=0,i_or_e=0;
i_or_e=MESA_dir_link_to_human(a_stream->routedir);
switch(a_stream->curdir)
{
case DIR_C2S:
if(i_or_e=='E' || i_or_e=='e')
{
direction='E';
}
else
{
direction='I';
}
break;
case DIR_S2C:
if(i_or_e=='E' || i_or_e=='e')
{
direction='I';
}
else
{
direction='E';
}
break;
default:
break;
}
TLD_append(_handle, _instance->id2field[LOG_COMMON_DIRECTION].name, (void *)(long)direction, TLD_TYPE_LONG);
return 1;
}
static int set_address_list(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{
int ret=0;
unsigned short tunnel_type=0;
char nest_addr_buf[1024];
int tunnel_type_size=sizeof(tunnel_type);
ret=MESA_get_stream_opt(a_stream, MSO_STREAM_TUNNEL_TYPE, &tunnel_type, &tunnel_type_size);
assert(ret==0);
if(tunnel_type==STREAM_TUNNLE_NON)
{
layer_addr_ntop_r(a_stream,nest_addr_buf, sizeof(nest_addr_buf));
}
else
{
stream_addr_list_ntop(a_stream,nest_addr_buf, sizeof(nest_addr_buf));
}
TLD_append(_handle, _instance->id2field[LOG_COMMON_ADDRESS_LIST].name, (void *)nest_addr_buf, TLD_TYPE_STRING);
return 1;
}
static int set_tuple4(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{
int addr_type=0;
unsigned short c_port=0, s_port=0;
struct layer_addr_ipv4 *ipv4=NULL;
struct layer_addr_ipv6 *ipv6=NULL;
char server_ip[MAX_IPV4_LEN*8]={0};
char client_ip[MAX_IPV4_LEN*8]={0};
switch(a_stream->addr.addrtype)
{
case ADDR_TYPE_IPV4:
case __ADDR_TYPE_IP_PAIR_V4:
ipv4=a_stream->addr.ipv4;
addr_type=4;
c_port=ntohs(ipv4->source);
s_port=ntohs(ipv4->dest);
inet_ntop(AF_INET, (void *)&ipv4->saddr, client_ip, sizeof(client_ip));
inet_ntop(AF_INET, (void *)&ipv4->daddr, server_ip, sizeof(server_ip));
break;
case ADDR_TYPE_IPV6:
case __ADDR_TYPE_IP_PAIR_V6:
ipv6=a_stream->addr.ipv6;
addr_type=6;
c_port=ntohs(ipv6->source);
s_port=ntohs(ipv6->dest);
inet_ntop(AF_INET6, (void *)ipv6->saddr, client_ip, sizeof(client_ip));
inet_ntop(AF_INET6, (void *)ipv6->daddr, server_ip, sizeof(server_ip));
break;
default:
break;
}
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_IP].name, (void *)server_ip, TLD_TYPE_STRING);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CLIENT_IP].name, (void *)client_ip, TLD_TYPE_STRING);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_PORT].name, (void *)(long)s_port, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CLIENT_PORT].name, (void *)(long)c_port, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)(long)a_stream->dir, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)(long)addr_type, TLD_TYPE_LONG);
return 1;
}
static int set_duraction(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{
int ret=0;
struct timespec tv;
long common_con_duration_ms=0;
unsigned long long create_time=0;
int size=sizeof(unsigned long long);
if(a_stream->ptcpdetail!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(a_stream->ptcpdetail->createtime), TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG);
ret=MESA_get_stream_opt(a_stream, MSO_STREAM_CREATE_TIMESTAMP_MS, (void *)&create_time, &size);
if(ret>=0)
{
clock_gettime(CLOCK_REALTIME, &tv);
common_con_duration_ms=tv.tv_sec*1000+tv.tv_nsec/1000/1000 - create_time;
}
if(common_con_duration_ms>0)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG);
}
}
else
{
time_t cur_time=time(NULL);
TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)cur_time, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)cur_time, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG);
}
return 1;
}
static int set_packet_bytes(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{
struct tcp_flow_stat *tflow_project=NULL;
struct udp_flow_stat *uflow_project=NULL;
switch(a_stream->type)
{
case STREAM_TYPE_TCP:
tflow_project=(struct tcp_flow_stat *)project_req_get_struct(a_stream, _instance->tcp_flow_project_id);
if(tflow_project!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)tflow_project->S2C_all_pkt, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)tflow_project->S2C_all_byte_raw, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)tflow_project->C2S_all_pkt, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)tflow_project->C2S_all_byte_raw, TLD_TYPE_LONG);
}
break;
case STREAM_TYPE_UDP:
uflow_project=(struct udp_flow_stat *)project_req_get_struct(a_stream, _instance->udp_flow_project_id);
if(uflow_project!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)uflow_project->S2C_pkt, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)uflow_project->S2C_all_byte_raw, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)uflow_project->C2S_pkt, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)uflow_project->C2S_all_byte_raw, TLD_TYPE_LONG);
}
break;
default:
break;
}
return 1;
}
static int get_l7_protocol(struct app_identify_result *result, char *protocol_list, int protocol_list_len, int *flag)
{
int i=0,offset=0;
char *name=NULL;
if((*flag)==1)
{
return 0;
}
for(i=0; i<result->app_id_num; i++)
{
(*flag)=1;
name=tsg_l7_protocol_id2name(result->app_id[i]);
if(name!=NULL)
{
offset+=snprintf(protocol_list+offset, protocol_list_len-offset, "%s", name);
}
}
return 1;
}
static int get_app_id_list(struct app_identify_result *result, char *app_list, int app_list_len, char *surrogate_list, int surrogate_list_len, int *flag)
{
int i=0;
int offset1=0,offset2=0;
if((*flag)==1)
{
return 0;
}
for(i=0; i<result->app_id_num; i++)
{
(*flag)=1;
offset1+=snprintf(app_list+offset1, app_list_len-offset1, "%d;", result->app_id[i]);
offset2+=snprintf(surrogate_list+offset2, surrogate_list_len-offset2, "%d;", result->surrogate_id[i]);
}
return 1;
}
static int set_app_id(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{
int app_id_flag=0;
int l7_protocol_flag=0;
char app_list[256]={0};
char protocol_list[256]={0};
char surrogate_list[256]={0};
struct gather_app_result *label=NULL;
label=(struct gather_app_result *)project_req_get_struct(a_stream, g_tsg_para.gather_app_project_id);
if(label!=NULL)
{
get_l7_protocol(&(label->result[ORIGIN_BASIC_PROTOCOL]), protocol_list, sizeof(protocol_list), &l7_protocol_flag);
get_app_id_list(&(label->result[ORIGIN_USER_DEFINE]), app_list, sizeof(app_list), surrogate_list, sizeof(surrogate_list), &app_id_flag);
if(app_id_flag!=1)
{
get_app_id_list(&(label->result[ORIGIN_DKPT]), app_list, sizeof(app_list), surrogate_list, sizeof(surrogate_list), &app_id_flag);
}
if(app_id_flag!=1)
{
get_app_id_list(&(label->result[ORIGIN_QM_ENGINE]), app_list, sizeof(app_list), surrogate_list, sizeof(surrogate_list), &app_id_flag);
}
if(app_id_flag==1)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_ID].name, (void *)app_list, TLD_TYPE_STRING);
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_SURROGATE_ID].name, (void *)surrogate_list, TLD_TYPE_STRING);
}
if(l7_protocol_flag==1)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)protocol_list, TLD_TYPE_STRING);
}
}
return 1;
}
static int set_vlan(struct tsg_log_instance_t *_instance, struct single_layer_vlan_addr *vlan_addr, int layer_num, cJSON *tunnel_object, tsg_log_field_id_t id)
{
if(layer_num==0)
{
return 0;
}
int i=0;
cJSON *vlan_array=cJSON_CreateArray();
for(i=0; i<layer_num; i++)
{
cJSON_AddNumberToObject(vlan_array, _instance->id2field[id].name, ntohs(vlan_addr[i].VID));
}
cJSON_AddItemToObject(tunnel_object, _instance->id2field[id].name, vlan_array);
return 1;
}
static int set_mpls(struct tsg_log_instance_t *_instance, struct single_layer_mpls_addr *mpls_addr, int layer_num, cJSON *tunnel_object, tsg_log_field_id_t id)
{
if(layer_num==0)
{
return 0;
}
int i=0;
cJSON *mpls_array=cJSON_CreateArray();
for(i=0; i<layer_num; i++)
{
cJSON_AddNumberToObject(mpls_array, _instance->id2field[id].name, ntohl(mpls_addr[i].label));
}
cJSON_AddItemToObject(tunnel_object, _instance->id2field[id].name, mpls_array);
return 1;
}
static int mac_to_string(unsigned char *mac, char *buff)
{ {
int i=0,len=0; int i=0,len=0;
@@ -87,6 +426,39 @@ static int convert_mac_to_string(unsigned char *mac, char *buff)
return 0; return 0;
} }
static int set_link_mac(struct tsg_log_instance_t *_instance, struct layer_addr_mac *mac, cJSON *tunnel_object)
{
char buff[128]={0};
cJSON *mac_object=NULL;
char default_mac[6]={0,0,0,0,0,0};
if((memcmp(mac->src_addr.h_source, default_mac, 6)))
{
mac_object=cJSON_CreateObject();
mac_to_string(mac->src_addr.h_source, buff);
cJSON_AddStringToObject(mac_object, _instance->id2field[LOG_COMMON_TUNNELS_MAC_SOURCE].name, buff);
mac_to_string(mac->src_addr.h_dest, buff);
cJSON_AddStringToObject(mac_object, _instance->id2field[LOG_COMMON_TUNNELS_MAC_DEST].name, buff);
cJSON_AddItemToObject(tunnel_object, "c2s_direction_mac", mac_object);
}
if((memcmp(mac->dst_addr.h_source, default_mac, 6)))
{
mac_object=cJSON_CreateObject();
mac_to_string(mac->dst_addr.h_source, buff);
cJSON_AddStringToObject(mac_object, _instance->id2field[LOG_COMMON_TUNNELS_MAC_SOURCE].name, buff);
mac_to_string(mac->dst_addr.h_dest, buff);
cJSON_AddStringToObject(mac_object, _instance->id2field[LOG_COMMON_TUNNELS_MAC_DEST].name, buff);
cJSON_AddItemToObject(tunnel_object, "s2c_direction_mac", mac_object);
}
return 1;
}
static int action2fs_id(int action) static int action2fs_id(int action)
{ {
switch(action) switch(action)
@@ -111,37 +483,6 @@ static int action2fs_id(int action)
return TSG_FS2_ABORT_UNKNOWN; return TSG_FS2_ABORT_UNKNOWN;
} }
int is_multi_hit_same_policy(struct Maat_rule_t *result, int *policy_id, int *policy_id_num)
{
int j=0;
for(j=0;j<*policy_id_num;j++)
{
if(policy_id[j]==result->config_id)
{
return 1;
}
}
policy_id[(*policy_id_num)++]=result->config_id;
return 0;
}
unsigned long long tsg_get_stream_id(struct streaminfo * a_stream)
{
int ret=0;
int device_id_size=sizeof(unsigned long long);
unsigned long long device_id=(unsigned long long)g_tsg_para.device_id;
ret=MESA_get_stream_opt(a_stream, MSO_GLOBAL_STREAM_ID, (void *)&device_id, &device_id_size);
if(ret==0)
{
return device_id;
}
return -1;
}
int TLD_cancel(struct TLD_handle_t *handle) int TLD_cancel(struct TLD_handle_t *handle)
{ {
struct TLD_handle_t *_handle=handle; struct TLD_handle_t *_handle=handle;
@@ -215,99 +556,6 @@ struct TLD_handle_t *TLD_create(int thread_id)
return _handle; return _handle;
} }
static int set_l7_protocol(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{
char *l7_protocol=NULL;
struct basic_proto_label *l7_proto_label=NULL;
l7_proto_label=(struct basic_proto_label *)project_req_get_struct(a_stream, g_tsg_para.l7_proto_project_id);
if(l7_proto_label!=NULL && l7_proto_label->proto_id!=g_tsg_para.mail_proto_id)
{
l7_protocol=tsg_l7_protocol_id2name(_instance, l7_proto_label->proto_id);
if(l7_protocol!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)l7_protocol, TLD_TYPE_STRING);
return 1;
}
}
return 0;
}
static int set_link_mac(struct tsg_log_instance_t *_instance, struct layer_addr_mac *mac, cJSON *tunnel_object)
{
char buff[128]={0};
cJSON *mac_object=NULL;
char default_mac[6]={0,0,0,0,0,0};
if((memcmp(mac->src_addr.h_source, default_mac, 6)))
{
mac_object=cJSON_CreateObject();
convert_mac_to_string(mac->src_addr.h_source, buff);
cJSON_AddStringToObject(mac_object, _instance->id2field[LOG_COMMON_TUNNELS_MAC_SOURCE].name, buff);
convert_mac_to_string(mac->src_addr.h_dest, buff);
cJSON_AddStringToObject(mac_object, _instance->id2field[LOG_COMMON_TUNNELS_MAC_DEST].name, buff);
cJSON_AddItemToObject(tunnel_object, "c2s_direction_mac", mac_object);
}
if((memcmp(mac->dst_addr.h_source, default_mac, 6)))
{
mac_object=cJSON_CreateObject();
convert_mac_to_string(mac->dst_addr.h_source, buff);
cJSON_AddStringToObject(mac_object, _instance->id2field[LOG_COMMON_TUNNELS_MAC_SOURCE].name, buff);
convert_mac_to_string(mac->dst_addr.h_dest, buff);
cJSON_AddStringToObject(mac_object, _instance->id2field[LOG_COMMON_TUNNELS_MAC_DEST].name, buff);
cJSON_AddItemToObject(tunnel_object, "s2c_direction_mac", mac_object);
}
return 1;
}
static int set_vlan(struct tsg_log_instance_t *_instance, struct single_layer_vlan_addr *vlan_addr, int layer_num, cJSON *tunnel_object, tsg_log_field_id_t id)
{
if(layer_num==0)
{
return 0;
}
int i=0;
cJSON *vlan_array=cJSON_CreateArray();
for(i=0; i<layer_num; i++)
{
cJSON_AddNumberToObject(vlan_array, _instance->id2field[id].name, ntohs(vlan_addr[i].VID));
}
cJSON_AddItemToObject(tunnel_object, _instance->id2field[id].name, vlan_array);
return 1;
}
static int set_mpls(struct tsg_log_instance_t *_instance, struct single_layer_mpls_addr *mpls_addr, int layer_num, cJSON *tunnel_object, tsg_log_field_id_t id)
{
if(layer_num==0)
{
return 0;
}
int i=0;
cJSON *mpls_array=cJSON_CreateArray();
for(i=0; i<layer_num; i++)
{
cJSON_AddNumberToObject(mpls_array, _instance->id2field[id].name, ntohl(mpls_addr[i].label));
}
cJSON_AddItemToObject(tunnel_object, _instance->id2field[id].name, mpls_array);
return 1;
}
static int get_gtp_ipv4v6_port(struct tsg_log_instance_t *_instance, struct streaminfo *a_stream, cJSON *object) static int get_gtp_ipv4v6_port(struct tsg_log_instance_t *_instance, struct streaminfo *a_stream, cJSON *object)
{ {
char ip_buff[64]={0}; char ip_buff[64]={0};
@@ -345,7 +593,7 @@ static int get_gtp_ipv4v6_port(struct tsg_log_instance_t *_instance, struct stre
return 0; return 0;
} }
static int get_common_tunnels(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream) static int set_common_tunnels(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{ {
int ret=0; int ret=0;
char ip_buff[64]={0}; char ip_buff[64]={0};
@@ -458,16 +706,36 @@ char *log_field_id2name(struct tsg_log_instance_t *instance, tsg_log_field_id_t
return NULL; return NULL;
} }
char *tsg_l7_protocol_id2name(struct tsg_log_instance_t *instance, unsigned short id) unsigned long long tsg_get_stream_id(struct streaminfo * a_stream)
{ {
struct tsg_log_instance_t *_instance=instance; int ret=0;
int device_id_size=sizeof(unsigned long long);
unsigned long long device_id=(unsigned long long)g_tsg_para.device_id;
if(_instance!=NULL && id>=MIN_L7_PROTO_ID && id<=MAX_L7_PROTO_ID) ret=MESA_get_stream_opt(a_stream, MSO_GLOBAL_STREAM_ID, (void *)&device_id, &device_id_size);
if(ret==0)
{ {
return _instance->l7_proto_id2field[id].name; return device_id;
} }
return NULL; return -1;
}
int is_multi_hit_same_policy(struct Maat_rule_t *result, int *policy_id, int *policy_id_num)
{
int j=0;
for(j=0;j<*policy_id_num;j++)
{
if(policy_id[j]==result->config_id)
{
return 1;
}
}
policy_id[(*policy_id_num)++]=result->config_id;
return 0;
} }
static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name, struct Maat_rule_t *p_result) static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name, struct Maat_rule_t *p_result)
@@ -505,33 +773,15 @@ static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name,
return 0; return 0;
} }
int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream) int set_session_attributes(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{ {
char buff[1024]={0}; char buff[1024]={0};
int l7_protocol_flag=0;
char *l7_protocol=NULL;
struct app_id_label *app_label=NULL;
struct _location_info_t *location=NULL; struct _location_info_t *location=NULL;
struct _session_attribute_label_t *attribute_label=NULL; struct _session_attribute_label_t *attribute_label=NULL;
l7_protocol_flag=set_l7_protocol(_instance, _handle, a_stream);
attribute_label=(struct _session_attribute_label_t *)project_req_get_struct(a_stream, _instance->internal_project_id); attribute_label=(struct _session_attribute_label_t *)project_req_get_struct(a_stream, _instance->internal_project_id);
if(attribute_label!=NULL) if(attribute_label!=NULL)
{ {
if(l7_protocol_flag==0)
{
l7_protocol=tsg_schema_index2string(attribute_label->proto);
if(l7_protocol!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)l7_protocol, TLD_TYPE_STRING);
}
else
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)"UNCATEGORIZED", TLD_TYPE_STRING);
}
}
TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)attribute_label->establish_latency_ms, TLD_TYPE_LONG); TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)attribute_label->establish_latency_ms, TLD_TYPE_LONG);
if(attribute_label->client_asn!=NULL) if(attribute_label->client_asn!=NULL)
@@ -565,157 +815,35 @@ int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD
TLD_append(_handle, _instance->id2field[LOG_SSL_JA3_FINGERPRINT].name, (void *)attribute_label->ja3_fingerprint, TLD_TYPE_STRING); TLD_append(_handle, _instance->id2field[LOG_SSL_JA3_FINGERPRINT].name, (void *)attribute_label->ja3_fingerprint, TLD_TYPE_STRING);
} }
} }
else
{
if(l7_protocol_flag==0)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)"UNCATEGORIZED", TLD_TYPE_STRING);
}
}
app_label=(struct app_id_label *)project_req_get_struct(a_stream, g_tsg_para.app_id_project_id);
if(app_label!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_ID].name, (void *)(long)app_label->app_id, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_SURROGATE_ID].name, (void *)(long)app_label->surrogate_id, TLD_TYPE_LONG);
}
return 0; return 0;
} }
int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, struct streaminfo *a_stream) int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, struct streaminfo *a_stream)
{ {
int i_or_e=0,direction=0;
int ret=0,addr_type=0;
unsigned short tunnel_type=0;
char nest_addr_buf[1024];
char *addr_proto=NULL; char *addr_proto=NULL;
struct timespec tv;
unsigned int client_isn=0,server_isn=0;
int size=sizeof(unsigned long long);
long common_con_duration_ms=0;
unsigned long long create_time=0;
unsigned long long stream_id=0; unsigned long long stream_id=0;
unsigned short c_port=0, s_port=0;
int tunnel_type_size=sizeof(tunnel_type);
struct layer_addr_ipv4 *ipv4=NULL;
struct layer_addr_ipv6 *ipv6=NULL;
char server_ip[MAX_IPV4_LEN*8]={0};
char client_ip[MAX_IPV4_LEN*8]={0};
struct tcp_flow_stat *tflow_project=NULL;
struct udp_flow_stat *uflow_project=NULL;
struct TLD_handle_t *_handle=handle; struct TLD_handle_t *_handle=handle;
struct tsg_log_instance_t *_instance=instance; struct tsg_log_instance_t *_instance=instance;
if(_instance==NULL || _handle==NULL || a_stream==NULL) if(_instance==NULL || _handle==NULL || a_stream==NULL)
{ {
MESA_handle_runtime_log(_instance->logger, MESA_handle_runtime_log(_instance->logger, RLOG_LV_DEBUG, "TLD_APPEND_STREAM", "instance==NULL || TLD_handle==NULL || addr==NULL");
RLOG_LV_DEBUG,
"TLD_APPEND_STREAM",
"instance==NULL || TLD_handle==NULL || addr==NULL"
);
return -1; return -1;
} }
switch(a_stream->addr.addrtype) set_app_id(_instance, _handle, a_stream);
set_tcp_isn(_instance, _handle, a_stream);
set_tuple4(_instance, _handle, a_stream);
set_direction(_instance, _handle, a_stream);
set_address_list(_instance, _handle, a_stream);
set_duraction(_instance, _handle, a_stream);
set_packet_bytes(_instance, _handle, a_stream);
set_session_attributes(_instance, _handle, a_stream);
if(is_tunnels(a_stream))
{ {
case ADDR_TYPE_IPV4: set_common_tunnels(_instance, _handle, a_stream);
case __ADDR_TYPE_IP_PAIR_V4:
ipv4=a_stream->addr.ipv4;
addr_type=4;
c_port=ntohs(ipv4->source);
s_port=ntohs(ipv4->dest);
inet_ntop(AF_INET, (void *)&ipv4->saddr, client_ip, sizeof(client_ip));
inet_ntop(AF_INET, (void *)&ipv4->daddr, server_ip, sizeof(server_ip));
break;
case ADDR_TYPE_IPV6:
case __ADDR_TYPE_IP_PAIR_V6:
ipv6=a_stream->addr.ipv6;
addr_type=6;
c_port=ntohs(ipv6->source);
s_port=ntohs(ipv6->dest);
inet_ntop(AF_INET6, (void *)ipv6->saddr, client_ip, sizeof(client_ip));
inet_ntop(AF_INET6, (void *)ipv6->daddr, server_ip, sizeof(server_ip));
break;
default:
break;
}
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_IP].name, (void *)server_ip, TLD_TYPE_STRING);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CLIENT_IP].name, (void *)client_ip, TLD_TYPE_STRING);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_PORT].name, (void *)(long)s_port, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CLIENT_PORT].name, (void *)(long)c_port, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_STREAM_DIR].name, (void *)(long)a_stream->dir, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_ADDRESS_TYPE].name, (void *)(long)addr_type, TLD_TYPE_LONG);
switch(a_stream->type)
{
case STREAM_TYPE_TCP:
tflow_project=(struct tcp_flow_stat *)project_req_get_struct(a_stream, _instance->tcp_flow_project_id);
if(tflow_project!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)tflow_project->S2C_all_pkt, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)tflow_project->S2C_all_byte_raw, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)tflow_project->C2S_all_pkt, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)tflow_project->C2S_all_byte_raw, TLD_TYPE_LONG);
}
size=sizeof(unsigned int);
ret=MESA_get_stream_opt(a_stream, MSO_TCP_ISN_C2S, &client_isn, &size);
if(ret==0)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_TCP_CLIENT_ISN].name, (void *)(long)client_isn, TLD_TYPE_LONG);
}
size=sizeof(unsigned int);
ret=MESA_get_stream_opt(a_stream, MSO_TCP_ISN_S2C, &server_isn, &size);
if(ret==0)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_TCP_SERVER_ISN].name, (void *)(long)server_isn, TLD_TYPE_LONG);
}
break;
case STREAM_TYPE_UDP:
uflow_project=(struct udp_flow_stat *)project_req_get_struct(a_stream, _instance->udp_flow_project_id);
if(uflow_project!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_PKT_NUM].name, (void *)(long)uflow_project->S2C_pkt, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_S2C_BYTE_NUM].name, (void *)(long)uflow_project->S2C_all_byte_raw, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_PKT_NUM].name, (void *)(long)uflow_project->C2S_pkt, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_C2S_BYTE_NUM].name, (void *)(long)uflow_project->C2S_all_byte_raw, TLD_TYPE_LONG);
}
break;
default:
break;
}
if(a_stream!=NULL && a_stream->ptcpdetail!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)(a_stream->ptcpdetail->createtime), TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)(a_stream->ptcpdetail->lastmtime), TLD_TYPE_LONG);
ret=MESA_get_stream_opt(a_stream, MSO_STREAM_CREATE_TIMESTAMP_MS, (void *)&create_time, &size);
if(ret>=0)
{
clock_gettime(CLOCK_REALTIME, &tv);
common_con_duration_ms=tv.tv_sec*1000+tv.tv_nsec/1000/1000 - create_time;
}
if(common_con_duration_ms>0)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG);
}
}
else
{
time_t cur_time=time(NULL);
TLD_append(_handle, _instance->id2field[LOG_COMMON_START_TIME].name, (void *)cur_time, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_END_TIME].name, (void *)cur_time, TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CON_DURATION_MS].name, (void *)(common_con_duration_ms), TLD_TYPE_LONG);
} }
stream_id=tsg_get_stream_id(a_stream); stream_id=tsg_get_stream_id(a_stream);
@@ -726,56 +854,6 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle
addr_proto=(char *)layer_addr_prefix_ntop(a_stream); addr_proto=(char *)layer_addr_prefix_ntop(a_stream);
TLD_append(_handle, _instance->id2field[LOG_COMMON_L4_PROTOCOL].name, (void *)addr_proto, TLD_TYPE_STRING); TLD_append(_handle, _instance->id2field[LOG_COMMON_L4_PROTOCOL].name, (void *)addr_proto, TLD_TYPE_STRING);
ret=MESA_get_stream_opt(a_stream, MSO_STREAM_TUNNEL_TYPE, &tunnel_type, &tunnel_type_size);
assert(ret==0);
if(tunnel_type==STREAM_TUNNLE_NON)
{
layer_addr_ntop_r(a_stream,nest_addr_buf, sizeof(nest_addr_buf));
}
else
{
stream_addr_list_ntop(a_stream,nest_addr_buf, sizeof(nest_addr_buf));
}
if(is_tunnels(a_stream))
{
get_common_tunnels(_instance, _handle, a_stream);
}
TLD_append(_handle, _instance->id2field[LOG_COMMON_ADDRESS_LIST].name, (void *)nest_addr_buf, TLD_TYPE_STRING);
set_common_field_from_label(_instance, _handle, a_stream);
i_or_e=MESA_dir_link_to_human(a_stream->routedir);
switch(a_stream->curdir)
{
case DIR_C2S:
if(i_or_e=='E' || i_or_e=='e')
{
direction='E';
}
else
{
direction='I';
}
break;
case DIR_S2C:
if(i_or_e=='E' || i_or_e=='e')
{
direction='I';
}
else
{
direction='E';
}
break;
default:
break;
}
TLD_append(_handle, _instance->id2field[LOG_COMMON_DIRECTION].name, (void *)(long)direction, TLD_TYPE_LONG);
return 0; return 0;
} }
@@ -874,7 +952,6 @@ int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t
struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
{ {
int i=0,ret=0; int i=0,ret=0;
int tmp_value=0;
char nic_name[32]={0}; char nic_name[32]={0};
char kafka_errstr[1024]={0}; char kafka_errstr[1024]={0};
unsigned int local_ip_nr=0; unsigned int local_ip_nr=0;
@@ -952,9 +1029,6 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
} }
inet_ntop(AF_INET,&(local_ip_nr),_instance->local_ip_str,sizeof(_instance->local_ip_str)); inet_ntop(AF_INET,&(local_ip_nr),_instance->local_ip_str,sizeof(_instance->local_ip_str));
MESA_load_profile_string_def(conffile, "TSG_LOG", "L7_PROTO_ID_FILE", _instance->l7_proto_id_file, sizeof(_instance->l7_proto_id_file), "./tsgconf/app_l7_proto_id.conf");
load_log_common_field(_instance->l7_proto_id_file, _instance->l7_proto_id2field, NULL, &tmp_value);
rdkafka_conf = rd_kafka_conf_new(); rdkafka_conf = rd_kafka_conf_new();
rd_kafka_conf_set(rdkafka_conf, "queue.buffering.max.messages", _instance->send_queue_max_msg, kafka_errstr, sizeof(kafka_errstr)); rd_kafka_conf_set(rdkafka_conf, "queue.buffering.max.messages", _instance->send_queue_max_msg, kafka_errstr, sizeof(kafka_errstr));
rd_kafka_conf_set(rdkafka_conf, "topic.metadata.refresh.interval.ms", _instance->refresh_interval_ms, kafka_errstr, sizeof(kafka_errstr)); rd_kafka_conf_set(rdkafka_conf, "topic.metadata.refresh.interval.ms", _instance->refresh_interval_ms, kafka_errstr, sizeof(kafka_errstr));

3
src/tsg_send_log_internal.h
View File

@@ -100,7 +100,7 @@ typedef enum _tsg_log_field_id
typedef struct _id2field typedef struct _id2field
{ {
TLD_TYPE type; int type;
int id; int id;
char name[MAX_STRING_LEN]; char name[MAX_STRING_LEN];
}id2field_t; }id2field_t;
@@ -137,7 +137,6 @@ struct tsg_log_instance_t
id2field_t id2field[LOG_COMMON_MAX]; id2field_t id2field[LOG_COMMON_MAX];
rd_kafka_topic_t **topic_rkt; rd_kafka_topic_t **topic_rkt;
id2field_t *service2topic; id2field_t *service2topic;
id2field_t l7_proto_id2field[MAX_L7_PROTO_ID+1];
void *logger; void *logger;
}; };

1150
src/uthash.h Normal file
View File

File diff suppressed because it is too large Load Diff