gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TSG-14945: 输出Application and Protocol Metrics

Browse Source
This commit is contained in:
刘学利
2023-06-02 08:19:50 +00:00
parent 2adf35914f
commit 380733e78b
15 changed files with 751 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
bin/main.conf
View File

@@ -27,17 +27,23 @@ UDP_MIN_BYTES=5
[SECURITY_HITS]
CYCLE=1000
TELEGRAF_PORT=8092
TELEGRAF_PORT=8400
TELEGRAF_IP="127.0.0.1"
APP_NAME="security_rule_hits"
[STATISTIC]
CYCLE=1
TELEGRAF_PORT=8092
CYCLE=5
TELEGRAF_PORT=8100
TELEGRAF_IP="127.0.0.1"
OUTPUT_PATH="./tsg_statistic.log"
APP_NAME="network_activity"
[APP_METRIC]
CYCLE=1000
TELEGRAF_PORT=8100
TELEGRAF_IP="127.0.0.1"
APP_NAME="app_metric"
[FIELD_STAT]
CYCLE=30
TELEGRAF_PORT=8125

29
inc/tsg_statistic.h
View File

@@ -11,7 +11,7 @@ enum _STATISTIC_OPT_TYPE
_OPT_TYPE_MAX
};
struct _traffic_info
struct traffic_info
{
long long con_num;
long long in_bytes;
@@ -20,8 +20,33 @@ struct _traffic_info
long long out_packets;
};
struct traffic_packet_info
{
uint32_t sessions;
uint32_t in_pkts;
uint32_t out_pkts;
uint32_t c2s_pkts;
uint32_t s2c_pkts;
uint32_t c2s_fragments;
uint32_t s2c_fragments;
uint32_t c2s_tcp_ooorder_pkts;
uint32_t s2c_tcp_ooorder_pkts;
uint32_t c2s_tcp_retransmitted_pkts;
uint32_t s2c_tcp_retransmitted_pkts;
uint64_t in_bytes;
uint64_t out_bytes;
uint64_t c2s_bytes;
uint64_t s2c_bytes;
uint64_t c2s_tcp_lost_bytes;
uint64_t s2c_tcp_lost_bytes;
uint64_t c2s_tcp_retransmitted_bytes;
uint64_t s2c_tcp_retransmitted_bytes;
};
int tsg_set_policy_flow(const struct streaminfo *a_stream, struct maat_rule *p_result, int thread_seq);
int tsg_set_intercept_flow(struct maat_rule *p_result, struct _traffic_info *traffic_info, int thread_seq);
int tsg_set_intercept_flow(struct maat_rule *p_result, struct traffic_info *traffic_info, int thread_seq);
int tsg_set_statistic_opt(int value, enum _STATISTIC_OPT_TYPE type, int thread_seq);
int tsg_set_application_metrics(const struct streaminfo *a_stream, const char *l4_protocol, const char *app_full_path, struct traffic_packet_info *app_statis, int thread_seq);
#endif

5
src/tsg_bridge.cpp
View File

@@ -135,6 +135,11 @@ void session_runtime_action_context_free(const struct streaminfo *a_stream, int
}
}
if (srt_action_context->l4_protocol != NULL)
{
dictator_free(a_stream->threadnum, srt_action_context->l4_protocol);
}
dictator_free(a_stream->threadnum, data);
data=NULL;
}

7
src/tsg_bridge.h
View File

@@ -6,6 +6,8 @@
#include "tsg_label.h"
#include "tsg_rule.h"
#include "tsg_leaky_bucket.h"
#include "tsg_send_log_internal.h"
#include "tsg_statistic.h"
#ifndef MAX_BRIDGE_NAME_LEN
#define MAX_BRIDGE_NAME_LEN 64
@@ -99,9 +101,10 @@ struct session_runtime_action_context
char udp_data_dropme;
char set_latency_flag;
char direction;
char padding[5];
char padding[1];
enum TSG_METHOD_TYPE method_type;
enum TSG_PROTOCOL protocol;
char *l4_protocol;
union
{
struct leaky_bucket *bucket;
@@ -110,6 +113,8 @@ struct session_runtime_action_context
struct hited_app_para hited_para;
void *para;
};
long long last_update_metric_time;
struct traffic_packet_info last_app_statis;
};
struct udp_session_runtime_context

474
src/tsg_entry.cpp
View File

@@ -687,6 +687,444 @@ static int srt_process_context_set_l7_protocol(struct session_runtime_process_co
return 0;
}
/*
L7(openvpn), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon
L7(openvpn), appSketch(psiphon), Thrid(openvpn.qq_web.wechat) = openvpn.psiphon
L7(openvpn), appSketch(psiphon), Thrid(openvpn.wechat) = openvpn.psiphon
L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn.wechat) = openvpn.ssl.psiphon
L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn) = openvpn.ssl.psiphon
L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon
L7(null), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon
L7(null), appSketch(psiphon), Thrid(wechat) = psiphon
L7(null), appSketch(null), Thrid(openvpn.ssl.wechat) = openvpn.ssl.wechat
L7(null), appSketch(null), Thrid(null) = unknown
*/
static int session_qm_result_l7_protocol_split(struct gather_app_result *qm_result, struct gather_app_result *qm_l7_result)
{
int i=0;
for(i=qm_result->app_num; i>0; i--)
{
if(tsg_l7_protocol_id2name(qm_result->attributes[i-1].app_id))
{
break;
}
}
qm_l7_result->app_num=i;
qm_l7_result->origin=ORIGIN_QM_ENGINE_L7;
memcpy(qm_l7_result->attributes, qm_result->attributes, sizeof(struct app_attributes)*i);
return 0;
}
static unsigned int get_max_app_id(struct gather_app_result *result)
{
int i=0;
unsigned int max_app_id=0;
for(i=0; i< result->app_num; i++)
{
if(max_app_id < result->attributes[i].app_id)
{
max_app_id=result->attributes[i].app_id;
}
}
return max_app_id;
}
static unsigned int get_lastest_app_id(struct gather_app_result *result)
{
if(result->app_num>0)
{
return result->attributes[result->app_num-1].app_id;
}
return 0;
}
/*
** Engine1 1.2 Engine2 1 -> 1.2
** Engine1 1.2 Engine2 1.3 -> 1.2
** Engine1 1 Engine2 1.2 -> 1.2
** Engine1 2 Engine2 1.2 -> 1.2
** Engine1 1 Engine2 2.3 -> 1
** Engine1 1 Engine2 2 -> 1
*/
static int copy_app_id(struct app_attributes *result, int result_num, unsigned int *combined_array, int combined_array_num)
{
int i=0;
for(i=0; i<result_num && i<combined_array_num; i++)
{
combined_array[i]=result[i].app_id;
}
return i;
}
#define COMBINED_APP_ID_NUM MAX_APP_ID_NUM*2
int session_application_full_path_combine(struct gather_app_result *result, char *out_full_path, int out_full_path_len)
{
int offset=0,combined_num=0;
unsigned int combined_array[COMBINED_APP_ID_NUM]={0};
if(result[ORIGIN_BASIC_PROTOCOL].app_num>0)
{
combined_num+=copy_app_id(result[ORIGIN_BASIC_PROTOCOL].attributes, result[ORIGIN_BASIC_PROTOCOL].app_num, combined_array, COMBINED_APP_ID_NUM);
}
else
{
combined_num+=copy_app_id(result[ORIGIN_QM_ENGINE_L7].attributes, result[ORIGIN_QM_ENGINE_L7].app_num, combined_array, COMBINED_APP_ID_NUM);
}
if(result[ORIGIN_HITED_APP].app_num > 0)
{
combined_array[combined_num]=get_lastest_app_id(&(result[ORIGIN_HITED_APP]));
combined_num+=1;
}
else if(result[ORIGIN_USER_DEFINE].app_num>0)
{
if(combined_num<COMBINED_APP_ID_NUM)
{
combined_array[combined_num]=get_max_app_id(&(result[ORIGIN_USER_DEFINE]));
combined_num+=1;
}
}
else if(result[ORIGIN_BUILT_IN].app_num>0)
{
combined_num+=copy_app_id(result[ORIGIN_BUILT_IN].attributes, result[ORIGIN_BUILT_IN].app_num, combined_array+combined_num, COMBINED_APP_ID_NUM-combined_num);
}
else if(result[ORIGIN_QM_ENGINE].app_num-result[ORIGIN_QM_ENGINE_L7].app_num > 0)
{
combined_num+=copy_app_id( result[ORIGIN_QM_ENGINE].attributes + result[ORIGIN_QM_ENGINE_L7].app_num,
result[ORIGIN_QM_ENGINE].app_num - result[ORIGIN_QM_ENGINE_L7].app_num,
combined_array+combined_num,
COMBINED_APP_ID_NUM-combined_num
);
}
if(combined_num==0)
{
offset=tsg_get_app_name_by_id(g_tsg_maat_feather, result[ORIGIN_UNKNOWN].attributes[0].app_id, out_full_path, out_full_path_len, 0);
if(offset>0)
{
return offset;
}
return 0;
}
for(int i=0; i<combined_num; i++)
{
offset+=tsg_get_app_name_by_id(g_tsg_maat_feather, combined_array[i], out_full_path+offset, out_full_path_len-offset, 0);
if(offset>0)
{
out_full_path[offset++]='.';
}
}
if(offset>0)
{
out_full_path[offset-1]='\0';
}
return offset;
}
int session_application_full_path_update(const struct streaminfo *a_stream, char *app_full_path, int app_full_path_len)
{
struct gather_app_result *gather_result=(struct gather_app_result *)session_gather_app_results_get(a_stream);
if(gather_result!=NULL)
{
session_qm_result_l7_protocol_split(&(gather_result[ORIGIN_QM_ENGINE]), &(gather_result[ORIGIN_QM_ENGINE_L7]));
session_application_full_path_combine(gather_result, app_full_path, app_full_path_len);
}
else
{
tsg_get_app_name_by_id(g_tsg_maat_feather, tsg_unknown_app_id_get(g_tsg_log_instance), app_full_path, app_full_path_len, 0);
}
return 1;
}
static const char *session_addr_type_string_convert(UCHAR addrtype)
{
switch (addrtype)
{
case ADDR_TYPE_IPV4:
case __ADDR_TYPE_IP_PAIR_V4:
return ".IPv4";
case ADDR_TYPE_IPV6:
case __ADDR_TYPE_IP_PAIR_V6:
return ".IPv6";
case ADDR_TYPE_VLAN:
return ".VLAN";
case ADDR_TYPE_MAC:
return ".ETHERNET";
case ADDR_TYPE_ARP:
return ".ARP";
case ADDR_TYPE_GRE:
return ".GRE";
case ADDR_TYPE_MPLS:
return ".MPLS";
case ADDR_TYPE_PPPOE_SES:
return ".PPPOE";
case ADDR_TYPE_TCP:
return ".TCP";
case ADDR_TYPE_UDP:
return ".UDP";
case ADDR_TYPE_L2TP:
return ".L2TP";
case ADDR_TYPE_PPP:
return ".PPP";
case ADDR_TYPE_PPTP:
return ".PPTP";
case ADDR_TYPE_MAC_IN_MAC:
return ".MAC_IN_MAC";
case ADDR_TYPE_GPRS_TUNNEL:
return ".GTP";
case ADDR_TYPE_VXLAN:
return ".VXLAN";
default:
return NULL;
}
}
#define MAX_L4_PROTOCOL_STR_LEN 512
char *session_l4_protocol_label_update(const struct streaminfo *a_stream, char **l4_protocol)
{
if(*l4_protocol!=NULL || a_stream == NULL)
{
return *l4_protocol;
}
const char *l4_protocol_str = NULL;
int combined_l4_protocol_offset=MAX_L4_PROTOCOL_STR_LEN;
char combined_l4_protocol_str[MAX_L4_PROTOCOL_STR_LEN]={0};
struct streaminfo *cur_stream = (struct streaminfo *)a_stream;
while (cur_stream != NULL)
{
switch (cur_stream->type)
{
case STREAM_TYPE_TCP:
if (cur_stream->addr.addrtype == ADDR_TYPE_IPV4)
{
l4_protocol_str = ".IPv4.TCP";
}
else if (cur_stream->addr.addrtype == ADDR_TYPE_IPV6)
{
l4_protocol_str = ".IPv6.TCP";
}
break;
case STREAM_TYPE_UDP:
if (cur_stream->addr.addrtype == ADDR_TYPE_IPV4)
{
l4_protocol_str = ".IPv4.UDP";
}
else if (cur_stream->addr.addrtype == ADDR_TYPE_IPV6)
{
l4_protocol_str = ".IPv6.UDP";
}
break;
case STREAM_TYPE_VLAN:
l4_protocol_str = ".VLAN"; break;
case STREAM_TYPE_PPPOE:
l4_protocol_str = ".PPPOE"; break;
case STREAM_TYPE_L2TP:
l4_protocol_str = ".L2TP"; break;
case STREAM_TYPE_OPENVPN:
l4_protocol_str = ".OpenVPN"; break;
case STREAM_TYPE_PPTP:
l4_protocol_str = ".PPTP"; break;
case STREAM_TYPE_NON:
case STREAM_TYPE_ISAKMP:
l4_protocol_str = session_addr_type_string_convert(cur_stream->addr.addrtype);
break;
// SOCKS4 SOCKS5 HTTP
default:
continue;
}
int n_l4_protocol_str = strlen(l4_protocol_str);
if (n_l4_protocol_str!=0)
{
if (combined_l4_protocol_offset - n_l4_protocol_str < 0)
{
break;
}
memcpy(combined_l4_protocol_str+combined_l4_protocol_offset-n_l4_protocol_str, l4_protocol_str, n_l4_protocol_str);
combined_l4_protocol_offset-=n_l4_protocol_str;
l4_protocol_str=NULL;
}
cur_stream = cur_stream->pfather;
}
if(combined_l4_protocol_offset>0 && combined_l4_protocol_offset<MAX_L4_PROTOCOL_STR_LEN)
{
int l4_protocol_len=MAX_L4_PROTOCOL_STR_LEN-combined_l4_protocol_offset-1;
*l4_protocol=(char *)malloc(l4_protocol_len+1);
memcpy(*l4_protocol, combined_l4_protocol_str+combined_l4_protocol_offset+1, l4_protocol_len); // +1 for del "."
(*l4_protocol)[l4_protocol_len]='\0';
}
return *l4_protocol;
}
int session_calculate_udp_packets_increment(const struct streaminfo *a_stream, struct traffic_packet_info *last_value, struct traffic_packet_info *increment)
{
struct udp_flow_stat *uflow_stat = (struct udp_flow_stat *)project_req_get_struct(a_stream, g_tsg_log_instance->udp_flow_project_id);
if (uflow_stat == NULL)
{
return -1;
}
increment->c2s_pkts = uflow_stat->C2S_pkt - last_value->c2s_pkts;
last_value->c2s_pkts = uflow_stat->C2S_pkt;
increment->s2c_pkts = uflow_stat->S2C_pkt - last_value->s2c_pkts;
last_value->s2c_pkts = uflow_stat->S2C_pkt;
increment->c2s_bytes = uflow_stat->C2S_all_byte_raw - last_value->c2s_bytes;
last_value->c2s_bytes = uflow_stat->C2S_all_byte_raw;
increment->s2c_bytes = uflow_stat->S2C_all_byte_raw - last_value->s2c_bytes;
last_value->s2c_bytes = uflow_stat->S2C_all_byte_raw;
increment->c2s_fragments = uflow_stat->C2S_ip_fragment_pkt - last_value->c2s_fragments;
last_value->c2s_fragments = uflow_stat->C2S_ip_fragment_pkt;
increment->s2c_fragments = uflow_stat->S2C_ip_fragment_pkt - last_value->s2c_fragments;
last_value->s2c_fragments = uflow_stat->S2C_ip_fragment_pkt;
return 0;
}
int session_calculate_tcp_packets_increment(const struct streaminfo *a_stream, struct traffic_packet_info *last_value, struct traffic_packet_info *increment)
{
struct tcp_flow_stat *tflow_stat = (struct tcp_flow_stat *)project_req_get_struct(a_stream, g_tsg_log_instance->tcp_flow_project_id);
if (tflow_stat == NULL)
{
return -1;
}
increment->c2s_pkts = tflow_stat->C2S_all_pkt - last_value->c2s_pkts;
last_value->c2s_pkts = tflow_stat->C2S_all_pkt;
increment->s2c_pkts = tflow_stat->S2C_all_pkt - last_value->s2c_pkts;
last_value->s2c_pkts = tflow_stat->S2C_all_pkt;
increment->c2s_bytes = tflow_stat->C2S_all_byte_raw - last_value->c2s_bytes;
last_value->c2s_bytes = tflow_stat->C2S_all_byte_raw;
increment->s2c_bytes = tflow_stat->S2C_all_byte_raw - last_value->s2c_bytes;
last_value->s2c_bytes = tflow_stat->S2C_all_byte_raw;
increment->c2s_fragments = tflow_stat->C2S_ip_fragment_pkt - last_value->c2s_fragments;
last_value->c2s_fragments = tflow_stat->C2S_ip_fragment_pkt;
increment->s2c_fragments = tflow_stat->S2C_ip_fragment_pkt - last_value->s2c_fragments;
last_value->s2c_fragments = tflow_stat->S2C_ip_fragment_pkt;
increment->c2s_tcp_lost_bytes = tflow_stat->C2S_seq_gap_loss_cnt - last_value->c2s_tcp_lost_bytes;
last_value->c2s_tcp_lost_bytes = tflow_stat->C2S_seq_gap_loss_cnt;
increment->s2c_tcp_lost_bytes = tflow_stat->S2C_seq_gap_loss_cnt - last_value->s2c_tcp_lost_bytes;
last_value->s2c_tcp_lost_bytes = tflow_stat->S2C_seq_gap_loss_cnt;
increment->c2s_tcp_ooorder_pkts = tflow_stat->C2S_ooorder_pkt - last_value->c2s_tcp_ooorder_pkts;
last_value->c2s_tcp_ooorder_pkts = tflow_stat->C2S_ooorder_pkt;
increment->s2c_tcp_ooorder_pkts = tflow_stat->S2C_ooorder_pkt - last_value->s2c_tcp_ooorder_pkts;
last_value->s2c_tcp_ooorder_pkts = tflow_stat->S2C_ooorder_pkt;
increment->c2s_tcp_retransmitted_pkts = tflow_stat->C2S_retransmission_pkt - last_value->c2s_tcp_retransmitted_pkts;
last_value->c2s_tcp_retransmitted_pkts = tflow_stat->C2S_retransmission_pkt;
increment->s2c_tcp_retransmitted_pkts = tflow_stat->S2C_retransmission_pkt - last_value->s2c_tcp_retransmitted_pkts;
last_value->s2c_tcp_retransmitted_pkts = tflow_stat->S2C_retransmission_pkt;
increment->c2s_tcp_retransmitted_bytes = tflow_stat->C2S_retransmission_byte - last_value->c2s_tcp_retransmitted_bytes;
last_value->c2s_tcp_retransmitted_bytes = tflow_stat->C2S_retransmission_byte;
increment->s2c_tcp_retransmitted_bytes = tflow_stat->S2C_retransmission_byte - last_value->s2c_tcp_retransmitted_bytes;
last_value->s2c_tcp_retransmitted_bytes = tflow_stat->S2C_retransmission_byte;
return 0;
}
int session_calculate_packets_increment(const struct streaminfo *a_stream, struct traffic_packet_info *last_value, struct traffic_packet_info *increment, int thread_seq)
{
if (a_stream == NULL || last_value == NULL || increment == NULL)
{
return -1;
}
unsigned long long value = 0;
int value_len = sizeof(unsigned long long);
increment->sessions = 1-last_value->sessions;
last_value->sessions = 1;
value = 0;
MESA_get_stream_opt(a_stream, MSO_TOTAL_INBOUND_BYTE_RAW, (void *)&value, &value_len);
increment->in_bytes = (uint64_t)value - last_value->in_bytes;
last_value->in_bytes = (uint64_t)value;
value = 0;
MESA_get_stream_opt(a_stream, MSO_TOTAL_INBOUND_PKT, (void *)&value, &value_len);
increment->in_pkts = (uint64_t)value - last_value->in_pkts;
last_value->in_pkts = (uint64_t)value;
value = 0;
MESA_get_stream_opt(a_stream, MSO_TOTAL_OUTBOUND_BYTE_RAW, (void *)&value, &value_len);
increment->out_bytes = (uint64_t)value - last_value->out_bytes;
last_value->out_bytes = (uint64_t)value;
value = 0;
MESA_get_stream_opt(a_stream, MSO_TOTAL_OUTBOUND_PKT, (void *)&value, &value_len);
increment->out_pkts = (uint64_t)value - last_value->out_pkts;
last_value->out_pkts = (uint64_t)value;
switch (a_stream->type)
{
case STREAM_TYPE_TCP:
session_calculate_tcp_packets_increment(a_stream, last_value, increment);
break;
case STREAM_TYPE_UDP:
session_calculate_udp_packets_increment(a_stream, last_value, increment);
break;
default:
break;
}
return 0;
}
int session_application_metrics_update(const struct streaminfo *a_stream, struct session_runtime_action_context *srt_action_context, int thread_seq)
{
if(tsg_get_current_time_ms() - srt_action_context->last_update_metric_time < g_tsg_para.app_metric_update_interval_ms)
{
return 0;
}
char app_full_path[256]={0};
struct traffic_packet_info increment_app_statis = {0};
session_application_full_path_update(a_stream, app_full_path, sizeof(app_full_path));
char *l4_protocol_string=session_l4_protocol_label_update(a_stream, &srt_action_context->l4_protocol);
session_calculate_packets_increment(a_stream, &srt_action_context->last_app_statis, &increment_app_statis, thread_seq);
tsg_set_application_metrics(a_stream, l4_protocol_string, app_full_path, &increment_app_statis, thread_seq);
srt_action_context->last_update_metric_time = tsg_get_current_time_ms();
return 1;
}
static int session_app_gather_results_set_l7_protocol(const struct streaminfo *a_stream, enum TSG_PROTOCOL protocol)
{
struct gather_app_result *gather_result=(struct gather_app_result *)session_gather_app_results_get(a_stream);
@@ -1955,13 +2393,18 @@ extern "C" unsigned char TSG_MASTER_UDP_ENTRY(const struct streaminfo *a_udp, vo
memset(*pme, 0, sizeof(struct udp_session_runtime_context));
udp_srt_context=(struct udp_session_runtime_context *)(*pme);
udp_srt_context->srt_action_context=(struct session_runtime_action_context *)dictator_malloc(thread_seq, sizeof(struct session_runtime_action_context));
memset(udp_srt_context->srt_action_context, 0, sizeof(struct session_runtime_action_context));
session_runtime_action_context_async(a_udp, (void *)(udp_srt_context->srt_action_context));
udp_srt_context->srt_action_context=(struct session_runtime_action_context *)session_runtime_action_context_get(a_udp);
if(udp_srt_context->srt_action_context==NULL)
{
udp_srt_context->srt_action_context=(struct session_runtime_action_context *)session_runtime_action_context_new(a_udp);
}
udp_srt_context->srt_action_context->last_update_metric_time = tsg_get_current_time_ms();
udp_srt_context->srt_action_context->direction=get_direction(a_udp);
}
session_application_metrics_update(a_udp, udp_srt_context->srt_action_context, thread_seq);
if(udp_srt_context->srt_action_context->udp_data_dropme==0)
{
state2=tsg_master_data_entry(a_udp, (void **)&(udp_srt_context->srt_process_context), thread_seq, a_packet);
@@ -1980,6 +2423,7 @@ extern "C" unsigned char TSG_MASTER_UDP_ENTRY(const struct streaminfo *a_udp, vo
if(state1&APP_STATE_DROPME || state2&APP_STATE_DROPME || a_udp->opstate==OP_STATE_CLOSE)
{
session_application_metrics_update(a_udp, udp_srt_context->srt_action_context, thread_seq);
dictator_free(thread_seq, *pme);
*pme=NULL;
@@ -1991,7 +2435,7 @@ extern "C" unsigned char TSG_MASTER_UDP_ENTRY(const struct streaminfo *a_udp, vo
extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp, void **pme, int thread_seq, const void *a_packet)
{
struct session_runtime_action_context *_context=(struct session_runtime_action_context *)(*pme);
struct session_runtime_action_context *srt_action_context=(struct session_runtime_action_context *)(*pme);
int is_ctrl_pkt=session_state_control_packet(a_tcp, a_tcp->ptcpdetail->pdata, a_tcp->ptcpdetail->datalen);
if(is_ctrl_pkt==1)
@@ -2005,18 +2449,19 @@ extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp,
*pme=(void *)session_runtime_action_context_get(a_tcp);
if(*pme==NULL)
{
*pme=(struct session_runtime_action_context *)dictator_malloc(thread_seq, sizeof(struct session_runtime_action_context));
memset(*pme, 0, sizeof(struct session_runtime_action_context));
session_runtime_action_context_async(a_tcp, (void *)(*pme));
srt_action_context=(struct session_runtime_action_context *)session_runtime_action_context_new(a_tcp);
*pme=(void *)srt_action_context;
}
_context=(struct session_runtime_action_context *)(*pme);
_context->direction=get_direction(a_tcp);
srt_action_context->direction=get_direction(a_tcp);
srt_action_context->last_update_metric_time = tsg_get_current_time_ms();
}
if(_context->set_latency_flag==0)
session_application_metrics_update(a_tcp, srt_action_context, thread_seq);
if(srt_action_context->set_latency_flag==0)
{
_context->set_latency_flag=session_tcp_establish_latency_ms_set(a_tcp, thread_seq, a_packet);
srt_action_context->set_latency_flag=session_tcp_establish_latency_ms_set(a_tcp, thread_seq, a_packet);
}
if (a_packet != NULL)
@@ -2028,6 +2473,7 @@ extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp,
if(state&APP_STATE_DROPME || a_tcp->pktstate==OP_STATE_CLOSE)
{
session_application_metrics_update(a_tcp, srt_action_context, thread_seq);
session_state_sync_in_opening_and_closing(a_tcp, a_tcp->pktstate);
}
@@ -2072,6 +2518,8 @@ extern "C" int TSG_MASTER_INIT()
MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "SCAN_TIME_INTERVAL", &g_tsg_para.scan_time_interval, 120);
MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "IENTIFY_APP_MAX_PKT_NUM", &g_tsg_para.identify_app_max_pkt_num, 20);
MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "APP_METRICS_INTERVAL", &g_tsg_para.app_metric_update_interval_ms, 30000);
MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "FEATURE_TAMPER", &g_tsg_para.feature_tamper, 0);
MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "SERVICE_CHAINING_SID", &g_tsg_para.service_chaining_sid, 0);
MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "SHAPING_SID", &g_tsg_para.shaping_sid, 0);
@@ -2126,10 +2574,10 @@ extern "C" int TSG_MASTER_INIT()
FS_start(g_tsg_para.fs2_handle);
ret=tsg_statistic_init(tsg_conffile, g_tsg_para.logger);
ret = tsg_metric_init(tsg_conffile, g_tsg_para.logger);
if(ret<0)
{
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_STATISTIC", "tsg_statistic_init failed ...");
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_STATISTIC", "tsg_metric_init failed ...");
return -1;
}

38
src/tsg_entry.h
View File

@@ -117,13 +117,28 @@ enum security_metric_columns
COLUMN_MAX
};
enum TRAFFIC_INFO_IDX
enum app_metric_columns
{
TRAFFIC_INFO_ALLOW=0,
TRAFFIC_INFO_DENY,
TRAFFIC_INFO_MONITOR,
TRAFFIC_INFO_INTERCEPT,
TRAFFIC_INFO_MAX
TRAFFIC_APP_SESSIONS = 0,
TRAFFIC_APP_IN_BYTES,
TRAFFIC_APP_OUT_BYTES,
TRAFFIC_APP_IN_PKTS,
TRAFFIC_APP_OUT_PKTS,
TRAFFIC_APP_C2S_PKTS,
TRAFFIC_APP_S2C_PKTS,
TRAFFIC_APP_C2S_BYTES,
TRAFFIC_APP_S2C_BYTES,
TRAFFIC_APP_C2S_FRAGMENTS,
TRAFFIC_APP_S2C_FRAGMENTS,
TRAFFIC_APP_C2S_LOST_BYTES,
TRAFFIC_APP_S2C_LOST_BYTES,
TRAFFIC_APP_C2S_ORDER_PKTS,
TRAFFIC_APP_S2C_ORDER_PKTS,
TRAFFIC_APP_C2S_RETRANSMITTED_PKTS,
TRAFFIC_APP_S2C_RETRANSMITTED_PKTS,
TRAFFIC_APP_C2S_RETRANSMITTED_BYTES,
TRAFFIC_APP_S2C_RETRANSMITTED_BYTES,
TRAFFIC_APP_MAX
};
struct tsg_statistic
@@ -143,10 +158,17 @@ struct tsg_statistic
int metric_table_id;
unsigned int security_column_ids[COLUMN_MAX];
struct fieldstat_dynamic_instance *security_metric_handle;
int app_metric_cycle;
int app_metric_table_id;
unsigned int app_column_ids[TRAFFIC_APP_MAX];
struct fieldstat_dynamic_instance *app_metric_handle;
};
int tsg_statistic_init(const char *conffile, void *logger);
void tsg_statistic_destroy(void);
int tsg_metric_init(const char *conffile, void *logger);
void tsg_metric_destroy(void);
int tsg_gtp_signaling_hash_init(const char* conffile, void *logger);
int session_application_full_path_update(const struct streaminfo *a_stream, char *app_full_path, int app_full_path_len);

3
src/tsg_rule.cpp
View File

@@ -126,8 +126,7 @@ unsigned short get_redis_port(char *redis_port_range)
}
else
{
int ret=sscanf(buf, "%hu-%hu", &s_port, &e_port);
assert(ret==2);
sscanf(buf, "%hu-%hu", &s_port, &e_port);
}
for(int i=s_port; i<=e_port && port_num<32; i++)

201
src/tsg_send_log.cpp
View File

@@ -688,175 +688,6 @@ static int set_userdefine_app(struct TLD_handle_t *_handle, char *field_name, st
return 1;
}
static unsigned int get_max_app_id(struct gather_app_result *result)
{
int i=0;
unsigned int max_app_id=0;
for(i=0; i< result->app_num; i++)
{
if(max_app_id < result->attributes[i].app_id)
{
max_app_id=result->attributes[i].app_id;
}
}
return max_app_id;
}
static unsigned int get_lastest_app_id(struct gather_app_result *result)
{
if(result->app_num>0)
{
return result->attributes[result->app_num-1].app_id;
}
return 0;
}
/*
L7(openvpn), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon
L7(openvpn), appSketch(psiphon), Thrid(openvpn.qq_web.wechat) = openvpn.psiphon
L7(openvpn), appSketch(psiphon), Thrid(openvpn.wechat) = openvpn.psiphon
L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn.wechat) = openvpn.ssl.psiphon
L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn) = openvpn.ssl.psiphon
L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon
L7(null), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon
L7(null), appSketch(psiphon), Thrid(wechat) = psiphon
L7(null), appSketch(null), Thrid(openvpn.ssl.wechat) = openvpn.ssl.wechat
L7(null), appSketch(null), Thrid(null) = unknown
*/
static int split_l7_and_app(struct gather_app_result *qm_result, struct gather_app_result *qm_l7_result)
{
int i=0;
for(i=qm_result->app_num; i>0; i--)
{
if(tsg_l7_protocol_id2name(qm_result->attributes[i-1].app_id))
{
break;
}
}
qm_l7_result->app_num=i;
qm_l7_result->origin=ORIGIN_QM_ENGINE_L7;
memcpy(qm_l7_result->attributes, qm_result->attributes, sizeof(struct app_attributes)*i);
return 0;
}
/*
** Engine1 1.2 Engine2 1 -> 1.2
** Engine1 1.2 Engine2 1.3 -> 1.2
** Engine1 1 Engine2 1.2 -> 1.2
** Engine1 2 Engine2 1.2 -> 1.2
** Engine1 1 Engine2 2.3 -> 1
** Engine1 1 Engine2 2 -> 1
*/
static int copy_app_id(struct app_attributes *result, int result_num, unsigned int *combined_array, int combined_array_num)
{
int i=0;
for(i=0; i<result_num && i<combined_array_num; i++)
{
combined_array[i]=result[i].app_id;
}
return i;
}
#define COMBINED_APP_ID_NUM MAX_APP_ID_NUM*2
int set_app_info(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct gather_app_result *result)
{
int offset=0;
int combined_num=0;
char app_full_path[256]={0};
unsigned int combined_array[COMBINED_APP_ID_NUM]={0};
if(result[ORIGIN_BASIC_PROTOCOL].app_num>0)
{
combined_num+=copy_app_id(result[ORIGIN_BASIC_PROTOCOL].attributes, result[ORIGIN_BASIC_PROTOCOL].app_num, combined_array, COMBINED_APP_ID_NUM);
}
else
{
combined_num+=copy_app_id(result[ORIGIN_QM_ENGINE_L7].attributes, result[ORIGIN_QM_ENGINE_L7].app_num, combined_array, COMBINED_APP_ID_NUM);
}
if(result[ORIGIN_HITED_APP].app_num > 0)
{
combined_array[combined_num]=get_lastest_app_id(&(result[ORIGIN_HITED_APP]));
combined_num+=1;
}
else if(result[ORIGIN_USER_DEFINE].app_num>0)
{
if(combined_num<COMBINED_APP_ID_NUM)
{
combined_array[combined_num]=get_max_app_id(&(result[ORIGIN_USER_DEFINE]));
combined_num+=1;
}
}
else if(result[ORIGIN_BUILT_IN].app_num>0)
{
combined_num+=copy_app_id(result[ORIGIN_BUILT_IN].attributes, result[ORIGIN_BUILT_IN].app_num, combined_array+combined_num, COMBINED_APP_ID_NUM-combined_num);
}
else if(result[ORIGIN_QM_ENGINE].app_num-result[ORIGIN_QM_ENGINE_L7].app_num > 0)
{
combined_num+=copy_app_id( result[ORIGIN_QM_ENGINE].attributes + result[ORIGIN_QM_ENGINE_L7].app_num,
result[ORIGIN_QM_ENGINE].app_num - result[ORIGIN_QM_ENGINE_L7].app_num,
combined_array+combined_num,
COMBINED_APP_ID_NUM-combined_num
);
}
if(combined_num==0)
{
offset=tsg_get_app_name_by_id(g_tsg_maat_feather, result[ORIGIN_UNKNOWN].attributes[0].app_id, app_full_path, sizeof(app_full_path), 0);
if(offset>0)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_FULL_PATH].name, (void *)app_full_path, TLD_TYPE_STRING);
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)app_full_path, TLD_TYPE_STRING);
return 1;
}
else
{
MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "APP_FULL_PATH", "app_full_path is null, combined_num=0");
}
return 0;
}
for(int i=0; i<combined_num; i++)
{
offset+=tsg_get_app_name_by_id(g_tsg_maat_feather, combined_array[i], app_full_path+offset, sizeof(app_full_path)-offset, 0);
if(offset>0)
{
app_full_path[offset++]='.';
}
}
if(offset>0)
{
app_full_path[offset-1]='\0';
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_FULL_PATH].name, (void *)app_full_path, TLD_TYPE_STRING);
char *app_label=rindex(app_full_path, '.');
if(app_label!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)(app_label+1), TLD_TYPE_STRING);
}
else
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)app_full_path, TLD_TYPE_STRING);
}
}
else
{
MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "APP_FULL_PATH", "app_full_path is null, combined_num=%d", combined_num);
}
return 1;
}
int set_app_id(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, const struct streaminfo *a_stream)
{
struct gather_app_result *gather_result=(struct gather_app_result *)session_gather_app_results_get(a_stream);
@@ -872,11 +703,21 @@ int set_app_id(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handl
return 0;
}
split_l7_and_app(&(gather_result[ORIGIN_QM_ENGINE]), &(gather_result[ORIGIN_QM_ENGINE_L7]));
set_userdefine_app(_handle, _instance->id2field[LOG_COMMON_USERDEFINE_APP].name, &(gather_result[ORIGIN_USER_DEFINE]));
set_app_info(_instance, _handle, gather_result);
char app_full_path[256]={0};
session_application_full_path_update(a_stream, app_full_path, sizeof(app_full_path));
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_FULL_PATH].name, (void *)app_full_path, TLD_TYPE_STRING);
char *app_label=rindex(app_full_path, '.');
if(app_label!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)(app_label+1), TLD_TYPE_STRING);
}
else
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)app_full_path, TLD_TYPE_STRING);
}
set_app_identify_info(_handle, _instance->id2field[LOG_COMMON_APP_IDENTIFY_INFO].name, gather_result);
@@ -1893,7 +1734,6 @@ static int session_record_limit(struct tsg_log_instance_t *_instance, const stru
int log_common_fields_new(const char *filename, id2field_t *id2field, struct topic_stat **service2topic, int *max_service)
{
int i=0,flag=0;
int ret=0,id=0;
FILE *fp=NULL;
char line[1024]={0};
char field_name[64]={0};
@@ -1915,9 +1755,9 @@ int log_common_fields_new(const char *filename, id2field_t *id2field, struct top
{
continue;
}
int id=0;
memset(type_name, 0, sizeof(type_name));
ret=sscanf(line, "%31s %63s %d", type_name, field_name, &id);
assert(ret==3);
sscanf(line, "%31s %63s %d", type_name, field_name, &id);
for(i=0; i<TLD_TYPE_MAX; i++)
{
@@ -2480,3 +2320,14 @@ int tsg_send_payload(struct tsg_log_instance_t *instance, int topic_id, char *pa
return 0;
}
int tsg_unknown_app_id_get(struct tsg_log_instance_t *instance)
{
if(instance!=NULL)
{
return instance->unknown_app_id;
}
return 0;
}

3
src/tsg_send_log_internal.h
View File

@@ -236,7 +236,8 @@ struct tsg_log_instance_t
void *logger;
};
int tsg_unknown_app_id_get(struct tsg_log_instance_t *instance);
char *log_field_id2name(struct tsg_log_instance_t *instance, enum LOG_FIELD_ID id);
struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile, screen_stat_handle_t fs2_handle);
void tsg_sendlog_destroy(struct tsg_log_instance_t * instance);

184
src/tsg_statistic.cpp
View File

@@ -2,7 +2,6 @@
#include <unistd.h>
#include <pthread.h>
#include <MESA/field_stat2.h>
#include <MESA/fieldstat.h>
#include <MESA/MESA_prof_load.h>
#include <MESA/MESA_handle_logger.h>
@@ -28,6 +27,14 @@ enum traffic_metric_tags
TRAFFIC_TAG_MAX
};
enum app_metric_tags
{
APP_TAG_VSYS_ID = 0,
APP_TAG_PROTOCOL_LABEL,
APP_TAG_APP_FULL_PATH,
APP_TAG_MAX
};
int tsg_set_statistic_opt(int value, enum _STATISTIC_OPT_TYPE type, int thread_seq)
{
switch(type)
@@ -37,16 +44,16 @@ int tsg_set_statistic_opt(int value, enum _STATISTIC_OPT_TYPE type, int thread_s
case OPT_TYPE_PINNING_YES:
case OPT_TYPE_PINNING_NOT:
case OPT_TYPE_PINNING_MAYBE:
atomic_add(&(g_tsg_statis_para.statistic_opt[type]), value);
break;
default:
break;
atomic_add(&(g_tsg_statis_para.statistic_opt[type]), value);
break;
default:
break;
}
return 0;
}
int tsg_set_intercept_flow(struct maat_rule *p_result, struct _traffic_info *traffic_info, int thread_seq)
int tsg_set_intercept_flow(struct maat_rule *p_result, struct traffic_info *traffic_info, int thread_seq)
{
if (p_result == NULL || traffic_info == NULL || thread_seq < 0 || thread_seq >= get_thread_count() || p_result->service_id != TSG_SERVICE_INTERCEPT)
{
@@ -182,11 +189,83 @@ static void *tsg_statistic_thread(void *arg)
return NULL;
}
int tsg_set_application_metrics(const struct streaminfo *a_stream, const char *l4_protocol, const char *app_full_path, struct traffic_packet_info *app_statis, int thread_seq)
{
if (a_stream == NULL || l4_protocol == NULL || app_full_path == NULL || app_statis == NULL)
{
return -1;
}
struct fieldstat_tag app_tags[APP_TAG_MAX] = {{"vsys_id", 0, -1}, {"protocol_label", 2, 0}, {"app_full_path", 2, 0}};
app_tags[APP_TAG_VSYS_ID].value_int = g_tsg_statis_para.vsystem_id;
app_tags[APP_TAG_PROTOCOL_LABEL].value_str = l4_protocol;
app_tags[APP_TAG_APP_FULL_PATH].value_str = app_full_path;
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_SESSIONS], "traffic_application_protocol_stat",
app_statis->sessions, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_IN_BYTES], "traffic_application_protocol_stat",
app_statis->in_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_IN_PKTS], "traffic_application_protocol_stat",
app_statis->in_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_OUT_BYTES], "traffic_application_protocol_stat",
app_statis->out_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_OUT_PKTS], "traffic_application_protocol_stat",
app_statis->out_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_PKTS], "traffic_application_protocol_stat",
app_statis->c2s_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_PKTS], "traffic_application_protocol_stat",
app_statis->s2c_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_BYTES], "traffic_application_protocol_stat",
app_statis->c2s_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_BYTES], "traffic_application_protocol_stat",
app_statis->s2c_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_FRAGMENTS], "traffic_application_protocol_stat",
app_statis->c2s_fragments, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_FRAGMENTS], "traffic_application_protocol_stat",
app_statis->s2c_fragments, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_LOST_BYTES], "traffic_application_protocol_stat",
app_statis->c2s_tcp_lost_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_LOST_BYTES], "traffic_application_protocol_stat",
app_statis->s2c_tcp_lost_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_ORDER_PKTS], "traffic_application_protocol_stat",
app_statis->c2s_tcp_ooorder_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_ORDER_PKTS], "traffic_application_protocol_stat",
app_statis->s2c_tcp_ooorder_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_RETRANSMITTED_PKTS], "traffic_application_protocol_stat",
app_statis->c2s_tcp_retransmitted_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_RETRANSMITTED_PKTS], "traffic_application_protocol_stat",
app_statis->s2c_tcp_retransmitted_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_RETRANSMITTED_BYTES], "traffic_application_protocol_stat",
app_statis->c2s_tcp_retransmitted_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq);
fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_RETRANSMITTED_BYTES], "traffic_application_protocol_stat",
app_statis->s2c_tcp_retransmitted_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq);
return 0;
}
int tsg_security_metric_init(const char *conffile, void *logger)
{
if (conffile == NULL || logger == NULL)
{
return -1;
return -1;
}
unsigned short fs_server_port=0;
@@ -200,21 +279,21 @@ int tsg_security_metric_init(const char *conffile, void *logger)
MESA_load_profile_int_def(conffile, "SECURITY_HITS", "CYCLE", &g_tsg_statis_para.metric_cycle, 1000);
if(g_tsg_statis_para.metric_cycle<=0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.metric_cycle error");
return -1;
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.metric_cycle error");
return -1;
}
g_tsg_statis_para.security_metric_handle = fieldstat_dynamic_instance_new(app_name, thread_num);
if (g_tsg_statis_para.security_metric_handle == NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.security_metric_handle error");
return -1;
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.security_metric_handle error");
return -1;
}
fieldstat_dynamic_set_output_interval(g_tsg_statis_para.security_metric_handle, g_tsg_statis_para.metric_cycle);
if (fs_server_port > 0 && strlen(fs_server_ip) > 0)
{
fieldstat_dynamic_set_line_protocol_server(g_tsg_statis_para.security_metric_handle, fs_server_ip, fs_server_port);
fieldstat_dynamic_set_line_protocol_server(g_tsg_statis_para.security_metric_handle, fs_server_ip, fs_server_port);
}
enum field_type security_metric_type[COLUMN_MAX] = {FIELD_TYPE_COUNTER};
@@ -222,8 +301,8 @@ int tsg_security_metric_init(const char *conffile, void *logger)
g_tsg_statis_para.metric_table_id = fieldstat_register_dynamic_table(g_tsg_statis_para.security_metric_handle, "security_rule_hits", security_metric_field, security_metric_type, (size_t)COLUMN_MAX, g_tsg_statis_para.security_column_ids);
if(g_tsg_statis_para.metric_table_id<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.metric_table_id error");
return -1;
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.metric_table_id error");
return -1;
}
fieldstat_dynamic_instance_start(g_tsg_statis_para.security_metric_handle);
@@ -231,7 +310,58 @@ int tsg_security_metric_init(const char *conffile, void *logger)
return 0;
}
int tsg_statistic_init(const char *conffile, void *logger)
int tsg_traffic_metric_init(const char *conffile, void *logger)
{
if (conffile == NULL || logger == NULL)
{
return -1;
}
unsigned short fs_server_port = 0;
char fs_server_ip[MAX_IPV4_LEN] = {0};
char app_name[128] = {0};
int thread_num = get_thread_count();
MESA_load_profile_short_nodef(conffile, "APP_METRIC", "TELEGRAF_PORT", (short *)&(fs_server_port));
MESA_load_profile_string_nodef(conffile, "APP_METRIC", "TELEGRAF_IP", fs_server_ip, sizeof(fs_server_ip));
MESA_load_profile_string_def(conffile, "APP_METRIC", "APP_NAME", app_name, sizeof(app_name), "app_metric");
MESA_load_profile_int_def(conffile, "APP_METRIC", "CYCLE", &g_tsg_statis_para.app_metric_cycle, 1000);
if (g_tsg_statis_para.app_metric_cycle <= 0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "APP_METRIC", "g_tsg_statis_para.app_metric_cycle error");
return -1;
}
g_tsg_statis_para.app_metric_handle = fieldstat_dynamic_instance_new(app_name, thread_num);
if (g_tsg_statis_para.app_metric_handle == NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "APP_METRIC", "g_tsg_statis_para.app_metric_handle error");
return -1;
}
fieldstat_dynamic_set_output_interval(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_cycle);
if (fs_server_port > 0 && strlen(fs_server_ip) > 0)
{
fieldstat_dynamic_set_line_protocol_server(g_tsg_statis_para.app_metric_handle, fs_server_ip, fs_server_port);
}
enum field_type app_metric_type[TRAFFIC_APP_MAX] = {FIELD_TYPE_COUNTER};
const char *app_metric_field[TRAFFIC_APP_MAX] = {"sessions", "in_bytes", "out_bytes", "in_pkts", "out_pkts", "c2s_pkts", "s2c_pkts", "c2s_bytes", "s2c_bytes", "c2s_fragments", "s2c_fragments",
"c2s_tcp_lost_bytes", "s2c_tcp_lost_bytes", "c2s_tcp_ooorder_pkts", "s2c_tcp_ooorder_pkts", "c2s_tcp_retransmitted_pkts", "s2c_tcp_retransmitted_pkts",
"c2s_tcp_retransmitted_bytes", "s2c_tcp_retransmitted_bytes"};
g_tsg_statis_para.app_metric_table_id = fieldstat_register_dynamic_table(g_tsg_statis_para.app_metric_handle, "traffic_application_protocol_stat", app_metric_field, app_metric_type, (size_t)TRAFFIC_APP_MAX, g_tsg_statis_para.app_column_ids);
if (g_tsg_statis_para.app_metric_table_id < 0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "APP_METRIC", "g_tsg_statis_para.app_metric_table_id error");
return -1;
}
fieldstat_dynamic_instance_start(g_tsg_statis_para.app_metric_handle);
return 0;
}
int tsg_metric_init(const char *conffile, void *logger)
{
unsigned short fs_server_port = 0;
char app_name[128]={0};
@@ -243,8 +373,8 @@ int tsg_statistic_init(const char *conffile, void *logger)
MESA_load_profile_int_def(conffile, "STATISTIC", "CYCLE", &g_tsg_statis_para.cycle, 1);
if(g_tsg_statis_para.cycle<=0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "STATISTIC", "Disabale traffic statistic");
return 0;
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "STATISTIC", "Disabale traffic statistic");
return 0;
}
MESA_load_profile_short_nodef(conffile, "STATISTIC", "TELEGRAF_PORT", (short *)&(fs_server_port));
@@ -263,7 +393,7 @@ int tsg_statistic_init(const char *conffile, void *logger)
if (fs_server_port > 0 && strlen(fs_server_ip) > 0)
{
fieldstat_set_line_protocol_server(g_tsg_statis_para.statistic_handle, fs_server_ip, fs_server_port);
fieldstat_set_line_protocol_server(g_tsg_statis_para.statistic_handle, fs_server_ip, fs_server_port);
}
const char *static_column_name[STATIS_MAX] = {"sessions", "active_sessions", "closed_sessions", "in_bytes", "out_bytes", "in_pkts", "out_pkts", "asymmetric_c2s_flows", "asymmetric_s2c_flows"};
@@ -279,8 +409,15 @@ int tsg_statistic_init(const char *conffile, void *logger)
int ret = tsg_security_metric_init(conffile, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "INIT_METRIC", "tsg_security_metric_init failed ...");
return -1;
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "INIT_METRIC", "tsg_security_metric_init failed ...");
return -1;
}
ret = tsg_traffic_metric_init(conffile, logger);
if (ret < 0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "INIT_METRIC", "tsg_traffic_metric_init failed ...");
return -1;
}
pthread_create(&g_tsg_statis_para.stat_thread_id, NULL, tsg_statistic_thread, NULL);
@@ -288,13 +425,12 @@ int tsg_statistic_init(const char *conffile, void *logger)
return 0;
}
void tsg_statistic_destroy(void)
void tsg_metric_destroy(void)
{
pthread_cancel(g_tsg_statis_para.stat_thread_id);
g_tsg_statis_para.thread_alive=0;
g_tsg_statis_para.thread_alive = 0;
sleep(g_tsg_statis_para.cycle);
fieldstat_instance_free(g_tsg_statis_para.statistic_handle);
return ;
return;
}

1
src/tsg_variable.h
View File

@@ -131,6 +131,7 @@ struct tsg_rt_para
int shaping_sid;
int intercept_sid;
int send_resetall;
int app_metric_update_interval_ms;
enum DEPLOY_MODE deploy_mode;
int scan_time_interval;
int identify_app_max_pkt_num;

9
test/src/CMakeLists.txt
View File

@@ -42,18 +42,19 @@ set(TSG_MASTER_SRC ${PROJECT_SOURCE_DIR}/src/tsg_entry.cpp
${PROJECT_SOURCE_DIR}/src/tsg_variable.cpp
${PROJECT_SOURCE_DIR}/src/tsg_protocol.cpp
${PROJECT_SOURCE_DIR}/src/tsg_proxy.cpp
${PROJECT_SOURCE_DIR}/src/mpack.c
)
add_executable(gtest_master ${TSG_MASTER_SRC} ${PROJECT_SOURCE_DIR}/src/mpack.c gtest_kafka.cpp gtest_common.cpp gtest_master.cpp)
add_executable(gtest_master ${TSG_MASTER_SRC} gtest_kafka.cpp gtest_common.cpp gtest_master.cpp)
target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3)
add_executable(gtest_sync_state ${PROJECT_SOURCE_DIR}/src/tsg_sync_state.cpp ${PROJECT_SOURCE_DIR}/src/mpack.c gtest_common.cpp gtest_sync_state.cpp)
target_link_libraries(gtest_sync_state gtest-static cjson ctemplate-static MESA_handle_logger)
add_executable(gtest_fieldstat3 ${PROJECT_SOURCE_DIR}/src/tsg_statistic.cpp gtest_common.cpp gtest_fieldstat3.cpp)
target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static MESA_field_stat2 fieldstat3 MESA_prof_load MESA_handle_logger)
add_executable(gtest_fieldstat3 ${TSG_MASTER_SRC} gtest_common.cpp gtest_kafka.cpp gtest_fieldstat3.cpp)
target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3)
add_executable(gtest_proxy ${TSG_MASTER_SRC} ${PROJECT_SOURCE_DIR}/src/mpack.c gtest_common.cpp gtest_proxy.cpp)
add_executable(gtest_proxy ${TSG_MASTER_SRC} gtest_common.cpp gtest_proxy.cpp)
target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap)
file(COPY ../pcap/ DESTINATION ./pcap/)

6
test/src/gtest_common.cpp
View File

@@ -7,7 +7,7 @@ struct gtest_ctrl_pkt
{
char buf[1024];
int len;
}ctrl_pkt;
};
void dictator_free(int thread_seq, void * pbuf)
{
@@ -40,9 +40,11 @@ int sapp_inject_pkt(struct streaminfo * stream, enum sapp_inject_opt sio, const
return 0;
}
struct gtest_ctrl_pkt ctrl_pkt;
int sapp_inject_ctrl_pkt(struct streaminfo * stream, enum sapp_inject_opt sio, const void * payload, int payload_len, unsigned char snd_routedir)
{
memset(&ctrl_pkt, 0, 1050);
memset(&ctrl_pkt, 0, sizeof(ctrl_pkt));
memcpy(ctrl_pkt.buf, payload, payload_len);
ctrl_pkt.len = payload_len;

10
test/src/gtest_fieldstat3.cpp
View File

@@ -17,7 +17,7 @@ pthread_t g_pid[8];
TEST(FIELDSATA3, InterceptIllegalParameter)
{
struct _traffic_info _info;
struct traffic_info _info;
struct maat_rule p_result;
int ret = tsg_set_intercept_flow(NULL, &_info, 0);
@@ -47,7 +47,7 @@ TEST(FIELDSATA3, PolicyIllegalParameter)
TEST(FIELDSATA3, Intercept)
{
struct _traffic_info _info;
struct traffic_info _info;
struct maat_rule p_result;
_info.con_num = 10;
@@ -97,7 +97,7 @@ void *run_time_funtion(void *arg)
struct maat_rule p_result;
struct streaminfo a_stream;
struct _traffic_info _info;
struct traffic_info _info;
while (1)
{
@@ -155,7 +155,7 @@ TEST(FIELDSATA3, MultiThreading)
int main(int argc, char *argv[])
{
void *logger = MESA_create_runtime_log_handle("log/gtest_fieldstat3.log", RLOG_LV_FATAL);
tsg_statistic_init(tsg_gtest_conffile, logger);
tsg_metric_init(tsg_gtest_conffile, logger);
testing::InitGoogleTest(&argc, argv);
int ret = RUN_ALL_TESTS();
// sleep(30);
@@ -164,7 +164,7 @@ int main(int argc, char *argv[])
// pthread_cancel(g_pid[i]);
// }
tsg_statistic_destroy();
tsg_metric_destroy();
return ret;
}

5
test/src/gtest_sendlog.cpp
View File

@@ -148,6 +148,11 @@ int tsg_set_policy_flow(const struct streaminfo * a_stream, struct maat_rule * p
return 0;
}
int session_application_full_path_update(const struct streaminfo * a_stream, char * app_full_path, int app_full_path_len)
{
return 0;
}
TEST(Master, SendInterimRecord)
{
struct streaminfo a_stream={0};