From 380733e78be9e5adb5fc0973689038a7ebcc70ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AD=A6=E5=88=A9?= Date: Fri, 2 Jun 2023 08:19:50 +0000 Subject: [PATCH] =?UTF-8?q?TSG-14945:=20=E8=BE=93=E5=87=BAApplication=20an?= =?UTF-8?q?d=20Protocol=20Metrics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/main.conf | 12 +- inc/tsg_statistic.h | 29 +- src/tsg_bridge.cpp | 5 + src/tsg_bridge.h | 7 +- src/tsg_entry.cpp | 480 ++++++++++++++++++++++++++++++++-- src/tsg_entry.h | 38 ++- src/tsg_rule.cpp | 3 +- src/tsg_send_log.cpp | 201 ++------------ src/tsg_send_log_internal.h | 3 +- src/tsg_statistic.cpp | 186 +++++++++++-- src/tsg_variable.h | 1 + test/src/CMakeLists.txt | 9 +- test/src/gtest_common.cpp | 6 +- test/src/gtest_fieldstat3.cpp | 10 +- test/src/gtest_sendlog.cpp | 5 + 15 files changed, 751 insertions(+), 244 deletions(-) diff --git a/bin/main.conf b/bin/main.conf index 6b8e5ed..24c7a5d 100644 --- a/bin/main.conf +++ b/bin/main.conf @@ -27,17 +27,23 @@ UDP_MIN_BYTES=5 [SECURITY_HITS] CYCLE=1000 -TELEGRAF_PORT=8092 +TELEGRAF_PORT=8400 TELEGRAF_IP="127.0.0.1" APP_NAME="security_rule_hits" [STATISTIC] -CYCLE=1 -TELEGRAF_PORT=8092 +CYCLE=5 +TELEGRAF_PORT=8100 TELEGRAF_IP="127.0.0.1" OUTPUT_PATH="./tsg_statistic.log" APP_NAME="network_activity" +[APP_METRIC] +CYCLE=1000 +TELEGRAF_PORT=8100 +TELEGRAF_IP="127.0.0.1" +APP_NAME="app_metric" + [FIELD_STAT] CYCLE=30 TELEGRAF_PORT=8125 diff --git a/inc/tsg_statistic.h b/inc/tsg_statistic.h index e13f3f8..d8a86b6 100644 --- a/inc/tsg_statistic.h +++ b/inc/tsg_statistic.h @@ -11,7 +11,7 @@ enum _STATISTIC_OPT_TYPE _OPT_TYPE_MAX }; -struct _traffic_info +struct traffic_info { long long con_num; long long in_bytes; @@ -20,8 +20,33 @@ struct _traffic_info long long out_packets; }; +struct traffic_packet_info +{ + uint32_t sessions; + uint32_t in_pkts; + uint32_t out_pkts; + uint32_t c2s_pkts; + uint32_t s2c_pkts; + uint32_t c2s_fragments; + uint32_t s2c_fragments; + uint32_t c2s_tcp_ooorder_pkts; + uint32_t s2c_tcp_ooorder_pkts; + uint32_t c2s_tcp_retransmitted_pkts; + uint32_t s2c_tcp_retransmitted_pkts; + uint64_t in_bytes; + uint64_t out_bytes; + uint64_t c2s_bytes; + uint64_t s2c_bytes; + uint64_t c2s_tcp_lost_bytes; + uint64_t s2c_tcp_lost_bytes; + uint64_t c2s_tcp_retransmitted_bytes; + uint64_t s2c_tcp_retransmitted_bytes; +}; + int tsg_set_policy_flow(const struct streaminfo *a_stream, struct maat_rule *p_result, int thread_seq); -int tsg_set_intercept_flow(struct maat_rule *p_result, struct _traffic_info *traffic_info, int thread_seq); +int tsg_set_intercept_flow(struct maat_rule *p_result, struct traffic_info *traffic_info, int thread_seq); int tsg_set_statistic_opt(int value, enum _STATISTIC_OPT_TYPE type, int thread_seq); +int tsg_set_application_metrics(const struct streaminfo *a_stream, const char *l4_protocol, const char *app_full_path, struct traffic_packet_info *app_statis, int thread_seq); + #endif diff --git a/src/tsg_bridge.cpp b/src/tsg_bridge.cpp index 3dab94f..f6032aa 100644 --- a/src/tsg_bridge.cpp +++ b/src/tsg_bridge.cpp @@ -135,6 +135,11 @@ void session_runtime_action_context_free(const struct streaminfo *a_stream, int } } + if (srt_action_context->l4_protocol != NULL) + { + dictator_free(a_stream->threadnum, srt_action_context->l4_protocol); + } + dictator_free(a_stream->threadnum, data); data=NULL; } diff --git a/src/tsg_bridge.h b/src/tsg_bridge.h index b35c1ec..276331a 100644 --- a/src/tsg_bridge.h +++ b/src/tsg_bridge.h @@ -6,6 +6,8 @@ #include "tsg_label.h" #include "tsg_rule.h" #include "tsg_leaky_bucket.h" +#include "tsg_send_log_internal.h" +#include "tsg_statistic.h" #ifndef MAX_BRIDGE_NAME_LEN #define MAX_BRIDGE_NAME_LEN 64 @@ -99,9 +101,10 @@ struct session_runtime_action_context char udp_data_dropme; char set_latency_flag; char direction; - char padding[5]; + char padding[1]; enum TSG_METHOD_TYPE method_type; enum TSG_PROTOCOL protocol; + char *l4_protocol; union { struct leaky_bucket *bucket; @@ -110,6 +113,8 @@ struct session_runtime_action_context struct hited_app_para hited_para; void *para; }; + long long last_update_metric_time; + struct traffic_packet_info last_app_statis; }; struct udp_session_runtime_context diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index 70f88ff..5abf504 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -687,6 +687,444 @@ static int srt_process_context_set_l7_protocol(struct session_runtime_process_co return 0; } +/* + L7(openvpn), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon + L7(openvpn), appSketch(psiphon), Thrid(openvpn.qq_web.wechat) = openvpn.psiphon + L7(openvpn), appSketch(psiphon), Thrid(openvpn.wechat) = openvpn.psiphon + L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn.wechat) = openvpn.ssl.psiphon + L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn) = openvpn.ssl.psiphon + L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon + L7(null), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon + L7(null), appSketch(psiphon), Thrid(wechat) = psiphon + L7(null), appSketch(null), Thrid(openvpn.ssl.wechat) = openvpn.ssl.wechat + L7(null), appSketch(null), Thrid(null) = unknown +*/ + +static int session_qm_result_l7_protocol_split(struct gather_app_result *qm_result, struct gather_app_result *qm_l7_result) +{ + int i=0; + for(i=qm_result->app_num; i>0; i--) + { + if(tsg_l7_protocol_id2name(qm_result->attributes[i-1].app_id)) + { + break; + } + } + + qm_l7_result->app_num=i; + qm_l7_result->origin=ORIGIN_QM_ENGINE_L7; + memcpy(qm_l7_result->attributes, qm_result->attributes, sizeof(struct app_attributes)*i); + + return 0; +} + +static unsigned int get_max_app_id(struct gather_app_result *result) +{ + int i=0; + unsigned int max_app_id=0; + + for(i=0; i< result->app_num; i++) + { + if(max_app_id < result->attributes[i].app_id) + { + max_app_id=result->attributes[i].app_id; + } + } + + return max_app_id; +} + +static unsigned int get_lastest_app_id(struct gather_app_result *result) +{ + if(result->app_num>0) + { + return result->attributes[result->app_num-1].app_id; + } + + return 0; +} + +/* +** Engine1 1.2 Engine2 1 -> 1.2 +** Engine1 1.2 Engine2 1.3 -> 1.2 +** Engine1 1 Engine2 1.2 -> 1.2 +** Engine1 2 Engine2 1.2 -> 1.2 +** Engine1 1 Engine2 2.3 -> 1 +** Engine1 1 Engine2 2 -> 1 +*/ + +static int copy_app_id(struct app_attributes *result, int result_num, unsigned int *combined_array, int combined_array_num) +{ + int i=0; + for(i=0; i0) + { + combined_num+=copy_app_id(result[ORIGIN_BASIC_PROTOCOL].attributes, result[ORIGIN_BASIC_PROTOCOL].app_num, combined_array, COMBINED_APP_ID_NUM); + } + else + { + combined_num+=copy_app_id(result[ORIGIN_QM_ENGINE_L7].attributes, result[ORIGIN_QM_ENGINE_L7].app_num, combined_array, COMBINED_APP_ID_NUM); + } + + if(result[ORIGIN_HITED_APP].app_num > 0) + { + combined_array[combined_num]=get_lastest_app_id(&(result[ORIGIN_HITED_APP])); + combined_num+=1; + } + else if(result[ORIGIN_USER_DEFINE].app_num>0) + { + if(combined_num0) + { + combined_num+=copy_app_id(result[ORIGIN_BUILT_IN].attributes, result[ORIGIN_BUILT_IN].app_num, combined_array+combined_num, COMBINED_APP_ID_NUM-combined_num); + } + else if(result[ORIGIN_QM_ENGINE].app_num-result[ORIGIN_QM_ENGINE_L7].app_num > 0) + { + combined_num+=copy_app_id( result[ORIGIN_QM_ENGINE].attributes + result[ORIGIN_QM_ENGINE_L7].app_num, + result[ORIGIN_QM_ENGINE].app_num - result[ORIGIN_QM_ENGINE_L7].app_num, + combined_array+combined_num, + COMBINED_APP_ID_NUM-combined_num + ); + } + + if(combined_num==0) + { + offset=tsg_get_app_name_by_id(g_tsg_maat_feather, result[ORIGIN_UNKNOWN].attributes[0].app_id, out_full_path, out_full_path_len, 0); + if(offset>0) + { + return offset; + } + + return 0; + } + + for(int i=0; i0) + { + out_full_path[offset++]='.'; + } + } + + if(offset>0) + { + out_full_path[offset-1]='\0'; + } + + return offset; +} + +int session_application_full_path_update(const struct streaminfo *a_stream, char *app_full_path, int app_full_path_len) +{ + struct gather_app_result *gather_result=(struct gather_app_result *)session_gather_app_results_get(a_stream); + if(gather_result!=NULL) + { + session_qm_result_l7_protocol_split(&(gather_result[ORIGIN_QM_ENGINE]), &(gather_result[ORIGIN_QM_ENGINE_L7])); + session_application_full_path_combine(gather_result, app_full_path, app_full_path_len); + } + else + { + tsg_get_app_name_by_id(g_tsg_maat_feather, tsg_unknown_app_id_get(g_tsg_log_instance), app_full_path, app_full_path_len, 0); + } + + return 1; +} + +static const char *session_addr_type_string_convert(UCHAR addrtype) +{ + switch (addrtype) + { + case ADDR_TYPE_IPV4: + case __ADDR_TYPE_IP_PAIR_V4: + return ".IPv4"; + case ADDR_TYPE_IPV6: + case __ADDR_TYPE_IP_PAIR_V6: + return ".IPv6"; + case ADDR_TYPE_VLAN: + return ".VLAN"; + case ADDR_TYPE_MAC: + return ".ETHERNET"; + case ADDR_TYPE_ARP: + return ".ARP"; + case ADDR_TYPE_GRE: + return ".GRE"; + case ADDR_TYPE_MPLS: + return ".MPLS"; + case ADDR_TYPE_PPPOE_SES: + return ".PPPOE"; + case ADDR_TYPE_TCP: + return ".TCP"; + case ADDR_TYPE_UDP: + return ".UDP"; + case ADDR_TYPE_L2TP: + return ".L2TP"; + case ADDR_TYPE_PPP: + return ".PPP"; + case ADDR_TYPE_PPTP: + return ".PPTP"; + case ADDR_TYPE_MAC_IN_MAC: + return ".MAC_IN_MAC"; + case ADDR_TYPE_GPRS_TUNNEL: + return ".GTP"; + case ADDR_TYPE_VXLAN: + return ".VXLAN"; + default: + return NULL; + } +} + +#define MAX_L4_PROTOCOL_STR_LEN 512 +char *session_l4_protocol_label_update(const struct streaminfo *a_stream, char **l4_protocol) +{ + if(*l4_protocol!=NULL || a_stream == NULL) + { + return *l4_protocol; + } + + const char *l4_protocol_str = NULL; + int combined_l4_protocol_offset=MAX_L4_PROTOCOL_STR_LEN; + char combined_l4_protocol_str[MAX_L4_PROTOCOL_STR_LEN]={0}; + struct streaminfo *cur_stream = (struct streaminfo *)a_stream; + + while (cur_stream != NULL) + { + switch (cur_stream->type) + { + case STREAM_TYPE_TCP: + if (cur_stream->addr.addrtype == ADDR_TYPE_IPV4) + { + l4_protocol_str = ".IPv4.TCP"; + } + else if (cur_stream->addr.addrtype == ADDR_TYPE_IPV6) + { + l4_protocol_str = ".IPv6.TCP"; + } + break; + case STREAM_TYPE_UDP: + if (cur_stream->addr.addrtype == ADDR_TYPE_IPV4) + { + l4_protocol_str = ".IPv4.UDP"; + } + else if (cur_stream->addr.addrtype == ADDR_TYPE_IPV6) + { + l4_protocol_str = ".IPv6.UDP"; + } + break; + case STREAM_TYPE_VLAN: + l4_protocol_str = ".VLAN"; break; + case STREAM_TYPE_PPPOE: + l4_protocol_str = ".PPPOE"; break; + case STREAM_TYPE_L2TP: + l4_protocol_str = ".L2TP"; break; + case STREAM_TYPE_OPENVPN: + l4_protocol_str = ".OpenVPN"; break; + case STREAM_TYPE_PPTP: + l4_protocol_str = ".PPTP"; break; + case STREAM_TYPE_NON: + case STREAM_TYPE_ISAKMP: + l4_protocol_str = session_addr_type_string_convert(cur_stream->addr.addrtype); + break; + // SOCKS4 SOCKS5 HTTP + default: + continue; + } + + int n_l4_protocol_str = strlen(l4_protocol_str); + if (n_l4_protocol_str!=0) + { + if (combined_l4_protocol_offset - n_l4_protocol_str < 0) + { + break; + } + memcpy(combined_l4_protocol_str+combined_l4_protocol_offset-n_l4_protocol_str, l4_protocol_str, n_l4_protocol_str); + combined_l4_protocol_offset-=n_l4_protocol_str; + l4_protocol_str=NULL; + } + + cur_stream = cur_stream->pfather; + } + + if(combined_l4_protocol_offset>0 && combined_l4_protocol_offsetudp_flow_project_id); + if (uflow_stat == NULL) + { + return -1; + } + + increment->c2s_pkts = uflow_stat->C2S_pkt - last_value->c2s_pkts; + last_value->c2s_pkts = uflow_stat->C2S_pkt; + + increment->s2c_pkts = uflow_stat->S2C_pkt - last_value->s2c_pkts; + last_value->s2c_pkts = uflow_stat->S2C_pkt; + + increment->c2s_bytes = uflow_stat->C2S_all_byte_raw - last_value->c2s_bytes; + last_value->c2s_bytes = uflow_stat->C2S_all_byte_raw; + + increment->s2c_bytes = uflow_stat->S2C_all_byte_raw - last_value->s2c_bytes; + last_value->s2c_bytes = uflow_stat->S2C_all_byte_raw; + + increment->c2s_fragments = uflow_stat->C2S_ip_fragment_pkt - last_value->c2s_fragments; + last_value->c2s_fragments = uflow_stat->C2S_ip_fragment_pkt; + + increment->s2c_fragments = uflow_stat->S2C_ip_fragment_pkt - last_value->s2c_fragments; + last_value->s2c_fragments = uflow_stat->S2C_ip_fragment_pkt; + + return 0; +} + +int session_calculate_tcp_packets_increment(const struct streaminfo *a_stream, struct traffic_packet_info *last_value, struct traffic_packet_info *increment) +{ + struct tcp_flow_stat *tflow_stat = (struct tcp_flow_stat *)project_req_get_struct(a_stream, g_tsg_log_instance->tcp_flow_project_id); + if (tflow_stat == NULL) + { + return -1; + } + + increment->c2s_pkts = tflow_stat->C2S_all_pkt - last_value->c2s_pkts; + last_value->c2s_pkts = tflow_stat->C2S_all_pkt; + + increment->s2c_pkts = tflow_stat->S2C_all_pkt - last_value->s2c_pkts; + last_value->s2c_pkts = tflow_stat->S2C_all_pkt; + + increment->c2s_bytes = tflow_stat->C2S_all_byte_raw - last_value->c2s_bytes; + last_value->c2s_bytes = tflow_stat->C2S_all_byte_raw; + + increment->s2c_bytes = tflow_stat->S2C_all_byte_raw - last_value->s2c_bytes; + last_value->s2c_bytes = tflow_stat->S2C_all_byte_raw; + + increment->c2s_fragments = tflow_stat->C2S_ip_fragment_pkt - last_value->c2s_fragments; + last_value->c2s_fragments = tflow_stat->C2S_ip_fragment_pkt; + + increment->s2c_fragments = tflow_stat->S2C_ip_fragment_pkt - last_value->s2c_fragments; + last_value->s2c_fragments = tflow_stat->S2C_ip_fragment_pkt; + + increment->c2s_tcp_lost_bytes = tflow_stat->C2S_seq_gap_loss_cnt - last_value->c2s_tcp_lost_bytes; + last_value->c2s_tcp_lost_bytes = tflow_stat->C2S_seq_gap_loss_cnt; + + increment->s2c_tcp_lost_bytes = tflow_stat->S2C_seq_gap_loss_cnt - last_value->s2c_tcp_lost_bytes; + last_value->s2c_tcp_lost_bytes = tflow_stat->S2C_seq_gap_loss_cnt; + + increment->c2s_tcp_ooorder_pkts = tflow_stat->C2S_ooorder_pkt - last_value->c2s_tcp_ooorder_pkts; + last_value->c2s_tcp_ooorder_pkts = tflow_stat->C2S_ooorder_pkt; + + increment->s2c_tcp_ooorder_pkts = tflow_stat->S2C_ooorder_pkt - last_value->s2c_tcp_ooorder_pkts; + last_value->s2c_tcp_ooorder_pkts = tflow_stat->S2C_ooorder_pkt; + + increment->c2s_tcp_retransmitted_pkts = tflow_stat->C2S_retransmission_pkt - last_value->c2s_tcp_retransmitted_pkts; + last_value->c2s_tcp_retransmitted_pkts = tflow_stat->C2S_retransmission_pkt; + + increment->s2c_tcp_retransmitted_pkts = tflow_stat->S2C_retransmission_pkt - last_value->s2c_tcp_retransmitted_pkts; + last_value->s2c_tcp_retransmitted_pkts = tflow_stat->S2C_retransmission_pkt; + + increment->c2s_tcp_retransmitted_bytes = tflow_stat->C2S_retransmission_byte - last_value->c2s_tcp_retransmitted_bytes; + last_value->c2s_tcp_retransmitted_bytes = tflow_stat->C2S_retransmission_byte; + + increment->s2c_tcp_retransmitted_bytes = tflow_stat->S2C_retransmission_byte - last_value->s2c_tcp_retransmitted_bytes; + last_value->s2c_tcp_retransmitted_bytes = tflow_stat->S2C_retransmission_byte; + + return 0; +} + +int session_calculate_packets_increment(const struct streaminfo *a_stream, struct traffic_packet_info *last_value, struct traffic_packet_info *increment, int thread_seq) +{ + if (a_stream == NULL || last_value == NULL || increment == NULL) + { + return -1; + } + + unsigned long long value = 0; + int value_len = sizeof(unsigned long long); + + increment->sessions = 1-last_value->sessions; + last_value->sessions = 1; + + value = 0; + MESA_get_stream_opt(a_stream, MSO_TOTAL_INBOUND_BYTE_RAW, (void *)&value, &value_len); + increment->in_bytes = (uint64_t)value - last_value->in_bytes; + last_value->in_bytes = (uint64_t)value; + + value = 0; + MESA_get_stream_opt(a_stream, MSO_TOTAL_INBOUND_PKT, (void *)&value, &value_len); + increment->in_pkts = (uint64_t)value - last_value->in_pkts; + last_value->in_pkts = (uint64_t)value; + + value = 0; + MESA_get_stream_opt(a_stream, MSO_TOTAL_OUTBOUND_BYTE_RAW, (void *)&value, &value_len); + increment->out_bytes = (uint64_t)value - last_value->out_bytes; + last_value->out_bytes = (uint64_t)value; + + value = 0; + MESA_get_stream_opt(a_stream, MSO_TOTAL_OUTBOUND_PKT, (void *)&value, &value_len); + increment->out_pkts = (uint64_t)value - last_value->out_pkts; + last_value->out_pkts = (uint64_t)value; + + switch (a_stream->type) + { + case STREAM_TYPE_TCP: + session_calculate_tcp_packets_increment(a_stream, last_value, increment); + break; + + case STREAM_TYPE_UDP: + session_calculate_udp_packets_increment(a_stream, last_value, increment); + break; + + default: + break; + } + + return 0; +} + +int session_application_metrics_update(const struct streaminfo *a_stream, struct session_runtime_action_context *srt_action_context, int thread_seq) +{ + if(tsg_get_current_time_ms() - srt_action_context->last_update_metric_time < g_tsg_para.app_metric_update_interval_ms) + { + return 0; + } + + char app_full_path[256]={0}; + struct traffic_packet_info increment_app_statis = {0}; + session_application_full_path_update(a_stream, app_full_path, sizeof(app_full_path)); + char *l4_protocol_string=session_l4_protocol_label_update(a_stream, &srt_action_context->l4_protocol); + session_calculate_packets_increment(a_stream, &srt_action_context->last_app_statis, &increment_app_statis, thread_seq); + tsg_set_application_metrics(a_stream, l4_protocol_string, app_full_path, &increment_app_statis, thread_seq); + srt_action_context->last_update_metric_time = tsg_get_current_time_ms(); + + return 1; +} + static int session_app_gather_results_set_l7_protocol(const struct streaminfo *a_stream, enum TSG_PROTOCOL protocol) { struct gather_app_result *gather_result=(struct gather_app_result *)session_gather_app_results_get(a_stream); @@ -1954,14 +2392,19 @@ extern "C" unsigned char TSG_MASTER_UDP_ENTRY(const struct streaminfo *a_udp, vo *pme=dictator_malloc(thread_seq, sizeof(struct udp_session_runtime_context)); memset(*pme, 0, sizeof(struct udp_session_runtime_context)); udp_srt_context=(struct udp_session_runtime_context *)(*pme); - - udp_srt_context->srt_action_context=(struct session_runtime_action_context *)dictator_malloc(thread_seq, sizeof(struct session_runtime_action_context)); - memset(udp_srt_context->srt_action_context, 0, sizeof(struct session_runtime_action_context)); - session_runtime_action_context_async(a_udp, (void *)(udp_srt_context->srt_action_context)); + udp_srt_context->srt_action_context=(struct session_runtime_action_context *)session_runtime_action_context_get(a_udp); + if(udp_srt_context->srt_action_context==NULL) + { + udp_srt_context->srt_action_context=(struct session_runtime_action_context *)session_runtime_action_context_new(a_udp); + } + + udp_srt_context->srt_action_context->last_update_metric_time = tsg_get_current_time_ms(); udp_srt_context->srt_action_context->direction=get_direction(a_udp); } + session_application_metrics_update(a_udp, udp_srt_context->srt_action_context, thread_seq); + if(udp_srt_context->srt_action_context->udp_data_dropme==0) { state2=tsg_master_data_entry(a_udp, (void **)&(udp_srt_context->srt_process_context), thread_seq, a_packet); @@ -1980,6 +2423,7 @@ extern "C" unsigned char TSG_MASTER_UDP_ENTRY(const struct streaminfo *a_udp, vo if(state1&APP_STATE_DROPME || state2&APP_STATE_DROPME || a_udp->opstate==OP_STATE_CLOSE) { + session_application_metrics_update(a_udp, udp_srt_context->srt_action_context, thread_seq); dictator_free(thread_seq, *pme); *pme=NULL; @@ -1991,7 +2435,7 @@ extern "C" unsigned char TSG_MASTER_UDP_ENTRY(const struct streaminfo *a_udp, vo extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp, void **pme, int thread_seq, const void *a_packet) { - struct session_runtime_action_context *_context=(struct session_runtime_action_context *)(*pme); + struct session_runtime_action_context *srt_action_context=(struct session_runtime_action_context *)(*pme); int is_ctrl_pkt=session_state_control_packet(a_tcp, a_tcp->ptcpdetail->pdata, a_tcp->ptcpdetail->datalen); if(is_ctrl_pkt==1) @@ -2005,18 +2449,19 @@ extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp, *pme=(void *)session_runtime_action_context_get(a_tcp); if(*pme==NULL) { - *pme=(struct session_runtime_action_context *)dictator_malloc(thread_seq, sizeof(struct session_runtime_action_context)); - memset(*pme, 0, sizeof(struct session_runtime_action_context)); - session_runtime_action_context_async(a_tcp, (void *)(*pme)); + srt_action_context=(struct session_runtime_action_context *)session_runtime_action_context_new(a_tcp); + *pme=(void *)srt_action_context; } - - _context=(struct session_runtime_action_context *)(*pme); - _context->direction=get_direction(a_tcp); + + srt_action_context->direction=get_direction(a_tcp); + srt_action_context->last_update_metric_time = tsg_get_current_time_ms(); } - - if(_context->set_latency_flag==0) + + session_application_metrics_update(a_tcp, srt_action_context, thread_seq); + + if(srt_action_context->set_latency_flag==0) { - _context->set_latency_flag=session_tcp_establish_latency_ms_set(a_tcp, thread_seq, a_packet); + srt_action_context->set_latency_flag=session_tcp_establish_latency_ms_set(a_tcp, thread_seq, a_packet); } if (a_packet != NULL) @@ -2028,6 +2473,7 @@ extern "C" unsigned char TSG_MASTER_TCPALL_ENTRY(const struct streaminfo *a_tcp, if(state&APP_STATE_DROPME || a_tcp->pktstate==OP_STATE_CLOSE) { + session_application_metrics_update(a_tcp, srt_action_context, thread_seq); session_state_sync_in_opening_and_closing(a_tcp, a_tcp->pktstate); } @@ -2071,6 +2517,8 @@ extern "C" int TSG_MASTER_INIT() MESA_load_profile_short_def(tsg_conffile, "SYSTEM", "TIMEOUT", (short *)&g_tsg_para.timeout, 300); MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "SCAN_TIME_INTERVAL", &g_tsg_para.scan_time_interval, 120); MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "IENTIFY_APP_MAX_PKT_NUM", &g_tsg_para.identify_app_max_pkt_num, 20); + + MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "APP_METRICS_INTERVAL", &g_tsg_para.app_metric_update_interval_ms, 30000); MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "FEATURE_TAMPER", &g_tsg_para.feature_tamper, 0); MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "SERVICE_CHAINING_SID", &g_tsg_para.service_chaining_sid, 0); @@ -2126,10 +2574,10 @@ extern "C" int TSG_MASTER_INIT() FS_start(g_tsg_para.fs2_handle); - ret=tsg_statistic_init(tsg_conffile, g_tsg_para.logger); + ret = tsg_metric_init(tsg_conffile, g_tsg_para.logger); if(ret<0) { - MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_STATISTIC", "tsg_statistic_init failed ..."); + MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_STATISTIC", "tsg_metric_init failed ..."); return -1; } diff --git a/src/tsg_entry.h b/src/tsg_entry.h index 8a8c92e..7727a6f 100644 --- a/src/tsg_entry.h +++ b/src/tsg_entry.h @@ -117,13 +117,28 @@ enum security_metric_columns COLUMN_MAX }; -enum TRAFFIC_INFO_IDX +enum app_metric_columns { - TRAFFIC_INFO_ALLOW=0, - TRAFFIC_INFO_DENY, - TRAFFIC_INFO_MONITOR, - TRAFFIC_INFO_INTERCEPT, - TRAFFIC_INFO_MAX + TRAFFIC_APP_SESSIONS = 0, + TRAFFIC_APP_IN_BYTES, + TRAFFIC_APP_OUT_BYTES, + TRAFFIC_APP_IN_PKTS, + TRAFFIC_APP_OUT_PKTS, + TRAFFIC_APP_C2S_PKTS, + TRAFFIC_APP_S2C_PKTS, + TRAFFIC_APP_C2S_BYTES, + TRAFFIC_APP_S2C_BYTES, + TRAFFIC_APP_C2S_FRAGMENTS, + TRAFFIC_APP_S2C_FRAGMENTS, + TRAFFIC_APP_C2S_LOST_BYTES, + TRAFFIC_APP_S2C_LOST_BYTES, + TRAFFIC_APP_C2S_ORDER_PKTS, + TRAFFIC_APP_S2C_ORDER_PKTS, + TRAFFIC_APP_C2S_RETRANSMITTED_PKTS, + TRAFFIC_APP_S2C_RETRANSMITTED_PKTS, + TRAFFIC_APP_C2S_RETRANSMITTED_BYTES, + TRAFFIC_APP_S2C_RETRANSMITTED_BYTES, + TRAFFIC_APP_MAX }; struct tsg_statistic @@ -143,10 +158,17 @@ struct tsg_statistic int metric_table_id; unsigned int security_column_ids[COLUMN_MAX]; struct fieldstat_dynamic_instance *security_metric_handle; + + int app_metric_cycle; + int app_metric_table_id; + unsigned int app_column_ids[TRAFFIC_APP_MAX]; + struct fieldstat_dynamic_instance *app_metric_handle; }; -int tsg_statistic_init(const char *conffile, void *logger); -void tsg_statistic_destroy(void); +int tsg_metric_init(const char *conffile, void *logger); +void tsg_metric_destroy(void); int tsg_gtp_signaling_hash_init(const char* conffile, void *logger); +int session_application_full_path_update(const struct streaminfo *a_stream, char *app_full_path, int app_full_path_len); + diff --git a/src/tsg_rule.cpp b/src/tsg_rule.cpp index 7de6a99..3b1164b 100644 --- a/src/tsg_rule.cpp +++ b/src/tsg_rule.cpp @@ -126,8 +126,7 @@ unsigned short get_redis_port(char *redis_port_range) } else { - int ret=sscanf(buf, "%hu-%hu", &s_port, &e_port); - assert(ret==2); + sscanf(buf, "%hu-%hu", &s_port, &e_port); } for(int i=s_port; i<=e_port && port_num<32; i++) diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index 46a9d3d..2938982 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -688,175 +688,6 @@ static int set_userdefine_app(struct TLD_handle_t *_handle, char *field_name, st return 1; } -static unsigned int get_max_app_id(struct gather_app_result *result) -{ - int i=0; - unsigned int max_app_id=0; - - for(i=0; i< result->app_num; i++) - { - if(max_app_id < result->attributes[i].app_id) - { - max_app_id=result->attributes[i].app_id; - } - } - - return max_app_id; -} - -static unsigned int get_lastest_app_id(struct gather_app_result *result) -{ - if(result->app_num>0) - { - return result->attributes[result->app_num-1].app_id; - } - - return 0; -} - -/* - L7(openvpn), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon - L7(openvpn), appSketch(psiphon), Thrid(openvpn.qq_web.wechat) = openvpn.psiphon - L7(openvpn), appSketch(psiphon), Thrid(openvpn.wechat) = openvpn.psiphon - L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn.wechat) = openvpn.ssl.psiphon - L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn) = openvpn.ssl.psiphon - L7(openvpn.ssl), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon - L7(null), appSketch(psiphon), Thrid(openvpn.ssl.wechat) = openvpn.ssl.psiphon - L7(null), appSketch(psiphon), Thrid(wechat) = psiphon - L7(null), appSketch(null), Thrid(openvpn.ssl.wechat) = openvpn.ssl.wechat - L7(null), appSketch(null), Thrid(null) = unknown -*/ - -static int split_l7_and_app(struct gather_app_result *qm_result, struct gather_app_result *qm_l7_result) -{ - int i=0; - for(i=qm_result->app_num; i>0; i--) - { - if(tsg_l7_protocol_id2name(qm_result->attributes[i-1].app_id)) - { - break; - } - } - - qm_l7_result->app_num=i; - qm_l7_result->origin=ORIGIN_QM_ENGINE_L7; - memcpy(qm_l7_result->attributes, qm_result->attributes, sizeof(struct app_attributes)*i); - - return 0; -} - -/* -** Engine1 1.2 Engine2 1 -> 1.2 -** Engine1 1.2 Engine2 1.3 -> 1.2 -** Engine1 1 Engine2 1.2 -> 1.2 -** Engine1 2 Engine2 1.2 -> 1.2 -** Engine1 1 Engine2 2.3 -> 1 -** Engine1 1 Engine2 2 -> 1 -*/ - -static int copy_app_id(struct app_attributes *result, int result_num, unsigned int *combined_array, int combined_array_num) -{ - int i=0; - for(i=0; i0) - { - combined_num+=copy_app_id(result[ORIGIN_BASIC_PROTOCOL].attributes, result[ORIGIN_BASIC_PROTOCOL].app_num, combined_array, COMBINED_APP_ID_NUM); - } - else - { - combined_num+=copy_app_id(result[ORIGIN_QM_ENGINE_L7].attributes, result[ORIGIN_QM_ENGINE_L7].app_num, combined_array, COMBINED_APP_ID_NUM); - } - - if(result[ORIGIN_HITED_APP].app_num > 0) - { - combined_array[combined_num]=get_lastest_app_id(&(result[ORIGIN_HITED_APP])); - combined_num+=1; - } - else if(result[ORIGIN_USER_DEFINE].app_num>0) - { - if(combined_num0) - { - combined_num+=copy_app_id(result[ORIGIN_BUILT_IN].attributes, result[ORIGIN_BUILT_IN].app_num, combined_array+combined_num, COMBINED_APP_ID_NUM-combined_num); - } - else if(result[ORIGIN_QM_ENGINE].app_num-result[ORIGIN_QM_ENGINE_L7].app_num > 0) - { - combined_num+=copy_app_id( result[ORIGIN_QM_ENGINE].attributes + result[ORIGIN_QM_ENGINE_L7].app_num, - result[ORIGIN_QM_ENGINE].app_num - result[ORIGIN_QM_ENGINE_L7].app_num, - combined_array+combined_num, - COMBINED_APP_ID_NUM-combined_num - ); - } - - if(combined_num==0) - { - offset=tsg_get_app_name_by_id(g_tsg_maat_feather, result[ORIGIN_UNKNOWN].attributes[0].app_id, app_full_path, sizeof(app_full_path), 0); - if(offset>0) - { - TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_FULL_PATH].name, (void *)app_full_path, TLD_TYPE_STRING); - TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)app_full_path, TLD_TYPE_STRING); - return 1; - } - else - { - MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "APP_FULL_PATH", "app_full_path is null, combined_num=0"); - } - - return 0; - } - - for(int i=0; i0) - { - app_full_path[offset++]='.'; - } - } - - if(offset>0) - { - app_full_path[offset-1]='\0'; - TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_FULL_PATH].name, (void *)app_full_path, TLD_TYPE_STRING); - - char *app_label=rindex(app_full_path, '.'); - if(app_label!=NULL) - { - TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)(app_label+1), TLD_TYPE_STRING); - } - else - { - TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)app_full_path, TLD_TYPE_STRING); - } - } - else - { - MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "APP_FULL_PATH", "app_full_path is null, combined_num=%d", combined_num); - } - - return 1; -} - int set_app_id(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, const struct streaminfo *a_stream) { struct gather_app_result *gather_result=(struct gather_app_result *)session_gather_app_results_get(a_stream); @@ -872,11 +703,21 @@ int set_app_id(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handl return 0; } - split_l7_and_app(&(gather_result[ORIGIN_QM_ENGINE]), &(gather_result[ORIGIN_QM_ENGINE_L7])); - set_userdefine_app(_handle, _instance->id2field[LOG_COMMON_USERDEFINE_APP].name, &(gather_result[ORIGIN_USER_DEFINE])); - set_app_info(_instance, _handle, gather_result); + char app_full_path[256]={0}; + session_application_full_path_update(a_stream, app_full_path, sizeof(app_full_path)); + + TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_FULL_PATH].name, (void *)app_full_path, TLD_TYPE_STRING); + char *app_label=rindex(app_full_path, '.'); + if(app_label!=NULL) + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)(app_label+1), TLD_TYPE_STRING); + } + else + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_APP_LABEL].name, (void *)app_full_path, TLD_TYPE_STRING); + } set_app_identify_info(_handle, _instance->id2field[LOG_COMMON_APP_IDENTIFY_INFO].name, gather_result); @@ -1893,7 +1734,6 @@ static int session_record_limit(struct tsg_log_instance_t *_instance, const stru int log_common_fields_new(const char *filename, id2field_t *id2field, struct topic_stat **service2topic, int *max_service) { int i=0,flag=0; - int ret=0,id=0; FILE *fp=NULL; char line[1024]={0}; char field_name[64]={0}; @@ -1915,9 +1755,9 @@ int log_common_fields_new(const char *filename, id2field_t *id2field, struct top { continue; } + int id=0; memset(type_name, 0, sizeof(type_name)); - ret=sscanf(line, "%31s %63s %d", type_name, field_name, &id); - assert(ret==3); + sscanf(line, "%31s %63s %d", type_name, field_name, &id); for(i=0; iunknown_app_id; + } + + return 0; +} + diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index 1307fb5..ff14578 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -236,7 +236,8 @@ struct tsg_log_instance_t void *logger; }; +int tsg_unknown_app_id_get(struct tsg_log_instance_t *instance); + char *log_field_id2name(struct tsg_log_instance_t *instance, enum LOG_FIELD_ID id); struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile, screen_stat_handle_t fs2_handle); void tsg_sendlog_destroy(struct tsg_log_instance_t * instance); - diff --git a/src/tsg_statistic.cpp b/src/tsg_statistic.cpp index c9147ad..e1c7d6b 100644 --- a/src/tsg_statistic.cpp +++ b/src/tsg_statistic.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -28,6 +27,14 @@ enum traffic_metric_tags TRAFFIC_TAG_MAX }; +enum app_metric_tags +{ + APP_TAG_VSYS_ID = 0, + APP_TAG_PROTOCOL_LABEL, + APP_TAG_APP_FULL_PATH, + APP_TAG_MAX +}; + int tsg_set_statistic_opt(int value, enum _STATISTIC_OPT_TYPE type, int thread_seq) { switch(type) @@ -37,16 +44,16 @@ int tsg_set_statistic_opt(int value, enum _STATISTIC_OPT_TYPE type, int thread_s case OPT_TYPE_PINNING_YES: case OPT_TYPE_PINNING_NOT: case OPT_TYPE_PINNING_MAYBE: - atomic_add(&(g_tsg_statis_para.statistic_opt[type]), value); - break; - default: - break; + atomic_add(&(g_tsg_statis_para.statistic_opt[type]), value); + break; + default: + break; } return 0; } -int tsg_set_intercept_flow(struct maat_rule *p_result, struct _traffic_info *traffic_info, int thread_seq) +int tsg_set_intercept_flow(struct maat_rule *p_result, struct traffic_info *traffic_info, int thread_seq) { if (p_result == NULL || traffic_info == NULL || thread_seq < 0 || thread_seq >= get_thread_count() || p_result->service_id != TSG_SERVICE_INTERCEPT) { @@ -182,11 +189,83 @@ static void *tsg_statistic_thread(void *arg) return NULL; } +int tsg_set_application_metrics(const struct streaminfo *a_stream, const char *l4_protocol, const char *app_full_path, struct traffic_packet_info *app_statis, int thread_seq) +{ + if (a_stream == NULL || l4_protocol == NULL || app_full_path == NULL || app_statis == NULL) + { + return -1; + } + + struct fieldstat_tag app_tags[APP_TAG_MAX] = {{"vsys_id", 0, -1}, {"protocol_label", 2, 0}, {"app_full_path", 2, 0}}; + app_tags[APP_TAG_VSYS_ID].value_int = g_tsg_statis_para.vsystem_id; + app_tags[APP_TAG_PROTOCOL_LABEL].value_str = l4_protocol; + app_tags[APP_TAG_APP_FULL_PATH].value_str = app_full_path; + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_SESSIONS], "traffic_application_protocol_stat", + app_statis->sessions, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_IN_BYTES], "traffic_application_protocol_stat", + app_statis->in_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_IN_PKTS], "traffic_application_protocol_stat", + app_statis->in_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_OUT_BYTES], "traffic_application_protocol_stat", + app_statis->out_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_OUT_PKTS], "traffic_application_protocol_stat", + app_statis->out_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_PKTS], "traffic_application_protocol_stat", + app_statis->c2s_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_PKTS], "traffic_application_protocol_stat", + app_statis->s2c_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_BYTES], "traffic_application_protocol_stat", + app_statis->c2s_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_BYTES], "traffic_application_protocol_stat", + app_statis->s2c_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_FRAGMENTS], "traffic_application_protocol_stat", + app_statis->c2s_fragments, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_FRAGMENTS], "traffic_application_protocol_stat", + app_statis->s2c_fragments, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_LOST_BYTES], "traffic_application_protocol_stat", + app_statis->c2s_tcp_lost_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_LOST_BYTES], "traffic_application_protocol_stat", + app_statis->s2c_tcp_lost_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_ORDER_PKTS], "traffic_application_protocol_stat", + app_statis->c2s_tcp_ooorder_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_ORDER_PKTS], "traffic_application_protocol_stat", + app_statis->s2c_tcp_ooorder_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_RETRANSMITTED_PKTS], "traffic_application_protocol_stat", + app_statis->c2s_tcp_retransmitted_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_RETRANSMITTED_PKTS], "traffic_application_protocol_stat", + app_statis->s2c_tcp_retransmitted_pkts, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_C2S_RETRANSMITTED_BYTES], "traffic_application_protocol_stat", + app_statis->c2s_tcp_retransmitted_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + fieldstat_dynamic_table_metric_value_incrby(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_table_id, g_tsg_statis_para.app_column_ids[TRAFFIC_APP_S2C_RETRANSMITTED_BYTES], "traffic_application_protocol_stat", + app_statis->s2c_tcp_retransmitted_bytes, app_tags, (size_t)APP_TAG_MAX, thread_seq); + + return 0; +} + int tsg_security_metric_init(const char *conffile, void *logger) { if (conffile == NULL || logger == NULL) { - return -1; + return -1; } unsigned short fs_server_port=0; @@ -200,21 +279,21 @@ int tsg_security_metric_init(const char *conffile, void *logger) MESA_load_profile_int_def(conffile, "SECURITY_HITS", "CYCLE", &g_tsg_statis_para.metric_cycle, 1000); if(g_tsg_statis_para.metric_cycle<=0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.metric_cycle error"); - return -1; + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.metric_cycle error"); + return -1; } g_tsg_statis_para.security_metric_handle = fieldstat_dynamic_instance_new(app_name, thread_num); if (g_tsg_statis_para.security_metric_handle == NULL) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.security_metric_handle error"); - return -1; + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.security_metric_handle error"); + return -1; } fieldstat_dynamic_set_output_interval(g_tsg_statis_para.security_metric_handle, g_tsg_statis_para.metric_cycle); if (fs_server_port > 0 && strlen(fs_server_ip) > 0) { - fieldstat_dynamic_set_line_protocol_server(g_tsg_statis_para.security_metric_handle, fs_server_ip, fs_server_port); + fieldstat_dynamic_set_line_protocol_server(g_tsg_statis_para.security_metric_handle, fs_server_ip, fs_server_port); } enum field_type security_metric_type[COLUMN_MAX] = {FIELD_TYPE_COUNTER}; @@ -222,8 +301,8 @@ int tsg_security_metric_init(const char *conffile, void *logger) g_tsg_statis_para.metric_table_id = fieldstat_register_dynamic_table(g_tsg_statis_para.security_metric_handle, "security_rule_hits", security_metric_field, security_metric_type, (size_t)COLUMN_MAX, g_tsg_statis_para.security_column_ids); if(g_tsg_statis_para.metric_table_id<0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.metric_table_id error"); - return -1; + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "SECURITY_HITS", "g_tsg_statis_para.metric_table_id error"); + return -1; } fieldstat_dynamic_instance_start(g_tsg_statis_para.security_metric_handle); @@ -231,7 +310,58 @@ int tsg_security_metric_init(const char *conffile, void *logger) return 0; } -int tsg_statistic_init(const char *conffile, void *logger) +int tsg_traffic_metric_init(const char *conffile, void *logger) +{ + if (conffile == NULL || logger == NULL) + { + return -1; + } + + unsigned short fs_server_port = 0; + char fs_server_ip[MAX_IPV4_LEN] = {0}; + char app_name[128] = {0}; + int thread_num = get_thread_count(); + + MESA_load_profile_short_nodef(conffile, "APP_METRIC", "TELEGRAF_PORT", (short *)&(fs_server_port)); + MESA_load_profile_string_nodef(conffile, "APP_METRIC", "TELEGRAF_IP", fs_server_ip, sizeof(fs_server_ip)); + MESA_load_profile_string_def(conffile, "APP_METRIC", "APP_NAME", app_name, sizeof(app_name), "app_metric"); + MESA_load_profile_int_def(conffile, "APP_METRIC", "CYCLE", &g_tsg_statis_para.app_metric_cycle, 1000); + + if (g_tsg_statis_para.app_metric_cycle <= 0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "APP_METRIC", "g_tsg_statis_para.app_metric_cycle error"); + return -1; + } + + g_tsg_statis_para.app_metric_handle = fieldstat_dynamic_instance_new(app_name, thread_num); + if (g_tsg_statis_para.app_metric_handle == NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "APP_METRIC", "g_tsg_statis_para.app_metric_handle error"); + return -1; + } + + fieldstat_dynamic_set_output_interval(g_tsg_statis_para.app_metric_handle, g_tsg_statis_para.app_metric_cycle); + if (fs_server_port > 0 && strlen(fs_server_ip) > 0) + { + fieldstat_dynamic_set_line_protocol_server(g_tsg_statis_para.app_metric_handle, fs_server_ip, fs_server_port); + } + + enum field_type app_metric_type[TRAFFIC_APP_MAX] = {FIELD_TYPE_COUNTER}; + const char *app_metric_field[TRAFFIC_APP_MAX] = {"sessions", "in_bytes", "out_bytes", "in_pkts", "out_pkts", "c2s_pkts", "s2c_pkts", "c2s_bytes", "s2c_bytes", "c2s_fragments", "s2c_fragments", + "c2s_tcp_lost_bytes", "s2c_tcp_lost_bytes", "c2s_tcp_ooorder_pkts", "s2c_tcp_ooorder_pkts", "c2s_tcp_retransmitted_pkts", "s2c_tcp_retransmitted_pkts", + "c2s_tcp_retransmitted_bytes", "s2c_tcp_retransmitted_bytes"}; + g_tsg_statis_para.app_metric_table_id = fieldstat_register_dynamic_table(g_tsg_statis_para.app_metric_handle, "traffic_application_protocol_stat", app_metric_field, app_metric_type, (size_t)TRAFFIC_APP_MAX, g_tsg_statis_para.app_column_ids); + if (g_tsg_statis_para.app_metric_table_id < 0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "APP_METRIC", "g_tsg_statis_para.app_metric_table_id error"); + return -1; + } + + fieldstat_dynamic_instance_start(g_tsg_statis_para.app_metric_handle); + return 0; +} + +int tsg_metric_init(const char *conffile, void *logger) { unsigned short fs_server_port = 0; char app_name[128]={0}; @@ -243,8 +373,8 @@ int tsg_statistic_init(const char *conffile, void *logger) MESA_load_profile_int_def(conffile, "STATISTIC", "CYCLE", &g_tsg_statis_para.cycle, 1); if(g_tsg_statis_para.cycle<=0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "STATISTIC", "Disabale traffic statistic"); - return 0; + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "STATISTIC", "Disabale traffic statistic"); + return 0; } MESA_load_profile_short_nodef(conffile, "STATISTIC", "TELEGRAF_PORT", (short *)&(fs_server_port)); @@ -263,7 +393,7 @@ int tsg_statistic_init(const char *conffile, void *logger) if (fs_server_port > 0 && strlen(fs_server_ip) > 0) { - fieldstat_set_line_protocol_server(g_tsg_statis_para.statistic_handle, fs_server_ip, fs_server_port); + fieldstat_set_line_protocol_server(g_tsg_statis_para.statistic_handle, fs_server_ip, fs_server_port); } const char *static_column_name[STATIS_MAX] = {"sessions", "active_sessions", "closed_sessions", "in_bytes", "out_bytes", "in_pkts", "out_pkts", "asymmetric_c2s_flows", "asymmetric_s2c_flows"}; @@ -279,22 +409,28 @@ int tsg_statistic_init(const char *conffile, void *logger) int ret = tsg_security_metric_init(conffile, logger); if(ret<0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "INIT_METRIC", "tsg_security_metric_init failed ..."); - return -1; + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "INIT_METRIC", "tsg_security_metric_init failed ..."); + return -1; + } + + ret = tsg_traffic_metric_init(conffile, logger); + if (ret < 0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "INIT_METRIC", "tsg_traffic_metric_init failed ..."); + return -1; } pthread_create(&g_tsg_statis_para.stat_thread_id, NULL, tsg_statistic_thread, NULL); - + return 0; } -void tsg_statistic_destroy(void) +void tsg_metric_destroy(void) { pthread_cancel(g_tsg_statis_para.stat_thread_id); - g_tsg_statis_para.thread_alive=0; + g_tsg_statis_para.thread_alive = 0; sleep(g_tsg_statis_para.cycle); fieldstat_instance_free(g_tsg_statis_para.statistic_handle); - return ; + return; } - diff --git a/src/tsg_variable.h b/src/tsg_variable.h index 2b9a0ef..6cf5413 100644 --- a/src/tsg_variable.h +++ b/src/tsg_variable.h @@ -131,6 +131,7 @@ struct tsg_rt_para int shaping_sid; int intercept_sid; int send_resetall; + int app_metric_update_interval_ms; enum DEPLOY_MODE deploy_mode; int scan_time_interval; int identify_app_max_pkt_num; diff --git a/test/src/CMakeLists.txt b/test/src/CMakeLists.txt index fc60733..664a1ef 100644 --- a/test/src/CMakeLists.txt +++ b/test/src/CMakeLists.txt @@ -42,18 +42,19 @@ set(TSG_MASTER_SRC ${PROJECT_SOURCE_DIR}/src/tsg_entry.cpp ${PROJECT_SOURCE_DIR}/src/tsg_variable.cpp ${PROJECT_SOURCE_DIR}/src/tsg_protocol.cpp ${PROJECT_SOURCE_DIR}/src/tsg_proxy.cpp + ${PROJECT_SOURCE_DIR}/src/mpack.c ) -add_executable(gtest_master ${TSG_MASTER_SRC} ${PROJECT_SOURCE_DIR}/src/mpack.c gtest_kafka.cpp gtest_common.cpp gtest_master.cpp) +add_executable(gtest_master ${TSG_MASTER_SRC} gtest_kafka.cpp gtest_common.cpp gtest_master.cpp) target_link_libraries(gtest_master gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3) add_executable(gtest_sync_state ${PROJECT_SOURCE_DIR}/src/tsg_sync_state.cpp ${PROJECT_SOURCE_DIR}/src/mpack.c gtest_common.cpp gtest_sync_state.cpp) target_link_libraries(gtest_sync_state gtest-static cjson ctemplate-static MESA_handle_logger) -add_executable(gtest_fieldstat3 ${PROJECT_SOURCE_DIR}/src/tsg_statistic.cpp gtest_common.cpp gtest_fieldstat3.cpp) -target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static MESA_field_stat2 fieldstat3 MESA_prof_load MESA_handle_logger) +add_executable(gtest_fieldstat3 ${TSG_MASTER_SRC} gtest_common.cpp gtest_kafka.cpp gtest_fieldstat3.cpp) +target_link_libraries(gtest_fieldstat3 gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 MESA_htable fieldstat3) -add_executable(gtest_proxy ${TSG_MASTER_SRC} ${PROJECT_SOURCE_DIR}/src/mpack.c gtest_common.cpp gtest_proxy.cpp) +add_executable(gtest_proxy ${TSG_MASTER_SRC} gtest_common.cpp gtest_proxy.cpp) target_link_libraries(gtest_proxy gtest-static ctemplate-static cjson MESA_prof_load MESA_handle_logger MESA_jump_layer MESA_field_stat2 maat4 rdkafka MESA_htable fieldstat3 pcap) file(COPY ../pcap/ DESTINATION ./pcap/) diff --git a/test/src/gtest_common.cpp b/test/src/gtest_common.cpp index e20f66f..71eebfd 100644 --- a/test/src/gtest_common.cpp +++ b/test/src/gtest_common.cpp @@ -7,7 +7,7 @@ struct gtest_ctrl_pkt { char buf[1024]; int len; -}ctrl_pkt; +}; void dictator_free(int thread_seq, void * pbuf) { @@ -40,9 +40,11 @@ int sapp_inject_pkt(struct streaminfo * stream, enum sapp_inject_opt sio, const return 0; } +struct gtest_ctrl_pkt ctrl_pkt; + int sapp_inject_ctrl_pkt(struct streaminfo * stream, enum sapp_inject_opt sio, const void * payload, int payload_len, unsigned char snd_routedir) { - memset(&ctrl_pkt, 0, 1050); + memset(&ctrl_pkt, 0, sizeof(ctrl_pkt)); memcpy(ctrl_pkt.buf, payload, payload_len); ctrl_pkt.len = payload_len; diff --git a/test/src/gtest_fieldstat3.cpp b/test/src/gtest_fieldstat3.cpp index ffb0ff8..4cf36cd 100644 --- a/test/src/gtest_fieldstat3.cpp +++ b/test/src/gtest_fieldstat3.cpp @@ -17,7 +17,7 @@ pthread_t g_pid[8]; TEST(FIELDSATA3, InterceptIllegalParameter) { - struct _traffic_info _info; + struct traffic_info _info; struct maat_rule p_result; int ret = tsg_set_intercept_flow(NULL, &_info, 0); @@ -47,7 +47,7 @@ TEST(FIELDSATA3, PolicyIllegalParameter) TEST(FIELDSATA3, Intercept) { - struct _traffic_info _info; + struct traffic_info _info; struct maat_rule p_result; _info.con_num = 10; @@ -97,7 +97,7 @@ void *run_time_funtion(void *arg) struct maat_rule p_result; struct streaminfo a_stream; - struct _traffic_info _info; + struct traffic_info _info; while (1) { @@ -155,7 +155,7 @@ TEST(FIELDSATA3, MultiThreading) int main(int argc, char *argv[]) { void *logger = MESA_create_runtime_log_handle("log/gtest_fieldstat3.log", RLOG_LV_FATAL); - tsg_statistic_init(tsg_gtest_conffile, logger); + tsg_metric_init(tsg_gtest_conffile, logger); testing::InitGoogleTest(&argc, argv); int ret = RUN_ALL_TESTS(); // sleep(30); @@ -164,7 +164,7 @@ int main(int argc, char *argv[]) // pthread_cancel(g_pid[i]); // } - tsg_statistic_destroy(); + tsg_metric_destroy(); return ret; } diff --git a/test/src/gtest_sendlog.cpp b/test/src/gtest_sendlog.cpp index e086cca..8a5d7d9 100644 --- a/test/src/gtest_sendlog.cpp +++ b/test/src/gtest_sendlog.cpp @@ -148,6 +148,11 @@ int tsg_set_policy_flow(const struct streaminfo * a_stream, struct maat_rule * p return 0; } +int session_application_full_path_update(const struct streaminfo * a_stream, char * app_full_path, int app_full_path_len) +{ + return 0; +} + TEST(Master, SendInterimRecord) { struct streaminfo a_stream={0};