gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into push-master

Browse Source
This commit is contained in:
liuxueli
2020-04-28 15:43:44 +08:00
parent e6eab6d0e6 acb61ea1a8
commit 32a58fb60c
9 changed files with 103 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bin/tsg_log_field.conf
View File

@@ -40,3 +40,4 @@ STRING common_stream_trace_id 33
STRING common_schema_type 34
STRING http_host 35
STRING ssl_sni 36
LONG common_establish_latency_ms 37

5
bin/tsg_static_tableinfo.conf
View File

@@ -16,7 +16,7 @@
5 TSG_OBJ_URL expr UTF8 UTF8/GBK yes 0
6 TSG_OBJ_FQDN expr UTF8 UTF8 yes 0
6 TSG_OBJ_FQDN_CAT expr UTF8 UTF8 yes 0
7 TSG_OBJ_KEYWORDS expr UTF8 UTF8/GBK yes 0
7 TSG_OBJ_KEYWORDS expr UTF8 UTF8/GBK/windows-1251 yes 0
8 TSG_OBJ_APP_ID expr UTF8 UTF8 yes 0
9 TSG_OBJ_HTTP_SIGNATURE expr_plus UTF8 UTF8/GBK yes 0
10 TSG_FIELD_HTTP_HOST virtual TSG_OBJ_FQDN --
@@ -40,3 +40,6 @@
28 TSG_FIELD_FTP_CONTENT virtual TSG_OBJ_KEYWORDS --
29 TSG_FIELD_FTP_ACCOUNT virtual TSG_OBJ_ACCOUNT --
30 FW_PROFILE_DNS_RECORDS plugin {"key":1,"valid":5} --
31 TSG_SECURITY_SOURCE_ADDR virtual TSG_OBJ_IP_ADDR --
32 TSG_SECURITY_DESTINATION_ADDR virtual TSG_OBJ_IP_ADDR --
33 TSG_SECURITY_ADDR composition {"source":"TSG_SECURITY_SOURCE_ADDR","destination":"TSG_SECURITY_DESTINATION_ADDR"}

4
preinstall/install.sh
View File

@@ -9,6 +9,10 @@ if [[ -z `grep -rn 'POLICY_PRIORITY' ${DST}/etc/project_list.conf` ]];then
echo 'POLICY_PRIORITY struct' >> ${DST}/etc/project_list.conf
fi
if [[ -z `grep -rn 'ESTABLISH_LATENCY' ${DST}/etc/project_list.conf` ]];then
echo 'ESTABLISH_LATENCY long' >> ${DST}/etc/project_list.conf
fi
if [[ -z `grep -rn 'tsg_master.inf' ${DST}/plug/conflist.inf` ]];then
sed -i '/\[platform\]/a\./plug/platform/tsg_master/tsg_master.inf' ${DST}/plug/conflist.inf
fi

1
preinstall/uninstall.sh
View File

@@ -3,3 +3,4 @@ DST=${RPM_INSTALL_PREFIX}
mkdir -p ${DST}/plug/platform/
sed -i '/tsg_master.inf/d' ${DST}/plug/conflist.inf
sed -i '/POLICY_PRIORITY/d' ${DST}/etc/project_list.conf
sed -i '/ESTABLISH_LATENCY/d' ${DST}/etc/project_list.conf

67
src/tsg_entry.cpp
View File

@@ -2,10 +2,12 @@
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#include <sys/time.h>
#include <unistd.h>
#include <MESA/http.h>
#include <MESA/ftp.h>
#include <MESA/mail.h>
#include <MESA/stream.h>
#include <MESA/MESA_prof_load.h>
#include <MESA/MESA_handle_logger.h>
@@ -39,7 +41,7 @@ static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL;
#endif
char TSG_MASTER_VERSION_20200119=0;
char TSG_MASTER_VERSION_20200330=0;
const char *tsg_conffile="tsgconf/main.conf";
g_tsg_para_t g_tsg_para;
@@ -402,6 +404,13 @@ static int identify_application_protocol(struct streaminfo *a_stream, struct _id
}
//mail
ret=mail_protocol_identify(a_stream);
if(ret>=SMTP_PROTOCOL&& ret<=IMAP_PROTOCOL)
{
identify_info->proto=PROTO_MAIL;
return 1;
}
return ret;
}
@@ -409,6 +418,8 @@ static int identify_application_protocol(struct streaminfo *a_stream, struct _id
extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet)
{
int ret=0,hit_num=0;
struct timespec tv;
long establish_latency_ms=0;
int state=APP_STATE_GIVEME;
scan_status_t mid=NULL;
Maat_rule_t *p_result=NULL;
@@ -425,6 +436,32 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
memset(&identify_info, 0, sizeof(identify_info));
identify_application_protocol(a_tcp, &identify_info);
if(identify_info.proto==PROTO_HTTP)
{
clock_gettime(CLOCK_REALTIME, &tv);
establish_latency_ms=(tv.tv_sec-a_tcp->ptcpdetail->createtime)*1000+tv.tv_nsec/1000/1000;
ret=project_req_add_long(a_tcp, g_tsg_para.establish_latency_project_id, establish_latency_ms);
if(ret<0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
"PROJECT_ADD",
"Add policy_priority_label failed, establish latency ms: %llu addr: %s",
establish_latency_ms,
printaddr(&a_tcp->addr, thread_seq)
);
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"PROJECT_ADD",
"Add policy_priority_label successfully, establish latency ms: %llu addr: %s",
establish_latency_ms,
printaddr(&a_tcp->addr, thread_seq)
);
}
}
ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, identify_info.proto, &mid, all_result+hit_num, MAX_RESULT_NUM-hit_num);
if(ret>0)
@@ -500,11 +537,23 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
if(q_result!=NULL && (p_result==q_result))
{
init_context(pme, identify_info.proto, p_result, thread_seq);
if(identify_info.proto>PROTO_UNKONWN && identify_info.proto<PROTO_MAX)
{
_context=(struct _master_context *)*pme;
memcpy(_context->domain, identify_info.domain, identify_info.domain_len);
_context->domain_len=identify_info.domain_len;
}
state=APP_STATE_GIVEME;
}
break;
case TSG_ACTION_BYPASS:
init_context(pme, identify_info.proto, p_result, thread_seq);
if(identify_info.proto>PROTO_UNKONWN && identify_info.proto<PROTO_MAX)
{
_context=(struct _master_context *)*pme;
memcpy(_context->domain, identify_info.domain, identify_info.domain_len);
_context->domain_len=identify_info.domain_len;
}
state=APP_STATE_GIVEME|APP_STATE_KILL_OTHER;
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_BYPASS], 0, FS_OP_ADD, 1);
break;
@@ -544,7 +593,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
(unsigned char)priority_label->result[0].action,
printaddr(&a_tcp->addr, thread_seq)
);
return APP_STATE_DROPME;
return APP_STATE_DROPME|APP_STATE_KILL_OTHER;
break;
case TSG_ACTION_NONE:
default:
@@ -563,7 +612,11 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
{
memset(&identify_info, 0, sizeof(identify_info));
identify_info.proto=_context->proto;
if(_context->domain_len>0)
{
memcpy(identify_info.domain, _context->domain, _context->domain_len);
identify_info.domain_len=_context->domain_len;
}
master_send_log(a_tcp, _context->result, _context->hit_cnt, &identify_info, thread_seq);
free_context(pme, thread_seq);
}
@@ -683,6 +736,13 @@ extern "C" int TSG_MASTER_INIT()
return -1;
}
MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "ESTABLISH_LATENCY_LABEL", label_buff, sizeof(label_buff), "ESTABLISH_LATENCY");
g_tsg_para.establish_latency_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_LONG, NULL);
if(g_tsg_para.establish_latency_project_id<0)
{
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_REGISTER", "Register %s failed ...", label_buff);
}
ret=tsg_rule_init(tsg_conffile, g_tsg_para.logger);
if(ret<0)
{
@@ -696,6 +756,7 @@ extern "C" int TSG_MASTER_INIT()
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_SENDLOG", "tsg_sendlog_init failed ...");
return -1;
}
g_tsg_log_instance->establish_latency_project_id=g_tsg_para.establish_latency_project_id;
MESA_load_profile_int_def(tsg_conffile, "FIELD_STAT", "CYCLE", &cycle, 30);
MESA_load_profile_short_nodef(tsg_conffile, "FIELD_STAT","TELEGRAF_PORT", (short *)&(fs_server_port));

4
src/tsg_entry.h
View File

@@ -53,6 +53,8 @@ struct _master_context
{
tsg_protocol_t proto;
int hit_cnt;
int domain_len;
char domain[MAX_DOAMIN_LEN];
struct Maat_rule_t *result;
};
@@ -64,6 +66,7 @@ typedef struct _tsg_para
int table_id[TABLE_MAX];
int dyn_subscribe_ip_table_id; //TSG_DYN_SUBSCRIBER_IP
int priority_project_id;
int establish_latency_project_id;
int fs2_field_id[TSG_FS2_MAX];
char table_name[TABLE_MAX][_MAX_TABLE_NAME_LEN];
void *logger;
@@ -138,6 +141,7 @@ typedef struct _tsg_statistic
int fs_line_id;
int fs_field_id[STATIS_MAX];
struct _traffic_info *traffic_info[TSG_ACTION_MAX+1];
struct _traffic_info default_total_info;
screen_stat_handle_t fs2_handle;
}tsg_statis_para_t;

13
src/tsg_send_log.cpp
View File

@@ -19,7 +19,7 @@
#include "tsg_send_log.h"
#include "tsg_send_log_internal.h"
char TSG_SEND_LOG_VERSION_20200119=0;
char TSG_SEND_LOG_VERSION_20200330=0;
struct tsg_log_instance_t *g_tsg_log_instance;
@@ -417,6 +417,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
{
int i=0,status=0;
char *payload=NULL;
long establish_latency_ms=0;
struct TLD_handle_t *_handle=handle;
struct tsg_log_instance_t *_instance=instance;
@@ -459,6 +460,15 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
break;
}
if(log_msg->result[i].action==TSG_ACTION_MONITOR && _instance->establish_latency_project_id>0)
{
establish_latency_ms=project_req_get_long(log_msg->a_stream, _instance->establish_latency_project_id);
if(establish_latency_ms>=0)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)establish_latency_ms, TLD_TYPE_LONG);
}
}
TLD_append(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(long)(log_msg->result[i].config_id), TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG);
@@ -485,6 +495,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
TLD_delete(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_SERVICE].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_ACTION].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name);
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LOG], 0, FS_OP_ADD, 1);
}

2
src/tsg_send_log_internal.h
View File

@@ -59,6 +59,7 @@ typedef enum _tsg_log_field_id
LOG_COMMON_SCHAME_TYPE,
LOG_HTTP_HOST,
LOG_SSL_SNI,
LOG_COMMON_ESTABLISH_LATENCY_MS,
LOG_COMMON_MAX
}tsg_log_field_id_t;
@@ -79,6 +80,7 @@ struct tsg_log_instance_t
{
int mode;
int max_service;
int establish_latency_project_id;
void *logger;
char common_field_file[MAX_STRING_LEN*4];
char broker_list[MAX_STRING_LEN*4];

17
src/tsg_statistic.cpp
View File

@@ -61,15 +61,20 @@ static int _get_traffic_info(struct _traffic_info *total, struct _traffic_info *
out->out_bytes=total->out_bytes-policy->out_bytes;
out->out_packets=total->out_packets-policy->out_packets;
if(out->con_num<0 || out->in_bytes<0 || out->in_packets<0 || out->out_bytes<0 || out->out_packets<0)
if(out->con_num-g_tsg_statis_para.default_total_info.con_num<0
|| out->in_bytes-g_tsg_statis_para.default_total_info.in_bytes<0
|| out->in_packets-g_tsg_statis_para.default_total_info.in_packets<0
|| out->out_bytes-g_tsg_statis_para.default_total_info.out_bytes<0
|| out->out_packets-g_tsg_statis_para.default_total_info.out_packets<0)
{
out->con_num=total->con_num;
out->in_bytes=total->in_bytes;
out->in_packets=total->in_packets;
out->out_bytes=total->out_bytes;
out->out_packets=total->out_packets;
memcpy(out, &g_tsg_statis_para.default_total_info, sizeof(struct _traffic_info));
}
else
{
memcpy(&g_tsg_statis_para.default_total_info, out, sizeof(struct _traffic_info));
}
}
return 0;
}