gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
liuxueli
2020-02-10 15:15:30 +08:00
parent f624b670d9 3291ebf2ba
commit 23c0bff4c5
12 changed files with 769 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
.gitlab-ci.yml
View File

@@ -10,10 +10,22 @@ variables:
stages:
- build
- fvt
- package
- release
- docker
.fvt:
image: git.mesalab.cn:7443/mesa_platform/mesa_protocol:master
script:
- rpm -ivh build/*.rpm
- source /etc/profile.d/MESA.sh
- cd /home/mesasoft/
- git clone https://$USER_NAME:$CI_TOKEN@git.mesalab.cn/tango/fvt.git
- cd fvt/
- sh +x ./fvt_verify.sh /home/mesasoft/sapp_run/ /home/mesasoft/sapp_run/tsgconf/tsg_maat.json /home/mesasoft/sapp_run/tsglog/tsglog /home/mesasoft/fvt/master
.build_before_script:
before_script:
- mkdir -p /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX/$CI_PROJECT_NAMESPACE/
@@ -55,6 +67,14 @@ build:
except:
- tags
fvt:
stage: fvt
extends: .fvt
tags:
- share
except:
- tags
tsg-master-release:
stage: package
extends: .package_master

4
bin/maat.conf
View File

@@ -1,5 +1,5 @@
[STATIC]
MAAT_MODE=2
MAAT_MODE=1
#EFFECTIVE_FLAG=
STAT_SWITCH=1
PERF_SWITCH=1
@@ -15,7 +15,7 @@ INC_CFG_DIR=tsgrule/inc/index/
FULL_CFG_DIR=tsgrule/full/index/
[DYNAMIC]
MAAT_MODE=2
MAAT_MODE=1
#EFFECTIVE_FLAG=
STAT_SWITCH=1
PERF_SWITCH=1

5
bin/main.conf
View File

@@ -6,11 +6,10 @@ CB_SUBSCRIBER_IP_TABLE=TSG_DYN_SUBSCRIBER_IP
[TSG_LOG]
MODE=1
NIC_NAME=enp8s0
MAX_SERVICE=1
NIC_NAME=lo
LOG_LEVEL=10
LOG_PATH=./tsglog/tsglog
BROKER_LIST=192.168.40.186:9092
BROKER_LIST=127.0.0.1:9092
COMMON_FIELD_FILE=tsgconf/tsg_log_field.conf
[STATISTIC]

4
bin/tsg_master.inf
View File

@@ -7,3 +7,7 @@ DESTROY_FUNC=TSG_MASTER_UNLOAD
[TCP]
FUNC_FLAG=ALL
FUNC_NAME=TSG_MASTER_TCP_ENTRY
[UDP]
FUNC_FLAG=ALL
FUNC_NAME=TSG_MASTER_UDP_ENTRY

45
bin/tsg_static_tableinfo.conf
View File

@@ -17,25 +17,26 @@
6 TSG_OBJ_FQDN expr UTF8 UTF8 yes 0
6 TSG_OBJ_FQDN_CAT expr UTF8 UTF8 yes 0
7 TSG_OBJ_KEYWORDS expr UTF8 UTF8/GBK yes 0
8 TSG_OBJ_HTTP_SIGNATURE expr_plus UTF8 UTF8/GBK yes 0
9 TSG_FIELD_HTTP_HOST virtual TSG_OBJ_FQDN --
10 TSG_FIELD_HTTP_URL virtual TSG_OBJ_URL --
11 TSG_FIELD_HTTP_REQ_HDR virtual TSG_OBJ_HTTP_SIGNATURE --
12 TSG_FIELD_HTTP_RES_HDR virtual TSG_OBJ_HTTP_SIGNATURE --
13 TSG_FIELD_HTTP_REQ_CONTENT virtual TSG_OBJ_KEYWORDS --
14 TSG_FIELD_HTTP_RES_CONTENT virtual TSG_OBJ_KEYWORDS --
15 TSG_FIELD_SSL_SNI virtual TSG_OBJ_FQDN --
16 TSG_FIELD_SSL_CN virtual TSG_OBJ_FQDN --
17 TSG_FIELD_SSL_SAN virtual TSG_OBJ_FQDN --
18 TSG_FIELD_DNS_QNAME virtual TSG_OBJ_FQDN --
19 TSG_FIELD_MAIL_ACCOUNT virtual TSG_OBJ_ACCOUNT --
20 TSG_FIELD_MAIL_FROM virtual TSG_OBJ_ACCOUNT --
21 TSG_FIELD_MAIL_TO virtual TSG_OBJ_ACCOUNT --
22 TSG_FIELD_MAIL_SUBJECT virtual TSG_OBJ_KEYWORDS --
23 TSG_FIELD_MAIL_CONTENT virtual TSG_OBJ_KEYWORDS --
24 TSG_FIELD_MAIL_ATT_NAME virtual TSG_OBJ_KEYWORDS --
25 TSG_FIELD_MAIL_ATT_CONTENT virtual TSG_OBJ_KEYWORDS --
26 TSG_FIELD_FTP_URI virtual TSG_OBJ_URL --
27 TSG_FIELD_FTP_CONTENT virtual TSG_OBJ_KEYWORDS --
28 TSG_FIELD_FTP_ACCOUNT virtual TSG_OBJ_ACCOUNT --
29 FW_PROFILE_DNS_RECORDS plugin {"key":1,"valid":5} --
8 TSG_OBJ_APP_ID expr UTF8 UTF8 yes 0
9 TSG_OBJ_HTTP_SIGNATURE expr_plus UTF8 UTF8/GBK yes 0
10 TSG_FIELD_HTTP_HOST virtual TSG_OBJ_FQDN --
11 TSG_FIELD_HTTP_URL virtual TSG_OBJ_URL --
12 TSG_FIELD_HTTP_REQ_HDR virtual TSG_OBJ_HTTP_SIGNATURE --
13 TSG_FIELD_HTTP_RES_HDR virtual TSG_OBJ_HTTP_SIGNATURE --
14 TSG_FIELD_HTTP_REQ_CONTENT virtual TSG_OBJ_KEYWORDS --
15 TSG_FIELD_HTTP_RES_CONTENT virtual TSG_OBJ_KEYWORDS --
16 TSG_FIELD_SSL_SNI virtual TSG_OBJ_FQDN --
17 TSG_FIELD_SSL_CN virtual TSG_OBJ_FQDN --
18 TSG_FIELD_SSL_SAN virtual TSG_OBJ_FQDN --
19 TSG_FIELD_DNS_QNAME virtual TSG_OBJ_FQDN --
20 TSG_FIELD_MAIL_ACCOUNT virtual TSG_OBJ_ACCOUNT --
21 TSG_FIELD_MAIL_FROM virtual TSG_OBJ_ACCOUNT --
22 TSG_FIELD_MAIL_TO virtual TSG_OBJ_ACCOUNT --
23 TSG_FIELD_MAIL_SUBJECT virtual TSG_OBJ_KEYWORDS --
24 TSG_FIELD_MAIL_CONTENT virtual TSG_OBJ_KEYWORDS --
25 TSG_FIELD_MAIL_ATT_NAME virtual TSG_OBJ_KEYWORDS --
26 TSG_FIELD_MAIL_ATT_CONTENT virtual TSG_OBJ_KEYWORDS --
27 TSG_FIELD_FTP_URI virtual TSG_OBJ_URL --
28 TSG_FIELD_FTP_CONTENT virtual TSG_OBJ_KEYWORDS --
29 TSG_FIELD_FTP_ACCOUNT virtual TSG_OBJ_ACCOUNT --
30 FW_PROFILE_DNS_RECORDS plugin {"key":1,"valid":5} --

10
inc/tsg_rule.h
View File

@@ -64,15 +64,7 @@ extern Maat_feather_t g_tsg_maat_feather;
int tsg_rule_init(const char *conffile, void *logger);
int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num);
//return 0 if failed, return >0 on success;
int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t *result, int result_num, struct _identify_info *identify_info);
//return -1 if failed, return 0 on success;
int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, void *logger);
//return value: -1: failed, 0: not hit, >0: hit count
int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num, struct _identify_info *identify_info, scan_status_t *mid, void *logger, int thread_seq);
int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t*result, int result_num, struct _identify_info *identify_info);
//return NULL if none exists, otherwise return one deny rule;
struct Maat_rule_t *tsg_fetch_deny_rule(Maat_rule_t *result, int result_num);

503
src/tsg_entry.cpp
View File

@@ -2,7 +2,10 @@
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#include <unistd.h>
#include <MESA/http.h>
#include <MESA/ftp.h>
#include <MESA/stream.h>
#include <MESA/MESA_prof_load.h>
#include <MESA/MESA_handle_logger.h>
@@ -12,6 +15,7 @@
#include "tsg_send_log.h"
#include "tsg_statistic.h"
#include "tsg_send_log_internal.h"
#include "tsg_ssl_utils.h"
#ifdef __cplusplus
extern "C"
@@ -35,7 +39,7 @@ static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL;
#endif
char TSG_MASTER_VERSION_20191226=0;
char TSG_MASTER_VERSION_20200119=0;
const char *tsg_conffile="tsgconf/main.conf";
g_tsg_para_t g_tsg_para;
@@ -47,27 +51,233 @@ id2field_t g_tsg_fs2_field[TSG_FS2_MAX]={{TLD_TYPE_UNKNOWN, TSG_FS2_LINKS, "link
{TLD_TYPE_UNKNOWN, TSG_FS2_LOG, "log"},
{TLD_TYPE_UNKNOWN, TSG_FS2_DENY, "deny"}
};
static void free_policy_label(int thread_seq, void *project_req_value)
{
dictator_free(thread_seq, project_req_value);
project_req_value=NULL;
}
static void free_context(void **pme, int thread_seq)
{
struct _master_context *_context=(struct _master_context *)*pme;
if(_context!=NULL)
{
if(_context->result!=NULL)
{
dictator_free(thread_seq, (void *)_context->result);
_context->result=NULL;
}
dictator_free(thread_seq, (void *)_context);
_context=NULL;
*pme=NULL;
}
}
static int init_context(void **pme, tsg_protocol_t proto, struct Maat_rule_t *p_result, int thread_seq)
{
struct _master_context *_context=(struct _master_context *)*pme;
*pme=dictator_malloc(thread_seq, sizeof(struct _master_context));
_context=(struct _master_context *)*pme;
_context->proto=proto;
_context->hit_cnt=1;
_context->result=(struct Maat_rule_t *)dictator_malloc(thread_seq, sizeof(struct Maat_rule_t));
memcpy(_context->result, p_result, sizeof(struct Maat_rule_t));
return 0;
}
static int master_method_type(struct streaminfo *a_stream, struct Maat_rule_t *p_result)
{
cJSON *item=NULL;
cJSON *object=NULL;
char *tmp_buff=NULL;
int method_type=-1;
if(p_result->serv_def_len<128)
{
object=cJSON_Parse(p_result->service_defined);
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"DO_ACTION",
"Hit policy_id: %d service: %d action: %d user_reagion: %s addr: %s",
p_result->config_id,
p_result->service_id,
(unsigned char)p_result->action,
p_result->service_defined,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
else
{
tmp_buff=(char *)calloc(1, p_result->serv_def_len+1);
Maat_read_rule(g_tsg_maat_feather, p_result, MAAT_RULE_SERV_DEFINE, tmp_buff, p_result->serv_def_len);
object=cJSON_Parse(tmp_buff);
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"DO_ACTION",
"Hit policy_id: %d service: %d action: %d user_reagion: %s addr: %s",
p_result->config_id,
p_result->service_id,
(unsigned char)p_result->action,
tmp_buff,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
if(object==NULL)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
"DO_ACTION",
"Hit policy_id: %d service: %d action: %d user_reagion: %s addr: %s",
p_result->config_id,
p_result->service_id,
(unsigned char)p_result->action,
(tmp_buff==NULL) ? p_result->service_defined : tmp_buff,
printaddr(&a_stream->addr, a_stream->threadnum)
);
if(tmp_buff!=NULL)
{
free(tmp_buff);
tmp_buff=NULL;
}
return -1;
}
item=cJSON_GetObjectItem(object, "method");
if(item!=NULL)
{
method_type=tsg_get_method_id(item->valuestring);
}
if(tmp_buff!=NULL)
{
free(tmp_buff);
tmp_buff=NULL;
}
cJSON_Delete(object);
object=NULL;
return method_type;
}
static int master_do_deny(struct streaminfo *a_stream, struct Maat_rule_t *p_result, int thread_seq)
{
int opt_value=0;
int method_type=-1;
struct rst_tcp_para rst_paras;
method_type=master_method_type(a_stream, p_result);
switch(method_type)
{
case TSG_METHOD_TYPE_DROP:
opt_value=1;
MESA_set_stream_opt(a_stream, MSO_DROP_STREAM, (void *)&opt_value, sizeof(opt_value));
break;
case TSG_METHOD_TYPE_BLOCK:
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
"TSG_ACTION_DENY",
"Unsupport block of deny, policy_id: %d service: %d action: %d addr: %s",
p_result[0].config_id,
p_result[0].service_id,
(unsigned char)p_result[0].action,
printaddr(&a_stream->addr, thread_seq)
);
//break; // not break
case TSG_METHOD_TYPE_RESET:
opt_value=1;
MESA_set_stream_opt(a_stream, MSO_TCP_RST_REMEDY, (void *)&opt_value, sizeof(opt_value));
rst_paras.dir=DIR_DOUBLE;
rst_paras.rst_pkt_num=1;
rst_paras.signature_seed1=65535;
rst_paras.signature_seed2=13;
rst_paras.th_flags=4;
rst_paras.__pad_no_use=0;
MESA_rst_tcp(a_stream, &rst_paras, sizeof(rst_paras));
opt_value=1;
MESA_set_stream_opt(a_stream, MSO_DROP_STREAM, (void *)&opt_value, sizeof(opt_value));
MESA_set_stream_opt(a_stream, MSO_TIMEOUT, (void *)&g_tsg_para.timeout, sizeof(g_tsg_para.timeout));
break;
default:
break;
}
return 0;
}
static char *schema_index2string(tsg_protocol_t proto)
{
char *schema_field_value=NULL;
switch(proto)
{
case PROTO_HTTP:
schema_field_value=(char *)"HTTP";
break;
case PROTO_SSL:
schema_field_value=(char *)"SSL";
break;
case PROTO_DNS:
schema_field_value=(char *)"DNS";
break;
case PROTO_FTP:
schema_field_value=(char *)"FTP";
break;
case PROTO_BGP:
schema_field_value=(char *)"BGP";
break;
case PROTO_SIP:
schema_field_value=(char *)"SIP";
break;
case PROTO_MAIL:
schema_field_value=(char *)"MAIL";
break;
case PROTO_STREAMING_MEDIA:
schema_field_value=(char *)"STREAMING_MEDIA";
break;
default:
break;
}
return schema_field_value;
}
static int master_send_log(struct streaminfo *a_stream, struct Maat_rule_t *p_result, int result_num, struct _identify_info *identify_info, int thread_seq)
{
tsg_log_t log_msg;
char *domain_field_name=NULL;
char *schema_field_name=NULL;
char *schema_field_value=NULL;
struct TLD_handle_t *TLD_handle=NULL;
TLD_handle=TLD_create(thread_seq);
if(identify_info!=NULL)
if(identify_info!=NULL && (identify_info->proto>PROTO_UNKONWN) && (identify_info->proto<PROTO_MAX))
{
schema_field_name=log_field_id2name(g_tsg_log_instance, LOG_COMMON_SCHAME_TYPE);
TLD_append(TLD_handle, schema_field_name, (void *)((identify_info->proto==PROTO_HTTP) ? "HTTP" : "SSL"), TLD_TYPE_STRING);
domain_field_name=log_field_id2name(g_tsg_log_instance, ((identify_info->proto==PROTO_HTTP) ? LOG_HTTP_HOST : LOG_SSL_SNI));
TLD_append(TLD_handle, domain_field_name, (void *)identify_info->domain, TLD_TYPE_STRING);
schema_field_value=schema_index2string(identify_info->proto);
if(schema_field_value!=NULL)
{
TLD_append(TLD_handle, schema_field_name, (void *)schema_field_value, TLD_TYPE_STRING);
}
if(identify_info->proto==PROTO_HTTP || identify_info->proto==PROTO_SSL)
{
domain_field_name=log_field_id2name(g_tsg_log_instance, ((identify_info->proto==PROTO_HTTP) ? LOG_HTTP_HOST : LOG_SSL_SNI));
TLD_append(TLD_handle, domain_field_name, (void *)identify_info->domain, TLD_TYPE_STRING);
}
}
log_msg.a_stream=a_stream;
@@ -110,17 +320,102 @@ static struct Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int
return p_result;
}
static int identify_application_protocol(struct streaminfo *a_stream, struct _identify_info *identify_info)
{
int ret=0;
identify_info->proto = PROTO_UNKONWN;
//http
char *host = NULL;
ret=http_host_parser((char *)a_stream->ptcpdetail->pdata, (unsigned int)a_stream->ptcpdetail->datalen, DIR_C2S, &host);
if(ret>=0)
{
identify_info->proto=PROTO_HTTP;
if(ret==0)
{
identify_info->domain_len=0;
}
else
{
identify_info->domain_len=MIN(ret, (int)sizeof(identify_info->domain) - 1);
strncpy(identify_info->domain, host, identify_info->domain_len);
}
return 1;
}
//ssl
enum chello_parse_result chello_status = CHELLO_PARSE_INVALID_FORMAT;
struct ssl_chello *chello = NULL;
chello=ssl_chello_parse((unsigned char *)a_stream->ptcpdetail->pdata, (unsigned int)a_stream->ptcpdetail->datalen, &chello_status);
if(chello_status==CHELLO_PARSE_SUCCESS)
{
identify_info->proto=PROTO_SSL;
if(chello->sni==NULL)
{
identify_info->domain_len = 0;
}
else
{
identify_info->domain_len = strnlen(chello->sni, sizeof(identify_info->domain) - 1);
strncpy(identify_info->domain, chello->sni, identify_info->domain_len);
}
ssl_chello_free(chello);
return 1;
}
ssl_chello_free(chello);
//dns
struct stream_tuple4_v4 *tpl4 = NULL;
struct stream_tuple4_v6 *tpl6 = NULL;
switch(a_stream->addr.addrtype)
{
case ADDR_TYPE_IPV4:
tpl4=a_stream->addr.tuple4_v4;
if((ntohs(tpl4->source)==53) || (ntohs(tpl4->dest)==53))
{
identify_info->proto=PROTO_DNS;
return 1;
}
break;
case ADDR_TYPE_IPV6:
tpl6=a_stream->addr.tuple4_v6;
if((ntohs(tpl6->source)==53) || (ntohs(tpl6->dest)==53))
{
identify_info->proto=PROTO_DNS;
return 1;
}
break;
default:
break;
}
//ftp
ret=ftp_control_identify(a_stream);
if(ret>0)
{
identify_info->proto=PROTO_FTP;
return 1;
}
//mail
return ret;
}
extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet)
{
int identify_flag=0;
int ret=0,hit_num=0;
int state=APP_STATE_DROPME;
int state=APP_STATE_GIVEME;
scan_status_t mid=NULL;
Maat_rule_t *p_result=NULL;
Maat_rule_t *q_result=NULL;
struct _identify_info identify_info;
Maat_rule_t all_result[MAX_RESULT_NUM];
policy_priority_label_t *priority_label=NULL;
policy_priority_label_t *priority_label=NULL;
struct _master_context *_context=(struct _master_context *)*pme;
switch(a_tcp->opstate)
@@ -128,40 +423,19 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
case OP_STATE_PENDING:
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LINKS], 0, FS_OP_ADD, 1);
ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, PROTO_MAX, &mid, all_result+hit_num, MAX_RESULT_NUM-hit_num);
memset(&identify_info, 0, sizeof(identify_info));
identify_application_protocol(a_tcp, &identify_info);
ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_tcp, identify_info.proto, &mid, all_result+hit_num, MAX_RESULT_NUM-hit_num);
if(ret>0)
{
hit_num+=ret;
q_result=tsg_policy_decision_criteria(all_result, hit_num);
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1);
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP",
"Hit addr: %s scan ret: %d policy_id: %d service: %d action: %d",
printaddr(&a_tcp->addr, thread_seq),
ret,
q_result->config_id,
q_result->service_id,
q_result->action);
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1);
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "SCAN_IP", "Not hit %s scan ret: %d",
printaddr(&a_tcp->addr, thread_seq), ret);
}
memset(&identify_info, 0, sizeof(identify_info));
ret=tsg_scan_shared_policy(g_tsg_maat_feather,
a_tcp->ptcpdetail->pdata,
a_tcp->ptcpdetail->datalen,
all_result+hit_num,
MAX_RESULT_NUM-hit_num,
&identify_info,
&mid,
g_tsg_para.logger,
thread_seq);
ret=tsg_scan_shared_policy(g_tsg_maat_feather, &identify_info, all_result+hit_num, MAX_RESULT_NUM-hit_num, &mid, thread_seq);
if(ret>0)
{
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1);
@@ -169,17 +443,15 @@ printaddr(&a_tcp->addr, thread_seq), ret);
RLOG_LV_DEBUG,
"SCAN_FQDN",
"Hit %s: %s policy_id: %d service: %d action: %d addr: %s",
(identify_info.proto==PROTO_HTTP) ? "host" : "sni",
identify_info.domain,
all_result[hit_num].config_id,
all_result[hit_num].service_id,
all_result[hit_num].action,
(unsigned char)all_result[hit_num].action,
printaddr(&a_tcp->addr, thread_seq)
);
hit_num+=ret;
identify_flag=1;
}
else
{
@@ -194,41 +466,45 @@ printaddr(&a_tcp->addr, thread_seq), ret);
printaddr(&a_tcp->addr, thread_seq)
);
}
if(mid!=NULL)
{
Maat_clean_status(&mid);
mid=NULL;
}
p_result=tsg_policy_decision_criteria(all_result, hit_num);
if(p_result!=NULL)
{
{
switch((unsigned char)p_result->action)
{
case TSG_ACTION_DENY:
MESA_kill_tcp(a_tcp, a_packet);
master_do_deny(a_tcp, p_result, thread_seq);
master_send_log(a_tcp, p_result, 1, &identify_info, thread_seq);
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_DENY], 0, FS_OP_ADD, 1);
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "DENY", "Hit deny policy, policy_id: %d action: %d addr: %s",
p_result[0].config_id, p_result[0].action, printaddr(&a_tcp->addr, thread_seq));
master_send_log(a_tcp, p_result, 1, ((identify_flag==1) ? &identify_info : NULL), thread_seq);
state|=APP_STATE_DROPPKT|APP_STATE_KILL_OTHER;
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"DENY",
"Hit deny policy, policy_id: %d service: %d action: %d addr: %s",
p_result[0].config_id,
p_result[0].service_id,
(unsigned char)p_result[0].action,
printaddr(&a_tcp->addr, thread_seq)
);
return APP_STATE_DROPPKT|APP_STATE_KILL_OTHER;
break;
case TSG_ACTION_MONITOR:
if(q_result!=NULL && (p_result==q_result))
{
*pme=dictator_malloc(thread_seq, sizeof(struct _master_context));
_context=(struct _master_context *)*pme;
_context->hit_cnt=1;
_context->result=(struct Maat_rule_t *)dictator_malloc(thread_seq, sizeof(struct Maat_rule_t));
memcpy(_context->result, p_result, sizeof(struct Maat_rule_t));
init_context(pme, identify_info.proto, p_result, thread_seq);
state=APP_STATE_GIVEME;
}
break;
case TSG_ACTION_BYPASS:
*pme=dictator_malloc(thread_seq, sizeof(struct _master_context));
_context=(struct _master_context *)*pme;
_context->hit_cnt=1;
_context->result=(struct Maat_rule_t *)dictator_malloc(thread_seq, sizeof(struct Maat_rule_t));
memcpy(_context->result, p_result, sizeof(struct Maat_rule_t));
init_context(pme, identify_info.proto, p_result, thread_seq);
state=APP_STATE_GIVEME|APP_STATE_KILL_OTHER;
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_BYPASS], 0, FS_OP_ADD, 1);
break;
@@ -250,29 +526,46 @@ printaddr(&a_tcp->addr, thread_seq), ret);
if(ret<0)
{
free_policy_label(thread_seq, (void *)priority_label);
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_ADD", "Add policy_priority_label failed ...");
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
"PROJECT_ADD",
"Add policy_priority_label failed, intercept policy, policy_id: %d action: %d addr: %s",
priority_label->result[0].config_id,
(unsigned char)priority_label->result[0].action,
printaddr(&a_tcp->addr, thread_seq)
);
}
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "INTERCEPT", "Hit intercept policy, policy_id: %d action: %d addr: %s",
priority_label->result[0].config_id, priority_label->result[0].action, printaddr(&a_tcp->addr, thread_seq));
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"INTERCEPT",
"Hit intercept policy, policy_id: %d action: %d addr: %s",
priority_label->result[0].config_id,
(unsigned char)priority_label->result[0].action,
printaddr(&a_tcp->addr, thread_seq)
);
return APP_STATE_DROPME;
break;
case TSG_ACTION_NONE:
default:
assert(0);
return APP_STATE_DROPME;
break;
}
}
break;
break;
case OP_STATE_DATA:
break;
case OP_STATE_CLOSE:
if(_context!=NULL)
{
if(_context->hit_cnt>0 && _context->result!=NULL)
{
master_send_log(a_tcp, _context->result, _context->hit_cnt, NULL, thread_seq);
dictator_free(thread_seq, (void *)_context->result);
_context->result=NULL;
memset(&identify_info, 0, sizeof(identify_info));
identify_info.proto=_context->proto;
master_send_log(a_tcp, _context->result, _context->hit_cnt, &identify_info, thread_seq);
free_context(pme, thread_seq);
}
}
default:
@@ -282,7 +575,79 @@ printaddr(&a_tcp->addr, thread_seq), ret);
return state;
}
extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int thread_seq,void *a_packet)
{
int ret=0,opt_value=0;
scan_status_t mid=NULL;
int state=APP_STATE_GIVEME;
Maat_rule_t *p_result=NULL;
Maat_rule_t result[MAX_RESULT_NUM];
struct _identify_info identify_info;
struct _master_context *_context=(struct _master_context *)*pme;
switch(a_udp->opstate)
{
case OP_STATE_PENDING:
memset(&identify_info, 0, sizeof(identify_info));
identify_application_protocol(a_udp, &identify_info);
ret=tsg_scan_nesting_addr(g_tsg_maat_feather, a_udp, identify_info.proto, &mid, result, MAX_RESULT_NUM);
if(mid!=NULL)
{
Maat_clean_status(&mid);
mid=NULL;
}
p_result=tsg_policy_decision_criteria(result, ret);
if(p_result!=NULL)
{
switch((unsigned char)p_result->action)
{
case TSG_ACTION_DENY:
opt_value=1;
MESA_set_stream_opt(a_udp, MSO_DROP_STREAM, (void *)&opt_value, sizeof(opt_value));
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_DENY], 0, FS_OP_ADD, 1);
return APP_STATE_DROPME|APP_STATE_DROPPKT;
break;
case TSG_ACTION_BYPASS:
init_context(pme, identify_info.proto, p_result, thread_seq);
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_BYPASS], 0, FS_OP_ADD, 1);
state=APP_STATE_GIVEME|APP_STATE_KILL_OTHER;
break;
case TSG_ACTION_MONITOR:
init_context(pme, identify_info.proto, p_result, thread_seq);
state=APP_STATE_GIVEME;
break;
case TSG_ACTION_INTERCEPT:
case TSG_ACTION_MANIPULATE:
default:
return APP_STATE_DROPME;
break;
}
}
break;
case OP_STATE_DATA:
break;
case OP_STATE_CLOSE:
if(_context!=NULL)
{
if(_context->hit_cnt>0 && _context->result!=NULL)
{
memset(&identify_info, 0, sizeof(identify_info));
identify_info.proto=_context->proto;
master_send_log(a_udp, _context->result, _context->hit_cnt, &identify_info, thread_seq);
free_context(pme, thread_seq);
}
}
break;
default:
break;
}
return state;
}
extern "C" int TSG_MASTER_INIT()
{
@@ -298,7 +663,7 @@ extern "C" int TSG_MASTER_INIT()
memset(&g_tsg_para, 0, sizeof(g_tsg_para));
MESA_load_profile_int_def(tsg_conffile, "SYSTEM","LOG_LEVEL", &level, 30);
MESA_load_profile_string_def(tsg_conffile, "SYSTEM","LOG_PATH", log_path, sizeof(log_path), NULL);
MESA_load_profile_string_def(tsg_conffile, "SYSTEM","LOG_PATH", log_path, sizeof(log_path), "tsglog/tsg_master");
g_tsg_para.logger=MESA_create_runtime_log_handle(log_path, level);
if(g_tsg_para.logger==NULL)
@@ -308,6 +673,7 @@ extern "C" int TSG_MASTER_INIT()
}
MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "DEVICE_ID", &g_tsg_para.device_id, 0);
MESA_load_profile_short_def(tsg_conffile, "SYSTEM", "TIMEOUT", (short *)&g_tsg_para.timeout, 300);
MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "POLICY_PRIORITY_LABEL", label_buff, sizeof(label_buff), "POLICY_PRIORITY");
g_tsg_para.priority_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_STRUCT, free_policy_label);
@@ -376,6 +742,7 @@ extern "C" int TSG_MASTER_INIT()
extern "C" int TSG_MASTER_UNLOAD()
{
sleep(5);
return 0;
}

26
src/tsg_entry.h
View File

@@ -5,6 +5,21 @@
#include <MESA/field_stat2.h>
#include "tsg_rule.h"
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
enum MASTER_TABLE{
TABLE_IP_ADDR=0,
TABLE_SUBSCRIBER_ID,
TABLE_APP_ID,
TABLE_HTTP_HOST,
TABLE_SSL_SNI,
TABLE_MAX
};
enum TSG_FS2_TYPE{
TSG_FS2_LINKS=0,
TSG_FS2_BYPASS,
@@ -36,18 +51,21 @@ typedef struct _policy_priority_label
struct _master_context
{
tsg_protocol_t proto;
int hit_cnt;
struct Maat_rule_t *result;
};
#define _MAX_TABLE_NAME_LEN 64
typedef struct _tsg_para
{
int device_id;
int ip_addr_table_id;
int subscribe_id_table_id;
unsigned short timeout;
int device_id;
int table_id[TABLE_MAX];
int dyn_subscribe_ip_table_id; //TSG_DYN_SUBSCRIBER_IP
int priority_project_id;
int fs2_field_id[TSG_FS2_MAX];
char table_name[TABLE_MAX][_MAX_TABLE_NAME_LEN];
void *logger;
screen_stat_handle_t fs2_handle;
}g_tsg_para_t;
@@ -125,4 +143,6 @@ typedef struct _tsg_statistic
int tsg_statistic_init(const char *conffile, void *logger);
int tsg_scan_shared_policy(Maat_feather_t maat_feather, struct _identify_info *identify_info, Maat_rule_t *result, int result_num, scan_status_t *mid, int thread_seq);
#endif

292
src/tsg_rule.cpp
View File

@@ -11,7 +11,6 @@
#include "Maat_rule.h"
#include "Maat_command.h"
#include "MESA/http.h"
#include "tsg_ssl_utils.h"
#include "tsg_rule.h"
#include "tsg_entry.h"
@@ -21,10 +20,6 @@ Maat_feather_t g_tsg_dynamic_maat_feather;
#define MAX_PATH_LEN 1024
#define MAX_IPV6_ADDR_LEN 128
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif
enum kni_scan_table{
TSG_FIELD_SSL_SNI,
TSG_FIELD_HTTP_HOST,
@@ -42,6 +37,21 @@ const struct _str2index method2index[TSG_METHOD_TYPE_MAX]={ {TSG_METHOD_TYPE_UNK
{TSG_METHOD_TYPE_RESET, 3, (char *)"rst"}
};
const struct _str2index g_tsg_proto_string[PROTO_MAX+1]={{PROTO_UNKONWN, 0, (char *)""},
{PROTO_IPv4, 5, (char *)"IPv4."},
{PROTO_IPv6, 5, (char *)"IPv6."},
{PROTO_TCP, 4, (char *)"TCP."},
{PROTO_UDP, 4, (char *)"UDP."},
{PROTO_HTTP, 5, (char *)"HTTP."},
{PROTO_MAIL, 5, (char *)"MAIL."},
{PROTO_DNS, 4, (char *)"DNS."},
{PROTO_FTP, 4, (char *)"FTP."},
{PROTO_SSL, 4, (char *)"SSL."},
{PROTO_SIP, 4, (char *)"SIP."},
{PROTO_BGP, 4, (char *)"BGP."},
{PROTO_STREAMING_MEDIA, 16, (char *)"STREAMING_MEDIA."},
{PROTO_MAX, 0, (char *)""}
};
void subscribe_id_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp)
{
@@ -50,7 +60,7 @@ void subscribe_id_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX
*to=calloc(1, strlen((char *)*from)+1);
memcpy(*to, *from, strlen((char *)*from));
MESA_handle_runtime_log(logger, RLOG_LV_INFO, "SUBSCRIBE_ID", "Dup subscribe_id: %s table_id: %d", (char *)*to, table_id);
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Dup subscribe_id: %s table_id: %d", (char *)*to, table_id);
return;
}
@@ -80,7 +90,7 @@ void subscribe_id_new_data(int table_id, const char* key, const char* table_line
memcpy(*ad, subscribe_id, strlen(subscribe_id));
MESA_handle_runtime_log(logger,
RLOG_LV_INFO,
RLOG_LV_DEBUG,
"SUBSCRIBE_ID",
"Add subscribe_id: %s table_id: %d key: %s table_line: %s",
*ad,
@@ -94,7 +104,7 @@ void subscribe_id_new_data(int table_id, const char* key, const char* table_line
void subscribe_id_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
void *logger=argp;
MESA_handle_runtime_log(logger, RLOG_LV_INFO, "SUBSCRIBE_ID", "Delete subscribe_id: %s table_id: %d", (char *)*ad, table_id);
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Delete subscribe_id: %s table_id: %d", (char *)*ad, table_id);
free(*ad);
*ad=NULL;
@@ -200,16 +210,16 @@ static Maat_feather_t init_maat_feather(const char* conffile, char* instance_nam
int tsg_rule_init(const char* conffile, void *logger)
{
int ret=0;
int i=0,ret=0;
char maat_conffile[256]={0};
char ip_addr_table[32]={0};
char subscriber_id_table[32]={0};
char cb_subscriber_ip_table[32]={0};
MESA_load_profile_string_def(conffile, "MAAT", "PROFILE", maat_conffile, sizeof(maat_conffile), "./tsgconf/maat_profile.conf");
MESA_load_profile_string_def(conffile, "MAAT", "IP_ADDR_TABLE", ip_addr_table, sizeof(ip_addr_table), "TSG_OBJ_IP_ADDR");
MESA_load_profile_string_def(conffile, "MAAT", "SUBSCRIBER_ID_TABLE", subscriber_id_table, sizeof(subscriber_id_table), "TSG_OBJ_SUBSCRIBER_ID");
MESA_load_profile_string_def(conffile, "MAAT", "IP_ADDR_TABLE", g_tsg_para.table_name[TABLE_IP_ADDR], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_ADDR");
MESA_load_profile_string_def(conffile, "MAAT", "SUBSCRIBER_ID_TABLE", g_tsg_para.table_name[TABLE_SUBSCRIBER_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_SUBSCRIBER_ID");
MESA_load_profile_string_def(conffile, "MAAT", "APP_ID_TABLE", g_tsg_para.table_name[TABLE_APP_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_APP_ID");
MESA_load_profile_string_def(conffile, "MAAT", "HTTP_HOST_TABLE", g_tsg_para.table_name[TABLE_HTTP_HOST], _MAX_TABLE_NAME_LEN, "TSG_FIELD_HTTP_HOST");
MESA_load_profile_string_def(conffile, "MAAT", "SSL_SNI_TABLE", g_tsg_para.table_name[TABLE_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_SSL_SNI");
//init dynamic maat feather
g_tsg_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_STATIC", (char *)"STATIC", logger);
@@ -218,29 +228,16 @@ int tsg_rule_init(const char* conffile, void *logger)
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "init_maat_feather failed, instance_name: %s module: %s", "TSG_STATIC", "STATIC");
return -1;
}
g_tsg_para.ip_addr_table_id=Maat_table_register(g_tsg_maat_feather, ip_addr_table);
if(g_tsg_para.ip_addr_table_id<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Maat_table_register %s failed", ip_addr_table);
return -1;
}
g_tsg_para.subscribe_id_table_id=Maat_table_register(g_tsg_maat_feather, subscriber_id_table);
if(g_tsg_para.subscribe_id_table_id<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "RULE_INIT", "Maat_table_register %s failed", subscriber_id_table);
return -1;
}
// init sni or host share table
ret=tsg_shared_table_init(conffile, g_tsg_maat_feather, logger);
if(ret<0)
for(i=0; i<TABLE_MAX; i++)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "RULE_INIT", "tsg_shared_table_init %s failed");
return -1;
}
g_tsg_para.table_id[i]=Maat_table_register(g_tsg_maat_feather, g_tsg_para.table_name[i]);
if(g_tsg_para.table_id[i]<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Maat_table_register %s failed, Please check tsgconf/tsg_static_tableinfo.conf", g_tsg_para.table_name[i]);
return -1;
}
}
//init dynamic maat feather
g_tsg_dynamic_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_DYNAMIC", (char *)"DYNAMIC", logger);
@@ -275,59 +272,6 @@ int tsg_rule_init(const char* conffile, void *logger)
return 0;
}
static void protocol_identify(char *buff, int buff_len, struct _identify_info *result){
result->proto = PROTO_UNKONWN;
//http
char *host = NULL;
int ret = http_host_parser(buff, (uint32_t)buff_len, DIR_C2S, &host);
//printf("http_host_parse: ret = %d, buff_len = %d, buff = %s\n", ret, buff_len, buff);
if(ret >= 0){
result->proto = PROTO_HTTP;
if(ret == 0){
result->domain_len = 0;
}
else{
result->domain_len = MIN(ret, (int)sizeof(result->domain) - 1);
strncpy(result->domain, host, result->domain_len);
}
return;
}
//ssl
enum chello_parse_result chello_status = CHELLO_PARSE_INVALID_FORMAT;
struct ssl_chello *chello = NULL;
chello = ssl_chello_parse((const unsigned char*)buff, buff_len, &chello_status);
if(chello_status == CHELLO_PARSE_SUCCESS){
result->proto = PROTO_SSL;
if(chello->sni == NULL){
result->domain_len = 0;
}
else{
result->domain_len = strnlen(chello->sni, sizeof(result->domain) - 1);
strncpy(result->domain, chello->sni, result->domain_len);
}
}
ssl_chello_free(chello);
return;
}
//return -1 if failed, return 0 on success;
int tsg_shared_table_init(const char *conffile, Maat_feather_t maat_feather, void *logger){
g_tsg_maat_feather = maat_feather;
g_kni_scan_table_name[TSG_FIELD_HTTP_HOST] = "TSG_FIELD_HTTP_HOST";
g_kni_scan_table_name[TSG_FIELD_SSL_SNI] = "TSG_FIELD_SSL_SNI";
int i;
for(i = 0; i < SCAN_TABLE_MAX; i++){
g_kni_scan_tableid[i] = Maat_table_register(maat_feather, g_kni_scan_table_name[i]);
if(g_kni_scan_tableid[i] < 0){
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Failed at Maat_table_register, tablename = %s, ret = %d",
g_kni_scan_table_name[i], g_kni_scan_tableid[i]);
return -1;
}
}
return 0;
}
int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_result_type, Maat_rule_t*result, int result_num, struct _identify_info *identify_info)
{
int num=0;
@@ -420,8 +364,13 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
const struct streaminfo *cur_stream = a_stream;
if(result == NULL || result_num <= 0 || a_stream == NULL || maat_feather == NULL)
if(result==NULL || result_num<=0 || a_stream==NULL || maat_feather==NULL)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
"SCAN_NESTING_ADDR",
"result==NULL || result_num<=0 || maat_feather==NULL || a_stream==%s",
(a_stream!=NULL) ? printaddr(&a_stream->addr, a_stream->threadnum) : "NULL");
return -1;
}
@@ -466,17 +415,38 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
}
maat_ret=Maat_scan_proto_addr(maat_feather,
g_tsg_para.ip_addr_table_id,
g_tsg_para.table_id[TABLE_IP_ADDR],
p_addr,
tans_proto,
result+hit_num,
result_num-hit_num,
mid,
cur_stream->threadnum);
if(maat_ret > 0)
if(maat_ret>0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP",
"Hit addr: %s scan ret: %d policy_id: %d service: %d action: %d",
printaddr(&cur_stream->addr, cur_stream->threadnum),
maat_ret,
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action
);
hit_num+=maat_ret;
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP",
"No hit addr: %s scan ret: %d",
printaddr(&cur_stream->addr, cur_stream->threadnum),
maat_ret
);
}
}
cur_stream = cur_stream->pfather;
@@ -484,6 +454,48 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
}while(cur_stream != NULL && hit_num < result_num);
if(hit_num<result_num && proto>PROTO_UNKONWN && proto<PROTO_MAX)
{
maat_ret=Maat_full_scan_string(maat_feather,
g_tsg_para.table_id[TABLE_APP_ID],
CHARSET_GBK,
g_tsg_proto_string[proto].type,
strlen(g_tsg_proto_string[proto].type),
result+hit_num,
&found_pos,
result_num-hit_num,
mid,
a_stream->threadnum);
if(maat_ret > 0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_PROTO",
"Hit PROTO: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
g_tsg_proto_string[proto].type,
maat_ret,
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
printaddr(&a_stream->addr, a_stream->threadnum)
);
hit_num+=maat_ret;
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_PROTO",
"No hit PROTO: %s scan ret: %d addr: %s",
g_tsg_proto_string[proto].type,
maat_ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
}
if(hit_num<result_num)
{
tsg_get_subscribe_id(a_stream, &source_subscribe_id, &dest_subscribe_id);
@@ -491,7 +503,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
if(source_subscribe_id!=NULL)
{
maat_ret=Maat_full_scan_string(maat_feather,
g_tsg_para.subscribe_id_table_id,
g_tsg_para.table_id[TABLE_SUBSCRIBER_ID],
CHARSET_GBK,
source_subscribe_id,
strlen(source_subscribe_id),
@@ -501,9 +513,32 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
mid,
a_stream->threadnum);
if(maat_ret > 0)
{
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_SUBSCRIBER",
"Hit source subscribe id: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
source_subscribe_id,
maat_ret,
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
printaddr(&a_stream->addr, a_stream->threadnum)
);
hit_num+=maat_ret;
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_SUBSCRIBER",
"No hit source subscribe id: %s scan ret: %d addr: %s",
source_subscribe_id,
maat_ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
subscribe_id_free_data(g_tsg_para.dyn_subscribe_ip_table_id,(MAAT_PLUGIN_EX_DATA *)&source_subscribe_id, 0, g_tsg_para.logger);
}
@@ -511,7 +546,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
if(dest_subscribe_id!=NULL)
{
maat_ret=Maat_full_scan_string(maat_feather,
g_tsg_para.subscribe_id_table_id,
g_tsg_para.table_id[TABLE_SUBSCRIBER_ID],
CHARSET_GBK,
dest_subscribe_id,
strlen(dest_subscribe_id),
@@ -522,8 +557,31 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
a_stream->threadnum);
if(maat_ret > 0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_SUBSCRIBER",
"Hit dest subscribe id: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
dest_subscribe_id,
maat_ret,
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
printaddr(&a_stream->addr, a_stream->threadnum)
);
hit_num+=maat_ret;
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_SUBSCRIBER",
"No hit dest subscribe id: %s scan ret: %d addr: %s",
dest_subscribe_id,
maat_ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
subscribe_id_free_data(g_tsg_para.dyn_subscribe_ip_table_id,(MAAT_PLUGIN_EX_DATA *)&dest_subscribe_id, 0, g_tsg_para.logger);
}
@@ -534,23 +592,39 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
//return value: -1: failed, 0: not hit, >0: hit count
int tsg_scan_shared_policy(Maat_feather_t maat_feather, void *pkt, int pkt_len, Maat_rule_t *result, int result_num,
struct _identify_info *identify_info, scan_status_t *mid, void *logger, int thread_seq)
int tsg_scan_shared_policy(Maat_feather_t maat_feather, struct _identify_info *identify_info, Maat_rule_t *result, int result_num, scan_status_t *mid, int thread_seq)
{
memset(identify_info, 0, sizeof(*identify_info));
protocol_identify((char*)pkt, pkt_len, identify_info);
if(identify_info->proto != PROTO_SSL && identify_info->proto != PROTO_HTTP){
return -1;
int ret=0,idx=0;
if(identify_info->proto!=PROTO_UNKONWN && identify_info->domain_len>0)
{
switch(identify_info->proto)
{
case PROTO_HTTP:
idx=TABLE_HTTP_HOST;
break;
case PROTO_SSL:
idx=TABLE_SSL_SNI;
break;
default:
return 0;
break;
}
ret=Maat_full_scan_string(g_tsg_maat_feather,
g_tsg_para.table_id[idx],
CHARSET_UTF8,
identify_info->domain,
identify_info->domain_len,
result,
NULL,
result_num,
mid,
thread_seq
);
}
int tableid;
if(identify_info->proto == PROTO_SSL){
tableid = g_kni_scan_tableid[TSG_FIELD_SSL_SNI];
}
else{
tableid = g_kni_scan_tableid[TSG_FIELD_HTTP_HOST];
}
return Maat_full_scan_string(g_tsg_maat_feather, tableid, CHARSET_UTF8, identify_info->domain, identify_info->domain_len,
result, NULL, result_num, mid, thread_seq);
return ret;
}

126
src/tsg_send_log.cpp
View File

@@ -19,7 +19,7 @@
#include "tsg_send_log.h"
#include "tsg_send_log_internal.h"
char TSG_SEND_LOG_VERSION_20191129=0;
char TSG_SEND_LOG_VERSION_20200119=0;
struct tsg_log_instance_t *g_tsg_log_instance;
@@ -238,7 +238,7 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle
return 0;
}
int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t *service2topic)
int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t **service2topic, int *max_service)
{
int i=0;
int ret=0,id=0;
@@ -246,7 +246,8 @@ int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t
char line[1024]={0};
char field_name[64]={0};
char type_name[32]={0};
id2field_t *_service2topic=NULL;
fp=fopen(filename, "r");
if(fp==NULL)
{
@@ -282,9 +283,36 @@ int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t
default:
if((strncasecmp("TOPIC", type_name, strlen("TOPIC")))==0)
{
service2topic[id].type = TLD_TYPE_MAX;
service2topic[id].id = id;
memcpy(service2topic[id].name, field_name, strlen(field_name));
if(_service2topic==NULL)
{
_service2topic=(id2field_t *)calloc(1, sizeof(id2field_t)*(id+1));
_service2topic[id].type = TLD_TYPE_MAX;
_service2topic[id].id = id;
memcpy(_service2topic[id].name, field_name, strlen(field_name));
*max_service=id+1;
}
else
{
if(*max_service<=id)
{
_service2topic=(id2field_t *)realloc(_service2topic, sizeof(id2field_t)*(id+1));
memset(&_service2topic[id], 0, sizeof(id2field_t));
_service2topic[id].type = TLD_TYPE_MAX;
_service2topic[id].id = id;
memcpy(_service2topic[id].name, field_name, strlen(field_name));
*max_service=id+1;
}
else
{
memset(&_service2topic[id], 0, sizeof(id2field_t));
_service2topic[id].type = TLD_TYPE_MAX;
_service2topic[id].id = id;
memcpy(_service2topic[id].name, field_name, strlen(field_name));
}
}
}
break;
}
@@ -296,6 +324,8 @@ int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t
fclose(fp);
fp=NULL;
*service2topic=_service2topic;
return 0;
}
@@ -314,7 +344,7 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
_instance=(struct tsg_log_instance_t *)calloc(1, sizeof(struct tsg_log_instance_t));
MESA_load_profile_int_def(conffile, "TSG_LOG", "LOG_LEVEL",&(level), 30);
MESA_load_profile_string_def(conffile, "TSG_LOG", "LOG_PATH", log_path, sizeof(log_path), NULL);
MESA_load_profile_string_def(conffile, "TSG_LOG", "LOG_PATH", log_path, sizeof(log_path), "./tsglog/tsglog");
_instance->logger=MESA_create_runtime_log_handle(log_path, level);
if(_instance->logger==NULL)
@@ -358,23 +388,26 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "KAFKA_INIT", "rd_kafka_brokers_add is error, broker_list: %s", _instance->broker_list);
return NULL;
}
MESA_load_profile_int_def(conffile, "TSG_LOG", "MAX_SERVICE",&(_instance->max_service), 0);
(_instance->topic_rkt)=(rd_kafka_topic_t **)calloc(1, sizeof(rd_kafka_topic_t*));
*(_instance->topic_rkt)=(rd_kafka_topic_t *)calloc(1, (1+_instance->max_service)*sizeof(rd_kafka_topic_t*));
_instance->service2topic=(id2field_t *)calloc(1, (1+_instance->max_service)*sizeof(id2field_t));
load_log_common_field(_instance->common_field_file, _instance->id2field, _instance->service2topic);
load_log_common_field(_instance->common_field_file, _instance->id2field, &(_instance->service2topic), &(_instance->max_service));
for(i=0; i<_instance->max_service+1; i++)
if(_instance->service2topic!=NULL)
{
if(_instance->service2topic[i].type==TLD_TYPE_MAX)
_instance->topic_rkt=(rd_kafka_topic_t **)calloc(1, (_instance->max_service)*sizeof(rd_kafka_topic_t*));
for(i=0; i<_instance->max_service+1; i++)
{
topic_conf=rd_kafka_topic_conf_new();
_instance->topic_rkt[_instance->service2topic[i].id]=rd_kafka_topic_new(kafka_handle, _instance->service2topic[i].name, topic_conf);
if(_instance->service2topic[i].type==TLD_TYPE_MAX)
{
topic_conf=rd_kafka_topic_conf_new();
_instance->topic_rkt[_instance->service2topic[i].id]=rd_kafka_topic_new(kafka_handle, _instance->service2topic[i].name, topic_conf);
}
}
}
else
{
MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "KAFKA_INIT", "load_log_common_field is error, please check %s", _instance->common_field_file);
}
return _instance;
}
@@ -382,7 +415,7 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, tsg_log_t *log_msg, int thread_id)
{
int i=0,ret=0,status=0;
int i=0,status=0;
char *payload=NULL;
struct TLD_handle_t *_handle=handle;
struct tsg_log_instance_t *_instance=instance;
@@ -400,37 +433,10 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
MESA_handle_runtime_log(_instance->logger, RLOG_LV_INFO, "TSG_SEND_LOG", "Disable tsg_send_log.");
return 0;
}
//TODO
//common_user_tags
//common_isp
//common_app_label
//common_app_id
//common_protocol_id
//common_has_dup_traffic
//common_stream_error
TLD_append_streaminfo(instance, handle, log_msg->a_stream);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SLED_IP].name, (void *)(_instance->local_ip_str), TLD_TYPE_STRING);
#if 0
struct vxlan_info vinfo;
int opt_val_len = sizeof(vinfo);
status=MESA_get_stream_opt(log_msg->a_stream, MSO_STREAM_VXLAN_INFO, &vinfo, &opt_val_len);
if(status < 0)
{
MESA_handle_runtime_log(_instance->logger, RLOG_LV_DEBUG, "TSG_SEND_LOG", "tsg log: get vxlan info error, tuple4: %s", printaddr(&log_msg->a_stream->addr, thread_id));
}
else
{
TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_LINK_ID].name, (void *)(long)vinfo.link_id, TLD_TYPE_LONG);
TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_DIRECTION].name, (void *)(long)vinfo.link_dir, TLD_TYPE_LONG);
TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_DEVICE_ID].name, (void *)(long)vinfo.dev_id, TLD_TYPE_LONG);
TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENTRANCE_ID].name, (void *)(long)vinfo.entrance_id, TLD_TYPE_LONG);
TLD_append((TLD_handle_t)_handle, _instance->id2field[LOG_COMMON_ENCAPSULATION].name, (void *)(long)vinfo.encap_type, TLD_TYPE_LONG);
}
#endif
for(i=0;i<log_msg->result_num; i++)
{
switch(log_msg->result[i].do_log)
@@ -457,33 +463,6 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG);
if(log_msg->result[i].serv_def_len<128)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)(log_msg->result[i].service_defined), TLD_TYPE_STRING);
}
else
{
char *service_defined=(char *)calloc(1, log_msg->result[i].serv_def_len+1);
ret=Maat_read_rule(g_tsg_maat_feather, &log_msg->result[i], MAAT_RULE_SERV_DEFINE, service_defined, log_msg->result[i].serv_def_len);
if(ret==log_msg->result[i].serv_def_len)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_USER_REGION].name, (void *)service_defined, TLD_TYPE_STRING);
}
else
{
MESA_handle_runtime_log(_instance->logger,
RLOG_LV_FATAL,
"TSG_SEND_LOG",
"Fetch service_defined failed, policy_id: %d service: %d action: %d addr: %s",
log_msg->result[i].config_id,
log_msg->result[i].service_id,
log_msg->result[i].action,
printaddr(&log_msg->a_stream->addr, thread_id));
}
free((void *)service_defined);
service_defined=NULL;
}
payload = cJSON_PrintUnformatted(_handle->object);
status = rd_kafka_produce(_instance->topic_rkt[log_msg->result[i].service_id], RD_KAFKA_PARTITION_UA, RD_KAFKA_MSG_F_COPY, payload, strlen(payload), NULL, 0, NULL);
@@ -506,7 +485,6 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
TLD_delete(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_SERVICE].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_ACTION].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_USER_REGION].name);
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LOG], 0, FS_OP_ADD, 1);
}

48
src/tsg_statistic.cpp
View File

@@ -60,6 +60,15 @@ static int _get_traffic_info(struct _traffic_info *total, struct _traffic_info *
out->in_packets=total->in_packets-policy->in_packets;
out->out_bytes=total->out_bytes-policy->out_bytes;
out->out_packets=total->out_packets-policy->out_packets;
if(out->con_num<0 || out->in_bytes<0 || out->in_packets<0 || out->out_bytes<0 || out->out_packets<0)
{
out->con_num=total->con_num;
out->in_bytes=total->in_bytes;
out->in_packets=total->in_packets;
out->out_bytes=total->out_bytes;
out->out_packets=total->out_packets;
}
}
return 0;
}
@@ -130,25 +139,13 @@ static int _set_traffic_info(struct _traffic_info *from, struct _traffic_info *t
in_packets_id=STATIS_DEFAULT_IN_PACKETS;
out_bytes_id=STATIS_DEFAULT_OUT_BYTES;
out_packets_id=STATIS_DEFAULT_OUT_PACKETS;
con_num_sum+=from->con_num;
in_bytes_sum+=from->in_bytes;
in_packets_sum+=from->in_packets;
out_bytes_sum+=from->out_bytes;
out_packets_sum+=from->out_packets;
break;
case -1:
con_num_id=STATIS_TOTAL_CON_NUM;
con_num_id=STATIS_NEW_CON_NUM;
in_bytes_id=STATIS_TOTAL_IN_BYTES;
in_packets_id=STATIS_TOTAL_IN_PACKETS;
out_bytes_id=STATIS_TOTAL_OUT_BYTES;
out_packets_id=STATIS_TOTAL_OUT_PACKETS;
con_num_sum+=from->con_num;
in_bytes_sum+=from->in_bytes;
in_packets_sum+=from->in_packets;
out_bytes_sum+=from->out_bytes;
out_packets_sum+=from->out_packets;
break;
default:
return 0;
@@ -180,21 +177,25 @@ static void *tsg_statistic_thread(void *arg)
memset(&total_traffic_info, 0, sizeof(total_traffic_info));
memset(&default_traffic_info, 0, sizeof(default_traffic_info));
value=0;
sapp_get_platform_opt(SPO_TCP_STREAM_NEW, (void *)&value, &value_len);
total_traffic_info.con_num+=value;
FS_operate(g_tsg_statis_para.fs2_handle, g_tsg_statis_para.fs_line_id, g_tsg_statis_para.fs_field_id[STATIS_NEW_CON_NUM], FS_OP_SET, value);
_set_traffic_info(g_tsg_statis_para.traffic_info[TSG_ACTION_BYPASS], &policy_traffic_info, TSG_ACTION_BYPASS, thread_num);
_set_traffic_info(g_tsg_statis_para.traffic_info[TSG_ACTION_DENY], &policy_traffic_info, TSG_ACTION_DENY, thread_num);
_set_traffic_info(g_tsg_statis_para.traffic_info[TSG_ACTION_MONITOR], &policy_traffic_info, TSG_ACTION_MONITOR, thread_num);
_set_traffic_info(g_tsg_statis_para.traffic_info[TSG_ACTION_INTERCEPT], &policy_traffic_info, TSG_ACTION_INTERCEPT, thread_num);
value=0;
sapp_get_platform_opt(SPO_TCP_STREAM_ESTAB, (void *)&value, &value_len);
total_traffic_info.con_num+=value;
FS_operate(g_tsg_statis_para.fs2_handle, g_tsg_statis_para.fs_line_id, g_tsg_statis_para.fs_field_id[STATIS_ESTABLISHED_CON_NUM], FS_OP_SET, value);
value=0;
sapp_get_platform_opt(SPO_TCP_STREAM_CLOSE, (void *)&value, &value_len);
total_traffic_info.con_num+=value;
FS_operate(g_tsg_statis_para.fs2_handle, g_tsg_statis_para.fs_line_id, g_tsg_statis_para.fs_field_id[STATIS_CLOSE_CON_NUM], FS_OP_SET, value);
value=0;
sapp_get_platform_opt(SPO_TCP_STREAM_NEW, (void *)&value, &value_len);
FS_operate(g_tsg_statis_para.fs2_handle, g_tsg_statis_para.fs_line_id, g_tsg_statis_para.fs_field_id[STATIS_NEW_CON_NUM], FS_OP_SET, value);
total_traffic_info.con_num+=value;
value_len=sizeof(total_traffic_info.in_bytes);
sapp_get_platform_opt(SPO_TOTAL_INBOUND_BYTE, (void *)&total_traffic_info.in_bytes, &value_len);
value_len=sizeof(total_traffic_info.in_packets);
@@ -204,16 +205,11 @@ static void *tsg_statistic_thread(void *arg)
sapp_get_platform_opt(SPO_TOTAL_OUTBOUND_BYTE, (void *)&total_traffic_info.out_bytes, &value_len);
value_len=sizeof(total_traffic_info.out_packets);
sapp_get_platform_opt(SPO_TOTAL_OUTBOUND_PKT, (void *)&total_traffic_info.out_packets, &value_len);
_set_traffic_info(g_tsg_statis_para.traffic_info[TSG_ACTION_BYPASS], &policy_traffic_info, TSG_ACTION_BYPASS, thread_num);
_set_traffic_info(g_tsg_statis_para.traffic_info[TSG_ACTION_DENY], &policy_traffic_info, TSG_ACTION_DENY, thread_num);
_set_traffic_info(g_tsg_statis_para.traffic_info[TSG_ACTION_MONITOR], &policy_traffic_info, TSG_ACTION_MONITOR, thread_num);
_set_traffic_info(g_tsg_statis_para.traffic_info[TSG_ACTION_INTERCEPT], &policy_traffic_info, TSG_ACTION_INTERCEPT, thread_num);
_set_traffic_info(&total_traffic_info, NULL, -1, 0);
_set_traffic_info(&total_traffic_info, NULL, -1, 1);
_get_traffic_info(&total_traffic_info, &policy_traffic_info, &default_traffic_info);
_set_traffic_info(&default_traffic_info, NULL, TSG_ACTION_NONE, 0);
_set_traffic_info(&default_traffic_info, NULL, TSG_ACTION_NONE, 1);
FS_passive_output(g_tsg_statis_para.fs2_handle);

1
src/version.map
View File

@@ -4,6 +4,7 @@ global:
g_*;
*TSG_MASTER_INIT*;
*TSG_MASTER_TCP_ENTRY*;
*TSG_MASTER_UDP_ENTRY*;
*TSG_MASTER_UNLOAD*;
*tsg_scan_nesting_addr*;
*tsg_pull_policy_result*;