gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature-fast-path' into develop

Browse Source
This commit is contained in:
liuxueli
2020-04-30 17:50:20 +08:00
parent acb61ea1a8 83241f8107
commit aaea800a56
9 changed files with 217 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bin/main.conf
View File

@@ -1,6 +1,6 @@
[MAAT]
PROFILE=./tsgconf/maat.conf
IP_ADDR_TABLE=TSG_OBJ_IP_ADDR
IP_ADDR_TABLE=TSG_SECURITY_ADDR
SUBSCRIBER_ID_TABLE=TSG_OBJ_SUBSCRIBER_ID
CB_SUBSCRIBER_IP_TABLE=TSG_DYN_SUBSCRIBER_IP

1
bin/tsg_log_field.conf
View File

@@ -41,3 +41,4 @@ STRING common_schema_type 34
STRING http_host 35
STRING ssl_sni 36
LONG common_establish_latency_ms 37
STRING common_sub_action 38

2
bin/tsg_static_tableinfo.conf
View File

@@ -43,3 +43,5 @@
31 TSG_SECURITY_SOURCE_ADDR virtual TSG_OBJ_IP_ADDR --
32 TSG_SECURITY_DESTINATION_ADDR virtual TSG_OBJ_IP_ADDR --
33 TSG_SECURITY_ADDR composition {"source":"TSG_SECURITY_SOURCE_ADDR","destination":"TSG_SECURITY_DESTINATION_ADDR"}
34 TSG_DECYPTION_EXCLUSION_SSL_SNI virtual TSG_OBJ_FQDN
35 TSG_PROFILE_RESPONSE_PAGES plugin {"key":1,"foreign":"4","valid":5} --

1
inc/tsg_rule.h
View File

@@ -18,6 +18,7 @@ enum TSG_ETHOD_TYPE
TSG_METHOD_TYPE_REDIRECTION,
TSG_METHOD_TYPE_BLOCK,
TSG_METHOD_TYPE_RESET,
TSG_METHOD_TYPE_ALERT,
TSG_METHOD_TYPE_MAX
};

140
src/tsg_entry.cpp
View File

@@ -41,7 +41,7 @@ static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL;
#endif
char TSG_MASTER_VERSION_20200330=0;
char TSG_MASTER_VERSION_20200430=0;
const char *tsg_conffile="tsgconf/main.conf";
g_tsg_para_t g_tsg_para;
@@ -291,24 +291,70 @@ static int master_send_log(struct streaminfo *a_stream, struct Maat_rule_t *p_re
return 1;
}
static struct Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int result_num)
static struct Maat_rule_t *tsg_policy_decision_criteria(struct streaminfo *a_stream, Maat_rule_t *result, int result_num, struct _identify_info *identify_info, int thread_seq)
{
int i=0;
int i=0,ret=0;
Maat_rule_t tmp_result;
Maat_rule_t *p_result=NULL;
Maat_rule_t *q_result=NULL;
scan_status_t mid=NULL;
if(result==NULL || result_num<=0)
{
return NULL;
}
#if 0
ret=Maat_rule_sort_by_evaluation_order(g_tsg_maat_feather, result, result_num);
if(ret!=result_num)
{
return NULL;
}
p_result=&result[0];
if(p_result->action==TSG_ACTION_INTERCEPT && domain!=NULL && domain_len>0)
{
ret=Maat_full_scan_string(g_tsg_maat_feather,
g_tsg_para.table_id[TABLE_EXCLUSION_SSL_SNI],
CHARSET_UTF8,
domain,
domain_len,
&tmp_result,
NULL,
1,
&mid,
thread_seq);
if(ret>0)
{
p_result=NULL;
for(i=1; i<result_num; i++)
{
if(p_result->action==TSG_ACTION_INTERCEPT)
{
continue;
}
p_result=&result[i];
}
}
}
return p_result;
#endif
p_result=&result[0];
for(i=1; i<result_num; i++)
{
if((unsigned char)result[i].action>(unsigned char)p_result->action)
{
q_result=p_result;
p_result=&result[i];
continue;
}
if(q_result==NULL)
{
q_result=&result[i];
}
if(result[i].action==p_result->action)
{
@@ -319,6 +365,49 @@ static struct Maat_rule_t *tsg_policy_decision_criteria(Maat_rule_t *result, int
}
}
if(p_result->action==TSG_ACTION_INTERCEPT && identify_info!=NULL && identify_info->domain_len>0)
{
ret=Maat_full_scan_string(g_tsg_maat_feather,
g_tsg_para.table_id[TABLE_EXCLUSION_SSL_SNI],
CHARSET_UTF8,
identify_info->domain,
identify_info->domain_len,
&tmp_result,
NULL,
1,
&mid,
thread_seq);
if(ret>0)
{
p_result=q_result;
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"EXCLUSION_SSL_SNI",
"Hit %s policy_id: %d service: %d action: %d addr: %s",
identify_info->domain,
tmp_result.config_id,
tmp_result.service_id,
(unsigned char)tmp_result.action,
printaddr(&a_stream->addr, thread_seq)
);
master_send_log(a_stream, &tmp_result, 1, identify_info, thread_seq);
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"EXCLUSION_SSL_SNI",
"Not hit %s stream_dir: %d addr: %s scan ret: %d",
identify_info->domain,
a_stream->dir,
printaddr(&a_stream->addr, thread_seq),
ret
);
}
}
return p_result;
}
@@ -467,7 +556,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
if(ret>0)
{
hit_num+=ret;
q_result=tsg_policy_decision_criteria(all_result, hit_num);
q_result=tsg_policy_decision_criteria(a_tcp, all_result, hit_num, NULL, thread_seq);
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1);
}
@@ -510,7 +599,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
mid=NULL;
}
p_result=tsg_policy_decision_criteria(all_result, hit_num);
p_result=tsg_policy_decision_criteria(a_tcp, all_result, hit_num, &identify_info, thread_seq);
if(p_result!=NULL)
{
@@ -543,6 +632,16 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
memcpy(_context->domain, identify_info.domain, identify_info.domain_len);
_context->domain_len=identify_info.domain_len;
}
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"MONITOR",
"Hit monitor policy, policy_id: %d service: %d action: %d addr: %s",
p_result[0].config_id,
p_result[0].service_id,
(unsigned char)p_result[0].action,
printaddr(&a_tcp->addr, thread_seq)
);
state=APP_STATE_GIVEME;
}
break;
@@ -554,6 +653,17 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
memcpy(_context->domain, identify_info.domain, identify_info.domain_len);
_context->domain_len=identify_info.domain_len;
}
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"ALLOW",
"Hit allow policy, policy_id: %d service: %d action: %d addr: %s",
p_result[0].config_id,
p_result[0].service_id,
(unsigned char)p_result[0].action,
printaddr(&a_tcp->addr, thread_seq)
);
state=APP_STATE_GIVEME|APP_STATE_KILL_OTHER;
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_BYPASS], 0, FS_OP_ADD, 1);
break;
@@ -651,7 +761,7 @@ extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int t
Maat_clean_status(&mid);
mid=NULL;
}
p_result=tsg_policy_decision_criteria(result, ret);
p_result=tsg_policy_decision_criteria(a_udp, result, ret, NULL, thread_seq);
if(p_result!=NULL)
{
switch((unsigned char)p_result->action)
@@ -732,7 +842,13 @@ extern "C" int TSG_MASTER_INIT()
g_tsg_para.priority_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_STRUCT, free_policy_label);
if(g_tsg_para.priority_project_id<0)
{
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_REGISTER", "Register %s failed ...", label_buff);
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
"PROJECT_REGISTER",
"Register %s failed; please check :%s and add <POLICY_PRIORITY struct>",
label_buff,
"etc/project_list.conf"
);
return -1;
}
@@ -740,7 +856,13 @@ extern "C" int TSG_MASTER_INIT()
g_tsg_para.establish_latency_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_LONG, NULL);
if(g_tsg_para.establish_latency_project_id<0)
{
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "PROJECT_REGISTER", "Register %s failed ...", label_buff);
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
"PROJECT_REGISTER",
"Register %s failed; please check :%s and add <ESTABLISH_LATENCY long>",
label_buff,
"etc/project_list.conf"
);
}
ret=tsg_rule_init(tsg_conffile, g_tsg_para.logger);

1
src/tsg_entry.h
View File

@@ -16,6 +16,7 @@ enum MASTER_TABLE{
TABLE_APP_ID,
TABLE_HTTP_HOST,
TABLE_SSL_SNI,
TABLE_EXCLUSION_SSL_SNI,
TABLE_MAX
};

21
src/tsg_rule.cpp
View File

@@ -34,7 +34,8 @@ const struct _str2index method2index[TSG_METHOD_TYPE_MAX]={ {TSG_METHOD_TYPE_UNK
{TSG_METHOD_TYPE_DROP, 4, (char *)"drop"},
{TSG_METHOD_TYPE_REDIRECTION, 8, (char *)"redirect"},
{TSG_METHOD_TYPE_BLOCK, 5, (char *)"block"},
{TSG_METHOD_TYPE_RESET, 3, (char *)"rst"}
{TSG_METHOD_TYPE_RESET, 3, (char *)"rst"},
{TSG_METHOD_TYPE_ALERT, 5, (char *)"alert"}
};
const struct _str2index g_tsg_proto_string[PROTO_MAX+1]={{PROTO_UNKONWN, 0, (char *)""},
@@ -161,7 +162,7 @@ static Maat_feather_t init_maat_feather(const char* conffile, char* instance_nam
Maat_set_feather_opt(_maat_feather, MAAT_OPT_PERF_ON, NULL, 0);
Maat_set_feather_opt(_maat_feather, MAAT_OPT_REDIS_INDEX, &redis_index, sizeof(redis_index));
Maat_set_feather_opt(_maat_feather, MAAT_OPT_SCAN_DETAIL, &scan_detail, sizeof(scan_detail));
//Maat_set_feather_opt(g_tsg_maat_feather, MAAT_OPT_DEFERRED_LOAD, NULL,0);
Maat_set_feather_opt(_maat_feather, MAAT_OPT_FOREIGN_CONT_DIR, "./alerts_files", strlen("./alerts_files")+1);
}
else
{
@@ -215,11 +216,12 @@ int tsg_rule_init(const char* conffile, void *logger)
char cb_subscriber_ip_table[32]={0};
MESA_load_profile_string_def(conffile, "MAAT", "PROFILE", maat_conffile, sizeof(maat_conffile), "./tsgconf/maat_profile.conf");
MESA_load_profile_string_def(conffile, "MAAT", "IP_ADDR_TABLE", g_tsg_para.table_name[TABLE_IP_ADDR], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_ADDR");
MESA_load_profile_string_def(conffile, "MAAT", "IP_ADDR_TABLE", g_tsg_para.table_name[TABLE_IP_ADDR], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_ADDR");
MESA_load_profile_string_def(conffile, "MAAT", "SUBSCRIBER_ID_TABLE", g_tsg_para.table_name[TABLE_SUBSCRIBER_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_SUBSCRIBER_ID");
MESA_load_profile_string_def(conffile, "MAAT", "APP_ID_TABLE", g_tsg_para.table_name[TABLE_APP_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_APP_ID");
MESA_load_profile_string_def(conffile, "MAAT", "HTTP_HOST_TABLE", g_tsg_para.table_name[TABLE_HTTP_HOST], _MAX_TABLE_NAME_LEN, "TSG_FIELD_HTTP_HOST");
MESA_load_profile_string_def(conffile, "MAAT", "SSL_SNI_TABLE", g_tsg_para.table_name[TABLE_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_SSL_SNI");
MESA_load_profile_string_def(conffile, "MAAT", "DECYPTION_EXCLUSION_SSL_SNI", g_tsg_para.table_name[TABLE_EXCLUSION_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_DECYPTION_EXCLUSION_SSL_SNI");
//init dynamic maat feather
g_tsg_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_STATIC", (char *)"STATIC", logger);
@@ -234,7 +236,11 @@ int tsg_rule_init(const char* conffile, void *logger)
g_tsg_para.table_id[i]=Maat_table_register(g_tsg_maat_feather, g_tsg_para.table_name[i]);
if(g_tsg_para.table_id[i]<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "Maat_table_register %s failed, Please check tsgconf/tsg_static_tableinfo.conf", g_tsg_para.table_name[i]);
MESA_handle_runtime_log(logger,
RLOG_LV_FATAL,
"Maat_table_register %s failed, Please check tsgconf/tsg_static_tableinfo.conf",
g_tsg_para.table_name[i]
);
return -1;
}
}
@@ -252,7 +258,12 @@ int tsg_rule_init(const char* conffile, void *logger)
g_tsg_para.dyn_subscribe_ip_table_id=Maat_table_register(g_tsg_dynamic_maat_feather, cb_subscriber_ip_table);
if(g_tsg_para.dyn_subscribe_ip_table_id<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, "RULE_INIT", "Maat_table_register %s failed", cb_subscriber_ip_table);
MESA_handle_runtime_log(logger,
RLOG_LV_FATAL,
"RULE_INIT",
"Maat_table_register %s failed, Please check tsgconf/tsg_static_tableinfo.conf",
cb_subscriber_ip_table
);
return -1;
}
ret=Maat_plugin_EX_register(g_tsg_dynamic_maat_feather,

69
src/tsg_send_log.cpp
View File

@@ -19,7 +19,7 @@
#include "tsg_send_log.h"
#include "tsg_send_log_internal.h"
char TSG_SEND_LOG_VERSION_20200330=0;
char TSG_SEND_LOG_VERSION_20200427=0;
struct tsg_log_instance_t *g_tsg_log_instance;
@@ -129,6 +129,42 @@ char *log_field_id2name(struct tsg_log_instance_t *instance, tsg_log_field_id_t
return NULL;
}
static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name, struct Maat_rule_t *p_result)
{
cJSON *item=NULL;
cJSON *object=NULL;
char *tmp_buff=NULL;
if(p_result->serv_def_len<128)
{
object=cJSON_Parse(p_result->service_defined);
}
else
{
tmp_buff=(char *)calloc(1, p_result->serv_def_len+1);
Maat_read_rule(g_tsg_maat_feather, p_result, MAAT_RULE_SERV_DEFINE, tmp_buff, p_result->serv_def_len);
object=cJSON_Parse(tmp_buff);
free(tmp_buff);
tmp_buff=NULL;
}
if(object!=NULL)
{
item=cJSON_GetObjectItem(object, "method");
if(item!=NULL && item->valuestring!=NULL)
{
TLD_append(handle, field_name, (void *)item->valuestring, TLD_TYPE_STRING);
}
cJSON_Delete(object);
object=NULL;
}
return 0;
}
int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, struct streaminfo *a_stream)
{
int ret=0,addr_type=0;
@@ -366,7 +402,12 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
ret=MESA_get_dev_ipv4(nic_name, (int *)&local_ip_nr);
if(ret<0)
{
MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "GET_LOCAL_IP", "MESA_get_dev_ipv4 is error, nic_name: %s", nic_name);
MESA_handle_runtime_log(_instance->logger,
RLOG_LV_FATAL,
"GET_LOCAL_IP",
"MESA_get_dev_ipv4 is error, nic_name: %s, please check tsgconf/main.conf",
nic_name
);
return NULL;
}
inet_ntop(AF_INET,&(local_ip_nr),_instance->local_ip_str,sizeof(_instance->local_ip_str));
@@ -385,7 +426,12 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
if(rd_kafka_brokers_add(kafka_handle, _instance->broker_list) == 0)
{
MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "KAFKA_INIT", "rd_kafka_brokers_add is error, broker_list: %s", _instance->broker_list);
MESA_handle_runtime_log(_instance->logger,
RLOG_LV_FATAL,
"KAFKA_INIT",
"rd_kafka_brokers_add is error, broker_list: %s, please check tsgconf/main.conf",
_instance->broker_list
);
return NULL;
}
@@ -395,7 +441,7 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
{
_instance->topic_rkt=(rd_kafka_topic_t **)calloc(1, (_instance->max_service)*sizeof(rd_kafka_topic_t*));
for(i=0; i<_instance->max_service+1; i++)
for(i=0; i<_instance->max_service; i++)
{
if(_instance->service2topic[i].type==TLD_TYPE_MAX)
{
@@ -406,7 +452,12 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
}
else
{
MESA_handle_runtime_log(_instance->logger, RLOG_LV_FATAL, "KAFKA_INIT", "load_log_common_field is error, please check %s", _instance->common_field_file);
MESA_handle_runtime_log(_instance->logger,
RLOG_LV_FATAL,
"KAFKA_INIT",
"load_log_common_field is error, please check %s",
_instance->common_field_file
);
}
return _instance;
@@ -472,7 +523,12 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
TLD_append(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(long)(log_msg->result[i].config_id), TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG);
if(log_msg->result[i].action==TSG_ACTION_DENY)
{
set_common_sub_action(_handle, _instance->id2field[LOG_COMMON_SUB_ACTION].name, &(log_msg->result[i]));
}
payload = cJSON_PrintUnformatted(_handle->object);
status = rd_kafka_produce(_instance->topic_rkt[log_msg->result[i].service_id], RD_KAFKA_PARTITION_UA, RD_KAFKA_MSG_F_COPY, payload, strlen(payload), NULL, 0, NULL);
@@ -495,6 +551,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
TLD_delete(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_SERVICE].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_ACTION].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_SUB_ACTION].name);
TLD_delete(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name);
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_LOG], 0, FS_OP_ADD, 1);

1
src/tsg_send_log_internal.h
View File

@@ -60,6 +60,7 @@ typedef enum _tsg_log_field_id
LOG_HTTP_HOST,
LOG_SSL_SNI,
LOG_COMMON_ESTABLISH_LATENCY_MS,
LOG_COMMON_SUB_ACTION,
LOG_COMMON_MAX
}tsg_log_field_id_t;