gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

适配FQDN接口

Browse Source
This commit is contained in:
liuxueli
2020-09-28 17:13:39 +08:00
parent f1150118de
commit f3ba19def0
8 changed files with 520 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
src/tsg_entry.cpp
View File

@@ -454,7 +454,7 @@ static struct Maat_rule_t *tsg_policy_decision_criteria(struct streaminfo *a_str
p_result->config_id,
p_result->service_id,
(unsigned char)p_result->action,
printaddr(&a_stream->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, thread_seq) : "")
);
p_result=q_result;
@@ -468,7 +468,7 @@ static struct Maat_rule_t *tsg_policy_decision_criteria(struct streaminfo *a_str
"Not hit %s stream_dir: %d addr: %s scan ret: %d",
identify_info->domain,
a_stream->dir,
printaddr(&a_stream->addr, thread_seq),
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, thread_seq) : ""),
ret
);
}
@@ -651,7 +651,7 @@ static int identify_application_protocol(struct streaminfo *a_stream, struct _id
extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet)
{
int ret=0,hit_num=0;
int i=0,ret=0,hit_num=0;
struct timespec tv;
int state=APP_STATE_GIVEME;
scan_status_t mid=NULL;
@@ -689,7 +689,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
"Add internal_label failed, establish latency ms: %llu proto: %d addr: %s",
internal_label->establish_latency_ms,
internal_label->proto,
printaddr(&a_tcp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_tcp->addr, thread_seq) : "")
);
}
else
@@ -700,7 +700,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
"Add internal_label successfully, establish latency ms: %llu proto: %d addr: %s",
internal_label->establish_latency_ms,
internal_label->proto,
printaddr(&a_tcp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_tcp->addr, thread_seq) : "")
);
}
@@ -708,7 +708,6 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
if(ret>0)
{
hit_num+=ret;
//q_result=tsg_policy_decision_criteria(a_tcp, all_result, hit_num, NULL, thread_seq);
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_ADDR], 0, FS_OP_ADD, 1);
}
@@ -727,7 +726,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
all_result[hit_num].config_id,
all_result[hit_num].service_id,
(unsigned char)all_result[hit_num].action,
printaddr(&a_tcp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_tcp->addr, thread_seq) : "")
);
hit_num+=ret;
@@ -742,9 +741,63 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
(ret==-1) ? "NULL" : identify_info.domain,
ret,
a_tcp->dir,
printaddr(&a_tcp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_tcp->addr, thread_seq) : "")
);
}
if(strlen(identify_info.domain)>0 && identify_info.domain_len>0)
{
internal_label->fqdn_category_id_num=tsg_get_fqdn_category_id(g_tsg_maat_feather,
identify_info.domain,
internal_label->fqdn_category_id,
MAX_CATEGORY_ID_NUM,
g_tsg_para.logger,
thread_seq
);
}
for(i=0; i<internal_label->fqdn_category_id_num; i++)
{
int idx=identify_info.proto==PROTO_HTTP ? TABLE_HTTP_HOST : TABLE_SSL_SNI;
ret=Maat_scan_intval(g_tsg_maat_feather,
g_tsg_para.table_id[idx],
(unsigned int)internal_label->fqdn_category_id[i],
all_result+hit_num,
MAX_RESULT_NUM-hit_num,
&mid,
thread_seq
);
if(ret>0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_FQDN_CAT",
"Hit %s: %s category_id: %d policy_id: %d service: %d action: %d addr: %s",
(identify_info.proto==PROTO_HTTP) ? "host" : "sni",
identify_info.domain,
internal_label->fqdn_category_id[i],
all_result[hit_num].config_id,
all_result[hit_num].service_id,
(unsigned char)all_result[hit_num].action,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_tcp->addr, thread_seq) : "")
);
hit_num+=ret;
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_FQDN_CAT",
"Not hit %s: %s category_id: %d ret: %d stream_dir: %d addr: %s",
(ret==-1) ? "NULL" : ((identify_info.proto==PROTO_HTTP) ? "host" : "sni"),
(ret==-1) ? "NULL" : identify_info.domain,
internal_label->fqdn_category_id[i],
ret,
a_tcp->dir,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_tcp->addr, thread_seq) : "")
);
}
}
}
if(mid!=NULL)
@@ -780,7 +833,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
p_result[0].config_id,
p_result[0].service_id,
(unsigned char)p_result[0].action,
printaddr(&a_tcp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_tcp->addr, thread_seq) : "")
);
state=APP_STATE_GIVEME|APP_STATE_KILL_OTHER;
@@ -808,7 +861,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
"Add policy_priority_label failed, intercept policy, policy_id: %d action: %d addr: %s",
priority_label->result[0].config_id,
(unsigned char)priority_label->result[0].action,
printaddr(&a_tcp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_tcp->addr, thread_seq) : "")
);
}
@@ -818,7 +871,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
"Hit intercept policy, policy_id: %d action: %d addr: %s",
priority_label->result[0].config_id,
(unsigned char)priority_label->result[0].action,
printaddr(&a_tcp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_tcp->addr, thread_seq) : "")
);
return APP_STATE_DROPME|APP_STATE_KILL_OTHER;
break;
@@ -857,7 +910,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int thread_seq,void *a_packet)
{
int ret=0;
int i=0,ret=0;
int hit_num=0;
scan_status_t mid=NULL;
int state=APP_STATE_GIVEME;
@@ -886,7 +939,7 @@ extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int t
"PROJECT_ADD",
"Add internal_label failed, proto: %d addr: %s",
internal_label->proto,
printaddr(&a_udp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_udp->addr, thread_seq) : "")
);
}
else
@@ -896,7 +949,7 @@ extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int t
"PROJECT_ADD",
"Add internal_label successfully, proto: %d addr: %s",
internal_label->proto,
printaddr(&a_udp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_udp->addr, thread_seq) : "")
);
}
@@ -922,7 +975,7 @@ extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int t
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
printaddr(&a_udp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_udp->addr, thread_seq) : "")
);
hit_num+=ret;
@@ -936,9 +989,62 @@ extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int t
"QUIC SNI",
identify_info.domain,
a_udp->dir,
printaddr(&a_udp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_udp->addr, thread_seq) : "")
);
}
if(strlen(identify_info.domain)>0 && identify_info.domain_len>0)
{
internal_label->fqdn_category_id_num=tsg_get_fqdn_category_id(g_tsg_maat_feather,
identify_info.domain,
internal_label->fqdn_category_id,
MAX_CATEGORY_ID_NUM,
g_tsg_para.logger,
thread_seq
);
}
for(i=0; i<internal_label->fqdn_category_id_num; i++)
{
ret=Maat_scan_intval(g_tsg_maat_feather,
g_tsg_para.table_id[TABLE_QUIC_SNI],
(unsigned int)internal_label->fqdn_category_id[i],
result+hit_num,
MAX_RESULT_NUM-hit_num,
&mid,
thread_seq
);
if(ret>0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_FQDN_CAT",
"Hit %s: %s category_id: %d policy_id: %d service: %d action: %d addr: %s",
"QUIC SNI",
identify_info.domain,
internal_label->fqdn_category_id[i],
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_udp->addr, thread_seq) : "")
);
hit_num+=ret;
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_FQDN_CAT",
"Not hit %s: %s category_id: %d ret: %d stream_dir: %d addr: %s",
(ret==-1) ? "NULL" : ((identify_info.proto==PROTO_HTTP) ? "host" : "sni"),
(ret==-1) ? "NULL" : identify_info.domain,
internal_label->fqdn_category_id[i],
ret,
a_udp->dir,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_udp->addr, thread_seq) : "")
);
}
}
}
if(mid!=NULL)
@@ -972,7 +1078,7 @@ extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int t
p_result[0].config_id,
p_result[0].service_id,
(unsigned char)p_result[0].action,
printaddr(&a_udp->addr, thread_seq)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_udp->addr, thread_seq) : "")
);
state=APP_STATE_GIVEME|APP_STATE_KILL_OTHER;
@@ -1016,11 +1122,10 @@ extern "C" char TSG_MASTER_UDP_ENTRY(struct streaminfo *a_udp, void **pme, int t
extern "C" int TSG_MASTER_INIT()
{
int i=0,ret=0,cycle=0;
int value=0,level=30;
int i=0,ret=0;
int value=0,cycle=0;
unsigned short fs_server_port=0;
char app_name[MAX_STRING_LEN]={0};
char log_path[MAX_STRING_LEN*4]={0};
char label_buff[MAX_STRING_LEN*4]={0};
char fs_server_ip[MAX_IPV4_LEN]={0};
char fs_output_path[MAX_STRING_LEN*4]={0};
@@ -1029,10 +1134,10 @@ extern "C" int TSG_MASTER_INIT()
memset(&g_tsg_para, 0, sizeof(g_tsg_para));
MESA_load_profile_int_def(tsg_conffile, "SYSTEM","LOG_LEVEL", &level, 30);
MESA_load_profile_string_def(tsg_conffile, "SYSTEM","LOG_PATH", log_path, sizeof(log_path), "tsglog/tsg_master");
MESA_load_profile_int_def(tsg_conffile, "SYSTEM","LOG_LEVEL", &g_tsg_para.level, RLOG_LV_FATAL);
MESA_load_profile_string_def(tsg_conffile, "SYSTEM","LOG_PATH", g_tsg_para.log_path, sizeof(g_tsg_para.log_path), "tsglog/tsg_master");
g_tsg_para.logger=MESA_create_runtime_log_handle(log_path, level);
g_tsg_para.logger=MESA_create_runtime_log_handle(g_tsg_para.log_path, g_tsg_para.level);
if(g_tsg_para.logger==NULL)
{
printf("MESA_create_runtime_log_handle failed ...\n");

13
src/tsg_entry.h
View File

@@ -40,6 +40,9 @@ enum MASTER_TABLE{
TABLE_LOCATION_USER_DEFINED,
TABLE_LOCATION_BUILT_IN,
TABLE_QUIC_SNI,
TABLE_FQDN_CAT_ID,
TABLE_FQDN_CAT_USER_DEFINED,
TABLE_FQDN_CAT_BUILT_IN,
TABLE_MAX
};
@@ -64,6 +67,14 @@ struct _str2index
char *type;
};
struct _fqdn_category_t
{
int ref_cnt;
unsigned int category_id;
int match_method;
char fqdn[MAX_DOAMIN_LEN/8];
};
typedef struct _policy_priority_label
{
@@ -87,6 +98,7 @@ struct _master_context
#define _MAX_TABLE_NAME_LEN 64
typedef struct _tsg_para
{
int level;
unsigned short timeout;
int app_id_table_type;
int device_id;
@@ -97,6 +109,7 @@ typedef struct _tsg_para
int proto_flag; //tsg_protocol_t
int fs2_field_id[TSG_FS2_MAX];
char device_sn[MAX_DOAMIN_LEN/8];
char log_path[MAX_DOAMIN_LEN/8];
char data_center[_MAX_TABLE_NAME_LEN];
char table_name[TABLE_MAX][_MAX_TABLE_NAME_LEN];
void *logger;

495
src/tsg_rule.cpp
View File

@@ -169,21 +169,24 @@ void ASN_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *fr
{
void *logger=argp;
struct _asn_info_t *asn=(struct _asn_info_t *)(*from);
atomic_inc(&asn->ref_cnt);
*to=*from;
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_ASN",
"Dup(table_id: %d) ip ASN: start_ip: %s end_ip: %s asn: %s organization: %s ref_cnt: %d",
table_id,
asn->start_ip,
asn->end_ip,
asn->asn,
asn->organization,
asn->ref_cnt
);
if(asn!=NULL)
{
atomic_inc(&asn->ref_cnt);
*to=*from;
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_ASN",
"Dup(table_id: %d) ip ASN: start_ip: %s end_ip: %s asn: %s organization: %s ref_cnt: %d",
table_id,
asn->start_ip,
asn->end_ip,
asn->asn,
asn->organization,
asn->ref_cnt
);
}
return;
}
@@ -230,25 +233,29 @@ void ASN_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
void *logger=argp;
struct _asn_info_t *asn=(struct _asn_info_t *)(*ad);
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_ASN",
"Delate(table_id: %d) ip ASN: start_ip: %s end_ip: %s asn: %s organization: %s ref_cnt: %d",
table_id,
asn->start_ip,
asn->end_ip,
asn->asn,
asn->organization,
asn->ref_cnt
);
atomic_dec(&asn->ref_cnt);
if(asn->ref_cnt<=0)
if(asn!=NULL)
{
free(*ad);
*ad=NULL;
atomic_dec(&asn->ref_cnt);
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_ASN",
"Delate(table_id: %d) ip ASN: start_ip: %s end_ip: %s asn: %s organization: %s ref_cnt: %d",
table_id,
asn->start_ip,
asn->end_ip,
asn->asn,
asn->organization,
asn->ref_cnt
);
if(asn->ref_cnt<=0)
{
free(*ad);
*ad=NULL;
}
}
return;
}
@@ -257,23 +264,27 @@ void location_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DAT
void *logger=argp;
struct _location_info_t *location=(struct _location_info_t *)(*from);
atomic_inc(&location->ref_cnt);
*to=*from;
if(location!=NULL)
{
atomic_inc(&location->ref_cnt);
*to=*from;
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_LOCATION",
"Dup(table_id: %d) location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s province: %s city: %s ref_cnt: %d",
table_id,
location->geoname_id,
location->start_ip,
location->end_ip,
location->continent_full,
location->country_full,
location->province_full,
location->city_full,
location->ref_cnt
);
}
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_LOCATION",
"Dup(table_id: %d) location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s province: %s city: %s ref_cnt: %d",
table_id,
location->geoname_id,
location->start_ip,
location->end_ip,
location->continent_full,
location->country_full,
location->province_full,
location->city_full,
location->ref_cnt
);
return;
}
@@ -347,41 +358,140 @@ void location_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void*
void *logger=argp;
struct _location_info_t *location=(struct _location_info_t *)(*ad);
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_LOCATION",
"Delete(table_id: %d) location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s province: %s city: %s ref_cnt: %d",
table_id,
location->geoname_id,
location->start_ip,
location->end_ip,
location->continent_full,
location->country_full,
location->province_full,
location->city_full,
location->ref_cnt
);
atomic_dec(&location->ref_cnt);
if(location->ref_cnt<=0)
if(location!=NULL)
{
free(*ad);
*ad=NULL;
atomic_dec(&location->ref_cnt);
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_LOCATION",
"Delete(table_id: %d) location: geoname_id: %d start_ip: %s end_ip: %s continent: %s country: %s province: %s city: %s ref_cnt: %d",
table_id,
location->geoname_id,
location->start_ip,
location->end_ip,
location->continent_full,
location->country_full,
location->province_full,
location->city_full,
location->ref_cnt
);
if(location->ref_cnt<=0)
{
free(*ad);
*ad=NULL;
}
}
return;
}
void fqdn_cat_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp)
{
void *logger=argp;
struct _fqdn_category_t *fqdn_cat=(struct _fqdn_category_t *)(*from);
if(fqdn_cat!=NULL)
{
atomic_inc(&fqdn_cat->ref_cnt);
*to=*from;
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_LOCATION",
"Dup(table_id: %d) fqdn category: category_id: %d fqdn: %s match_method: %d ref_cnt: %d",
table_id,
fqdn_cat->category_id,
fqdn_cat->fqdn,
fqdn_cat->match_method,
fqdn_cat->ref_cnt
);
}
return;
}
void fqdn_cat_new_data(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
void *logger=argp;
int ret=0,id=0,is_valid=0;
struct _fqdn_category_t *fqdn_cat=NULL;
fqdn_cat=(struct _fqdn_category_t *)calloc(1, sizeof(struct _fqdn_category_t));
ret=sscanf(table_line,
"%d\t%u\t%s\t\t%d\t%d",
&id,
&fqdn_cat->category_id,
fqdn_cat->fqdn,
&fqdn_cat->match_method,
&is_valid
);
if(ret!=5)
{
free(fqdn_cat);
fqdn_cat=NULL;
MESA_handle_runtime_log(logger,
RLOG_LV_FATAL,
"IP_LOCATION",
"Parse fqdn category failed, ret: %d table_id: %d table_line: %s",
ret,
table_id,
table_line
);
return;
}
atomic_inc(&fqdn_cat->ref_cnt);
*ad=(MAAT_PLUGIN_EX_DATA)fqdn_cat;
return;
}
void fqdn_cat_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
void *logger=argp;
struct _fqdn_category_t *fqdn_cat=(struct _fqdn_category_t *)(*ad);
if(fqdn_cat!=NULL)
{
atomic_dec(&fqdn_cat->ref_cnt);
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_LOCATION",
"Delete(table_id: %d) fqdn category: category_id: %d fqdn: %s match_method: %d ref_cnt: %d",
table_id,
fqdn_cat->category_id,
fqdn_cat->fqdn,
fqdn_cat->match_method,
fqdn_cat->ref_cnt
);
if(fqdn_cat->ref_cnt<=0)
{
free(*ad);
*ad=NULL;
}
}
return;
}
void subscribe_id_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp)
{
void *logger=argp;
struct _subscribe_id_info_t *subscribe_id=(struct _subscribe_id_info_t *)(*from);
atomic_inc(&subscribe_id->ref_cnt);
*to=*from;
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Dup(table_id: %d ) subscribe_id: %s ref_cnt: %d", table_id, subscribe_id->subscribe_id, subscribe_id->ref_cnt);
if(subscribe_id!=NULL)
{
atomic_inc(&subscribe_id->ref_cnt);
*to=*from;
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Dup(table_id: %d ) subscribe_id: %s ref_cnt: %d", table_id, subscribe_id->subscribe_id, subscribe_id->ref_cnt);
}
return;
}
@@ -426,13 +536,17 @@ void subscribe_id_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, vo
void *logger=argp;
struct _subscribe_id_info_t *subscribe_id=(struct _subscribe_id_info_t *)(*ad);
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Delete(table_id: %d ) subscribe_id: %s ref_cnt: %d", table_id, subscribe_id->subscribe_id, subscribe_id->ref_cnt);
atomic_dec(&subscribe_id->ref_cnt);
if(subscribe_id->ref_cnt<=0)
if(subscribe_id!=NULL)
{
free(*ad);
*ad=NULL;
atomic_dec(&subscribe_id->ref_cnt);
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Delete(table_id: %d ) subscribe_id: %s ref_cnt: %d", table_id, subscribe_id->subscribe_id, subscribe_id->ref_cnt);
if(subscribe_id->ref_cnt<=0)
{
free(*ad);
*ad=NULL;
}
}
return;
@@ -582,6 +696,10 @@ int tsg_rule_init(const char* conffile, void *logger)
MESA_load_profile_string_def(conffile, "MAAT", "QUIC_SNI_TABLE", g_tsg_para.table_name[TABLE_QUIC_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_QUIC_SNI");
MESA_load_profile_string_def(conffile, "MAAT", "FQDN_CAT_ID_TABLE", g_tsg_para.table_name[TABLE_FQDN_CAT_ID], _MAX_TABLE_NAME_LEN, "TSG_OBJ_FQDN_CAT");
MESA_load_profile_string_def(conffile, "MAAT", "FQDN_CAT_BUILT_IN_TABLE", g_tsg_para.table_name[TABLE_FQDN_CAT_BUILT_IN], _MAX_TABLE_NAME_LEN, "TSG_FQDN_CATEGORY_BUILT_IN");
MESA_load_profile_string_def(conffile, "MAAT", "FQDN_CAT_USER_DEFINED_TABLE", g_tsg_para.table_name[TABLE_FQDN_CAT_USER_DEFINED], _MAX_TABLE_NAME_LEN, "TSG_FQDN_CATEGORY_USER_DEFINED");
//init static maat feather
g_tsg_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_STATIC", (char *)"STATIC", logger);
if(g_tsg_maat_feather==NULL)
@@ -647,6 +765,28 @@ int tsg_rule_init(const char* conffile, void *logger)
}
}
for(i=TABLE_FQDN_CAT_USER_DEFINED; i<=TABLE_FQDN_CAT_BUILT_IN; i++)
{
ret=Maat_fqdn_plugin_EX_register(g_tsg_maat_feather,
g_tsg_para.table_id[i],
fqdn_cat_new_data,
fqdn_cat_free_data,
fqdn_cat_dup_data,
0,
logger
);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,
"RULE_INIT",
"Maat_ip_plugin_EX_register failed, table_name: %s table_id: %d",
g_tsg_para.table_name[i],
g_tsg_para.table_id[i]
);
return -1;
}
}
//init dynamic maat feather
g_tsg_dynamic_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_DYNAMIC", (char *)"DYNAMIC", logger);
if(g_tsg_maat_feather==NULL)
@@ -707,23 +847,29 @@ int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_re
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "PULL_RESULT",
"pull policy failed, hit: %s %s: %s policy_id: %d service: %d action: %d addr: %s",
(label->result_type==PULL_KNI_RESULT) ? "KNI" : "FW",
label->proto==PROTO_HTTP ? "host" : "sni",
label->domain,
label->result->config_id,
label->result->service_id,
label->result->action,
printaddr(&a_stream->addr, a_stream->threadnum));
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"PULL_RESULT",
"pull policy failed, hit: %s %s: %s policy_id: %d service: %d action: %d addr: %s",
(label->result_type==PULL_KNI_RESULT) ? "KNI" : "FW",
label->proto==PROTO_HTTP ? "host" : "sni",
label->domain,
label->result->config_id,
label->result->service_id,
label->result->action,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "PULL_RESULT",
"pull policy failed, Not hit, label is %s addr: %s",
(label==NULL) ? "NULL" : label->domain,
printaddr(&a_stream->addr, a_stream->threadnum));
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"PULL_RESULT",
"pull policy failed, Not hit, label is %s addr: %s",
(label==NULL) ? "NULL" : label->domain,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
return 0;
@@ -860,28 +1006,28 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
if(ret > 0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"Hit IP_ASN: %s scan ret: %d table_name: %s policy_id: %d service: %d action: %d addr: %s",
asn->asn,
ret,
g_tsg_para.table_name[idx],
result[0].config_id,
result[0].service_id,
(unsigned char)result[0].action,
printaddr(&a_stream->addr, a_stream->threadnum)
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"Hit IP_ASN: %s scan ret: %d table_name: %s policy_id: %d service: %d action: %d addr: %s",
asn->asn,
ret,
g_tsg_para.table_name[idx],
result[0].config_id,
result[0].service_id,
(unsigned char)result[0].action,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"No hit IP_ASN: %s scan ret: %d table_name: %s addr: %s",
asn->asn,
ret,
g_tsg_para.table_name[idx],
printaddr(&a_stream->addr, a_stream->threadnum)
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"No hit IP_ASN: %s scan ret: %d table_name: %s addr: %s",
asn->asn,
ret,
g_tsg_para.table_name[idx],
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
}
@@ -893,8 +1039,8 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
"IP_ASN is NULL scan ret: %d table_name: %s addr: %s",
ret,
g_tsg_para.table_name[idx],
printaddr(&a_stream->addr, a_stream->threadnum)
);
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
return (ret>0 ? ret : 0);
@@ -922,28 +1068,28 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
if(ret > 0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"Hit IP_LOCATION: %s scan ret: %d table_name: %s policy_id: %d service: %d action: %d addr: %s",
buff,
ret,
g_tsg_para.table_name[idx],
result[0].config_id,
result[0].service_id,
(unsigned char)result[0].action,
printaddr(&a_stream->addr, a_stream->threadnum)
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"Hit IP_LOCATION: %s scan ret: %d table_name: %s policy_id: %d service: %d action: %d addr: %s",
buff,
ret,
g_tsg_para.table_name[idx],
result[0].config_id,
result[0].service_id,
(unsigned char)result[0].action,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"No hit IP_LOCATION: %s scan ret: %d table_name: %s addr: %s",
buff,
ret,
g_tsg_para.table_name[idx],
printaddr(&a_stream->addr, a_stream->threadnum)
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"No hit IP_LOCATION: %s scan ret: %d table_name: %s addr: %s",
buff,
ret,
g_tsg_para.table_name[idx],
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
}
@@ -955,8 +1101,8 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
"IP_LOCATION is NULL scan ret: %d table_name: %s addr: %s",
ret,
g_tsg_para.table_name[idx],
printaddr(&a_stream->addr, a_stream->threadnum)
);
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
return (ret>0 ? ret : 0);
@@ -975,10 +1121,10 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
if(result==NULL || result_num<=0 || a_stream==NULL || maat_feather==NULL)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
RLOG_LV_DEBUG,
"SCAN_NESTING_ADDR",
"result==NULL || result_num<=0 || maat_feather==NULL || a_stream==%s",
(a_stream!=NULL) ? printaddr(&a_stream->addr, a_stream->threadnum) : "NULL");
"result==NULL || result_num<=0 || maat_feather==NULL || a_stream==NULL"
);
return -1;
}
@@ -1036,7 +1182,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
RLOG_LV_DEBUG,
"SCAN_IP",
"Hit addr: %s scan ret: %d policy_id: %d service: %d action: %d",
printaddr(&cur_stream->addr, cur_stream->threadnum),
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&cur_stream->addr, cur_stream->threadnum) : ""),
maat_ret,
result[hit_num].config_id,
result[hit_num].service_id,
@@ -1051,7 +1197,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
RLOG_LV_DEBUG,
"SCAN_IP",
"No hit addr: %s scan ret: %d",
printaddr(&cur_stream->addr, cur_stream->threadnum),
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&cur_stream->addr, cur_stream->threadnum) : ""),
maat_ret
);
}
@@ -1095,15 +1241,15 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
if(maat_ret > 0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_PROTO",
"Hit PROTO: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
g_tsg_proto_string[proto].type,
maat_ret,
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
printaddr(&a_stream->addr, a_stream->threadnum)
RLOG_LV_DEBUG,
"SCAN_PROTO",
"Hit PROTO: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
g_tsg_proto_string[proto].type,
maat_ret,
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
hit_num+=maat_ret;
@@ -1111,13 +1257,13 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_PROTO",
"No hit PROTO: %s(%d) scan ret: %d addr: %s",
g_tsg_proto_string[proto].type,
proto_str2id(proto),
maat_ret,
printaddr(&a_stream->addr, a_stream->threadnum)
RLOG_LV_DEBUG,
"SCAN_PROTO",
"No hit PROTO: %s(%d) scan ret: %d addr: %s",
g_tsg_proto_string[proto].type,
proto_str2id(proto),
maat_ret,
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
}
@@ -1175,7 +1321,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
printaddr(&a_stream->addr, a_stream->threadnum)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
hit_num+=maat_ret;
@@ -1188,7 +1334,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
"No hit source subscribe id: %s scan ret: %d addr: %s",
internal_label->client_subscribe_id->subscribe_id,
maat_ret,
printaddr(&a_stream->addr, a_stream->threadnum)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
}
@@ -1216,7 +1362,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
result[hit_num].config_id,
result[hit_num].service_id,
(unsigned char)result[hit_num].action,
printaddr(&a_stream->addr, a_stream->threadnum)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
hit_num+=maat_ret;
@@ -1229,7 +1375,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
"No hit dest subscribe id: %s scan ret: %d addr: %s",
internal_label->server_subscribe_id->subscribe_id,
maat_ret,
printaddr(&a_stream->addr, a_stream->threadnum)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
}
@@ -1243,7 +1389,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
"ADD_INTERNAL_LABEL",
"Add internal label failed, ret: %d addr: %s",
ret,
printaddr(&a_stream->addr, a_stream->threadnum)
(g_tsg_para.level<RLOG_LV_FATAL ? printaddr(&a_stream->addr, a_stream->threadnum) : "")
);
}
@@ -1337,3 +1483,46 @@ int tsg_get_method_id(char *method)
return -1;
}
int tsg_get_fqdn_category_id(Maat_feather_t maat_feather, char *fqdn, int *category_id, int category_id_num, void *logger, int thread_seq)
{
int i=0,ret=0;
struct _fqdn_category_t *ex_data_array[8]={0};
if(category_id!=NULL && category_id_num>0)
{
ret=Maat_fqdn_plugin_get_EX_data(maat_feather, g_tsg_para.table_id[TABLE_FQDN_CAT_USER_DEFINED], fqdn, (MAAT_PLUGIN_EX_DATA *)ex_data_array, 8);
if(ret>0)
{
for(i=0; i<ret; i++)
{
if(i<category_id_num)
{
category_id[i]=ex_data_array[i]->category_id;
}
fqdn_cat_free_data(g_tsg_para.table_id[TABLE_FQDN_CAT_USER_DEFINED], (MAAT_PLUGIN_EX_DATA *)&(ex_data_array[i]), 0, logger);
}
return i<category_id_num ? i : category_id_num;
}
ret=Maat_fqdn_plugin_get_EX_data(maat_feather, g_tsg_para.table_id[TABLE_FQDN_CAT_BUILT_IN], fqdn, (MAAT_PLUGIN_EX_DATA *)ex_data_array, 8);
if(ret>0)
{
for(i=0; i<ret && i<category_id_num; i++)
{
if(i<category_id_num)
{
category_id[i]=ex_data_array[i]->category_id;
}
fqdn_cat_free_data(g_tsg_para.table_id[TABLE_FQDN_CAT_BUILT_IN], (MAAT_PLUGIN_EX_DATA *)&(ex_data_array[i]), 0, logger);
}
return i<category_id_num ? i : category_id_num;
}
}
return 0;
}

22
src/tsg_send_log.cpp
View File

@@ -255,10 +255,9 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle
if(_instance==NULL || _handle==NULL || a_stream==NULL)
{
MESA_handle_runtime_log(_instance->logger,
((a_stream==NULL) ? RLOG_LV_DEBUG: RLOG_LV_FATAL),
RLOG_LV_DEBUG,
"TLD_APPEND_STREAM",
"instance==NULL || TLD_handle==NULL || addr: %s",
((a_stream==NULL) ? "NULL": (printaddr(&a_stream->addr, a_stream->threadnum)))
"instance==NULL || TLD_handle==NULL || addr==NULL"
);
return -1;
}
@@ -456,11 +455,10 @@ int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t
struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
{
int i=0,ret=0,level=30;
int i=0,ret=0;
char nic_name[32]={0};
char kafka_errstr[1024]={0};
unsigned int local_ip_nr=0;
char log_path[MAX_STRING_LEN*4]={0};
rd_kafka_t *kafka_handle = NULL;
rd_kafka_conf_t *rdkafka_conf = NULL;
rd_kafka_topic_conf_t *topic_conf;
@@ -478,13 +476,13 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
_instance->send_log_percent[i]=100;
}
MESA_load_profile_int_def(conffile, "TSG_LOG", "LOG_LEVEL",&(level), 30);
MESA_load_profile_string_def(conffile, "TSG_LOG", "LOG_PATH", log_path, sizeof(log_path), "./tsglog/tsglog");
MESA_load_profile_int_def(conffile, "TSG_LOG", "LOG_LEVEL",&(_instance->level), 30);
MESA_load_profile_string_def(conffile, "TSG_LOG", "LOG_PATH", _instance->log_path, sizeof(_instance->log_path), "./tsglog/tsglog");
_instance->logger=MESA_create_runtime_log_handle(log_path, level);
_instance->logger=MESA_create_runtime_log_handle(_instance->log_path, _instance->level);
if(_instance->logger==NULL)
{
printf("MESA_create_runtime_log_handle failed ..., path: %s level: %d", log_path, level);
printf("MESA_create_runtime_log_handle failed ..., path: %s level: %d", _instance->log_path, _instance->level);
return NULL;
}
@@ -633,7 +631,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
"tsg same log:cfg_id=%d service=%d addr=%s",
log_msg->result[i].config_id,
log_msg->result[i].service_id,
(log_msg->a_stream==NULL ? "" : printaddr(&(log_msg->a_stream->addr), thread_id))
(_instance->level<RLOG_LV_FATAL ? (log_msg->a_stream==NULL ? "" : printaddr(&(log_msg->a_stream->addr), thread_id)) : "")
);
continue;
}
@@ -648,7 +646,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
log_msg->result[i].config_id,
log_msg->result[i].service_id,
_instance->send_log_percent[thread_id],
(log_msg->a_stream==NULL ? "" : printaddr(&(log_msg->a_stream->addr), thread_id))
(_instance->level<RLOG_LV_FATAL ? (log_msg->a_stream==NULL ? "" : printaddr(&(log_msg->a_stream->addr), thread_id)) : "")
);
continue;
}
@@ -661,7 +659,7 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
"tsg abort log:cfg_id=%d service=%d addr=%s",
log_msg->result[i].config_id,
log_msg->result[i].service_id,
(log_msg->a_stream==NULL ? "" : printaddr(&(log_msg->a_stream->addr), thread_id))
(_instance->level<RLOG_LV_FATAL ? (log_msg->a_stream==NULL ? "" : printaddr(&(log_msg->a_stream->addr), thread_id)) : "")
);
continue;

2
src/tsg_send_log_internal.h
View File

@@ -89,6 +89,7 @@ struct TLD_handle_t
struct tsg_log_instance_t
{
int mode;
int level;
int max_service;
int recovery_interval;
int internal_project_id;
@@ -97,6 +98,7 @@ struct tsg_log_instance_t
int *send_log_percent;
int *fs_status_ids;
struct timespec *drop_start;
char log_path[MAX_STRING_LEN*2];
char tcp_label[MAX_STRING_LEN];
char udp_label[MAX_STRING_LEN];
char common_field_file[MAX_STRING_LEN*4];