gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持IP归属地定位功能

Browse Source
This commit is contained in:
liuxueli
2020-05-14 15:52:54 +08:00
parent 7c22c8d28a
commit 278c54ab1f
7 changed files with 727 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
bin/tsg_log_field.conf
View File

@@ -42,3 +42,7 @@ STRING http_host 35
STRING ssl_sni 36
LONG common_establish_latency_ms 37
STRING common_sub_action 38
STRING common_clinet_asn 39
STRING common_server_asn 40
STRING common_clinet_location 41
STRING common_server_location 42

6
bin/tsg_static_tableinfo.conf
View File

@@ -45,3 +45,9 @@
33 TSG_SECURITY_ADDR composition {"source":"TSG_SECURITY_SOURCE_ADDR","destination":"TSG_SECURITY_DESTINATION_ADDR"}
34 TSG_DECYPTION_EXCLUSION_SSL_SNI virtual TSG_OBJ_FQDN
35 TSG_PROFILE_RESPONSE_PAGES plugin {"key":1,"foreign":"4","valid":5} --
36 TSG_IP_ASN_BUILT_IN ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7}
37 TSG_IP_ASN_USER_DEFINED ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7}
38 TSG_IP_LOCATION_BUILT_IN ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":17}
39 TSG_IP_LOCATION_USER_DEFINED ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":17}
40 TSG_OBJ_IP_ASN expr UTF8 UTF8/GBK yes 0
41 TSG_OBJ_IP_LOCATION expr UTF8 UTF8/GBK yes 0

76
src/tsg_entry.cpp
View File

@@ -41,7 +41,7 @@ static __attribute__((__used__)) const char * GIT_VERSION_UNKNOWN = NULL;
#endif
char TSG_MASTER_VERSION_20200507=0;
char TSG_MASTER_VERSION_20200514=0;
const char *tsg_conffile="tsgconf/main.conf";
g_tsg_para_t g_tsg_para;
@@ -60,6 +60,50 @@ static void free_policy_label(int thread_seq, void *project_req_value)
project_req_value=NULL;
}
static void free_internal_label(int thread_seq, void *project_req_value)
{
struct _internal_label *label=(struct _internal_label *)project_req_value;
if(label->client_asn!=NULL)
{
ASN_free_data(label->client_asn->table_id, (MAAT_PLUGIN_EX_DATA *)&(label->client_asn), 0, g_tsg_para.logger);
label->client_asn=NULL;
}
if(label->server_asn!=NULL)
{
ASN_free_data(label->server_asn->table_id, (MAAT_PLUGIN_EX_DATA *)&(label->server_asn), 0, g_tsg_para.logger);
label->server_asn=NULL;
}
if(label->client_location!=NULL)
{
location_free_data(label->client_location->table_id, (MAAT_PLUGIN_EX_DATA *)&(label->client_location), 0, g_tsg_para.logger);
label->client_location=NULL;
}
if(label->server_location!=NULL)
{
location_free_data(label->server_location->table_id, (MAAT_PLUGIN_EX_DATA *)&(label->server_location), 0, g_tsg_para.logger);
label->server_location=NULL;
}
if(label->client_subscribe_id!=NULL)
{
free(label->client_subscribe_id);
label->client_subscribe_id=NULL;
}
if(label->server_subscribe_id!=NULL)
{
free(label->server_subscribe_id);
label->server_subscribe_id=NULL;
}
dictator_free(thread_seq, project_req_value);
project_req_value=NULL;
}
static void free_context(void **pme, int thread_seq)
{
struct _master_context *_context=(struct _master_context *)*pme;
@@ -92,6 +136,7 @@ static int init_context(void **pme, tsg_protocol_t proto, struct Maat_rule_t *p_
return 0;
}
#if 0
static int master_method_type(struct streaminfo *a_stream, struct Maat_rule_t *p_result)
{
cJSON *item=NULL;
@@ -172,6 +217,7 @@ static int master_method_type(struct streaminfo *a_stream, struct Maat_rule_t *p
return method_type;
}
static int master_do_deny(struct streaminfo *a_stream, struct Maat_rule_t *p_result, int thread_seq)
{
int opt_value=0;
@@ -218,6 +264,7 @@ static int master_do_deny(struct streaminfo *a_stream, struct Maat_rule_t *p_res
return 0;
}
#endif
static char *schema_index2string(tsg_protocol_t proto)
{
@@ -345,13 +392,16 @@ static struct Maat_rule_t *tsg_policy_decision_criteria(struct streaminfo *a_str
for(i=1; i<result_num; i++)
{
if((unsigned char)result[i].action>(unsigned char)p_result->action)
{
if(p_result->action!=TSG_ACTION_INTERCEPT)
{
q_result=p_result;
}
p_result=&result[i];
continue;
}
if(q_result==NULL)
if(q_result==NULL && result[i].action!=TSG_ACTION_INTERCEPT)
{
q_result=&result[i];
}
@@ -517,7 +567,7 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
Maat_rule_t all_result[MAX_RESULT_NUM];
policy_priority_label_t *priority_label=NULL;
struct _master_context *_context=(struct _master_context *)*pme;
struct _internal_label *internal_label=NULL;
switch(a_tcp->opstate)
{
case OP_STATE_PENDING:
@@ -527,9 +577,13 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
identify_application_protocol(a_tcp, &identify_info);
if(identify_info.proto==PROTO_HTTP)
{
internal_label=(struct _internal_label *)dictator_malloc(1, sizeof(struct _internal_label));
memset(internal_label, 0, sizeof(struct _internal_label));
clock_gettime(CLOCK_REALTIME, &tv);
establish_latency_ms=(tv.tv_sec-a_tcp->ptcpdetail->createtime)*1000+tv.tv_nsec/1000/1000;
ret=project_req_add_long(a_tcp, g_tsg_para.establish_latency_project_id, establish_latency_ms);
internal_label->establish_latency_ms=(tv.tv_sec-a_tcp->ptcpdetail->createtime)*1000+tv.tv_nsec/1000/1000;
ret=project_req_add_struct(a_tcp, g_tsg_para.internal_project_id, (const void *)internal_label);
if(ret<0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
@@ -561,7 +615,6 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
}
ret=tsg_scan_shared_policy(g_tsg_maat_feather, &identify_info, all_result+hit_num, MAX_RESULT_NUM-hit_num, &mid, thread_seq);
if(ret>0)
{
FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_HIT_SHARE], 0, FS_OP_ADD, 1);
@@ -585,7 +638,6 @@ extern "C" char TSG_MASTER_TCP_ENTRY(struct streaminfo *a_tcp, void **pme, int t
RLOG_LV_DEBUG,
"SCAN_FQDN",
"Not hit %s: %s stream_dir: %d addr: %s",
(ret==-1) ? "NULL" : ((identify_info.proto==PROTO_HTTP) ? "host" : "sni"),
(ret==-1) ? "NULL" : identify_info.domain,
a_tcp->dir,
@@ -854,14 +906,14 @@ extern "C" int TSG_MASTER_INIT()
return -1;
}
MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "ESTABLISH_LATENCY_LABEL", label_buff, sizeof(label_buff), "ESTABLISH_LATENCY");
g_tsg_para.establish_latency_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_LONG, NULL);
if(g_tsg_para.establish_latency_project_id<0)
MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "TSG_MASTER_INTERNAL_LABEL", label_buff, sizeof(label_buff), "TSG_MASTER_INTERNAL_LABEL");
g_tsg_para.internal_project_id=project_producer_register(label_buff, PROJECT_VAL_TYPE_STRUCT, free_internal_label);
if(g_tsg_para.internal_project_id<0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
"PROJECT_REGISTER",
"Register %s failed; please check :%s and add <ESTABLISH_LATENCY long>",
"Register %s failed; please check :%s and add <TSG_MASTER_INTERNAL_LABEL struct>",
label_buff,
"etc/project_list.conf"
);
@@ -880,7 +932,7 @@ extern "C" int TSG_MASTER_INIT()
MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_FATAL, "INIT_SENDLOG", "tsg_sendlog_init failed ...");
return -1;
}
g_tsg_log_instance->establish_latency_project_id=g_tsg_para.establish_latency_project_id;
g_tsg_log_instance->internal_project_id=g_tsg_para.internal_project_id;
MESA_load_profile_int_def(tsg_conffile, "FIELD_STAT", "CYCLE", &cycle, 30);
MESA_load_profile_short_nodef(tsg_conffile, "FIELD_STAT","TELEGRAF_PORT", (short *)&(fs_server_port));

75
src/tsg_entry.h
View File

@@ -5,6 +5,18 @@
#include <MESA/field_stat2.h>
#include "tsg_rule.h"
#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411)
#define atomic_inc(x) __sync_add_and_fetch((x),1)
#define atomic_dec(x) __sync_sub_and_fetch((x),1)
#define atomic_add(x,y) __sync_add_and_fetch((x),(y))
#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y))
typedef int atomic_t;
#define ATOMIC_INIT(i) { (i) }
#define atomic_read(x) __sync_add_and_fetch((x),0)
#define atomic_set(x,y) __sync_lock_test_and_set((x),y)
#else
#include <alsa/iatomic.h>
#endif
#ifndef MIN
#define MIN(a, b) (((a) < (b)) ? (a) : (b))
@@ -17,6 +29,12 @@ enum MASTER_TABLE{
TABLE_HTTP_HOST,
TABLE_SSL_SNI,
TABLE_EXCLUSION_SSL_SNI,
TABLE_IP_ASN,
TABLE_IP_LOCATION,
TABLE_ASN_USER_DEFINED,
TABLE_ASN_BUILT_IN,
TABLE_LOCATION_USER_DEFINED,
TABLE_LOCATION_BUILT_IN,
TABLE_MAX
};
@@ -39,6 +57,57 @@ struct _str2index
char *type;
};
struct _asn_info
{
int ref_cnt;
int addr_type;
char start_ip[40];
char end_ip[40];
char asn[40];
char organization[256];
int table_id;
};
struct _location_info
{
int ref_cnt;
int addr_type;
char start_ip[40];
char end_ip[40];
double latitude;
double longitude;
double coords;
char language[40];
char continent_abbr[256];
char continent_full[256];
char country_abbr[256];
char country_full[256];
char province_abbr[256];
char province_full[256];
char city_full[256];
char time_zone[128];
int parent_location_id;
int table_id;
};
struct _subscribe_id_info
{
int ref_cnt;
char subscribe_id[256];
};
struct _internal_label
{
long establish_latency_ms;
struct _asn_info *client_asn;
struct _asn_info *server_asn;
struct _location_info *client_location;
struct _location_info *server_location;
struct _subscribe_id_info *client_subscribe_id;
struct _subscribe_id_info *server_subscribe_id;
};
typedef struct _policy_priority_label
{
@@ -67,7 +136,7 @@ typedef struct _tsg_para
int table_id[TABLE_MAX];
int dyn_subscribe_ip_table_id; //TSG_DYN_SUBSCRIBER_IP
int priority_project_id;
int establish_latency_project_id;
int internal_project_id;
int fs2_field_id[TSG_FS2_MAX];
char table_name[TABLE_MAX][_MAX_TABLE_NAME_LEN];
void *logger;
@@ -150,4 +219,8 @@ int tsg_statistic_init(const char *conffile, void *logger);
int tsg_scan_shared_policy(Maat_feather_t maat_feather, struct _identify_info *identify_info, Maat_rule_t *result, int result_num, scan_status_t *mid, int thread_seq);
void location_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp);
void ASN_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp);
#endif

565
src/tsg_rule.cpp
View File

@@ -54,14 +54,221 @@ const struct _str2index g_tsg_proto_string[PROTO_MAX+1]={{PROTO_UNKONWN, 0, (cha
{PROTO_MAX, 0, (char *)""}
};
void ASN_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp)
{
void *logger=argp;
struct _asn_info *asn=(struct _asn_info *)(*from);
atomic_inc(&asn->ref_cnt);
*to=*from;
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_ASN",
"Add ip ASN: [ start_ip: %s end_ip: %s asn: %d organization: %s ref_cnt: %d ] table_id: %d",
asn->start_ip,
asn->end_ip,
asn->asn,
asn->organization,
asn->ref_cnt,
table_id
);
return;
}
void ASN_new_data(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
void *logger=argp;
int ret=0,id=0,is_valid=0;
struct _asn_info *asn=NULL;
asn=(struct _asn_info *)calloc(1, sizeof(struct _asn_info));
ret=sscanf(table_line, "%d\t%d\t%s\t%s\t%s\t%s\t%d", &id, &asn->addr_type, asn->start_ip, asn->end_ip, asn->asn, asn->organization, &is_valid);
if(ret!=7)
{
free(asn);
asn=NULL;
MESA_handle_runtime_log(logger,
RLOG_LV_FATAL,
"IP_ASN",
"Parse ip ASN failed, ret: %d table_id: %d table_line: %s",
ret,
table_id,
table_line
);
return;
}
atomic_inc(&asn->ref_cnt);
*ad=(MAAT_PLUGIN_EX_DATA)asn;
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_ASN",
"Add ip ASN: [ start_ip: %s end_ip: %s asn: %d organization: %s ] table_id: %d table_line: %s",
asn->start_ip,
asn->end_ip,
asn->asn,
asn->organization,
table_id,
table_line
);
return;
}
void ASN_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
void *logger=argp;
struct _asn_info *asn=(struct _asn_info *)(*ad);
atomic_dec(&asn->ref_cnt);
if(asn->ref_cnt<=0)
{
free(*ad);
*ad=NULL;
}
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_ASN",
"Add ip ASN: [ start_ip: %s end_ip: %s asn: %d organization: %s ref_cnt: %d ] table_id: %d",
asn->start_ip,
asn->end_ip,
asn->asn,
asn->organization,
asn->ref_cnt,
table_id
);
return;
}
void location_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp)
{
void *logger=argp;
struct _location_info *location=(struct _location_info *)(*from);
atomic_inc(&location->ref_cnt);
*to=*from;
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_LOCATION",
"Dup location: [ start_ip: %s end_ip: %s continent: %s country: %s city: %s ref_cnt: %d ] table_id: %d",
location->start_ip,
location->end_ip,
location->continent_full,
location->country_full,
location->city_full,
table_id);
return;
}
void location_new_data(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
void *logger=argp;
int ret=0,id=0,is_valid=0;
struct _location_info *location=NULL;
location=(struct _location_info *)calloc(1, sizeof(struct _location_info));
ret=sscanf(table_line,
"%d\t%d\t%s\t%s\t%lf\t%lf\t%lf\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%u\t%d",
&id,
&location->addr_type,
location->start_ip,
location->end_ip,
&location->latitude,
&location->longitude,
&location->coords,
location->language,
location->continent_abbr,
location->continent_full,
location->country_abbr,
location->country_full,
location->province_abbr,
location->province_full,
location->city_full,
location->time_zone,
&location->parent_location_id,
&is_valid);
if(ret!=18)
{
free(location);
location=NULL;
MESA_handle_runtime_log(logger,
RLOG_LV_FATAL,
"IP_LOCATION",
"Parse ip location failed, ret: %d table_id: %d table_line: %s",
ret,
table_id,
table_line
);
return;
}
atomic_inc(&location->ref_cnt);
location->table_id=table_id;
*ad=(MAAT_PLUGIN_EX_DATA)location;
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_LOCATION",
"Add ip location: [ start_ip: %s end_ip: %s continent: %s country: %s city: %s ref_cnt: %d ] table_id: %d table_line: %s",
location->start_ip,
location->end_ip,
location->continent_full,
location->country_full,
location->city_full,
location->ref_cnt,
table_id,
table_line
);
return;
}
void location_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
void *logger=argp;
struct _location_info *location=(struct _location_info *)(*ad);
atomic_dec(&location->ref_cnt);
if(location->ref_cnt<=0)
{
free(*ad);
*ad=NULL;
}
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"IP_LOCATION",
"Delete location: [ start_ip: %s end_ip: %s continent: %s country: %s city: %s ref_cnt: %d ] table_id: %d",
location->start_ip,
location->end_ip,
location->continent_full,
location->country_full,
location->city_full,
location->ref_cnt,
table_id
);
return;
}
void subscribe_id_dup_data(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void* argp)
{
void *logger=argp;
struct _subscribe_id_info *subscribe_id=(struct _subscribe_id_info *)(*from);
*to=calloc(1, strlen((char *)*from)+1);
memcpy(*to, *from, strlen((char *)*from));
atomic_inc(&subscribe_id->ref_cnt);
*to=*from;
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Dup subscribe_id: %s ref_cnt: %d table_id: %d", subscribe_id->subscribe_id, subscribe_id->ref_cnt, table_id);
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Dup subscribe_id: %s table_id: %d", (char *)*to, table_id);
return;
}
@@ -69,34 +276,39 @@ void subscribe_id_new_data(int table_id, const char* key, const char* table_line
{
void *logger=argp;
int ret=0,id=0,type=0,is_valid=0;
char subscribe_id[256]={0};
char ip_addr[MAX_IPV6_ADDR_LEN]={0};
struct _subscribe_id_info *subscribe_id=NULL;
ret=sscanf(table_line, "%d\t%d\t%s\t%s\t%d", &id, &type, ip_addr, subscribe_id, &is_valid);
subscribe_id=(struct _subscribe_id_info *)calloc(1, sizeof(struct _subscribe_id_info));
ret=sscanf(table_line, "%d\t%d\t%s\t%s\t%d", &id, &type, ip_addr, subscribe_id->subscribe_id, &is_valid);
if(ret!=5)
{
free(subscribe_id);
subscribe_id=NULL;
MESA_handle_runtime_log(logger,
RLOG_LV_FATAL,
"SUBSCRIBE_ID",
"Parse subscribe_id failed, ret: %d table_id: %d key: %s table_line: %s",
"Parse subscribe_id failed, ret: %d table_id: %d table_line: %s",
ret,
table_id,
key,
table_line
);
return;
}
*ad=calloc(1, strlen(subscribe_id)+1);
memcpy(*ad, subscribe_id, strlen(subscribe_id));
atomic_inc(&subscribe_id->ref_cnt);
*ad=(MAAT_PLUGIN_EX_DATA)subscribe_id;
MESA_handle_runtime_log(logger,
RLOG_LV_DEBUG,
"SUBSCRIBE_ID",
"Add subscribe_id: %s table_id: %d key: %s table_line: %s",
*ad,
"Add subscribe_id: %s table_id: %d table_line: %s",
subscribe_id->subscribe_id,
table_id,
key,
table_line
);
return;
@@ -105,9 +317,16 @@ void subscribe_id_new_data(int table_id, const char* key, const char* table_line
void subscribe_id_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
{
void *logger=argp;
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Delete subscribe_id: %s table_id: %d", (char *)*ad, table_id);
struct _subscribe_id_info *subscribe_id=(struct _subscribe_id_info *)(*ad);
atomic_dec(&subscribe_id->ref_cnt);
if(subscribe_id->ref_cnt<=0)
{
free(*ad);
*ad=NULL;
}
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, "SUBSCRIBE_ID", "Delete subscribe_id: %s ref_cnt: %d table_id: %d", subscribe_id->subscribe_id, subscribe_id->ref_cnt, table_id);
return;
}
@@ -223,7 +442,15 @@ int tsg_rule_init(const char* conffile, void *logger)
MESA_load_profile_string_def(conffile, "MAAT", "SSL_SNI_TABLE", g_tsg_para.table_name[TABLE_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_SSL_SNI");
MESA_load_profile_string_def(conffile, "MAAT", "DECYPTION_EXCLUSION_SSL_SNI", g_tsg_para.table_name[TABLE_EXCLUSION_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_DECYPTION_EXCLUSION_SSL_SNI");
//init dynamic maat feather
MESA_load_profile_string_def(conffile, "MAAT", "IP_ASN_TABLE", g_tsg_para.table_name[TABLE_IP_ASN], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_ASN");
MESA_load_profile_string_def(conffile, "MAAT", "IP_LOCATION_TABLE", g_tsg_para.table_name[TABLE_IP_LOCATION], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_LOCATION");
MESA_load_profile_string_def(conffile, "MAAT", "ASN_BUILT_IN_TABLE", g_tsg_para.table_name[TABLE_ASN_BUILT_IN], _MAX_TABLE_NAME_LEN, "TSG_IP_ASN_BUILT_IN");
MESA_load_profile_string_def(conffile, "MAAT", "ASN_USER_DEFINED_TABLE", g_tsg_para.table_name[TABLE_ASN_USER_DEFINED], _MAX_TABLE_NAME_LEN, "TSG_IP_ASN_USER_DEFINED");
MESA_load_profile_string_def(conffile, "MAAT", "LOCATION_BUILT_IN_TABLE", g_tsg_para.table_name[TABLE_LOCATION_BUILT_IN], _MAX_TABLE_NAME_LEN, "TSG_IP_LOCATION_BUILT_IN");
MESA_load_profile_string_def(conffile, "MAAT", "LOCATION_USER_DEFINED_TABLE", g_tsg_para.table_name[TABLE_LOCATION_USER_DEFINED], _MAX_TABLE_NAME_LEN, "TSG_IP_LOCATION_USER_DEFINED");
//init static maat feather
g_tsg_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_STATIC", (char *)"STATIC", logger);
if(g_tsg_maat_feather==NULL)
{
@@ -245,6 +472,49 @@ int tsg_rule_init(const char* conffile, void *logger)
}
}
for(i=TABLE_ASN_USER_DEFINED; i<=TABLE_ASN_BUILT_IN; i++)
{
ret=Maat_ip_plugin_EX_register(g_tsg_maat_feather,
g_tsg_para.table_id[i],
ASN_new_data,
ASN_free_data,
ASN_dup_data,
0,
logger);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,
"RULE_INIT",
"Maat_ip_plugin_EX_register failed, table_name: %s table_id: %d",
g_tsg_para.table_name[i],
g_tsg_para.table_id[i]
);
return -1;
}
}
for(i=TABLE_LOCATION_USER_DEFINED; i<=TABLE_LOCATION_BUILT_IN; i++)
{
ret=Maat_ip_plugin_EX_register(g_tsg_maat_feather,
g_tsg_para.table_id[i],
location_new_data,
location_free_data,
location_dup_data,
0,
logger);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL,
"RULE_INIT",
"Maat_ip_plugin_EX_register failed, table_name: %s table_id: %d",
g_tsg_para.table_name[i],
g_tsg_para.table_id[i]
);
return -1;
}
}
//init dynamic maat feather
g_tsg_dynamic_maat_feather=init_maat_feather(maat_conffile, (char *)"TSG_DYNAMIC", (char *)"DYNAMIC", logger);
if(g_tsg_maat_feather==NULL)
@@ -327,8 +597,83 @@ int tsg_pull_policy_result(struct streaminfo *a_stream, PULL_RESULT_TYPE pull_re
return 0;
}
int tsg_get_ip_asn(const struct streaminfo *a_stream, int table_id, MAAT_PLUGIN_EX_DATA* client_asn, MAAT_PLUGIN_EX_DATA* server_asn)
{
struct ip_address dest_ip={0}, source_ip={0};
int tsg_get_subscribe_id(const struct streaminfo *a_stream, char **source_subscribe_id, char **dest_subscribe_id)
switch(a_stream->addr.addrtype)
{
case ADDR_TYPE_IPV4:
source_ip.ip_type=4;
source_ip.ipv4=a_stream->addr.tuple4_v4->saddr;
dest_ip.ip_type=4;
dest_ip.ipv4=a_stream->addr.tuple4_v4->daddr;
break;
case ADDR_TYPE_IPV6:
source_ip.ip_type=6;
memcpy((char *)(source_ip.ipv6), a_stream->addr.tuple4_v6->saddr, IPV6_ADDR_LEN);
dest_ip.ip_type=6;
memcpy((char *)(source_ip.ipv6), a_stream->addr.tuple4_v6->daddr, IPV6_ADDR_LEN);
break;
default:
return 0;
break;
}
if(*client_asn==NULL)
{
Maat_ip_plugin_get_EX_data(g_tsg_maat_feather, table_id, &source_ip, client_asn, 1);
}
if(*server_asn==NULL)
{
Maat_ip_plugin_get_EX_data(g_tsg_maat_feather, table_id, &dest_ip, server_asn, 1);
}
return 0;
}
int tsg_get_ip_location(const struct streaminfo *a_stream, int table_id, MAAT_PLUGIN_EX_DATA *client_location, MAAT_PLUGIN_EX_DATA *server_location)
{
struct ip_address dest_ip={0}, source_ip={0};
switch(a_stream->addr.addrtype)
{
case ADDR_TYPE_IPV4:
source_ip.ip_type=4;
source_ip.ipv4=a_stream->addr.tuple4_v4->saddr;
dest_ip.ip_type=4;
dest_ip.ipv4=a_stream->addr.tuple4_v4->daddr;
break;
case ADDR_TYPE_IPV6:
source_ip.ip_type=6;
memcpy((char *)(source_ip.ipv6), a_stream->addr.tuple4_v6->saddr, IPV6_ADDR_LEN);
dest_ip.ip_type=6;
memcpy((char *)(source_ip.ipv6), a_stream->addr.tuple4_v6->daddr, IPV6_ADDR_LEN);
break;
default:
return 0;
break;
}
if(*client_location==NULL)
{
Maat_ip_plugin_get_EX_data(g_tsg_maat_feather, table_id, &source_ip, client_location, 1);
}
if(*server_location==NULL)
{
Maat_ip_plugin_get_EX_data(g_tsg_maat_feather, table_id, &dest_ip, server_location, 1);
}
return 0;
}
int tsg_get_subscribe_id(const struct streaminfo *a_stream, struct _subscribe_id_info **source_subscribe_id, struct _subscribe_id_info **dest_subscribe_id)
{
char source_ip[MAX_IPV6_ADDR_LEN]={0};
char dest_ip[MAX_IPV6_ADDR_LEN]={0};
@@ -351,29 +696,141 @@ int tsg_get_subscribe_id(const struct streaminfo *a_stream, char **source_subscr
break;
}
if(strlen(dest_ip)>0)
if(strlen(dest_ip)>0 && *dest_subscribe_id==NULL)
{
*dest_subscribe_id = (char*)Maat_plugin_get_EX_data(g_tsg_dynamic_maat_feather, g_tsg_para.dyn_subscribe_ip_table_id, dest_ip);
*dest_subscribe_id = (struct _subscribe_id_info *)Maat_plugin_get_EX_data(g_tsg_dynamic_maat_feather, g_tsg_para.dyn_subscribe_ip_table_id, dest_ip);
}
if(strlen(source_ip)>0)
if(strlen(source_ip)>0 && *source_subscribe_id==NULL)
{
*source_subscribe_id = (char*)Maat_plugin_get_EX_data(g_tsg_dynamic_maat_feather, g_tsg_para.dyn_subscribe_ip_table_id, source_ip);
*source_subscribe_id = (struct _subscribe_id_info *)Maat_plugin_get_EX_data(g_tsg_dynamic_maat_feather, g_tsg_para.dyn_subscribe_ip_table_id, source_ip);
}
return 0;
}
int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _asn_info *asn, scan_status_t *mid, Maat_rule_t*result, int result_num)
{
int ret=0;
if(asn!=NULL)
{
ret=Maat_full_scan_string(maat_feather,
g_tsg_para.table_id[TABLE_IP_ASN],
CHARSET_GBK,
asn->asn,
strlen(asn->asn),
result,
NULL,
result_num,
mid,
a_stream->threadnum);
if(ret > 0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"Hit IP_ASN: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
asn->asn,
ret,
result[0].config_id,
result[0].service_id,
(unsigned char)result[0].action,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"No hit IP_ASN: %s scan ret: %d addr: %s",
asn->asn,
ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"IP_ASN is NULL scan ret: %d addr: %s",
ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
return ret;
}
int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _location_info *location, scan_status_t *mid, Maat_rule_t*result, int result_num)
{
int ret=0;
if(location!=NULL)
{
ret=Maat_full_scan_string(maat_feather,
g_tsg_para.table_id[TABLE_IP_LOCATION],
CHARSET_GBK,
location->country_full,
strlen(location->country_full),
result,
NULL,
result_num,
mid,
a_stream->threadnum);
if(ret > 0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"Hit IP_LOCATION: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
location->country_full,
ret,
result[0].config_id,
result[0].service_id,
(unsigned char)result[0].action,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"No hit IP_LOCATION: %s scan ret: %d addr: %s",
location->country_full,
ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
}
else
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"IP_LOCATION is NULL scan ret: %d addr: %s",
ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
return ret;
}
int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num)
{
int ret=0;
struct ipaddr t_addr;
struct ipaddr* p_addr=NULL;
int hit_num=0,tans_proto=0;
char *source_subscribe_id=NULL;
char *dest_subscribe_id=NULL;
int is_scan_addr=1, maat_ret=0,found_pos=0;
const struct streaminfo *cur_stream = a_stream;
struct _internal_label *internal_label=NULL;
if(result==NULL || result_num<=0 || a_stream==NULL || maat_feather==NULL)
{
@@ -464,7 +921,6 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
}while(cur_stream != NULL && hit_num < result_num);
if(hit_num<result_num && proto>PROTO_UNKONWN && proto<PROTO_MAX)
{
maat_ret=Maat_full_scan_string(maat_feather,
@@ -506,18 +962,43 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
}
}
internal_label=(struct _internal_label *)project_req_get_struct(a_stream, g_tsg_para.internal_project_id);
if(internal_label==NULL)
{
internal_label=(struct _internal_label *)calloc(1, sizeof(struct _internal_label));
memset(internal_label, 0, sizeof(struct _internal_label));
}
if(hit_num<result_num)
{
tsg_get_subscribe_id(a_stream, &source_subscribe_id, &dest_subscribe_id);
tsg_get_ip_location(a_stream, g_tsg_para.table_id[TABLE_LOCATION_USER_DEFINED], (void **)&(internal_label->client_location), (void **)&(internal_label->server_location));
tsg_get_ip_location(a_stream, g_tsg_para.table_id[TABLE_LOCATION_BUILT_IN], (void **)&(internal_label->client_location), (void **)&(internal_label->server_location));
if(source_subscribe_id!=NULL)
hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->client_location, mid, result+hit_num, result_num-hit_num);
hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->server_location, mid, result+hit_num, result_num-hit_num);
}
if(hit_num<result_num)
{
tsg_get_ip_asn(a_stream, g_tsg_para.table_id[TABLE_ASN_USER_DEFINED], (void **)&(internal_label->client_asn), (void **)&(internal_label->server_asn));
tsg_get_ip_asn(a_stream, g_tsg_para.table_id[TABLE_ASN_BUILT_IN], (void **)&(internal_label->client_asn), (void **)&(internal_label->server_asn));
hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->client_asn, mid, result+hit_num, result_num-hit_num);
hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->server_asn, mid, result+hit_num, result_num-hit_num);
}
if(hit_num<result_num)
{
tsg_get_subscribe_id(a_stream, &internal_label->client_subscribe_id, &internal_label->server_subscribe_id);
if(internal_label->client_subscribe_id!=NULL)
{
maat_ret=Maat_full_scan_string(maat_feather,
g_tsg_para.table_id[TABLE_SUBSCRIBER_ID],
CHARSET_GBK,
source_subscribe_id,
strlen(source_subscribe_id),
internal_label->client_subscribe_id->subscribe_id,
strlen(internal_label->client_subscribe_id->subscribe_id),
result+hit_num,
&found_pos,
result_num-hit_num,
@@ -529,7 +1010,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
RLOG_LV_DEBUG,
"SCAN_SUBSCRIBER",
"Hit source subscribe id: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
source_subscribe_id,
internal_label->client_subscribe_id->subscribe_id,
maat_ret,
result[hit_num].config_id,
result[hit_num].service_id,
@@ -545,22 +1026,20 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
RLOG_LV_DEBUG,
"SCAN_SUBSCRIBER",
"No hit source subscribe id: %s scan ret: %d addr: %s",
source_subscribe_id,
internal_label->client_subscribe_id->subscribe_id,
maat_ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
subscribe_id_free_data(g_tsg_para.dyn_subscribe_ip_table_id,(MAAT_PLUGIN_EX_DATA *)&source_subscribe_id, 0, g_tsg_para.logger);
}
if(dest_subscribe_id!=NULL)
if(internal_label->server_subscribe_id!=NULL)
{
maat_ret=Maat_full_scan_string(maat_feather,
g_tsg_para.table_id[TABLE_SUBSCRIBER_ID],
CHARSET_GBK,
dest_subscribe_id,
strlen(dest_subscribe_id),
internal_label->server_subscribe_id->subscribe_id,
strlen(internal_label->server_subscribe_id->subscribe_id),
result+hit_num,
&found_pos,
result_num-hit_num,
@@ -572,7 +1051,7 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
RLOG_LV_DEBUG,
"SCAN_SUBSCRIBER",
"Hit dest subscribe id: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
dest_subscribe_id,
internal_label->server_subscribe_id->subscribe_id,
maat_ret,
result[hit_num].config_id,
result[hit_num].service_id,
@@ -588,16 +1067,26 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
RLOG_LV_DEBUG,
"SCAN_SUBSCRIBER",
"No hit dest subscribe id: %s scan ret: %d addr: %s",
dest_subscribe_id,
internal_label->server_subscribe_id->subscribe_id,
maat_ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
subscribe_id_free_data(g_tsg_para.dyn_subscribe_ip_table_id,(MAAT_PLUGIN_EX_DATA *)&dest_subscribe_id, 0, g_tsg_para.logger);
}
}
ret=project_req_add_struct((struct streaminfo *)a_stream, g_tsg_para.internal_project_id, (void *)internal_label);
if(ret<0)
{
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_FATAL,
"ADD_INTERNAL_LABEL",
"Add internal label failed, ret: %d addr: %s",
ret,
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
return hit_num;
}

54
src/tsg_send_log.cpp
View File

@@ -19,7 +19,7 @@
#include "tsg_send_log.h"
#include "tsg_send_log_internal.h"
char TSG_SEND_LOG_VERSION_20200427=0;
char TSG_SEND_LOG_VERSION_20200514=0;
struct tsg_log_instance_t *g_tsg_log_instance;
@@ -164,6 +164,46 @@ static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name,
return 0;
}
int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
{
char buff[512]={0};
struct _location_info *location=NULL;
struct _internal_label *internal_label=NULL;
internal_label=(struct _internal_label *)project_req_get_struct(a_stream, _instance->internal_project_id);
if(internal_label!=NULL)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)internal_label->establish_latency_ms, TLD_TYPE_LONG);
if(internal_label->client_asn!=NULL)
{
snprintf(buff, sizeof(buff), "%s(%s)", internal_label->client_asn->asn, internal_label->client_asn->organization);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_ASN].name, (void *)buff, TLD_TYPE_STRING);
}
if(internal_label->server_asn!=NULL)
{
snprintf(buff, sizeof(buff), "%s(%s)", internal_label->server_asn->asn, internal_label->server_asn->organization);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_ASN].name, (void *)buff, TLD_TYPE_STRING);
}
if(internal_label->client_location!=NULL)
{
location=internal_label->client_location;
snprintf(buff, sizeof(buff), "%s%s%s%s%s", location->country_full, (strlen(location->province_full)>0) ? "/" : "", location->province_full, (strlen(location->city_full)>0) ? "/" : "", location->city_full);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_LOCATION].name, (void *)buff, TLD_TYPE_STRING);
}
if(internal_label->server_location!=NULL)
{
location=internal_label->server_location;
snprintf(buff, sizeof(buff), "%s%s%s%s%s", location->country_full, (strlen(location->province_full)>0) ? "/" : "", location->province_full, (strlen(location->city_full)>0) ? "/" : "", location->city_full);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_LOCATION].name, (void *)buff, TLD_TYPE_STRING);
}
}
return 0;
}
int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, struct streaminfo *a_stream)
{
@@ -468,7 +508,6 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
{
int i=0,status=0;
char *payload=NULL;
long establish_latency_ms=0;
struct TLD_handle_t *_handle=handle;
struct tsg_log_instance_t *_instance=instance;
@@ -489,6 +528,8 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
TLD_append_streaminfo(instance, handle, log_msg->a_stream);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SLED_IP].name, (void *)(_instance->local_ip_str), TLD_TYPE_STRING);
set_common_field_from_label(_instance, _handle, log_msg->a_stream);
for(i=0;i<log_msg->result_num; i++)
{
switch(log_msg->result[i].do_log)
@@ -511,15 +552,6 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
break;
}
if(log_msg->result[i].action==TSG_ACTION_MONITOR && _instance->establish_latency_project_id>0)
{
establish_latency_ms=project_req_get_long(log_msg->a_stream, _instance->establish_latency_project_id);
if(establish_latency_ms>=0)
{
TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)establish_latency_ms, TLD_TYPE_LONG);
}
}
TLD_append(_handle, _instance->id2field[LOG_COMMON_POLICY_ID].name, (void *)(long)(log_msg->result[i].config_id), TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVICE].name, (void *)(long)(log_msg->result[i].service_id), TLD_TYPE_LONG);
TLD_append(_handle, _instance->id2field[LOG_COMMON_ACTION].name, (void *)(long)((unsigned char)log_msg->result[i].action), TLD_TYPE_LONG);

6
src/tsg_send_log_internal.h
View File

@@ -61,6 +61,10 @@ typedef enum _tsg_log_field_id
LOG_SSL_SNI,
LOG_COMMON_ESTABLISH_LATENCY_MS,
LOG_COMMON_SUB_ACTION,
LOG_COMMON_CLINET_ASN,
LOG_COMMON_SERVER_ASN,
LOG_COMMON_CLINET_LOCATION,
LOG_COMMON_SERVER_LOCATION,
LOG_COMMON_MAX
}tsg_log_field_id_t;
@@ -81,7 +85,7 @@ struct tsg_log_instance_t
{
int mode;
int max_service;
int establish_latency_project_id;
int internal_project_id;
void *logger;
char common_field_file[MAX_STRING_LEN*4];
char broker_list[MAX_STRING_LEN*4];