gfwleak/tango-tsg-master
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

调整扫描的属地表名称,同时调整table_info,IP归属地表不能使用composition聚合表

Browse Source 
支持扫描quic协议，增加相关代码
调整发送IP归属地字段的位置，保证拦截日志能正常填充IP归属地字段
This commit is contained in:
liuxueli
2020-05-29 14:40:15 +08:00
parent 0ebd134553
commit fdd6c8ab2b
6 changed files with 151 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
bin/tsg_maat.json
View File

@@ -1,44 +1,10 @@
{
"compile_table": "TSG_SECURITY_COMPILE",
"group_table": "POLICY_OBJECT",
"group_table": "GROUP_COMPILE_RELATION",
"rules": [
{
"compile_id": 1,
"service": 0,
"action": 16,
"do_blacklist": 0,
"do_log": 1,
"effective_rage": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "group_1",
"regions": [
{
"table_name": "TSG_OBJ_IP_ADDR",
"table_type": "ip",
"table_content": {
"addr_type": "ipv4",
"src_ip": "61.135.169.125",
"mask_src_ip": "255.255.255.255",
"src_port": "80",
"mask_src_port": "65535",
"dst_ip": "192.168.41.228",
"mask_dst_ip": "255.255.255.255",
"dst_port": "0",
"mask_dst_port": "65535",
"protocol": 6,
"direction": "double"
}
}
]
}
]
},
{
"compile_id": 2,
"service": 0,
"action": 128,
"do_blacklist": 0,
"do_log": 1,
@@ -47,14 +13,83 @@
"is_valid": "yes",
"groups": [
{
"group_name": "FQDN_SNI",
"group_name": "IP_ADDR_1",
"regions": [
{
"table_name": "TSG_OBJ_FQDN",
"table_name": "TSG_OBJ_IP_ADDR",
"table_type": "ip_plus",
"table_content": {
"addr_type": "ipv4",
"src_ip": "192.168.100.5",
"mask_src_ip": "255.255.255.255",
"src_port": "0",
"mask_src_port": "65535",
"dst_ip": "0.0.0.0",
"mask_dst_ip": "255.255.255.255",
"dst_port": "0",
"mask_dst_port": "65535",
"protocol": 6,
"direction": "double"
}
}
]
},
{
"group_name": "IP_ADDR_2",
"regions": [
{
"table_name": "TSG_OBJ_IP_ADDR",
"table_type": "ip_plus",
"table_content": {
"addr_type": "ipv4",
"src_ip": "192.168.50.37",
"mask_src_ip": "255.255.255.255",
"src_port": "0",
"mask_src_port": "65535",
"dst_ip": "0.0.0.0",
"mask_dst_ip": "255.255.255.255",
"dst_port": "0",
"mask_dst_port": "65535",
"protocol": 6,
"direction": "double"
}
}
]
},
{
"group_name": "ASN",
"regions": [
{
"table_name": "TSG_OBJ_AS_NUMBER",
"table_type": "expr",
"table_content": {
"keywords": "baidu.com",
"expr_type": "and",
"keywords": "101",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
},
{
"table_name": "TSG_OBJ_AS_NUMBER",
"table_type": "expr",
"table_content": {
"keywords": "102",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"group_name": "LOCATION",
"regions": [
{
"table_name": "TSG_OBJ_GEO_LOCATION",
"table_type": "expr",
"table_content": {
"keywords": "China",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
@@ -64,21 +99,52 @@
]
},
{
"compile_id": 3,
"compile_id": 5,
"service": 0,
"action": 128,
"do_blacklist": 0,
"do_log": 1,
"effective_rage": 0,
"user_region": "Virtual",
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name":"FQDN_SNI",
"virtual_table":"TSG_FIELD_SSL_SNI",
"not_flag" : 0
"group_name":"IP_ADDR_1",
"virtual_table":"TSG_SECURITY_SOURCE_ADDR",
"not_flag":0
},
{
"group_name":"IP_ADDR_2",
"virtual_table":"TSG_SECURITY_DESTINATION_ADDR",
"not_flag":0
},
{
"group_name":"ASN",
"virtual_table":"TSG_SECURITY_SOURCE_ASN",
"not_flag":0
},
{
"group_name":"LOCATION",
"virtual_table":"TSG_SECURITY_DESTINATION_LOCATION",
"not_flag":0
}
]
}
],
"plugin_table": [
{
"table_name": "TSG_IP_ASN_USER_DEFINED",
"table_content": [
"101\t4\t192.168.50.1\t192.168.50.255\t101\tmesa\t1",
"102\t4\t192.168.100.1\t192.168.100.255\t102\tgeedge\t1"
]
},
{
"table_name": "TSG_IP_LOCATION_USER_DEFINED",
"table_content": [
"201\t20100\t4\t192.168.50.1\t192.168.50.255\t11.12\t11.12\t0\ten\tAS\tAsia\tCN\tChina\tBJ\tBeijing\tBeijing\tAsia/Singapore\t1",
"202\t20200\t4\t192.168.100.1\t192.168.100.255\t11.12\t11.12\t0\ten\tAS\tAsia\tCN\tChina\tSH\tShanghai\tShanghai\tAsia/Singapore\t1"
]
}
]
}

16
bin/tsg_static_tableinfo.conf
View File

@@ -49,12 +49,10 @@
37 TSG_IP_ASN_USER_DEFINED ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7}
38 TSG_IP_LOCATION_BUILT_IN ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18}
39 TSG_IP_LOCATION_USER_DEFINED ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18}
40 TSG_OBJ_IP_ASN expr UTF8 UTF8/GBK yes 0
41 TSG_SECURITY_SOURCE_ASN virtual TSG_OBJ_IP_ASN --
42 TSG_SECURITY_DESTINATION_ASN virtual TSG_OBJ_IP_ASN --
43 TSG_SECURITY_ASN composition {"source":"TSG_SECURITY_SOURCE_ASN","destination":"TSG_SECURITY_DESTINATION_ASN"}
44 TSG_OBJ_IP_LOCATION expr UTF8 UTF8/GBK yes 0
45 TSG_SECURITY_SOURCE_LOCATION virtual TSG_OBJ_IP_LOCATION --
46 TSG_SECURITY_DESTINATION_LOCATION virtual TSG_OBJ_IP_LOCATION --
47 TSG_SECURITY_LOCATION composition {"source":"TSG_SECURITY_SOURCE_LOCATION","destination":"TSG_SECURITY_DESTINATION_LOCATION"}
48 TSG_FIELD_QUIC_SNI virtual TSG_OBJ_FQDN --
40 TSG_OBJ_AS_NUMBER expr UTF8 UTF8/GBK yes 0
41 TSG_SECURITY_SOURCE_ASN virtual TSG_OBJ_AS_NUMBER --
42 TSG_SECURITY_DESTINATION_ASN virtual TSG_OBJ_AS_NUMBER --
43 TSG_OBJ_GEO_LOCATION expr UTF8 UTF8/GBK yes 0
44 TSG_SECURITY_SOURCE_LOCATION virtual TSG_OBJ_GEO_LOCATION --
45 TSG_SECURITY_DESTINATION_LOCATION virtual TSG_OBJ_GEO_LOCATION --
46 TSG_FIELD_QUIC_SNI virtual TSG_OBJ_FQDN --

1
inc/tsg_rule.h
View File

@@ -38,6 +38,7 @@ typedef enum _tsg_protocol
PROTO_SIP,
PROTO_BGP,
PROTO_STREAMING_MEDIA,
PROTO_QUIC,
PROTO_MAX
}tsg_protocol_t;

6
src/tsg_entry.h
View File

@@ -30,8 +30,10 @@ enum MASTER_TABLE{
TABLE_HTTP_HOST,
TABLE_SSL_SNI,
TABLE_EXCLUSION_SSL_SNI,
TABLE_IP_ASN,
TABLE_IP_LOCATION,
TABLE_SRC_ASN,
TABLE_DST_ASN,
TABLE_SRC_LOCATION,
TABLE_DST_LOCATION,
TABLE_ASN_USER_DEFINED,
TABLE_ASN_BUILT_IN,
TABLE_LOCATION_USER_DEFINED,

51
src/tsg_rule.cpp
View File

@@ -51,6 +51,7 @@ const struct _str2index g_tsg_proto_string[PROTO_MAX+1]={{PROTO_UNKONWN, 0, (cha
{PROTO_SIP, 4, (char *)"SIP."},
{PROTO_BGP, 4, (char *)"BGP."},
{PROTO_STREAMING_MEDIA, 16, (char *)"STREAMING_MEDIA."},
{PROTO_QUIC, 5, (char *)"QUIC."},
{PROTO_MAX, 0, (char *)""}
};
@@ -449,8 +450,10 @@ int tsg_rule_init(const char* conffile, void *logger)
MESA_load_profile_string_def(conffile, "MAAT", "SSL_SNI_TABLE", g_tsg_para.table_name[TABLE_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_SSL_SNI");
MESA_load_profile_string_def(conffile, "MAAT", "DECYPTION_EXCLUSION_SSL_SNI", g_tsg_para.table_name[TABLE_EXCLUSION_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_DECYPTION_EXCLUSION_SSL_SNI");
MESA_load_profile_string_def(conffile, "MAAT", "IP_ASN_TABLE", g_tsg_para.table_name[TABLE_IP_ASN], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_ASN");
MESA_load_profile_string_def(conffile, "MAAT", "IP_LOCATION_TABLE", g_tsg_para.table_name[TABLE_IP_LOCATION], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_LOCATION");
MESA_load_profile_string_def(conffile, "MAAT", "SRC_ASN_TABLE", g_tsg_para.table_name[TABLE_SRC_ASN], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_SOURCE_ASN");
MESA_load_profile_string_def(conffile, "MAAT", "DST_ASN_TABLE", g_tsg_para.table_name[TABLE_DST_ASN], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_DESTINATION_ASN");
MESA_load_profile_string_def(conffile, "MAAT", "SRC_LOCATION_TABLE", g_tsg_para.table_name[TABLE_SRC_LOCATION], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_SOURCE_LOCATION");
MESA_load_profile_string_def(conffile, "MAAT", "DST_LOCATION_TABLE", g_tsg_para.table_name[TABLE_DST_LOCATION], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_DESTINATION_LOCATION");
MESA_load_profile_string_def(conffile, "MAAT", "ASN_BUILT_IN_TABLE", g_tsg_para.table_name[TABLE_ASN_BUILT_IN], _MAX_TABLE_NAME_LEN, "TSG_IP_ASN_BUILT_IN");
MESA_load_profile_string_def(conffile, "MAAT", "ASN_USER_DEFINED_TABLE", g_tsg_para.table_name[TABLE_ASN_USER_DEFINED], _MAX_TABLE_NAME_LEN, "TSG_IP_ASN_USER_DEFINED");
@@ -716,14 +719,14 @@ int tsg_get_subscribe_id(const struct streaminfo *a_stream, struct _subscribe_id
return 0;
}
int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _asn_info *asn, scan_status_t *mid, Maat_rule_t*result, int result_num)
int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _asn_info *asn, enum MASTER_TABLE idx, scan_status_t *mid, Maat_rule_t*result, int result_num)
{
int ret=0;
if(asn!=NULL)
{
ret=Maat_full_scan_string(maat_feather,
g_tsg_para.table_id[TABLE_IP_ASN],
g_tsg_para.table_id[idx],
CHARSET_GBK,
asn->asn,
strlen(asn->asn),
@@ -737,9 +740,10 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"Hit IP_ASN: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
"Hit IP_ASN: %s scan ret: %d table_name: %s policy_id: %d service: %d action: %d addr: %s",
asn->asn,
ret,
g_tsg_para.table_name[idx],
result[0].config_id,
result[0].service_id,
(unsigned char)result[0].action,
@@ -751,9 +755,10 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"No hit IP_ASN: %s scan ret: %d addr: %s",
"No hit IP_ASN: %s scan ret: %d table_name: %s addr: %s",
asn->asn,
ret,
g_tsg_para.table_name[idx],
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
@@ -763,8 +768,9 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_ASN",
"IP_ASN is NULL scan ret: %d addr: %s",
"IP_ASN is NULL scan ret: %d table_name: %s addr: %s",
ret,
g_tsg_para.table_name[idx],
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
@@ -773,17 +779,19 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
}
int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _location_info *location, scan_status_t *mid, Maat_rule_t*result, int result_num)
int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _location_info *location, enum MASTER_TABLE idx, scan_status_t *mid, Maat_rule_t*result, int result_num)
{
int ret=0;
char buff[1024]={0};
if(location!=NULL)
{
snprintf(buff, sizeof(buff), "%s.%s.", location->country_full, location->city_full);
ret=Maat_full_scan_string(maat_feather,
g_tsg_para.table_id[TABLE_IP_LOCATION],
g_tsg_para.table_id[idx],
CHARSET_GBK,
location->country_full,
strlen(location->country_full),
buff,
strlen(buff),
result,
NULL,
result_num,
@@ -794,9 +802,10 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"Hit IP_LOCATION: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
location->country_full,
"Hit IP_LOCATION: %s scan ret: %d table_name: %d policy_id: %d service: %d action: %d addr: %s",
buff,
ret,
g_tsg_para.table_name[idx],
result[0].config_id,
result[0].service_id,
(unsigned char)result[0].action,
@@ -808,9 +817,10 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"No hit IP_LOCATION: %s scan ret: %d addr: %s",
location->country_full,
"No hit IP_LOCATION: %s scan ret: %d table_name: %s addr: %s",
buff,
ret,
g_tsg_para.table_name[idx],
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
@@ -820,8 +830,9 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
MESA_handle_runtime_log(g_tsg_para.logger,
RLOG_LV_DEBUG,
"SCAN_IP_LOCATION",
"IP_LOCATION is NULL scan ret: %d addr: %s",
"IP_LOCATION is NULL scan ret: %d table_name: %s addr: %s",
ret,
g_tsg_para.table_name[idx],
printaddr(&a_stream->addr, a_stream->threadnum)
);
}
@@ -981,8 +992,8 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
tsg_get_ip_location(a_stream, g_tsg_para.table_id[TABLE_LOCATION_USER_DEFINED], (void **)&(internal_label->client_location), (void **)&(internal_label->server_location));
tsg_get_ip_location(a_stream, g_tsg_para.table_id[TABLE_LOCATION_BUILT_IN], (void **)&(internal_label->client_location), (void **)&(internal_label->server_location));
hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->client_location, mid, result+hit_num, result_num-hit_num);
hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->server_location, mid, result+hit_num, result_num-hit_num);
hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->client_location, TABLE_SRC_LOCATION, mid, result+hit_num, result_num-hit_num);
hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->server_location, TABLE_DST_LOCATION, mid, result+hit_num, result_num-hit_num);
}
if(hit_num<result_num)
@@ -990,8 +1001,8 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
tsg_get_ip_asn(a_stream, g_tsg_para.table_id[TABLE_ASN_USER_DEFINED], (void **)&(internal_label->client_asn), (void **)&(internal_label->server_asn));
tsg_get_ip_asn(a_stream, g_tsg_para.table_id[TABLE_ASN_BUILT_IN], (void **)&(internal_label->client_asn), (void **)&(internal_label->server_asn));
hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->client_asn, mid, result+hit_num, result_num-hit_num);
hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->server_asn, mid, result+hit_num, result_num-hit_num);
hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->client_asn, TABLE_SRC_ASN, mid, result+hit_num, result_num-hit_num);
hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->server_asn, TABLE_DST_ASN, mid, result+hit_num, result_num-hit_num);
}

7
src/tsg_send_log.cpp
View File

@@ -191,14 +191,14 @@ int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD
if(internal_label->client_location!=NULL)
{
location=internal_label->client_location;
snprintf(buff, sizeof(buff), "%s%s%s%s%s", location->country_full, (strlen(location->province_full)>0) ? "/" : "", location->province_full, (strlen(location->city_full)>0) ? "/" : "", location->city_full);
snprintf(buff, sizeof(buff), "%s%s%s", location->country_full, (strlen(location->city_full)>0) ? "." : "", location->city_full);
TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_LOCATION].name, (void *)buff, TLD_TYPE_STRING);
}
if(internal_label->server_location!=NULL)
{
location=internal_label->server_location;
snprintf(buff, sizeof(buff), "%s%s%s%s%s", location->country_full, (strlen(location->province_full)>0) ? "/" : "", location->province_full, (strlen(location->city_full)>0) ? "/" : "", location->city_full);
snprintf(buff, sizeof(buff), "%s%s%s", location->country_full, (strlen(location->city_full)>0) ? "." : "", location->city_full);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_LOCATION].name, (void *)buff, TLD_TYPE_STRING);
}
}
@@ -312,6 +312,8 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle
TLD_append(_handle, _instance->id2field[LOG_COMMON_ADDRESS_LIST].name, (void *)nest_addr_buf, TLD_TYPE_STRING);
set_common_field_from_label(_instance, _handle, a_stream);
return 0;
}
@@ -527,7 +529,6 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
TLD_append_streaminfo(instance, handle, log_msg->a_stream);
TLD_append(_handle, _instance->id2field[LOG_COMMON_SLED_IP].name, (void *)(_instance->local_ip_str), TLD_TYPE_STRING);
TLD_append(_handle, _instance->id2field[LOG_COMMON_DEVICE_ID].name, (void *)(g_tsg_para.device_sn), TLD_TYPE_STRING);
set_common_field_from_label(_instance, _handle, log_msg->a_stream);
for(i=0;i<log_msg->result_num; i++)
{