调整扫描的属地表名称,同时调整table_info,IP归属地表不能使用composition聚合表

支持扫描quic协议，增加相关代码
调整发送IP归属地字段的位置，保证拦截日志能正常填充IP归属地字段

bin/tsg_maat.json

@@ -1,44 +1,10 @@
 {
     "compile_table": "TSG_SECURITY_COMPILE",
-    "group_table": "POLICY_OBJECT",
+    "group_table": "GROUP_COMPILE_RELATION",
     "rules": [
         {
             "compile_id": 1,
             "service": 0,
-            "action": 16,
-            "do_blacklist": 0,
-            "do_log": 1,
-            "effective_rage": 0,
-            "user_region": "anything",
-            "is_valid": "yes",
-            "groups": [
-                {
-                    "group_name": "group_1",
-                    "regions": [
-                        {
-                            "table_name": "TSG_OBJ_IP_ADDR",
-                            "table_type": "ip",
-                            "table_content": {
-                                "addr_type": "ipv4",
-                                "src_ip": "61.135.169.125",
-                                "mask_src_ip": "255.255.255.255",
-                                "src_port": "80",
-                                "mask_src_port": "65535",
-                                "dst_ip": "192.168.41.228",
-                                "mask_dst_ip": "255.255.255.255",
-                                "dst_port": "0",
-                                "mask_dst_port": "65535",
-                                "protocol": 6,
-                                "direction": "double"
-                            }
-                        }
-                    ]
-                }
-            ]    
-        },
-        {
-            "compile_id": 2,
-            "service": 0,
             "action": 128,
             "do_blacklist": 0,
             "do_log": 1,
@@ -47,14 +13,83 @@
             "is_valid": "yes",
             "groups": [
                 {
-                    "group_name": "FQDN_SNI",
+                    "group_name": "IP_ADDR_1",
                     "regions": [
                         {
-                            "table_name": "TSG_OBJ_FQDN",
+                            "table_name": "TSG_OBJ_IP_ADDR",
+                            "table_type": "ip_plus",
+                            "table_content": {
+                                "addr_type": "ipv4",
+                                "src_ip": "192.168.100.5",
+                                "mask_src_ip": "255.255.255.255",
+                                "src_port": "0",
+                                "mask_src_port": "65535",
+                                "dst_ip": "0.0.0.0",
+                                "mask_dst_ip": "255.255.255.255",
+                                "dst_port": "0",
+                                "mask_dst_port": "65535",
+                                "protocol": 6,
+                                "direction": "double"
+                            }
+                        }
+                    ]
+                },
+                {
+                    "group_name": "IP_ADDR_2",
+                    "regions": [
+                        {
+                            "table_name": "TSG_OBJ_IP_ADDR",
+                            "table_type": "ip_plus",
+                            "table_content": {
+                                "addr_type": "ipv4",
+                                "src_ip": "192.168.50.37",
+                                "mask_src_ip": "255.255.255.255",
+                                "src_port": "0",
+                                "mask_src_port": "65535",
+                                "dst_ip": "0.0.0.0",
+                                "mask_dst_ip": "255.255.255.255",
+                                "dst_port": "0",
+                                "mask_dst_port": "65535",
+                                "protocol": 6,
+                                "direction": "double"
+                            }
+                        }
+                    ]
+                },
+                {
+                    "group_name": "ASN",
+                    "regions": [
+                        {
+                            "table_name": "TSG_OBJ_AS_NUMBER",
                             "table_type": "expr",
                             "table_content": {
-                                "keywords": "baidu.com",
+                                "keywords": "101",
-                                "expr_type": "and",
+                                "expr_type": "none",
+                                "match_method": "sub",
+                                "format": "uncase plain"
+                            }
+                        },
+                        {
+                            "table_name": "TSG_OBJ_AS_NUMBER",
+                            "table_type": "expr",
+                            "table_content": {
+                                "keywords": "102",
+                                "expr_type": "none",
+                                "match_method": "sub",
+                                "format": "uncase plain"
+                            }
+                        }
+                    ]
+                },
+                {
+                    "group_name": "LOCATION",
+                    "regions": [
+                        {
+                            "table_name": "TSG_OBJ_GEO_LOCATION",
+                            "table_type": "expr",
+                            "table_content": {
+                                "keywords": "China",
+                                "expr_type": "none",
                                 "match_method": "sub",
                                 "format": "uncase plain"
                             }
@@ -64,21 +99,52 @@
             ]    
         },
         {
-            "compile_id": 3,
+            "compile_id": 5,
             "service": 0,
             "action": 128,
             "do_blacklist": 0,
             "do_log": 1,
             "effective_rage": 0,
-            "user_region": "Virtual",
+            "user_region": "anything",
             "is_valid": "yes",
             "groups": [
                 {
-			"group_name":"FQDN_SNI",
+                    "group_name":"IP_ADDR_1",
-                        "virtual_table":"TSG_FIELD_SSL_SNI",
+                    "virtual_table":"TSG_SECURITY_SOURCE_ADDR",
-                        "not_flag" : 0
+                    "not_flag":0                   
+                },
+                {
+                    "group_name":"IP_ADDR_2",
+                    "virtual_table":"TSG_SECURITY_DESTINATION_ADDR",
+                    "not_flag":0                   
+                },
+                {
+                    "group_name":"ASN",
+                    "virtual_table":"TSG_SECURITY_SOURCE_ASN",
+                    "not_flag":0                   
+                },
+                {
+                    "group_name":"LOCATION",
+                    "virtual_table":"TSG_SECURITY_DESTINATION_LOCATION",
+                    "not_flag":0                   
                 }
             ]
         }
+    ],
+    "plugin_table": [
+        {
+            "table_name": "TSG_IP_ASN_USER_DEFINED",
+            "table_content": [
+                "101\t4\t192.168.50.1\t192.168.50.255\t101\tmesa\t1",
+                "102\t4\t192.168.100.1\t192.168.100.255\t102\tgeedge\t1"
+            ]
+        },
+        {
+            "table_name": "TSG_IP_LOCATION_USER_DEFINED",
+            "table_content": [
+                "201\t20100\t4\t192.168.50.1\t192.168.50.255\t11.12\t11.12\t0\ten\tAS\tAsia\tCN\tChina\tBJ\tBeijing\tBeijing\tAsia/Singapore\t1",
+                "202\t20200\t4\t192.168.100.1\t192.168.100.255\t11.12\t11.12\t0\ten\tAS\tAsia\tCN\tChina\tSH\tShanghai\tShanghai\tAsia/Singapore\t1"
+            ]
+        }
     ]
 }

bin/tsg_static_tableinfo.conf

@@ -49,12 +49,10 @@
 37	TSG_IP_ASN_USER_DEFINED	ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":7}
 38	TSG_IP_LOCATION_BUILT_IN	ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18}
 39	TSG_IP_LOCATION_USER_DEFINED	ip_plugin {"row_id":1,"ip_type":3,"start_ip":4,"end_ip":5,"valid":18}
-40	TSG_OBJ_IP_ASN	expr	UTF8	UTF8/GBK	yes	0
+40	TSG_OBJ_AS_NUMBER	expr	UTF8	UTF8/GBK	yes	0
-41	TSG_SECURITY_SOURCE_ASN	virtual	TSG_OBJ_IP_ASN	--
+41	TSG_SECURITY_SOURCE_ASN	virtual	TSG_OBJ_AS_NUMBER	--
-42	TSG_SECURITY_DESTINATION_ASN	virtual	TSG_OBJ_IP_ASN	--
+42	TSG_SECURITY_DESTINATION_ASN	virtual	TSG_OBJ_AS_NUMBER	--
-43	TSG_SECURITY_ASN	composition	{"source":"TSG_SECURITY_SOURCE_ASN","destination":"TSG_SECURITY_DESTINATION_ASN"}
+43	TSG_OBJ_GEO_LOCATION	expr	UTF8	UTF8/GBK	yes	0
-44	TSG_OBJ_IP_LOCATION	expr	UTF8	UTF8/GBK	yes	0
+44	TSG_SECURITY_SOURCE_LOCATION	virtual	TSG_OBJ_GEO_LOCATION	--
-45	TSG_SECURITY_SOURCE_LOCATION	virtual	TSG_OBJ_IP_LOCATION	--
+45	TSG_SECURITY_DESTINATION_LOCATION	virtual	TSG_OBJ_GEO_LOCATION	--
-46	TSG_SECURITY_DESTINATION_LOCATION	virtual	TSG_OBJ_IP_LOCATION	--
+46	TSG_FIELD_QUIC_SNI	virtual	TSG_OBJ_FQDN	--
-47	TSG_SECURITY_LOCATION	composition	{"source":"TSG_SECURITY_SOURCE_LOCATION","destination":"TSG_SECURITY_DESTINATION_LOCATION"}
-48	TSG_FIELD_QUIC_SNI	virtual	TSG_OBJ_FQDN	--

inc/tsg_rule.h

@@ -38,6 +38,7 @@ typedef enum _tsg_protocol
    PROTO_SIP,
    PROTO_BGP,
    PROTO_STREAMING_MEDIA,
+   PROTO_QUIC,
    PROTO_MAX
 }tsg_protocol_t;
 

src/tsg_entry.h

@@ -30,8 +30,10 @@ enum MASTER_TABLE{
 	TABLE_HTTP_HOST,
 	TABLE_SSL_SNI,
 	TABLE_EXCLUSION_SSL_SNI,
-	TABLE_IP_ASN,
+	TABLE_SRC_ASN,
-	TABLE_IP_LOCATION,
+	TABLE_DST_ASN,
+	TABLE_SRC_LOCATION,
+	TABLE_DST_LOCATION,
 	TABLE_ASN_USER_DEFINED,
 	TABLE_ASN_BUILT_IN,
 	TABLE_LOCATION_USER_DEFINED,

src/tsg_rule.cpp

@@ -51,6 +51,7 @@ const struct _str2index g_tsg_proto_string[PROTO_MAX+1]={{PROTO_UNKONWN, 0, (cha
 										 {PROTO_SIP, 4, (char *)"SIP."},
 										 {PROTO_BGP, 4, (char *)"BGP."},
 										 {PROTO_STREAMING_MEDIA, 16, (char *)"STREAMING_MEDIA."},
+										 {PROTO_QUIC, 5, (char *)"QUIC."},
 										 {PROTO_MAX, 0, (char *)""}
 };
 
@@ -449,8 +450,10 @@ int tsg_rule_init(const char* conffile, void *logger)
 	MESA_load_profile_string_def(conffile, "MAAT", "SSL_SNI_TABLE", g_tsg_para.table_name[TABLE_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_FIELD_SSL_SNI");
 	MESA_load_profile_string_def(conffile, "MAAT", "DECYPTION_EXCLUSION_SSL_SNI", g_tsg_para.table_name[TABLE_EXCLUSION_SSL_SNI], _MAX_TABLE_NAME_LEN, "TSG_DECYPTION_EXCLUSION_SSL_SNI");
 
-	MESA_load_profile_string_def(conffile, "MAAT", "IP_ASN_TABLE", g_tsg_para.table_name[TABLE_IP_ASN], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_ASN");
+	MESA_load_profile_string_def(conffile, "MAAT", "SRC_ASN_TABLE", g_tsg_para.table_name[TABLE_SRC_ASN], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_SOURCE_ASN");
-	MESA_load_profile_string_def(conffile, "MAAT", "IP_LOCATION_TABLE", g_tsg_para.table_name[TABLE_IP_LOCATION], _MAX_TABLE_NAME_LEN, "TSG_OBJ_IP_LOCATION");
+	MESA_load_profile_string_def(conffile, "MAAT", "DST_ASN_TABLE", g_tsg_para.table_name[TABLE_DST_ASN], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_DESTINATION_ASN");
+	MESA_load_profile_string_def(conffile, "MAAT", "SRC_LOCATION_TABLE", g_tsg_para.table_name[TABLE_SRC_LOCATION], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_SOURCE_LOCATION");
+	MESA_load_profile_string_def(conffile, "MAAT", "DST_LOCATION_TABLE", g_tsg_para.table_name[TABLE_DST_LOCATION], _MAX_TABLE_NAME_LEN, "TSG_SECURITY_DESTINATION_LOCATION");
 	
 	MESA_load_profile_string_def(conffile, "MAAT", "ASN_BUILT_IN_TABLE", g_tsg_para.table_name[TABLE_ASN_BUILT_IN], _MAX_TABLE_NAME_LEN, "TSG_IP_ASN_BUILT_IN");
 	MESA_load_profile_string_def(conffile, "MAAT", "ASN_USER_DEFINED_TABLE", g_tsg_para.table_name[TABLE_ASN_USER_DEFINED], _MAX_TABLE_NAME_LEN, "TSG_IP_ASN_USER_DEFINED");
@@ -716,14 +719,14 @@ int tsg_get_subscribe_id(const struct streaminfo *a_stream, struct _subscribe_id
 	return 0;
 }
 
-int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _asn_info *asn, scan_status_t *mid, Maat_rule_t*result, int result_num)
+int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _asn_info *asn, enum MASTER_TABLE idx, scan_status_t *mid, Maat_rule_t*result, int result_num)
 {
 	int ret=0;
 
 	if(asn!=NULL)
 	{
 		ret=Maat_full_scan_string(maat_feather,
-								g_tsg_para.table_id[TABLE_IP_ASN],
+								g_tsg_para.table_id[idx],
 								CHARSET_GBK,
 								asn->asn,
 								strlen(asn->asn),
@@ -737,9 +740,10 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
 			MESA_handle_runtime_log(g_tsg_para.logger,
 									RLOG_LV_DEBUG,
 									"SCAN_IP_ASN",
-									"Hit IP_ASN: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
+									"Hit IP_ASN: %s scan ret: %d table_name: %s policy_id: %d service: %d action: %d addr: %s",
 									asn->asn,
 									ret,
+									g_tsg_para.table_name[idx],
 									result[0].config_id,
 									result[0].service_id,
 									(unsigned char)result[0].action,
@@ -751,9 +755,10 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
 			MESA_handle_runtime_log(g_tsg_para.logger,
 									RLOG_LV_DEBUG,
 									"SCAN_IP_ASN",
-									"No hit IP_ASN: %s scan ret: %d addr: %s",
+									"No hit IP_ASN: %s scan ret: %d table_name: %s addr: %s",
 									asn->asn,
 									ret,
+									g_tsg_para.table_name[idx],
 									printaddr(&a_stream->addr, a_stream->threadnum)
 									);
 		}
@@ -763,8 +768,9 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
 		MESA_handle_runtime_log(g_tsg_para.logger,
 									RLOG_LV_DEBUG,
 									"SCAN_IP_ASN",
-									"IP_ASN is NULL scan ret: %d addr: %s",
+									"IP_ASN is NULL scan ret: %d table_name: %s addr: %s",
 									ret,
+									g_tsg_para.table_name[idx],
 									printaddr(&a_stream->addr, a_stream->threadnum)
 									);
 	}
@@ -773,17 +779,19 @@ int tsg_scan_ip_asn(Maat_feather_t maat_feather, const struct streaminfo *a_stre
 }
 
 
-int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _location_info *location, scan_status_t *mid, Maat_rule_t*result, int result_num)
+int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct _location_info *location, enum MASTER_TABLE idx, scan_status_t *mid, Maat_rule_t*result, int result_num)
 {
 	int ret=0;
+	char buff[1024]={0};
 	
 	if(location!=NULL)
 	{
+		snprintf(buff, sizeof(buff), "%s.%s.", location->country_full, location->city_full);
 		ret=Maat_full_scan_string(maat_feather,
-								g_tsg_para.table_id[TABLE_IP_LOCATION],
+								g_tsg_para.table_id[idx],
 								CHARSET_GBK,
-								location->country_full,
+								buff,
-								strlen(location->country_full),
+								strlen(buff),
 								result,
 								NULL,
 								result_num,
@@ -794,9 +802,10 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
 			MESA_handle_runtime_log(g_tsg_para.logger,
 									RLOG_LV_DEBUG,
 									"SCAN_IP_LOCATION",
-									"Hit IP_LOCATION: %s scan ret: %d policy_id: %d service: %d action: %d addr: %s",
+									"Hit IP_LOCATION: %s scan ret: %d table_name: %d policy_id: %d service: %d action: %d addr: %s",
-									location->country_full,
+									buff,
 									ret,
+									g_tsg_para.table_name[idx],
 									result[0].config_id,
 									result[0].service_id,
 									(unsigned char)result[0].action,
@@ -808,9 +817,10 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
 			MESA_handle_runtime_log(g_tsg_para.logger,
 									RLOG_LV_DEBUG,
 									"SCAN_IP_LOCATION",
-									"No hit IP_LOCATION: %s scan ret: %d addr: %s",
+									"No hit IP_LOCATION: %s scan ret: %d table_name: %s addr: %s",
-									location->country_full,
+									buff,
 									ret,
+									g_tsg_para.table_name[idx],
 									printaddr(&a_stream->addr, a_stream->threadnum)
 									);
 		}
@@ -820,8 +830,9 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
 		MESA_handle_runtime_log(g_tsg_para.logger,
 									RLOG_LV_DEBUG,
 									"SCAN_IP_LOCATION",
-									"IP_LOCATION is NULL scan ret: %d addr: %s",
+									"IP_LOCATION is NULL scan ret: %d table_name: %s addr: %s",
 									ret,
+									g_tsg_para.table_name[idx],
 									printaddr(&a_stream->addr, a_stream->threadnum)
 									);
 	}
@@ -981,8 +992,8 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 		tsg_get_ip_location(a_stream, g_tsg_para.table_id[TABLE_LOCATION_USER_DEFINED], (void **)&(internal_label->client_location), (void **)&(internal_label->server_location));
 		tsg_get_ip_location(a_stream, g_tsg_para.table_id[TABLE_LOCATION_BUILT_IN], (void **)&(internal_label->client_location), (void **)&(internal_label->server_location));
 		
-		hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->client_location, mid, result+hit_num, result_num-hit_num);
+		hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->client_location, TABLE_SRC_LOCATION, mid, result+hit_num, result_num-hit_num);
-		hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->server_location, mid, result+hit_num, result_num-hit_num);
+		hit_num+=tsg_scan_ip_location(maat_feather, a_stream, internal_label->server_location, TABLE_DST_LOCATION, mid, result+hit_num, result_num-hit_num);
 	}
 
 	if(hit_num<result_num)
@@ -990,8 +1001,8 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 		tsg_get_ip_asn(a_stream, g_tsg_para.table_id[TABLE_ASN_USER_DEFINED], (void **)&(internal_label->client_asn), (void **)&(internal_label->server_asn));
 		tsg_get_ip_asn(a_stream, g_tsg_para.table_id[TABLE_ASN_BUILT_IN], (void **)&(internal_label->client_asn), (void **)&(internal_label->server_asn));
 		
-		hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->client_asn, mid, result+hit_num, result_num-hit_num);
+		hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->client_asn, TABLE_SRC_ASN, mid, result+hit_num, result_num-hit_num);
-		hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->server_asn, mid, result+hit_num, result_num-hit_num);
+		hit_num+=tsg_scan_ip_asn(maat_feather, a_stream, internal_label->server_asn, TABLE_DST_ASN, mid, result+hit_num, result_num-hit_num);
 	}
 
 

src/tsg_send_log.cpp

@@ -191,14 +191,14 @@ int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD
 		if(internal_label->client_location!=NULL)
 		{
 			location=internal_label->client_location;
-			snprintf(buff, sizeof(buff), "%s%s%s%s%s", location->country_full, (strlen(location->province_full)>0) ? "/" : "", location->province_full, (strlen(location->city_full)>0) ? "/" : "", location->city_full);
+			snprintf(buff, sizeof(buff), "%s%s%s", location->country_full, (strlen(location->city_full)>0) ? "." : "", location->city_full);
 			TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_LOCATION].name, (void *)buff, TLD_TYPE_STRING);
 		}
 
 		if(internal_label->server_location!=NULL)
 		{
 			location=internal_label->server_location;
-			snprintf(buff, sizeof(buff), "%s%s%s%s%s", location->country_full, (strlen(location->province_full)>0) ? "/" : "", location->province_full, (strlen(location->city_full)>0) ? "/" : "", location->city_full);
+			snprintf(buff, sizeof(buff), "%s%s%s", location->country_full, (strlen(location->city_full)>0) ? "." : "", location->city_full);
 			TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_LOCATION].name, (void *)buff, TLD_TYPE_STRING);
 		}
 	}
@@ -312,6 +312,8 @@ int TLD_append_streaminfo(struct tsg_log_instance_t *instance, struct TLD_handle
 	
 	TLD_append(_handle, _instance->id2field[LOG_COMMON_ADDRESS_LIST].name, (void *)nest_addr_buf, TLD_TYPE_STRING);
 
+	set_common_field_from_label(_instance, _handle, a_stream);
+	
 	return 0;
 }
 
@@ -527,7 +529,6 @@ int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handl
 	TLD_append_streaminfo(instance, handle, log_msg->a_stream);
 	TLD_append(_handle, _instance->id2field[LOG_COMMON_SLED_IP].name, (void *)(_instance->local_ip_str), TLD_TYPE_STRING);
 	TLD_append(_handle, _instance->id2field[LOG_COMMON_DEVICE_ID].name, (void *)(g_tsg_para.device_sn), TLD_TYPE_STRING);
-	set_common_field_from_label(_instance, _handle, log_msg->a_stream);
 
 	for(i=0;i<log_msg->result_num; i++)
 	{