支持每隔N秒扫描一次，支持配置文件修改，默认10S

发送common_service_category日志字段

bin/tsg_log_field.conf

@@ -75,3 +75,6 @@ STRING		source		64
 STRING		destination		65
 STRING		client_ip		66
 STRING		server_ip		67
+STRING		c2s_id		68
+STRING		s2c_id		69
+STRING		common_service_category	70

inc/app_label.h

@@ -20,7 +20,7 @@ struct basic_proto_label
 
 enum APP_IDENTIFY_ORIGIN
 {
-	ORIGIN_BASIC_PROTOCOL,
+	ORIGIN_BASIC_PROTOCOL=0,
 	ORIGIN_USER_DEFINE,
 	ORIGIN_DKPT,
 	ORIGIN_QM_ENGINE,

src/tsg_entry.cpp

@@ -168,6 +168,22 @@ static int is_only_monitor(struct Maat_rule_t *result, int hit_cnt)
 	return 1;
 }
 
+static int is_hited_allow(struct Maat_rule_t *result, int hit_cnt)
+{
+	int i=0;
+
+	for(i=0; i<hit_cnt; i++)
+	{
+		if(result[i].action==TSG_ACTION_BYPASS)
+		{
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+
 static int set_drop_stream(const struct streaminfo *a_stream)
 {
 	int opt_value=1;
@@ -1104,14 +1120,6 @@ static unsigned char master_deal_scan_result(const struct streaminfo *a_stream,
 	unsigned char state=APP_STATE_GIVEME;
 
 	p_result=tsg_policy_decision_criteria(result, hit_num);
-	if(g_tsg_para.default_compile_switch==1 && p_result==NULL)
-	{
-		if(get_default_policy(g_tsg_para.default_compile_id, &result[0]))
-		{
-			p_result=&result[0];			
-		}
-	}
-	
 	if(p_result!=NULL)
 	{	
 		switch((unsigned char)p_result->action)
@@ -1120,7 +1128,17 @@ static unsigned char master_deal_scan_result(const struct streaminfo *a_stream,
 				state=deal_deny_action(a_stream, context, p_result);
 				if((state&APP_STATE_DROPPKT)==APP_STATE_DROPPKT)
 				{
+					context->hit_cnt=0;
 					master_send_log(a_stream, p_result, 1, context->domain, context->proto, a_stream->threadnum);
+					MESA_handle_runtime_log(g_tsg_para.logger,
+												RLOG_LV_DEBUG,
+												"DENY",
+												"Hit deny policy, policy_id: %d service: %d action: %d addr: %s",
+												p_result->config_id,
+												p_result->service_id,
+												(unsigned char)p_result->action,
+												PRINTADDR(a_stream, g_tsg_para.level)
+											);
 				}
 				break;
 			case	TSG_ACTION_MONITOR:
@@ -1188,6 +1206,7 @@ static int app_identify_result_cb(const struct streaminfo *a_stream, int bridge_
 	if(context==NULL)
 	{
 		init_context((void **)&context, a_stream->threadnum);
+		set_struct_project(a_stream, g_tsg_para.context_project_id, (void *)context);
 	}
 	
 	record_time_start(&context->last_scan_time);
@@ -1208,6 +1227,7 @@ static int master_deal_pending_state(const struct streaminfo *a_stream, struct m
 	ret=identify_application_protocol(a_stream, context, a_packet);
 	if(ret==1)
 	{				
+		set_session_attribute_label(a_stream, TSG_ATTRIBUTE_TYPE_ESTABLISH_LATECY, NULL, a_stream->threadnum);
 		set_session_attribute_label(a_stream, TSG_ATTRIBUTE_TYPE_PROTOCOL, (void *)&(context->proto), a_stream->threadnum);
 
 		if(context->proto==PROTO_SSL)
@@ -1244,21 +1264,24 @@ static int master_deal_pending_state(const struct streaminfo *a_stream, struct m
 	return hit_num;
 }
 
-extern "C" unsigned char TSG_MASTER_TCP_ENTRY(const struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet)
+
+static unsigned char tsg_master_entry(const struct streaminfo *a_stream, void **pme, int thread_seq,void *a_packet)
 {
-	int hit_num=0;	
+	int i=0, hit_num=0;	
 	unsigned char state=APP_STATE_GIVEME;
-	Maat_rule_t result[MAX_RESULT_NUM];
+	Maat_rule_t scan_result[MAX_RESULT_NUM];
+	Maat_rule_t *p_result=NULL;
+	struct gather_app_result *identify_result=NULL;
 	struct master_context *context=(struct master_context *)*pme;
 
 	if(*pme==NULL)
 	{
-		context=(struct master_context *)get_struct_project(a_tcp, g_tsg_para.context_project_id);
+		context=(struct master_context *)get_struct_project(a_stream, g_tsg_para.context_project_id);
 		if(context==NULL)
 		{
 			init_context(pme, thread_seq);
 			context=(struct master_context *)*pme;
-			set_struct_project(a_tcp, g_tsg_para.context_project_id, *pme);
+			set_struct_project(a_stream, g_tsg_para.context_project_id, *pme);
 		}
 		else
 		{
@@ -1267,70 +1290,79 @@ extern "C" unsigned char TSG_MASTER_TCP_ENTRY(const struct streaminfo *a_tcp, vo
 		record_time_start(&context->last_scan_time);
 	}
 	
-	if(a_tcp->opstate==OP_STATE_PENDING)
+	switch(a_stream->opstate)
 	{
-		FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_TCP_LINKS], 0, FS_OP_ADD, 1);			
+		case	OP_STATE_PENDING:
-		set_session_attribute_label(a_tcp, TSG_ATTRIBUTE_TYPE_ESTABLISH_LATECY, NULL, thread_seq);
+			hit_num+=master_deal_pending_state(a_stream, context, scan_result+hit_num, MAX_RESULT_NUM-hit_num, a_packet);
-		hit_num+=master_deal_pending_state(a_tcp, context, result-hit_num, MAX_RESULT_NUM-hit_num, a_packet);
+			p_result=tsg_policy_decision_criteria(scan_result, hit_num);
+			if(g_tsg_para.default_compile_switch==1 && p_result==NULL)
+			{
+				if(get_default_policy(g_tsg_para.default_compile_id, &scan_result[0]))
+				{
+					p_result=&scan_result[0];			
+				}
+			}
+			
+			state=master_deal_scan_result(a_stream, context, scan_result, hit_num, a_packet);
+			break;
+		case	OP_STATE_DATA:
+		case	OP_STATE_CLOSE:
+			if(is_hited_allow(context->result, context->hit_cnt))
+			{
+				break;
+			}
+			
+			if(record_time_elapse_us(&context->last_scan_time) < (g_tsg_para.scan_time_interval*1000000))
+			{
+				break;
+			}
+			
+			record_time_start(&context->last_scan_time);
+			hit_num+=tsg_scan_nesting_addr(g_tsg_maat_feather, a_stream, context->proto, &context->mid, scan_result+hit_num, MAX_RESULT_NUM-hit_num);
+			
+			identify_result=(struct gather_app_result *)get_struct_project(a_stream, g_tsg_para.gather_app_project_id);
+			for(i=0; i<ORIGIN_MAX && identify_result!=NULL; i++)
+			{
+				hit_num+=scan_application_id_and_properties(a_stream, scan_result+hit_num, MAX_RESULT_NUM-hit_num, &context->mid, &(identify_result->result[i]), thread_seq);
+			}
+
+			p_result=tsg_policy_decision_criteria(scan_result, hit_num);
+			if(p_result!=NULL && p_result->action!=TSG_ACTION_MONITOR)
+			{
+				state=master_deal_scan_result(a_stream, context, scan_result, hit_num, a_packet);
+			}
+			break;
+		default:
+			break;
 	}
 	
-	if(record_time_elapse_us(&context->last_scan_time)/100000 > g_tsg_para.scan_time_interval)
+	if((a_stream->opstate==OP_STATE_CLOSE) || (state&APP_STATE_DROPME)==APP_STATE_DROPME)
 	{
-		record_time_start(&context->last_scan_time);
+		close_stream_free_context(a_stream, context, thread_seq);
-	}
-
-	state=master_deal_scan_result(a_tcp, context, result, hit_num, a_packet);
-	if((a_tcp->opstate==OP_STATE_CLOSE) || (state&APP_STATE_DROPME)==APP_STATE_DROPME)
-	{
-		close_stream_free_context(a_tcp, context, thread_seq);
 		*pme=NULL;
 	}
 
 	return state;
 }
 
+extern "C" unsigned char TSG_MASTER_TCP_ENTRY(const struct streaminfo *a_tcp, void **pme, int thread_seq,void *a_packet)
+{
+	if(a_tcp->type==STREAM_TYPE_TCP)
+	{		
+		FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_TCP_LINKS], 0, FS_OP_ADD, 1); 		
+	}
+	
+	return tsg_master_entry(a_tcp, pme, thread_seq, a_packet);
+}
+
 extern "C" unsigned char TSG_MASTER_UDP_ENTRY(const struct streaminfo *a_udp, void **pme, int thread_seq,void *a_packet)
 {
-	int hit_num=0;	
+	if(a_udp->type==STREAM_TYPE_UDP)
-	unsigned char state=APP_STATE_GIVEME;
-	Maat_rule_t result[MAX_RESULT_NUM]={0};
-	struct master_context *context=(struct master_context *)*pme;
-
-	if(*pme==NULL)
-	{
-		context=(struct master_context *)get_struct_project(a_udp, g_tsg_para.context_project_id);
-		if(context==NULL)
-		{
-			init_context(pme, thread_seq);
-			context=(struct master_context *)*pme;
-			set_struct_project(a_udp, g_tsg_para.context_project_id, *pme);
-		}
-		else
-		{
-			*pme=(void *)context;
-		}
-		record_time_start(&context->last_scan_time);
-	}
-	
-	if(a_udp->opstate==OP_STATE_PENDING)
 	{		
 		FS_operate(g_tsg_para.fs2_handle, g_tsg_para.fs2_field_id[TSG_FS2_UDP_LINKS], 0, FS_OP_ADD, 1); 		
-		hit_num+=master_deal_pending_state(a_udp, context, result-hit_num, MAX_RESULT_NUM-hit_num, a_packet);
 	}
 
-	if(record_time_elapse_us(&context->last_scan_time)/100000 > g_tsg_para.scan_time_interval)
+	return tsg_master_entry(a_udp, pme, thread_seq, a_packet);
-	{
-		record_time_start(&context->last_scan_time);
-	}
-	
-	state=master_deal_scan_result(a_udp, context, result, hit_num, a_packet);
-	
-	if((a_udp->opstate==OP_STATE_CLOSE) || (state&APP_STATE_DROPME)==APP_STATE_DROPME)
-	{
-		close_stream_free_context(a_udp, context, thread_seq);
-		*pme=NULL;
-	}
-	return state;
 }
 
 extern "C" int TSG_MASTER_INIT()
@@ -1369,7 +1401,7 @@ extern "C" int TSG_MASTER_INIT()
 	MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "ENTRANCE_ID", &g_tsg_para.entrance_id, 0);
 	MESA_load_profile_short_def(tsg_conffile, "SYSTEM", "TIMEOUT", (short *)&g_tsg_para.timeout, 300);
 	MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "MAIL_PROTOCOL_ID",&(g_tsg_para.mail_proto_id), 110);	
-	MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "SCAN_TIME_INTERVAL", &g_tsg_para.scan_time_interval, 10);
+	MESA_load_profile_int_def(tsg_conffile, "SYSTEM", "SCAN_TIME_INTERVAL", &g_tsg_para.scan_time_interval, 5);
 
 	MESA_load_profile_string_def(tsg_conffile, "SYSTEM", "DEVICE_ID_COMMAND", g_tsg_para.device_id_command, sizeof(g_tsg_para.device_id_command), NULL);
 	g_tsg_para.device_id=get_device_id(g_tsg_para.device_id_command, g_tsg_para.entrance_id);

src/tsg_entry.h

@@ -255,6 +255,7 @@ void ASN_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp)
 void subscribe_id_free_data(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void* argp);
 void security_compile_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp);
 struct Maat_rule_t *tsg_policy_decision_criteria(struct streaminfo *a_stream, Maat_rule_t *result, int result_num, int thread_seq);
+int tsg_scan_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num);
 int tsg_scan_shared_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, char *domain, Maat_rule_t *result, int result_num,  scan_status_t *mid, int table_id, int thread_seq);
 int tsg_scan_app_id_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *name, unsigned int id, int thread_seq);
 int tsg_scan_app_properties_policy(Maat_feather_t maat_feather, const struct streaminfo *a_stream, struct Maat_rule_t *result, int result_num, scan_status_t *mid, char *property, char *district, int thread_seq);

src/tsg_rule.cpp

@@ -1152,22 +1152,13 @@ int tsg_scan_ip_location(Maat_feather_t maat_feather, const struct streaminfo *a
 	return 0;
 }
 
-int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num)
+int tsg_scan_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num)
 {
-	int ret=0;
+	int hit_num=0,tans_proto=0;
-	unsigned int proto_id=0;
 	struct ipaddr t_addr;
 	struct ipaddr* p_addr=NULL;	
-	int hit_num=0,tans_proto=0;
 	int is_scan_addr=1, maat_ret=0;	
 	const struct streaminfo *cur_stream = a_stream;
-	struct _session_attribute_label_t *attribute_label=NULL;
-
-	if(result==NULL || result_num<=0 || a_stream==NULL || maat_feather==NULL)
-	{
-		MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "SCAN_NESTING_ADDR", "result==NULL || result_num<=0 || maat_feather==NULL || a_stream==NULL");
-		return -1;
-	}
 
 	do
 	{
@@ -1235,6 +1226,24 @@ int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *
 		
 	}while(cur_stream != NULL && hit_num < result_num);
 
+	return hit_num;
+}
+
+int tsg_scan_nesting_addr(Maat_feather_t maat_feather, const struct streaminfo *a_stream, tsg_protocol_t proto, scan_status_t *mid, Maat_rule_t*result, int result_num)
+{
+	int ret=0;
+	unsigned int proto_id=0;
+	int hit_num=0;
+	struct _session_attribute_label_t *attribute_label=NULL;
+
+	if(result==NULL || result_num<=0 || a_stream==NULL || maat_feather==NULL)
+	{
+		MESA_handle_runtime_log(g_tsg_para.logger, RLOG_LV_DEBUG, "SCAN_NESTING_ADDR", "result==NULL || result_num<=0 || maat_feather==NULL || a_stream==NULL");
+		return -1;
+	}
+
+	hit_num+=tsg_scan_addr(maat_feather, a_stream, proto, mid, result+hit_num, result_num-hit_num);
+
 	if(hit_num<result_num && proto>PROTO_UNKONWN && proto<PROTO_MAX)
 	{
 		proto_id=tsg_l7_protocol_name2id(g_tsg_proto_name2id[proto].name);

src/tsg_send_log.cpp

@@ -112,6 +112,34 @@ static int set_tcp_isn(struct tsg_log_instance_t *_instance, struct TLD_handle_t
 	
 	return 1;
 }
+
+static int set_asn(struct TLD_handle_t *_handle, struct streaminfo *a_stream, char *field_name, struct _asn_info_t *asn_info)
+{
+	char buff[1024]={0};
+	
+	if(asn_info!=NULL)
+	{
+		snprintf(buff, sizeof(buff), "%s(%s)", asn_info->asn, asn_info->organization);
+		TLD_append(_handle, field_name, (void *)buff, TLD_TYPE_STRING);
+	}
+
+	return 1;
+}
+
+static int set_location(struct TLD_handle_t *_handle, struct streaminfo *a_stream, char *field_name, struct _location_info_t *location_info)
+{
+	char buff[1024]={0};
+	
+	if(location_info!=NULL)
+	{
+		snprintf(buff, sizeof(buff), "%s,%s,%s", location_info->city_full, location_info->province_full, location_info->country_full);
+		TLD_append(_handle, field_name, (void *)buff, TLD_TYPE_STRING);
+	}
+
+	return 1;
+}
+
+
 static int set_direction(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
 {
 	int direction=0,i_or_e=0;
@@ -256,6 +284,24 @@ static int set_duraction(struct tsg_log_instance_t *_instance, struct TLD_handle
 	return 1;
 }
 
+static int set_fqdn_category(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream, unsigned int *category_id, int category_id_num)
+{
+	int i=0,offset=0;
+	char category_str[128]={0};
+	
+	for(i=0; i< category_id_num; i++)
+	{
+		offset+=snprintf(category_str+offset, sizeof(category_str)-offset, "%d,", category_id[i]);
+	}
+
+	if(offset>0)
+	{
+		TLD_append(_handle, _instance->id2field[LOG_COMMON_FQDN_CATEGORY].name, (void *)category_str, TLD_TYPE_STRING);
+	}
+
+	return 1;
+}
+
 
 static int set_packet_bytes(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
 {
@@ -775,8 +821,6 @@ static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name,
 
 int set_session_attributes(struct tsg_log_instance_t *_instance, struct TLD_handle_t *_handle, struct streaminfo *a_stream)
 {
-	char buff[1024]={0};
-	struct _location_info_t *location=NULL;
 	struct _session_attribute_label_t *attribute_label=NULL;
 
 	attribute_label=(struct _session_attribute_label_t *)project_req_get_struct(a_stream, _instance->internal_project_id);
@@ -784,31 +828,13 @@ int set_session_attributes(struct tsg_log_instance_t *_instance, struct TLD_hand
 	{
 		TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)attribute_label->establish_latency_ms, TLD_TYPE_LONG);
 
-		if(attribute_label->client_asn!=NULL)
+		set_asn(_handle, a_stream, _instance->id2field[LOG_COMMON_CLINET_ASN].name, attribute_label->client_asn);		
-		{
+		set_asn(_handle, a_stream, _instance->id2field[LOG_COMMON_SERVER_ASN].name, attribute_label->server_asn);
-			snprintf(buff, sizeof(buff), "%s(%s)", attribute_label->client_asn->asn, attribute_label->client_asn->organization);
-			TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_ASN].name, (void *)buff, TLD_TYPE_STRING);
-		}
 		
-		if(attribute_label->server_asn!=NULL)
+		set_location(_handle, a_stream, _instance->id2field[LOG_COMMON_CLINET_LOCATION].name, attribute_label->client_location);
-		{
+		set_location(_handle, a_stream, _instance->id2field[LOG_COMMON_SERVER_LOCATION].name, attribute_label->server_location);
-			snprintf(buff, sizeof(buff), "%s(%s)", attribute_label->server_asn->asn, attribute_label->server_asn->organization);
-			TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_ASN].name, (void *)buff, TLD_TYPE_STRING);
-		}
 		
-		if(attribute_label->client_location!=NULL)
+		set_fqdn_category(_instance, _handle, a_stream, attribute_label->fqdn_category_id, attribute_label->fqdn_category_id_num);
-		{
-			location=attribute_label->client_location;
-			snprintf(buff, sizeof(buff), "%s,%s,%s", location->city_full, location->province_full, location->country_full);
-			TLD_append(_handle, _instance->id2field[LOG_COMMON_CLINET_LOCATION].name, (void *)buff, TLD_TYPE_STRING);
-		}
-
-		if(attribute_label->server_location!=NULL)
-		{
-			location=attribute_label->server_location;
-			snprintf(buff, sizeof(buff), "%s,%s,%s", location->city_full, location->province_full, location->country_full);
-			TLD_append(_handle, _instance->id2field[LOG_COMMON_SERVER_LOCATION].name, (void *)buff, TLD_TYPE_STRING);
-		}
 
 		if(attribute_label->ja3_fingerprint!=NULL)
 		{

src/tsg_send_log_internal.h

@@ -95,6 +95,7 @@ typedef enum _tsg_log_field_id
 	LOG_COMMON_TUNNELS_SERVER_IP,
 	LOG_COMMON_TUNNELS_PPTP_C2S_ID,
 	LOG_COMMON_TUNNELS_PPTP_S2C_ID,
+	LOG_COMMON_FQDN_CATEGORY,
 	LOG_COMMON_MAX
 }tsg_log_field_id_t;