同一发送common_l7_protocol字段

2020-11-14 15:50:44 +06:00
parent 39335c9543
commit 23f1e9238c
7 changed files with 280 additions and 14 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -41,10 +41,6 @@ SET(CPACK_RPM_LIBRARIES_PRE_UNINSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/cmake/P

 set(CPACK_RPM_LIBRARIES_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/main.conf"
                           		 "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/maat.conf"
-			 "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/tsg_static_tableinfo.conf"
-			 "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/tsg_dynamic_tableinfo.conf"
-			 "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/tsg_log_field.conf"
-			 "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/tsg_maat.json"
 )

 install(FILES bin/main.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
@@ -52,10 +48,13 @@ install(FILES bin/maat.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONEN
 install(FILES bin/tsg_static_tableinfo.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
 install(FILES bin/tsg_dynamic_tableinfo.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
 install(FILES bin/tsg_log_field.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
+install(FILES bin/app_l7_proto_id.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
 install(FILES bin/tsg_maat.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE)
+
 install(FILES inc/tsg_send_log.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER)
 install(FILES inc/tsg_rule.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER)
 install(FILES inc/tsg_statistic.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER)
 install(FILES inc/tsg_label.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER)
+install(FILES inc/app_label.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER)

 include(Package)
--- a/bin/app_l7_proto_id.conf
+++ b/bin/app_l7_proto_id.conf
@@ -0,0 +1,51 @@
+#TYPE：1:UCHAR,2:USHORT,3:USTRING,4:ULOG,5:USTRING,6:FILE,7:UBASE64,8:PACKET
+#TYPE	FIELD		VALUE
+STRING	UNKNOWN_TCP	100
+STRING	UNKNOWN_UDP	101
+STRING	UNKNOWN_OTHER	102
+STRING	DNS		103
+STRING	FTP		104
+STRING	FTPS		105
+STRING	HTTP		106
+STRING	HTTPS		107
+STRING	ICMP		108
+STRING	IKE		109
+STRING	MAIL		110
+STRING	IMAPS		111
+STRING	IPSEC		112
+STRING	XMPP		113
+STRING	L2TP		114
+STRING	NTP		115
+STRING	POP3S		117
+STRING	PPTP		118
+STRING	QUIC		119
+STRING	SIP		120
+STRING	SMB		121
+STRING	SMTPS		123
+STRING	SPDY		124
+STRING	SSH		125
+STRING	SSL		126
+STRING	SOCKS		127
+STRING	TELNET		128
+STRING	DHCP		129
+STRING	RADIUS		130
+STRING	OPENVPN		131
+STRING	STUN		132
+STRING	TEREDO		133
+STRING	DTLS		134
+STRING	DOH		135
+STRING	ISAKMP		136
+STRING	MDNS		137
+STRING	NETBIOS		138
+STRING	NETFLOW		139
+STRING	RDP		140
+STRING	RTCP		141
+STRING	RTP		142
+STRING	SLP		143
+STRING	SNMP		144
+STRING	SSDP		145
+STRING	TFTP		146
+STRING	BJNP		147
+STRING	LDAP		148
+STRING	RTMP		149
+STRING	RTSP		150
--- a/inc/app_label.h
+++ b/inc/app_label.h
@@ -0,0 +1,157 @@
+#ifndef	__APP_LABEL_H__
+#define	__APP_LABEL_H__
+
+#define	MAX_APP_ID_PROPERTY_LEN	128
+
+
+struct _quic_attribute_t
+{
+	char *version;
+	char *sni;
+	char *user_agent;
+};
+
+struct _dns_attribute_t
+{
+	unsigned short id;
+	unsigned short flag;
+	unsigned short qdcount;
+	unsigned short ancount;
+	unsigned short aucount;//authority count
+	unsigned short adcount;//additional count
+
+	unsigned short qtype;
+	unsigned short qclass;
+	char *qname;
+};
+
+struct _http_attribute_t
+{
+	char *host;
+	char *uri;
+	char *user_agent;
+	char *content_type;
+	char *content_encoding;
+	char *referer;
+	char *cookie;
+	char *set_cookie;
+};
+
+struct _ssl_attribute_t
+{
+	char *cn;
+	char *sni;
+	char *san;
+	char *version;
+	char *fingerprint;
+	char *serial_number;
+	char *issuer_common_name;
+	char *issuer_organition_name;
+	char *issuer_country_name;
+	char *subject_common_name;
+	char *subject_organition_name;
+	char *subject_country_name;
+	char *not_valid_before;
+	char *not_valid_after;
+	char *algotithm_id;
+	char *ja3_fingerprint;
+};
+
+struct _app_id_dict_t
+{
+	int ref_cnt;
+	int app_id;
+	int deny_action;
+	int continue_scanning;
+	int tcp_timeout;
+	int udp_timeout;
+	int tcp_time_wait;
+	int tcp_half_close;
+	int is_valid;
+	char risk[MAX_APP_ID_PROPERTY_LEN*4];
+	char app_name[MAX_APP_ID_PROPERTY_LEN];
+	char category[MAX_APP_ID_PROPERTY_LEN*4];
+	char subcategroy[MAX_APP_ID_PROPERTY_LEN*4];
+	char technology[MAX_APP_ID_PROPERTY_LEN*4];
+	char characteristics[MAX_APP_ID_PROPERTY_LEN*4];
+	char depends_on_app_ids[MAX_APP_ID_PROPERTY_LEN];
+	char implicitly_uses_app_ids[MAX_APP_ID_PROPERTY_LEN];
+};
+
+
+#define	MAX_APP_ID_NUM	8
+
+enum _APP_SESSION_PROTO
+{
+	SESSION_PROTO_HTTP=1,
+	SESSION_PROTO_SSL,
+	SESSION_PROTO_QUIC,
+	SESSION_PROTO_DNS,
+	_SESSION_PROTO_MAX
+};
+
+enum _APP_SESSION_FLAG
+{
+	SESSION_FLAG_TCP_CREATE_WITH_SYN=1,
+	SESSION_FLAG_SSL_USE_SELFSIGNED_CERT=2,
+	SESSION_FLAG_SSL_INCOMPLETE_CERT_CHAIN=4
+};
+
+
+typedef	struct _app_id_label_add_flag_t
+{
+	volatile char flag;
+}APP_ADD_ID_LABEL_T;
+
+struct _app_id_label_t
+{
+	char ref_cnt;
+	char session_flag; // enum SESSION_FLAG
+	char session_proto; //enum _APP_SESSION_PROTO
+	char app_id_num;
+	APP_ADD_ID_LABEL_T flag; //0: no; 1: yes
+	char pad[3];
+	int surrogate_id;
+	int app_id;
+	struct _app_id_dict_t *dict;
+	union
+	{
+		struct _ssl_attribute_t *ssl_attribute;
+		struct _dns_attribute_t *dns_attribute;
+		struct _http_attribute_t *http_attribute;
+		struct _quic_attribute_t *quic_attribute;
+		void *attribute; //quic sni or dns query qname
+	};
+};
+
+struct _basic_proto_label
+{
+	unsigned char continue_scan_flag;  //0: stop; 1: continue
+	unsigned char pad;
+	unsigned short proto_id;
+};
+
+enum _ATTRIBUTE_TYPE
+{
+	ATTRIBUTE_TYPE_UNKNOWN,
+	ATTRIBUTE_TYPE_IP,
+	ATTRIBUTE_TYPE_STRING,
+	ATTRIBUTE_TYPE_NUMERIC,
+	ATTRIBUTE_TYPE_BOOL,
+	_ATTRIBUTE_TYPE_MAX
+};
+
+struct _attribute_kv_t
+{
+	enum _ATTRIBUTE_TYPE type;
+	char *name;
+	void *value;
+};
+
+struct _user_defined_attribute_label_t
+{
+	int attribute_num;
+	struct _attribute_kv_t *attribute;
+};
+
+#endif
--- a/inc/tsg_send_log.h
+++ b/inc/tsg_send_log.h
@@ -36,6 +36,6 @@ int TLD_cancel(struct TLD_handle_t *handle);
 int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, tsg_log_t *log_msg, int thread_id);

 unsigned long long tsg_get_stream_id(struct streaminfo *a_stream);
-
+char *tsg_l7_protocol_id2name(struct tsg_log_instance_t *instance, unsigned short id);

 #endif
--- a/src/tsg_entry.cpp
+++ b/src/tsg_entry.cpp
@@ -186,6 +186,9 @@ static int init_context(void **pme, tsg_protocol_t proto, struct Maat_rule_t *p_
 	*pme=dictator_malloc(thread_seq, sizeof(struct _master_context));
 	_context=(struct _master_context *)*pme;
 	_context->proto=proto;
+	_context->domain_len=0;
+	memset(_context->domain, 0, sizeof(_context->domain));
+	
 	_context->hit_cnt=1;
 	_context->result=(struct Maat_rule_t *)dictator_malloc(thread_seq, sizeof(struct Maat_rule_t));
 	memcpy(_context->result, p_result, sizeof(struct Maat_rule_t));
--- a/src/tsg_send_log.cpp
+++ b/src/tsg_send_log.cpp
@@ -16,6 +16,7 @@
 #include <MESA/MESA_prof_load.h>
 #include <MESA/MESA_handle_logger.h>

+#include "app_label.h"
 #include "tsg_entry.h"
 #include "tsg_send_log.h"
 #include "tsg_send_log_internal.h"
@@ -150,6 +151,18 @@ char *log_field_id2name(struct tsg_log_instance_t *instance, tsg_log_field_id_t
 	return NULL;
 }

+char *tsg_l7_protocol_id2name(struct tsg_log_instance_t *instance, unsigned short id)
+{
+	struct tsg_log_instance_t *_instance=instance;
+	
+	if(_instance!=NULL && id>=MIN_L7_PROTO_ID && id<=MAX_L7_PROTO_ID)
+	{
+		return _instance->l7_proto_id2field[id].name;
+	}
+
+	return NULL;
+}
+
 static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name, struct Maat_rule_t *p_result)
 {
 	cJSON *item=NULL;
@@ -189,17 +202,34 @@ int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD
 {
 	char buff[1024]={0};
 	char *l7_protocol=NULL;
+	struct _basic_proto_label *l7_proto_label=NULL;
 	struct _location_info_t *location=NULL;
 	struct _session_attribute_label_t *internal_label=NULL;

 	internal_label=(struct _session_attribute_label_t *)project_req_get_struct(a_stream, _instance->internal_project_id);
 	if(internal_label!=NULL)
+	{		
+		l7_proto_label=(struct _basic_proto_label *)project_req_get_struct(a_stream, _instance->l7_proto_project_id);
+		if(l7_proto_label!=NULL)
+		{
+			l7_protocol=tsg_l7_protocol_id2name(_instance, l7_proto_label->proto_id);
+			if(l7_protocol!=NULL)
+			{
+				TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)l7_protocol, TLD_TYPE_STRING);
+			}
+		}
+		else
 		{
 			l7_protocol=tsg_schema_index2string(internal_label->proto);
 			if(l7_protocol!=NULL)
 			{			
 				TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)l7_protocol, TLD_TYPE_STRING);
 			}
+			else
+			{
+				TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)"UNCATEGORIZED", TLD_TYPE_STRING);
+			}
+		}
 		
 		TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)internal_label->establish_latency_ms, TLD_TYPE_LONG);

@@ -234,6 +264,10 @@ int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD
 			TLD_append(_handle, _instance->id2field[LOG_SSL_JA3_FINGERPRINT].name, (void *)internal_label->ja3_fingerprint, TLD_TYPE_STRING);
 		}
 	}
+	else
+	{
+		TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)"UNCATEGORIZED", TLD_TYPE_STRING);
+	}

 	return 0;
 }
@@ -430,7 +464,7 @@ int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t
 		}
 		memset(type_name, 0, sizeof(type_name));
 		ret=sscanf(line, "%s %s %d", type_name, field_name, &id);
-		assert(ret==3 && id<LOG_COMMON_MAX);
+		assert(ret==3);

 		for(i=0; i<TLD_TYPE_MAX; i++)
 		{
@@ -487,7 +521,10 @@ int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t
 	fclose(fp);
 	fp=NULL;

+	if(service2topic!=NULL)
+	{
 		*service2topic=_service2topic;
+	}

 	return 0;
 }
@@ -495,6 +532,7 @@ int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t
 struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
 {
 	int i=0,ret=0;
+	int tmp_value=0;
 	char nic_name[32]={0};
 	char kafka_errstr[1024]={0};
 	unsigned int local_ip_nr=0;
@@ -543,6 +581,7 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)

 	MESA_load_profile_string_def(conffile, "TSG_LOG", "TCP_LABEL", _instance->tcp_label, sizeof(_instance->tcp_label), "tcp_flow_stat");
 	MESA_load_profile_string_def(conffile, "TSG_LOG", "UDP_LABEL", _instance->udp_label, sizeof(_instance->udp_label), "udp_flow_stat");
+	MESA_load_profile_string_def(conffile, "TSG_LOG", "L7_PROTO_LABEL", _instance->l7_proto_label, sizeof(_instance->l7_proto_label), "BASIC_PROTO_LABEL");

 	_instance->tcp_flow_project_id=project_customer_register(_instance->tcp_label, "struct");
 	_instance->udp_flow_project_id=project_customer_register(_instance->udp_label, "struct");
@@ -557,6 +596,16 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
 								);
 	}
 	
+	_instance->l7_proto_project_id=project_customer_register(_instance->l7_proto_label, "struct");
+	if(_instance->l7_proto_project_id<0)
+	{
+		MESA_handle_runtime_log(_instance->logger,
+									RLOG_LV_FATAL,
+									"L7_PROTO_LABEL",
+									"project_customer_register is error, l7_proto_label: %s, please check etc/project.conf",
+									_instance->l7_proto_label
+								);	
+	}
 	
 	MESA_load_profile_string_def(conffile, "TSG_LOG", "NIC_NAME", nic_name, sizeof(nic_name), "eth0");	
 	ret=MESA_get_dev_ipv4(nic_name, (int *)&local_ip_nr);
@@ -572,6 +621,8 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile)
 	}
 	inet_ntop(AF_INET,&(local_ip_nr),_instance->local_ip_str,sizeof(_instance->local_ip_str));
 	
+	MESA_load_profile_string_def(conffile, "TSG_LOG", "L7_PROTO_ID_FILE", _instance->l7_proto_id_file, sizeof(_instance->l7_proto_id_file), "./tsgconf/app_l7_proto_id.conf");
+	load_log_common_field(_instance->l7_proto_id_file, _instance->l7_proto_id2field, NULL, &tmp_value);

 	rdkafka_conf = rd_kafka_conf_new();	
 	rd_kafka_conf_set(rdkafka_conf, "queue.buffering.max.messages", _instance->send_queue_max_msg, kafka_errstr, sizeof(kafka_errstr));
--- a/src/tsg_send_log_internal.h
+++ b/src/tsg_send_log_internal.h
@@ -5,7 +5,8 @@
 #include <MESA/cJSON.h>
 #include <time.h>

-
+#define	MIN_L7_PROTO_ID	100
+#define	MAX_L7_PROTO_ID	150

 #define	MAX_IPV4_LEN	16
 #define	MAX_STRING_LEN	32
@@ -93,6 +94,7 @@ struct tsg_log_instance_t
 	int level;
 	int max_service;	
 	int recovery_interval;
+	int l7_proto_project_id;
 	int internal_project_id;
 	int tcp_flow_project_id;
 	int udp_flow_project_id;
@@ -102,15 +104,18 @@ struct tsg_log_instance_t
 	char log_path[MAX_STRING_LEN*2];
 	char tcp_label[MAX_STRING_LEN];
 	char udp_label[MAX_STRING_LEN];
+	char l7_proto_label[MAX_STRING_LEN];
 	char common_field_file[MAX_STRING_LEN*4];
 	char broker_list[MAX_STRING_LEN*4];
 	char send_queue_max_msg[MAX_STRING_LEN];
 	char require_ack[MAX_STRING_LEN];
 	char refresh_interval_ms[MAX_STRING_LEN];
 	char local_ip_str[MAX_IPV4_LEN];
+	char l7_proto_id_file[MAX_STRING_LEN*4];
 	id2field_t id2field[LOG_COMMON_MAX];
 	rd_kafka_topic_t **topic_rkt;
 	id2field_t *service2topic;	
+	id2field_t l7_proto_id2field[MAX_L7_PROTO_ID+1];
 	void *logger;
 };