diff --git a/CMakeLists.txt b/CMakeLists.txt index 8190556..6d051ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,10 +41,6 @@ SET(CPACK_RPM_LIBRARIES_PRE_UNINSTALL_SCRIPT_FILE "${PROJECT_SOURCE_DIR}/cmake/P set(CPACK_RPM_LIBRARIES_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/main.conf" "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/maat.conf" - "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/tsg_static_tableinfo.conf" - "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/tsg_dynamic_tableinfo.conf" - "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/tsg_log_field.conf" - "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/tsgconf/tsg_maat.json" ) install(FILES bin/main.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) @@ -52,10 +48,13 @@ install(FILES bin/maat.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONEN install(FILES bin/tsg_static_tableinfo.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) install(FILES bin/tsg_dynamic_tableinfo.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) install(FILES bin/tsg_log_field.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) +install(FILES bin/app_l7_proto_id.conf DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) install(FILES bin/tsg_maat.json DESTINATION ${CMAKE_INSTALL_PREFIX}/tsgconf COMPONENT PROFILE) + install(FILES inc/tsg_send_log.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER) install(FILES inc/tsg_rule.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER) install(FILES inc/tsg_statistic.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER) install(FILES inc/tsg_label.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER) +install(FILES inc/app_label.h DESTINATION /opt/MESA/include/tsg COMPONENT HEADER) include(Package) diff --git a/bin/app_l7_proto_id.conf b/bin/app_l7_proto_id.conf new file mode 100644 index 0000000..6be9dff --- /dev/null +++ b/bin/app_l7_proto_id.conf @@ -0,0 +1,51 @@ +#TYPE：1:UCHAR,2:USHORT,3:USTRING,4:ULOG,5:USTRING,6:FILE,7:UBASE64,8:PACKET +#TYPE FIELD VALUE +STRING UNKNOWN_TCP 100 +STRING UNKNOWN_UDP 101 +STRING UNKNOWN_OTHER 102 +STRING DNS 103 +STRING FTP 104 +STRING FTPS 105 +STRING HTTP 106 +STRING HTTPS 107 +STRING ICMP 108 +STRING IKE 109 +STRING MAIL 110 +STRING IMAPS 111 +STRING IPSEC 112 +STRING XMPP 113 +STRING L2TP 114 +STRING NTP 115 +STRING POP3S 117 +STRING PPTP 118 +STRING QUIC 119 +STRING SIP 120 +STRING SMB 121 +STRING SMTPS 123 +STRING SPDY 124 +STRING SSH 125 +STRING SSL 126 +STRING SOCKS 127 +STRING TELNET 128 +STRING DHCP 129 +STRING RADIUS 130 +STRING OPENVPN 131 +STRING STUN 132 +STRING TEREDO 133 +STRING DTLS 134 +STRING DOH 135 +STRING ISAKMP 136 +STRING MDNS 137 +STRING NETBIOS 138 +STRING NETFLOW 139 +STRING RDP 140 +STRING RTCP 141 +STRING RTP 142 +STRING SLP 143 +STRING SNMP 144 +STRING SSDP 145 +STRING TFTP 146 +STRING BJNP 147 +STRING LDAP 148 +STRING RTMP 149 +STRING RTSP 150 diff --git a/inc/app_label.h b/inc/app_label.h new file mode 100644 index 0000000..b80da8c --- /dev/null +++ b/inc/app_label.h @@ -0,0 +1,157 @@ +#ifndef __APP_LABEL_H__ +#define __APP_LABEL_H__ + +#define MAX_APP_ID_PROPERTY_LEN 128 + + +struct _quic_attribute_t +{ + char *version; + char *sni; + char *user_agent; +}; + +struct _dns_attribute_t +{ + unsigned short id; + unsigned short flag; + unsigned short qdcount; + unsigned short ancount; + unsigned short aucount;//authority count + unsigned short adcount;//additional count + + unsigned short qtype; + unsigned short qclass; + char *qname; +}; + +struct _http_attribute_t +{ + char *host; + char *uri; + char *user_agent; + char *content_type; + char *content_encoding; + char *referer; + char *cookie; + char *set_cookie; +}; + +struct _ssl_attribute_t +{ + char *cn; + char *sni; + char *san; + char *version; + char *fingerprint; + char *serial_number; + char *issuer_common_name; + char *issuer_organition_name; + char *issuer_country_name; + char *subject_common_name; + char *subject_organition_name; + char *subject_country_name; + char *not_valid_before; + char *not_valid_after; + char *algotithm_id; + char *ja3_fingerprint; +}; + +struct _app_id_dict_t +{ + int ref_cnt; + int app_id; + int deny_action; + int continue_scanning; + int tcp_timeout; + int udp_timeout; + int tcp_time_wait; + int tcp_half_close; + int is_valid; + char risk[MAX_APP_ID_PROPERTY_LEN*4]; + char app_name[MAX_APP_ID_PROPERTY_LEN]; + char category[MAX_APP_ID_PROPERTY_LEN*4]; + char subcategroy[MAX_APP_ID_PROPERTY_LEN*4]; + char technology[MAX_APP_ID_PROPERTY_LEN*4]; + char characteristics[MAX_APP_ID_PROPERTY_LEN*4]; + char depends_on_app_ids[MAX_APP_ID_PROPERTY_LEN]; + char implicitly_uses_app_ids[MAX_APP_ID_PROPERTY_LEN]; +}; + + +#define MAX_APP_ID_NUM 8 + +enum _APP_SESSION_PROTO +{ + SESSION_PROTO_HTTP=1, + SESSION_PROTO_SSL, + SESSION_PROTO_QUIC, + SESSION_PROTO_DNS, + _SESSION_PROTO_MAX +}; + +enum _APP_SESSION_FLAG +{ + SESSION_FLAG_TCP_CREATE_WITH_SYN=1, + SESSION_FLAG_SSL_USE_SELFSIGNED_CERT=2, + SESSION_FLAG_SSL_INCOMPLETE_CERT_CHAIN=4 +}; + + +typedef struct _app_id_label_add_flag_t +{ + volatile char flag; +}APP_ADD_ID_LABEL_T; + +struct _app_id_label_t +{ + char ref_cnt; + char session_flag; // enum SESSION_FLAG + char session_proto; //enum _APP_SESSION_PROTO + char app_id_num; + APP_ADD_ID_LABEL_T flag; //0: no; 1: yes + char pad[3]; + int surrogate_id; + int app_id; + struct _app_id_dict_t *dict; + union + { + struct _ssl_attribute_t *ssl_attribute; + struct _dns_attribute_t *dns_attribute; + struct _http_attribute_t *http_attribute; + struct _quic_attribute_t *quic_attribute; + void *attribute; //quic sni or dns query qname + }; +}; + +struct _basic_proto_label +{ + unsigned char continue_scan_flag; //0: stop; 1: continue + unsigned char pad; + unsigned short proto_id; +}; + +enum _ATTRIBUTE_TYPE +{ + ATTRIBUTE_TYPE_UNKNOWN, + ATTRIBUTE_TYPE_IP, + ATTRIBUTE_TYPE_STRING, + ATTRIBUTE_TYPE_NUMERIC, + ATTRIBUTE_TYPE_BOOL, + _ATTRIBUTE_TYPE_MAX +}; + +struct _attribute_kv_t +{ + enum _ATTRIBUTE_TYPE type; + char *name; + void *value; +}; + +struct _user_defined_attribute_label_t +{ + int attribute_num; + struct _attribute_kv_t *attribute; +}; + +#endif diff --git a/inc/tsg_send_log.h b/inc/tsg_send_log.h index 711f45b..6c76568 100644 --- a/inc/tsg_send_log.h +++ b/inc/tsg_send_log.h @@ -36,6 +36,6 @@ int TLD_cancel(struct TLD_handle_t *handle); int tsg_send_log(struct tsg_log_instance_t *instance, struct TLD_handle_t *handle, tsg_log_t *log_msg, int thread_id); unsigned long long tsg_get_stream_id(struct streaminfo *a_stream); - +char *tsg_l7_protocol_id2name(struct tsg_log_instance_t *instance, unsigned short id); #endif diff --git a/src/tsg_entry.cpp b/src/tsg_entry.cpp index cd8b738..0c3309d 100644 --- a/src/tsg_entry.cpp +++ b/src/tsg_entry.cpp @@ -186,6 +186,9 @@ static int init_context(void **pme, tsg_protocol_t proto, struct Maat_rule_t *p_ *pme=dictator_malloc(thread_seq, sizeof(struct _master_context)); _context=(struct _master_context *)*pme; _context->proto=proto; + _context->domain_len=0; + memset(_context->domain, 0, sizeof(_context->domain)); + _context->hit_cnt=1; _context->result=(struct Maat_rule_t *)dictator_malloc(thread_seq, sizeof(struct Maat_rule_t)); memcpy(_context->result, p_result, sizeof(struct Maat_rule_t)); diff --git a/src/tsg_send_log.cpp b/src/tsg_send_log.cpp index e371969..4aa160a 100644 --- a/src/tsg_send_log.cpp +++ b/src/tsg_send_log.cpp @@ -16,6 +16,7 @@ #include #include +#include "app_label.h" #include "tsg_entry.h" #include "tsg_send_log.h" #include "tsg_send_log_internal.h" @@ -150,6 +151,18 @@ char *log_field_id2name(struct tsg_log_instance_t *instance, tsg_log_field_id_t return NULL; } +char *tsg_l7_protocol_id2name(struct tsg_log_instance_t *instance, unsigned short id) +{ + struct tsg_log_instance_t *_instance=instance; + + if(_instance!=NULL && id>=MIN_L7_PROTO_ID && id<=MAX_L7_PROTO_ID) + { + return _instance->l7_proto_id2field[id].name; + } + + return NULL; +} + static int set_common_sub_action(struct TLD_handle_t *handle, char *field_name, struct Maat_rule_t *p_result) { cJSON *item=NULL; @@ -189,16 +202,33 @@ int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD { char buff[1024]={0}; char *l7_protocol=NULL; + struct _basic_proto_label *l7_proto_label=NULL; struct _location_info_t *location=NULL; struct _session_attribute_label_t *internal_label=NULL; internal_label=(struct _session_attribute_label_t *)project_req_get_struct(a_stream, _instance->internal_project_id); if(internal_label!=NULL) { - l7_protocol=tsg_schema_index2string(internal_label->proto); - if(l7_protocol!=NULL) - { - TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)l7_protocol, TLD_TYPE_STRING); + l7_proto_label=(struct _basic_proto_label *)project_req_get_struct(a_stream, _instance->l7_proto_project_id); + if(l7_proto_label!=NULL) + { + l7_protocol=tsg_l7_protocol_id2name(_instance, l7_proto_label->proto_id); + if(l7_protocol!=NULL) + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)l7_protocol, TLD_TYPE_STRING); + } + } + else + { + l7_protocol=tsg_schema_index2string(internal_label->proto); + if(l7_protocol!=NULL) + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)l7_protocol, TLD_TYPE_STRING); + } + else + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)"UNCATEGORIZED", TLD_TYPE_STRING); + } } TLD_append(_handle, _instance->id2field[LOG_COMMON_ESTABLISH_LATENCY_MS].name, (void *)internal_label->establish_latency_ms, TLD_TYPE_LONG); @@ -234,6 +264,10 @@ int set_common_field_from_label(struct tsg_log_instance_t *_instance, struct TLD TLD_append(_handle, _instance->id2field[LOG_SSL_JA3_FINGERPRINT].name, (void *)internal_label->ja3_fingerprint, TLD_TYPE_STRING); } } + else + { + TLD_append(_handle, _instance->id2field[LOG_COMMON_L7_PROTOCOL].name, (void *)"UNCATEGORIZED", TLD_TYPE_STRING); + } return 0; } @@ -430,7 +464,7 @@ int load_log_common_field(const char *filename, id2field_t *id2field, id2field_t } memset(type_name, 0, sizeof(type_name)); ret=sscanf(line, "%s %s %d", type_name, field_name, &id); - assert(ret==3 && idtcp_label, sizeof(_instance->tcp_label), "tcp_flow_stat"); MESA_load_profile_string_def(conffile, "TSG_LOG", "UDP_LABEL", _instance->udp_label, sizeof(_instance->udp_label), "udp_flow_stat"); + MESA_load_profile_string_def(conffile, "TSG_LOG", "L7_PROTO_LABEL", _instance->l7_proto_label, sizeof(_instance->l7_proto_label), "BASIC_PROTO_LABEL"); _instance->tcp_flow_project_id=project_customer_register(_instance->tcp_label, "struct"); _instance->udp_flow_project_id=project_customer_register(_instance->udp_label, "struct"); @@ -557,7 +596,17 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) ); } - + _instance->l7_proto_project_id=project_customer_register(_instance->l7_proto_label, "struct"); + if(_instance->l7_proto_project_id<0) + { + MESA_handle_runtime_log(_instance->logger, + RLOG_LV_FATAL, + "L7_PROTO_LABEL", + "project_customer_register is error, l7_proto_label: %s, please check etc/project.conf", + _instance->l7_proto_label + ); + } + MESA_load_profile_string_def(conffile, "TSG_LOG", "NIC_NAME", nic_name, sizeof(nic_name), "eth0"); ret=MESA_get_dev_ipv4(nic_name, (int *)&local_ip_nr); if(ret<0) @@ -571,7 +620,9 @@ struct tsg_log_instance_t *tsg_sendlog_init(const char *conffile) return NULL; } inet_ntop(AF_INET,&(local_ip_nr),_instance->local_ip_str,sizeof(_instance->local_ip_str)); - + + MESA_load_profile_string_def(conffile, "TSG_LOG", "L7_PROTO_ID_FILE", _instance->l7_proto_id_file, sizeof(_instance->l7_proto_id_file), "./tsgconf/app_l7_proto_id.conf"); + load_log_common_field(_instance->l7_proto_id_file, _instance->l7_proto_id2field, NULL, &tmp_value); rdkafka_conf = rd_kafka_conf_new(); rd_kafka_conf_set(rdkafka_conf, "queue.buffering.max.messages", _instance->send_queue_max_msg, kafka_errstr, sizeof(kafka_errstr)); diff --git a/src/tsg_send_log_internal.h b/src/tsg_send_log_internal.h index be01b7c..4f3a60d 100644 --- a/src/tsg_send_log_internal.h +++ b/src/tsg_send_log_internal.h @@ -5,7 +5,8 @@ #include #include - +#define MIN_L7_PROTO_ID 100 +#define MAX_L7_PROTO_ID 150 #define MAX_IPV4_LEN 16 #define MAX_STRING_LEN 32 @@ -93,6 +94,7 @@ struct tsg_log_instance_t int level; int max_service; int recovery_interval; + int l7_proto_project_id; int internal_project_id; int tcp_flow_project_id; int udp_flow_project_id; @@ -102,15 +104,18 @@ struct tsg_log_instance_t char log_path[MAX_STRING_LEN*2]; char tcp_label[MAX_STRING_LEN]; char udp_label[MAX_STRING_LEN]; + char l7_proto_label[MAX_STRING_LEN]; char common_field_file[MAX_STRING_LEN*4]; char broker_list[MAX_STRING_LEN*4]; char send_queue_max_msg[MAX_STRING_LEN]; char require_ack[MAX_STRING_LEN]; char refresh_interval_ms[MAX_STRING_LEN]; char local_ip_str[MAX_IPV4_LEN]; + char l7_proto_id_file[MAX_STRING_LEN*4]; id2field_t id2field[LOG_COMMON_MAX]; rd_kafka_topic_t **topic_rkt; id2field_t *service2topic; + id2field_t l7_proto_id2field[MAX_L7_PROTO_ID+1]; void *logger; };