diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index f825658..d22aabd 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -319,6 +319,15 @@ struct ip_address int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data); +int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + long argl, void *argp); + +//Return order: Longger suffix first, then fqdn with bigger index first. +int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data); + enum MAAT_RULE_OPT { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7be2077..db37b65 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/FQDN_engine.cpp b/src/entry/FQDN_engine.cpp new file mode 100644 index 0000000..d402f66 --- /dev/null +++ b/src/entry/FQDN_engine.cpp @@ -0,0 +1,352 @@ +/* + * + * Copyright (c) 2020 + * String Algorithms Research Group + * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) + * National Engineering Laboratory for Information Security Technologies (NELIST) + * All rights reserved + * + * Written by: LIU YANBING (liuyanbing@iie.ac.cn) + * Last modification: 2020-09-01 + * + * This code is the exclusive and proprietary property of IIE-CAS and NELIST. + * Usage for direct or indirect commercial advantage is not allowed without + * written permission from the authors. + * + */ + +#include "FQDN_engine.h" +#include +#include +#include +#include +#include + +/*************************************************************************************/ + +//#include +//#define popcnt_u64 _mm_popcnt_u64 +//Use gcc builtin function to replace SSE4.2 instruction for portability + +#ifdef _MSC_VER +#include +#define popcnt_u64 __popcnt +#else +#define popcnt_u64 __builtin_popcountl +#endif + +#define FOR(i, n) for(int i=0, _n=(int)(n); i<_n; i++) + +struct packedRT_t +{ + unsigned long long bitmap[4]; + unsigned int A; + unsigned char B[4]; +}; + +void * aligned_malloc(size_t size, size_t align) +{ + void * malloc_ptr; + void * aligned_ptr; + + /* Error if align is not a power of two. */ + if (align & (align - 1)) + { + return ((void*) 0); + } + + if (align==0 || size == 0) + { + return ((void *) 0); + } + + malloc_ptr = malloc (sizeof(void *) + align - 1 + size); + if (!malloc_ptr) + { + return ((void *) 0); + } + + aligned_ptr = (void *) (((size_t)malloc_ptr + sizeof(void *) + align-1) & ~(align-1)); + + ((void **) aligned_ptr) [-1] = malloc_ptr; + + return aligned_ptr; +} + +void aligned_free(void * aligned_ptr) +{ + if (aligned_ptr) + { + free (((void **) aligned_ptr) [-1]); + } +} + +/*************************************************************************************/ +struct domain_impl_t +{ + unsigned int id; + int suf_match; + unsigned int len; + unsigned long long hash; /*用64位哈希值唯一表示一个域名*/ + domain_impl_t * next; + void * utag; +}; + +class CHashTrieFQDN +{ +public: + CHashTrieFQDN(); + ~CHashTrieFQDN(); + + int initialize(const struct FQDN_rule * rules, size_t n_rule); + int search(const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result); + +protected: + unsigned int rank(unsigned int h); + +protected: + unsigned int m_num; + domain_impl_t * m_domains; + unsigned int m_H; + unsigned int m_max_pat_len; + unsigned long long * m_B; + packedRT_t * m_RT; + domain_impl_t ** m_matched; + unsigned char m_case_tab[256]; +}; + + +const unsigned long long ONE=1; + +#define is_bit_set(tbl, off) ( tbl[off>>6] & ( ONE<<(off&63) ) ) +#define set_bit(tbl, off) ( tbl[off>>6] |= ( ONE<<(off&63) ) ) + +const unsigned long long A=6364136223846793005LL; + +CHashTrieFQDN::CHashTrieFQDN() +{ + m_num=0; + m_domains=NULL; + m_B=NULL; + m_RT=NULL; + m_matched=NULL; + FOR(c, 256) m_case_tab[c]=tolower(c); +} + +CHashTrieFQDN::~CHashTrieFQDN() +{ + if(m_domains!=NULL) + { + delete [] m_domains; + } + + if(m_B!=NULL) + { + delete [] m_B; + } + + if(m_RT!=NULL) + { + aligned_free(m_RT); + } + + if(m_matched!=NULL) + { + delete [] m_matched; + } +} + +int CHashTrieFQDN::initialize(const struct FQDN_rule * rules, size_t n_rule) +{ + long long mem_bytes=0; + + if(n_rule==0) return -1; + m_num=n_rule; + m_domains=new domain_impl_t[m_num]; + mem_bytes+=m_num*sizeof(domain_impl_t); + + unsigned int N=m_num; + m_max_pat_len=0; + + FOR(k, m_num) + { + m_domains[k].id =rules[k].id; + m_domains[k].suf_match=rules[k].is_suffix_match; + m_domains[k].len=rules[k].len; + m_domains[k].next=NULL; + m_domains[k].utag=rules[k].user_tag; + + FOR(j, rules[k].len) + { + if(rules[k].FQDN[j]=='.') ++N; + } + + if(m_max_pat_len>6]; + mem_bytes+=(m_H>>6)*sizeof(unsigned long long); + FOR(i, (m_H>>6)) m_B[i]=0; + + m_RT=(packedRT_t *)aligned_malloc(sizeof(packedRT_t)*((m_H>>8)+1), 64); + mem_bytes+=((m_H>>8)+1)*sizeof(packedRT_t); + FOR(i, (m_H>>8)) + { + FOR(j, 4) m_RT[i].bitmap[j]=0; + } + + FOR(k, m_num) + { + const unsigned char * pb=(const unsigned char *)rules[k].FQDN; + m_domains[k].hash=0; + + for(int j=rules[k].len-1; j>=-1; --j) + { + if(j==-1 || pb[j]=='.') + { + unsigned int h=m_domains[k].hash&(m_H-1); + set_bit(m_B, h); + if(j==-1) + { + int q=h&255; + m_RT[h>>8].bitmap[q>>6]|=(ONE<<(q&63)); + } + } + else + { + m_domains[k].hash=A*m_domains[k].hash+m_case_tab[pb[j]]; + } + } + } + + m_RT[0].A=0; + FOR(i, (m_H>>8)) + { + m_RT[i].B[0]=0; + m_RT[i].B[1]= popcnt_u64(m_RT[i].bitmap[0]); + m_RT[i].B[2]= m_RT[i].B[1]+popcnt_u64(m_RT[i].bitmap[1]); + m_RT[i].B[3]= m_RT[i].B[2]+popcnt_u64(m_RT[i].bitmap[2]); + m_RT[i+1].A=m_RT[i].A+m_RT[i].B[3]+popcnt_u64(m_RT[i].bitmap[3]); + } + + int tn=m_RT[m_H>>8].A; + + m_matched=new domain_impl_t *[tn]; + mem_bytes+=tn*sizeof(domain_impl_t *); + FOR(i, tn) m_matched[i]=NULL; + + FOR(k, m_num) + { + unsigned int h=m_domains[k].hash&(m_H-1); + unsigned idx=rank(h); + m_domains[k].next=m_matched[idx]; + m_matched[idx]=&(m_domains[k]); + } + +// printf("mem_bytes=%u(MB)\n", mem_bytes/(1U<<20)); + + return 1; +} + +unsigned int CHashTrieFQDN::rank(unsigned int h) +{ + int p=(h>>8); + int r=((h&255)>>6); + int s=(h&63); + unsigned long long e=m_RT[p].bitmap[r]&((ONE<=-1; --j) + { + if(j==-1 || pb[j]=='.') + { + unsigned int h=hash&(m_H-1); + if(is_bit_set(m_B, h)==0) break; + HASH[t]=hash; + P[t]=j+1; + ++t; + } + else if(j+m_max_pat_len=0; t--) + { + unsigned int h=HASH[t]&(m_H-1); + int q=h&255; + + if(m_RT[h>>8].bitmap[q>>6]&(ONE<<(q&63))) + { + unsigned idx=rank(h); + + for(domain_impl_t * pt=m_matched[idx]; pt!=NULL; pt=pt->next) + { + if(P[t]!=0 && pt->suf_match==0) continue; + if(pt->len+P[t]==FQDN_len && pt->hash==HASH[t]) + { + //if(match_num>0 && P[t]!=results[match_num-1].offset) return match_num; + results[match_num].id=pt->id; + results[match_num].offset=P[t]; + results[match_num].user_tag=pt->utag; + ++match_num; + if(match_num==n_result) return match_num; + } + } + } + } + + return match_num; +} + +/*************************************************************************************/ +struct FQDN_engine +{ + CHashTrieFQDN ht; +}; + +struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule) +{ + struct FQDN_engine * instance=new struct FQDN_engine; + if(instance->ht.initialize(rules, n_rule)<0) + { + delete instance; + return NULL; + } + else + { + return instance; + } +} + +int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result) +{ + if(instance==NULL) return -1; + return instance->ht.search(FQDN, FQDN_len, results, n_result); +} + +void FQDN_engine_free(struct FQDN_engine * instance) +{ + if(instance!=NULL) delete instance; +} diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index c8c5f8b..95088c0 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1125,13 +1125,30 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id struct Maat_table_schema *table_desc=NULL; struct Maat_table_runtime *table_rt=NULL; MAAT_RULE_EX_DATA exdata=NULL; + struct timespec start,end; + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&start); + } + if(_feather->scanner==NULL) { return NULL; } - table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); + + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&end); + maat_stat_table(table_rt, 0, &start, &end, 0); + } + else + { + maat_stat_table(table_rt, 0, NULL, NULL, 0); + } + return exdata; } int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, @@ -1163,6 +1180,35 @@ int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, return 0; } +int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + long argl, void *argp) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + int ret=-1; + struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); + pthread_mutex_lock(&(_feather->background_update_mutex)); + ret=Maat_table_fqdn_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger); + + if(ret<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + + struct Maat_table_runtime* table_rt=NULL; + if(_feather->scanner!=NULL) + { + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + Maat_table_runtime_fqdn_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger); + } + pthread_mutex_unlock(&(_feather->background_update_mutex)); + + return 0; + +} int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) { @@ -1170,11 +1216,18 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc struct Maat_table_schema *table_schema=NULL; struct Maat_table_runtime *table_rt=NULL; int n_get=0; + + struct timespec start,end; + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&start); + } if(_feather->scanner==NULL) { return 0; } - table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + + table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN) { @@ -1190,9 +1243,57 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc ipv6_ntoh(ip_data.ipv6); } n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data); + + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&end); + maat_stat_table(table_rt, 0, &start, &end, 0); + } + else + { + maat_stat_table(table_rt, 0, NULL, NULL, 0); + } + return n_get; } +int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_schema *table_schema=NULL; + struct Maat_table_runtime *table_rt=NULL; + int n_get=0; + + struct timespec start,end; + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&start); + } + if(_feather->scanner==NULL) + { + return 0; + } + table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN) + { + return -1; + } + n_get=Maat_table_runtime_fqdn_plugin_get_N_ex_data(table_rt, table_schema, fqdn, ex_data_array, n_ex_data); + + + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&end); + maat_stat_table(table_rt, 0, &start, &end, 0); + } + else + { + maat_stat_table(table_rt, 0, NULL, NULL, 0); + } + + return n_get; +} int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len @@ -1227,7 +1328,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id return 0; } - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1365,7 +1466,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id _mid->scan_cnt++; int virtual_table_id=0; - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1467,7 +1568,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id _mid->scan_cnt++; int virtual_table_id=0; - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1531,7 +1632,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id return compile_ret; } -static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_CHILD_TYPE child_type, int table_id) +static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, unsigned short int proto, enum MAAT_TABLE_COMPONENT_TYPE child_type, int table_id) { memset(scan_data, 0, sizeof(scan_data_t)); scan_data->sub_type=make_sub_type(table_id, CHARSET_NONE, 0); @@ -1542,15 +1643,15 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u scan_data->ipv4_data.proto=proto; switch(child_type) { - case CHILD_TABLE_TYPE_SOURCE_IP: + case COMPONENT_TABLE_TYPE_SOURCE_IP: scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr); scan_data->ipv4_data.sport=ntohs(addr->v4->source); break; - case CHILD_TABLE_TYPE_DESTINATION_IP: + case COMPONENT_TABLE_TYPE_DESTINATION_IP: scan_data->ipv4_data.saddr=ntohl(addr->v4->daddr); scan_data->ipv4_data.sport=ntohs(addr->v4->dest); break; - case CHILD_TABLE_TYPE_SESSION: + case COMPONENT_TABLE_TYPE_SESSION: scan_data->ipv4_data.saddr=ntohl(addr->v4->saddr); scan_data->ipv4_data.sport=ntohs(addr->v4->source); scan_data->ipv4_data.daddr=ntohl(addr->v4->daddr); @@ -1566,17 +1667,17 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u scan_data->ipv6_data.proto=proto; switch(child_type) { - case CHILD_TABLE_TYPE_SOURCE_IP: + case COMPONENT_TABLE_TYPE_SOURCE_IP: memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr)); ipv6_ntoh(scan_data->ipv6_data.saddr); scan_data->ipv6_data.sport=ntohs(addr->v6->source); break; - case CHILD_TABLE_TYPE_DESTINATION_IP: + case COMPONENT_TABLE_TYPE_DESTINATION_IP: memcpy(scan_data->ipv6_data.saddr, addr->v6->daddr, sizeof(scan_data->ipv6_data.saddr)); ipv6_ntoh(scan_data->ipv6_data.saddr); scan_data->ipv6_data.sport=ntohs(addr->v6->dest); break; - case CHILD_TABLE_TYPE_SESSION: + case COMPONENT_TABLE_TYPE_SESSION: memcpy(scan_data->ipv6_data.saddr, addr->v6->saddr, sizeof(scan_data->ipv6_data.saddr)); ipv6_ntoh(scan_data->ipv6_data.saddr); scan_data->ipv6_data.sport=ntohs(addr->v6->source); @@ -1596,7 +1697,7 @@ static int ip_scan_data_set(scan_data_t* scan_data, const struct ipaddr* addr, u } return 0; } -static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, Maat_table_schema* parent_table, enum MAAT_TABLE_CHILD_TYPE child_type, +static int IP_composition_scan(const struct ipaddr* addr, unsigned short int proto, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE child_type, scan_result_t *region_result, unsigned int result_num, int* virtual_table_id, rule_scanner_t ip_scanner, struct Maat_table_manager* table_mgr, struct Maat_table_runtime_manager* table_rt_mgr, @@ -1604,21 +1705,21 @@ static int IP_composition_scan(const struct ipaddr* addr, unsigned short int pro { int child_table_id=0; - if(child_type==CHILD_TABLE_TYPE_NONE) + if(child_type==COMPONENT_TABLE_TYPE_NONE) { - child_table_id=parent_table->table_id; - child_type=CHILD_TABLE_TYPE_SESSION; + child_table_id=parent_table_id; + child_type=COMPONENT_TABLE_TYPE_SESSION; } else { - child_table_id=Maat_table_get_child_id(parent_table, child_type); + child_table_id=Maat_table_get_child_id(table_mgr, parent_table_id, child_type); } if(child_table_id<0) { return 0; } - Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, TABLE_TYPE_IP, virtual_table_id); + Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id); if(real_table==NULL) { return 0; @@ -1659,7 +1760,6 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id int region_ret=0, compile_ret=0; struct _OUTER_scan_status_t* _mid=NULL; scan_result_t *region_result=NULL; - Maat_table_schema* p_table=NULL; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct Maat_scanner* my_scanner=NULL; struct timespec start,end; @@ -1670,8 +1770,9 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id _mid=grab_mid(mid, _feather, thread_num, 0); _mid->scan_cnt++; int virtual_table_id=0; - p_table=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); - if(p_table==NULL) + enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID; + table_type=Maat_table_get_type_by_id(_feather->table_mgr, table_id); + if(table_type==TABLE_TYPE_INVALID) { _feather->scan_err_cnt++; return -1; @@ -1690,12 +1791,12 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); INC_SCANNER_REF(my_scanner, thread_num); - if(p_table->table_type==TABLE_TYPE_COMPOSITION) + if(table_type==TABLE_TYPE_COMPOSITION) { - enum MAAT_TABLE_CHILD_TYPE childs[3]={CHILD_TABLE_TYPE_SOURCE_IP, CHILD_TABLE_TYPE_DESTINATION_IP, CHILD_TABLE_TYPE_SESSION}; + enum MAAT_TABLE_COMPONENT_TYPE childs[3]={COMPONENT_TABLE_TYPE_SOURCE_IP, COMPONENT_TABLE_TYPE_DESTINATION_IP, COMPONENT_TABLE_TYPE_SESSION}; for(int i=0; i<3; i++) { - region_ret=IP_composition_scan(addr, proto, p_table, childs[i], + region_ret=IP_composition_scan(addr, proto, table_id, childs[i], region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id, my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num); if(region_ret<0) @@ -1713,7 +1814,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id } else { - region_ret=IP_composition_scan(addr, proto, p_table, CHILD_TABLE_TYPE_NONE, + region_ret=IP_composition_scan(addr, proto, table_id, COMPONENT_TABLE_TYPE_NONE, region_result+region_hit_cnt, MAX_SCANNER_HIT_NUM-region_hit_cnt, &virtual_table_id, my_scanner->region, _feather->table_mgr, _feather->scanner->table_rt_mgr, thread_num); if(region_ret<0) @@ -1733,7 +1834,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id _mid=grab_mid(mid, _feather, thread_num, 1); scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_hit_cnt, _mid->is_last_region, virtual_table_id, _mid->scan_cnt); - if(p_table->table_type==TABLE_TYPE_COMPOSITION) + if(table_type==TABLE_TYPE_COMPOSITION) { region_hit_wraper.virtual_table_ids=region_rslt_virtual_table_id; } @@ -1783,7 +1884,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct Maat_table_schema *p_table=NULL; int virtual_table_id=0; assert(thread_num<_feather->scan_thread_num); - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2071,7 +2172,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sfh_instance_t * tmp_fuzzy_handle=NULL; struct Maat_table_schema *p_table=NULL; int virtual_table_id=0; - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_DIGEST, &virtual_table_id); + p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index fa48b80..02ce3ca 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -201,6 +201,7 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_ break; case TABLE_TYPE_PLUGIN: case TABLE_TYPE_IP_PLUGIN: + case TABLE_TYPE_FQDN_PLUGIN: if(valid_column_seq<0) { return -1; @@ -1624,7 +1625,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m int update_type=CM_UPDATE_TYPE_INC; long long new_version=0; enum MAAT_TABLE_TYPE table_type; - struct Maat_table_schema* table_desc=NULL; + enum MAAT_SCAN_TYPE scan_type; + struct Maat_table_schema* table_schema=NULL; void* logger=feather->logger; if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write @@ -1713,26 +1715,14 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id); if(rule_list[i].op==MAAT_OP_DEL) { - if(table_type==TABLE_TYPE_PLUGIN||table_type==TABLE_TYPE_IP_PLUGIN) - { - table_desc=Maat_table_get_scan_by_id(feather->table_mgr, table_id, table_type, NULL); - if(table_type==TABLE_TYPE_PLUGIN) - { - valid_column=table_desc->plugin.valid_flag_column; - } - else - { - valid_column=table_desc->ip_plugin.valid_flag_column; - } - } - else - { - valid_column=-1; - } + + scan_type=Maat_table_get_scan_type(table_type); + table_schema=Maat_table_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL); + valid_column=Maat_table_xx_plugin_table_get_valid_flag_column(table_schema); ret=invalidate_line(rule_list[i].table_line, table_type, valid_column); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invaid format %s ." + MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invalidate line failed, invaid format %s ." ,rule_list[i].table_line); continue; } @@ -1918,22 +1908,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_ goto error_out; } int valid_flag_column=0; - switch(p_table->table_type) + + valid_flag_column=Maat_table_xx_plugin_table_get_valid_flag_column(p_table); + if(valid_flag_column<0) { - case TABLE_TYPE_PLUGIN: - valid_flag_column=p_table->plugin.valid_flag_column; - plugin_desc=&(p_table->plugin); - break; - case TABLE_TYPE_IP_PLUGIN: - valid_flag_column=p_table->ip_plugin.valid_flag_column; - break; - default: - MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command - ,"Command set line id %d failed: table %s is not a plugin or ip_plugin table." - , line_rule[i]->rule_id - , line_rule[i]->table_name); - ret=-1; - goto error_out; + MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command + ,"Command set line id %d failed: table %s is not a plugin or ip_plugin table." + , line_rule[i]->rule_id + , line_rule[i]->table_name); + ret=-1; + goto error_out; + } if(op==MAAT_OP_ADD) diff --git a/src/entry/Maat_ex_data.cpp b/src/entry/Maat_ex_data.cpp index 7cf2a77..a98645b 100644 --- a/src/entry/Maat_ex_data.cpp +++ b/src/entry/Maat_ex_data.cpp @@ -57,7 +57,7 @@ void cache_row_free(void*p) } UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free}; -struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)) +struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)) { struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1); p->hash_key2ex=NULL; @@ -207,3 +207,7 @@ void* EX_data_container_get_user_data(struct EX_data_container* ex_container) { return ex_container->user_data; } +size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt) +{ + return HASH_COUNT(ex_rt->hash_key2ex); +} diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index e6c30e1..6b01597 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -34,7 +34,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_3_0_20200917=1; +int MAAT_FRAME_VERSION_3_0_20200927=1; int is_valid_table_name(const char* str) { @@ -1392,6 +1392,8 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r scanner->gie_update_q_size++; return 0; } + + int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id, int rule_type, struct Maat_scanner *maat_scanner, void* logger) { int i=0; @@ -2163,7 +2165,6 @@ error_out: digest_rule=NULL; return; } - void update_plugin_table(struct Maat_table_schema* table_schema, const char* row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int ret=1, matched_tag=1; @@ -2210,21 +2211,34 @@ void update_plugin_table(struct Maat_table_schema* table_schema, const char* row } Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger); } -void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) +void update_xx_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int ret=1, matched_tag=1; - struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id); char* copy=NULL; size_t accept_tag_offset=0, accept_tag_len=0; - if(ip_plugin_schema->rule_tag_column>0&&n_tags>0) + int rule_tag_column=-1; + if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN) { - ret=Maat_helper_read_column(table_row, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len); + rule_tag_column=table_schema->ip_plugin.rule_tag_column; + } + else if(table_schema->table_type==TABLE_TYPE_FQDN_PLUGIN) + { + rule_tag_column=table_schema->fqdn_plugin.rule_tag_column; + } + else + { + assert(0); + return; + } + if(rule_tag_column>0&&n_tags>0) + { + ret=Maat_helper_read_column(table_row, rule_tag_column, &accept_tag_offset, &accept_tag_len); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, could not locate tag in column %d of plugin table_schema %s:%s", - ip_plugin_schema->rule_tag_column, + "update error, could not locate tag in column %d of table %s:%s", + rule_tag_column, table_schema->table_name[table_schema->updating_name], table_row); table_schema->udpate_err_cnt++; @@ -2254,8 +2268,15 @@ void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* return; } } - Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger); - scanner->ip_plugin_update_q_size++; + if(table_schema->table_type==TABLE_TYPE_IP_PLUGIN) + { + Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger); + } + else //TABLE_TYPE_FQDN_PLUGIN + { + Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger); + } + scanner->xx_plugin_update_q_size++; return; } @@ -2263,9 +2284,9 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* { MESA_htable_handle tmp_map=NULL; struct Maat_table_runtime* table_rt=NULL; - struct ip_matcher* old_ip_matcher=NULL; int i=0, ret=0; - + struct ip_matcher* old_ip_matcher=NULL; + struct FQDN_engine* old_fqdn_engine=NULL; if(scanner->to_update_compile_cnt+scanner->to_update_group_cnt>0) { ret=Maat_hierarchy_rebuild(scanner->hier); @@ -2299,8 +2320,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* switch(table_rt->table_type) { case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - + case TABLE_TYPE_SIMILARITY: ret=Maat_table_runtime_digest_batch_udpate(table_rt); if(ret<0) { @@ -2308,7 +2328,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* "GIE_update error."); } break; - case TABLE_TYPE_IP_PLUGIN: + case TABLE_TYPE_IP_PLUGIN: ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt); if(ret) { @@ -2319,6 +2339,18 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* } } break; + case TABLE_TYPE_FQDN_PLUGIN: + + ret=Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt); + if(ret) + { + old_fqdn_engine=Maat_table_runtime_apply_new_fqdn_engine(table_rt); + if(old_fqdn_engine) + { + Maat_garbage_bagging(scanner->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free); + } + } + break; default: break; } @@ -2335,7 +2367,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* scanner->gie_update_q_size=0; scanner->to_update_group_cnt=0; scanner->to_update_compile_cnt=0; - scanner->ip_plugin_update_q_size=0; + scanner->xx_plugin_update_q_size=0; return; } @@ -2400,7 +2432,7 @@ void maat_finish_cb(void* u_para) feather->scanner->cfg_num=scanner_rule_num(feather->scanner); feather->scanner->version=feather->maat_version; expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); - feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->ip_plugin_update_q_size; + feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->xx_plugin_update_q_size; if(time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000) { do_scanner_update(feather->scanner, @@ -2480,8 +2512,9 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); break; case TABLE_TYPE_IP_PLUGIN: - update_ip_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); - break; + case TABLE_TYPE_FQDN_PLUGIN: + update_xx_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + break; default: break; diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index 2994445..552eb94 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -18,7 +18,45 @@ struct Maat_table_manager MESA_htable_handle map_tablename2id; int active_plugin_table_num; int is_last_plugin_table_updating; + void* logger; }; +enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type) +{ + enum MAAT_SCAN_TYPE ret=SCAN_TYPE_INVALID; + switch(table_type) + { + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + case TABLE_TYPE_SIMILARITY: + case TABLE_TYPE_DIGEST: + ret=SCAN_TYPE_STRING; + break; + case TABLE_TYPE_INTERVAL: + case TABLE_TYPE_INTERVAL_PLUS: + ret=SCAN_TYPE_INTERVAL; + break; + case TABLE_TYPE_IP: + case TABLE_TYPE_IP_PLUS: + case TABLE_TYPE_COMPOSITION: + ret=SCAN_TYPE_IP; + break; + case TABLE_TYPE_PLUGIN: + ret=SCAN_TYPE_PLUGIN; + break; + case TABLE_TYPE_IP_PLUGIN: + ret=SCAN_TYPE_IP; + break; + case TABLE_TYPE_FQDN_PLUGIN: + ret=SCAN_TYPE_FQDN_PLUGIN; + break; + case TABLE_TYPE_COMPILE: + ret=SCAN_TYPE_NONE; + break; + default: + break; + } + return ret; +} int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map) { @@ -74,25 +112,7 @@ int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA } return 0; } -int read_virtual_table_schema(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map) -{ - int ret=0; - char table_type[16]; - ret=sscanf(line, "%d\t%s\t%s\t%s", &(table->table_id), - table->table_name[0], - table_type, - table->virtual_table.real_table_name); - if(ret!=4) - { - return -1; - } - ret=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); - if(ret<0) - { - return -1; - } - return 0; -} + Maat_table_schema* table_info_new(void) { struct Maat_table_schema*p=ALLOC(struct Maat_table_schema, 1); @@ -119,7 +139,83 @@ int _read_integer_arrary(char* string, int *array, int size) } #define COLUMN_PLUGIN_SCHEMA_JSON 4 #define COLUMN_IP_PLUGIN_SCHEMA_JSON 4 +#define COLUMN_FQDN_PLUGIN_SHCEMA_JSON 4 #define COLUMN_COMPOSITION_SCHEMA_JSON 4 +#define COLUMN_VIRUTAL_SCHEMA_JSON 4 +int read_virtual_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* table, MESA_htable_handle reserved_word_map) +{ + int ret=0, tmp_table_id=0; + enum MAAT_TABLE_TYPE physical_table_type=TABLE_TYPE_INVALID; + enum MAAT_SCAN_TYPE physical_table_scan_type=SCAN_TYPE_INVALID; + cJSON* json=NULL, *tmp=NULL; + char *json_str; + + + size_t offset=0, len=0; + char* copy_line=NULL; + copy_line=_maat_strdup(line); + ret=get_column_pos(copy_line, COLUMN_VIRUTAL_SCHEMA_JSON, &offset, &len); + if(ret<0) + { + goto error_out; + } + if(offset+lentype!=cJSON_Array) + { + goto error_out; + } + cJSON_ArrayForEach(tmp, json) + { + if(tmp->type!=cJSON_String) + { + goto error_out; + } + ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &tmp_table_id); + if(ret<0) + { + goto error_out; + } + physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type; + physical_table_scan_type=Maat_table_get_scan_type(physical_table_type); + if(physical_table_scan_typevirtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id; + } + } + else //For compatible non-json physical description + { + ret=map_str2int(table_mgr->map_tablename2id, json_str, &tmp_table_id); + if(ret<0) + { + goto error_out; + } + physical_table_type=table_mgr->p_table_info[tmp_table_id]->table_type; + physical_table_scan_type=Maat_table_get_scan_type(physical_table_type); + table->virtual_table.physical_table_id[physical_table_scan_type]=tmp_table_id; + } + cJSON_Delete(json); + free(copy_line); + return 0; + +error_out: + if(json) cJSON_Delete(json); + free(copy_line); + return -1; + +} int read_plugin_table_schema(const char* line, struct Maat_table_schema* p) { @@ -136,7 +232,7 @@ int read_plugin_table_schema(const char* line, struct Maat_table_schema* p) } if(offset+lentype==cJSON_Number); plugin_desc->rule_tag_column=tmp->valueint; } - tmp=cJSON_GetObjectItem(json, "estimate_size"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->estimate_size=tmp->valueint; - } tmp=cJSON_GetObjectItem(json, "foreign"); if(tmp!=NULL) { @@ -220,7 +310,7 @@ int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p) } if(offset+lenestimate_size=4096; - tmp=cJSON_GetObjectItem(json, "estimate_size"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - ip_plugin_schema->estimate_size=tmp->valueint; - //read_cnt++; estimate_size is optional, so NOT ++ intentionally. - } - cJSON_Delete(json); free(copy_line); @@ -296,8 +377,85 @@ error_out: return -1; } +int read_fqdn_plugin_table_schema(const char* line, struct Maat_table_schema* p) +{ + int ret=0, read_cnt=0; + size_t offset=0, len=0; + cJSON* json=NULL, *tmp=NULL; + char* copy_line=NULL, *fqnd_plugin_schema_json=NULL; + struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(p->fqdn_plugin); -int read_composition_table_schema(const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map) + copy_line=_maat_strdup(line); + ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SHCEMA_JSON, &offset, &len); + if(ret<0) + { + goto error_out; + } + if(offset+lentype==cJSON_Number) + { + fqdn_plugin_schema->row_id_column=tmp->valueint; + read_cnt++; + } + + tmp=cJSON_GetObjectItem(json, "is_suffix_match"); + if(tmp!=NULL && tmp->type==cJSON_Number) + { + fqdn_plugin_schema->is_suffix_flag_column=tmp->valueint; + read_cnt++; + } + tmp=cJSON_GetObjectItem(json, "fqdn"); + if(tmp!=NULL && tmp->type==cJSON_Number) + { + fqdn_plugin_schema->fqdn_column=tmp->valueint; + read_cnt++; + } + + tmp=cJSON_GetObjectItem(json, "valid"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + fqdn_plugin_schema->valid_flag_column=tmp->valueint; + read_cnt++; + } + fqdn_plugin_schema->rule_tag_column=-1; + tmp=cJSON_GetObjectItem(json, "tag"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + fqdn_plugin_schema->rule_tag_column=tmp->valueint; + //read_cnt++; Tag is optional, so NOT ++ intentionally. + } + + cJSON_Delete(json); + + free(copy_line); + if(read_cnt<4) + { + return -1; + } + else + { + return 0; + } +error_out: + free(copy_line); + return -1; + +} + +int read_composition_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map) { int ret=0; size_t offset=0, len=0; @@ -312,7 +470,7 @@ int read_composition_table_schema(const char* line, struct Maat_table_schema* p, } if(offset+lentype==cJSON_String) { - strncpy(composition_schema->source_table.real_table_name, tmp->valuestring, sizeof(composition_schema->source_table.real_table_name)); + ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SOURCE_IP])); + if(ret<0) + { + MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, + "Child table %s of table %s (id=%d) are not defined.", + tmp->valuestring, + p->table_name[0], + p->table_id); + goto error_out; + } } tmp=cJSON_GetObjectItem(json, "destination"); if(tmp!=NULL && tmp->type==cJSON_String) { - strncpy(composition_schema->destination_table.real_table_name, tmp->valuestring, sizeof(composition_schema->destination_table.real_table_name)); + ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DESTINATION_IP])); + if(ret<0) + { + MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, + "Child table %s of table %s (id=%d) are not defined.", + tmp->valuestring, + p->table_name[0], + p->table_id); + goto error_out; + } + } tmp=cJSON_GetObjectItem(json, "session"); if(tmp!=NULL && tmp->type==cJSON_String) { - strncpy(composition_schema->session_table.real_table_name, tmp->valuestring, sizeof(composition_schema->session_table.real_table_name)); + ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION])); + if(ret<0) + { + MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, + "Child table %s of table %s (id=%d) are not defined.", + tmp->valuestring, + p->table_name[0], + p->table_id); + goto error_out; + } } cJSON_Delete(json); - free(copy_line); return 0; + error_out: free(copy_line); return -1; } -static int Maat_table_build_map(struct Maat_table_manager* table_mgr, void* logger) -{ - struct Maat_table_schema** p_table_info=table_mgr->p_table_info; - size_t n_table=MAX_TABLE_NUM; - - MESA_htable_handle map_tablename2id=map_create(); - size_t i=0; - int j=0, ret=0; - for(i=0;itable_type) - { - case TABLE_TYPE_VIRTUAL: - ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Undefined real table %s, virtual table %s of table id %d.", - p_table_info[i]->virtual_table.real_table_name, - p_table_info[i]->table_name[0], - p_table_info[i]->table_id); - goto failed; - } - break; - case TABLE_TYPE_COMPOSITION: - if(strlen(p_table_info[i]->composition.source_table.real_table_name)>0) - { - ret=map_str2int(map_tablename2id, p_table_info[i]->composition.source_table.real_table_name, - &(p_table_info[i]->composition.source_table.real_table_id)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Child table %s of table %s (id=%d) are not defined.", - p_table_info[i]->composition.source_table.real_table_name, - p_table_info[i]->table_name[0], - p_table_info[i]->table_id); - goto failed; - } - } - - if(strlen(p_table_info[i]->composition.destination_table.real_table_name)>0) - { - ret=map_str2int(map_tablename2id, p_table_info[i]->composition.destination_table.real_table_name, - &(p_table_info[i]->composition.destination_table.real_table_id)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Child table %s of table %s (id=%d) are not defined.", - p_table_info[i]->composition.destination_table.real_table_name, - p_table_info[i]->table_name[0], - p_table_info[i]->table_id); - goto failed; - } - } - if(strlen(p_table_info[i]->composition.session_table.real_table_name)>0) - { - ret=map_str2int(map_tablename2id, p_table_info[i]->composition.session_table.real_table_name, - &(p_table_info[i]->composition.session_table.real_table_id)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Child table %s of table %s (id=%d) are not defined.", - p_table_info[i]->composition.session_table.real_table_name, - p_table_info[i]->table_name[0], - p_table_info[i]->table_id); - goto failed; - } - } - default: - break; - } - - - for(j=0; jconj_cnt; j++) - { - ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Duplicate table %s of table id %d", - p_table_info[i]->table_name[j], - p_table_info[i]->table_id); - continue; - } - } - - } - table_mgr->map_tablename2id=map_tablename2id; - return 0; -failed: - map_destroy(map_tablename2id); - return -1; -} - void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr) { size_t i=0; @@ -469,7 +556,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path char line[MAX_TABLE_LINE_SIZE]; int i=0, ret=0; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; - MESA_htable_handle string2int_map=NULL;; + MESA_htable_handle reserved_word_map=NULL;; struct Maat_table_schema*p=NULL; struct Maat_table_schema*conj_table=NULL; fp=fopen(table_info_path,"r"); @@ -483,37 +570,40 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path table_mgr=ALLOC(struct Maat_table_manager, 1); struct Maat_table_schema** p_table_info=table_mgr->p_table_info; size_t n_table=MAX_TABLE_NUM; + table_mgr->logger=logger; + table_mgr->map_tablename2id=map_create(); - string2int_map=map_create(); - map_register(string2int_map,"expr", TABLE_TYPE_EXPR); - map_register(string2int_map,"ip", TABLE_TYPE_IP); - map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS); - map_register(string2int_map,"compile", TABLE_TYPE_COMPILE); - map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); - map_register(string2int_map,"ip_plugin", TABLE_TYPE_IP_PLUGIN); - map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL); - map_register(string2int_map,"interval", TABLE_TYPE_INTERVAL); - map_register(string2int_map,"intval_plus", TABLE_TYPE_INTERVAL_PLUS); - map_register(string2int_map,"interval_plus", TABLE_TYPE_INTERVAL_PLUS); - map_register(string2int_map,"digest", TABLE_TYPE_DIGEST); - map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS); - map_register(string2int_map,"group", TABLE_TYPE_GROUP); - map_register(string2int_map,"group2group", TABLE_TYPE_GROUP2GROUP); - map_register(string2int_map,"group2compile", TABLE_TYPE_GROUP2COMPILE); - map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY); - map_register(string2int_map,"virtual", TABLE_TYPE_VIRTUAL); - map_register(string2int_map,"composition", TABLE_TYPE_COMPOSITION); - map_register(string2int_map,"quickoff", 0); - map_register(string2int_map,"quickon", 1); - map_register(string2int_map,"escape", USER_REGION_ENCODE_ESCAPE); -// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet + reserved_word_map=map_create(); + map_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); + map_register(reserved_word_map, "ip", TABLE_TYPE_IP); + map_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); + map_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE); + map_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); + map_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); + map_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); + map_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL); + map_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL); + map_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); + map_register(reserved_word_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS); + map_register(reserved_word_map, "digest", TABLE_TYPE_DIGEST); + map_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); + map_register(reserved_word_map, "group", TABLE_TYPE_GROUP); + map_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); + map_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); + map_register(reserved_word_map, "similar", TABLE_TYPE_SIMILARITY); + map_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); + map_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION); + map_register(reserved_word_map, "quickoff", 0); + map_register(reserved_word_map, "quickon", 1); + map_register(reserved_word_map, "escape", USER_REGION_ENCODE_ESCAPE); +// map_register(reserved_word_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet const char** charset_name_list=charset_get_all_name(); for(i=0;i0) { - map_register(string2int_map, charset_name_list[i], i); + map_register(reserved_word_map, charset_name_list[i], i); } else { @@ -521,8 +611,8 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path } } - map_register(string2int_map,"yes", 1); - map_register(string2int_map,"no", 0); + map_register(reserved_word_map,"yes", 1); + map_register(reserved_word_map,"no", 0); i=0; @@ -546,7 +636,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path "Maat read table info %s line %d error: not enough column.",table_info_path,i); continue; } - ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type)); + ret=map_str2int(reserved_word_map,str_tolower(table_type_str),(int*)&(p->table_type)); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, @@ -557,7 +647,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - ret=read_expr_table_info(line, p, string2int_map); + ret=read_expr_table_info(line, p, reserved_word_map); if(ret<0) { fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i); @@ -570,9 +660,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_plugin_table_schema(line, p); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n", table_info_path,i); + fprintf(stderr,"Maat read table info %s line %d error:illegal plugin table schema.\n", table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal plugin info.", table_info_path,i); + "Maat read table info %s line %d error:illegal plugin table schema.", table_info_path,i); goto invalid_table; } break; @@ -580,29 +670,39 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_ip_plugin_table_schema(line, p); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin info.\n", table_info_path,i); + fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin table schema.\n", table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal ip_plugin info.", table_info_path,i); + "Maat read table info %s line %d error:illegal ip_plugin table schema.", table_info_path,i); goto invalid_table; } - break; - case TABLE_TYPE_COMPOSITION: - ret=read_composition_table_schema(line, p, string2int_map); + break; + case TABLE_TYPE_FQDN_PLUGIN: + ret=read_fqdn_plugin_table_schema(line, p); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal composition info.\n", table_info_path,i); + fprintf(stderr,"Maat read table info %s line %d error:illegal fqdn_plugin table schema.\n", table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal composition info.", table_info_path,i); + "Maat read table info %s line %d error:illegal fqdn_plugin table schema.", table_info_path,i); + goto invalid_table; + } + break; + case TABLE_TYPE_COMPOSITION: + ret=read_composition_table_schema(table_mgr, line, p, reserved_word_map); + if(ret<0) + { + fprintf(stderr,"Maat read table info %s line %d error:illegal composition table schema.\n", table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal composition table schema.", table_info_path,i); goto invalid_table; } break; case TABLE_TYPE_VIRTUAL: - ret=read_virtual_table_schema(line, p, string2int_map); + ret=read_virtual_table_schema(table_mgr, line, p, reserved_word_map); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n", table_info_path,i); + fprintf(stderr,"Maat read table info %s line %d error:illegal virtual table schema.\n", table_info_path,i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal virtual info.", table_info_path,i); + "Maat read table info %s line %d error:illegal virtual table schema.", table_info_path,i); goto invalid_table; } break; @@ -610,12 +710,13 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=sscanf(not_care,"%[a-z0-9]",tmp_str); if(ret>0) { - ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); + ret=map_str2int(reserved_word_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); } if(ret!=1) { p->compile.user_region_encoding=USER_REGION_ENCODE_NONE; } + break; default: break; } @@ -628,6 +729,15 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path goto invalid_table; } + ret=map_register(table_mgr->map_tablename2id, p->table_name[0], p->table_id); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Duplicate table %s of table id %d", + p->table_name[0], + p->table_id); + goto invalid_table; + } if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table; { conj_table=p_table_info[p->table_id]; @@ -647,21 +757,16 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path //use goto to free the conjunctioned table_info goto invalid_table; } - p_table_info[p->table_id]=p; table_mgr->table_cnt++; continue; + invalid_table: table_info_free(p); p=NULL; } fclose(fp); - ret=Maat_table_build_map(table_mgr, logger); - if(ret<0) - { - return NULL; - } - map_destroy(string2int_map); + map_destroy(reserved_word_map); return table_mgr; } size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr) @@ -747,13 +852,13 @@ struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* t return table_mgr->p_table_info[table_id]; } -struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id) +struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id) { - + enum MAAT_SCAN_TYPE tab_scan_type; struct Maat_table_schema **p_table_info=table_mgr->p_table_info; size_t n_table=MAX_TABLE_NUM; - struct Maat_table_schema *p_table=NULL, *p_real_table=NULL; + struct Maat_table_schema *p_table=NULL, *p_physical_table=NULL; if((unsigned int) table_id>n_table) { return NULL; @@ -769,23 +874,20 @@ struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* } if(p_table->table_type==TABLE_TYPE_VIRTUAL) { - p_real_table=p_table_info[p_table->virtual_table.real_table_id]; + p_physical_table=p_table_info[p_table->virtual_table.physical_table_id[scan_type]]; *virutal_table_id=table_id; } else { - p_real_table=p_table; + p_physical_table=p_table; if(virutal_table_id) *virutal_table_id=0; } - if(p_real_table->table_type!=expect_type) + tab_scan_type=Maat_table_get_scan_type(p_physical_table->table_type); + if(tab_scan_type!=scan_type) { - if((expect_type==TABLE_TYPE_EXPR && p_real_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| - (expect_type==TABLE_TYPE_IP && p_real_table->table_type!=TABLE_TYPE_IP_PLUS)) - { - return NULL; - } + return NULL; } - return p_real_table; + return p_physical_table; } int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name) { @@ -808,7 +910,7 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, void* u_para) { int idx=0; - struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); struct plugin_table_schema *plugin_desc=&(p_table->plugin); if(p_table==NULL) { @@ -838,7 +940,7 @@ struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_tabl { return NULL; } - p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); if(!p_table) { return NULL; @@ -863,7 +965,7 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c { return -1; } - p_table=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); if(!p_table) { return -1; @@ -970,7 +1072,40 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche new_func, free_func, dup_func, key2index_func, argl, argp); table_schema->ip_plugin.have_exdata=1; return 0; -} +} +int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger) +{ + if(new_func==NULL || free_func==NULL || dup_func==NULL ) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); + return -1; + } + if(table_schema->table_type!=TABLE_TYPE_FQDN_PLUGIN) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a fqdn_plugin table.", __FUNCTION__); + return -1; + } + if(table_schema->fqdn_plugin.have_exdata) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__); + return -1; + } + Maat_table_EX_data_schema_set(&table_schema->fqdn_plugin.ex_schema, + new_func, free_func, dup_func, key2index_func, argl, argp); + table_schema->fqdn_plugin.have_exdata=1; + return 0; +} + + void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type) { table_mgr->active_plugin_table_num=0; @@ -1067,28 +1202,35 @@ void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* assert(i<=p_table->conj_cnt); } -int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type) +int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type) { int ret=-1; + struct Maat_table_schema* p_table=Maat_table_get_by_id_raw(table_mgr, parent_table_id); if(p_table->table_type!=TABLE_TYPE_COMPOSITION) { return -1; } - switch (type) - { - case CHILD_TABLE_TYPE_SOURCE_IP: - ret=p_table->composition.source_table.real_table_id; - break; - case CHILD_TABLE_TYPE_DESTINATION_IP: - ret=p_table->composition.destination_table.real_table_id; - break; - case CHILD_TABLE_TYPE_SESSION: - ret=p_table->composition.session_table.real_table_id; - break; - default: - assert(0); - break; - } + ret=p_table->composition.component_table_id[type]; return ret; } +int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table) +{ + int valid_flag_column=-1; + switch(p_table->table_type) + { + case TABLE_TYPE_PLUGIN: + valid_flag_column=p_table->plugin.valid_flag_column; + break; + case TABLE_TYPE_IP_PLUGIN: + valid_flag_column=p_table->ip_plugin.valid_flag_column; + break; + case TABLE_TYPE_FQDN_PLUGIN: + valid_flag_column=p_table->fqdn_plugin.valid_flag_column; + break; + default: + valid_flag_column=-1; + break; + } + return valid_flag_column; +} diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp index fe0c574..292c3e2 100644 --- a/src/entry/Maat_table_runtime.cpp +++ b/src/entry/Maat_table_runtime.cpp @@ -113,6 +113,57 @@ static void destroy_digest_rule(GIE_digest_t*rule) rule=NULL; return; } +struct FQDN_rule* fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match) +{ + struct FQDN_rule* fqdn_rule=ALLOC(struct FQDN_rule, 1); + //Todo: check FQDN format with regex ^([a-zA-Z0-9._-])+$ + if(fqdn[0]=='.') + { + fqdn++; + fqdn_len--; + } + if(fqdn[fqdn_len]=='/') + { + fqdn_len--; + } + fqdn_rule->FQDN=ALLOC(char, fqdn_len+1); + memcpy(fqdn_rule->FQDN, fqdn, fqdn_len); + fqdn_rule->len=fqdn_len; + fqdn_rule->is_suffix_match=is_suffix_match; + fqdn_rule->id=id; + return fqdn_rule; +} +void fqdn_rule_free(struct FQDN_rule* fqdn_rule) +{ + assert(fqdn_rule->user_tag==NULL); + free(fqdn_rule->FQDN); + fqdn_rule->FQDN=NULL; + free(fqdn_rule); + return; +} +void _notype_fqdn_rule_free(void* p) +{ + fqdn_rule_free((struct FQDN_rule*)p); + return; +} +struct xx_plugin_ex_free_wrapper +{ + struct EX_data_rt* ex_data_rt; + char* row; + size_t key_offset; + size_t key_len; + void* logger; +}; +void xx_plugin_ex_data_wrapper_free(void* ex_data) +{ + struct xx_plugin_ex_free_wrapper* wrapper=(struct xx_plugin_ex_free_wrapper*)ex_data; + EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger); + free(wrapper->row); + wrapper->key_offset=0; + wrapper->key_len=0; + free(wrapper); + return; +} static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num) { @@ -127,7 +178,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche break; case TABLE_TYPE_PLUGIN: table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, - table_schema->plugin.estimate_size, table_schema->plugin.ex_schema.key2index_func, NULL); if(table_schema->plugin.have_exdata) @@ -136,8 +186,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche } break; case TABLE_TYPE_IP_PLUGIN: - table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, - table_schema->ip_plugin.estimate_size, + table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, table_schema->ip_plugin.ex_schema.key2index_func, free); if(table_schema->ip_plugin.have_exdata) @@ -146,6 +195,16 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche } table_rt->ip_plugin.bin=Maat_garbage_bin_new(0); break; + case TABLE_TYPE_FQDN_PLUGIN: + table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, + table_schema->fqdn_plugin.ex_schema.key2index_func, + _notype_fqdn_rule_free); + if(table_schema->fqdn_plugin.have_exdata) + { + EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema); + } + table_rt->fqdn_plugin.bin=Maat_garbage_bin_new(0); + break; default: break; } @@ -196,6 +255,12 @@ static void table_runtime_free(struct Maat_table_runtime* p) EX_data_rt_free(p->ip_plugin.ex_data_rt); assert(p->ip_plugin.new_ip_matcher==NULL); break; + case TABLE_TYPE_FQDN_PLUGIN: + FQDN_engine_free(p->fqdn_plugin.fqdn_engine); + Maat_garbage_bin_free(p->fqdn_plugin.bin); + EX_data_rt_free(p->fqdn_plugin.ex_data_rt); + assert(p->fqdn_plugin.new_fqdn_engine==NULL); + break; case TABLE_TYPE_PLUGIN: EX_data_rt_free(p->plugin.ex_data_rt); break; @@ -358,6 +423,156 @@ void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*)); return; } + +void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) +{ + struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(table_schema->fqdn_plugin); + struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin); + size_t is_valid_offset=0, valid_len=0; + size_t is_suffix_flag_offset=0, is_suffix_flag_len=0; + size_t row_id_offset=0, row_id_len=0; + size_t fqdn_offset=0, fqdn_len=0; + struct FQDN_rule* fqdn_rule=NULL; + int ret=0; + struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL; + if(fqdn_plugin_schema->have_exdata) + { + ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: cannot find is_valid column %d of %s", + fqdn_plugin_schema->valid_flag_column, row); + return; + } + ret=Maat_helper_read_column(row, fqdn_plugin_schema->row_id_column, &row_id_offset, &row_id_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: cannot find row id column %d of %s", + fqdn_plugin_schema->row_id_column, row); + return; + } + ret=Maat_helper_read_column(row, fqdn_plugin_schema->is_suffix_flag_column, &is_suffix_flag_offset, &is_suffix_flag_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: cannot find is_suffix_match column %d of %s", + fqdn_plugin_schema->is_suffix_flag_column, row); + return; + } + ret=Maat_helper_read_column(row, fqdn_plugin_schema->fqdn_column, &fqdn_offset, &fqdn_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: cannot find fqdn column %d of %s", + fqdn_plugin_schema->fqdn_column, row); + return; + } + fqdn_rule=fqdn_rule_new((unsigned int)atoi(row+row_id_offset), row+fqdn_offset, fqdn_len, atoi(row+is_suffix_flag_offset)); + + if(atoi(row+is_valid_offset)==1)//add + { + + EX_data_rt_row2EX_data(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, fqdn_rule, logger); + } + else + { + wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1); + wrapper_for_free->row=_maat_strdup(row); + wrapper_for_free->ex_data_rt=fqdn_plugin_rt->ex_data_rt; + wrapper_for_free->key_len=row_id_len; + wrapper_for_free->key_offset=row_id_offset; + wrapper_for_free->logger=logger; + Maat_garbage_bagging(fqdn_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free); + } + } + else + { + EX_data_rt_cache_row_put(fqdn_plugin_rt->ex_data_rt, row); + } + fqdn_plugin_rt->changed_flag=1; + return; +} + +int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt) +{ + struct FQDN_engine* new_fqdn_engine=NULL; + struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin; + assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); + struct EX_data_container **exc_array=NULL; + Maat_garbage_collect_routine(fqdn_rt->bin); + struct FQDN_rule* rules=NULL; + size_t rule_cnt=0, i=0; + if(!fqdn_rt->changed_flag) + { + assert(0==Maat_garbage_bin_get_size(fqdn_rt->bin)); + return 0; + } + + rule_cnt=EX_data_rt_list_all_ex_container(fqdn_rt->ex_data_rt, &exc_array); + rules=ALLOC(struct FQDN_rule, rule_cnt); + for(i=0; i0) + { + new_fqdn_engine=FQDN_engine_new(rules, rule_cnt); + } + fqdn_rt->new_fqdn_engine=new_fqdn_engine; + free(rules); + free(exc_array); + return 1; +} +struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt) +{ + struct FQDN_engine* old_one=table_rt->fqdn_plugin.fqdn_engine; + table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine; + assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); + table_rt->fqdn_plugin.new_fqdn_engine=NULL; + table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt); + return old_one; +} + +int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) +{ + size_t i=0; + const char* row=NULL; + struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin); + EX_data_rt_set_schema(fqdn_plugin_rt->ex_data_rt, &table_schema->fqdn_plugin.ex_schema); + for(i=0; iex_data_rt); i++) + { + row=EX_data_rt_cached_row_get(fqdn_plugin_rt->ex_data_rt, i); + Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger); + } + EX_data_rt_clear_row_cache(fqdn_plugin_rt->ex_data_rt); + Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt); + Maat_table_runtime_apply_new_fqdn_engine(table_rt);//returned NULL. + return 0; +} +int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) +{ + struct FQDN_match results[size]; + int n_result=0, i=0; + if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN) + { + return -1; + } + if(!table_rt->fqdn_plugin.fqdn_engine) + { + return 0; + } + n_result=FQDN_engine_search(table_rt->fqdn_plugin.fqdn_engine, query_fqdn, strlen(query_fqdn), results, size); + for(i=0; ifqdn_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag); + } + return n_result; +} + int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) { long i=0,data_size=0; @@ -459,41 +674,28 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* ip_plugin->changed_flag=0; return 1; } + + + struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt) { struct ip_matcher* old_one=table_rt->ip_plugin.ip_matcher; table_rt->ip_plugin.ip_matcher=table_rt->ip_plugin.new_ip_matcher; assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN); table_rt->ip_plugin.new_ip_matcher=NULL; + table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt); return old_one; } -struct ip_plugin_ex_free_wrapper -{ - struct EX_data_rt* ex_data_rt; - char* row; - size_t key_offset; - size_t key_len; - void* logger; -}; -void ip_plugin_ex_data_wrapper_free(void* ex_data) -{ - struct ip_plugin_ex_free_wrapper* wrapper=(struct ip_plugin_ex_free_wrapper*)ex_data; - EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger); - free(wrapper->row); - wrapper->key_offset=0; - wrapper->key_len=0; - free(wrapper); - return; -} + void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) { struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin); size_t is_valid_offset=0, valid_len=0; - size_t key_offset=0, key_len=0; + size_t row_id_offset=0, row_id_len=0; struct ip_rule* ip_rule=NULL; int ret=0; - struct ip_plugin_ex_free_wrapper* wrapper_for_free=NULL; + struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL; if(ip_plugin_schema->have_exdata) { ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); @@ -504,7 +706,7 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s ip_plugin_schema->row_id_column, row); return; } - ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &key_offset, &key_len); + ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &row_id_offset, &row_id_len); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, @@ -523,17 +725,18 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s if(atoi(row+is_valid_offset)==1)//add { - EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, ip_rule, logger); + EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, ip_rule, logger); } else { - wrapper_for_free=ALLOC(struct ip_plugin_ex_free_wrapper, 1); + wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1); wrapper_for_free->row=_maat_strdup(row); wrapper_for_free->ex_data_rt=ip_plugin_rt->ex_data_rt; - wrapper_for_free->key_len=key_len; - wrapper_for_free->key_offset=key_offset; + wrapper_for_free->key_len=row_id_len; + wrapper_for_free->key_offset=row_id_offset; wrapper_for_free->logger=logger; - Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, ip_plugin_ex_data_wrapper_free); + Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free); + free(ip_rule); } } else diff --git a/src/inc_internal/FQDN_engine.h b/src/inc_internal/FQDN_engine.h new file mode 100644 index 0000000..d2213fc --- /dev/null +++ b/src/inc_internal/FQDN_engine.h @@ -0,0 +1,69 @@ +/* + * + * Copyright (c) 2020 + * String Algorithms Research Group + * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) + * National Engineering Laboratory for Information Security Technologies (NELIST) + * All rights reserved + * + * Written by: LIU YANBING (liuyanbing@iie.ac.cn) + * Last modification: 2020-09-01 + * + * This code is the exclusive and proprietary property of IIE-CAS and NELIST. + * Usage for direct or indirect commercial advantage is not allowed without + * written permission from the authors. + * + */ + +#ifndef H_FQDN_ENGINE_H +#define H_FQDN_ENGINE_H + +#ifdef __cplusplus +extern "C" { +#endif + + #include + + struct FQDN_rule + { + unsigned int id; + int is_suffix_match; /* is_suffix_match==0: exact match; is_suffix_match==1: longest suffix matching. */ + size_t len; + char * FQDN; /* Non-ASCII character is allowed. */ + void * user_tag; /* A transparent user tag for convenient accessing, the caller is responsible for its memory management. */ + }; + + struct FQDN_engine; + + struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule); + + struct FQDN_match + { + unsigned int id; + unsigned int offset; /* offset==0 for exact matching; offset>0 for longest suffix matching. */ + void * user_tag; + }; + + /* + *Function: + * Search FQDN in the rule base + *Paramters: + * instance[in]: Instance of FQDN engine + * FQDN[in]: FQDN for search + * FQDN_len[in]: Length of FQDN + * results[out]: An array to store matched FQDNs + * n_result[in]: Number of element in the result array + * Return: + * 0: No matched FQDN; + * >0: Number of matched FQNDs which were stored in results; + * <0: Error. + */ + int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result); + + void FQDN_engine_free(struct FQDN_engine * instance); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/inc_internal/Maat_ex_data.h b/src/inc_internal/Maat_ex_data.h index baa3d15..17230ab 100644 --- a/src/inc_internal/Maat_ex_data.h +++ b/src/inc_internal/Maat_ex_data.h @@ -3,7 +3,7 @@ struct EX_data_rt; -struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)); +struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)); void EX_data_rt_free(struct EX_data_rt* p); void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema); void EX_data_rt_cache_row_put(struct EX_data_rt* p, const char* row); @@ -23,4 +23,5 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container); size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array); void* EX_data_container_get_user_data(struct EX_data_container* ex_container); +size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt); diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 39016f3..dba92e2 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -16,6 +16,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" #include "alignment_int64.h" +#include "FQDN_engine.h" #include #include #include @@ -78,6 +79,15 @@ struct db_digest_rule short confidence_degree; int is_valid; }; +struct db_fqdn_rule +{ + int region_id; + int group_id; + int is_suffix_match; + char* fqdn; + int is_valid; +}; + struct Maat_rule_head { int config_id; @@ -204,7 +214,7 @@ struct Maat_scanner mcore_long_t ref_cnt; rule_scanner_t region; size_t gie_update_q_size; - size_t ip_plugin_update_q_size; + size_t xx_plugin_update_q_size; size_t to_update_group_cnt; size_t to_update_compile_cnt; diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index f607827..c34a1de 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -17,6 +17,28 @@ enum USER_REGION_ENCODE USER_REGION_ENCODE_ESCAPE, USER_REGION_ENCODE_BASE64 }; +enum MAAT_SCAN_TYPE +{ + SCAN_TYPE_INVALID=-1, + SCAN_TYPE_NONE=0, + SCAN_TYPE_PLUGIN, + SCAN_TYPE_IP_PLUGIN, + SCAN_TYPE_FQDN_PLUGIN, + SCAN_TYPE_IP, + SCAN_TYPE_INTERVAL, + SCAN_TYPE_STRING, + __SCAN_TYPE_MAX +}; + +enum MAAT_TABLE_COMPONENT_TYPE +{ + COMPONENT_TABLE_TYPE_NONE=-1, + COMPONENT_TABLE_TYPE_SOURCE_IP=0, + COMPONENT_TABLE_TYPE_DESTINATION_IP, + COMPONENT_TABLE_TYPE_SESSION, + __COMPONENT_TABLE_TYPE_MAX +}; + enum MAAT_TABLE_TYPE { @@ -28,6 +50,8 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_DIGEST, TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_SIMILARITY, + TABLE_TYPE_INTERVAL_PLUS, + //Above are physical table for scan TABLE_TYPE_VIRTUAL, TABLE_TYPE_COMPOSITION, TABLE_TYPE_GROUP2GROUP, @@ -36,7 +60,7 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_COMPILE, TABLE_TYPE_PLUGIN, TABLE_TYPE_IP_PLUGIN, - TABLE_TYPE_INTERVAL_PLUS + TABLE_TYPE_FQDN_PLUGIN }; struct compile_ex_data_idx @@ -68,12 +92,11 @@ struct expr_table_schema }; struct virtual_table_schema { - int real_table_id; - char real_table_name[MAX_TABLE_NAME_LEN]; + int physical_table_id[__SCAN_TYPE_MAX]; }; struct composition_table_schema { - struct virtual_table_schema source_table, destination_table, session_table; + int component_table_id[__COMPONENT_TABLE_TYPE_MAX]; }; struct plugin_table_callback_schema { @@ -100,7 +123,6 @@ struct plugin_table_schema int foreign_columns[MAX_FOREIGN_CLMN_NUM]; int cb_plug_cnt; int have_exdata; - long long estimate_size; struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE]; struct EX_data_schema ex_schema; }; @@ -112,7 +134,17 @@ struct ip_plugin_table_schema int end_ip_column; int valid_flag_column; int rule_tag_column; - long long estimate_size; + int have_exdata; + struct EX_data_schema ex_schema; +}; +struct fqdn_plugin_table_schema +{ + + int row_id_column; + int is_suffix_flag_column; + int fqdn_column; + int valid_flag_column; + int rule_tag_column; int have_exdata; struct EX_data_schema ex_schema; }; @@ -129,6 +161,7 @@ struct Maat_table_schema struct expr_table_schema expr; struct plugin_table_schema plugin; struct ip_plugin_table_schema ip_plugin; + struct fqdn_plugin_table_schema fqdn_plugin; struct virtual_table_schema virtual_table; struct composition_table_schema composition; void* others;//group, ip, interval and digest don't have any special schema. @@ -144,8 +177,9 @@ void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr); size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr); size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr); -struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id); +struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id); struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id); +enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type); int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name); int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, @@ -181,6 +215,13 @@ int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_sche Maat_plugin_EX_key2index_func_t* key2index_func, long argl, void *argp, void* logger); +int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger); void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type); void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr); @@ -188,12 +229,7 @@ void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mg int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr); struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name); void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* table_name); -enum MAAT_TABLE_CHILD_TYPE -{ - CHILD_TABLE_TYPE_NONE=-1, - CHILD_TABLE_TYPE_SOURCE_IP=0, - CHILD_TABLE_TYPE_DESTINATION_IP, - CHILD_TABLE_TYPE_SESSION -}; -int Maat_table_get_child_id(struct Maat_table_schema* p_table, enum MAAT_TABLE_CHILD_TYPE type); + +int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type); +int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table); diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h index 15fc7a6..0ac2846 100644 --- a/src/inc_internal/Maat_table_runtime.h +++ b/src/inc_internal/Maat_table_runtime.h @@ -4,6 +4,7 @@ #include "IPMatcher.h" #include "gram_index_engine.h" +#include "FQDN_engine.h" #include "alignment_int64.h" #include "dynamic_array.h" #include @@ -14,7 +15,14 @@ struct similar_runtime GIE_handle_t* gie_handle; MESA_lqueue_head update_q; }; - +struct fqdn_plugin_runtime +{ + struct FQDN_engine* fqdn_engine; + struct FQDN_engine* new_fqdn_engine; + struct EX_data_rt* ex_data_rt; //for fqdn_plugin ONLY + struct Maat_garbage_bin* bin; + int changed_flag; +}; struct plugin_runtime { struct EX_data_rt* ex_data_rt; @@ -51,6 +59,7 @@ struct Maat_table_runtime union { struct similar_runtime similar; //for digest and similarity + struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin struct plugin_runtime plugin; struct ip_plugin_runtime ip_plugin; struct expr_runtime expr; @@ -86,4 +95,12 @@ int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt); struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt); +int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt); + +void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); +int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt); +struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt); +int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger); +int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); + diff --git a/test/maat_json.json b/test/maat_json.json index f7d8d59..8f835c3 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -60,7 +60,7 @@ "regions": [ { "table_name": "GeoLocation", - "table_type": "string", + "table_type": "expr", "table_content": { "keywords": "Greece.Sparta", "expr_type": "none", @@ -69,6 +69,35 @@ } } ] + }, + { + "group_name": "FQDN_OBJ1", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "sports.example.com", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "FQDN_CAT1", + "regions": [ + { + "table_name": "INTERGER_PLUS", + "table_type": "intval_plus", + "table_content": { + "district": "fqdn_cat_id", + "low_boundary": 1724, + "up_boundary": 1724 + } + } + ] } ], "rules": [ @@ -1220,7 +1249,7 @@ "do_blacklist": 0, "do_log": 0, "effective_rage": 0, - "user_region": "VirtualWithPhysical", + "user_region": "VirtualWithOnePhysical", "is_valid": "yes", "groups": [ { @@ -1850,51 +1879,85 @@ ] } ] - } + }, + { + "compile_id": 180, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "Hierarchy_VirtualWithTwoPhysical", + "is_valid": "yes", + "groups": [ + { + "group_name":"FQDN_OBJ1", + "virtual_table":"VIRTUAL_SSL_SNI", + "not_flag" : 0, + "clause_index":0 + }, + { + "group_name":"FQDN_CAT1", + "virtual_table":"VIRTUAL_SSL_SNI", + "not_flag" : 0, + "clause_index":0 + } + ] + } ], "plugin_table": [ { "table_name": "QD_ENTRY_INFO", "table_content": [ - "1\t192.168.0.1\t101\t1", - "2\t192.168.0.2\t101\t1", - "3\t192.168.1.1\t102\t1" + "1\t192.168.0.1\t101\t1", + "2\t192.168.0.2\t101\t1", + "3\t192.168.1.1\t102\t1" ] }, { "table_name": "TEST_PLUGIN_TABLE", "table_content": [ - "1\t3388\t99\t1", - "2\t3355\t66\t1", - "3\tcccc\t11\t1" + "1\t3388\t99\t1", + "2\t3355\t66\t1", + "3\tcccc\t11\t1" ] }, { "table_name": "TEST_EFFECTIVE_RANGE_TABLE", "table_content": [ - "1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"绉诲姩\"]}]]}\t1111", - "2\tSUCCESS\t66\t1\t0\t222", - "3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\",\"涓婃捣/娴︿笢/闄嗗鍢碶"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"鑱旈歕"]}],[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]},{\"tag\":\"isp\",\"value\":[\"鑱旈歕"]}]]}\t333", - "4\tSUCCESS\t66\t1\t{}\t444", - "5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]}]]}\t444", - "6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444" + "1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"绉诲姩\"]}]]}\t1111", + "2\tSUCCESS\t66\t1\t0\t222", + "3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\",\"涓婃捣/娴︿笢/闄嗗鍢碶"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"鑱旈歕"]}],[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]},{\"tag\":\"isp\",\"value\":[\"鑱旈歕"]}]]}\t333", + "4\tSUCCESS\t66\t1\t{}\t444", + "5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]}]]}\t444", + "6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444" ] }, { "table_name": "IR_INTERCEPT_IP", "table_content": [ - "1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34", - "161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42" + "1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34", + "161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42" ] }, { "table_name": "TEST_IP_PLUGIN_WITH_EXDATA", "table_content": [ - "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1", + "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1", "102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1", - "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1", + "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1", "104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1" ] + }, + { + "table_name": "TEST_FQDN_PLUGIN_WITH_EXDATA", + "table_content": [ + "201\t0\twww.example1.com\tcatid=1\t1", + "202\t1\t.example1.com\tcatid=1\t1", + "203\t0\tnews.example1.com\tcatid=2\t1", + "204\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1", + "205\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1" + ] } ] } diff --git a/test/table_info.conf b/test/table_info.conf index 64387a3..adfaf31 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -55,4 +55,6 @@ 32 DESTINATION_IP_ASN virtual AS_NUMBER -- 33 GeoLocation expr UTF8 UTF8 yes 0 34 SOURCE_IP_GEO virtual GeoLocation -- -35 INTERGER_PLUS intval_plus -- \ No newline at end of file +35 INTERGER_PLUS intval_plus -- +36 TEST_FQDN_PLUGIN_WITH_EXDATA fqdn_plugin {"row_id":1,"is_suffix_match":2,"fqdn":3,"valid":5} -- +37 VIRTUAL_SSL_SNI virtual ["KEYWORDS_TABLE","INTERGER_PLUS"] -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 6d9e01a..2e6876e 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -310,6 +310,86 @@ TEST(IP_Plugin_Table, EX_DATA) } +#define FQDN_PLUGIN_EX_DATA +struct fqdn_plugin_ud +{ + int rule_id; + int catid; + int ref_cnt; +}; +void fqdn_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + int *counter=(int *)argp, ret=0; + size_t column_offset=0, column_len=0; + struct fqdn_plugin_ud* ud=(struct fqdn_plugin_ud*)calloc(sizeof(struct fqdn_plugin_ud), 1); + ret=Maat_helper_read_column(table_line, 1, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + ud->rule_id=atoi(table_line+column_offset); + ret=Maat_helper_read_column(table_line, 4, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + sscanf(table_line+column_offset, "catid=%d",&ud->catid); + ud->ref_cnt=1; + *ad=ud; + (*counter)++; + return; +} +void fqdn_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*ad); + u->ref_cnt--; + if(u->ref_cnt>0) return; + free(u); + *ad=NULL; +} +void fqdn_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) +{ + struct fqdn_plugin_ud* u=(struct fqdn_plugin_ud*)(*from); + u->ref_cnt++; + *to=u; +} +TEST(FQDN_Plugin_Table, EX_DATA) +{ + + int fqdn_plugin_ex_data_counter=0, i=0; + const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA"; + int table_id=0, ret=0; + table_id=Maat_table_register(g_feather, table_name); + ASSERT_GT(table_id, 0); + ret=Maat_fqdn_plugin_EX_register(g_feather, table_id, + fqdn_plugin_EX_new_cb, + fqdn_plugin_EX_free_cb, + fqdn_plugin_EX_dup_cb, + 0, &fqdn_plugin_ex_data_counter); + ASSERT_TRUE(ret>=0); + EXPECT_EQ(fqdn_plugin_ex_data_counter, 5); + + struct fqdn_plugin_ud* result[4]; + + ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "www.example1.com", (void**)result, 4); + ASSERT_EQ(ret, 2); + EXPECT_EQ(result[0]->rule_id, 201); + EXPECT_EQ(result[1]->rule_id, 202); + + for(i=0; irule_id, 205); + EXPECT_EQ(result[1]->rule_id, 204); + for(i=0; i=0); + EXPECT_EQ(fqdn_plugin_ex_data_counter, 5); + + + struct fqdn_plugin_ud* result[4]; + + ret=Maat_fqdn_plugin_get_EX_data(g_feather, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4); + ASSERT_EQ(ret, 2); + for(i=0; i