diff --git a/include/maat/maat.h b/include/maat/maat.h index 1a83790..c6bc624 100644 --- a/include/maat/maat.h +++ b/include/maat/maat.h @@ -92,6 +92,15 @@ int maat_table_callback_register(struct maat *instance, int table_id, maat_update_callback_t *update, maat_finish_callback_t *finish, void *u_para); + +/* maat compile table API */ +int maat_compile_table_ex_schema_register(struct maat *instance, int table_id, + maat_rule_ex_new_func_t *new_func, + maat_rule_ex_free_func_t *free_func, + maat_rule_ex_dup_func_t *dup_func, + long argl, void *argp); +void *maat_compile_table_get_ex_data(struct maat *instance, int table_id, int compile_id, size_t idx); + /* maat plugin table API */ int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id, maat_plugin_ex_new_func_t *new_func, @@ -100,8 +109,9 @@ int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id, long argl, void *argp); /* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register, caller is responsible to free the data. */ -void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id, - const char *key, size_t key_len); +int maat_plugin_table_get_ex_data(struct maat *instance, int table_id, + const char *key, size_t key_len, + void **ex_data_array, size_t n_ex_data); /* maat scan API */ struct maat_state; diff --git a/scanner/CMakeLists.txt b/scanner/CMakeLists.txt index 3e1fb3b..d07b230 100644 --- a/scanner/CMakeLists.txt +++ b/scanner/CMakeLists.txt @@ -5,7 +5,5 @@ include_directories(${PROJECT_SOURCE_DIR}/deps) include_directories(${PROJECT_SOURCE_DIR}/include) include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) -add_library(adapter-static adapter_hs.cpp bool_matcher.cpp) - -#add_executable(adapter_gtest adapter_hs_gtest.cpp) +add_library(adapter-static adapter_hs.cpp bool_matcher.cpp fqdn_engine.cpp) target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static ipmatcher-static) diff --git a/scanner/adapter_hs.cpp b/scanner/adapter_hs.cpp index fb3dad9..d669896 100644 --- a/scanner/adapter_hs.cpp +++ b/scanner/adapter_hs.cpp @@ -12,6 +12,9 @@ #include #include #include +#include +#include +#include #include "adapter_hs.h" #include "uthash/utarray.h" @@ -19,7 +22,20 @@ #include "maat_utils.h" #include "bool_matcher.h" -#define MODULE_ADAPTER_HS module_name_str("maat.adapter_hs") +pid_t hs_gettid() +{ + return syscall(SYS_gettid); +} + +static const char *hs_module_name_str(const char *name) +{ + static __thread char module[64]; + snprintf(module,sizeof(module),"%s(%d)", name, hs_gettid()); + + return module; +} + +#define MODULE_ADAPTER_HS hs_module_name_str("maat.adapter_hs") struct adpt_hs_compile_data { unsigned int *ids; @@ -58,6 +74,33 @@ struct adapter_hs_stream { UT_array *pattern_id_set; }; +struct matched_pattern { + unsigned int pattern_id; + unsigned long matched_l_offset; + unsigned long matched_r_offset; + UT_hash_handle hh; +}; + +struct matched_pattern_set { + UT_array *pat_ids; + unsigned int pattern_id; + unsigned long long l_matched; + unsigned long long r_matched; + struct matched_pattern *pat_hash; +}; + +struct pattern_offset { + unsigned int pattern_id; + unsigned long l_offset; + unsigned long r_offset; +}; + +struct hs_tag { + size_t n_pat_offset; + struct pattern_offset *pat_offset; + void *user_tag; +}; + static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread, int pattern_type, struct log_handle *logger) @@ -214,6 +257,11 @@ adapter_hs_initialize(int scan_mode, size_t n_worker_thread, } for (size_t j = 0; j < expr_array[i].n_patterns; j++) { + /* pat_len should not 0 */ + if (0 == expr_array[i].patterns[j].pat_len) { + return NULL; + } + if (expr_array[i].patterns[j].type == PATTERN_TYPE_STR) { literal_pattern_num++; } else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) { @@ -245,12 +293,26 @@ adapter_hs_initialize(int scan_mode, size_t n_worker_thread, /* populate adpt_hs_compile_data and bool_expr */ for (size_t i = 0; i < n_expr_array; i++) { + struct hs_tag *hs_tag = ALLOC(struct hs_tag, 1); + hs_tag->pat_offset = ALLOC(struct pattern_offset, expr_array[i].n_patterns); + hs_tag->n_pat_offset = expr_array[i].n_patterns; + hs_tag->user_tag = expr_array[i].user_tag; + for (size_t j = 0; j < expr_array[i].n_patterns; j++) { size_t pat_len = 0; + hs_tag->pat_offset[j].pattern_id = pattern_id; + hs_tag->pat_offset[j].l_offset = expr_array[i].patterns[j].l_offset; + hs_tag->pat_offset[j].r_offset = expr_array[i].patterns[j].r_offset; + if (expr_array[i].patterns[j].type == PATTERN_TYPE_STR) { literal_cd->ids[literal_index] = pattern_id; - literal_cd->flags[literal_index] = HS_FLAG_CASELESS; + + /* set flags */ + literal_cd->flags[literal_index] = HS_FLAG_SOM_LEFTMOST; + if (expr_array[i].patterns[j].case_sensitive == 1) { + literal_cd->flags[literal_index] |= HS_FLAG_CASELESS; + } pat_len = expr_array[i].patterns[j].pat_len; literal_cd->pattern_lens[literal_index] = pat_len; @@ -261,8 +323,13 @@ adapter_hs_initialize(int scan_mode, size_t n_worker_thread, literal_index++; } else { regex_cd->ids[regex_index] = pattern_id; - regex_cd->flags[regex_index] = HS_FLAG_CASELESS; + /* set flags */ + regex_cd->flags[regex_index] = HS_FLAG_SOM_LEFTMOST; + if (expr_array[i].patterns[j].case_sensitive == 1) { + regex_cd->flags[literal_index] |= HS_FLAG_CASELESS; + } + pat_len = expr_array[i].patterns[j].pat_len; regex_cd->pattern_lens[regex_index] = pat_len; regex_cd->patterns[regex_index] = ALLOC(char, pat_len); @@ -276,7 +343,7 @@ adapter_hs_initialize(int scan_mode, size_t n_worker_thread, } exprs[i].expr_id = expr_array[i].expr_id; exprs[i].item_num = expr_array[i].n_patterns; - exprs[i].user_tag = expr_array[i].user_tag; + exprs[i].user_tag = hs_tag; } if (literal_cd != NULL) { @@ -376,9 +443,9 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance) FREE(hs_instance); } -static inline int compare_pattern_id(const void* a, const void* b) +static inline int compare_pattern_id(const void *a, const void *b) { - long long ret= *(unsigned long long *)a - *(unsigned long long *)b; + long long ret = *(unsigned long long *)a - *(unsigned long long *)b; if (0 == ret) { return 0; @@ -389,21 +456,28 @@ static inline int compare_pattern_id(const void* a, const void* b) } } -UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; - +UT_icd ut_pattern_id_icd = {sizeof(unsigned int), NULL, NULL, NULL}; /** * @param id: pattern id */ int matched_event_cb(unsigned int id, unsigned long long from, - unsigned long long to, unsigned int flags, void *ctx) { + unsigned long long to, unsigned int flags, + void *ctx) { // put id in set - UT_array *pattern_id_set = (UT_array *)ctx; - unsigned long long pattern_id = (unsigned long long)id; - if (utarray_find(pattern_id_set, &pattern_id, compare_pattern_id)) { + printf("matched_event_cb, expr_id:%u, from:%llu to:%llu\n", id, from, to); + + struct matched_pattern_set *matched_pattern_set = (struct matched_pattern_set *)ctx; + unsigned int pattern_id = id; + + if (utarray_find(matched_pattern_set->pat_ids, &pattern_id, compare_pattern_id)) { return -1; } - utarray_push_back(pattern_id_set, &pattern_id); - utarray_sort(pattern_id_set, compare_pattern_id); + + matched_pattern_set->pattern_id = pattern_id; + matched_pattern_set->l_matched = from; + matched_pattern_set->r_matched = to; + utarray_push_back(matched_pattern_set->pat_ids, &pattern_id); + utarray_sort(matched_pattern_set->pat_ids, compare_pattern_id); return 0; } @@ -420,16 +494,18 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, struct adapter_hs_runtime *hs_rt = hs_instance->hs_rt; hs_scratch_t *scratch = hs_rt->scratchs[thread_id]; - UT_array *pattern_id_set; hs_error_t err; - utarray_new(pattern_id_set, &ut_pattern_id_icd); - utarray_reserve(pattern_id_set, hs_instance->n_patterns); + struct matched_pattern_set matched_pat_set; + + matched_pat_set.pat_hash = NULL; + utarray_new(matched_pat_set.pat_ids, &ut_pattern_id_icd); + utarray_reserve(matched_pat_set.pat_ids, hs_instance->n_patterns); int err_count = 0; if (hs_rt->literal_db != NULL) { err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, - matched_event_cb, pattern_id_set); + matched_event_cb, &matched_pat_set); if (err != HS_SUCCESS) { //log_error() err_count++; @@ -438,7 +514,7 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, if (hs_rt->regex_db != NULL) { err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, - matched_event_cb, pattern_id_set); + matched_event_cb, &matched_pat_set); if (err != HS_SUCCESS) { //log_error() err_count++; @@ -449,37 +525,66 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, return -1; } - size_t pattern_set_size = utarray_len(pattern_id_set); - unsigned long long items[pattern_set_size]; - memset(items, 0, sizeof(unsigned long long) * pattern_set_size); - for (size_t i = 0; i < pattern_set_size; i++) { - items[i] = *(unsigned long long *)utarray_eltptr(pattern_id_set, i); + struct matched_pattern *matched_pat = ALLOC(struct matched_pattern, 1); + unsigned int pattern_id = matched_pat_set.pattern_id; + matched_pat->pattern_id = pattern_id; + matched_pat->matched_l_offset = matched_pat_set.l_matched; + matched_pat->matched_r_offset = matched_pat_set.r_matched; + + HASH_ADD_INT(matched_pat_set.pat_hash, pattern_id, matched_pat); + + size_t matched_pattern_ids_cnt = utarray_len(matched_pat_set.pat_ids); + unsigned long long items[matched_pattern_ids_cnt]; + memset(items, 0, sizeof(unsigned long long) * matched_pattern_ids_cnt); + + for (size_t i = 0; i < matched_pattern_ids_cnt; i++) { + items[i] = *(unsigned long long *)utarray_eltptr(matched_pat_set.pat_ids, i); } int ret = 0; int matched_index = 0; - + int real_matched_index = 0; struct bool_expr_match *bool_matcher_results = NULL; bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr); - int bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, pattern_set_size, + int bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, matched_pattern_ids_cnt, bool_matcher_results, hs_instance->n_expr); if (bool_matcher_ret < 0) { ret = -1; goto next; } - if (bool_matcher_ret > n_result) { + if (bool_matcher_ret > (int)n_result) { bool_matcher_ret = n_result; } for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { - results[matched_index].item_id = bool_matcher_results[matched_index].expr_id; - results[matched_index].user_tag = bool_matcher_results[matched_index].user_tag; + //results[matched_index].item_id = bool_matcher_results[matched_index].expr_id; + struct hs_tag *hs_tag = (struct hs_tag *)bool_matcher_results[matched_index].user_tag; + for (size_t i = 0; i < hs_tag->n_pat_offset; i++) { + //命中的 item = pat1 & pat2 + int pattern_id = hs_tag->pat_offset[i].pattern_id; + struct matched_pattern *matched_pat = NULL; + HASH_FIND_INT(matched_pat_set.pat_hash, &pattern_id, matched_pat); + if (matched_pat) { + if (matched_pat->matched_l_offset >= hs_tag->pat_offset[i].l_offset && + matched_pat->matched_r_offset <= hs_tag->pat_offset[i].r_offset) { + results[real_matched_index].item_id = bool_matcher_results[matched_index].expr_id; + results[real_matched_index].user_tag = hs_tag->user_tag; + real_matched_index++; + break; + } + } + } } *n_hit_result = bool_matcher_ret; next: FREE(bool_matcher_results); - utarray_free(pattern_id_set); + struct matched_pattern *pattern = NULL, *tmp_pattern = NULL; + HASH_ITER(hh, matched_pat_set.pat_hash, pattern, tmp_pattern) { + HASH_DELETE(hh, matched_pat_set.pat_hash, pattern); + FREE(pattern); + } + utarray_free(matched_pat_set.pat_ids); return ret; } @@ -560,7 +665,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data goto next; } - if (bool_matcher_ret > n_result) { + if (bool_matcher_ret > (int)n_result) { bool_matcher_ret = n_result; } diff --git a/scanner/adapter_hs.h b/scanner/adapter_hs.h index 73335b5..4c42e03 100644 --- a/scanner/adapter_hs.h +++ b/scanner/adapter_hs.h @@ -47,6 +47,16 @@ typedef struct { /* pattern type */ int type; + /* default 0(sensitive) 1(insensitive) */ + int case_sensitive; + + /* (0)sub string match (1)complete match */ + int match_mode; + + /* just match in scan_data's range of [l_offset, r_offset] */ + int l_offset; + int r_offset; + /* start pointer of pattern */ char *pat; /* pattern length */ diff --git a/scanner/fqdn_engine.cpp b/scanner/fqdn_engine.cpp new file mode 100644 index 0000000..99ef716 --- /dev/null +++ b/scanner/fqdn_engine.cpp @@ -0,0 +1,352 @@ +/* + * + * Copyright (c) 2020 + * String Algorithms Research Group + * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) + * National Engineering Laboratory for Information Security Technologies (NELIST) + * All rights reserved + * + * Written by: LIU YANBING (liuyanbing@iie.ac.cn) + * Last modification: 2020-09-01 + * + * This code is the exclusive and proprietary property of IIE-CAS and NELIST. + * Usage for direct or indirect commercial advantage is not allowed without + * written permission from the authors. + * + */ + +#include "fqdn_engine.h" +#include +#include +#include +#include +#include + +/*************************************************************************************/ + +//#include +//#define popcnt_u64 _mm_popcnt_u64 +//Use gcc builtin function to replace SSE4.2 instruction for portability + +#ifdef _MSC_VER +#include +#define popcnt_u64 __popcnt +#else +#define popcnt_u64 __builtin_popcountl +#endif + +#define FOR(i, n) for(int i=0, _n=(int)(n); i<_n; i++) + +struct packedRT_t +{ + unsigned long long bitmap[4]; + unsigned int A; + unsigned char B[4]; +}; + +void * aligned_malloc(size_t size, size_t align) +{ + void * malloc_ptr; + void * aligned_ptr; + + /* Error if align is not a power of two. */ + if (align & (align - 1)) + { + return ((void*) 0); + } + + if (align==0 || size == 0) + { + return ((void *) 0); + } + + malloc_ptr = malloc (sizeof(void *) + align - 1 + size); + if (!malloc_ptr) + { + return ((void *) 0); + } + + aligned_ptr = (void *) (((size_t)malloc_ptr + sizeof(void *) + align-1) & ~(align-1)); + + ((void **) aligned_ptr) [-1] = malloc_ptr; + + return aligned_ptr; +} + +void aligned_free(void * aligned_ptr) +{ + if (aligned_ptr) + { + free (((void **) aligned_ptr) [-1]); + } +} + +/*************************************************************************************/ +struct domain_impl_t +{ + unsigned int id; + int suf_match; + unsigned int len; + unsigned long long hash; /*��64λ��ϣֵΨһ��ʾһ������*/ + domain_impl_t * next; + void * utag; +}; + +class CHashTrieFQDN +{ +public: + CHashTrieFQDN(); + ~CHashTrieFQDN(); + + int initialize(const struct FQDN_rule * rules, size_t n_rule); + int search(const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result); + +protected: + unsigned int rank(unsigned int h); + +protected: + unsigned int m_num; + domain_impl_t * m_domains; + unsigned int m_H; + unsigned int m_max_pat_len; + unsigned long long * m_B; + packedRT_t * m_RT; + domain_impl_t ** m_matched; + unsigned char m_case_tab[256]; +}; + + +const unsigned long long ONE=1; + +#define is_bit_set(tbl, off) ( tbl[off>>6] & ( ONE<<(off&63) ) ) +#define set_bit(tbl, off) ( tbl[off>>6] |= ( ONE<<(off&63) ) ) + +const unsigned long long A=6364136223846793005LL; + +CHashTrieFQDN::CHashTrieFQDN() +{ + m_num=0; + m_domains=NULL; + m_B=NULL; + m_RT=NULL; + m_matched=NULL; + FOR(c, 256) m_case_tab[c]=tolower(c); +} + +CHashTrieFQDN::~CHashTrieFQDN() +{ + if(m_domains!=NULL) + { + delete [] m_domains; + } + + if(m_B!=NULL) + { + delete [] m_B; + } + + if(m_RT!=NULL) + { + aligned_free(m_RT); + } + + if(m_matched!=NULL) + { + delete [] m_matched; + } +} + +int CHashTrieFQDN::initialize(const struct FQDN_rule * rules, size_t n_rule) +{ + long long mem_bytes=0; + + if(n_rule==0) return -1; + m_num=n_rule; + m_domains=new domain_impl_t[m_num]; + mem_bytes+=m_num*sizeof(domain_impl_t); + + unsigned int N=m_num; + m_max_pat_len=0; + + FOR(k, m_num) + { + m_domains[k].id =rules[k].id; + m_domains[k].suf_match=rules[k].is_suffix_match; + m_domains[k].len=rules[k].len; + m_domains[k].next=NULL; + m_domains[k].utag=rules[k].user_tag; + + FOR(j, rules[k].len) + { + if(rules[k].FQDN[j]=='.') ++N; + } + + if(m_max_pat_len>6]; + mem_bytes+=(m_H>>6)*sizeof(unsigned long long); + FOR(i, (m_H>>6)) m_B[i]=0; + + m_RT=(packedRT_t *)aligned_malloc(sizeof(packedRT_t)*((m_H>>8)+1), 64); + mem_bytes+=((m_H>>8)+1)*sizeof(packedRT_t); + FOR(i, (m_H>>8)) + { + FOR(j, 4) m_RT[i].bitmap[j]=0; + } + + FOR(k, m_num) + { + const unsigned char * pb=(const unsigned char *)rules[k].FQDN; + m_domains[k].hash=0; + + for(int j=rules[k].len-1; j>=-1; --j) + { + if(j==-1 || pb[j]=='.') + { + unsigned int h=m_domains[k].hash&(m_H-1); + set_bit(m_B, h); + if(j==-1) + { + int q=h&255; + m_RT[h>>8].bitmap[q>>6]|=(ONE<<(q&63)); + } + } + else + { + m_domains[k].hash=A*m_domains[k].hash+m_case_tab[pb[j]]; + } + } + } + + m_RT[0].A=0; + FOR(i, (m_H>>8)) + { + m_RT[i].B[0]=0; + m_RT[i].B[1]= popcnt_u64(m_RT[i].bitmap[0]); + m_RT[i].B[2]= m_RT[i].B[1]+popcnt_u64(m_RT[i].bitmap[1]); + m_RT[i].B[3]= m_RT[i].B[2]+popcnt_u64(m_RT[i].bitmap[2]); + m_RT[i+1].A=m_RT[i].A+m_RT[i].B[3]+popcnt_u64(m_RT[i].bitmap[3]); + } + + int tn=m_RT[m_H>>8].A; + + m_matched=new domain_impl_t *[tn]; + mem_bytes+=tn*sizeof(domain_impl_t *); + FOR(i, tn) m_matched[i]=NULL; + + FOR(k, m_num) + { + unsigned int h=m_domains[k].hash&(m_H-1); + unsigned idx=rank(h); + m_domains[k].next=m_matched[idx]; + m_matched[idx]=&(m_domains[k]); + } + +// printf("mem_bytes=%u(MB)\n", mem_bytes/(1U<<20)); + + return 1; +} + +unsigned int CHashTrieFQDN::rank(unsigned int h) +{ + int p=(h>>8); + int r=((h&255)>>6); + int s=(h&63); + unsigned long long e=m_RT[p].bitmap[r]&((ONE<=-1; --j) + { + if(j==-1 || pb[j]=='.') + { + unsigned int h=hash&(m_H-1); + if(is_bit_set(m_B, h)==0) break; + HASH[t]=hash; + P[t]=j+1; + ++t; + } + else if(j+m_max_pat_len=0; t--) + { + unsigned int h=HASH[t]&(m_H-1); + int q=h&255; + + if(m_RT[h>>8].bitmap[q>>6]&(ONE<<(q&63))) + { + unsigned idx=rank(h); + + for(domain_impl_t * pt=m_matched[idx]; pt!=NULL; pt=pt->next) + { + if(P[t]!=0 && pt->suf_match==0) continue; + if(pt->len+P[t]==FQDN_len && pt->hash==HASH[t]) + { + //if(match_num>0 && P[t]!=results[match_num-1].offset) return match_num; + results[match_num].id=pt->id; + results[match_num].offset=P[t]; + results[match_num].user_tag=pt->utag; + ++match_num; + if(match_num==n_result) return match_num; + } + } + } + } + + return match_num; +} + +/*************************************************************************************/ +struct FQDN_engine +{ + CHashTrieFQDN ht; +}; + +struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule) +{ + struct FQDN_engine * instance=new struct FQDN_engine; + if(instance->ht.initialize(rules, n_rule)<0) + { + delete instance; + return NULL; + } + else + { + return instance; + } +} + +int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result) +{ + if(instance==NULL) return -1; + return instance->ht.search(FQDN, FQDN_len, results, n_result); +} + +void FQDN_engine_free(struct FQDN_engine * instance) +{ + if(instance!=NULL) delete instance; +} diff --git a/scanner/fqdn_engine.h b/scanner/fqdn_engine.h new file mode 100644 index 0000000..d2213fc --- /dev/null +++ b/scanner/fqdn_engine.h @@ -0,0 +1,69 @@ +/* + * + * Copyright (c) 2020 + * String Algorithms Research Group + * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) + * National Engineering Laboratory for Information Security Technologies (NELIST) + * All rights reserved + * + * Written by: LIU YANBING (liuyanbing@iie.ac.cn) + * Last modification: 2020-09-01 + * + * This code is the exclusive and proprietary property of IIE-CAS and NELIST. + * Usage for direct or indirect commercial advantage is not allowed without + * written permission from the authors. + * + */ + +#ifndef H_FQDN_ENGINE_H +#define H_FQDN_ENGINE_H + +#ifdef __cplusplus +extern "C" { +#endif + + #include + + struct FQDN_rule + { + unsigned int id; + int is_suffix_match; /* is_suffix_match==0: exact match; is_suffix_match==1: longest suffix matching. */ + size_t len; + char * FQDN; /* Non-ASCII character is allowed. */ + void * user_tag; /* A transparent user tag for convenient accessing, the caller is responsible for its memory management. */ + }; + + struct FQDN_engine; + + struct FQDN_engine * FQDN_engine_new(const struct FQDN_rule * rules, size_t n_rule); + + struct FQDN_match + { + unsigned int id; + unsigned int offset; /* offset==0 for exact matching; offset>0 for longest suffix matching. */ + void * user_tag; + }; + + /* + *Function: + * Search FQDN in the rule base + *Paramters: + * instance[in]: Instance of FQDN engine + * FQDN[in]: FQDN for search + * FQDN_len[in]: Length of FQDN + * results[out]: An array to store matched FQDNs + * n_result[in]: Number of element in the result array + * Return: + * 0: No matched FQDN; + * >0: Number of matched FQNDs which were stored in results; + * <0: Error. + */ + int FQDN_engine_search(struct FQDN_engine * instance, const char * FQDN, size_t FQDN_len, struct FQDN_match * results, size_t n_result); + + void FQDN_engine_free(struct FQDN_engine * instance); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4d8babe..475ebf7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,7 +11,8 @@ add_definitions(-fPIC) set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_flag.cpp maat_interval.cpp - maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_virtual.cpp) + maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_bool_plugin.cpp maat_fqdn_plugin.cpp + maat_virtual.cpp) set(LIB_SOURCE_FILES ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c) diff --git a/src/inc_internal/maat_bool_plugin.h b/src/inc_internal/maat_bool_plugin.h new file mode 100644 index 0000000..5ec7d0d --- /dev/null +++ b/src/inc_internal/maat_bool_plugin.h @@ -0,0 +1,56 @@ +/* +********************************************************************************************** +* File: maat_bool_plugin.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_BOOL_PLUGIN_H_ +#define _MAAT_BOOL_PLUGIN_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include "log/log.h" +#include "maat/maat.h" +#include "cJSON/cJSON.h" + +struct bool_plugin_runtime; + +/* bool plugin schema API */ +void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); +void bool_plugin_schema_free(void *bool_plugin_schema); + +/* ip plugin table ex data API */ +struct ex_data_schema *bool_plugin_table_get_ex_data_schema(void *bool_plugin_schema); + +int bool_plugin_table_set_ex_data_schema(void *bool_plugin_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger); + +/* ip plugin runtime API */ +void *bool_plugin_runtime_new(void *bool_plugin_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void bool_plugin_runtime_free(void *bool_plugin_runtime); + +int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_schema, + const char *line, int valid_column); +int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name); + +struct ex_data_runtime *bool_plugin_runtime_get_ex_data_rt(void *bool_plugin_runtime); + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index c1ffb85..5cb11b7 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -49,8 +49,7 @@ int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, maat_rule_ex_dup_func_t *dup_func, long argl, void *argp, struct log_handle *logger); -struct compile_ex_data_schema * -compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, size_t idx); +void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, int compile_id, size_t idx); size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema); @@ -62,7 +61,7 @@ void compile_runtime_free(void *compile_runtime); int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, int valid_column); -int compile_runtime_commit(void *compile_runtime); +int compile_runtime_commit(void *compile_runtime, const char *table_name); int compile_runtime_match(struct compile_runtime *compile_rt, int *compile_ids, size_t compile_ids_size, struct maat_state *state); diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index fa230f0..3bf7432 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -33,7 +33,7 @@ void expr_runtime_free(void *expr_runtime); int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column); -int expr_runtime_commit(void *expr_runtime); +int expr_runtime_commit(void *expr_runtime, const char *table_name); /* expr runtime scan API */ /** diff --git a/src/inc_internal/maat_flag.h b/src/inc_internal/maat_flag.h index 2818858..ca642e5 100644 --- a/src/inc_internal/maat_flag.h +++ b/src/inc_internal/maat_flag.h @@ -35,7 +35,7 @@ void flag_runtime_free(void *flag_runtime); int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *line, int valid_column); -int flag_runtime_commit(void *flag_runtime); +int flag_runtime_commit(void *flag_runtime, const char *table_name); /* flag runtime scan API */ /** diff --git a/src/inc_internal/maat_fqdn_plugin.h b/src/inc_internal/maat_fqdn_plugin.h new file mode 100644 index 0000000..5d6fb21 --- /dev/null +++ b/src/inc_internal/maat_fqdn_plugin.h @@ -0,0 +1,56 @@ +/* +********************************************************************************************** +* File: maat_fqdn_plugin.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_FQDN_PLUGIN_H_ +#define _MAAT_FQDN_PLUGIN_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include "log/log.h" +#include "maat/maat.h" +#include "cJSON/cJSON.h" + +struct fqdn_plugin_runtime; + +/* fqdn plugin schema API */ +void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); +void fqdn_plugin_schema_free(void *fqdn_plugin_schema); + +/* fqdn plugin table ex data API */ +struct ex_data_schema *fqdn_plugin_table_get_ex_data_schema(void *fqdn_plugin_schema); + +int fqdn_plugin_table_set_ex_data_schema(void *fqdn_plugin_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger); + +/* fqdn plugin runtime API */ +void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime); + +int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_schema, + const char *line, int valid_column); +int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name); + +struct ex_data_runtime *fqdn_plugin_runtime_get_ex_data_rt(void *fqdn_plugin_runtime); + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/src/inc_internal/maat_group.h b/src/inc_internal/maat_group.h index 45202b0..5bdbdea 100644 --- a/src/inc_internal/maat_group.h +++ b/src/inc_internal/maat_group.h @@ -47,7 +47,7 @@ int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column); -int group2group_runtime_commit(void *g2g_runtime); +int group2group_runtime_commit(void *g2g_runtime, const char *table_name); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_interval.h b/src/inc_internal/maat_interval.h index 7b13399..ab40f9f 100644 --- a/src/inc_internal/maat_interval.h +++ b/src/inc_internal/maat_interval.h @@ -35,7 +35,7 @@ void interval_runtime_free(void *interval_runtime); int interval_runtime_update(void *interval_runtime, void *interval_schema, const char *line, int valid_column); -int interval_runtime_commit(void *interval_runtime); +int interval_runtime_commit(void *interval_runtime, const char *table_name); /* interval runtime scan API */ /** diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h index 4130fc7..3352612 100644 --- a/src/inc_internal/maat_ip.h +++ b/src/inc_internal/maat_ip.h @@ -30,7 +30,7 @@ void ip_runtime_free(void *ip_runtime); int ip_runtime_update(void *ip_runtime, void *ip_schema, const char *line, int valid_column); -int ip_runtime_commit(void *ip_runtime); +int ip_runtime_commit(void *ip_runtime, const char *table_name); struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt); diff --git a/src/inc_internal/maat_ip_plugin.h b/src/inc_internal/maat_ip_plugin.h index 94ec278..2b8332f 100644 --- a/src/inc_internal/maat_ip_plugin.h +++ b/src/inc_internal/maat_ip_plugin.h @@ -45,7 +45,7 @@ void ip_plugin_runtime_free(void *ip_plugin_runtime); int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line, int valid_column); -int ip_plugin_runtime_commit(void *ip_plugin_runtime); +int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name); struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime); diff --git a/src/inc_internal/maat_plugin.h b/src/inc_internal/maat_plugin.h index a990184..34567cf 100644 --- a/src/inc_internal/maat_plugin.h +++ b/src/inc_internal/maat_plugin.h @@ -55,7 +55,7 @@ void plugin_runtime_free(void *plugin_runtime); int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line, int valid_column); -int plugin_runtime_commit(void *plugin_runtime); +int plugin_runtime_commit(void *plugin_runtime, const char *table_name); struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime); diff --git a/src/json2iris.cpp b/src/json2iris.cpp index 5d87927..65c3e07 100644 --- a/src/json2iris.cpp +++ b/src/json2iris.cpp @@ -136,6 +136,7 @@ int set_iris_descriptor(const char *json_file, cJSON *json, maat_kv_register(iris_cfg->str2int_map, "yes", 1); maat_kv_register(iris_cfg->str2int_map, "no", 0); + maat_kv_register(iris_cfg->str2int_map, "flag", TABLE_TYPE_FLAG); maat_kv_register(iris_cfg->str2int_map, "ip_plus", TABLE_TYPE_IP_PLUS); maat_kv_register(iris_cfg->str2int_map, "string", TABLE_TYPE_EXPR); maat_kv_register(iris_cfg->str2int_map, "expr", TABLE_TYPE_EXPR); @@ -405,6 +406,37 @@ error_out: return ret; } +int write_flag_line(cJSON *region_json, struct iris_description *p_iris, + struct iris_table *table, struct log_handle *logger) +{ + struct translate_command json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt = 0; + memset(json_cmd, 0, sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string = "region_id"; + json_cmd[cmd_cnt].json_type=cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "group_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "flag"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "flag_mask"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "is_valid"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map, + json_cmd, cmd_cnt, table, logger); +} + int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, struct log_handle *logger) { @@ -715,6 +747,9 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_table *table_info = query_table_info(p_iris, table_name, table_type); switch(table_type) { + case TABLE_TYPE_FLAG: + ret = write_flag_line(table_content, p_iris, table_info, logger); + break; case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: ret = write_expr_line(table_content, p_iris, table_info, logger); diff --git a/src/maat_api.cpp b/src/maat_api.cpp index 4d3d088..1781528 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -460,6 +460,34 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id, return 0; } +int maat_compile_table_ex_schema_register(struct maat *maat_instance, int table_id, + maat_rule_ex_new_func_t *new_func, + maat_rule_ex_free_func_t *free_func, + maat_rule_ex_dup_func_t *dup_func, + long argl, void *argp) +{ + if (NULL == maat_instance || table_id < 0 || table_id > MAX_TABLE_NUM) { + return -1; + } + + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + assert(table_type == TABLE_TYPE_COMPILE); + + void *compile_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + assert(compile_schema != NULL); + + return compile_table_set_rule_ex_data_schema((struct compile_schema *)compile_schema, table_id, + new_func, free_func, dup_func, + argl, argp, maat_instance->logger); +} + +void *maat_compile_table_get_ex_data(struct maat *maat_instance, int table_id, int compile_id, size_t idx) +{ + struct compile_schema *schema = (struct compile_schema *)table_manager_get_schema(maat_instance->tbl_mgr, + table_id); + return compile_table_get_rule_ex_data(schema, compile_id, idx); +} + int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int table_id, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, @@ -556,10 +584,10 @@ void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, switch (table_type) { case TABLE_TYPE_PLUGIN: - plugin_runtime_commit(runtime); + plugin_runtime_commit(runtime, "false_plugin_table"); break; case TABLE_TYPE_IP_PLUGIN: - ip_plugin_runtime_commit(runtime); + ip_plugin_runtime_commit(runtime, "false_plugin_table"); break; default: break; @@ -598,18 +626,19 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i return 0; } -void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, - const char *key, size_t key_len) +int maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, + const char *key, size_t key_len, + void **ex_data_array, size_t n_ex_data) { struct maat_runtime *maat_rt = maat_instance->maat_rt; if (NULL == maat_rt) { - return NULL; + return -1; } enum table_type table_type = table_manager_get_table_type(maat_rt->ref_tbl_mgr, table_id); void *runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, table_id); if (NULL == runtime) { - return NULL; + return -1; } struct ex_data_runtime *ex_data_rt = NULL; @@ -625,10 +654,11 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, } if (NULL == ex_data_rt) { - return NULL; + return -1; } - return ex_data_runtime_get_ex_data(ex_data_rt, key, key_len); + *ex_data_array = ex_data_runtime_get_ex_data(ex_data_rt, key, key_len); + return 0; } struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id) @@ -681,17 +711,12 @@ inline int scan_status_should_compile_NOT(struct maat_state *state) return 0; } -int hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile_ids_size, - size_t *n_hit_compile_id, struct maat_state *mid) +size_t hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile_ids_size, + struct maat_state *mid) { - int compile_id_cnt = compile_runtime_match((struct compile_runtime *)compile_runtime, - compile_ids, compile_ids_size, mid); - *n_hit_compile_id = compile_id_cnt; - if (compile_id_cnt > 0) { - return MAAT_SCAN_HIT; - } else { - return MAAT_SCAN_HALF_HIT; - } + size_t n_hit_compile = compile_runtime_match((struct compile_runtime *)compile_runtime, + compile_ids, compile_ids_size, mid); + return n_hit_compile; } int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, @@ -746,7 +771,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_ERR; } - int compile_ret = 0; + size_t n_hit_compile = 0; if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT if (group_hit_cnt > 0) { @@ -761,7 +786,8 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, } void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid); + n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + *n_hit_result = n_hit_compile; assert(mid->is_last_scan < LAST_SCAN_FINISHED); if (LAST_SCAN_SET == mid->is_last_scan) { @@ -769,22 +795,23 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, } } - if (compile_ret > 0) { + if (n_hit_compile > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - } - - if (0 == group_hit_cnt && compile_ret > 0) { - // hit NOT group - alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + if (0 == group_hit_cnt) { + //hit NOT group + alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + } + return MAAT_SCAN_HIT; + } else { + // n_hit_compile == 0 + if (group_hit_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } } maat_runtime_ref_dec(maat_instance->maat_rt, thread_id); - if (0 == compile_ret && group_hit_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } - - return MAAT_SCAN_HIT; + return MAAT_SCAN_OK; } int maat_scan_integer(struct maat *instance, int table_id, int thread_id, @@ -837,7 +864,7 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_ERR; } - int compile_ret = 0; + size_t n_hit_compile = 0; if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT if (group_hit_cnt > 0) { @@ -852,7 +879,8 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, } void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid); + n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + *n_hit_result = n_hit_compile; assert(mid->is_last_scan < LAST_SCAN_FINISHED); if (LAST_SCAN_SET == mid->is_last_scan) { @@ -860,22 +888,23 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, } } - if (compile_ret > 0) { + if (n_hit_compile > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - } - - if (0 == group_hit_cnt && compile_ret > 0) { - // hit NOT group - alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + if (0 == group_hit_cnt) { + //hit NOT group + alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + } + return MAAT_SCAN_HIT; + } else { + // n_hit_compile == 0 + if (group_hit_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } } maat_runtime_ref_dec(maat_instance->maat_rt, thread_id); - if (0 == compile_ret && group_hit_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } - - return MAAT_SCAN_HIT; + return MAAT_SCAN_OK; } @@ -921,7 +950,7 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_ERR; } - int compile_ret = 0; + size_t n_hit_compile = 0; if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT if (group_hit_cnt > 0) { @@ -936,7 +965,8 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, } void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid); + n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + *n_hit_result = n_hit_compile; assert(mid->is_last_scan < LAST_SCAN_FINISHED); if (LAST_SCAN_SET == mid->is_last_scan) { @@ -944,22 +974,23 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, } } - if (compile_ret > 0) { + if (n_hit_compile > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - } - - if (0 == group_hit_cnt && compile_ret > 0) { - // hit NOT group - alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + if (0 == group_hit_cnt) { + //hit NOT group + alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + } + return MAAT_SCAN_HIT; + } else { + // n_hit_compile == 0 + if (group_hit_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } } maat_runtime_ref_dec(maat_instance->maat_rt, thread_id); - if (0 == compile_ret && group_hit_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } - - return MAAT_SCAN_HIT; + return MAAT_SCAN_OK; } int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, @@ -1023,7 +1054,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_ERR; } - int compile_ret = 0; + size_t n_hit_compile = 0; if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT if (group_hit_cnt > 0) { @@ -1038,7 +1069,8 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, } void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid); + n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + *n_hit_result = n_hit_compile; assert(mid->is_last_scan < LAST_SCAN_FINISHED); if (LAST_SCAN_SET == mid->is_last_scan) { @@ -1046,22 +1078,23 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, } } - if (compile_ret > 0) { + if (n_hit_compile > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - } - - if (0 == group_hit_cnt && compile_ret > 0) { - // hit NOT group - alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + if (0 == group_hit_cnt) { + //hit NOT group + alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + } + return MAAT_SCAN_HIT; + } else { + // n_hit_compile == 0 + if (group_hit_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } } maat_runtime_ref_dec(maat_instance->maat_rt, thread_id); - if (0 == compile_ret && group_hit_cnt > 0) { - return MAAT_SCAN_HALF_HIT; - } - - return MAAT_SCAN_HIT; + return MAAT_SCAN_OK; } struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id) diff --git a/src/maat_bool_plugin.cpp b/src/maat_bool_plugin.cpp new file mode 100644 index 0000000..39f38de --- /dev/null +++ b/src/maat_bool_plugin.cpp @@ -0,0 +1,491 @@ +/* +********************************************************************************************** +* File: maat_bool_plugin.cpp +* Description: +* Authors: Liu wentan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include + +#include "maat_bool_plugin.h" +#include "bool_matcher.h" +#include "maat_ex_data.h" +#include "log/log.h" +#include "maat_utils.h" +#include "maat_rule.h" +#include "maat_garbage_collection.h" + +#define MODULE_BOOL_PLUGIN module_name_str("maat.bool_plugin") + +struct bool_plugin_schema { + int item_id_column; + int bool_expr_column; + int rule_tag_column; + struct ex_data_schema *ex_schema; + int table_id; + struct table_manager *ref_tbl_mgr; + + unsigned long long update_err_cnt; + unsigned long long unmatch_tag_cnt; +}; + +struct bool_plugin_item { + int item_id; + size_t n_bool_item; + unsigned long long bool_item_id[MAX_ITEMS_PER_BOOL_EXPR]; +}; + +struct bool_plugin_runtime { + struct bool_matcher *matcher; + struct ex_data_runtime *ex_data_rt; + + uint32_t rule_num; + uint32_t updating_rule_num; + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; +}; + +/* bool plugin schema API */ +void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) +{ + size_t read_cnt = 0; + struct bool_plugin_schema *schema = ALLOC(struct bool_plugin_schema, 1); + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (item != NULL && item->type == cJSON_Number) { + schema->table_id = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "custom"); + if (NULL == item || item->type != cJSON_Object) { + log_error(logger, MODULE_BOOL_PLUGIN, + "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "item_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->item_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "bool_expr"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->bool_expr_column = custom_item->valueint; + read_cnt++; + } + + // rule_tag is optional + custom_item = cJSON_GetObjectItem(item, "rule_tag"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->rule_tag_column = custom_item->valueint; + } + + schema->ref_tbl_mgr = tbl_mgr; + + if (read_cnt < 3) { + goto error; + } + + return schema; +error: + FREE(schema); + return NULL; +} + +void bool_plugin_schema_free(void *bool_plugin_schema) +{ + if (NULL == bool_plugin_schema) { + return; + } + struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema; + if (schema->ex_schema != NULL) { + ex_data_schema_free(schema->ex_schema); + schema->ex_schema = NULL; + } + + FREE(schema); +} + +/* ip plugin table ex data API */ +struct ex_data_schema *bool_plugin_table_get_ex_data_schema(void *bool_plugin_schema) +{ + if (NULL == bool_plugin_schema) { + return NULL; + } + + struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema; + + return schema->ex_schema; +} + +int bool_plugin_table_set_ex_data_schema(void *bool_plugin_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger) +{ + if (NULL == bool_plugin_schema) { + return -1; + } + + struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema; + if (schema->ex_schema != NULL) { + assert(0); + log_error(logger, MODULE_BOOL_PLUGIN, + "Error: %s, EX data schema already registed", __FUNCTION__); + return -1; + } + + schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp); + + return 0; +} + +void *bool_plugin_runtime_new(void *bool_plugin_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == bool_plugin_schema) { + return NULL; + } + + struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema; + struct bool_plugin_runtime *bool_plugin_rt = ALLOC(struct bool_plugin_runtime, 1); + + bool_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, + logger); + bool_plugin_rt->ref_garbage_bin = garbage_bin; + bool_plugin_rt->logger = logger; + + return bool_plugin_rt; +} + +void bool_plugin_runtime_free(void *bool_plugin_runtime) +{ + if (NULL == bool_plugin_runtime) { + return; + } + + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + if (bool_plugin_rt->matcher != NULL) { + bool_matcher_free(bool_plugin_rt->matcher); + bool_plugin_rt->matcher = NULL; + } + + if (bool_plugin_rt->ex_data_rt != NULL) { + ex_data_runtime_free(bool_plugin_rt->ex_data_rt); + bool_plugin_rt->ex_data_rt = NULL; + } + + FREE(bool_plugin_rt); +} + +int bool_plugin_table_ex_data_schema_flag(struct bool_plugin_schema *bool_plugin_schema) +{ + return 0; +} + +int bool_plugin_runtime_update_row(struct bool_plugin_runtime *rt, + struct bool_plugin_schema *schema, + const char *row, char *key, size_t key_len, + struct bool_expr *expr, int is_valid) +{ + int ret = -1; + struct ex_data_runtime *ex_data_rt = rt->ex_data_rt; + int set_flag = bool_plugin_table_ex_data_schema_flag(schema); + + if (1 == set_flag) { + if (0 == is_valid) { + //delete + ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len); + if (ret < 0) { + return -1; + } + } else { + //add + void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); + struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)expr); + ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); + if (ret < 0) { + return -1; + } + } + } else { + ex_data_runtime_cache_row_put(ex_data_rt, row); + } + + return 0; +} + +int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *line, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr); + + if (schema->rule_tag_column > 0 && n_tag > 0) { + int ret = get_column_pos(line, schema->rule_tag_column, + &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_BOOL_PLUGIN, + "bool_plugin table(table_id:%d) has no rule_tag, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (column_len > 2) { + char *tag_str = ALLOC(char, column_len + 1); + memcpy(tag_str, (line + column_offset), column_len); + ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str); + FREE(tag_str); + if (TAG_MATCH_ERR == ret) { + log_error(logger, MODULE_BOOL_PLUGIN, + "bool_plugin table(table_id:%d) has invalid tag format, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (TAG_MATCH_UNMATCHED == ret) { + schema->unmatch_tag_cnt++; + return TAG_MATCH_UNMATCHED; + } + } + } + + return TAG_MATCH_MATCHED; +} + +struct bool_plugin_item * +bool_plugin_item_new(const char *line, struct bool_plugin_schema *schema, + struct log_handle *logger) +{ + int ret = bool_plugin_accept_tag_match(schema, line, logger); + if (ret == TAG_MATCH_UNMATCHED) { + return NULL; + } + + size_t column_offset = 0; + size_t column_len = 0; + size_t n_item = 0; + char expr_buffer[BUFSIZ] = {0}; + char *token = NULL, *sub_token = NULL, *saveptr; + struct bool_plugin_item *item = ALLOC(struct bool_plugin_item, 1); + + ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_BOOL_PLUGIN, + "bool_plugin table(table_id:%d) line:%s has no item_id column", + schema->table_id, line); + goto error; + } + item->item_id = atoi(line + column_offset); + + ret = get_column_pos(line, schema->bool_expr_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_BOOL_PLUGIN, + "bool_plugin table(table_id:%d) line:%s has no bool_expr column", + schema->table_id, line); + goto error; + } + + memcpy(expr_buffer, line + column_offset, column_len); + for (token = expr_buffer; ; token = NULL) { + sub_token = strtok_r(token, "&", &saveptr); + if (NULL == sub_token) { + break; + } + + ret = sscanf(sub_token, "%llu", item->bool_item_id + n_item); + n_item++; + if (ret != 1 || n_item > MAX_ITEMS_PER_BOOL_EXPR) { + log_error(logger, MODULE_BOOL_PLUGIN, + "bool_plugin table(table_id:%d) invalid format of bool_expr column, line:%s", + schema->table_id, line); + goto error; + } + } + item->n_bool_item = n_item; + + return item; +error: + FREE(item); + return NULL; +} + +void bool_plugin_item_free(struct bool_plugin_item *item) +{ + FREE(item); +} + +static int cmp_ull_p(const void *p1, const void *p2) +{ + if(* (unsigned long long*) p1 > * (unsigned long long*) p2) { + return 1; + } else if(* (unsigned long long*) p1 < * (unsigned long long*) p2) { + return -1; + } else { + return 0; + } +} + +size_t ull_dedup(unsigned long long item_ids[], size_t n_item) +{ + size_t index = 0; + + qsort(item_ids, n_item, sizeof(unsigned long long), cmp_ull_p); + + for (size_t i = 1; i < n_item; i++) { + if (item_ids[i] != item_ids[index]) { + item_ids[++index] = item_ids[i]; + } + } + + return index + 1; +} + +struct bool_expr *bool_expr_new(int item_id, struct bool_plugin_item *item) +{ + struct bool_expr *expr = ALLOC(struct bool_expr, 1); + + expr->expr_id = item_id; + size_t n_item = ull_dedup(item->bool_item_id, item->n_bool_item); + for (size_t i = 0; i < n_item; i++) { + expr->items[i].item_id = item->bool_item_id[i]; + expr->items[i].not_flag = 0; + } + + expr->item_num = n_item; + + return expr; +} + +int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_schema, + const char *line, int valid_column) +{ + if (NULL == bool_plugin_runtime || NULL == bool_plugin_schema || + NULL == line) { + return -1; + } + + struct bool_plugin_item *item = NULL; + struct bool_expr *expr = NULL; + struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema; + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + int item_id = get_column_value(line, schema->item_id_column); + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } + + if (schema->ex_schema != NULL) { + if (1 == is_valid) { + // add + item = bool_plugin_item_new(line, schema, bool_plugin_rt->logger); + if (NULL == item) { + return -1; + } + + expr = bool_expr_new(item_id, item); + assert(expr != NULL); + bool_plugin_item_free(item); + } + + char *key = (char *)&item_id; + int ret = bool_plugin_runtime_update_row(bool_plugin_rt, schema, line, key, + sizeof(int), expr, is_valid); + if (ret < 0) { + if (item != NULL) { + FREE(item); + } + return -1; + } else { + if (0 == is_valid) { + bool_plugin_rt->rule_num--; + } else { + bool_plugin_rt->rule_num++; + } + } + } else { + //ex_schema not set + ex_data_runtime_cache_row_put(bool_plugin_rt->ex_data_rt, line); + bool_plugin_rt->rule_num = ex_data_runtime_cached_row_count(bool_plugin_rt->ex_data_rt); + } + + return 0; +} + +int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name) +{ + if (NULL == bool_plugin_runtime) { + return -1; + } + + int ret = 0; + struct ex_data_container **ex_container = NULL; + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + struct ex_data_runtime *ex_data_rt = bool_plugin_rt->ex_data_rt; + + size_t expr_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); + if (0 == expr_cnt) { + FREE(ex_container); + return 0; + } + + struct bool_expr *exprs = ALLOC(struct bool_expr, expr_cnt); + + for (size_t i = 0; i < expr_cnt; i++) { + exprs[i] = *(struct bool_expr *)(struct bool_expr *)ex_container[i]->custom_data; + assert(exprs[i].user_tag == ex_container[i] || NULL == exprs[i].user_tag); + exprs[i].user_tag = ex_container[i]; + } + + struct bool_matcher *new_bool_matcher = NULL; + struct bool_matcher *old_bool_matcher = NULL; + size_t mem_used = 0; + + log_info(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN, + "table[%s] committing %zu bool_plugin rules for rebuilding bool_matcher engine", + table_name, expr_cnt); + + new_bool_matcher = bool_matcher_new(exprs, expr_cnt, &mem_used); + if (NULL == new_bool_matcher) { + log_error(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN, + "table[%s] rebuild bool_matcher engine failed when update %zu bool_plugin rules", + table_name, expr_cnt); + ret = -1; + } + + old_bool_matcher = bool_plugin_rt->matcher; + bool_plugin_rt->matcher = new_bool_matcher; + maat_garbage_bagging(bool_plugin_rt->ref_garbage_bin, old_bool_matcher, + (void (*)(void*))bool_matcher_free); + ex_data_runtime_commit(ex_data_rt); + + FREE(exprs); + FREE(ex_container); + + return ret; +} + +struct ex_data_runtime *bool_plugin_runtime_get_ex_data_rt(void *bool_plugin_runtime) +{ + if (NULL == bool_plugin_runtime) { + return NULL; + } + + struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; + + return bool_plugin_rt->ex_data_rt; +} \ No newline at end of file diff --git a/src/maat_compile.cpp b/src/maat_compile.cpp index a65b124..f151557 100644 --- a/src/maat_compile.cpp +++ b/src/maat_compile.cpp @@ -48,7 +48,7 @@ struct compile_schema { size_t n_ex_schema; struct compile_ex_data_schema ex_schema[MAX_COMPILE_EX_DATA_NUM]; int table_id; //ugly - struct table_manager *tbl_mgr; + struct table_manager *ref_tbl_mgr; size_t unmatched_tag_cnt; }; @@ -59,9 +59,8 @@ struct group2compile_schema { int vtable_name_column; int clause_index_column; char associated_compile_table_id; - struct table_manager *ref_tbl_mgr; int table_id;//ugly - struct table_manager *tbl_mgr; + struct table_manager *ref_tbl_mgr; }; struct compile_item { @@ -192,18 +191,45 @@ int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, return idx; } -struct compile_ex_data_schema * -compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, size_t idx) +void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, int compile_id, int is_dettach) +{ + struct maat_compile *compile = NULL; + void *ret = NULL; + + //TODO: add mutex + HASH_FIND_INT(compile_rt->compile_hash, &compile_id, compile); + if (compile != NULL) { + ret = compile->user_data; + if (is_dettach) { + compile->user_data = NULL; + } + } + + return ret; +} + +void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, int compile_id, size_t idx) { if (NULL == compile_schema) { return NULL; } - if (idx < compile_schema->n_ex_schema) { - return (compile_schema->ex_schema + idx); - } + struct compile_rule *compile_rule = NULL; + struct compile_runtime *compile_rt = NULL; + compile_rt = (struct compile_runtime *)table_manager_get_runtime(compile_schema->ref_tbl_mgr, + compile_schema->table_id); + + compile_rule = (struct compile_rule *)compile_runtime_get_user_data(compile_rt, compile_id, 0); + if (NULL == compile_rule) { + return NULL; + } - return NULL; + void *ex_data = NULL; + assert(idx < compile_schema->n_ex_schema); + struct compile_ex_data_schema *ex_schema = &(compile_schema->ex_schema[idx]); + ex_schema->dup_func(ex_schema->idx, &ex_data, compile_rule->ex_data + idx, ex_schema->argl, ex_schema->argp); + + return ex_data; } size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema) @@ -295,7 +321,7 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } - compile_schema->tbl_mgr = tbl_mgr; + compile_schema->ref_tbl_mgr = tbl_mgr; if (read_cnt < 10) { goto error; @@ -367,7 +393,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } - g2c_schema->tbl_mgr = tbl_mgr; + g2c_schema->ref_tbl_mgr = tbl_mgr; if (read_cnt < 7) { goto error; @@ -461,11 +487,11 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, } memcpy(tag_str, (line + column_offset), column_len); - n_accept_tag = table_manager_accept_tags_count(compile_schema->tbl_mgr); + n_accept_tag = table_manager_accept_tags_count(compile_schema->ref_tbl_mgr); if (n_accept_tag > 0 && strlen(tag_str) > 2) { str_unescape(tag_str); - ret = table_manager_accept_tags_match(compile_schema->tbl_mgr, tag_str); + ret = table_manager_accept_tags_match(compile_schema->ref_tbl_mgr, tag_str); if (TAG_MATCH_ERR == ret) { log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s is invalid tag", @@ -836,9 +862,24 @@ struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash, return compile; } -size_t maat_compile_hash_count(struct maat_compile *compile_hash) +size_t maat_compile_in_use_count(struct maat_compile *compile_hash) { - return HASH_COUNT(compile_hash); + struct maat_compile *compile = NULL, *tmp_compile = NULL; + size_t in_use_compile_cnt = 0; + struct maat_clause_state *clause_state = NULL; + + HASH_ITER(hh, compile_hash, compile, tmp_compile) { + //find how much compile whose clause is in_use + for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states + i; + if (clause_state->in_use) { + in_use_compile_cnt++; + break; + } + } + } + + return in_use_compile_cnt; } int compare_literal_id(const void *pa, const void *pb) @@ -982,7 +1023,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, //STEP 2, serial compile clause states to a bool expression array size_t expr_cnt = 0; - size_t compile_cnt = maat_compile_hash_count(compile_hash); + size_t compile_cnt = maat_compile_in_use_count(compile_hash); struct bool_expr *bool_expr_array = ALLOC(struct bool_expr, compile_cnt); HASH_ITER(hh, compile_hash, compile, tmp_compile) { for (i = 0, j = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { @@ -992,7 +1033,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, } //TODO:mytest need to delete - #if 1 + #if 0 struct maat_literal_id *p = NULL; for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) { printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n", @@ -1067,7 +1108,7 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan size_t ud_result_cnt = 0; //TODO:mytest need to delete - #if 1 + #if 0 unsigned long long *p; printf("utarray_len:%u\n", utarray_len(compile_state->all_hit_clauses)); for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clauses); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clauses, p)) @@ -1098,6 +1139,7 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan } } + compile_state->this_scan_hit_item_cnt = 0; return ud_result_cnt; } @@ -1120,11 +1162,12 @@ int maat_add_group_to_compile(struct maat_compile **compile_hash, void *g2g_runt } struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vtable_id}; + ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, g2c_item->not_flag); if (ret < 0) { log_error(logger, MODULE_COMPILE, - "add literal_id{group_id:%d, vtable_id:%d} to clause %d of compile %d failed", + "add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d of compile %d failed", g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, g2c_item->compile_id); ret = -1; @@ -1474,9 +1517,9 @@ void compile_item_to_compile_rule(struct compile_item *compile_item, compile_rule->head.serv_def_len); compile_rule->evaluation_order = compile_item->evaluation_order; - size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(compile_schema); + size_t n_rule_ex_schema =compile_schema->n_ex_schema; for (size_t i = 0; i < n_rule_ex_schema; i++) { - struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(compile_schema, i); + struct compile_ex_data_schema *ex_schema = &(compile_schema->ex_schema[i]); compile_rule->ex_data[i] = rule_ex_data_new(&compile_rule->head, compile_rule->service_defined, ex_schema); } @@ -1489,9 +1532,9 @@ void destroy_compile_rule(struct compile_rule *compile_rule) struct compile_schema *schema = compile_rule->ref_table; assert(compile_rule->magic_num==COMPILE_RULE_MAGIC); - size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(schema); + size_t n_rule_ex_schema = schema->n_ex_schema; for (size_t i = 0; i < n_rule_ex_schema; i++) { - struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(schema, i); + struct compile_ex_data_schema *ex_schema = &(schema->ex_schema[i]); rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, ex_schema); compile_rule->ex_data[i] = NULL; @@ -1612,14 +1655,14 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, return ret; } -int compile_runtime_commit(void *compile_runtime) +int compile_runtime_commit(void *compile_runtime, const char *table_name) { if (NULL == compile_runtime) { return -1; } struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; - size_t compile_cnt = maat_compile_hash_count(compile_rt->compile_hash); + size_t compile_cnt = maat_compile_in_use_count(compile_rt->compile_hash); if (0 == compile_cnt) { return 0; } @@ -1628,8 +1671,8 @@ int compile_runtime_commit(void *compile_runtime) struct bool_matcher *new_bool_matcher = NULL; log_info(compile_rt->logger, MODULE_COMPILE, - "committing %zu compile rules for rebuilding compile bool_matcher engine", - compile_cnt); + "table[%s] committing %zu compile rules for rebuilding compile bool_matcher engine", + table_name, compile_cnt); int ret = 0; new_bool_matcher = maat_compile_bool_matcher_new(compile_rt->compile_hash, @@ -1637,8 +1680,8 @@ int compile_runtime_commit(void *compile_runtime) compile_rt->logger); if (NULL == new_bool_matcher) { log_error(compile_rt->logger, MODULE_COMPILE, - "rebuild compile bool_matcher engine failed when update %zu compile rules", - compile_cnt); + "table[%s] rebuild compile bool_matcher engine failed when update %zu compile rules", + table_name, compile_cnt); ret = -1; } @@ -1701,10 +1744,6 @@ int compile_runtime_match(struct compile_runtime *compile_rt, int *compile_ids, size_t compile_ids_size, struct maat_state *state) { - if (NULL == compile_rt) { - return -1; - } - struct maat_compile_state *compile_state = state->compile_state; int is_last_scan = state->is_last_scan; struct compile_rule *compile_rules[compile_ids_size]; @@ -1780,10 +1819,10 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i], 1, top_group_ids); - if (0 == top_group_cnt) { - maat_compile_state_update_hit_clause(state->compile_state, compile_rt, - group_ids[i], vtable_id); - } + // if (0 == top_group_cnt) { + // maat_compile_state_update_hit_clause(state->compile_state, compile_rt, + // group_ids[i], vtable_id); + // } for (int j = 0; j < top_group_cnt; j++) { maat_compile_state_update_hit_clause(state->compile_state, compile_rt, diff --git a/src/maat_expr.cpp b/src/maat_expr.cpp index 8ddd425..9b76f67 100644 --- a/src/maat_expr.cpp +++ b/src/maat_expr.cpp @@ -10,6 +10,7 @@ #include #include +#include #include "maat_expr.h" #include "adapter_hs.h" @@ -42,6 +43,7 @@ enum expr_type { EXPR_TYPE_STRING = 0, EXPR_TYPE_AND, EXPR_TYPE_REGEX, + EXPR_TYPE_OFFSET, EXPR_TYPE_MAX }; @@ -99,6 +101,9 @@ enum expr_type int_to_expr_type(int expr_type) case 2: type = EXPR_TYPE_REGEX; break; + case 3: + type = EXPR_TYPE_OFFSET; + break; default: break; } @@ -179,22 +184,6 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem str_unescape(district); expr_item->district_id = table_manager_get_district_id(expr_schema->ref_tbl_mgr, district); } - - ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_EXPR, - "expr table(table_id:%d) line:%s has no keywords", - expr_schema->table_id, line); - goto error; - } - - if (column_len >= MAX_KEYWORDS_STR) { - log_error(logger, MODULE_EXPR, - "expr table(table_id:%d) line:%s keywords length too long", - expr_schema->table_id, line); - goto error; - } - memcpy(expr_item->keywords, (line + column_offset), column_len); ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len); if (ret < 0) { @@ -247,6 +236,23 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem goto error; } + ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_EXPR, + "expr table(table_id:%d) line:%s has no keywords", + expr_schema->table_id, line); + goto error; + } + + if (column_len >= MAX_KEYWORDS_STR) { + log_error(logger, MODULE_EXPR, + "expr table(table_id:%d) line:%s keywords length too long", + expr_schema->table_id, line); + goto error; + } + memcpy(expr_item->keywords, (line + column_offset), column_len); + + return expr_item; error: FREE(expr_item); @@ -478,6 +484,7 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type) switch (expr_type) { case EXPR_TYPE_STRING: case EXPR_TYPE_AND: + case EXPR_TYPE_OFFSET: break; case EXPR_TYPE_REGEX: pattern_type = PATTERN_TYPE_REG; @@ -489,6 +496,33 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type) return pattern_type; } +int converHextoint(char srctmp) +{ + if (isdigit(srctmp)) { + return srctmp - '0'; + } else { + char temp = toupper(srctmp); + temp = temp - 'A' + 10; + return temp; + } +} + +size_t hex2bin(char *hex, int hex_len, char *binary, size_t size) +{ + size_t resultlen = 0; + int high,low; + for (int i = 0; i < hex_len && size > resultlen; i += 2, resultlen++) { + high = converHextoint(hex[i]); + low = converHextoint(hex[i+1]); + binary[resultlen] = high * 16 + low; + } + + size = resultlen; + binary[resultlen] = '\0'; + + return resultlen; +} + #define MAAT_MAX_EXPR_ITEM_NUM 8 and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, struct log_handle *logger) @@ -496,15 +530,18 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, size_t i = 0; size_t sub_expr_cnt = 0; char *pos = NULL; + char *tmp = NULL; char *saveptr = NULL; char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM]; + int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM] = {-1}; + int key_right_offset[MAAT_MAX_EXPR_ITEM_NUM] = {-1}; and_expr_t *expr_rule = ALLOC(and_expr_t, 1); switch (expr_item->expr_type) { case EXPR_TYPE_AND: case EXPR_TYPE_REGEX: for (i = 0, pos = expr_item->keywords; ; i++, pos = NULL) { - char *tmp = strtok_r_esc(pos, '&', &saveptr); + tmp = strtok_r_esc(pos, '&', &saveptr); if (NULL == tmp) { break; } @@ -525,6 +562,42 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, } sub_expr_cnt = i; break; + case EXPR_TYPE_OFFSET: + for (i = 0, pos = expr_item->keywords; ; i++, pos = NULL) { + tmp = strtok_r_esc(pos, '&', &saveptr); + if (NULL == tmp) { + break; + } + + if (i >= MAAT_MAX_EXPR_ITEM_NUM) { + log_error(logger, MODULE_EXPR, + "expr item_id:%d too many patterns", + expr_item->item_id); + return NULL; + } + + sub_key_array[i] = tmp; + sscanf(sub_key_array[i], "%d-%d:", &(key_left_offset[i]),&(key_right_offset[i])); + if (!(key_left_offset[i] >= 0 && key_right_offset[i] > 0 + && key_left_offset[i] <= key_right_offset[i])) { + log_error(logger, MODULE_EXPR, + "expr item:%d has invalid offset.", expr_item->item_id); + return NULL; + } + + sub_key_array[i] = (char *)memchr(sub_key_array[i], ':', strlen(sub_key_array[i])); + if (NULL == sub_key_array[i]) { + log_error(logger, MODULE_EXPR, + "expr item:%d has invalid offset keyword format.", + expr_item->item_id); + return NULL; + } + + sub_key_array[i]++;//jump over ':' + sub_key_array[i] = str_unescape(sub_key_array[i]); + } + sub_expr_cnt = i; + break; case EXPR_TYPE_STRING: sub_expr_cnt = 1; sub_key_array[0] = expr_item->keywords; @@ -534,12 +607,36 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, break; } + size_t region_str_len = 0; + char *region_string = NULL; + size_t sub_key_len = 0; + for (i = 0; i < sub_expr_cnt; i++) { expr_rule->expr_id = expr_item->item_id; - expr_rule->patterns[i].pat = ALLOC(char, strlen(sub_key_array[i])); - memcpy(expr_rule->patterns[i].pat, sub_key_array[i], strlen(sub_key_array[i])); - expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]); + if (FALSE == expr_item->is_case_sensitive) { + // insensitive + expr_rule->patterns[i].case_sensitive = 1; + } + expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); + + if (TRUE == expr_item->is_hexbin && expr_rule->patterns[i].type != EXPR_TYPE_REGEX) { + region_str_len = strlen(sub_key_array[i]) + 1; + region_string = ALLOC(char, region_str_len); + region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]), region_string, region_str_len); + } + + if (region_string != NULL) { + expr_rule->patterns[i].pat = ALLOC(char, region_str_len); + memcpy(expr_rule->patterns[i].pat, region_string, region_str_len); + expr_rule->patterns[i].pat_len = region_str_len; + FREE(region_string); + } else { + sub_key_len = strlen(sub_key_array[i]); + expr_rule->patterns[i].pat = ALLOC(char, sub_key_len); + memcpy(expr_rule->patterns[i].pat, sub_key_array[i], sub_key_len); + expr_rule->patterns[i].pat_len = sub_key_len; + } } expr_rule->user_tag = user_data; @@ -631,7 +728,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, return 0; } -int expr_runtime_commit(void *expr_runtime) +int expr_runtime_commit(void *expr_runtime, const char *table_name) { if (NULL == expr_runtime) { return -1; @@ -648,26 +745,24 @@ int expr_runtime_commit(void *expr_runtime) } and_expr_t *rules = ALLOC(and_expr_t, rule_cnt); - - for (size_t i = 0; i < rule_cnt; i++) { + for (size_t i = 0 ; i < rule_cnt; i++) { rules[i] = *(and_expr_t *)ex_data_array[i]; - printf("rule_id:%d\n", rules[i].expr_id); } - printf("\n\n"); + struct adapter_hs *new_adapter_hs = NULL; struct adapter_hs *old_adapter_hs = NULL; log_info(expr_rt->logger, MODULE_EXPR, - "committing %zu expr rules for rebuilding adapter_hs engine", - rule_cnt); + "table[%s] committing %zu expr rules for rebuilding adapter_hs engine", + table_name, rule_cnt); new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger); if (NULL == new_adapter_hs) { log_error(expr_rt->logger, MODULE_EXPR, - "rebuild adapter_hs engine failed when update %zu expr rules", - rule_cnt); + "table[%s] rebuild adapter_hs engine failed when update %zu expr rules", + table_name, rule_cnt); ret = -1; } diff --git a/src/maat_flag.cpp b/src/maat_flag.cpp index 402b6dd..3e50dcd 100644 --- a/src/maat_flag.cpp +++ b/src/maat_flag.cpp @@ -359,7 +359,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, return 0; } -int flag_runtime_commit(void *flag_runtime) +int flag_runtime_commit(void *flag_runtime, const char *table_name) { if (NULL == flag_runtime) { return -1; @@ -385,13 +385,13 @@ int flag_runtime_commit(void *flag_runtime) struct flag_matcher *old_flag_matcher = NULL; log_info(flag_rt->logger, MODULE_FLAG, - "committing %zu flag rules for rebuilding flag_matcher engine", - rule_cnt); + "table[%s] committing %zu flag rules for rebuilding flag_matcher engine", + table_name, rule_cnt); new_flag_matcher = flag_matcher_new(rules, rule_cnt); if (NULL == new_flag_matcher) { log_error(flag_rt->logger, MODULE_FLAG, - "rebuild flag_matcher engine failed when update %zu flag rules", - rule_cnt); + "table[%s] rebuild flag_matcher engine failed when update %zu flag rules", + table_name, rule_cnt); ret = -1; } diff --git a/src/maat_fqdn_plugin.cpp b/src/maat_fqdn_plugin.cpp new file mode 100644 index 0000000..28f2d26 --- /dev/null +++ b/src/maat_fqdn_plugin.cpp @@ -0,0 +1,430 @@ +/* +********************************************************************************************** +* File: maat_fqdn_plugin.cpp +* Description: +* Authors: Liu wentan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include + +#include "maat_fqdn_plugin.h" +#include "maat_ex_data.h" +#include "fqdn_engine.h" +#include "log/log.h" +#include "maat_utils.h" +#include "maat_table.h" +#include "maat_rule.h" +#include "maat_garbage_collection.h" + +#define MODULE_FQDN_PLUGIN module_name_str("maat.bool_plugin") + +struct fqdn_plugin_schema { + int item_id_column; + int suffix_flag_column; + int fqdn_column; + int rule_tag_column; + struct ex_data_schema *ex_schema; + int table_id; + struct table_manager *ref_tbl_mgr; + + unsigned long long update_err_cnt; + unsigned long long unmatch_tag_cnt; +}; + +struct fqdn_plugin_item { + int item_id; + int suffix_flag; +}; + +struct fqdn_plugin_runtime { + struct FQDN_engine *engine; + struct ex_data_runtime *ex_data_rt; + + uint32_t rule_num; + uint32_t updating_rule_num; + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; +}; + +void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) +{ + size_t read_cnt = 0; + struct fqdn_plugin_schema *schema = ALLOC(struct fqdn_plugin_schema, 1); + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (item != NULL && item->type == cJSON_Number) { + schema->table_id = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "custom"); + if (NULL == item || item->type != cJSON_Object) { + log_error(logger, MODULE_FQDN_PLUGIN, + "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "item_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->item_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "suffix_match_method"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->suffix_flag_column = custom_item->valueint; + read_cnt++; + } + + // rule_tag is optional + custom_item = cJSON_GetObjectItem(item, "fqdn"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->fqdn_column = custom_item->valueint; + read_cnt++; + } + + // rule_tag is optional + custom_item = cJSON_GetObjectItem(item, "rule_tag"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->rule_tag_column = custom_item->valueint; + } + + schema->ref_tbl_mgr = tbl_mgr; + + if (read_cnt < 4) { + goto error; + } + + return schema; +error: + FREE(schema); + return NULL; +} + +void fqdn_plugin_schema_free(void *fqdn_plugin_schema) +{ + if (NULL == fqdn_plugin_schema) { + return; + } + struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema; + if (schema->ex_schema != NULL) { + ex_data_schema_free(schema->ex_schema); + schema->ex_schema = NULL; + } + + FREE(schema); +} + +/* fqdn plugin table ex data API */ +struct ex_data_schema *fqdn_plugin_table_get_ex_data_schema(void *fqdn_plugin_schema) +{ + if (NULL == fqdn_plugin_schema) { + return NULL; + } + + struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema; + + return schema->ex_schema; +} + +int fqdn_plugin_table_set_ex_data_schema(void *fqdn_plugin_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger) +{ + if (NULL == fqdn_plugin_schema) { + return -1; + } + + struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema; + if (schema->ex_schema != NULL) { + assert(0); + log_error(logger, MODULE_FQDN_PLUGIN, + "Error: %s, EX data schema already registed", __FUNCTION__); + return -1; + } + + schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp); + + return 0; +} + +void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == fqdn_plugin_schema) { + return NULL; + } + + struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema; + struct fqdn_plugin_runtime *fqdn_plugin_rt = ALLOC(struct fqdn_plugin_runtime, 1); + + fqdn_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, + logger); + fqdn_plugin_rt->ref_garbage_bin = garbage_bin; + fqdn_plugin_rt->logger = logger; + + return fqdn_plugin_rt; +} + +void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime) +{ + if (NULL == fqdn_plugin_runtime) { + return; + } + + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + if (fqdn_plugin_rt->engine != NULL) { + FQDN_engine_free(fqdn_plugin_rt->engine); + fqdn_plugin_rt->engine = NULL; + } + + if (fqdn_plugin_rt->ex_data_rt != NULL) { + ex_data_runtime_free(fqdn_plugin_rt->ex_data_rt); + fqdn_plugin_rt->ex_data_rt = NULL; + } + + FREE(fqdn_plugin_rt); +} + +int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *line, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr); + + if (schema->rule_tag_column > 0 && n_tag > 0) { + int ret = get_column_pos(line, schema->rule_tag_column, + &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_FQDN_PLUGIN, + "fqdn_plugin table(table_id:%d) has no rule_tag, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (column_len > 2) { + char *tag_str = ALLOC(char, column_len + 1); + memcpy(tag_str, (line + column_offset), column_len); + ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str); + FREE(tag_str); + if (TAG_MATCH_ERR == ret) { + log_error(logger, MODULE_FQDN_PLUGIN, + "fqdn_plugin table(table_id:%d) has invalid tag format, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (TAG_MATCH_UNMATCHED == ret) { + schema->unmatch_tag_cnt++; + return TAG_MATCH_UNMATCHED; + } + } + } + + return TAG_MATCH_MATCHED; +} + +struct fqdn_plugin_item * +fqdn_plugin_item_new(const char *line, struct fqdn_plugin_schema *schema, + struct log_handle *logger) +{ + int ret = fqdn_plugin_accept_tag_match(schema, line, logger); + if (ret == TAG_MATCH_UNMATCHED) { + return NULL; + } + + size_t column_offset = 0; + size_t column_len = 0; + struct fqdn_plugin_item *item = ALLOC(struct fqdn_plugin_item, 1); + + ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_FQDN_PLUGIN, + "fqdn_plugin table(table_id:%d) line:%s has no item_id column", + schema->table_id, line); + goto error; + } + item->item_id = atoi(line + column_offset); + + ret = get_column_pos(line, schema->suffix_flag_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_FQDN_PLUGIN, + "fqdn_plugin table(table_id:%d) line:%s has no suffix_match_method column", + schema->table_id, line); + goto error; + } + item->suffix_flag = atoi(line + column_offset); + + ret = get_column_pos(line, schema->fqdn_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_FQDN_PLUGIN, + "fqdn_plugin table(table_id:%d) line:%s has no fqdn column", + schema->table_id, line); + goto error; + } + + return item; +error: + FREE(item); + return NULL; +} + +struct FQDN_rule *fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match) +{ + struct FQDN_rule *fqdn_rule=ALLOC(struct FQDN_rule, 1); + //Todo: check FQDN format with regex ^([a-zA-Z0-9._-])+$ + if(fqdn[0]=='.') + { + fqdn++; + fqdn_len--; + } + if(fqdn[fqdn_len]=='/') + { + fqdn_len--; + } + fqdn_rule->FQDN=ALLOC(char, fqdn_len+1); + memcpy(fqdn_rule->FQDN, fqdn, fqdn_len); + fqdn_rule->len=fqdn_len; + fqdn_rule->is_suffix_match=is_suffix_match; + fqdn_rule->id=id; + return fqdn_rule; +} + +int fqdn_plugin_runtime_update_row(struct fqdn_plugin_runtime *rt, + struct fqdn_plugin_schema *schema, + const char *row, char *key, size_t key_len, + struct FQDN_rule *rule, int is_valid) +{ + return 0; +} + +int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_schema, + const char *line, int valid_column) +{ + if (NULL == fqdn_plugin_runtime || NULL == fqdn_plugin_schema || + NULL == line) { + return -1; + } + + struct fqdn_plugin_item *item = NULL; + struct FQDN_rule *rule = NULL; + struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema; + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + int item_id = get_column_value(line, schema->item_id_column); + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } + + if (schema->ex_schema != NULL) { + if (1 == is_valid) { + // add + item = fqdn_plugin_item_new(line, schema, fqdn_plugin_rt->logger); + if (NULL == item) { + return -1; + } + + //rule = fqdn_rule_new(line, schema, fqdn_plugin_rt->logger); + assert(rule != NULL); + //fqdn_plugin_item_free(item); + } + + char *key = (char *)&item_id; + int ret = fqdn_plugin_runtime_update_row(fqdn_plugin_rt, schema, line, key, + sizeof(int), rule, is_valid); + if (ret < 0) { + if (item != NULL) { + FREE(item); + } + return -1; + } else { + if (0 == is_valid) { + fqdn_plugin_rt->rule_num--; + } else { + fqdn_plugin_rt->rule_num++; + } + } + } else { + //ex_schema not set + ex_data_runtime_cache_row_put(fqdn_plugin_rt->ex_data_rt, line); + fqdn_plugin_rt->rule_num = ex_data_runtime_cached_row_count(fqdn_plugin_rt->ex_data_rt); + } + + return 0; +} + +int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name) +{ + if (NULL == fqdn_plugin_runtime) { + return -1; + } + + int ret = 0; + struct ex_data_container **ex_container = NULL; + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + struct ex_data_runtime *ex_data_rt = fqdn_plugin_rt->ex_data_rt; + + size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); + if (0 == rule_cnt) { + FREE(ex_container); + return 0; + } + + struct FQDN_rule *rules = ALLOC(struct FQDN_rule, rule_cnt); + + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(struct FQDN_rule *)ex_container[i]->custom_data; + assert(rules[i].user_tag == ex_container[i] || NULL == rules[i].user_tag); + rules[i].user_tag = ex_container[i]; + } + + struct FQDN_engine *new_fqdn_engine = NULL; + struct FQDN_engine *old_fqdn_engine = NULL; + + log_info(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN, + "table[%s] committing %zu fqdn_plugin rules for rebuilding FQDN engine", + table_name, rule_cnt); + + new_fqdn_engine = FQDN_engine_new(rules, rule_cnt); + if (NULL == new_fqdn_engine) { + log_error(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN, + "table[%s] rebuild FQDN engine failed when update %zu fqdn_plugin rules", + table_name, rule_cnt); + ret = -1; + } + + old_fqdn_engine = fqdn_plugin_rt->engine; + fqdn_plugin_rt->engine = new_fqdn_engine; + maat_garbage_bagging(fqdn_plugin_rt->ref_garbage_bin, old_fqdn_engine, + (void (*)(void*))FQDN_engine_free); + ex_data_runtime_commit(ex_data_rt); + + FREE(rules); + FREE(ex_container); + + return ret; +} + +struct ex_data_runtime *fqdn_plugin_runtime_get_ex_data_rt(void *fqdn_plugin_runtime) +{ + if (NULL == fqdn_plugin_runtime) { + return NULL; + } + + struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; + + return fqdn_plugin_rt->ex_data_rt; +} \ No newline at end of file diff --git a/src/maat_garbage_collection.cpp b/src/maat_garbage_collection.cpp index d778a35..4801b64 100644 --- a/src/maat_garbage_collection.cpp +++ b/src/maat_garbage_collection.cpp @@ -45,7 +45,7 @@ void maat_garbage_bin_free(struct maat_garbage_bin* bin) { struct maat_garbage_bag *p = NULL; - while (p = TAILQ_FIRST(&bin->garbage_q)) { + while ((p = TAILQ_FIRST(&bin->garbage_q)) != NULL) { p->garbage_free(p->garbage); TAILQ_REMOVE(&bin->garbage_q, p, entries); FREE(p); @@ -96,7 +96,7 @@ void maat_garbage_collect_routine(struct maat_garbage_bin* bin) void maat_garbage_collect_by_force(struct maat_garbage_bin* bin) { struct maat_garbage_bag *p = NULL; - while (p = TAILQ_FIRST(&bin->garbage_q)) { + while ((p = TAILQ_FIRST(&bin->garbage_q)) != NULL) { p->garbage_free(p->garbage); TAILQ_REMOVE(&bin->garbage_q, p, entries); FREE(p); diff --git a/src/maat_group.cpp b/src/maat_group.cpp index f9b62fe..386a2f0 100644 --- a/src/maat_group.cpp +++ b/src/maat_group.cpp @@ -28,7 +28,7 @@ struct group2group_schema { int group_id_column; int super_group_id_column; int table_id;//ugly - struct table_manager *tbl_mgr; + struct table_manager *ref_tbl_mgr; }; struct maat_group { @@ -99,7 +99,7 @@ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } - g2g_schema->tbl_mgr = tbl_mgr; + g2g_schema->ref_tbl_mgr = tbl_mgr; if (read_cnt < 3) { goto error; @@ -554,13 +554,20 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, return ret; } -int group2group_runtime_commit(void *g2g_runtime) +int group2group_runtime_commit(void *g2g_runtime, const char *table_name) { if (NULL == g2g_runtime) { return -1; } - return group2group_runtime_build_top_groups(g2g_runtime); + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + int ret = group2group_runtime_build_top_groups(g2g_runtime); + if (ret < 0) { + log_error(g2g_rt->logger, MODULE_GROUP, + "table[%s] group2group runtime commit failed", table_name); + } + + return ret; } int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, diff --git a/src/maat_interval.cpp b/src/maat_interval.cpp index fda0621..89ed40a 100644 --- a/src/maat_interval.cpp +++ b/src/maat_interval.cpp @@ -387,7 +387,7 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, return 0; } -int interval_runtime_commit(void *interval_runtime) +int interval_runtime_commit(void *interval_runtime, const char *table_name) { if (NULL == interval_runtime) { return -1; @@ -413,14 +413,14 @@ int interval_runtime_commit(void *interval_runtime) struct interval_matcher *old_interval_matcher = NULL; log_info(interval_rt->logger, MODULE_INTERVAL, - "committing %zu interval rules for rebuilding interval_matcher engine", - rule_cnt); + "table[%s] committing %zu interval rules for rebuilding interval_matcher engine", + table_name, rule_cnt); new_interval_matcher = interval_matcher_new(rules, rule_cnt); if (NULL == new_interval_matcher) { log_error(interval_rt->logger, MODULE_INTERVAL, - "rebuild interval_matcher engine failed when update %zu interval rules", - rule_cnt); + "table[%s]rebuild interval_matcher engine failed when update %zu interval rules", + table_name, rule_cnt); ret = -1; } diff --git a/src/maat_ip.cpp b/src/maat_ip.cpp index 5757ba8..c4be7f3 100644 --- a/src/maat_ip.cpp +++ b/src/maat_ip.cpp @@ -32,7 +32,7 @@ struct ip_schema { int sip1_column; int sip2_column; int table_id; //ugly - struct table_manager *tbl_mgr; + struct table_manager *ref_tbl_mgr; }; struct ipv4_item_rule { @@ -127,7 +127,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } - ip_schema->tbl_mgr = tbl_mgr; + ip_schema->ref_tbl_mgr = tbl_mgr; if (read_cnt < 7) { goto error; @@ -208,13 +208,8 @@ struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema, size_t column_offset = 0; size_t column_len = 0; char saddr_format[16] = {0}; - char sport_format[16] = {0}; char sip1_str[40] = {0}; char sip2_str[40] = {0}; - uint16_t sport1 = 0; - uint16_t sport2 = 0; - uint16_t protocol = 0; - uint16_t direction = 0; struct ip_item *ip_item = ALLOC(struct ip_item, 1); int ret = get_column_pos(line, ip_schema->item_id_column, &column_offset, &column_len); @@ -431,7 +426,7 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, return 0; } -int ip_runtime_commit(void *ip_runtime) +int ip_runtime_commit(void *ip_runtime, const char *table_name) { if (NULL == ip_runtime) { return -1; @@ -461,13 +456,13 @@ int ip_runtime_commit(void *ip_runtime) if (rule_cnt > 0) { log_info(ip_rt->logger, MODULE_IP, - "committing %zu ip rules for rebuilding ip_matcher engine", - rule_cnt); + "table[%s] committing %zu ip rules for rebuilding ip_matcher engine", + table_name, rule_cnt); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_ip_matcher) { log_error(ip_rt->logger, MODULE_IP, - "rebuild ip_matcher engine failed when update %zu ip rules", - rule_cnt); + "table[%s] rebuild ip_matcher engine failed when update %zu ip rules", + table_name, rule_cnt); ret = -1; } } diff --git a/src/maat_ip_plugin.cpp b/src/maat_ip_plugin.cpp index 1a00c1f..e555641 100644 --- a/src/maat_ip_plugin.cpp +++ b/src/maat_ip_plugin.cpp @@ -36,7 +36,7 @@ struct ip_plugin_schema { int rule_tag_column; struct ex_data_schema *ex_schema; int table_id; //ugly - struct table_manager *tbl_mgr; + struct table_manager *ref_tbl_mgr; unsigned long long update_err_cnt; unsigned long long unmatch_tag_cnt; @@ -57,12 +57,12 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger) { size_t read_cnt = 0; - struct ip_plugin_schema *ip_plugin_schema = ALLOC(struct ip_plugin_schema, 1); + struct ip_plugin_schema *schema = ALLOC(struct ip_plugin_schema, 1); cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); if (item != NULL && item->type == cJSON_Number) { - ip_plugin_schema->table_id = item->valueint; + schema->table_id = item->valueint; read_cnt++; } @@ -75,37 +75,43 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, custom_item = cJSON_GetObjectItem(item, "item_id"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plugin_schema->item_id_column = custom_item->valueint; + schema->item_id_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "ip_type"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plugin_schema->ip_type_column = custom_item->valueint; + schema->ip_type_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "start_ip"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plugin_schema->start_ip_column = custom_item->valueint; + schema->start_ip_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "end_ip"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plugin_schema->end_ip_column = custom_item->valueint; + schema->end_ip_column = custom_item->valueint; read_cnt++; } - ip_plugin_schema->tbl_mgr = tbl_mgr; + // rule_tag is optional + custom_item = cJSON_GetObjectItem(item, "rule_tag"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->rule_tag_column = custom_item->valueint; + } + + schema->ref_tbl_mgr = tbl_mgr; if (read_cnt < 5) { goto error; } - return ip_plugin_schema; + return schema; error: - FREE(ip_plugin_schema); + FREE(schema); return NULL; } @@ -139,7 +145,7 @@ int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line { size_t column_offset = 0; size_t column_len = 0; - size_t n_tag = table_manager_accept_tags_count(schema->tbl_mgr); + size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr); if (schema->rule_tag_column > 0 && n_tag > 0) { int ret = get_column_pos(line, schema->rule_tag_column, @@ -155,7 +161,7 @@ int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line if (column_len > 2) { char *tag_str = ALLOC(char, column_len + 1); memcpy(tag_str, (line + column_offset), column_len); - ret = table_manager_accept_tags_match(schema->tbl_mgr, tag_str); + ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str); FREE(tag_str); if (TAG_MATCH_ERR == ret) { log_error(logger, MODULE_IP_PLUGIN, @@ -372,36 +378,42 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; - } - - if (1 == is_valid) { - //add - ip_plugin_item = ip_plugin_item_new(line, schema, ip_plugin_rt->logger); - if (NULL == ip_plugin_item) { - return -1; - } } - char *key = (char *)&item_id; - int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key, - sizeof(int), ip_plugin_item, is_valid); - if (ret < 0) { - if (ip_plugin_item != NULL) { - FREE(ip_plugin_item); + if (schema->ex_schema != NULL) { + if (1 == is_valid) { + // add + ip_plugin_item = ip_plugin_item_new(line, schema, ip_plugin_rt->logger); + if (NULL == ip_plugin_item) { + return -1; + } } - return -1; - } else { - if (0 == is_valid) { - ip_plugin_rt->rule_num--; + + char *key = (char *)&item_id; + int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key, + sizeof(int), ip_plugin_item, is_valid); + if (ret < 0) { + if (ip_plugin_item != NULL) { + FREE(ip_plugin_item); + } + return -1; } else { - ip_plugin_rt->rule_num++; + if (0 == is_valid) { + ip_plugin_rt->rule_num--; + } else { + ip_plugin_rt->rule_num++; + } } + } else { + //ex_schema not set + ex_data_runtime_cache_row_put(ip_plugin_rt->ex_data_rt, line); + ip_plugin_rt->rule_num = ex_data_runtime_cached_row_count(ip_plugin_rt->ex_data_rt); } return 0; } -int ip_plugin_runtime_commit(void *ip_plugin_runtime) +int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name) { if (NULL == ip_plugin_runtime) { return -1; @@ -431,13 +443,13 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime) if (rule_cnt > 0) { log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, - "committing %zu ip_plugin rules for rebuilding ip_matcher engine", - rule_cnt); + "table[%s] committing %zu ip_plugin rules for rebuilding ip_matcher engine", + table_name, rule_cnt); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_ip_matcher) { log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, - "rebuild ip_matcher engine failed when update %zu ip_plugin rules", - rule_cnt); + "table[%s] rebuild ip_matcher engine failed when update %zu ip_plugin rules", + table_name, rule_cnt); ret = -1; } } diff --git a/src/maat_kv.cpp b/src/maat_kv.cpp index 2b19e17..03ab275 100644 --- a/src/maat_kv.cpp +++ b/src/maat_kv.cpp @@ -120,7 +120,7 @@ int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t key strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase)); HASH_FIND(hh, store->hash, key_lowercase, keylen, kv); if (kv) { - *value=kv->val; + *value = kv->val; return 1; } else { return -1; diff --git a/src/maat_plugin.cpp b/src/maat_plugin.cpp index 1eddfaa..78ace61 100644 --- a/src/maat_plugin.cpp +++ b/src/maat_plugin.cpp @@ -41,7 +41,6 @@ struct plugin_runtime { #define MAX_PLUGIN_PER_TABLE 32 struct plugin_schema { - int item_id_column; int key_column; int rule_tag_column; int n_foreign; @@ -50,7 +49,7 @@ struct plugin_schema { struct plugin_callback_schema cb[MAX_PLUGIN_PER_TABLE]; struct ex_data_schema *ex_schema; int table_id; //ugly - struct table_manager *tbl_mgr; + struct table_manager *ref_tbl_mgr; unsigned long long update_err_cnt; unsigned long long unmatch_tag_cnt; @@ -76,67 +75,57 @@ static int read_integer_array(char *string, int *array, int size) void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger) { - size_t read_cnt = 0; - struct plugin_schema *plugin_schema = ALLOC(struct plugin_schema, 1); + struct plugin_schema *schema = ALLOC(struct plugin_schema, 1); cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (item != NULL && item->type == cJSON_Number) { - plugin_schema->table_id = item->valueint; - read_cnt++; + if (NULL == item || item->type != cJSON_Number) { + log_error(logger, MODULE_PLUGIN, + "plugin table %s has no table_id column", table_name); + goto error; } + schema->table_id = item->valueint; item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { log_error(logger, MODULE_PLUGIN, - "table %s has no custom column", table_name); + "plugin table %s has no custom column", table_name); goto error; } - custom_item = cJSON_GetObjectItem(item, "item_id"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - plugin_schema->item_id_column = custom_item->valueint; - read_cnt++; - } - custom_item = cJSON_GetObjectItem(item, "key"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - plugin_schema->key_column = custom_item->valueint; - read_cnt++; + if (NULL == custom_item || custom_item->type != cJSON_Number) { + log_error(logger, MODULE_PLUGIN, + "plugin table: %s has no key column", table_name); + goto error; } - + schema->key_column = custom_item->valueint; + custom_item = cJSON_GetObjectItem(item, "tag"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - plugin_schema->rule_tag_column = custom_item->valueint; - read_cnt++; + schema->rule_tag_column = custom_item->valueint; } custom_item = cJSON_GetObjectItem(item, "foreign"); if (custom_item != NULL) { - read_cnt++; if (custom_item->type == cJSON_String) { - plugin_schema->n_foreign = read_integer_array(custom_item->valuestring, - plugin_schema->foreign_columns, - MAX_FOREIGN_CLMN_NUM); + schema->n_foreign = read_integer_array(custom_item->valuestring, + schema->foreign_columns, + MAX_FOREIGN_CLMN_NUM); } else if (custom_item->type == cJSON_Array) { - plugin_schema->n_foreign = cJSON_GetArraySize(custom_item); - for (int i = 0; i < plugin_schema->n_foreign; i++) { + schema->n_foreign = cJSON_GetArraySize(custom_item); + for (int i = 0; i < schema->n_foreign; i++) { cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i); assert(foreign_item->type == cJSON_Number); - plugin_schema->foreign_columns[i] = foreign_item->valueint; + schema->foreign_columns[i] = foreign_item->valueint; } } } - plugin_schema->tbl_mgr = tbl_mgr; - - if (read_cnt < 5) { - goto error; - } - - return plugin_schema; + schema->ref_tbl_mgr = tbl_mgr; + return schema; error: - FREE(plugin_schema); + FREE(schema); return NULL; } @@ -331,7 +320,7 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *line, { size_t column_offset = 0; size_t column_len = 0; - size_t n_tag = table_manager_accept_tags_count(schema->tbl_mgr); + size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr); if (schema->rule_tag_column > 0 && n_tag > 0) { int ret = get_column_pos(line, schema->rule_tag_column, &column_offset, &column_len); @@ -346,7 +335,7 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *line, if (column_len > 2) { char *tag_str = ALLOC(char, column_len + 1); memcpy(tag_str, (line + column_offset), column_len); - ret = table_manager_accept_tags_match(schema->tbl_mgr, tag_str); + ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str); FREE(tag_str); if (TAG_MATCH_ERR == ret) { log_error(logger, MODULE_PLUGIN, @@ -386,7 +375,7 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, return -1; } - int item_id = get_column_value(line, schema->item_id_column); + int item_id = get_column_value(line, schema->key_column); char *key = (char *)&item_id; ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid); if (ret < 0) { @@ -398,7 +387,7 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, return 0; } -int plugin_runtime_commit(void *plugin_runtime) +int plugin_runtime_commit(void *plugin_runtime, const char *table_name) { if (NULL == plugin_runtime) { return -1; diff --git a/src/maat_redis_monitor.cpp b/src/maat_redis_monitor.cpp index 305205d..3f57f01 100644 --- a/src/maat_redis_monitor.cpp +++ b/src/maat_redis_monitor.cpp @@ -1351,8 +1351,6 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, int call_update_num = 0; int valid_column = -1; enum table_type table_type; - enum scan_type scan_type; - //void *table_schema = NULL; struct maat *maat_instance = (struct maat *)u_param; //authorized to write diff --git a/src/maat_table.cpp b/src/maat_table.cpp index 6a645de..6ef6d8b 100644 --- a/src/maat_table.cpp +++ b/src/maat_table.cpp @@ -24,6 +24,9 @@ #include "maat_flag.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" +#include "maat_bool_plugin.h" +#include "maat_fqdn_plugin.h" +#include "maat_interval.h" #include "maat_virtual.h" #define MODULE_TABLE module_name_str("maat.table") @@ -72,21 +75,18 @@ struct table_operations { void (*free_runtime)(void *runtime); int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column); - int (*commit_runtime)(void *runtime); + int (*commit_runtime)(void *runtime, const char *table_name); }; struct table_operations table_ops[TABLE_TYPE_MAX] = { - // { - // .type = TABLE_TYPE_FLAG, - // .new_schema = flag_schema_new, - // .free_schema = flag_schema_free, - // .new_runtime = flag_runtime_new, - // .free_runtime = flag_runtime_free, - // .update_runtime = flag_runtime_update, - // .commit_runtime = flag_runtime_commit - // }, { - .type = TABLE_TYPE_FLAG + .type = TABLE_TYPE_FLAG, + .new_schema = flag_schema_new, + .free_schema = flag_schema_free, + .new_runtime = flag_runtime_new, + .free_runtime = flag_runtime_free, + .update_runtime = flag_runtime_update, + .commit_runtime = flag_runtime_commit }, { .type = TABLE_TYPE_EXPR, @@ -117,21 +117,21 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { }, { .type = TABLE_TYPE_INTERVAL, - .new_schema = NULL, - .free_schema = NULL, - .new_runtime = NULL, - .free_runtime = NULL, - .update_runtime = NULL, - .commit_runtime = NULL + .new_schema = interval_schema_new, + .free_schema = interval_schema_free, + .new_runtime = interval_runtime_new, + .free_runtime = interval_runtime_free, + .update_runtime = interval_runtime_update, + .commit_runtime = interval_runtime_commit }, { .type = TABLE_TYPE_INTERVAL_PLUS, - .new_schema = NULL, - .free_schema = NULL, - .new_runtime = NULL, - .free_runtime = NULL, - .update_runtime = NULL, - .commit_runtime = NULL + .new_schema = interval_schema_new, + .free_schema = interval_schema_free, + .new_runtime = interval_runtime_new, + .free_runtime = interval_runtime_free, + .update_runtime = interval_runtime_update, + .commit_runtime = interval_runtime_commit }, { .type = TABLE_TYPE_DIGEST @@ -168,21 +168,21 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { }, { .type = TABLE_TYPE_FQDN_PLUGIN, - .new_schema = NULL, - .free_schema = NULL, - .new_runtime = NULL, - .free_runtime = NULL, - .update_runtime = NULL, - .commit_runtime = NULL + .new_schema = fqdn_plugin_schema_new, + .free_schema = fqdn_plugin_schema_free, + .new_runtime = fqdn_plugin_runtime_new, + .free_runtime = fqdn_plugin_runtime_free, + .update_runtime = fqdn_plugin_runtime_update, + .commit_runtime = fqdn_plugin_runtime_commit }, { .type = TABLE_TYPE_BOOL_PLUGIN, - .new_schema = NULL, - .free_schema = NULL, - .new_runtime = NULL, - .free_runtime = NULL, - .update_runtime = NULL, - .commit_runtime = NULL + .new_schema = bool_plugin_schema_new, + .free_schema = bool_plugin_schema_free, + .new_runtime = bool_plugin_runtime_new, + .free_runtime = bool_plugin_runtime_free, + .update_runtime = bool_plugin_runtime_update, + .commit_runtime = bool_plugin_runtime_commit }, { .type = TABLE_TYPE_VIRTUAL, @@ -252,6 +252,7 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map) maat_kv_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE); maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); + maat_kv_register(reserved_word_map, "flag", TABLE_TYPE_FLAG); maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL); @@ -314,7 +315,7 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo "table(table_id:%d) has no table name", ptable->table_id); goto error; } - + if (strlen(item->valuestring) >= NAME_MAX) { log_error(logger, MODULE_TABLE, "table(table_id:%d) name %s length too long", @@ -440,7 +441,6 @@ table_manager_create(const char *table_info_path, const char *accept_tags, if (json != NULL && json->type == cJSON_Object) { struct maat_table *maat_tbl = maat_table_new(json, reserved_word_map, logger); if (NULL == maat_tbl) { - log_error(logger, MODULE_TABLE, "Maat table new failed."); continue; } @@ -452,7 +452,9 @@ table_manager_create(const char *table_info_path, const char *accept_tags, maat_table_free(maat_tbl); continue; } - + log_info(logger, MODULE_TABLE, "register table[%s]->table_id:%d", + maat_tbl->table_name, maat_tbl->table_id); + if (maat_tbl->table_type == TABLE_TYPE_COMPILE) { if (maat_tbl->table_id < default_compile_table_id) { default_compile_table_id = maat_tbl->table_id; @@ -778,8 +780,8 @@ void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) return; } - printf("table_id:%d\n", table_id); + struct maat_table *ptable = tbl_mgr->tbl[table_id]; if ( table_ops[table_type].commit_runtime != NULL) { - table_ops[table_type].commit_runtime(runtime); + table_ops[table_type].commit_runtime(runtime, ptable->table_name);; } } \ No newline at end of file diff --git a/src/maat_virtual.cpp b/src/maat_virtual.cpp index 364d462..8785f08 100644 --- a/src/maat_virtual.cpp +++ b/src/maat_virtual.cpp @@ -23,7 +23,7 @@ struct virtual_schema { int physical_table_id[SCAN_TYPE_MAX]; int table_id; - struct table_manager *tbl_mgr; + struct table_manager *ref_tbl_mgr; }; void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -37,7 +37,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, } struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); - vt_schema->tbl_mgr = tbl_mgr; + vt_schema->ref_tbl_mgr = tbl_mgr; int cnt = cJSON_GetArraySize(item); for (int i = 0; i < cnt; i++) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 54a6ab1..79788f0 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,6 +14,9 @@ target_link_libraries(maat_input_mode_gtest maat_frame_static gtest_static) add_executable(maat_framework_gtest maat_framework_gtest.cpp) target_link_libraries(maat_framework_gtest maat_frame_static gtest_static) +add_executable(adapter_hs_gtest adapter_hs_gtest.cpp) +target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static) + file(COPY rule DESTINATION ./) file(COPY table_info.conf DESTINATION ./) file(COPY and_expr.conf DESTINATION ./) diff --git a/scanner/adapter_hs_gtest.cpp b/test/adapter_hs_gtest.cpp similarity index 51% rename from scanner/adapter_hs_gtest.cpp rename to test/adapter_hs_gtest.cpp index 6b4cd26..2b0145f 100644 --- a/scanner/adapter_hs_gtest.cpp +++ b/test/adapter_hs_gtest.cpp @@ -1,6 +1,10 @@ #include +#include "log/log.h" #include "adapter_hs.h" +#include "maat_utils.h" + +struct log_handle *g_logger = NULL; int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr) { @@ -10,6 +14,7 @@ int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr) size_t i = 0; while (NULL != fgets(line, sizeof(line), fp)) { if (line[0] == '#' || line[0] == ' ' || line[0] == '\t') { + memset(line, 0, sizeof(line)); continue; } @@ -17,6 +22,7 @@ int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr) int ret = sscanf(line, "%u\t%lu\t%s", &(expr[i].expr_id), &(expr[i].n_patterns), pattern_buf); EXPECT_NE(ret, 0); + str_unescape(pattern_buf); char *expr_token = NULL; char *sub_expr_token = NULL; char *save_expr_ptr = NULL; @@ -38,6 +44,7 @@ int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr) j++; } i++; + memset(line, 0, sizeof(line)); } *n_expr = i; fclose(fp); @@ -58,18 +65,18 @@ void expr_array_free(and_expr_t expr_array[], size_t n_expr_array) TEST(block_mode_initialize, invalid_input_parameter) { struct adapter_hs *hs_instance = NULL; - and_expr_t exprs[5]; + and_expr_t exprs[64]; /* case1: invalid scan_mode parameter */ - hs_instance = adapter_hs_initialize(0, 1, exprs, 1); + hs_instance = adapter_hs_initialize(0, 1, exprs, 1, g_logger); EXPECT_EQ(hs_instance, nullptr); /* case2: invalid expr parameter */ - hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 1); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 1, g_logger); EXPECT_EQ(hs_instance, nullptr); /* case3: invalid expr num */ - hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, exprs, 0); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, exprs, 0, g_logger); EXPECT_EQ(hs_instance, nullptr); } @@ -78,16 +85,16 @@ TEST(block_mode_scan, invalid_input_parameter) and_expr_t expr_array[64]; size_t n_expr_array = 0; - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 0); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 0, g_logger); EXPECT_EQ(hs_instance, nullptr); - hs_instance = adapter_hs_initialize(0, 1, expr_array, n_expr_array); + hs_instance = adapter_hs_initialize(0, 1, expr_array, n_expr_array, g_logger); EXPECT_EQ(hs_instance, nullptr); n_expr_array = 1; expr_array[0].expr_id = 101; expr_array[0].n_patterns = 10; - hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); EXPECT_EQ(hs_instance, nullptr); memset(expr_array, 0, sizeof(expr_array)); @@ -95,197 +102,295 @@ TEST(block_mode_scan, invalid_input_parameter) expr_array[0].expr_id = 101; expr_array[0].n_patterns = 1; expr_array[0].patterns[0].type = 0; - hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); EXPECT_EQ(hs_instance, nullptr); } -TEST(block_mode_scan, hit_one_expr) +TEST(block_mode_scan, literal_hit1) { and_expr_t expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_expr_array, 6); + EXPECT_EQ(n_expr_array, 10); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); char data0[64] = "luis"; - int result0[64] = {0}; + struct hs_scan_result result0[64] = {0}; size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0); + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result0, 0); char data1[64] = "hello"; - int result1[64] = {0}; + struct hs_scan_result result1[64] = {0}; size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1); + ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result1, 1); - EXPECT_EQ(result1[0], 101); + EXPECT_EQ(result1[0].item_id, 101); char data2[64] = "world"; - int result2[64] = {0}; + struct hs_scan_result result2[64] = {0}; size_t n_result2 = 0; - ret = adapter_hs_scan(hs_instance, 0, data2, strlen(data2), result2, &n_result2); + ret = adapter_hs_scan(hs_instance, 0, data2, strlen(data2), result2, 64, &n_result2); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result2, 1); - EXPECT_EQ(result2[0], 102); + EXPECT_EQ(result2[0].item_id, 102); adapter_hs_destroy(hs_instance); hs_instance = nullptr; } -TEST(block_mode_scan, hit_two_expr) +TEST(block_mode_scan, literal_hit2) { and_expr_t expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_expr_array, 6); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, + expr_array, n_expr_array, g_logger); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); char data0[64] = "hello maat"; - int result0[64] = {0}; + struct hs_scan_result result0[64] = {0}; size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0); + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result0, 2); - EXPECT_EQ(result0[0], 103); - EXPECT_EQ(result0[1], 101); + EXPECT_EQ(result0[0].item_id, 103); + EXPECT_EQ(result0[1].item_id, 101); char data1[64] = "maat World"; - int result1[64] = {0}; + struct hs_scan_result result1[64] = {0}; size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1); + ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result1, 2); - EXPECT_EQ(result1[0], 103); - EXPECT_EQ(result1[1], 102); + EXPECT_EQ(result1[0].item_id, 103); + EXPECT_EQ(result1[1].item_id, 102); adapter_hs_destroy(hs_instance); hs_instance = nullptr; } -TEST(block_mode_scan, hit_three_expr) +TEST(block_mode_scan, literal_hit3) { and_expr_t expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_expr_array, 6); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); char data0[64] = "hello world"; - int result0[64] = {0}; + struct hs_scan_result result0[64] = {0}; size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0); + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result0, 3); - EXPECT_EQ(result0[0], 104); - EXPECT_EQ(result0[1], 102); - EXPECT_EQ(result0[2], 101); + EXPECT_EQ(result0[0].item_id, 104); + EXPECT_EQ(result0[1].item_id, 102); + EXPECT_EQ(result0[2].item_id, 101); char data1[64] = "hello World"; - int result1[64] = {0}; + struct hs_scan_result result1[64] = {0}; size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1); + ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result1, 3); - EXPECT_EQ(result1[0], 104); - EXPECT_EQ(result1[1], 102); - EXPECT_EQ(result1[2], 101); + EXPECT_EQ(n_result1, 2); + EXPECT_EQ(result1[0].item_id, 102); + EXPECT_EQ(result1[1].item_id, 101); adapter_hs_destroy(hs_instance); hs_instance = nullptr; } -TEST(block_mode_scan, hit_four_expr) +TEST(block_mode_scan, literal_russian_hit3) { and_expr_t expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_expr_array, 6); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); + EXPECT_NE(hs_instance, nullptr); + expr_array_free(expr_array, n_expr_array); + + char data0[64] = "يىلىدىكى"; + struct hs_scan_result result0[64] = {0}; + size_t n_result0 = 0; + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result0, 1); + EXPECT_EQ(result0[0].item_id, 109); + + adapter_hs_destroy(hs_instance); + hs_instance = nullptr; +} + +TEST(block_mode_scan, literal_case_insensitive_hit4) +{ + and_expr_t expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); + EXPECT_NE(hs_instance, nullptr); + expr_array_free(expr_array, n_expr_array); + + char data0[64] = "today"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char data1[64] = "Today"; + n_result = 0; + memset(result, 0, sizeof(result)); + ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 110); + + adapter_hs_destroy(hs_instance); + hs_instance = nullptr; +} + +TEST(block_mode_scan, literal_and_regex1) +{ + and_expr_t expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, + expr_array, n_expr_array, g_logger); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); char data0[64] = "1hello world"; - int result0[64] = {0}; + struct hs_scan_result result0[64] = {0}; size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0); + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result0, 4); - EXPECT_EQ(result0[0], 105); - EXPECT_EQ(result0[1], 104); - EXPECT_EQ(result0[2], 102); - EXPECT_EQ(result0[3], 101); + EXPECT_EQ(result0[0].item_id, 105); + EXPECT_EQ(result0[1].item_id, 104); + EXPECT_EQ(result0[2].item_id, 102); + EXPECT_EQ(result0[3].item_id, 101); char data1[64] = "8hello World"; - int result1[64] = {0}; + struct hs_scan_result result1[64] = {0}; size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1); + ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result1, 4); - EXPECT_EQ(result1[0], 105); - EXPECT_EQ(result1[1], 104); - EXPECT_EQ(result1[2], 102); - EXPECT_EQ(result1[3], 101); + EXPECT_EQ(n_result1, 2); + EXPECT_EQ(result1[0].item_id, 102); + EXPECT_EQ(result1[1].item_id, 101); adapter_hs_destroy(hs_instance); hs_instance = nullptr; } -TEST(block_mode_scan, hit_five_expr) +TEST(block_mode_scan, literal_and_regex2) { and_expr_t expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_expr_array, 6); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, + expr_array, n_expr_array, g_logger); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); char data0[64] = "1hello 2world"; - int result0[64] = {0}; + struct hs_scan_result result0[64] = {0}; size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, &n_result0); + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result0, 5); - EXPECT_EQ(result0[0], 106); - EXPECT_EQ(result0[1], 105); - EXPECT_EQ(result0[2], 104); - EXPECT_EQ(result0[3], 102); - EXPECT_EQ(result0[4], 101); + EXPECT_EQ(result0[0].item_id, 106); + EXPECT_EQ(result0[1].item_id, 105); + EXPECT_EQ(result0[2].item_id, 104); + EXPECT_EQ(result0[3].item_id, 102); + EXPECT_EQ(result0[4].item_id, 101); char data1[64] = "8hello 9World"; - int result1[64] = {0}; + struct hs_scan_result result1[64] = {0}; size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, &n_result1); + ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result1, 5); - EXPECT_EQ(result1[0], 106); - EXPECT_EQ(result1[1], 105); - EXPECT_EQ(result1[2], 104); - EXPECT_EQ(result1[3], 102); - EXPECT_EQ(result1[4], 101); + EXPECT_EQ(n_result1, 2); + EXPECT_EQ(result1[0].item_id, 102); + EXPECT_EQ(result1[1].item_id, 101); + + adapter_hs_destroy(hs_instance); + hs_instance = nullptr; +} + +TEST(block_mode_scan, regex_hit1) +{ + and_expr_t expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, + expr_array, n_expr_array, g_logger); + EXPECT_NE(hs_instance, nullptr); + expr_array_free(expr_array, n_expr_array); + + char data0[64] = "Cookie: Txa123aheadBCAxd"; + struct hs_scan_result result0[64] = {0}; + size_t n_result0 = 0; + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result0, 1); + EXPECT_EQ(result0[0].item_id, 107); + + adapter_hs_destroy(hs_instance); + hs_instance = nullptr; +} + +TEST(block_mode_scan, chinese_hit1) +{ + and_expr_t expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, + expr_array, n_expr_array, g_logger); + EXPECT_NE(hs_instance, nullptr); + expr_array_free(expr_array, n_expr_array); + + char data0[64] = "中国 你好"; + struct hs_scan_result result0[64] = {0}; + size_t n_result0 = 0; + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result0, 1); + EXPECT_EQ(result0[0].item_id, 108); adapter_hs_destroy(hs_instance); hs_instance = nullptr; @@ -295,6 +400,10 @@ int main(int argc, char **argv) { int ret = 0; ::testing::InitGoogleTest(&argc, argv); + g_logger = log_handle_create("./tmp.log", 0); + ret = RUN_ALL_TESTS(); + + log_handle_destroy(g_logger); return ret; } diff --git a/test/and_expr.conf b/test/and_expr.conf index 852393c..f795066 100644 --- a/test/and_expr.conf +++ b/test/and_expr.conf @@ -1,10 +1,14 @@ # logic AND expressions sample # expr_id pattern_num expression(several patterns) # pattern_type:pattern_string&pattern_type:pattern_string&... -# PATTERN_TYPE_STR(1) PATTERN_TYPE_REG(2) -101 1 1:hello -102 1 2:[W|w]orld -103 1 1:maat -104 2 1:hello&1:world -105 2 2:[0-9]hello&1:world -106 2 2:[0-9]hello&2:[0-9]world \ No newline at end of file +# PATTERN_TYPE_STR(0) PATTERN_TYPE_REG(1) +101 1 0:hello +102 1 1:[W|w]orld +103 1 0:maat +104 2 0:hello&0:world +105 2 1:[0-9]hello&0:world +106 2 1:[0-9]hello&1:[0-9]world +107 2 1:Cookie:\\s&1:head +108 1 0:你好 +109 1 0:يىلىدىكى +110 1 0:Today \ No newline at end of file diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index c6c3576..b84d6b1 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -15,11 +15,126 @@ const char *json_path="./maat_json.json"; const char *json_filename = "maat_json.json"; struct maat *g_maat_instance = NULL; +class MaatFlagScan : public testing::Test +{ +protected: + static void SetUpTestCase() { + flag_table_id = maat_table_get_id(g_maat_instance, "FLAG_CONFIG"); + expr_table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); + } + + static void TearDownTestCase() { + + } + + static int flag_table_id; + static int expr_table_id; +}; +int MaatFlagScan::flag_table_id; +int MaatFlagScan::expr_table_id; + +TEST_F(MaatFlagScan, basic) { + int flag_table_id = MaatFlagScan::flag_table_id; + + //compile_id:192 flag: 0000 0001 mask: 0000 0011 + //scan_data: 0000 1001 or 0000 1101 should hit + uint64_t scan_data = 9; + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data, results, + 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 192); + + struct maat_hit_path hit_path[128] = {0}; + int n_read = 0; + n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, sizeof(hit_path)); + EXPECT_NE(n_read, 0); + maat_state_free(&state); + + scan_data = 13; + memset(results, 0, sizeof(results)); + n_hit_result = 0; + ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data, results, + 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 192); + maat_state_free(&state); + + scan_data = 6; + memset(results, 0, sizeof(results)); + n_hit_result = 0; + ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data, results, + 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + EXPECT_EQ(n_hit_result, 0); + maat_state_free(&state); +} + +TEST_F(MaatFlagScan, withExprRegion) { + int flag_table_id = MaatFlagScan::flag_table_id; + int expr_table_id = MaatFlagScan::expr_table_id; + + //compile_id:193 flag: 0000 0010 mask: 0000 0011 + //scan_data: 0000 0010 or 0000 0100 should hit + uint64_t flag_scan_data = 2; + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + + int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, flag_scan_data, results, + 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + EXPECT_EQ(n_hit_result, 0); + + struct maat_hit_path hit_path[128] = {0}; + int n_read = 0; + n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, sizeof(hit_path)); + EXPECT_NE(n_read, 0); + + const char *expr_scan_data = "hello world"; + ret = maat_scan_string(g_maat_instance, expr_table_id, 0, expr_scan_data, strlen(expr_scan_data), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 193); + maat_state_free(&state); +} + +TEST_F(MaatFlagScan, hitMultiCompile) { + int flag_table_id = MaatFlagScan::flag_table_id; + + //compile_id:192 flag: 0000 0001 mask: 0000 0011 + //compile_id:194 flag: 0001 0101 mask: 0001 1111 + //scan_data: 0001 0101 should hit compile192 and compile194 + uint64_t flag_scan_data = 21; + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + + int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, flag_scan_data, results, + 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 2); + EXPECT_EQ(results[0], 194); + EXPECT_EQ(results[1], 192); + + struct maat_hit_path hit_path[128] = {0}; + int n_read = 0; + n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, sizeof(hit_path)); + EXPECT_NE(n_read, 0); + + maat_state_free(&state); +} + class MaatStringScan : public testing::Test { protected: static void SetUpTestCase() { - table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); + table_id = maat_table_get_id(g_maat_instance, "KEYWORDS_TABLE"); } static void TearDownTestCase() { @@ -30,16 +145,17 @@ protected: }; int MaatStringScan::table_id; -TEST_F(MaatStringScan, hit_one_expr) { +TEST_F(MaatStringScan, Expr8) { int table_id = MaatStringScan::table_id; - char scan_data[128] = "hello"; + char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); + int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), + results, 5, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 191); + EXPECT_EQ(results[0], 182); struct maat_hit_path hit_path[128] = {0}; int n_read = 0; @@ -48,13 +164,200 @@ TEST_F(MaatStringScan, hit_one_expr) { maat_state_free(&state); } +TEST_F(MaatStringScan, Regex) { + int ret = 0; + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + const char *cookie = "Cookie: Txa123aheadBCAxd"; + int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); + ASSERT_GT(table_id, 0); + + ret = maat_scan_string(g_maat_instance, table_id, 0, cookie, strlen(cookie), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(results[0], 146); + maat_state_free(&state); + //TODO: +#if 0 + size_t i = 0; + n_hit_result = 0; + memset(results, 0, sizeof(results)); + const char *sni_should_not_hit[] = {"instagram.fbcdn.net", "a.instagram.fbcdn.net"}; + const char *sni_should_hit[] = {"xx.fbcdn.net", "ainstagram.fbcdn.net"}; + for (i = 0; i < sizeof(sni_should_not_hit)/sizeof(const char *); i++) { + ret = maat_scan_string(g_maat_instance, table_id, 0, sni_should_not_hit[i], strlen(sni_should_not_hit[i]), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, 0); + maat_state_free(&state); + } + + for (i = 0; i < sizeof(sni_should_hit)/sizeof(const char *); i++) { + ret = maat_scan_string(g_maat_instance, table_id, 0, sni_should_hit[i], strlen(sni_should_hit[i]), + results, 5, &n_hit_result, &state); + EXPECT_GE(ret, 1); + EXPECT_EQ(results[0], 149); + maat_state_free(&state); + } +#endif +} + +TEST_F(MaatStringScan, ExprPlus) { + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + const char *region_name1 ="HTTP URL"; + const char *region_name2 ="我的diStricT"; + const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567"; + const char *scan_data2 = "Addis Sapphire Hotel"; + + int table_id = maat_table_get_id(g_maat_instance, "HTTP_SIGNATURE"); + ASSERT_GT(table_id, 0); + + int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data1, strlen(scan_data1), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_ERR);//Should return error for district not setting. + + ret = maat_state_set_scan_district(g_maat_instance, &state, region_name1, strlen(region_name1)); + ASSERT_EQ(ret, 0); + ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data1, strlen(scan_data1), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(results[0], 128); + maat_state_free(&state); + + ret = maat_state_set_scan_district(g_maat_instance, &state, region_name2, strlen(region_name2)); + ASSERT_EQ(ret, 0); + ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data2, strlen(scan_data2), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(results[0], 190); + maat_state_free(&state); +} +//TODO: +TEST_F(MaatStringScan, ShouldNotHitExprPlus) { + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + const char *region_name = "tcp.payload"; + unsigned char udp_payload_not_hit[] = { /* Stun packet */ + 0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42, + 0x4f, 0xc2, 0xc2, 0x70, 0xb3, 0xa8, 0x4e, 0x22, + 0xf5, 0x22, 0x87, 0x4c, 0x40, 0x00, 0x00, 0x46, + 0x03, 0x02, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01, + 0x3a, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a, + 0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9, + 0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b, + 0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2, + 0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f, + 0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64, + 0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a, + 0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a, 0xab, 0x00 }; + + int table_id = maat_table_get_id(g_maat_instance, "APP_PAYLOAD"); + ASSERT_GT(table_id, 0); + + int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name)); + ASSERT_EQ(ret, 0); + ret = maat_scan_string(g_maat_instance, table_id, 0, (char *)udp_payload_not_hit, sizeof(udp_payload_not_hit), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + maat_state_free(&state); +} + +TEST_F(MaatStringScan, ExprPlusWithHex) { + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + const char *scan_data1 = "text/html; charset=UTF-8"; + const char *scan_data2 = "Batman\\:Take me Home.Superman/:Fine,stay with me."; + const char *region_name1 = "Content-Type"; + const char *region_name2 = "User-Agent"; + + int table_id = maat_table_get_id(g_maat_instance, "HTTP_SIGNATURE"); + ASSERT_GT(table_id, 0); + + int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name1, strlen(region_name1)); + ASSERT_EQ(ret, 0); + ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data1, strlen(scan_data1), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(results[0], 156); + + ret = maat_state_set_scan_district(g_maat_instance, &state, region_name2, strlen(region_name2)); + ASSERT_EQ(ret, 0); + ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data1, strlen(scan_data1), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + table_id = maat_table_get_id(g_maat_instance, "KEYWORDS_TABLE"); + ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data2, strlen(scan_data2), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(results[0], 132); + maat_state_free(&state); +} + +TEST_F(MaatStringScan, ExprPlusWithOffset) +{ + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + const char *region_name = "Payload"; + unsigned char udp_payload_not_hit[] = { /* Stun packet */ + 0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42, + 0x4f, 0xc2, 0xc2, 0x70, 0xb3, 0xa8, 0x4e, 0x22, + 0xf5, 0x22, 0x87, 0x4c, 0x40, 0x00, 0x00, 0x46, + 0x03, 0x02, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01, + 0x3a, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a, + 0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9, + 0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b, + 0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2, + 0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f, + 0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64, + 0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a, + 0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a }; + unsigned char udp_payload_hit[] = { /* Stun packet */ //rule:"1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d" + 0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42, //1-1:03 + 0x4f, 0xc2, 0x2d, 0x70, 0xb3, 0xa8, 0x4e, 0x2d, //10-10:2d + 0x34, 0x22, 0x87, 0x4c, 0x2d, 0x00, 0x00, 0x46, //15-16:2d34&20-20:2d + 0x2d, 0x34, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01, //24-24:2d + 0x03, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a, + 0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9, + 0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b, + 0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2, + 0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f, + 0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64, + 0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a, + 0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a }; + + int table_id = maat_table_get_id(g_maat_instance, "APP_PAYLOAD"); + ASSERT_GT(table_id, 0); + + int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name)); + EXPECT_EQ(ret, 0); + + ret = maat_scan_string(g_maat_instance, table_id, 0, (char*)udp_payload_not_hit, sizeof(udp_payload_not_hit), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(g_maat_instance, table_id, 0, (char*)udp_payload_hit, sizeof(udp_payload_hit), + results, 5, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(results[0], 148); + + maat_state_free(&state); +} + +#if 0 TEST_F(MaatStringScan, hit_two_expr) { int table_id = MaatStringScan::table_id; char data[128] = "should hit aaa bbb"; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, + sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 2); EXPECT_EQ(results[0], 139); @@ -70,7 +373,8 @@ TEST_F(MaatStringScan, hit_three_expr) { int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, + sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 3); EXPECT_EQ(results[0], 139); @@ -95,9 +399,9 @@ protected: }; int MaatIPScan::table_id; -TEST_F(MaatIPScan, hit_ip_and_port) { +TEST_F(MaatIPScan, hit_ip) { int table_id = MaatIPScan::table_id; - char ip_str[32] = "192.168.58.19"; + char ip_str[32] = "192.168.58.20"; uint32_t sip; int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); @@ -105,20 +409,22 @@ TEST_F(MaatIPScan, hit_ip_and_port) { int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), + &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 7); maat_state_free(&state); state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), + &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } -#if 0 + TEST_F(MaatStringScan, hit_ip_and_port_range) { int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "192.168.50.24"; @@ -326,7 +632,6 @@ TEST(maat_scan_ip, dynamic_config) { } #endif - int count_line_num_cb(const char *table_name, const char *line, void *u_para) { (*((unsigned int *)u_para))++; @@ -390,7 +695,11 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db, return -1; } - if ((access(json_iris_path, F_OK)) < 0) { + if (0 == access(json_iris_path, F_OK)) { + system_cmd_rmdir(json_iris_path); + } + + if (access(json_iris_path, F_OK) < 0) { char tmp_iris_path[128] = {0}; char *json_buff = NULL; size_t json_buff_sz = 0; @@ -458,7 +767,7 @@ int main(int argc, char ** argv) log_handle_destroy(logger); return -1; } - + struct maat_options *opts = maat_options_new(); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, logger); diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 2b920da..f178c14 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -9,14 +9,23 @@ const char *table_info_path = "./table_info.conf"; const char *json_filename = "maat_json.json"; +struct log_handle *g_logger = NULL; -#if 0 TEST(json_mode, maat_scan_string) { + char tmp_iris_path[128] = {0}; char json_iris_path[128] = {0}; snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); - if ((access(json_iris_path, F_OK)) == 0) { - system_cmd_rmdir(json_iris_path); + if (access(json_iris_path, F_OK) < 0) { + char *json_buff = NULL; + size_t json_buff_sz = 0; + + int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz); + EXPECT_NE(ret, -1); + + ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), + NULL, NULL, g_logger); + EXPECT_NE(ret, -1); } struct maat_options *opts = maat_options_new(); @@ -29,14 +38,15 @@ TEST(json_mode, maat_scan_string) { int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE"); - char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me."; + char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), + results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 132); + EXPECT_EQ(results[0], 182); maat_options_free(opts); maat_state_free(&state); @@ -48,14 +58,15 @@ TEST(iris_mode, maat_scan_string) { char json_iris_path[128] = {0}; snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); - if ((access(json_iris_path, F_OK)) < 0) { + if (access(json_iris_path, F_OK) < 0) { char *json_buff = NULL; size_t json_buff_sz = 0; int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz); EXPECT_NE(ret, -1); - ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL); + ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), + NULL, NULL, g_logger); EXPECT_NE(ret, -1); } @@ -72,20 +83,21 @@ TEST(iris_mode, maat_scan_string) { int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE"); - char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me."; + char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), + results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 132); + EXPECT_EQ(results[0], 182); maat_options_free(opts); maat_state_free(&state); maat_free(maat_instance); } -#endif + int count_line_num_cb(const char *table_name, const char *line, void *u_para) { (*((unsigned int *)u_para))++; @@ -132,25 +144,21 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para) return 0; } - -#if 1 TEST(redis_mode, maat_scan_string) { char json_iris_path[128] = {0}; char redis_ip[64] = "127.0.0.1"; int redis_port = 6379; int redis_db = 0; - struct log_handle *logger = log_handle_create("./tmp.log", 0); - snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); - redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger); + redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, g_logger); EXPECT_NE(c, nullptr); redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb"); EXPECT_NE(reply, nullptr); - if ((access(json_iris_path, F_OK)) < 0) { + if (access(json_iris_path, F_OK) < 0) { char tmp_iris_path[128] = {0}; char *json_buff = NULL; size_t json_buff_sz = 0; @@ -158,25 +166,27 @@ TEST(redis_mode, maat_scan_string) { int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); EXPECT_NE(ret, -1); - ret = json2iris(json_buff, json_filename, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger); + ret = json2iris(json_buff, json_filename, c, tmp_iris_path, sizeof(tmp_iris_path), + NULL, NULL, g_logger); EXPECT_NE(ret, -1); } size_t total_line_cnt = 0; char tmp_iris_full_idx_path[128] = {0}; snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path); - config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger); + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, + &total_line_cnt, g_logger); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); long long server_time = maat_cmd_redis_server_time_s(c); EXPECT_NE(server_time, -1); absolute_expire_time = server_time + 300; - config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger); - + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, + s_rule, g_logger); int success_cnt = 0; do { - success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger); + success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, g_logger); } while (success_cnt < 0); EXPECT_EQ(success_cnt, (int)total_line_cnt); @@ -189,30 +199,40 @@ TEST(redis_mode, maat_scan_string) { struct maat_options *opts = maat_options_new(); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); - maat_options_set_logger(opts, logger); + maat_options_set_logger(opts, g_logger); struct maat *maat_instance = maat_new(opts, table_info_path); int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE"); - char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me."; + char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), + results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 132); + EXPECT_EQ(results[0], 182); maat_options_free(opts); maat_state_free(&state); - log_handle_destroy(maat_instance->logger); maat_free(maat_instance); } -#endif + int main(int argc, char ** argv) { int ret=0; ::testing::InitGoogleTest(&argc, argv); + g_logger = log_handle_create("./input_mode_gtest.log", 0); + + char json_iris_path[128] = {0}; + snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); + if ((access(json_iris_path, F_OK)) == 0) { + system_cmd_rmdir(json_iris_path); + } + ret=RUN_ALL_TESTS(); + + log_handle_destroy(g_logger); return ret; } diff --git a/test/maat_json.json b/test/maat_json.json index 94971d4..f4de4af 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -3,183 +3,179 @@ "group2compile_table": "GROUP2COMPILE", "group2group_table": "GROUP2GROUP", "groups": [ - { - "group_name": "ASN1234", - "regions": [ - { - "table_name": "AS_NUMBER", - "table_type": "expr", - "table_content": { - "keywords": "AS1234", - "expr_type": "none", - "match_method": "exact", - "format": "uncase plain" - } - } - ] - }, - { - "group_name": "ASN2345", - "regions": [ - { - "table_name": "AS_NUMBER", - "table_type": "expr", - "table_content": { - "keywords": "AS2345", - "expr_type": "none", - "match_method": "exact", - "format": "uncase plain" - } - } - ] - }, - { - "group_name": "financial-department-ip", - "regions": [ - { - "table_name": "IP_PLUS_CONFIG", - "table_type": "ip_plus", - "table_content": { - "addr_type": "ipv4", - "saddr_format": "mask", - "src_ip1": "192.168.40.88", - "src_ip2": "255.255.255.255", - "sport_format": "mask", - "src_port1": "0", - "src_port2": "65535", - "daddr_format": "mask", - "dst_ip1": "0.0.0.0", - "dst_ip2": "255.255.255.255", - "dport_format": "mask", - "dst_port1": "0", - "dst_port2": "65535", - "protocol": 6, - "direction": "double" - } - } - ] + { + "group_name": "ASN1234", + "regions": [ + { + "table_name": "AS_NUMBER", + "table_type": "expr", + "table_content": { + "keywords": "AS1234", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "ASN2345", + "regions": [ + { + "table_name": "AS_NUMBER", + "table_type": "expr", + "table_content": { + "keywords": "AS2345", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "financial-department-ip", + "regions": [ + { + "table_name": "IP_CONFIG", + "table_type": "ip_plus", + "table_content": { + "addr_type": "ipv4", + "src_ip": "192.168.40.88", + "mask_src_ip": "255.255.255.255", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0.0.0.0", + "mask_dst_ip": "255.255.255.255", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + } + ] }, - { - "group_name": "Country-Sparta-IP", - "regions": [ - { - "table_name": "GeoLocation", - "table_type": "expr", - "table_content": { - "keywords": "Greece.Sparta", - "expr_type": "none", - "match_method": "exact", - "format": "uncase plain" - } - } - ] - }, - { - "group_name": "IPv4-composition-source-only", - "regions": [ - { - "table_type": "ip_plus", - "table_name": "IP_PLUS_CONFIG", - "table_content": { - "addr_type": "ipv4", - "saddr_format": "range", - "src_ip1": "192.168.50.24", - "src_ip2": "192.168.50.24", - "sport_format": "range", - "src_port1": "1", - "src_port2": "40000", - "daddr_format": "mask", - "dst_ip1": "0.0.0.0", - "dst_ip2": "255.255.255.0", - "dport_format": "range", - "dst_port1": "0", - "dst_port2": "65535", - "protocol": 6, - "direction": "double" - } - } - ] + { + "group_name": "Country-Sparta-IP", + "regions": [ + { + "table_name": "GeoLocation", + "table_type": "expr", + "table_content": { + "keywords": "Greece.Sparta", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "IPv4-composition-source-only", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "192.168.50.24", + "src_ip2": "192.168.50.24", + "sport_format": "range", + "src_port1": "1", + "src_port2": "40000", + "daddr_format": "mask", + "dst_ip1": "0.0.0.0", + "dst_ip2": "255.255.255.0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ] }, - { - "group_name": "FQDN_OBJ1", - "regions": [ - { - "table_name": "KEYWORDS_TABLE", - "table_type": "expr", - "table_content": { - "keywords": "sports.example.com", - "expr_type": "none", - "match_method": "exact", - "format": "uncase plain" - } - } - ] - }, - { - "group_name": "FQDN_CAT1", - "regions": [ - { - "table_name": "INTERGER_PLUS", - "table_type": "interval_plus", - "table_content": { - "district": "fqdn_cat_id", - "low_boundary": 1724, - "up_boundary": 1724 - } - } - ] - }, - { - "group_name": "IPv4-composition-NOT-client-ip", - "regions": [ - { - "table_type": "ip_plus", - "table_name": "IP_PLUS_CONFIG", - "table_content": { - "addr_type": "ipv4", - "saddr_format": "range", - "src_ip1": "192.168.58.19", - "src_ip2": "192.168.58.19", - "sport_format": "range", - "src_port1": "20000", - "src_port2": "20000", - "daddr_format": "mask", - "dst_ip1": "0.0.0.0", - "dst_ip2": "255.255.255.0", - "dport_format": "range", - "dst_port1": "0", - "dst_port2": "65535", - "protocol": 6, - "direction": "double" - } - } - ] + { + "group_name": "FQDN_OBJ1", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "sports.example.com", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "FQDN_CAT1", + "regions": [ + { + "table_name": "INTERGER_PLUS", + "table_type": "interval_plus", + "table_content": { + "district": "fqdn_cat_id", + "low_boundary": 1724, + "up_boundary": 1724 + } + } + ] + }, + { + "group_name": "IPv4-composition-NOT-client-ip", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "192.168.58.19", + "src_ip2": "192.168.58.19", + "sport_format": "range", + "src_port1": "20000", + "src_port2": "20000", + "daddr_format": "mask", + "dst_ip1": "0.0.0.0", + "dst_ip2": "255.255.255.0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ] }, - { - "group_name": "IPv4-composition-NOT-server-ip", - "regions": [ - { - "table_type": "ip_plus", - "table_name": "IP_PLUS_CONFIG", - "table_content": { - "addr_type": "ipv4", - "saddr_format": "range", - "src_ip1": "10.0.1.20", - "src_ip2": "10.0.1.25", - "sport_format": "range", - "src_port1": "1", - "src_port2": "443", - "daddr_format": "mask", - "dst_ip1": "0.0.0.0", - "dst_ip2": "255.255.255.0", - "dport_format": "range", - "dst_port1": "0", - "dst_port2": "65535", - "protocol": 6, - "direction": "double" - } - } - ] + { + "group_name": "IPv4-composition-NOT-server-ip", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "10.0.1.20", + "src_ip2": "10.0.1.25", + "sport_format": "range", + "src_port1": "1", + "src_port2": "443", + "daddr_format": "mask", + "dst_ip1": "0.0.0.0", + "dst_ip2": "255.255.255.0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ] } ], "rules": [ @@ -196,43 +192,35 @@ "group_name": "123_IP_group", "regions": [ { - "table_name": "IP_PLUS_CONFIG", + "table_name": "IP_CONFIG", "table_type": "ip_plus", "table_content": { "addr_type": "ipv4", - "saddr_format": "mask", - "src_ip1": "10.0.6.201", - "src_ip2": "255.255.0.0", - "sport_format": "mask", - "src_port1": "0", - "src_port2": "65535", - "daddr_format":"mask", - "dst_ip1": "0.0.0.0", - "dst_ip2": "255.255.255.255", - "dport_format":"mask", - "dst_port1": "0", - "dst_port2": "65535", + "src_ip": "10.0.6.201", + "mask_src_ip": "255.255.0.0", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0.0.0.0", + "mask_dst_ip": "255.255.255.255", + "dst_port": "0", + "mask_dst_port": "65535", "protocol": 6, "direction": "double" } }, { - "table_name": "IP_PLUS_CONFIG", + "table_name": "IP_CONFIG", "table_type": "ip_plus", "table_content": { "addr_type": "ipv6", - "saddr_format":"mask", - "src_ip1": "2001:da8:205:1::101", - "src_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000", - "sport_format":"mask", - "src_port1": "0", - "src_port2": "65535", - "daddr_format":"mask", - "dst_ip1": "0::0", - "dst_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - "dport_format":"mask", - "dst_port1": "0", - "dst_port2": "65535", + "src_ip": "2001:da8:205:1::101", + "mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0::0", + "mask_dst_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "dst_port": "0", + "mask_dst_port": "65535", "protocol": 6, "direction": "double" } @@ -283,31 +271,7 @@ } ] }, - { - "compile_id": 125, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "regions": [ - { - "table_name": "HTTP_URL", - "table_type": "expr", - "table_content": { - "keywords": "action=search\\&query=(.*)", - "expr_type": "regex", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - }, + { "compile_id": 126, "service": 1, @@ -953,7 +917,31 @@ } ] }, - + { + "compile_id": 149, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "StringScan.Regex", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "CORNER_CASE_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "^((?!.*\\binstagram\\b)).*\\.fbcdn\\.net$", + "expr_type": "regex", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, { "compile_id": 150, "service": 0, @@ -2012,6 +2000,32 @@ } ] }, + { + "compile_id": 183, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "StringScan.RegexWithNotContains", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "CORNER_CASE_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "^(?=.*/rain/a/TWF2021042600418000)(?!new.qq.com).*", + "expr_type": "regex", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, { "compile_id": 184, "user_region": "APP_ID=6006740;Liumengyan-Bugreport-20210515", @@ -2027,24 +2041,20 @@ "regions": [ { - "table_name": "IP_PLUS_CONFIG", + "table_name": "IP_CONFIG", "table_type": "ip_plus", - "table_content": { - "addr_type": "ipv6", - "saddr_format":"mask", - "src_ip1": "::", - "src_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - "sport_format":"mask", - "src_port1": "0", - "src_port2": "65535", - "daddr_format":"mask", - "dst_ip1": "2620:100:3000::", - "dst_ip2": "ffff:ffff:ff00:0000:0000:0000:0000:0000", - "dport_format":"mask", - "dst_port1": "0", - "dst_port2": "65535", + "table_content": { "protocol": 0, - "direction": "double" + "addr_type": "ipv6", + "direction": "double", + "src_ip": "::", + "dst_ip": "2620:100:3000::", + "src_port": "0", + "dst_port": "0", + "mask_src_port": "65535", + "mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "mask_dst_port": "65535", + "mask_dst_ip": "ffff:ffff:ff00:0000:0000:0000:0000:0000" } } ] @@ -2286,15 +2296,76 @@ }, { "compile_id": 191, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "StringScan.HexBinCaseSensitive", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "expr", + "table_name": "KEYWORDS_TABLE", + "table_content": { + "keywords": "54455354", + "expr_type": "none", + "format": "hexbin", + "match_method": "sub" + } + } + ], + "group_name": "Untitled" + } + ] + }, + { + "compile_id": 192, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, "user_region": "anything", "is_valid": "yes", "groups": [ { - "group_name": "Untitled", + "regions": [ + { + "table_type": "flag", + "table_name": "FLAG_CONFIG", + "table_content": { + "flag": 1, + "flag_mask": 3 + } + } + ] + } + ] + }, + { + "compile_id": 193, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "flag", + "table_name": "FLAG_CONFIG", + "table_content": { + "flag": 2, + "flag_mask": 3 + } + } + ] + }, + { "regions": [ { "table_name": "HTTP_URL", @@ -2309,6 +2380,29 @@ ] } ] + }, + { + "compile_id": 194, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "flag", + "table_name": "FLAG_CONFIG", + "table_content": { + "flag": 21, + "flag_mask": 31 + } + } + ] + } + ] } ], "plugin_table": [ diff --git a/test/table_info.conf b/test/table_info.conf index 5a5c2cd..768eb06 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -84,7 +84,21 @@ "match_method":5, "is_hexbin":6 } - }, + }, + { + "table_id":5, + "table_name":"IP_CONFIG", + "table_type":"ip_plus", + "valid_column":7, + "custom": { + "item_id":1, + "group_id":2, + "addr_type":3, + "saddr_format":4, + "sip1":5, + "sip2":6 + } + }, { "table_id":6, "table_name":"CONTENT_SIZE", @@ -120,7 +134,7 @@ "keywords":4, "expr_type":5, "match_method":6, - "db_hexbin":7 + "is_hexbin":7 } }, { @@ -372,8 +386,8 @@ "table_type":"fqdn_plugin", "valid_column":5, "custom": { - "row_id":1, - "is_suffix_match":2, + "item_id":1, + "suffix_match_method":2, "fqdn":3 } },