diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index ca557f3..cea3938 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -293,7 +293,6 @@ typedef void Maat_plugin_EX_free_func_t(int table_id, MAAT_PLUGIN_EX_DATA* ad, l typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp); typedef int Maat_plugin_EX_key2index_func_t(const char* key); -//For IP plugin, key2index_func MUST be NULL. int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, @@ -302,6 +301,14 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, long argl, void *argp); //Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to FREE the data. MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key); + + +int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + long argl, void *argp); + struct ip_address { int ip_type; //4: IPv4, 6: IPv6 @@ -312,7 +319,7 @@ struct ip_address }; }; -MAAT_PLUGIN_EX_DATA Maat_IP_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip); +int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data); enum MAAT_RULE_OPT diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index db42f65..9ebce39 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index cd031a2..b25867a 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1331,38 +1331,24 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int idx=-1; + int ret=0; + struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); - if(new_func==NULL || free_func==NULL || dup_func==NULL ) - { - assert(0); - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); - return -1; - } pthread_mutex_lock(&(_feather->background_update_mutex)); + ret=Maat_table_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, key2index_func,argl, argp, _feather->logger); - idx=Maat_table_plugin_new_ex_index(_feather->table_mgr, table_id, - new_func, - free_func, - dup_func, - key2index_func, - argl, argp); - if(idx<0) + if(ret<0) { pthread_mutex_unlock(&(_feather->background_update_mutex)); return -1; } - struct Maat_table_schema *table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); struct Maat_table_runtime* table_rt=NULL; - - if(_feather->scanner!=NULL) { table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - Maat_table_runtime_plugin_new_ex_idx(table_rt, table_desc, _feather->logger); + Maat_table_runtime_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger); } pthread_mutex_unlock(&(_feather->background_update_mutex)); - return 0; } MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) @@ -1380,24 +1366,62 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); return exdata; } -int Maat_IP_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) +int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + long argl, void *argp) + { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_table_schema *table_desc=NULL; + int ret=-1; + struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); + pthread_mutex_lock(&(_feather->background_update_mutex)); + ret=Maat_table_ip_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger); + + if(ret<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + + struct Maat_table_runtime* table_rt=NULL; + if(_feather->scanner!=NULL) + { + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + Maat_table_runtime_ip_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger); + } + pthread_mutex_unlock(&(_feather->background_update_mutex)); + + return 0; +} + +int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_schema *table_schema=NULL; struct Maat_table_runtime *table_rt=NULL; int n_get=0; if(_feather->scanner==NULL) { return 0; } - table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN) { return -1; } - - n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_desc, ip, ex_data_array, n_ex_data); + struct ip_data ip_data=*(const struct ip_data*)ip; + if(ip_data.type==IPv4) + { + ip_data.ipv4=ntohl(ip_data.ipv4); + } + else + { + ipv6_ntoh(ip_data.ipv6); + } + n_get=Maat_table_runtime_ip_plugin_get_N_ex_data(table_rt, table_schema, &ip_data, ex_data_array, n_ex_data); return n_get; } diff --git a/src/entry/Maat_ex_data.cpp b/src/entry/Maat_ex_data.cpp new file mode 100644 index 0000000..25f6bee --- /dev/null +++ b/src/entry/Maat_ex_data.cpp @@ -0,0 +1,193 @@ +#include "Maat_ex_data.h" +#include "Maat_table.h" +#include "Maat_utils.h" + +#include +#include + +void EX_data_container_free(void *data) +{ + struct EX_data_container* wrap_data=(struct EX_data_container*)data; + const struct EX_data_schema* ex_schema=wrap_data->rt->ex_schema; + ex_schema->free_func(wrap_data->rt->table_id, &(wrap_data->ex_data), ex_schema->argl, ex_schema->argp); + if(wrap_data->user_data && wrap_data->rt->user_data_free) + { + wrap_data->rt->user_data_free(wrap_data->user_data); + } + wrap_data->user_data=NULL; + wrap_data->rt=NULL; + free(wrap_data); + return; +} +static MESA_htable_handle EX_data_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) +{ + MESA_htable_handle key2ex_hash=NULL; + unsigned int slot_size=1; + while(estimate_size!=0) + { + estimate_size=estimate_size>>1; + slot_size*=2; + } + if(slot_size==1) + { + slot_size=4096; + } + + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=8; + hargs.hash_slot_size = slot_size; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = NULL; //Not supported yet. + hargs.recursive = 1; + hargs.data_free = EX_data_container_free; + hargs.data_expire_with_condition = NULL; + key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(key2ex_hash, 0); + return key2ex_hash; +} + +struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)) +{ + struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1); + p->key2ex_hash=EX_data_hash_new(estimate_size, key2index); + p->cache_rows=dynamic_array_create(4, 1024); + p->table_id=table_id; + p->user_data_free=user_data_free; + return p; +}; +void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema) +{ + p->ex_schema=schema; +} +void EX_data_rt_free(struct EX_data_rt* p) +{ + if(p->cache_rows) + { + dynamic_array_destroy(p->cache_rows, free); + p->cache_rows=NULL; + } + MESA_htable_destroy(p->key2ex_hash, NULL); + return; +} +void EX_data_rt_cache_row(struct EX_data_rt* p, const char* row) +{ + size_t len=strlen(row)+1; + char* row_copy=ALLOC(char, len); + memcpy(row_copy, row, len); + p->cache_size+=len; + dynamic_array_write(p->cache_rows, p->cache_row_num, row_copy); + p->cache_row_num++; + return; +} +const char* EX_data_rt_get_cached_row(struct EX_data_rt* p, int i) +{ + const char* row=NULL; + row=(const char*)dynamic_array_read(p->cache_rows, i); + return row; +} +void EX_data_rt_clear_row_cache(struct EX_data_rt* p) +{ + dynamic_array_destroy(p->cache_rows, free); + p->cache_rows=NULL; + p->cache_row_num=0; + p->cache_size=0; +} +int EX_data_rt_get_row_num(struct EX_data_rt* p) +{ + return p->cache_row_num; +} +struct EX_data_container* EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt, + const char* row, const char* key, size_t key_len, + void* user_data, void* logger) +{ + + MAAT_RULE_EX_DATA ex_data=NULL; + int ret=0; + const struct EX_data_schema* ex_schema=ex_rt->ex_schema; + struct EX_data_container* ex_container=ALLOC(struct EX_data_container, 1); + ex_schema->new_func(ex_rt->table_id, key, row, &ex_data, + ex_schema->argl, ex_schema->argp); + ex_container->ex_data=ex_data; + ex_container->rt=ex_rt; + ex_container->user_data=user_data; + ret=MESA_htable_add(ex_rt->key2ex_hash, (unsigned char*)key, key_len, ex_container); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "EX data add error: duplicated key %.*s of %s", + key_len, key, row); + EX_data_container_free(ex_container); + return NULL; + } + return ex_container; +} +int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len, + void *logger) +{ + int ret=0; + ret=MESA_htable_del(ex_rt->key2ex_hash, (const unsigned char*)key, key_len, NULL); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "EX data del error: no such key %.*s of %s", + key_len, key, row); + return -1; + } + return 0; +} + +MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len) +{ + struct EX_data_container* container=NULL; + MAAT_RULE_EX_DATA ex_data=NULL; + + if(!ex_rt->ex_schema) + { + assert(0); + return NULL; + } + container=(struct EX_data_container*)MESA_htable_search(ex_rt->key2ex_hash, + (const unsigned char*)key, strlen(key)); + if(container!=NULL) + { + ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(container->ex_data), + ex_rt->ex_schema->argl, ex_rt->ex_schema->argp); + } + return ex_data; +} +MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container) +{ + MAAT_RULE_EX_DATA ex_data=NULL; + ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(container->ex_data), + ex_rt->ex_schema->argl, ex_rt->ex_schema->argp); + return ex_data; +} +struct key2EX_hash_walker +{ + EX_data_container_q* listed; + size_t count; +}; + +void walk_key2EX_hash(const uchar * key, uint size, void * data, void * user) +{ + struct key2EX_hash_walker *walker=(struct key2EX_hash_walker*)user; + struct EX_data_container* ex_container=(struct EX_data_container*)data; + TAILQ_INSERT_TAIL(walker->listed, ex_container, entries); + walker->count++; + return; +} +size_t EX_data_rt_list_all(struct EX_data_rt* ex_rt, EX_data_container_q* listed) +{ + size_t ex_data_cnt; + struct key2EX_hash_walker walker={listed, 0}; + TAILQ_INIT(listed); + MESA_htable_iterate(ex_rt->key2ex_hash, walk_key2EX_hash, &walker); + ex_data_cnt=(size_t)MESA_htable_get_elem_num(ex_rt->key2ex_hash); + assert(walker.count==ex_data_cnt); + return ex_data_cnt; +} + diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 34c2eed..902fab7 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -3000,44 +3000,41 @@ void garbage_bury(MESA_lqueue_head garbage_q, int timeout, void *logger) q_cnt,bury_cnt); } } -void update_plugin_table(struct Maat_table_schema* table, const char* table_line, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) +void update_plugin_table(struct Maat_table_schema* table_schema, const char* row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) { - int i=0, ret=1, matched_tag=1; - unsigned int len=strlen(table_line)+1; - struct plugin_table_schema* plugin_desc=&(table->plugin); - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - char *p=NULL; + int ret=1, matched_tag=1; + struct plugin_table_schema* plugin_desc=&(table_schema->plugin); + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id); char* copy=NULL; - size_t is_valid_offset=0, valid_len=0; size_t accept_tag_offset=0, accept_tag_len=0; if(plugin_desc->rule_tag_column>0&&n_tags>0) { - ret=Maat_helper_read_column(table_line, plugin_desc->rule_tag_column, &accept_tag_offset, &accept_tag_len); + ret=Maat_helper_read_column(row, plugin_desc->rule_tag_column, &accept_tag_offset, &accept_tag_len); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, could not locate tag in column %d of plugin table %s:%s", + "update error, could not locate tag in column %d of plugin table_schema %s:%s", plugin_desc->rule_tag_column, - table->table_name[table->updating_name], - table_line); - table->udpate_err_cnt++; + table_schema->table_name[table_schema->updating_name], + row); + table_schema->udpate_err_cnt++; return; } if(accept_tag_len>2) { copy=ALLOC(char, accept_tag_len+1); - memcpy(copy, table_line+accept_tag_offset, accept_tag_len); + memcpy(copy, row+accept_tag_offset, accept_tag_len); matched_tag=compare_accept_tag(copy, tags, n_tags); if(matched_tag<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error,invalid tag format of plugin table %s:%s" - ,table->table_name[table->updating_name],table_line); - table->udpate_err_cnt++; + "update error,invalid tag format of plugin table_schema %s:%s" + ,table_schema->table_name[table_schema->updating_name],row); + table_schema->udpate_err_cnt++; } if(matched_tag==0) { - table->unmatch_tag_cnt++; + table_schema->unmatch_tag_cnt++; } free(copy); copy=NULL; @@ -3047,79 +3044,43 @@ void update_plugin_table(struct Maat_table_schema* table, const char* table_line return; } } - - table_rt->plugin.acc_line_num++; - if(plugin_desc->have_exdata || plugin_desc->cb_plug_cnt>0) - { - - if(plugin_desc->have_exdata) - { - ret=Maat_helper_read_column(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len); - //thread safe is protected by background_update_mutex - if(atoi(table_line+is_valid_offset)==1) - { - plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger); - } - else - { - plugin_EX_data_free(table_line, plugin_desc->key_column, table_rt->plugin.key2ex_hash, logger); - } - } - if(plugin_desc->cb_plug_cnt>0) - { - for(i=0;icb_plug_cnt;i++) - { - plugin_desc->cb_plug[i].update(table->table_id, table_line, plugin_desc->cb_plug[i].u_para); - } - } - - } - else - { - p=ALLOC(char, len); - memcpy(p,table_line,len); - table_rt->plugin.cache_size+=len; - dynamic_array_write(table_rt->plugin.cache_lines, table_rt->plugin.cache_line_num, p); - table_rt->plugin.cache_line_num++; - } + Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger); } -void update_ip_plugin_table(struct Maat_table_schema* table,const char* table_line,Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) +void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char* table_row, Maat_scanner* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int ret=1, matched_tag=1; - struct ip_plugin_table_schema* ip_plugin_schema=&(table->ip_plugin); - struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); - struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin); + struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_schema->table_id); char* copy=NULL; - size_t is_valid_offset=0, valid_len=0; size_t accept_tag_offset=0, accept_tag_len=0; if(ip_plugin_schema->rule_tag_column>0&&n_tags>0) { - ret=Maat_helper_read_column(table_line, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len); + ret=Maat_helper_read_column(table_row, ip_plugin_schema->rule_tag_column, &accept_tag_offset, &accept_tag_len); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "update error, could not locate tag in column %d of plugin table %s:%s", + "update error, could not locate tag in column %d of plugin table_schema %s:%s", ip_plugin_schema->rule_tag_column, - table->table_name[table->updating_name], - table_line); - table->udpate_err_cnt++; + table_schema->table_name[table_schema->updating_name], + table_row); + table_schema->udpate_err_cnt++; return; } if(accept_tag_len>2) { copy=ALLOC(char, accept_tag_len+1); - memcpy(copy, table_line+accept_tag_offset, accept_tag_len); + memcpy(copy, table_row+accept_tag_offset, accept_tag_len); matched_tag=compare_accept_tag(copy, tags, n_tags); if(matched_tag<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "update error, invalid tag format of ip_plugin table %s:%s", - table->table_name[table->updating_name], table_line); - table->udpate_err_cnt++; + "update error, invalid tag format of ip_plugin table_schema %s:%s", + table_schema->table_name[table_schema->updating_name], table_row); + table_schema->udpate_err_cnt++; } if(matched_tag==0) { - table->unmatch_tag_cnt++; + table_schema->unmatch_tag_cnt++; } free(copy); copy=NULL; @@ -3129,25 +3090,8 @@ void update_ip_plugin_table(struct Maat_table_schema* table,const char* table_li return; } } - ret=Maat_helper_read_column(table_line, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); - //thread safe is protected by background_update_mutex - if(atoi(table_line+is_valid_offset)==1) - { - ret=plugin_EX_data_new(table, table_line, ip_plugin_rt->rowid2ex_hash, logger); - if(ret==0) - { - ip_plugin_rt->row_num++; - } - } - else - { - ret=plugin_EX_data_free(table_line, ip_plugin_schema->row_id_column, ip_plugin_rt->rowid2ex_hash, logger); - if(ret==0) - { - ip_plugin_rt->row_num--; - } - - } + Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger); + return; } void vector_print(igraph_vector_t *v) { @@ -3293,7 +3237,7 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, } break; case TABLE_TYPE_IP_PLUGIN: - Maat_table_runtime_rebuild_ip_matcher(table_rt); + Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt); old_ip_matcher=Maat_table_runtime_dettach_old_ip_matcher(table_rt); garbage_bagging(GARBAGE_IP_MATCHER, old_ip_matcher, garbage_q); break; diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index f26d1d9..c283618 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -219,7 +219,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) switch(p_table->table_type) { case TABLE_TYPE_PLUGIN: - plugin_cache_num+=table_rt->plugin.cache_line_num; + plugin_cache_num+=table_rt->plugin.ex_data_rt->cache_row_num; plugin_acc_num+=table_rt->plugin.acc_line_num; break; case TABLE_TYPE_GROUP: diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index 3771bef..e73ed79 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -285,11 +285,11 @@ int read_ip_plugin_table_schema(const char* line, struct Maat_table_schema* p) free(copy_line); if(read_cnt<5) { - return 0; + return -1; } else { - return -1; + return 0; } error_out: free(copy_line); @@ -869,32 +869,89 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c return idx; } -int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, +void Maat_table_EX_data_schema_set(struct EX_data_schema* ex_schema, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, Maat_plugin_EX_dup_func_t* dup_func, Maat_plugin_EX_key2index_func_t* key2index_func, long argl, void *argp) - { - struct Maat_table_schema *table_desc=NULL;; - table_desc=Maat_table_get_scan_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); - struct plugin_table_schema* plugin_desc=&(table_desc->plugin); - - if(plugin_desc->have_exdata - || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) + ex_schema->new_func=new_func; + ex_schema->free_func=free_func; + ex_schema->dup_func=dup_func; + ex_schema->key2index_func=key2index_func;//Set but not used. + ex_schema->argl=argl; + ex_schema->argp=argp; + return; +} +int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger) +{ + if(new_func==NULL || free_func==NULL || dup_func==NULL ) { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); return -1; } - plugin_desc->ex_desc.new_func=new_func; - plugin_desc->ex_desc.free_func=free_func; - plugin_desc->ex_desc.dup_func=dup_func; - plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. - plugin_desc->ex_desc.argl=argl; - plugin_desc->ex_desc.argp=argp; - plugin_desc->have_exdata=1; + if(table_schema->table_type!=TABLE_TYPE_PLUGIN) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, Regist target is not a plugin table.", __FUNCTION__); + return -1; + } + if(table_schema->plugin.have_exdata) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__); + return -1; + } + if(table_schema->plugin.key_column==0 || table_schema->plugin.valid_flag_column==0) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, not enough schema information.", __FUNCTION__); + return -1; + } + Maat_table_EX_data_schema_set(&table_schema->plugin.ex_schema, + new_func, free_func, dup_func, key2index_func, argl, argp); + table_schema->plugin.have_exdata=1; return 0; } +int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger) +{ + if(new_func==NULL || free_func==NULL || dup_func==NULL ) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); + return -1; + } + if(table_schema->table_type!=TABLE_TYPE_IP_PLUGIN) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a ip_plugin table.", __FUNCTION__); + return -1; + } + if(table_schema->ip_plugin.have_exdata) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__); + return -1; + } + Maat_table_EX_data_schema_set(&table_schema->ip_plugin.ex_schema, + new_func, free_func, dup_func, key2index_func, argl, argp); + table_schema->ip_plugin.have_exdata=1; + return 0; +} void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type) { table_mgr->active_plugin_table_num=0; diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp index ede0861..accdc83 100644 --- a/src/entry/Maat_table_runtime.cpp +++ b/src/entry/Maat_table_runtime.cpp @@ -7,96 +7,52 @@ #include #include -struct wrap_plugin_EX_data -{ - MAAT_RULE_EX_DATA exdata; - int table_id; - const struct plugin_table_ex_data_schema* ex_desc; - struct ip_rule range_rule;//for ip_plugin - TAILQ_ENTRY(wrap_plugin_EX_data) entries; -}; -void wrap_plugin_EX_data_free(void *data) -{ - struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; - wrap_data->ex_desc->free_func(wrap_data->table_id, &(wrap_data->exdata), wrap_data->ex_desc->argl, wrap_data->ex_desc->argp); - wrap_data->ex_desc=NULL; - wrap_data->table_id=-1; - free(wrap_data); - return; -} -MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) -{ - MESA_htable_handle key2ex_hash=NULL; - unsigned int slot_size=1; - while(estimate_size!=0) - { - estimate_size=estimate_size>>1; - slot_size*=2; - } - if(slot_size==1) - { - slot_size=4096; - } - MESA_htable_create_args_t hargs; - memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=8; - hargs.hash_slot_size = slot_size; - hargs.max_elem_num = 0; - hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; - hargs.expire_time = 0; - hargs.key_comp = NULL; - hargs.key2index = NULL; //Not supported yet. - hargs.recursive = 1; - hargs.data_free = wrap_plugin_EX_data_free; - hargs.data_expire_with_condition = NULL; - key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(key2ex_hash, 0); - return key2ex_hash; -} - -int plugin_EX_data_free(const char* line, int key_column, +int plugin_EX_data_free(const char* row, int key_column, MESA_htable_handle key2ex_hash, void *logger) { size_t key_offset=0, key_len=0; int ret=0; - ret=get_column_pos(line, key_column, &key_offset, &key_len); + ret=get_column_pos(row, key_column, &key_offset, &key_len); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "plugin/ip_plugin EX data del error: cannot find column %d of %s", - key_column, line); + key_column, row); return -1; } - ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); + ret=MESA_htable_del(key2ex_hash, (const unsigned char*)row+key_offset, key_len, NULL); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "plugin/ip_plugin EX data del error: no such key %.*s of %s", - key_len, line+key_offset, line); + key_len, row+key_offset, row); return -1; } return 0; } -int ip_plugin_line_read(const struct ip_plugin_table_schema* schema, const char* line, struct ip_rule* range_rule) +struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row) { + struct ip_rule* range_rule=ALLOC(struct ip_rule, 1); + int ret[4]={0}; size_t column_offset=0, column_len=0; char start_ip[128]={0}, end_ip[128]={0}; - ret[0]=get_column_pos(line, schema->row_id_column, &column_offset, &column_len); - range_rule->rule_id=atoi(line+column_offset); + ret[0]=get_column_pos(row, schema->row_id_column, &column_offset, &column_len); + range_rule->rule_id=atoi(row+column_offset); - ret[1]=get_column_pos(line, schema->ip_type_column, &column_offset, &column_len); - int ip_type=atoi(line+column_offset); + ret[1]=get_column_pos(row, schema->ip_type_column, &column_offset, &column_len); + int ip_type=atoi(row+column_offset); - ret[2]=get_column_pos(line, schema->start_ip_column, &column_offset, &column_len); - strncpy(start_ip, line+column_offset, MIN(column_len, sizeof(start_ip))); + ret[2]=get_column_pos(row, schema->start_ip_column, &column_offset, &column_len); + strncpy(start_ip, row+column_offset, MIN(column_len, sizeof(start_ip))); - ret[3]=get_column_pos(line, schema->end_ip_column, &column_offset, &column_len); - strncpy(end_ip, line+column_offset, MIN(column_len, sizeof(end_ip))); + ret[3]=get_column_pos(row, schema->end_ip_column, &column_offset, &column_len); + strncpy(end_ip, row+column_offset, MIN(column_len, sizeof(end_ip))); if(ret[0]<0||ret[1]<0||ret[2]<0||ret[3]<0) { - return -1; + free(range_rule); + return NULL; } if(ip_type==4) @@ -111,76 +67,17 @@ int ip_plugin_line_read(const struct ip_plugin_table_schema* schema, const char* } else { - return -1; + free(range_rule); + return NULL; } if(ret[0]<0) { - return -1; + free(range_rule); + return NULL; } - range_rule->rule_id=(unsigned int)atoi(line+column_offset); range_rule->user_tag=NULL; - return 0; + return range_rule; } -int plugin_EX_data_new(const struct Maat_table_schema* table_schema, - const char* line, - MESA_htable_handle key2ex_hash, void *logger) -{ - char* key=NULL; - size_t key_offset=0, key_len=0; - MAAT_RULE_EX_DATA exdata=NULL; - const struct plugin_table_ex_data_schema* ex_desc=NULL; - int key_column=-1; - struct wrap_plugin_EX_data* wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); - switch(table_schema->table_type) - { - case TABLE_TYPE_PLUGIN: - ex_desc=&(table_schema->plugin.ex_desc); - key_column=table_schema->plugin.key_column; - break; - case TABLE_TYPE_IP_PLUGIN: - ex_desc=&(table_schema->ip_plugin.ex_desc); - key_column=table_schema->ip_plugin.row_id_column; - ip_plugin_line_read(&table_schema->ip_plugin, line, &(wrap_data->range_rule)); - wrap_data->range_rule.user_tag=wrap_data; - break; - default: - assert(0); - break; - } - - int ret=0; - ret=get_column_pos(line, key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "plugin/ip_plugin EX data add error: cannot find column %d of %s", - key_column, line); - goto error_out; - } - key=ALLOC(char, key_len+1); - memcpy(key, line+key_offset, key_len); - ex_desc->new_func(table_schema->table_id, key, line, &exdata, - ex_desc->argl, ex_desc->argp); - wrap_data->exdata=exdata; - wrap_data->ex_desc=ex_desc; - wrap_data->table_id=table_schema->table_id; - ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); - free(key); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "plugin/ip_plugin EX data add error: duplicated key %.*s of %s", - key_len, line+key_offset, line); - wrap_plugin_EX_data_free(wrap_data); - return -1; - } - return 0; -error_out: - free(wrap_data); - free(key); - return -1; -} - struct Maat_table_runtime_manager { struct Maat_table_runtime** table_rt; @@ -217,28 +114,36 @@ static void destroy_digest_rule(GIE_digest_t*rule) return; } -static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_desc, int max_thread_num) +static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num) { struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); - table_rt->table_type=table_desc->table_type; - switch(table_desc->table_type) + table_rt->table_type=table_schema->table_type; + switch(table_schema->table_type) { case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: table_rt->similar.update_q=MESA_lqueue_create(0,0); break; case TABLE_TYPE_PLUGIN: - table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); - if(table_desc->plugin.have_exdata) + table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, + table_schema->plugin.estimate_size, + table_schema->plugin.ex_schema.key2index_func, + NULL); + if(table_schema->plugin.have_exdata) { - table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, - table_desc->plugin.ex_desc.key2index_func); + EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema); } break; case TABLE_TYPE_IP_PLUGIN: - table_rt->ip_plugin.rowid2ex_hash=wrap_plugin_EX_hash_new(table_desc->ip_plugin.estimate_size, - NULL); + table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, + table_schema->plugin.estimate_size, + table_schema->plugin.ex_schema.key2index_func, + free); + if(table_schema->ip_plugin.have_exdata) + { + EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema); + } break; default: break; @@ -283,18 +188,14 @@ static void table_runtime_free(struct Maat_table_runtime* p) MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); } break; - case TABLE_TYPE_PLUGIN: - dynamic_array_destroy(p->plugin.cache_lines, free); - p->plugin.cache_lines=NULL; - if(p->plugin.key2ex_hash!=NULL) - { - MESA_htable_destroy(p->plugin.key2ex_hash, NULL); - } - break; + case TABLE_TYPE_IP_PLUGIN: - MESA_htable_destroy(p->ip_plugin.rowid2ex_hash, NULL); ip_matcher_free(p->ip_plugin.ip_matcher); - p->ip_plugin.row_num=0; + EX_data_rt_free(p->ip_plugin.ex_data_rt); + assert(p->ip_plugin.old_ip_matcher==NULL); + break; + case TABLE_TYPE_PLUGIN: + EX_data_rt_free(p->plugin.ex_data_rt); break; default: break; @@ -349,81 +250,83 @@ struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_mana long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt) { struct plugin_runtime* plugin_rt=&(table_rt->plugin); - return plugin_rt->cache_line_num; + return plugin_rt->ex_data_rt->cache_row_num; } const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line) { const char* line=NULL; struct plugin_runtime* plugin_rt=&(table_rt->plugin); - line=(const char*)dynamic_array_read(plugin_rt->cache_lines, Nth_line); + line=(const char*)dynamic_array_read(plugin_rt->ex_data_rt->cache_rows, Nth_line); return line; } -MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_schema* plugin_table, - struct dynamic_array_t* lines, size_t line_cnt, void* logger) +int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) { - MESA_htable_handle key2ex_hash=NULL; - size_t i=0; - const char* line=NULL; - const struct plugin_table_schema* plugin_desc= &(plugin_table->plugin); + int i=0; + const char* row=NULL; + EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema); - key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); - - for(i=0; i< line_cnt; i++) + for(i=0; iplugin.ex_data_rt); i++) { - line=(const char*)dynamic_array_read(lines, i); - plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); + row=EX_data_rt_get_cached_row(table_rt->plugin.ex_data_rt, i); + Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger); } - return key2ex_hash; -} -int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger) -{ - assert(table_rt->plugin.key2ex_hash==NULL); - if(table_rt->plugin.key2ex_hash) - { - return -1; - } - table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, - table_rt->plugin.cache_line_num, logger); return 0; } -MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key) +MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* key) { - struct plugin_table_schema* plugin_desc=NULL; - struct wrap_plugin_EX_data* wrap_data=NULL; - MAAT_RULE_EX_DATA exdata=NULL; - - plugin_desc=&(table_desc->plugin); - if(!plugin_desc->have_exdata) + MAAT_RULE_EX_DATA ex_data=NULL; + if(!table_schema->plugin.have_exdata) { assert(0); return NULL; } - wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, - (const unsigned char*)key, strlen(key)); - if(wrap_data!=NULL) - { - plugin_desc->ex_desc.dup_func(table_desc->table_id, &(exdata), &(wrap_data->exdata), - plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); - } - return exdata; + ex_data=EX_data_rt_get_EX_data_by_key(table_rt->plugin.ex_data_rt, key, strlen(key)); + return ex_data; } -int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) +void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) { - struct ip_plugin_table_schema* ip_plugin_desc=NULL; - struct scan_result results[size]; - int n_result=0, i=0; + int ret=0, i=0; + size_t is_valid_offset=0, valid_len=0; + size_t key_offset=0, key_len=0; - ip_plugin_desc=&(table_schema->ip_plugin); - n_result=ip_matcher_match(table_rt->ip_plugin.ip_matcher, (struct ip_data*)ip, results, size); - for(i=0; iplugin; + struct plugin_runtime* plugin_rt=&table_rt->plugin; + ret=Maat_helper_read_column(row, plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); + plugin_rt->acc_line_num++; + if(plugin_schema->have_exdata) { - - ip_plugin_desc->ex_desc.dup_func(table_schema->table_id, &(ex_data_array[i]), &(results[i].tag), - ip_plugin_desc->ex_desc.argl, ip_plugin_desc->ex_desc.argp); + ret=get_column_pos(row, plugin_schema->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "plugin EX data process error: cannot find column %d of %s", + plugin_schema->key_column, row); + return; + } + if(atoi(row+is_valid_offset)==1) + { + EX_data_rt_row2EX_data(plugin_rt->ex_data_rt, row, row+key_offset, key_len, NULL, logger); + } + else + { + EX_data_rt_delete_by_row(plugin_rt->ex_data_rt, row, row+key_offset, key_len, logger); + } } - return n_result; + if(plugin_schema->cb_plug_cnt>0) + { + for(i=0; icb_plug_cnt;i++) + { + plugin_schema->cb_plug[i].update(table_schema->table_id, row, plugin_schema->cb_plug[i].u_para); + } + } + if(!plugin_schema->have_exdata && !plugin_schema->cb_plug_cnt) + { + EX_data_rt_cache_row(plugin_rt->ex_data_rt, row); + } + + return; } void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag) @@ -518,34 +421,29 @@ int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) return q_cnt; } -TAILQ_HEAD(ip_range_rule_q, wrap_plugin_EX_data); - -void walk_ip_plugin_hash(const uchar * key, uint size, void * data, void * user) -{ - struct wrap_plugin_EX_data* wrap_plugin_ex=(struct wrap_plugin_EX_data*)data; - struct ip_range_rule_q* queue=(struct ip_range_rule_q*)user; - TAILQ_INSERT_TAIL(queue, wrap_plugin_ex, entries); - return; -} - -int Maat_table_runtime_rebuild_ip_matcher(struct Maat_table_runtime* table_rt) +int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt) { struct ip_matcher* new_ip_matcher=NULL; - struct ip_range_rule_q queue;//This is for index, no need to free. + struct EX_data_container_q queue;//This is for index, no need to free. size_t rule_cnt=0; size_t i=0, mem_use=0; struct ip_rule *rules=NULL; - struct wrap_plugin_EX_data *p=NULL; + struct EX_data_container *p=NULL; TAILQ_INIT(&queue); - MESA_htable_iterate(table_rt->ip_plugin.rowid2ex_hash, walk_ip_plugin_hash, &queue); - rule_cnt=(size_t)MESA_htable_get_elem_num(table_rt->ip_plugin.rowid2ex_hash); + rule_cnt=EX_data_rt_list_all(table_rt->ip_plugin.ex_data_rt, &queue); rules=ALLOC(struct ip_rule, rule_cnt); TAILQ_FOREACH(p, &queue, entries) { - rules[i]=p->range_rule; + rules[i]=*((struct ip_rule *)(p->user_data)); + assert(rules[i].user_tag==p||rules[i].user_tag==NULL); + rules[i].user_tag=p; i++; } assert(i==rule_cnt); + if(rule_cnt==0) + { + return 0; + } new_ip_matcher=ip_matcher_new(rules, rule_cnt, &mem_use); table_rt->ip_plugin.old_ip_matcher=table_rt->ip_plugin.ip_matcher; table_rt->ip_plugin.ip_matcher=new_ip_matcher; @@ -558,5 +456,82 @@ struct ip_matcher* Maat_table_runtime_dettach_old_ip_matcher(struct Maat_table_r table_rt->ip_plugin.old_ip_matcher=NULL; return old_one; } +void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) +{ + struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); + struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin); + size_t is_valid_offset=0, valid_len=0; + size_t key_offset=0, key_len=0; + struct ip_rule* ip_rule=NULL; + int ret=0; + + if(ip_plugin_schema->have_exdata) + { + ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "ip_plugin EX data process error: cannot find is_valid column %d of %s", + ip_plugin_schema->row_id_column, row); + return; + } + ret=Maat_helper_read_column(row, ip_plugin_schema->row_id_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "ip_plugin EX data process error: cannot find row id column %d of %s", + ip_plugin_schema->row_id_column, row); + return; + } + ip_rule=ip_plugin_row2ip_rule(ip_plugin_schema, row); + if(ip_rule==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "ip_plugin read ip error: %s", row); + return; + + } + if(atoi(row+is_valid_offset)==1)//add + { + EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, ip_rule, logger); + } + else + { + EX_data_rt_delete_by_row(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, logger); + } + } + else + { + EX_data_rt_cache_row(ip_plugin_rt->ex_data_rt, row); + } + return; +} +int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) +{ + int i=0; + const char* row=NULL; + EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema); + for(i=0; iplugin.ex_data_rt); i++) + { + row=EX_data_rt_get_cached_row(table_rt->plugin.ex_data_rt, i); + Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger); + } + EX_data_rt_clear_row_cache(table_rt->plugin.ex_data_rt); + Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt); + + return 0; +} + +int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) +{ + struct scan_result results[size]; + int n_result=0, i=0; + n_result=ip_matcher_match(table_rt->ip_plugin.ip_matcher, (struct ip_data*)ip, results, size); + for(i=0; iip_plugin.ex_data_rt, (struct EX_data_container *)results[i].tag); + } + return n_result; +} diff --git a/src/inc_internal/Maat_ex_data.h b/src/inc_internal/Maat_ex_data.h new file mode 100644 index 0000000..47d7e7f --- /dev/null +++ b/src/inc_internal/Maat_ex_data.h @@ -0,0 +1,45 @@ + +#include "dynamic_array.h" +#include "Maat_rule.h" +#include + +#include + +struct EX_data_rt +{ + dynamic_array_t *cache_rows; + long long cache_row_num; + long long cache_size; + MESA_htable_handle key2ex_hash; + const struct EX_data_schema* ex_schema; + int table_id; + void (* user_data_free)(void *user_data); +}; +struct EX_data_container +{ + MAAT_RULE_EX_DATA ex_data; + const struct EX_data_rt* rt; + void* user_data; + TAILQ_ENTRY(EX_data_container) entries; +}; + +TAILQ_HEAD(EX_data_container_q, EX_data_container); +struct EX_data_rt* EX_data_rt_new(int table_id, long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data)); +void EX_data_rt_free(struct EX_data_rt* p); +void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema); +void EX_data_rt_cache_row(struct EX_data_rt* p, const char* row); + +const char* EX_data_rt_get_cached_row(struct EX_data_rt* p, int i); + +void EX_data_rt_clear_row_cache(struct EX_data_rt* p); +int EX_data_rt_get_row_num(struct EX_data_rt* p); + +struct EX_data_container* EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt, + const char* row, const char* key, size_t key_len, + void* user_data, void* logger); + +int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len, void *logger); +MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len); +MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container); +size_t EX_data_rt_list_all(struct EX_data_rt* ex_rt, EX_data_container_q* listed); + diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 8746343..4a0d04c 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -457,9 +457,6 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p); void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len); MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); -MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index); -int plugin_EX_data_new(const struct Maat_table_schema* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); -int plugin_EX_data_free(const char* line, int key_column, MESA_htable_handle key2ex_hash, void *logger); void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout); diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index 8e0b75c..3249b7b 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -79,7 +79,7 @@ struct plugin_table_callback_schema Maat_finish_callback_t *finish; void* u_para; }; -struct plugin_table_ex_data_schema +struct EX_data_schema { Maat_plugin_EX_new_func_t* new_func; Maat_plugin_EX_free_func_t* free_func; @@ -99,7 +99,7 @@ struct plugin_table_schema int have_exdata; long long estimate_size; struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE]; - struct plugin_table_ex_data_schema ex_desc; + struct EX_data_schema ex_schema; }; struct ip_plugin_table_schema { @@ -110,7 +110,8 @@ struct ip_plugin_table_schema int valid_flag_column; int rule_tag_column; long long estimate_size; - struct plugin_table_ex_data_schema ex_desc; + int have_exdata; + struct EX_data_schema ex_schema; }; struct Maat_table_schema { @@ -161,12 +162,21 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c Maat_rule_EX_dup_func_t* dup_func, long argl, void *argp); struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx); -int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, +int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, Maat_plugin_EX_dup_func_t* dup_func, Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp); + long argl, void *argp, + void* logger); +int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp, + void* logger); + void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type); void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr); diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h index 57c6e2e..936e7c1 100644 --- a/src/inc_internal/Maat_table_runtime.h +++ b/src/inc_internal/Maat_table_runtime.h @@ -1,4 +1,5 @@ #include "Maat_table.h" +#include "Maat_ex_data.h" #include "IPMatcher.h" #include "gram_index_engine.h" #include "alignment_int64.h" @@ -14,16 +15,13 @@ struct similar_runtime struct plugin_runtime { - dynamic_array_t *cache_lines; - long long cache_line_num; + struct EX_data_rt* ex_data_rt; long long acc_line_num; - long long cache_size; - MESA_htable_handle key2ex_hash; }; + struct ip_plugin_runtime { - long long row_num; - MESA_htable_handle rowid2ex_hash; + struct EX_data_rt* ex_data_rt; struct ip_matcher* ip_matcher; struct ip_matcher* old_ip_matcher; }; @@ -69,13 +67,18 @@ struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_mana long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt); const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line); -int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger); +int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger); MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key); void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag); void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id); int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt); -int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); -int Maat_table_runtime_rebuild_ip_matcher(struct Maat_table_runtime* table_rt); + +void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); +void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); + +int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger); +int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); +int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt); struct ip_matcher* Maat_table_runtime_dettach_old_ip_matcher(struct Maat_table_runtime* table_rt); diff --git a/test/maat_json.json b/test/maat_json.json index 00e00bb..5692650 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -1754,6 +1754,15 @@ "1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34", "161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42" ] + }, + { + "table_name": "TEST_IP_PLUGIN_WITH_EXDATA", + "table_content": [ + "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1", + "102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1", + "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1", + "104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1" + ] } ] } diff --git a/test/table_info.conf b/test/table_info.conf index 82e6924..f577ffb 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -47,4 +47,5 @@ 24 COMPOSITION_IP_SOURCE virtual IP_PLUS_CONFIG -- 25 COMPOSITION_IP_DESTINATION virtual IP_PLUS_CONFIG -- 26 COMPOSITION_IP_SESSION virtual IP_PLUS_CONFIG -- -27 COMPOSITION_IP composition {"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"} \ No newline at end of file +27 COMPOSITION_IP composition {"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"} +28 TEST_IP_PLUGIN_WITH_EXDATA ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":6} -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 103f4fb..d17c940 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -101,7 +101,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int is_old) if(!is_old) { EXPECT_EQ(ret, 1); - EXPECT_TRUE(result.config_id==2); + EXPECT_EQ(result.config_id, 2); } else { @@ -219,7 +219,7 @@ void test_plugin_table(Maat_feather_t feather,const char* table_name, ASSERT_GT(ret, 0); } - +#define Plugin_callback TEST(PluginTable, Callback) { test_plugin_table(g_feather, "QD_ENTRY_INFO", @@ -229,6 +229,85 @@ TEST(PluginTable, Callback) g_feather, g_logger); +} +#define IP_PLUGIN_EX_DATA +struct ip_plugin_ud +{ + int rule_id; + char* buffer; + int ref_cnt; +}; +void ip_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + int *counter=(int *)argp, ret=0; + size_t column_offset=0, column_len=0; + struct ip_plugin_ud* ud=(struct ip_plugin_ud*)calloc(sizeof(struct ip_plugin_ud), 1); + ret=Maat_helper_read_column(table_line, 1, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + ud->rule_id=atoi(table_line+column_offset); + ret=Maat_helper_read_column(table_line, 5, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + ud->buffer=(char*)calloc(sizeof(char), column_len+1); + strncpy(ud->buffer, table_line+column_offset, column_len); + ud->ref_cnt=1; + *ad=ud; + (*counter)++; + return; +} +void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad); + u->ref_cnt--; + if(u->ref_cnt>0) return; + free(u->buffer); + free(u); + *ad=NULL; +} +void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) +{ + struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from); + u->ref_cnt++; + *to=u; +} + +TEST(IP_Plugin_Table, EX_DATA) +{ + + int ip_plugin_ex_data_counter=0, i=0; + const char* table_name="TEST_IP_PLUGIN_WITH_EXDATA"; + int table_id=0, ret=0; + table_id=Maat_table_register(g_feather, table_name); + ASSERT_GT(table_id, 0); + ret=Maat_ip_plugin_EX_register(g_feather, table_id, + ip_plugin_EX_new_cb, + ip_plugin_EX_free_cb, + ip_plugin_EX_dup_cb, + 0, &ip_plugin_ex_data_counter); + ASSERT_TRUE(ret>=0); + EXPECT_EQ(ip_plugin_ex_data_counter, 4); + struct ip_address ipv4, ipv6; + struct ip_plugin_ud* result[4]; + ipv4.ip_type=4; + inet_pton(AF_INET, "192.168.30.100", &(ipv4.ipv4)); + ret=Maat_ip_plugin_get_EX_data(g_feather, table_id, &ipv4, (void**)result, 4); + ASSERT_EQ(ret, 2); + EXPECT_EQ(result[0]->rule_id, 101); + EXPECT_EQ(result[1]->rule_id, 102); + for(i=0; irule_id, 104); + EXPECT_EQ(result[1]->rule_id, 103); + for(i=0; i