diff --git a/inc/bool_matcher.h b/inc/bool_matcher.h index 08df846..25a0554 100644 --- a/inc/bool_matcher.h +++ b/inc/bool_matcher.h @@ -49,11 +49,11 @@ extern "C" struct bool_matcher; - struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, unsigned int max_thread_num, size_t * mem_size); + struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, size_t * mem_size); /* Returned results are sorted by expr_id in descending order. */ //Input item_ids MUST be ASCENDING order and NO duplication. - int bool_matcher_match(struct bool_matcher * matcher, unsigned int thread_id, unsigned long long * item_ids, size_t item_num, struct bool_expr_match * results, size_t n_result); + int bool_matcher_match(struct bool_matcher * matcher, unsigned long long * item_ids, size_t item_num, struct bool_expr_match * results, size_t n_result); void bool_matcher_free(struct bool_matcher * matcher); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2cbe127..7531a9b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table_schema.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp entry/Maat_ex_data.cpp entry/Maat_hierarchy.cpp entry/Maat_garbage_collection.cpp entry/Maat_command.cpp entry/FQDN_engine.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 8f94508..d113ba6 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -577,9 +577,9 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* { goto failed; } - Maat_table_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn)); - Maat_table_get_group2compile_table_name(feather->table_mgr, feather->group2compile_tn, sizeof(feather->group2compile_tn)); - Maat_table_get_group2group_table_name(feather->table_mgr, feather->group2group_tn, sizeof(feather->group2group_tn)); + Maat_table_manager_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn)); + Maat_table_manager_get_group2compile_table_name(feather->table_mgr, feather->group2compile_tn, sizeof(feather->group2compile_tn)); + Maat_table_manager_get_group2group_table_name(feather->table_mgr, feather->group2group_tn, sizeof(feather->group2group_tn)); feather->logger=logger; feather->scan_thread_num=max_thread_num; @@ -1035,7 +1035,7 @@ void Maat_burn_feather(Maat_feather_t feather) int Maat_table_register(Maat_feather_t feather,const char* table_name) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - return Maat_table_get_id_by_name(_feather->table_mgr, table_name); + return Maat_table_manager_get_id_by_name(_feather->table_mgr, table_name); } int Maat_table_callback_register(Maat_feather_t feather,short table_id, Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para @@ -1047,7 +1047,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, int i=0,ret=0; pthread_mutex_lock(&(_feather->background_update_mutex)); - ret=Maat_table_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para); + ret=Maat_table_manager_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para); if(ret<0) { pthread_mutex_unlock(&(_feather->background_update_mutex)); @@ -1117,7 +1117,7 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table return -1; } pthread_mutex_lock(&(_feather->background_update_mutex)); - idx=Maat_table_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name, + idx=Maat_table_manager_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name, new_func, free_func, dup_func, @@ -1128,7 +1128,7 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table pthread_mutex_unlock(&(_feather->background_update_mutex)); return -1; } - struct compile_ex_data_idx* compile_ex_desc=Maat_table_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx); + struct compile_ex_data_idx* compile_ex_desc=Maat_table_manager_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx); if(_feather->scanner!=NULL) { @@ -1162,35 +1162,7 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa return ad; } -int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp) -{ - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int ret=0; - struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); - - pthread_mutex_lock(&(_feather->background_update_mutex)); - ret=Maat_table_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, key2index_func,argl, argp, _feather->logger); - - if(ret<0) - { - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return -1; - } - struct Maat_table_runtime* table_rt=NULL; - if(_feather->scanner!=NULL) - { - table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - Maat_table_runtime_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger); - } - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return 0; -} MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; @@ -1207,7 +1179,7 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id { return NULL; } - table_desc=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); + table_desc=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); @@ -1223,18 +1195,17 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id return exdata; } -int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, +int Maat_generic_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t *new_func, + Maat_plugin_EX_free_func_t *free_func, + Maat_plugin_EX_dup_func_t *dup_func, long argl, void *argp) - { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; int ret=-1; - struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); + struct Maat_table_schema *table_schema=Maat_table_manager_get_by_id_raw(_feather->table_mgr, table_id); pthread_mutex_lock(&(_feather->background_update_mutex)); - ret=Maat_table_ip_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger); + ret=Maat_table_schema_set_EX_data_schema(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger); if(ret<0) { @@ -1246,50 +1217,76 @@ int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, if(_feather->scanner!=NULL) { table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - Maat_table_runtime_ip_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger); + Maat_table_runtime_commit_EX_data_schema(table_rt, table_schema, _feather->logger); } pthread_mutex_unlock(&(_feather->background_update_mutex)); return 0; + +} +int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp) + +{ + int ret=-1; + ret=Maat_generic_plugin_EX_register(feather, table_id, + new_func, free_func, dup_func, + argl, argp); + return ret; + +} + +int Maat_ip_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t *new_func, + Maat_plugin_EX_free_func_t *free_func, + Maat_plugin_EX_dup_func_t *dup_func, + long argl, void *argp) + +{ + int ret=-1; + ret=Maat_generic_plugin_EX_register(feather, table_id, + new_func, free_func, dup_func, + argl, argp); + return ret; } int Maat_fqdn_plugin_EX_register(Maat_feather_t feather, int table_id, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_new_func_t *new_func, + Maat_plugin_EX_free_func_t *free_func, + Maat_plugin_EX_dup_func_t *dup_func, long argl, void *argp) { - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; int ret=-1; - struct Maat_table_schema *table_schema=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); - pthread_mutex_lock(&(_feather->background_update_mutex)); - ret=Maat_table_fqdn_plugin_EX_data_schema_set(table_schema, new_func, free_func, dup_func, NULL, argl, argp, _feather->logger); - - if(ret<0) - { - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return -1; - } - - struct Maat_table_runtime* table_rt=NULL; - if(_feather->scanner!=NULL) - { - table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); - Maat_table_runtime_fqdn_plugin_commit_ex_schema(table_rt, table_schema, _feather->logger); - } - pthread_mutex_unlock(&(_feather->background_update_mutex)); - - return 0; - + ret=Maat_generic_plugin_EX_register(feather, table_id, + new_func, free_func, dup_func, + argl, argp); + return ret; } +int Maat_bool_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t *new_func, + Maat_plugin_EX_free_func_t *free_func, + Maat_plugin_EX_dup_func_t *dup_func, + long argl, void *argp) +{ + int ret=-1; + ret=Maat_generic_plugin_EX_register(feather, table_id, + new_func, free_func, dup_func, + argl, argp); + return ret; +} + int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struct ip_address* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) { - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather; struct Maat_table_schema *table_schema=NULL; struct Maat_table_runtime *table_rt=NULL; int n_get=0; - struct timespec start,end; + struct timespec start, end; if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&start); @@ -1299,7 +1296,7 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc return 0; } - table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL); + table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt); if(table_type!=TABLE_TYPE_IP_PLUGIN) @@ -1332,12 +1329,12 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc } int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) { - struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather; struct Maat_table_schema *table_schema=NULL; struct Maat_table_runtime *table_rt=NULL; int n_get=0; - struct timespec start,end; + struct timespec start, end; if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&start); @@ -1346,7 +1343,7 @@ int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const cha { return 0; } - table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL); + table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL); table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt); if(table_type!=TABLE_TYPE_FQDN_PLUGIN) @@ -1368,11 +1365,48 @@ int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const cha return n_get; } +int Maat_bool_plugin_get_EX_data(Maat_feather_t feather, int table_id, unsigned long long item_id[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t n_ex_data) +{ + struct _Maat_feather_t *_feather=(_Maat_feather_t*)feather; + struct Maat_table_schema *table_schema=NULL; + struct Maat_table_runtime *table_rt=NULL; + int n_get=0; -int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num - ,int* detail_ret,scan_status_t* mid,int thread_num) + struct timespec start, end; + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&start); + } + if(_feather->scanner==NULL) + { + return 0; + } + table_schema=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_BOOL_PLUGIN, NULL); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt); + if(table_type!=TABLE_TYPE_BOOL_PLUGIN) + { + return -1; + } + n_get=Maat_table_runtime_bool_plugin_get_N_ex_data(table_rt, table_schema, item_id, n_item, ex_data_array, n_ex_data); + + if(_feather->perf_on==1) + { + clock_gettime(CLOCK_MONOTONIC,&end); + Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0); + } + else + { + Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0); + } + + return n_get; +} + +int Maat_full_scan_string_detail(Maat_feather_t feather, int table_id, + enum MAAT_CHARSET charset, const char* data, int data_len, + struct Maat_rule_t*result, int rule_num, struct Maat_hit_detail_t *hit_detail, int detail_num, + int* detail_ret, scan_status_t* mid, int thread_num) { int region_ret=0, compile_ret=0; int district_id=DISTRICT_ANY; @@ -1404,7 +1438,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id return 0; } - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); + p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1515,10 +1549,10 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } return compile_ret; } -int Maat_full_scan_string(Maat_feather_t feather,int table_id - ,enum MAAT_CHARSET charset,const char* data,int data_len - ,struct Maat_rule_t*result,int* found_pos,int rule_num - ,scan_status_t* mid,int thread_num) +int Maat_full_scan_string(Maat_feather_t feather, int table_id, + enum MAAT_CHARSET charset, const char* data, int data_len, + struct Maat_rule_t*result, int* found_pos, int rule_num, + scan_status_t* mid, int thread_num) { int detail_ret=0,compile_ret=0; compile_ret=Maat_full_scan_string_detail(feather,table_id, @@ -1550,7 +1584,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id scan_staus_count_inc(_mid); int virtual_table_id=0; - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id); + p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_INTERVAL, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1656,7 +1690,7 @@ int Maat_similar_scan_string(Maat_feather_t feather, int table_id, scan_staus_count_inc(_mid); int virtual_table_id=0; - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); + p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1809,14 +1843,14 @@ static int IP_composition_scan(const struct ipaddr* addr, unsigned short int pro } else { - child_table_id=Maat_table_get_child_id(table_mgr, parent_table_id, child_type); + child_table_id=Maat_table_manager_get_child_id(table_mgr, parent_table_id, child_type); } if(child_table_id<0) { return 0; } - Maat_table_schema* real_table=Maat_table_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id); + Maat_table_schema* real_table=Maat_table_manager_get_scan_by_id(table_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id); if(real_table==NULL) { return 0; @@ -1868,7 +1902,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id scan_staus_count_inc(_mid); int virtual_table_id=0; enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID; - table_type=Maat_table_get_type_by_id(_feather->table_mgr, table_id); + table_type=Maat_table_manager_get_type_by_id(_feather->table_mgr, table_id); if(table_type==TABLE_TYPE_INVALID) { _feather->scan_err_cnt++; @@ -1981,7 +2015,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct Maat_table_schema *p_table=NULL; int virtual_table_id=0; assert(thread_num<_feather->scan_thread_num); - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); + p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2278,7 +2312,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sfh_instance_t * tmp_fuzzy_handle=NULL; struct Maat_table_schema *p_table=NULL; int virtual_table_id=0; - p_table=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); + p_table=Maat_table_manager_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_STRING, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 10ec908..724fc35 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -732,7 +732,7 @@ FULL_UPDATE: } if(table_mgr) { - table_id=Maat_table_get_id_by_name(table_mgr, s_rule_array[full_idx].table_name); + table_id=Maat_table_manager_get_id_by_name(table_mgr, s_rule_array[full_idx].table_name); if(table_id<0)//Unrecognized table. { continue; @@ -1536,7 +1536,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, { continue; } - p_table=Maat_table_get_desc_by_name(feather->table_mgr, rule_list[i].table_name); + p_table=Maat_table_manager_get_desc_by_name(feather->table_mgr, rule_list[i].table_name); if(!p_table||p_table->table_type!=TABLE_TYPE_PLUGIN) { continue; @@ -1808,19 +1808,19 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m { continue; } - table_id=Maat_table_get_id_by_name(feather->table_mgr, rule_list[i].table_name); + table_id=Maat_table_manager_get_id_by_name(feather->table_mgr, rule_list[i].table_name); if(table_id<0)//Unrecognized table. { no_table_num++; continue; } - table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id); + table_type=Maat_table_manager_get_type_by_id(feather->table_mgr, table_id); if(rule_list[i].op==MAAT_OP_DEL) { scan_type=Maat_table_get_scan_type(table_type); - table_schema=Maat_table_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL); - valid_column=Maat_table_xx_plugin_table_get_valid_flag_column(table_schema); + table_schema=Maat_table_manager_get_scan_by_id(feather->table_mgr, table_id, scan_type, NULL); + valid_column=Maat_table_schema_get_valid_flag_column(table_schema); ret=invalidate_line(rule_list[i].table_line, table_type, valid_column); if(ret<0) { @@ -1935,7 +1935,7 @@ int Maat_command_raw_set_lines(Maat_feather_t feather,const struct Maat_cmd_line s_rule=ALLOC(struct serial_rule_t, n_line); for(i=0;itable_mgr, line_rule[i]->table_name); + table_id=Maat_table_manager_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name); if(table_id<0) { MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, @@ -1998,7 +1998,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_ s_rule=ALLOC(struct serial_rule_t, line_num); for(i=0;itable_mgr, line_rule[i]->table_name); + table_id=Maat_table_manager_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name); if(table_id<0) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command @@ -2008,7 +2008,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_ ret=-1; goto error_out; } - p_table=Maat_table_get_by_id_raw(_feather->table_mgr, table_id); + p_table=Maat_table_manager_get_by_id_raw(_feather->table_mgr, table_id); if(!p_table) { ret=-1; @@ -2016,7 +2016,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_ } int valid_flag_column=0; - valid_flag_column=Maat_table_xx_plugin_table_get_valid_flag_column(p_table); + valid_flag_column=Maat_table_schema_get_valid_flag_column(p_table); if(valid_flag_column<0) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command diff --git a/src/entry/Maat_ex_data.cpp b/src/entry/Maat_ex_data.cpp index 954209c..ec73c61 100644 --- a/src/entry/Maat_ex_data.cpp +++ b/src/entry/Maat_ex_data.cpp @@ -1,5 +1,5 @@ #include "Maat_ex_data.h" -#include "Maat_table.h" +#include "Maat_table_schema.h" #include "Maat_utils.h" #include "Maat_garbage_collection.h" diff --git a/src/entry/Maat_hierarchy.cpp b/src/entry/Maat_hierarchy.cpp index 96875e8..2188c6e 100644 --- a/src/entry/Maat_hierarchy.cpp +++ b/src/entry/Maat_hierarchy.cpp @@ -998,7 +998,7 @@ static struct bool_matcher* Maat_hierarchy_build_bool_matcher(struct Maat_hierar "No bool expression to build."); goto error_out; } - bm=bool_matcher_new(bool_expr_array, expr_cnt, hier->thread_num, &mem_size); + bm=bool_matcher_new(bool_expr_array, expr_cnt, &mem_size); if(bm!=NULL) { MESA_handle_runtime_log(hier->logger, RLOG_LV_INFO, module_maat_hierarchy, @@ -1365,7 +1365,7 @@ size_t Maat_hierarchy_get_hit_paths(struct Maat_hierarchy* hier, struct Maat_hie } } - bool_match_ret=bool_matcher_match(hier->bm, mid->thread_num, + bool_match_ret=bool_matcher_match(hier->bm, (unsigned long long*)utarray_eltptr(mid->_all_hit_clause_array, 0), utarray_len(mid->_all_hit_clause_array), expr_match, MAX_SCANNER_HIT_NUM); for(i=0; ithis_scan_region_hit_cnt=0; return 0; } - bool_match_ret=bool_matcher_match(hier->bm, mid->thread_num, + bool_match_ret=bool_matcher_match(hier->bm, (unsigned long long*)utarray_eltptr(mid->_all_hit_clause_array, 0), utarray_len(mid->_all_hit_clause_array), expr_match, MAX_SCANNER_HIT_NUM); for(i=0; itable_type==TABLE_TYPE_IP_PLUGIN) + switch(table_schema->table_type) { - Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger); - } - else //TABLE_TYPE_FQDN_PLUGIN - { - Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger); + case TABLE_TYPE_IP_PLUGIN: + Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger); + break; + case TABLE_TYPE_FQDN_PLUGIN: + Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger); + break; + case TABLE_TYPE_BOOL_PLUGIN: + Maat_table_runtime_bool_plugin_new_row(table_rt, table_schema, table_row, logger); + break; + default: + assert(0); + break; } scanner->xx_plugin_rule_to_update_cnt++; return; @@ -2361,10 +2368,10 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* } break; case TABLE_TYPE_PLUGIN: - Maat_table_runtime_plugin_update_commit(table_rt); + Maat_table_runtime_plugin_commit_update(table_rt); break; case TABLE_TYPE_IP_PLUGIN: - ret=Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt); + ret=Maat_table_runtime_ip_plugin_commit_update(table_rt); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, @@ -2374,12 +2381,21 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* break; case TABLE_TYPE_FQDN_PLUGIN: - ret=Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(table_rt); + ret=Maat_table_runtime_fqdn_plugin_commit_update(table_rt); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "FQDN plugin table_id %d build failed.", i); + } + break; + case TABLE_TYPE_BOOL_PLUGIN: + ret=Maat_table_runtime_bool_plugin_commit_update(table_rt); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Boolean expression plugin table_id %d build failed.", i); + } break; default: @@ -2501,13 +2517,13 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) scanner=feather->scanner; } // MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line); - p_table=Maat_table_get_desc_by_name(feather->table_mgr, table_name); + p_table=Maat_table_manager_get_desc_by_name(feather->table_mgr, table_name); if(!p_table) { MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s", table_name); return -1; } - Maat_table_set_updating_name(p_table, table_name); + Maat_table_schema_set_updating_name(p_table, table_name); switch(p_table->table_type) { diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index 8f395fc..78e0307 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -1,5 +1,5 @@ #include "Maat_rule_internal.h" -#include "Maat_table.h" +#include "Maat_table_schema.h" #include "Maat_garbage_collection.h" #include "alignment_int64.h" #include @@ -128,7 +128,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr); for(i=0; i<(int)max_table_num; i++) { - p_table=Maat_table_get_by_id_raw(feather->table_mgr, i); + p_table=Maat_table_manager_get_by_id_raw(feather->table_mgr, i); if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN ||p_table->table_type==TABLE_TYPE_GROUP ||p_table->table_type==TABLE_TYPE_COMPILE) @@ -203,7 +203,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_hit_cnt=0; table_regex_ipv6_num=0; - p_table=Maat_table_get_by_id_raw(feather->table_mgr, i); + p_table=Maat_table_manager_get_by_id_raw(feather->table_mgr, i); if(p_table==NULL) { continue; diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp index 14dcd3e..13a44fa 100644 --- a/src/entry/Maat_table_runtime.cpp +++ b/src/entry/Maat_table_runtime.cpp @@ -99,9 +99,9 @@ static void destroy_digest_rule(GIE_digest_t*rule) rule=NULL; return; } -struct FQDN_rule* fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match) +struct FQDN_rule *fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_len, int is_suffix_match) { - struct FQDN_rule* fqdn_rule=ALLOC(struct FQDN_rule, 1); + struct FQDN_rule *fqdn_rule=ALLOC(struct FQDN_rule, 1); //Todo: check FQDN format with regex ^([a-zA-Z0-9._-])+$ if(fqdn[0]=='.') { @@ -119,7 +119,7 @@ struct FQDN_rule* fqdn_rule_new(unsigned int id, const char* fqdn, size_t fqdn_l fqdn_rule->id=id; return fqdn_rule; } -void fqdn_rule_free(struct FQDN_rule* fqdn_rule) +void fqdn_rule_free(struct FQDN_rule *fqdn_rule) { assert(fqdn_rule->user_tag==NULL); free(fqdn_rule->FQDN); @@ -127,6 +127,23 @@ void fqdn_rule_free(struct FQDN_rule* fqdn_rule) free(fqdn_rule); return; } +struct bool_expr *bool_expr_new(unsigned int id, unsigned long long item_id[], size_t n_item) +{ + struct bool_expr *expr=ALLOC(struct bool_expr, 1); + expr->expr_id=id; + size_t i=0; + for(i=0; iitems[i].item_id=item_id[i]; + expr->items[i].not_flag=0; + } + return expr; +} +void bool_expr_free(struct bool_expr *expr) +{ + free(expr); + return; +} void _notype_fqdn_rule_free(void* p) { fqdn_rule_free((struct FQDN_rule*)p); @@ -147,7 +164,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, table_schema->plugin.ex_schema.key2index_func, NULL); - if(table_schema->plugin.have_exdata) + if(table_schema->plugin.ex_schema.set_flag) { EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema); } @@ -156,7 +173,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, table_schema->ip_plugin.ex_schema.key2index_func, free); - if(table_schema->ip_plugin.have_exdata) + if(table_schema->ip_plugin.ex_schema.set_flag) { EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema); } @@ -165,11 +182,20 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, table_schema->fqdn_plugin.ex_schema.key2index_func, _notype_fqdn_rule_free); - if(table_schema->fqdn_plugin.have_exdata) + if(table_schema->fqdn_plugin.ex_schema.set_flag) { EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema); } break; + case TABLE_TYPE_BOOL_PLUGIN: + table_rt->bool_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, + table_schema->fqdn_plugin.ex_schema.key2index_func, + (void (*)(void*))bool_expr_free); + if(table_schema->bool_plugin.ex_schema.set_flag) + { + EX_data_rt_set_schema(table_rt->bool_plugin.ex_data_rt, &table_schema->bool_plugin.ex_schema); + } + break; default: break; } @@ -249,7 +275,7 @@ struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt); for(i=0; in_table_rt; i++) { - table_desc=Maat_table_get_by_id_raw(table_manager, i); + table_desc=Maat_table_manager_get_by_id_raw(table_manager, i); if(!table_desc) { continue; @@ -305,25 +331,10 @@ const char* Maat_table_runtime_plugin_get_cached_row(struct Maat_table_runtime* line=EX_data_rt_cached_row_get(plugin_rt->ex_data_rt, Nth_row); return line; } -int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) -{ - size_t i=0; - const char* row=NULL; - EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema); - - for(i=0; iplugin.ex_data_rt); i++) - { - row=EX_data_rt_cached_row_get(table_rt->plugin.ex_data_rt, i); - Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger); - } - EX_data_rt_update_commit(table_rt->plugin.ex_data_rt); - table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->plugin.ex_data_rt); - return 0; -} MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* key) { MAAT_RULE_EX_DATA ex_data=NULL; - if(!table_schema->plugin.have_exdata) + if(!table_schema->plugin.ex_schema.set_flag) { assert(0); return NULL; @@ -342,7 +353,7 @@ void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, stru struct plugin_runtime* plugin_rt=&table_rt->plugin; ret=Maat_helper_read_column(row, plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); plugin_rt->acc_line_num++; - if(plugin_schema->have_exdata) + if(plugin_schema->ex_schema.set_flag) { ret=get_column_pos(row, plugin_schema->key_column, &key_offset, &key_len); if(ret<0) @@ -368,14 +379,14 @@ void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, stru plugin_schema->cb_plug[i].update(table_schema->table_id, row, plugin_schema->cb_plug[i].u_para); } } - if(!plugin_schema->have_exdata && !plugin_schema->cb_plug_cnt) + if(!plugin_schema->ex_schema.set_flag && !plugin_schema->cb_plug_cnt) { EX_data_rt_cache_row_put(plugin_rt->ex_data_rt, row); } return; } -void Maat_table_runtime_plugin_update_commit(struct Maat_table_runtime* table_rt) +void Maat_table_runtime_plugin_commit_update(struct Maat_table_runtime* table_rt) { EX_data_rt_update_commit(table_rt->plugin.ex_data_rt); table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->plugin.ex_data_rt); @@ -420,7 +431,7 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, size_t fqdn_offset=0, fqdn_len=0; struct FQDN_rule* fqdn_rule=NULL; int ret=0; - if(fqdn_plugin_schema->have_exdata) + if(fqdn_plugin_schema->ex_schema.set_flag) { ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); if(ret<0) @@ -479,7 +490,7 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, return; } -int Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(struct Maat_table_runtime* table_rt) +int Maat_table_runtime_fqdn_plugin_commit_update(struct Maat_table_runtime* table_rt) { struct FQDN_engine* new_fqdn_engine=NULL, *old_fqdn_engine=NULL; struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin; @@ -518,23 +529,80 @@ int Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(struct Maat_table_runtime table_rt->fqdn_plugin.changed_flag=0; return ret; } - -int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) +struct EX_data_rt *Maat_table_runtime_get_EX_data_rt(struct Maat_table_runtime* table_rt) +{ + struct EX_data_rt *rt=NULL; + + switch(table_rt->table_type) + { + case TABLE_TYPE_PLUGIN: + rt=table_rt->plugin.ex_data_rt; + break; + case TABLE_TYPE_IP_PLUGIN: + rt=table_rt->ip_plugin.ex_data_rt; + break; + case TABLE_TYPE_FQDN_PLUGIN: + rt=table_rt->fqdn_plugin.ex_data_rt; + break; + case TABLE_TYPE_BOOL_PLUGIN: + rt=table_rt->bool_plugin.ex_data_rt; + break; + default: + break; + } + return rt; +} +int Maat_table_runtime_commit_EX_data_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) { size_t i=0; const char* row=NULL; - struct fqdn_plugin_runtime* fqdn_plugin_rt=&(table_rt->fqdn_plugin); - EX_data_rt_set_schema(fqdn_plugin_rt->ex_data_rt, &table_schema->fqdn_plugin.ex_schema); - for(i=0; iex_data_rt); i++) + assert(table_rt->table_type==table_schema->table_type); + struct EX_data_rt *ex_data_rt=NULL; + struct EX_data_schema *ex_data_schema=Maat_table_schema_get_EX_data_schema(table_schema); + ex_data_rt=Maat_table_runtime_get_EX_data_rt(table_rt); + EX_data_rt_set_schema(ex_data_rt, ex_data_schema); + for(i=0; iex_data_rt, i); - Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger); + row=EX_data_rt_cached_row_get(ex_data_rt, i); + switch(table_rt->table_type) + { + case TABLE_TYPE_PLUGIN: + Maat_table_runtime_plugin_new_row(table_rt, table_schema, row, logger); + break; + case TABLE_TYPE_IP_PLUGIN: + Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger); + break; + case TABLE_TYPE_FQDN_PLUGIN: + Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger); + break; + case TABLE_TYPE_BOOL_PLUGIN: + Maat_table_runtime_bool_plugin_new_row(table_rt, table_schema, row, logger); + break; + default: + break; + } + } + EX_data_rt_clear_row_cache(ex_data_rt); + switch(table_rt->table_type) + { + case TABLE_TYPE_PLUGIN: + Maat_table_runtime_plugin_commit_update(table_rt); + break; + case TABLE_TYPE_IP_PLUGIN: + Maat_table_runtime_ip_plugin_commit_update(table_rt); + break; + case TABLE_TYPE_FQDN_PLUGIN: + Maat_table_runtime_fqdn_plugin_commit_update(table_rt); + break; + case TABLE_TYPE_BOOL_PLUGIN: + Maat_table_runtime_bool_plugin_commit_update(table_rt); + break; + default: + break; } - EX_data_rt_clear_row_cache(fqdn_plugin_rt->ex_data_rt); - Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(table_rt); - table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(fqdn_plugin_rt->ex_data_rt); return 0; } + int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) { struct FQDN_match results[size]; @@ -555,6 +623,145 @@ int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* tabl return n_result; } +void Maat_table_runtime_bool_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) +{ + struct bool_plugin_table_schema *bool_plugin_schema=&(table_schema->bool_plugin); + struct bool_plugin_runtime *bool_plugin_rt=&(table_rt->bool_plugin); + size_t is_valid_offset=0, valid_len=0; + size_t row_id_offset=0, row_id_len=0; + size_t bool_expr_offset=0, bool_expr_len=0; + struct bool_expr *expr=NULL; + int ret=0; + unsigned long long item_id[MAX_ITEMS_PER_BOOL_EXPR]; + size_t n_item=0; + if(bool_plugin_schema->ex_schema.set_flag) + { + ret=Maat_helper_read_column(row, bool_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "bool_plugin EX data process error: cannot find is_valid column %d of %s", + bool_plugin_schema->valid_flag_column, row); + return; + } + ret=Maat_helper_read_column(row, bool_plugin_schema->row_id_column, &row_id_offset, &row_id_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "bool_plugin EX data process error: cannot find row id column %d of %s", + bool_plugin_schema->row_id_column, row); + return; + } + ret=Maat_helper_read_column(row, bool_plugin_schema->bool_expr_column, &bool_expr_offset, &bool_expr_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: cannot find bool_expr column %d of %s", + bool_plugin_schema->bool_expr_column, row); + return; + } + char *token=NULL, *sub_token=NULL, *saveptr; + char expr_buffer[256]; + memset(expr_buffer, 0, sizeof(expr_buffer)); + memcpy(expr_buffer, row+bool_expr_offset, bool_expr_len); + for (token = expr_buffer; ; token= NULL) + { + sub_token= strtok_r(token, "&", &saveptr); + if (sub_token == NULL) + break; + ret=sscanf(sub_token, "%llu", item_id+n_item); + n_item++; + if(ret!=1||n_item==MAX_ITEMS_PER_BOOL_EXPR) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "fqdn_plugin EX data process error: invalid format of bool_expr column %d of %s", + bool_plugin_schema->bool_expr_column, row); + return; + } + } + if(atoi(row+is_valid_offset)==1)//add + { + expr=bool_expr_new((unsigned int)atoi(row+row_id_offset), item_id, n_item); + if(expr==NULL) + ret=EX_data_rt_row2EX_data(bool_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, expr, logger); + if(ret<0) + { + bool_expr_free(expr); + expr=NULL; + } + } + else + { + EX_data_rt_delete_by_row(bool_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger); + } + } + else + { + EX_data_rt_cache_row_put(bool_plugin_rt->ex_data_rt, row); + table_rt->origin_rule_num=EX_data_rt_get_cached_row_num(bool_plugin_rt->ex_data_rt); + } + bool_plugin_rt->changed_flag=1; + return; +} +int Maat_table_runtime_bool_plugin_commit_update(struct Maat_table_runtime* table_rt) +{ + struct bool_matcher* new_bool_matcher=NULL, *old_bool_matcher=NULL; + struct bool_plugin_runtime* bool_plugin_rt=&table_rt->bool_plugin; + assert(table_rt->table_type==TABLE_TYPE_BOOL_PLUGIN); + struct EX_data_container **exc_array=NULL; + struct bool_expr* exprs=NULL; + size_t expr_cnt=0, i=0, ret=0, mem_usage=0; + if(!bool_plugin_rt->changed_flag) + { + return ret; + } + expr_cnt=EX_data_rt_list_updating_ex_containers(bool_plugin_rt->ex_data_rt, &exc_array); + exprs=ALLOC(struct bool_expr, expr_cnt); + for(i=0; i0) + { + new_bool_matcher=bool_matcher_new(exprs, expr_cnt, &mem_usage); + if(!new_bool_matcher) + { + ret=-1; + } + } + old_bool_matcher=bool_plugin_rt->matcher; + bool_plugin_rt->matcher=new_bool_matcher; + Maat_garbage_bagging(table_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))bool_matcher_free); + EX_data_rt_update_commit(bool_plugin_rt->ex_data_rt); + table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(bool_plugin_rt->ex_data_rt); + + free(exprs); + free(exc_array); + table_rt->fqdn_plugin.changed_flag=0; + return ret; +} + +int Maat_table_runtime_bool_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, unsigned long long item_id[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) +{ + struct bool_expr_match results[size]; + int n_result=0, i=0; + if(table_rt->table_type!=TABLE_TYPE_BOOL_PLUGIN) + { + return -1; + } + if(!table_rt->bool_plugin.matcher) + { + return 0; + } + n_result=bool_matcher_match(table_rt->bool_plugin.matcher, item_id, n_item, results, size); + for(i=0; ibool_plugin.ex_data_rt, (struct EX_data_container *)results[i].user_tag); + } + return n_result; +} int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) { long i=0,data_size=0; @@ -621,7 +828,7 @@ int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) return q_cnt; } -int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt) +int Maat_table_runtime_ip_plugin_commit_update(struct Maat_table_runtime* table_rt) { struct ip_matcher* new_ip_matcher=NULL, *old_ip_matcher=NULL; size_t rule_cnt=0; @@ -673,7 +880,7 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s size_t row_id_offset=0, row_id_len=0; struct ip_rule* ip_rule=NULL; int ret=0; - if(ip_plugin_schema->have_exdata) + if(ip_plugin_schema->ex_schema.set_flag) { ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); if(ret<0) @@ -724,23 +931,6 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s ip_plugin_rt->changed_flag=1; return; } -int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) -{ - size_t i=0; - const char* row=NULL; - struct ip_plugin_runtime* ip_plugin_rt=&(table_rt->ip_plugin); - EX_data_rt_set_schema(ip_plugin_rt->ex_data_rt, &table_schema->ip_plugin.ex_schema); - for(i=0; iex_data_rt); i++) - { - row=EX_data_rt_cached_row_get(ip_plugin_rt->ex_data_rt, i); - Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger); - } - EX_data_rt_clear_row_cache(ip_plugin_rt->ex_data_rt); - Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt); - table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(ip_plugin_rt->ex_data_rt); - return 0; -} - int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) { struct scan_result results[size]; diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table_schema.cpp similarity index 83% rename from src/entry/Maat_table.cpp rename to src/entry/Maat_table_schema.cpp index 3d8615b..0b68e29 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table_schema.cpp @@ -1,4 +1,4 @@ -#include "Maat_table.h" +#include "Maat_table_schema.h" #include "map_str2int.h" #include "Maat_utils.h" #include "cJSON.h" @@ -49,6 +49,9 @@ enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type) case TABLE_TYPE_FQDN_PLUGIN: ret=SCAN_TYPE_FQDN_PLUGIN; break; + case TABLE_TYPE_BOOL_PLUGIN: + ret=SCAN_TYPE_BOOL_PLUGIN; + break; case TABLE_TYPE_COMPILE: ret=SCAN_TYPE_NONE; break; @@ -140,7 +143,8 @@ int _read_integer_arrary(char* string, int *array, int size) } #define COLUMN_PLUGIN_SCHEMA_JSON 4 #define COLUMN_IP_PLUGIN_SCHEMA_JSON 4 -#define COLUMN_FQDN_PLUGIN_SHCEMA_JSON 4 +#define COLUMN_FQDN_PLUGIN_SCHEMA_JSON 4 +#define COLUMN_BOOL_PLUGIN_SCHEMA_JSON 4 #define COLUMN_COMPOSITION_SCHEMA_JSON 4 #define COLUMN_VIRUTAL_SCHEMA_JSON 4 int read_virtual_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* table, MESA_htable_handle reserved_word_map) @@ -387,7 +391,7 @@ int read_fqdn_plugin_table_schema(const char* line, struct Maat_table_schema* p) struct fqdn_plugin_table_schema* fqdn_plugin_schema=&(p->fqdn_plugin); copy_line=_maat_strdup(line); - ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SHCEMA_JSON, &offset, &len); + ret=get_column_pos(copy_line, COLUMN_FQDN_PLUGIN_SCHEMA_JSON, &offset, &len); if(ret<0) { goto error_out; @@ -454,6 +458,76 @@ error_out: free(copy_line); return -1; +} +int read_bool_plugin_table_schema(const char* line, struct Maat_table_schema* p) +{ + int ret=0, read_cnt=0; + size_t offset=0, len=0; + cJSON* json=NULL, *tmp=NULL; + char* copy_line=NULL, *schema_json=NULL; + struct bool_plugin_table_schema* bool_plugin_schema=&(p->bool_plugin); + + copy_line=_maat_strdup(line); + ret=get_column_pos(copy_line, COLUMN_BOOL_PLUGIN_SCHEMA_JSON, &offset, &len); + if(ret<0) + { + goto error_out; + } + if(offset+lentype==cJSON_Number) + { + bool_plugin_schema->row_id_column=tmp->valueint; + read_cnt++; + } + tmp=cJSON_GetObjectItem(json, "bool_expr"); + if(tmp!=NULL && tmp->type==cJSON_Number) + { + bool_plugin_schema->bool_expr_column=tmp->valueint; + read_cnt++; + } + + tmp=cJSON_GetObjectItem(json, "valid"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + bool_plugin_schema->valid_flag_column=tmp->valueint; + read_cnt++; + } + bool_plugin_schema->rule_tag_column=-1; + tmp=cJSON_GetObjectItem(json, "tag"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + bool_plugin_schema->rule_tag_column=tmp->valueint; + //read_cnt++; Tag is optional, so NOT ++ intentionally. + } + + cJSON_Delete(json); + + free(copy_line); + if(read_cnt<4) + { + return -1; + } + else + { + return 0; + } +error_out: + free(copy_line); + return -1; + } int read_composition_table_schema(struct Maat_table_manager* table_mgr, const char* line, struct Maat_table_schema* p, MESA_htable_handle string2int_map) @@ -582,6 +656,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); + maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN); maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL); maat_kv_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL); maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); @@ -687,6 +762,16 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path goto invalid_table; } break; + case TABLE_TYPE_BOOL_PLUGIN: + ret=read_bool_plugin_table_schema(line, p); + if(ret<0) + { + fprintf(stderr, "Maat read table info %s line %d error: illegal bool_plugin table schema.\n", table_info_path, i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Maat read table info %s line %d error: illegal bool_plugin table schema.", table_info_path, i); + goto invalid_table; + } + break; case TABLE_TYPE_COMPOSITION: ret=read_composition_table_schema(table_mgr, line, p, reserved_word_map); if(ret<0) @@ -778,7 +863,7 @@ size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr) { return table_mgr->table_cnt; } -int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +int Maat_table_manager_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) { int i=0; for(i=0; i< MAX_TABLE_NUM; i++) @@ -791,7 +876,7 @@ int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char } return 0; } -int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +int Maat_table_manager_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) { int i=0; for(i=0; i< MAX_TABLE_NUM; i++) @@ -804,7 +889,7 @@ int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr } return 0; } -int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +int Maat_table_manager_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) { int i=0; for(i=0; i< MAX_TABLE_NUM; i++) @@ -817,20 +902,7 @@ int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, } return 0; } - -const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id) -{ - if(table_id>MAX_TABLE_NUM) - { - return NULL; - } - if(table_mgr->p_table_info[table_id]) - { - return table_mgr->p_table_info[table_id]->table_name[0]; - } - return NULL; -} -enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id) +enum MAAT_TABLE_TYPE Maat_table_manager_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id) { if(table_id>MAX_TABLE_NUM) { @@ -843,17 +915,8 @@ enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_ return TABLE_TYPE_INVALID; } -struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id) -{ - if(table_id>MAX_TABLE_NUM||table_id<0) - { - return NULL; - } - return table_mgr->p_table_info[table_id]; -} - -struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id) +struct Maat_table_schema * Maat_table_manager_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id) { enum MAAT_SCAN_TYPE tab_scan_type; struct Maat_table_schema **p_table_info=table_mgr->p_table_info; @@ -890,7 +953,7 @@ struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* } return p_physical_table; } -int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name) +int Maat_table_manager_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name) { int table_id=-1,ret=0; ret=maat_kv_read(table_mgr->map_tablename2id, table_name, &table_id); @@ -903,7 +966,7 @@ int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* return -1; } } -int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, +int Maat_table_manager_add_callback_func(struct Maat_table_manager* table_mgr, int table_id, Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para Maat_update_callback_t *update,//table line ,u_para @@ -911,7 +974,7 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, void* u_para) { int idx=0; - struct Maat_table_schema *p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); + struct Maat_table_schema *p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_PLUGIN, NULL); struct plugin_table_schema *plugin_desc=&(p_table->plugin); if(p_table==NULL) { @@ -931,17 +994,17 @@ int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, return 1; } -struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx) +struct compile_ex_data_idx* Maat_table_manager_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx) { int table_id=-1; struct Maat_table_schema *p_table=NULL; - table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name); + table_id=Maat_table_manager_get_id_by_name(table_mgr, compile_table_name); if(table_id<0) { return NULL; } - p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); + p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); if(!p_table) { return NULL; @@ -953,7 +1016,7 @@ struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_tabl return NULL; } -int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, +int Maat_table_manager_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, Maat_rule_EX_new_func_t *new_func, Maat_rule_EX_free_func_t* free_func, Maat_rule_EX_dup_func_t* dup_func, @@ -961,12 +1024,12 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c { int table_id=-1; struct Maat_table_schema *p_table=NULL; - table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name); + table_id=Maat_table_manager_get_id_by_name(table_mgr, compile_table_name); if(table_id<0) { return -1; } - p_table=Maat_table_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); + p_table=Maat_table_manager_get_scan_by_id(table_mgr, table_id, SCAN_TYPE_NONE, NULL); if(!p_table) { return -1; @@ -991,122 +1054,66 @@ int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, c return idx; } -void Maat_table_EX_data_schema_set(struct EX_data_schema* ex_schema, +struct EX_data_schema *Maat_table_schema_get_EX_data_schema(struct Maat_table_schema *table_schema) +{ + struct EX_data_schema *ex_schema=NULL; + switch(table_schema->table_type) + { + case TABLE_TYPE_PLUGIN: + ex_schema=&table_schema->plugin.ex_schema; + break; + case TABLE_TYPE_IP_PLUGIN: + ex_schema=&table_schema->ip_plugin.ex_schema; + break; + case TABLE_TYPE_FQDN_PLUGIN: + ex_schema=&table_schema->fqdn_plugin.ex_schema; + break; + case TABLE_TYPE_BOOL_PLUGIN: + ex_schema=&table_schema->bool_plugin.ex_schema; + break; + default: + break; + } + return ex_schema; +} + +int Maat_table_schema_set_EX_data_schema(struct Maat_table_schema *table_schema, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, Maat_plugin_EX_dup_func_t* dup_func, Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp) + long argl, void *argp, + void* logger) { + struct EX_data_schema *ex_schema=NULL; + if(new_func==NULL || free_func==NULL || dup_func==NULL ) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); + return -1; + } + ex_schema=Maat_table_schema_get_EX_data_schema(table_schema); + if(ex_schema==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a valid plugin table.", __FUNCTION__); + return -1; + } + if(ex_schema->set_flag) + { + assert(0); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data schema already registed.", __FUNCTION__); + return -1; + } ex_schema->new_func=new_func; ex_schema->free_func=free_func; ex_schema->dup_func=dup_func; ex_schema->key2index_func=key2index_func;//Set but not used. ex_schema->argl=argl; ex_schema->argp=argp; - return; -} -int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp, - void* logger) -{ - if(new_func==NULL || free_func==NULL || dup_func==NULL ) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); - return -1; - } - if(table_schema->table_type!=TABLE_TYPE_PLUGIN) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, Regist target is not a plugin table.", __FUNCTION__); - return -1; - } - if(table_schema->plugin.have_exdata) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__); - return -1; - } - if(table_schema->plugin.key_column==0 || table_schema->plugin.valid_flag_column==0) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, not enough schema information.", __FUNCTION__); - return -1; - } - Maat_table_EX_data_schema_set(&table_schema->plugin.ex_schema, - new_func, free_func, dup_func, key2index_func, argl, argp); - table_schema->plugin.have_exdata=1; + ex_schema->set_flag=1; return 0; -} -int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp, - void* logger) -{ - if(new_func==NULL || free_func==NULL || dup_func==NULL ) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); - return -1; - } - if(table_schema->table_type!=TABLE_TYPE_IP_PLUGIN) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a ip_plugin table.", __FUNCTION__); - return -1; - } - if(table_schema->ip_plugin.have_exdata) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__); - return -1; - } - Maat_table_EX_data_schema_set(&table_schema->ip_plugin.ex_schema, - new_func, free_func, dup_func, key2index_func, argl, argp); - table_schema->ip_plugin.have_exdata=1; - return 0; -} -int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp, - void* logger) -{ - if(new_func==NULL || free_func==NULL || dup_func==NULL ) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); - return -1; - } - if(table_schema->table_type!=TABLE_TYPE_FQDN_PLUGIN) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, target table is not a fqdn_plugin table.", __FUNCTION__); - return -1; - } - if(table_schema->fqdn_plugin.have_exdata) - { - assert(0); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Error: %s, EX data already registed.", __FUNCTION__); - return -1; - } - Maat_table_EX_data_schema_set(&table_schema->fqdn_plugin.ex_schema, - new_func, free_func, dup_func, key2index_func, argl, argp); - table_schema->fqdn_plugin.have_exdata=1; - return 0; -} - +} void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type) { table_mgr->active_plugin_table_num=0; @@ -1178,11 +1185,11 @@ int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* { return table_mgr->is_last_plugin_table_updating; } -struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name) +struct Maat_table_schema* Maat_table_manager_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name) { struct Maat_table_schema * p_table=NULL; int table_id=0; - table_id=Maat_table_get_id_by_name(table_mgr, table_name); + table_id=Maat_table_manager_get_id_by_name(table_mgr, table_name); if(table_id<0) { return NULL; @@ -1190,7 +1197,7 @@ struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* p_table=table_mgr->p_table_info[table_id]; return p_table; } -void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* table_name) +void Maat_table_schema_set_updating_name(struct Maat_table_schema* p_table, const char* table_name) { int i=0; for(i=0; iconj_cnt; i++) @@ -1202,11 +1209,20 @@ void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* } assert(i<=p_table->conj_cnt); } +struct Maat_table_schema * Maat_table_manager_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM||table_id<0) + { + return NULL; + } -int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type) + return table_mgr->p_table_info[table_id]; +} + +int Maat_table_manager_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type) { int ret=-1; - struct Maat_table_schema* p_table=Maat_table_get_by_id_raw(table_mgr, parent_table_id); + struct Maat_table_schema* p_table=Maat_table_manager_get_by_id_raw(table_mgr, parent_table_id); if(p_table->table_type!=TABLE_TYPE_COMPOSITION) { return -1; @@ -1214,7 +1230,7 @@ int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_tab ret=p_table->composition.component_table_id[type]; return ret; } -int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table) +int Maat_table_schema_get_valid_flag_column(struct Maat_table_schema* p_table) { int valid_flag_column=-1; switch(p_table->table_type) @@ -1228,6 +1244,9 @@ int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p case TABLE_TYPE_FQDN_PLUGIN: valid_flag_column=p_table->fqdn_plugin.valid_flag_column; break; + case TABLE_TYPE_BOOL_PLUGIN: + valid_flag_column=p_table->bool_plugin.valid_flag_column; + break; default: valid_flag_column=-1; break; diff --git a/src/entry/bool_matcher.cpp b/src/entry/bool_matcher.cpp index cabc552..c3992c3 100644 --- a/src/entry/bool_matcher.cpp +++ b/src/entry/bool_matcher.cpp @@ -31,9 +31,9 @@ bool operator<(const struct bool_item & lhs, const struct bool_item & rhs) return lhs.item_id diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h index 0fbc0de..4eca371 100644 --- a/src/inc_internal/Maat_table_runtime.h +++ b/src/inc_internal/Maat_table_runtime.h @@ -1,8 +1,9 @@ -#include "Maat_table.h" +#include "Maat_table_schema.h" #include "Maat_ex_data.h" #include "IPMatcher.h" #include "gram_index_engine.h" +#include "bool_matcher.h" #include "FQDN_engine.h" #include "alignment_int64.h" #include @@ -18,6 +19,12 @@ struct fqdn_plugin_runtime struct EX_data_rt* ex_data_rt; int changed_flag; }; +struct bool_plugin_runtime +{ + struct bool_matcher *matcher; + struct EX_data_rt* ex_data_rt; + int changed_flag; +}; struct plugin_runtime { struct EX_data_rt* ex_data_rt; @@ -54,6 +61,7 @@ struct Maat_table_runtime { struct similar_runtime similar; //for digest and similarity struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin + struct bool_plugin_runtime bool_plugin; struct plugin_runtime plugin; struct ip_plugin_runtime ip_plugin; struct expr_runtime expr; @@ -80,24 +88,25 @@ enum MAAT_TABLE_TYPE Maat_table_runtime_get_type(struct Maat_table_runtime* tabl size_t Maat_table_runtime_plugin_cached_row_count(struct Maat_table_runtime* table_rt); const char* Maat_table_runtime_plugin_get_cached_row(struct Maat_table_runtime* table_rt, size_t Nth_row); -int Maat_table_runtime_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, void* logger); -MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key); +int Maat_table_runtime_commit_EX_data_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger); void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag); void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id); int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt); void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); -void Maat_table_runtime_plugin_update_commit(struct Maat_table_runtime* table_rt); +void Maat_table_runtime_plugin_commit_update(struct Maat_table_runtime* table_rt); +MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_desc, const char* key); void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); - -int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger); int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); -int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt); +int Maat_table_runtime_ip_plugin_commit_update(struct Maat_table_runtime* table_rt); void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); -int Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(struct Maat_table_runtime* table_rt); -int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger); +int Maat_table_runtime_fqdn_plugin_commit_update(struct Maat_table_runtime* table_rt); int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); +void Maat_table_runtime_bool_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); +int Maat_table_runtime_bool_plugin_commit_update(struct Maat_table_runtime* table_rt); +int Maat_table_runtime_bool_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, unsigned long long item_id[], size_t n_item, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size); + diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table_schema.h similarity index 68% rename from src/inc_internal/Maat_table.h rename to src/inc_internal/Maat_table_schema.h index c34a1de..05f7daa 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table_schema.h @@ -24,6 +24,7 @@ enum MAAT_SCAN_TYPE SCAN_TYPE_PLUGIN, SCAN_TYPE_IP_PLUGIN, SCAN_TYPE_FQDN_PLUGIN, + SCAN_TYPE_BOOL_PLUGIN, SCAN_TYPE_IP, SCAN_TYPE_INTERVAL, SCAN_TYPE_STRING, @@ -60,7 +61,8 @@ enum MAAT_TABLE_TYPE TABLE_TYPE_COMPILE, TABLE_TYPE_PLUGIN, TABLE_TYPE_IP_PLUGIN, - TABLE_TYPE_FQDN_PLUGIN + TABLE_TYPE_FQDN_PLUGIN, + TABLE_TYPE_BOOL_PLUGIN }; struct compile_ex_data_idx @@ -113,6 +115,7 @@ struct EX_data_schema Maat_plugin_EX_key2index_func_t* key2index_func; long argl; void *argp; + int set_flag; }; struct plugin_table_schema { @@ -122,7 +125,6 @@ struct plugin_table_schema int n_foreign; int foreign_columns[MAX_FOREIGN_CLMN_NUM]; int cb_plug_cnt; - int have_exdata; struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE]; struct EX_data_schema ex_schema; }; @@ -148,6 +150,16 @@ struct fqdn_plugin_table_schema int have_exdata; struct EX_data_schema ex_schema; }; +struct bool_plugin_table_schema +{ + int row_id_column; + int bool_expr_column; + int rule_tag_column; + int valid_flag_column; + int have_exdata; + struct EX_data_schema ex_schema; + +}; struct Maat_table_schema { int table_id; @@ -162,6 +174,7 @@ struct Maat_table_schema struct plugin_table_schema plugin; struct ip_plugin_table_schema ip_plugin; struct fqdn_plugin_table_schema fqdn_plugin; + struct bool_plugin_table_schema bool_plugin; struct virtual_table_schema virtual_table; struct composition_table_schema composition; void* others;//group, ip, interval and digest don't have any special schema. @@ -177,59 +190,47 @@ void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr); size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr); size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr); -struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id); -struct Maat_table_schema * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id); +struct Maat_table_schema * Maat_table_manager_get_scan_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_SCAN_TYPE scan_type, int* virutal_table_id); +struct Maat_table_schema * Maat_table_manager_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id); enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type); -int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name); -int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, +int Maat_table_manager_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name); +int Maat_table_manager_add_callback_func(struct Maat_table_manager* table_mgr, int table_id, Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para Maat_update_callback_t *update,//table line ,u_para Maat_finish_callback_t *finish,//u_para void* u_para); -int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); -int Maat_table_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); -int Maat_table_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +int Maat_table_manager_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +int Maat_table_manager_get_group2compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +int Maat_table_manager_get_group2group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); -const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id); -enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id); +const char* Maat_table_manager_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id); +enum MAAT_TABLE_TYPE Maat_table_manager_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id); -int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, +int Maat_table_manager_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, Maat_rule_EX_new_func_t *new_func, Maat_rule_EX_free_func_t* free_func, Maat_rule_EX_dup_func_t* dup_func, long argl, void *argp); -struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx); -int Maat_table_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp, - void* logger); -int Maat_table_ip_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, - Maat_plugin_EX_new_func_t* new_func, - Maat_plugin_EX_free_func_t* free_func, - Maat_plugin_EX_dup_func_t* dup_func, - Maat_plugin_EX_key2index_func_t* key2index_func, - long argl, void *argp, - void* logger); -int Maat_table_fqdn_plugin_EX_data_schema_set(struct Maat_table_schema *table_schema, +struct compile_ex_data_idx* Maat_table_manager_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx); +int Maat_table_schema_set_EX_data_schema(struct Maat_table_schema *table_schema, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, Maat_plugin_EX_dup_func_t* dup_func, Maat_plugin_EX_key2index_func_t* key2index_func, long argl, void *argp, void* logger); +struct EX_data_schema *Maat_table_schema_get_EX_data_schema(struct Maat_table_schema *table_schema); + void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type); void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr); int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr); -struct Maat_table_schema* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name); -void Maat_table_set_updating_name(struct Maat_table_schema* p_table, const char* table_name); +struct Maat_table_schema* Maat_table_manager_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name); +void Maat_table_schema_set_updating_name(struct Maat_table_schema* p_table, const char* table_name); -int Maat_table_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type); -int Maat_table_xx_plugin_table_get_valid_flag_column(struct Maat_table_schema* p_table); +int Maat_table_manager_get_child_id(struct Maat_table_manager* table_mgr, int parent_table_id, enum MAAT_TABLE_COMPONENT_TYPE type); +int Maat_table_schema_get_valid_flag_column(struct Maat_table_schema* p_table); diff --git a/test/perf_test_maatframe.cpp b/test/perf_test_maatframe.cpp index e7d7721..8a16dd5 100644 --- a/test/perf_test_maatframe.cpp +++ b/test/perf_test_maatframe.cpp @@ -71,7 +71,7 @@ TEST(BoolMatcher, Match) i++; } size_t mem_size=0; - bm=bool_matcher_new(expr_array, expr_num, 4, &mem_size); + bm=bool_matcher_new(expr_array, expr_num, &mem_size); unsigned long long test_count=2*1000*1000, match_count=0, unmatch_count=0; long int j=0; size_t k=0; @@ -97,7 +97,7 @@ TEST(BoolMatcher, Match) input_item_num++; } sort(input_item_ids, input_item_ids+input_item_num); - ret=bool_matcher_match(bm, 1, input_item_ids, input_item_num, result_array, 1024); + ret=bool_matcher_match(bm, input_item_ids, input_item_num, result_array, 1024); if(ret>0) { match_count++; @@ -124,7 +124,7 @@ TEST(BoolMatcher, Match) sort(input_item_ids, input_item_ids+input_item_num); for(i=0; i