diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index abce127..1ef76dd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 494055c..abcc421 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -16,42 +16,7 @@ #include "rulescan.h" #include "json2iris.h" -struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id) -{ - struct Maat_table_desc *p_table=NULL, *p_real_table=NULL; - if(table_id>MAX_TABLE_NUM) - { - return NULL; - } - if(_feather->p_table_info[table_id]==NULL) - { - return NULL; - } - p_table=_feather->p_table_info[table_id]; - if(p_table==NULL) - { - return NULL; - } - if(p_table->table_type==TABLE_TYPE_VIRTUAL) - { - p_real_table=_feather->p_table_info[p_table->virtual_table.real_table_id]; - *virutal_table_id=table_id; - } - else - { - p_real_table=p_table; - *virutal_table_id=0; - } - if(p_real_table->table_type!=expect_type) - { - if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| - (expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS)) - { - return NULL; - } - } - return p_real_table; -} + inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num) { alignment_int64_array_add(scanner->ref_cnt, thread_num, 1); @@ -547,19 +512,14 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* return NULL; } _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); - feather->table_cnt=Maat_table_read_table_info(feather->p_table_info, MAX_TABLE_NUM, table_info_path, logger); - if(feather->table_cnt==0) - { - goto failed; - } - feather->map_tablename2id=Maat_table_build(feather->p_table_info, MAX_TABLE_NUM, - feather->compile_tn, sizeof(feather->compile_tn), - feather->group_tn, sizeof(feather->group_tn), - logger); - if(feather->map_tablename2id==NULL) + feather->table_mgr=Maat_table_manager_create(table_info_path, logger); + if(feather->table_mgr==NULL) { goto failed; } + Maat_table_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn)); + Maat_table_get_group_table_name(feather->table_mgr, feather->group_tn, sizeof(feather->group_tn)); + feather->logger=logger; feather->scan_thread_num=max_thread_num; feather->garbage_q=MESA_lqueue_create(0,0); @@ -994,16 +954,7 @@ void Maat_burn_feather(Maat_feather_t feather) int Maat_table_register(Maat_feather_t feather,const char* table_name) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int table_id=-1,ret=0; - ret=map_str2int(_feather->map_tablename2id, table_name, &table_id); - if(ret>0) - { - return table_id; - } - else - { - return -1; - } + return Maat_table_get_id_by_name(_feather->table_mgr, table_name); } int Maat_table_callback_register(Maat_feather_t feather,short table_id, Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para @@ -1012,52 +963,35 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, void* u_para) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int idx=0,i=0; - struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; - struct plugin_table_desc *plugin_desc=&(p_table->plugin); - struct plugin_runtime* plugin_aux=NULL; - const char* lines=NULL; - if(p_table==NULL) - { - return -1; - } - if(p_table->table_type!=TABLE_TYPE_PLUGIN) - { - return -1; - } - //plugin table register blocks background update. + int i=0,ret=0; + pthread_mutex_lock(&(_feather->background_update_mutex)); - idx=plugin_desc->cb_plug_cnt; - if(idx==MAX_PLUGIN_PER_TABLE) + ret=Maat_table_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para); + if(ret<0) { pthread_mutex_unlock(&(_feather->background_update_mutex)); return -1; } - plugin_desc->cb_plug_cnt++; - plugin_desc->cb_plug[idx].start=start; - plugin_desc->cb_plug[idx].update=update; - plugin_desc->cb_plug[idx].finish=finish; - plugin_desc->cb_plug[idx].u_para=u_para; - if(_feather->scanner==NULL) - { - pthread_mutex_unlock(&(_feather->background_update_mutex)); - return 1; - } - plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin); - if(plugin_aux->cache_line_num>0) + + const char* line=NULL; + struct Maat_table_runtime* table_rt=NULL; + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + long long line_cnt=Maat_table_runtime_plugin_cached_line_count(table_rt); + + if(line_cnt>0) { if(start!=NULL) { start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); } - for(i=0;icache_line_num;i++) + for(i=0; icache_lines,i); - if(lines==NULL) + line=Maat_table_runtime_plugin_get_cached_line(table_rt, i); + if(line==NULL) { break; } - update(table_id,lines,u_para); + update(table_id,line,u_para); } if(finish!=NULL) { @@ -1095,39 +1029,31 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table long argl, void *argp) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int table_id=-1,ret=0, idx=-1; - ret=map_str2int(_feather->map_tablename2id, compile_table_name, &table_id); - if(ret<0) + int idx=-1; + + if(new_func==NULL || free_func==NULL || dup_func==NULL) { return -1; } - struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; - if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL || dup_func==NULL) - { - return -1; - } - struct compile_table_desc* compile_desc=&(p_table->compile); pthread_mutex_lock(&(_feather->background_update_mutex)); - if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM) - { - ret=-1; - goto failed; - } - idx=compile_desc->ex_data_num; - compile_desc->ex_desc[idx].idx=idx; - compile_desc->ex_desc[idx].table_id=table_id; - compile_desc->ex_desc[idx].argl=argl; - compile_desc->ex_desc[idx].argp=argp; - compile_desc->ex_desc[idx].new_func=new_func; - compile_desc->ex_desc[idx].free_func=free_func; - compile_desc->ex_desc[idx].dup_func=dup_func; + idx=Maat_table_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name, + new_func, + free_func, + dup_func, + argl, argp); - compile_desc->ex_data_num++; + if(idx<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + struct compile_ex_data_idx* compile_ex_desc=Maat_table_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx); + if(_feather->scanner!=NULL) { - MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_desc->ex_desc+idx); + MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_ex_desc); } -failed: + pthread_mutex_unlock(&(_feather->background_update_mutex)); return idx; @@ -1155,129 +1081,7 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa pthread_rwlock_unlock(&(relation->rwlock)); return ad; } -struct wrap_plugin_EX_data -{ - MAAT_RULE_EX_DATA exdata; - const struct Maat_table_desc* ref_plugin_table; -}; -void wrap_plugin_EX_data_free(void *data) -{ - struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; - const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); - ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); - wrap_data->ref_plugin_table=NULL; - free(wrap_data); - return; -} -MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) -{ - MESA_htable_handle key2ex_hash=NULL; - unsigned int slot_size=1; - while(estimate_size!=0) - { - estimate_size=estimate_size>>1; - slot_size*=2; - } - if(slot_size==1) - { - slot_size=4096; - } - MESA_htable_create_args_t hargs; - memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=8; - hargs.hash_slot_size = slot_size; - hargs.max_elem_num = 0; - hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; - hargs.expire_time = 0; - hargs.key_comp = NULL; - hargs.key2index = NULL; //Not supported yet. - hargs.recursive = 1; - hargs.data_free = wrap_plugin_EX_data_free; - hargs.data_expire_with_condition = NULL; - key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(key2ex_hash, 0); - return key2ex_hash; -} - -int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, - MESA_htable_handle key2ex_hash, void *logger) -{ - size_t key_offset=0, key_len=0; - const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); - int ret=0; - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data del error: cannot find column %d of %s", - plugin_desc->key_column, line); - return -1; - } - ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data del error: no such key %.*s of %s", - key_len, line+key_offset, line); - return -1; - } - return 0; -} - -int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, - MESA_htable_handle key2ex_hash, void *logger) -{ - char* key=NULL; - size_t key_offset=0, key_len=0; - MAAT_RULE_EX_DATA exdata=NULL; - struct wrap_plugin_EX_data* wrap_data=NULL; - const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); - int ret=0; - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data add error: cannot find column %d of %s", - plugin_desc->key_column, line); - return -1; - } - key=ALLOC(char, key_len+1); - memcpy(key, line+key_offset, key_len); - plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, - plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); - wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); - wrap_data->exdata=exdata; - wrap_data->ref_plugin_table=plugin_table; - ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); - free(key); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data add error: duplicated key %.*s of %s", - key_len, line+key_offset, line); - wrap_plugin_EX_data_free(wrap_data); - return -1; - } - return 0; -} -MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table, - struct dynamic_array_t* lines, size_t line_cnt, void* logger) -{ - MESA_htable_handle key2ex_hash=NULL; - size_t i=0; - const char* line=NULL; - const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); - - key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); - - for(i=0; i< line_cnt; i++) - { - line=(const char*)dynamic_array_read(lines, i); - plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); - } - return key2ex_hash; -} int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, @@ -1287,39 +1091,35 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; - struct plugin_table_desc* plugin_desc=&(table_desc->plugin); - struct Maat_table_runtime* table_rt=NULL; + int idx=-1; + if(new_func==NULL || free_func==NULL || dup_func==NULL ) { assert(0); MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); return -1; } - if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata - || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) - { - assert(0); - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, - "%s failed: key or valid flag column are not specified", __FUNCTION__); - return -1; - } - pthread_mutex_lock(&(_feather->background_update_mutex)); - plugin_desc->ex_desc.new_func=new_func; - plugin_desc->ex_desc.free_func=free_func; - plugin_desc->ex_desc.dup_func=dup_func; - plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. - plugin_desc->ex_desc.argl=argl; - plugin_desc->ex_desc.argp=argp; - plugin_desc->have_exdata=1; + idx=Maat_table_plugin_new_ex_index(_feather->table_mgr, table_id, + new_func, + free_func, + dup_func, + key2index_func, + argl, argp); + if(idx<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + struct Maat_table_desc *table_desc=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct Maat_table_runtime* table_rt=NULL; + + if(_feather->scanner!=NULL) { - table_rt=_feather->scanner->table_rt[table_id]; - assert(table_rt->plugin.key2ex_hash==NULL); - table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, - table_rt->plugin.cache_line_num, _feather->logger); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + Maat_table_runtime_plugin_new_ex_idx(table_rt, table_desc, _feather->logger); } pthread_mutex_unlock(&(_feather->background_update_mutex)); @@ -1328,28 +1128,16 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; - struct Maat_table_runtime *table_rt= NULL; - struct plugin_table_desc* plugin_desc=&(table_desc->plugin); - struct wrap_plugin_EX_data* wrap_data=NULL; + struct Maat_table_desc *table_desc=NULL; + struct Maat_table_runtime *table_rt=NULL; MAAT_RULE_EX_DATA exdata=NULL; - if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0) - { - assert(0); - return NULL; - } if(_feather->scanner==NULL) { return NULL; } - table_rt= _feather->scanner->table_rt[table_id]; - wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, - (const unsigned char*)key, strlen(key)); - if(wrap_data!=NULL) - { - plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata), - plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); - } + table_desc=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); return exdata; } @@ -1384,7 +1172,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } _mid=grab_mid(mid,_feather, thread_num, 0); - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1410,7 +1198,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id scan_data.text_data.tlen=data_len; scan_data.text_data.toffset=0; - struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -1518,7 +1306,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } int virutal_table_id=0; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1529,7 +1317,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -1609,7 +1397,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } int virtual_table_id=0; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_IP, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1620,7 +1408,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -1728,7 +1516,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct Maat_table_desc *p_table=NULL; int virtual_table_id=0; assert(thread_num<_feather->scan_thread_num); - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1748,7 +1536,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { return sp; } - struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id]; + + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); if(table_rt->origin_rule_num==0) { return sp; @@ -1796,7 +1585,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para { return 0; } - struct Maat_table_runtime* table_rt=scanner->table_rt[sp->p_real_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC, &start); @@ -1969,7 +1758,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) struct Maat_table_runtime* table_rt=NULL; if(scanner!=NULL) { - table_rt=scanner->table_rt[sp->p_real_table->table_id]; + table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); } @@ -2011,7 +1800,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sfh_instance_t * tmp_fuzzy_handle=NULL; struct Maat_table_desc *p_table=NULL; int virtual_table_id=0; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_DIGEST, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2028,7 +1817,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, { return sp; } - struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_id); tmp_fuzzy_handle=SFH_instance(total_len); if(tmp_fuzzy_handle==NULL) { @@ -2086,7 +1875,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return 0; } - struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id]; + struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id); GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; unsigned long long digest_len=0; char* digest_buff=NULL; @@ -2169,7 +1958,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct Maat_scanner* scanner=sp->feather->scanner; - struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->p_real_table->table_id]; + struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id); alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); if(scanner!=NULL) { @@ -2273,7 +2062,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } int virtual_table_id=0; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2284,7 +2073,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[p_table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -2356,7 +2145,7 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu } break; case MAAT_STATE_LAST_UPDATING_TABLE: - *int_val=_feather->is_last_plugin_table_updating; + *int_val=Maat_table_manager_is_last_plugin_table_updating(_feather->table_mgr); break; case MAAT_STATE_IN_UPDATING: if(size!=sizeof(int)) diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index a86d303..16d206b 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -915,7 +915,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ continue; } region_cmd=&(group_cmd->regions[j]); - region_cmd->table_name=_maat_strdup(feather->p_table_info[region_inner->table_id]->table_name[0]); + region_cmd->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, region_inner->table_id)); region_cmd->region_id=region_inner->region_id; //NOTICE: region_type only avilable when OP_ADD, region_cmd->region_type=REGION_EXPR; @@ -1374,7 +1374,7 @@ error_out: int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) { - int i=0,j=0,ret=0; + int i=0, j=0; const char *table_name=NULL; int table_id=0; struct Maat_group_t* p_group=NULL; @@ -1399,8 +1399,8 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) { p_region=&(p_group->regions[j]); table_name=p_region->table_name; - ret=map_str2int(feather->map_tablename2id, table_name, &table_id); - if(ret<0) + table_id=Maat_table_get_id_by_name(feather->table_mgr, table_name); + if(table_id<0) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module ,"Unknown table %s of Maat_cmd_t[%d]->group[%d]->region[%d]." @@ -1409,7 +1409,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) return -1; } table_type=type_region2table(p_region); - if(table_type!=feather->p_table_info[table_id]->table_type) + if(table_type!=Maat_table_get_type_by_id(feather->table_mgr, table_id)) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module ,"Table %s not support region type %d of Maat_cmd_t[%d]->group[%d]->region[%d]." @@ -1419,7 +1419,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) return -1; } free((char*)p_region->table_name); - p_region->table_name=_maat_strdup(feather->p_table_info[table_id]->table_name[0]); + p_region->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, table_id)); } } return 0; @@ -1651,7 +1651,7 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n } int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger) { - int ret=0, table_id=0, i=0; + int i=0; int rule_with_foreign_key=0; struct Maat_table_desc* p_table=NULL; struct plugin_table_desc* plugin_desc=NULL; @@ -1661,14 +1661,13 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, { continue; } - ret=map_str2int(feather->map_tablename2id, rule_list[i].table_name, &table_id); - if(ret<0) + p_table=Maat_table_get_desc_by_name(feather->table_mgr, rule_list[i].table_name); + if(!p_table||p_table->table_type!=TABLE_TYPE_PLUGIN) { continue; } - p_table=feather->p_table_info[table_id]; plugin_desc= &(p_table->plugin); - if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0) + if(plugin_desc->n_foreign==0) { continue; } @@ -1808,6 +1807,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m int update_type=CM_UPDATE_TYPE_INC; long long new_version=0; enum MAAT_TABLE_TYPE table_type; + struct Maat_table_desc* table_desc=NULL; const struct plugin_table_desc* plugin_desc=NULL; void* logger=feather->logger; @@ -1889,17 +1889,18 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m { continue; } - ret=map_str2int(feather->map_tablename2id,rule_list[i].table_name,&table_id); - if(ret<0)//Unrecognized table. + table_id=Maat_table_get_id_by_name(feather->table_mgr, rule_list[i].table_name); + if(table_id<0)//Unrecognized table. { continue; } - table_type=feather->p_table_info[table_id]->table_type; + table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id); if(rule_list[i].op==MAAT_OP_DEL) { if(table_type==TABLE_TYPE_PLUGIN) { - plugin_desc=&(feather->p_table_info[table_id]->plugin); + table_desc=Maat_table_get_by_id(feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + plugin_desc=&(table_desc->plugin); valid_column=plugin_desc->valid_flag_column; } else @@ -2105,8 +2106,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),line_num); for(i=0;imap_tablename2id, line_rule[i]->table_name, &table_id); - if(ret<0) + table_id=Maat_table_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name); + if(table_id<0) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command ,"Command set line id %d failed: unknown table %s." @@ -2115,9 +2116,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru ret=-1; goto error_out; } - p_table=_feather->p_table_info[table_id]; - plugin_desc=&(p_table->plugin); - if(TABLE_TYPE_PLUGIN!=p_table->table_type) + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + if(!p_table) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command ,"Command set line id %d failed: table %s is not a plugin table." @@ -2126,6 +2126,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru ret=-1; goto error_out; } + plugin_desc=&(p_table->plugin); if(op==MAAT_OP_ADD) { ret=get_valid_flag_offset(line_rule[i]->table_line diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 7f091f5..7c12f23 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -479,11 +480,7 @@ error_out: cJSON_Delete(json); return ret; } -int lqueue_destroy_cb(void *data, long data_len, void *arg) -{ - assert(0); - return 0; -} + void * HASH_fetch_by_id(MESA_htable_handle hash,int id) { return MESA_htable_search(hash,(unsigned char*)&(id),sizeof(id)); @@ -938,127 +935,10 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) op_expr->rule_type=p_rule->rule_type; return; } -GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, - short cfds_lvl,struct Maat_group_inner* tag) -{ - GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); - int digest_len=0; - rule->id=id; - rule->operation=op; - if(digest!=NULL) - { - digest_len=strlen(digest); - rule->sfh=(char*)calloc(sizeof(char),digest_len+1); - memcpy(rule->sfh,digest,digest_len); - - } - rule->sfh_length=digest_len; - rule->cfds_lvl=cfds_lvl; - rule->tag=(void*)tag; - return rule; -} -void destroy_digest_rule(GIE_digest_t*rule) -{ - if(rule->sfh!=NULL) - { - free(rule->sfh); - rule->sfh=NULL; - } - free(rule); - rule=NULL; - return; -} - - -struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num) -{ - - struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); - table_rt->table_type=table_desc->table_type; - switch(table_desc->table_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - table_rt->similar.update_q=MESA_lqueue_create(0,0); - break; - case TABLE_TYPE_PLUGIN: - table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); - if(table_desc->plugin.have_exdata) - { - table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, - table_desc->plugin.ex_desc.key2index_func); - } - break; - default: - break; - } - - table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); - table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); - table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); - table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); - table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); - return table_rt; -} -void table_runtime_free(struct Maat_table_runtime* p) -{ - long q_cnt=0,data_size=0; - int i=0; - UNUSED int q_ret=0; - - GIE_digest_t* digest_rule=NULL; - if(p==NULL) - { - return; - } - switch(p->table_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - if(p->similar.gie_handle!=NULL) - { - GIE_destory(p->similar.gie_handle); - } - if(p->similar.update_q!=NULL) - { - q_cnt=MESA_lqueue_get_count(p->similar.update_q); - for(i=0;isimilar.update_q,&digest_rule,&data_size); - assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); - destroy_digest_rule(digest_rule); - } - MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); - } - break; - case TABLE_TYPE_PLUGIN: - dynamic_array_destroy(p->plugin.cache_lines, free); - p->plugin.cache_lines=NULL; - if(p->plugin.key2ex_hash!=NULL) - { - MESA_htable_destroy(p->plugin.key2ex_hash, NULL); - } - default: - break; - } - - alignment_int64_array_free(p->scan_cnt); - alignment_int64_array_free(p->scan_cpu_time); - alignment_int64_array_free(p->input_bytes); - alignment_int64_array_free(p->stream_num); - alignment_int64_array_free(p->hit_cnt); - free(p); - return; -} struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *feather) { int scan_thread_num=feather->scan_thread_num; -// int rs_scan_type=feather->rule_scan_type; - struct Maat_table_desc ** pp_table_desc=feather->p_table_info; - struct Maat_table_runtime* table_rt=NULL; - int i=0; UNUSED int ret=0; MESA_htable_create_args_t hargs; @@ -1115,8 +995,8 @@ struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *f scanner->region_update_q=MESA_lqueue_create(0, 0); scanner->region=rulescan_initialize(scan_thread_num); - //For best performance test: - //1.Do NOT set this option,rulescan return no hit detail as default; + //For best scan performance: + //1.Do NOT set this option, rulescan return no hit detail as default; //2.Set necessary STR rule to QUICK; if(feather->rule_scan_type==1) { @@ -1130,16 +1010,8 @@ struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *f scanner->tomb_ref=feather->garbage_q; scanner->logger_ref=feather->logger; scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); - - for(i=0;iscan_thread_num); - scanner->table_rt[i]=table_rt; - } + scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num); + scanner->max_table_num=Maat_table_manager_get_size(feather->table_mgr); return scanner; } @@ -1189,10 +1061,9 @@ void destroy_maat_scanner(struct Maat_scanner*scanner) } } } - for(i=0;itable_rt[i]); - } + Maat_table_rt_manager_destroy(scanner->table_rt_mgr); + scanner->table_rt_mgr=NULL; + igraph_destroy(&scanner->group_graph); free(scanner); return; @@ -1235,9 +1106,8 @@ struct _region_stat_t int ipv6_rule_cnt; }; }; -void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, int size) +void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, size_t size) { - assert(op_expr->table_idp_expr->operation==0)//add { @@ -1277,7 +1147,7 @@ void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_sta void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner* maat_scanner) { - long i=0,data_size=0; + long data_size=0, i=0; unsigned int j=0; int ret=0; unsigned int failed_ids[MAX_FAILED_NUM]; @@ -1288,7 +1158,8 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, const long q_cnt=MESA_lqueue_get_count(expr_queue); struct timespec start,end; unsigned long long update_interval=0; - struct _region_stat_t region_counter[MAX_TABLE_NUM]; + size_t max_table_num=maat_scanner->max_table_num; + struct _region_stat_t region_counter[max_table_num]; memset(region_counter, 0, sizeof(region_counter)); struct Maat_table_runtime* table_rt=NULL; if(q_cnt==0) @@ -1317,7 +1188,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, } } - count_rs_region(op_expr,region_counter,MAX_TABLE_NUM); + count_rs_region(op_expr, region_counter, max_table_num); destroy_op_expr(op_expr); op_expr=NULL; } @@ -1344,9 +1215,9 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, ,(double)update_interval ,update_interval); //update scanner's region cnt; - for(i=0;itable_rt[i]; + table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, i); if(table_rt==NULL) { continue; @@ -1384,52 +1255,6 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, free(to_update_expr); } -void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct Maat_scanner* maat_scanner,int table_id) -{ - long i=0,data_size=0; - int ret=0; - GIE_digest_t* digest_rule=NULL; - GIE_digest_t** update_array=NULL; - UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; - const long q_cnt=MESA_lqueue_get_count(update_q); - if(q_cnt==0) - { - return; - } - struct Maat_table_runtime* table_rt=maat_scanner->table_rt[table_id]; - update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); - for(i=0;ioperation==GIE_INSERT_OPT) - { - table_rt->origin_rule_num++; - } - else - { - table_rt->origin_rule_num--; - } - destroy_digest_rule(update_array[i]); - update_array[i]=NULL; - } - free(update_array); - update_array=NULL; - return; -} struct region_group_relation { int region_id; @@ -2049,34 +1874,25 @@ int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule* intval_ MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule* db_digest_rule,struct Maat_scanner *scanner,void* logger) +int add_digest_rule(struct Maat_table_desc* table, struct db_digest_rule* db_rule, struct Maat_scanner *scanner,void* logger) { struct Maat_group_inner* group_rule=NULL; - GIE_digest_t* digest_rule=NULL; struct Maat_group_inner* u_para=NULL; - struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime * table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); int expr_id=0,district_id=-1; - group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_digest_rule->group_id, 0, scanner); + group_rule=create_group_rule(db_rule->group_id, 0, scanner); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, table->table_id, db_digest_rule->region_id, district_id, expr_id, TABLE_TYPE_DIGEST, scanner); + u_para=add_region_to_group(group_rule, table->table_id, db_rule->region_id, district_id, expr_id, TABLE_TYPE_DIGEST, scanner); if(u_para==NULL) { return -1; } - if(table->table_type==TABLE_TYPE_SIMILARITY) - { - db_digest_rule->digest_string=str_unescape(db_digest_rule->digest_string); - } - digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT - ,db_digest_rule->digest_string - ,db_digest_rule->confidence_degree - ,group_rule); - MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); + Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, group_rule); scanner->gie_update_q_size++; return 0; } @@ -2086,8 +1902,8 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; int expr_num=0; struct Maat_group_inner* group_rule=NULL; + struct Maat_table_runtime* table_rt=NULL; struct op_expr_t* op_expr=NULL; - GIE_digest_t* digest_rule=NULL; group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); if(group_rule==NULL) { @@ -2126,11 +1942,8 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int case TABLE_TYPE_SIMILARITY: case TABLE_TYPE_DIGEST: assert(expr_num==1); - digest_rule=create_digest_rule(expr_id[0], GIE_DELETE_OPT //del digest - ,NULL - ,0 - ,NULL); - MESA_lqueue_join_tail(maat_scanner->table_rt[table->table_id]->similar.update_q,&digest_rule, sizeof(void*)); + table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id); + Maat_table_runtime_digest_del(table_rt, expr_id[0]); maat_scanner->gie_update_q_size++; break; default: @@ -2331,10 +2144,10 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_ scanner->to_update_compile_cnt++; return 1; } -void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, MESA_htable_handle map_tablename2id, void* logger) +void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, struct Maat_table_manager* table_mgr, void* logger) { struct db_group_rule_t db_group_rule; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); int ret=0; char virtual_table_name[MAX_TABLE_NAME_LEN]={0}; memset(&db_group_rule, 0, sizeof(db_group_rule)); @@ -2364,10 +2177,10 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru table->udpate_err_cnt++; return; } - if(strlen(virtual_table_name)>0) + if(strlen(virtual_table_name)>0&&!strcasecmp(virtual_table_name, "null")) { - ret=map_str2int(map_tablename2id, virtual_table_name, &(db_group_rule.virtual_table_id)); - if(ret<0) + db_group_rule.virtual_table_id=Maat_table_get_id_by_name(table_mgr, virtual_table_name); + if(db_group_rule.virtual_table_id<0) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , "update error, unknown virutal table name: %s of group table %s:%s.", @@ -2381,7 +2194,8 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru { ret=del_group_rule(table, &db_group_rule, scanner, logger); //leave no trace when compatible_group_update calling - if(table->table_type==TABLE_TYPE_GROUP&&ret==1) + assert(table->table_type==TABLE_TYPE_GROUP); + if(ret==1) { table_rt->origin_rule_num--; assert(table_rt->origin_rule_num>=0); @@ -2422,7 +2236,7 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); switch(table->table_type) { case TABLE_TYPE_EXPR: @@ -2711,7 +2525,7 @@ void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struc struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0}; char saddr_format[16]={0}, sport_format[16]={0}, daddr_format[16]={0}, dport_format[16]={0}; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); unsigned short src_port1=0, src_port2=0, dst_port1=0, dst_port2=0; int protocol=0,direction=0; int ret=0; @@ -2918,7 +2732,7 @@ error_out: void update_intval_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1); - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); int ret=0; ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) ,&(intval_rule->group_id) @@ -2984,7 +2798,7 @@ error_out: void update_compile_rule(struct Maat_table_desc* table,const char* table_line ,struct Maat_scanner *scanner, const struct rule_tag* tags, int n_tags,void* logger) { struct compile_table_desc* compile_desc=&(table->compile); - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); struct Maat_compile_rule *p_compile=NULL; struct Maat_rule_head m_rule_tmp; @@ -3069,7 +2883,7 @@ error_out: void update_digest_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1); int ret=0; char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'}; @@ -3270,7 +3084,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma int i=0, ret=1, matched_tag=1; unsigned int len=strlen(table_line)+1; struct plugin_table_desc* plugin_desc=&(table->plugin); - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); char *p=NULL; char* copy=NULL; size_t is_valid_offset=0, valid_len=0; @@ -3413,11 +3227,7 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, struct bool_matcher *tmp1=NULL,*tmp2=NULL; MESA_htable_handle tmp_map=NULL; struct Maat_table_runtime* table_rt=NULL; - int i=0; - long q_cnt; - GIE_create_para_t para; - para.gram_value=7; - para.position_accuracy=10; + int i=0, ret=0; igraph_bool_t is_dag; igraph_is_dag(&(scanner->group_graph), &is_dag); if(!is_dag) @@ -3451,9 +3261,9 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, scanner->region_update_q, logger, scanner); - for(i=0;imax_table_num; i++) { - table_rt=scanner->table_rt[i]; + table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i); if(table_rt==NULL) { continue; @@ -3462,30 +3272,13 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, { case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); - if(q_cnt==0) + + ret=Maat_table_runtime_digest_batch_udpate(table_rt); + if(ret<0) { - continue; + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "GIE_update error."); } - if(table_rt->similar.gie_handle==NULL) - { - if(table_rt->table_type==TABLE_TYPE_SIMILARITY) - { - para.ED_reexamine=1; - para.format=GIE_INPUT_FORMAT_PLAIN; - } - else - { - para.ED_reexamine=0; - para.format=GIE_INPUT_FORMAT_SFH; - } - table_rt->similar.gie_handle=GIE_create(¶); - } - digest_batch_update(table_rt->similar.gie_handle, - table_rt->similar.update_q, - logger, - scanner, - i); break; case TABLE_TYPE_PLUGIN: break; @@ -3509,12 +3302,9 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, } -void maat_start_cb(long long new_version,int update_type,void*u_para) +void maat_start_cb(long long new_version, int update_type, void*u_para) { - struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct Maat_table_desc* p_table=NULL; - struct plugin_table_desc* plugin_desc=NULL; - int i=0,j=0; + struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; feather->new_version=new_version; if(update_type==CM_UPDATE_TYPE_FULL) @@ -3532,26 +3322,7 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) feather->maat_version,new_version); feather->maat_version=new_version; } - feather->active_plugin_table_num=0; - for(i=0;ip_table_info[i]; - plugin_desc=&(p_table->plugin); - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0) - { - continue; - } - - feather->active_plugin_table_num++; - - for(j=0;jcb_plug_cnt;j++) - { - if(plugin_desc->cb_plug[j].start!=NULL) - { - plugin_desc->cb_plug[j].start(update_type,plugin_desc->cb_plug[j].u_para); - } - } - } + Maat_table_manager_all_plugin_cb_start(feather->table_mgr, update_type); return; } long long scanner_rule_num(struct Maat_scanner *scanner) @@ -3559,9 +3330,9 @@ long long scanner_rule_num(struct Maat_scanner *scanner) long long total=0; struct Maat_table_runtime* table_rt=NULL; int i=0; - for(i=0;imax_table_num; i++) { - table_rt=scanner->table_rt[i]; + table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i); if(table_rt!=NULL) { total+=table_rt->origin_rule_num; @@ -3572,42 +3343,9 @@ long long scanner_rule_num(struct Maat_scanner *scanner) void maat_finish_cb(void* u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct Maat_table_desc* p_table=NULL; - struct plugin_table_desc* plugin_desc=NULL; long expr_wait_q_cnt=0; - int i=0, j=0; - - int call_plugin_table_cnt=0; - for(i=0;ip_table_info[i]; - if(p_table==NULL) - { - continue; - } - switch(p_table->table_type) - { - case TABLE_TYPE_PLUGIN: - plugin_desc=&(p_table->plugin); - call_plugin_table_cnt++; - if(call_plugin_table_cnt==feather->active_plugin_table_num) - { - feather->is_last_plugin_table_updating=1; - } - for(j=0;jcb_plug_cnt;j++) - { - if(plugin_desc->cb_plug[j].finish!=NULL) - { - plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para); - } - } - feather->is_last_plugin_table_updating=0; - break; - default: - break; - } - } + Maat_table_manager_all_plugin_cb_finish(feather->table_mgr); if(feather->update_tmp_scanner!=NULL) { @@ -3651,14 +3389,11 @@ void maat_finish_cb(void* u_para) feather->maat_version); } feather->new_version=-1; - feather->active_plugin_table_num=0; return; } int maat_update_cb(const char* table_name,const char* line,void *u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - int ret=-1,i=0; - int table_id=-1; Maat_scanner* scanner=NULL; struct Maat_table_desc* p_table=NULL; if(feather->update_tmp_scanner!=NULL) @@ -3670,47 +3405,39 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) scanner=feather->scanner; } // MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line); - ret=map_str2int(feather->map_tablename2id,table_name,&table_id); - if(ret<0) + p_table=Maat_table_get_desc_by_name(feather->table_mgr, table_name); + if(!p_table) { - MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s",table_name); + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s", table_name); return -1; } - p_table=feather->p_table_info[table_id]; - for(i=0;iconj_cnt;i++) - { - if(0==memcmp(p_table->table_name[i],table_name,strlen(table_name))) - { - p_table->updating_name=i; - } - } - assert(i<=p_table->conj_cnt); + Maat_table_set_updating_name(p_table, table_name); - switch(feather->p_table_info[table_id]->table_type) + switch(p_table->table_type) { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - update_expr_rule(feather->p_table_info[table_id], line, scanner, feather->logger); + update_expr_rule(p_table, line, scanner, feather->logger); break; case TABLE_TYPE_IP: case TABLE_TYPE_IP_PLUS: - update_ip_rule(feather->p_table_info[table_id], line, scanner, feather->logger); + update_ip_rule(p_table, line, scanner, feather->logger); break; case TABLE_TYPE_INTERVAL: - update_intval_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + update_intval_rule(p_table, line, scanner,feather->logger); break; case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - update_digest_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + update_digest_rule(p_table, line, scanner,feather->logger); break; case TABLE_TYPE_COMPILE: - update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + update_compile_rule(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); break; case TABLE_TYPE_GROUP: - update_group_rule(feather->p_table_info[table_id], line, scanner, feather->map_tablename2id, feather->logger); + update_group_rule(p_table, line, scanner, feather->table_mgr, feather->logger); break; case TABLE_TYPE_PLUGIN: - update_plugin_table(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); default: break; @@ -3884,12 +3611,11 @@ void *thread_rule_monitor(void *arg) } } - MESA_htable_destroy(feather->map_tablename2id,free); + Maat_table_manager_destroy(feather->table_mgr); destroy_maat_scanner(feather->scanner); garbage_bury(feather->garbage_q,0,feather->logger); assert(0==MESA_lqueue_get_count(feather->garbage_q)); MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); - Maat_table_clear(feather->p_table_info, MAX_TABLE_NUM); alignment_int64_array_free(feather->thread_call_cnt); alignment_int64_array_free(feather->inner_mid_cnt); diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index f0573eb..bcf3bfb 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -1,4 +1,5 @@ #include "Maat_rule_internal.h" +#include "Maat_table.h" #include "alignment_int64.h" #include #include @@ -121,10 +122,11 @@ void maat_stat_init(struct _Maat_feather_t* feather) FS_STYLE_COLUMN, FS_CALC_SPEED, "hit_rate"); - feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,"Sum"); - for(i=0;itotal_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, "Sum"); + size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr); + for(i=0; i<(int)max_table_num; i++) { - p_table=feather->p_table_info[i]; + p_table=Maat_table_get_by_id_raw(feather->table_mgr, i); if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN ||p_table->table_type==TABLE_TYPE_GROUP ||p_table->table_type==TABLE_TYPE_COMPILE) @@ -185,7 +187,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,Maat_table_manager_get_count(feather->table_mgr)); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt); @@ -197,7 +200,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value); feather->update_err_cnt=0; feather->iconv_err_cnt=0; - for(i=0;itable_mgr); + for(i=0; i<(int)max_table_num; i++) { table_stream_num=0; table_scan_cnt=0; @@ -205,12 +209,13 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_scan_cpu_time=0; table_hit_cnt=0; table_regex_ipv6_num=0; - p_table=feather->p_table_info[i]; + + p_table=Maat_table_get_by_id_raw(feather->table_mgr, i); if(p_table==NULL) { continue; } - table_rt=feather->scanner->table_rt[i]; + table_rt=Maat_table_runtime_get(feather->scanner->table_rt_mgr, i); switch(p_table->table_type) { case TABLE_TYPE_PLUGIN: diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index bfc45e7..5cd8a64 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -9,6 +9,16 @@ #include #include +#define MAX_TABLE_NUM 256 + +struct Maat_table_manager +{ + struct Maat_table_desc* p_table_info[MAX_TABLE_NUM]; + size_t table_cnt; + MESA_htable_handle map_tablename2id; + int active_plugin_table_num; + int is_last_plugin_table_updating; +}; int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map) { @@ -192,11 +202,85 @@ error_out: free(copy_line); return -1; } -int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger) + +static int Maat_table_build_map(struct Maat_table_manager* table_mgr, void* logger) { + struct Maat_table_desc** p_table_info=table_mgr->p_table_info; + size_t n_table=MAX_TABLE_NUM; + + MESA_htable_handle map_tablename2id=map_create(); + size_t i=0; + int j=0, ret=0; + for(i=0;itable_type) + { + case TABLE_TYPE_VIRTUAL: + ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id)); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Undefined real table %s, virtual table %s of table id %d.", + p_table_info[i]->virtual_table.real_table_name, + p_table_info[i]->table_name[j], + p_table_info[i]->table_id); + goto failed; + } + break; + default: + break; + } + + + for(j=0; jconj_cnt; j++) + { + ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Duplicate table %s of table id %d", + p_table_info[i]->table_name[j], + p_table_info[i]->table_id); + continue; + } + } + + } + table_mgr->map_tablename2id=map_tablename2id; + return 0; +failed: + map_destroy(map_tablename2id); + return -1; +} + +void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr) +{ + size_t i=0; + for(i=0;ip_table_info[i]==NULL) + { + continue; + } + table_info_free(table_mgr->p_table_info[i]); + table_mgr->p_table_info[i]=NULL; + } + MESA_htable_destroy(table_mgr->map_tablename2id, free); + free(table_mgr); + return; +} + +struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger) +{ + struct Maat_table_manager* table_mgr=NULL; FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; - int i=0,ret=0,table_cnt=0; + int i=0, ret=0; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; MESA_htable_handle string2int_map=NULL;; struct Maat_table_desc*p=NULL; @@ -207,9 +291,12 @@ int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_t fprintf(stderr,"Maat read table info %s error.\n",table_info_path); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, "Maat read table info %s failed: %s.\n", table_info_path, strerror(errno)); - return 0; + return NULL; } - + table_mgr=ALLOC(struct Maat_table_manager, 1); + struct Maat_table_desc** p_table_info=table_mgr->p_table_info; + size_t n_table=MAX_TABLE_NUM; + string2int_map=map_create(); map_register(string2int_map,"expr", TABLE_TYPE_EXPR); map_register(string2int_map,"ip", TABLE_TYPE_IP); @@ -348,83 +435,350 @@ int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_t } p_table_info[p->table_id]=p; - table_cnt++; + table_mgr->table_cnt++; continue; invalid_table: table_info_free(p); p=NULL; } fclose(fp); - map_destroy(string2int_map); - return table_cnt; -} -MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger) -{ - MESA_htable_handle map_tablename2id=map_create(); - size_t i=0; - int j=0, ret=0; - for(i=0;itable_cnt; +} +int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +{ + int i=0; + for(i=0; i< MAX_TABLE_NUM; i++) + { + if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_COMPILE) + { + strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); + return 1; + } + } + return 0; +} +int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +{ + int i=0; + for(i=0; i< MAX_TABLE_NUM; i++) + { + if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP) + { + strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); + return 1; + } + } + return 0; +} +const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM) + { + return NULL; + } + if(table_mgr->p_table_info[table_id]) + { + return table_mgr->p_table_info[table_id]->table_name[0]; + } + return NULL; +} +enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM) + { + return TABLE_TYPE_INVALID; + } + if(table_mgr->p_table_info[table_id]) + { + return table_mgr->p_table_info[table_id]->table_type; + } + return TABLE_TYPE_INVALID; + +} +struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM) + { + return NULL; + } + + return table_mgr->p_table_info[table_id]; +} + +struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id) +{ + + struct Maat_table_desc **p_table_info=table_mgr->p_table_info; + size_t n_table=MAX_TABLE_NUM; + + struct Maat_table_desc *p_table=NULL, *p_real_table=NULL; + if((unsigned int) table_id>n_table) + { + return NULL; + } + if(p_table_info[table_id]==NULL) + { + return NULL; + } + p_table=p_table_info[table_id]; + if(p_table==NULL) + { + return NULL; + } + if(p_table->table_type==TABLE_TYPE_VIRTUAL) + { + p_real_table=p_table_info[p_table->virtual_table.real_table_id]; + *virutal_table_id=table_id; + } + else + { + p_real_table=p_table; + if(virutal_table_id) *virutal_table_id=0; + } + if(p_real_table->table_type!=expect_type) + { + if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| + (expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS)) + { + return NULL; + } + } + return p_real_table; +} +int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name) +{ + int table_id=-1,ret=0; + ret=map_str2int(table_mgr->map_tablename2id, table_name, &table_id); + if(ret>0) + { + return table_id; + } + else + { + return -1; + } +} +int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, + int table_id, + Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para + Maat_update_callback_t *update,//table line ,u_para + Maat_finish_callback_t *finish,//u_para + void* u_para) +{ + int idx=0; + struct Maat_table_desc *p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct plugin_table_desc *plugin_desc=&(p_table->plugin); + if(p_table==NULL) + { + return -1; + } + + idx=plugin_desc->cb_plug_cnt; + if(idx==MAX_PLUGIN_PER_TABLE) + { + return -1; + } + plugin_desc->cb_plug_cnt++; + plugin_desc->cb_plug[idx].start=start; + plugin_desc->cb_plug[idx].update=update; + plugin_desc->cb_plug[idx].finish=finish; + plugin_desc->cb_plug[idx].u_para=u_para; + return 1; +} + +struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx) +{ + int table_id=-1; + struct Maat_table_desc *p_table=NULL; + + table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name); + if(table_id<0) + { + return NULL; + } + p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + if(!p_table) + { + return NULL; + } + if(idxcompile.ex_data_num) + { + return p_table->compile.ex_desc+idx; + } + return NULL; + +} +int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, + Maat_rule_EX_new_func_t *new_func, + Maat_rule_EX_free_func_t* free_func, + Maat_rule_EX_dup_func_t* dup_func, + long argl, void *argp) +{ + int table_id=-1; + struct Maat_table_desc *p_table=NULL; + table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name); + if(table_id<0) + { + return -1; + } + p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + if(!p_table) + { + return -1; + } + int idx=-1; + + struct compile_table_desc* compile_desc=&(p_table->compile); + if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM) + { + return -1; + } + idx=compile_desc->ex_data_num; + compile_desc->ex_desc[idx].idx=idx; + compile_desc->ex_desc[idx].table_id=table_id; + compile_desc->ex_desc[idx].argl=argl; + compile_desc->ex_desc[idx].argp=argp; + compile_desc->ex_desc[idx].new_func=new_func; + compile_desc->ex_desc[idx].free_func=free_func; + compile_desc->ex_desc[idx].dup_func=dup_func; + + compile_desc->ex_data_num++; + + return idx; +} +int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp) + +{ + struct Maat_table_desc *table_desc=NULL;; + table_desc=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct plugin_table_desc* plugin_desc=&(table_desc->plugin); + + if(plugin_desc->have_exdata + || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) + { + return -1; + } + plugin_desc->ex_desc.new_func=new_func; + plugin_desc->ex_desc.free_func=free_func; + plugin_desc->ex_desc.dup_func=dup_func; + plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. + plugin_desc->ex_desc.argl=argl; + plugin_desc->ex_desc.argp=argp; + plugin_desc->have_exdata=1; + return 0; +} +void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type) +{ + table_mgr->active_plugin_table_num=0; + int i=0, j=0; + struct Maat_table_desc* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; + + for(i=0; ip_table_info[i]; + plugin_desc=&(p_table->plugin); + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0) { continue; } + + table_mgr->active_plugin_table_num++; - switch(p_table_info[i]->table_type) + for(j=0;jcb_plug_cnt;j++) { - case TABLE_TYPE_GROUP: - strncpy(group_tn, p_table_info[i]->table_name[0], n_gtn); - break; - case TABLE_TYPE_COMPILE: - strncpy(compile_tn, p_table_info[i]->table_name[0], n_ctn); - break; - case TABLE_TYPE_VIRTUAL: - ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id)); - if(ret<0) + if(plugin_desc->cb_plug[j].start!=NULL) + { + plugin_desc->cb_plug[j].start(update_type, plugin_desc->cb_plug[j].u_para); + } + } + } + +} +void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr) +{ + int i=0, j=0; + struct Maat_table_desc* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; + + int call_plugin_table_cnt=0; + for(i=0;ip_table_info[i]; + if(p_table==NULL) + { + continue; + } + switch(p_table->table_type) + { + case TABLE_TYPE_PLUGIN: + plugin_desc=&(p_table->plugin); + call_plugin_table_cnt++; + if(call_plugin_table_cnt==table_mgr->active_plugin_table_num) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Undefined real table %s, virtual table %s of table id %d.", - p_table_info[i]->virtual_table.real_table_name, - p_table_info[i]->table_name[j], - p_table_info[i]->table_id); - goto failed; + table_mgr->is_last_plugin_table_updating=1; } + for(j=0;jcb_plug_cnt;j++) + { + if(plugin_desc->cb_plug[j].finish!=NULL) + { + plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para); + } + } + table_mgr->is_last_plugin_table_updating=0; break; default: break; - } - - - for(j=0; jconj_cnt; j++) - { - ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Duplicate table %s of table id %d", - p_table_info[i]->table_name[j], - p_table_info[i]->table_id); - continue; - } - } - - } - return map_tablename2id; -failed: - map_destroy(map_tablename2id); - return NULL; -} -void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table) -{ - size_t i=0; - for(i=0;iactive_plugin_table_num=0; return; } +int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr) +{ + return table_mgr->is_last_plugin_table_updating; +} +struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name) +{ + struct Maat_table_desc * p_table=NULL; + int table_id=0; + table_id=Maat_table_get_id_by_name(table_mgr, table_name); + if(table_id<0) + { + return NULL; + } + p_table=table_mgr->p_table_info[table_id]; + return p_table; +} +void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name) +{ + int i=0; + for(i=0; iconj_cnt; i++) + { + if(0==strcmp(p_table->table_name[i], table_name)) + { + p_table->updating_name=i; + } + } + assert(i<=p_table->conj_cnt); +} + diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp new file mode 100644 index 0000000..5b9dbc0 --- /dev/null +++ b/src/entry/Maat_table_runtime.cpp @@ -0,0 +1,427 @@ +#include "Maat_table_runtime.h" +#include "Maat_rule.h" +#include "Maat_utils.h" + +#include +#include +#include + +struct wrap_plugin_EX_data +{ + MAAT_RULE_EX_DATA exdata; + const struct Maat_table_desc* ref_plugin_table; +}; +void wrap_plugin_EX_data_free(void *data) +{ + struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; + const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); + ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); + wrap_data->ref_plugin_table=NULL; + free(wrap_data); + return; +} +MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) +{ + MESA_htable_handle key2ex_hash=NULL; + unsigned int slot_size=1; + while(estimate_size!=0) + { + estimate_size=estimate_size>>1; + slot_size*=2; + } + if(slot_size==1) + { + slot_size=4096; + } + + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=8; + hargs.hash_slot_size = slot_size; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = NULL; //Not supported yet. + hargs.recursive = 1; + hargs.data_free = wrap_plugin_EX_data_free; + hargs.data_expire_with_condition = NULL; + key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(key2ex_hash, 0); + return key2ex_hash; +} + +int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + size_t key_offset=0, key_len=0; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: no such key %.*s of %s", + key_len, line+key_offset, line); + return -1; + } + return 0; +} + +int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + char* key=NULL; + size_t key_offset=0, key_len=0; + MAAT_RULE_EX_DATA exdata=NULL; + struct wrap_plugin_EX_data* wrap_data=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + key=ALLOC(char, key_len+1); + memcpy(key, line+key_offset, key_len); + plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); + wrap_data->exdata=exdata; + wrap_data->ref_plugin_table=plugin_table; + ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); + free(key); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: duplicated key %.*s of %s", + key_len, line+key_offset, line); + wrap_plugin_EX_data_free(wrap_data); + return -1; + } + return 0; +} + +struct Maat_table_runtime_manager +{ + struct Maat_table_runtime** table_rt; + size_t n_table_rt; +}; +static GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, + short cfds_lvl, void* tag) +{ + GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); + int digest_len=0; + rule->id=id; + rule->operation=op; + if(digest!=NULL) + { + digest_len=strlen(digest); + rule->sfh=(char*)calloc(sizeof(char),digest_len+1); + memcpy(rule->sfh,digest,digest_len); + + } + rule->sfh_length=digest_len; + rule->cfds_lvl=cfds_lvl; + rule->tag=tag; + return rule; +} +static void destroy_digest_rule(GIE_digest_t*rule) +{ + if(rule->sfh!=NULL) + { + free(rule->sfh); + rule->sfh=NULL; + } + free(rule); + rule=NULL; + return; +} + +static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num) +{ + + struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); + table_rt->table_type=table_desc->table_type; + switch(table_desc->table_type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + table_rt->similar.update_q=MESA_lqueue_create(0,0); + break; + case TABLE_TYPE_PLUGIN: + table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); + if(table_desc->plugin.have_exdata) + { + table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, + table_desc->plugin.ex_desc.key2index_func); + } + break; + default: + break; + } + + table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); + table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); + table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); + table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); + table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); + return table_rt; +} +static void table_runtime_free(struct Maat_table_runtime* p) +{ + long q_cnt=0,data_size=0; + int i=0; + UNUSED int q_ret=0; + + GIE_digest_t* digest_rule=NULL; + if(p==NULL) + { + return; + } + switch(p->table_type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + if(p->similar.gie_handle!=NULL) + { + GIE_destory(p->similar.gie_handle); + } + if(p->similar.update_q!=NULL) + { + q_cnt=MESA_lqueue_get_count(p->similar.update_q); + for(i=0;isimilar.update_q,&digest_rule,&data_size); + assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); + destroy_digest_rule(digest_rule); + } + MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); + } + break; + case TABLE_TYPE_PLUGIN: + dynamic_array_destroy(p->plugin.cache_lines, free); + p->plugin.cache_lines=NULL; + if(p->plugin.key2ex_hash!=NULL) + { + MESA_htable_destroy(p->plugin.key2ex_hash, NULL); + } + default: + break; + } + + alignment_int64_array_free(p->scan_cnt); + alignment_int64_array_free(p->scan_cpu_time); + alignment_int64_array_free(p->input_bytes); + alignment_int64_array_free(p->stream_num); + alignment_int64_array_free(p->hit_cnt); + free(p); + return; +} + +struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num) +{ + const struct Maat_table_desc* table_desc=NULL; + struct Maat_table_runtime* table_rt=NULL; + struct Maat_table_runtime_manager* table_rt_mgr=ALLOC(struct Maat_table_runtime_manager, 1); + size_t i=0; + table_rt_mgr->n_table_rt=Maat_table_manager_get_size(table_manager); + table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt); + for(i=0; in_table_rt; i++) + { + table_desc=Maat_table_get_by_id_raw(table_manager, i); + if(!table_desc) + { + continue; + } + table_rt=table_runtime_new(table_desc, max_thread_num); + table_rt_mgr->table_rt[i]=table_rt; + } + return table_rt_mgr; +} +void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr) +{ + size_t i=0; + for(i=0; in_table_rt; i++) + { + table_runtime_free(table_rt_mgr->table_rt[i]); + table_rt_mgr->table_rt[i]=NULL; + } + free(table_rt_mgr->table_rt); + table_rt_mgr->table_rt=NULL; + free(table_rt_mgr); +} +struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id) +{ + assert(table_id<(int)table_rt_mgr->n_table_rt); + return table_rt_mgr->table_rt[table_id]; +} +long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt) +{ + struct plugin_runtime* plugin_rt=&(table_rt->plugin); + return plugin_rt->cache_line_num; +} +const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line) +{ + const char* line=NULL; + struct plugin_runtime* plugin_rt=&(table_rt->plugin); + + line=(const char*)dynamic_array_read(plugin_rt->cache_lines, Nth_line); + return line; +} +MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table, + struct dynamic_array_t* lines, size_t line_cnt, void* logger) +{ + MESA_htable_handle key2ex_hash=NULL; + size_t i=0; + const char* line=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + + key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); + + for(i=0; i< line_cnt; i++) + { + line=(const char*)dynamic_array_read(lines, i); + plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); + } + return key2ex_hash; +} +int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger) +{ + assert(table_rt->plugin.key2ex_hash==NULL); + if(table_rt->plugin.key2ex_hash) + { + return -1; + } + table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, + table_rt->plugin.cache_line_num, logger); + return 0; +} +MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key) +{ + struct plugin_table_desc* plugin_desc=NULL; + struct wrap_plugin_EX_data* wrap_data=NULL; + MAAT_RULE_EX_DATA exdata=NULL; + + plugin_desc=&(table_desc->plugin); + if(!plugin_desc->have_exdata) + { + assert(0); + return NULL; + } + wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, + (const unsigned char*)key, strlen(key)); + if(wrap_data!=NULL) + { + plugin_desc->ex_desc.dup_func(table_desc->table_id, &(exdata), &(wrap_data->exdata), + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + } + return exdata; + +} + +void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag) +{ + GIE_digest_t* digest_rule=NULL; + char *dup_digest=_maat_strdup(digest); + + if(table_rt->table_type==TABLE_TYPE_SIMILARITY) + { + dup_digest=str_unescape(dup_digest); + } + digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT, + dup_digest, + confidence_degree, + tag); + MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); + return; +} +void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id) +{ + GIE_digest_t* digest_rule=NULL; + digest_rule=create_digest_rule(expr_id, GIE_DELETE_OPT //del digest + ,NULL + ,0 + ,NULL); + MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*)); + return; +} +int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) +{ + long i=0,data_size=0; + int ret=0; + GIE_digest_t* digest_rule=NULL; + GIE_digest_t** update_array=NULL; + UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; + + + GIE_create_para_t para; + para.gram_value=7; + para.position_accuracy=10; + + const long q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); + if(q_cnt==0) + { + return 0; + } + if(table_rt->similar.gie_handle==NULL) + { + if(table_rt->table_type==TABLE_TYPE_SIMILARITY) + { + para.ED_reexamine=1; + para.format=GIE_INPUT_FORMAT_PLAIN; + } + else + { + para.ED_reexamine=0; + para.format=GIE_INPUT_FORMAT_SFH; + } + table_rt->similar.gie_handle=GIE_create(¶); + } + + update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); + for(i=0;isimilar.update_q, &digest_rule, &data_size); + assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); + update_array[i]=digest_rule; + digest_rule=NULL; + } + ret=GIE_update(table_rt->similar.gie_handle, update_array, (int)q_cnt); + for(i=0;ioperation==GIE_INSERT_OPT) + { + table_rt->origin_rule_num++; + } + else + { + table_rt->origin_rule_num--; + } + destroy_digest_rule(update_array[i]); + update_array[i]=NULL; + } + free(update_array); + update_array=NULL; + + if(ret!=(int)q_cnt) + { + return -1; + } + return q_cnt; +} + + diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 39f7461..892fe09 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -3,7 +3,7 @@ #include #include #include - +#include #include "Maat_utils.h" pid_t gettid() { @@ -252,3 +252,9 @@ const char* charset_get_name(enum MAAT_CHARSET charset) { return CHARSET_STRING[charset]; } +int lqueue_destroy_cb(void *data, long data_len, void *arg) +{ + assert(0); + return 0; +} + diff --git a/src/inc_internal/Maat_limits.h b/src/inc_internal/Maat_limits.h index feef2a8..6d025f6 100644 --- a/src/inc_internal/Maat_limits.h +++ b/src/inc_internal/Maat_limits.h @@ -1,4 +1,3 @@ -#define MAX_TABLE_NUM 256 #define MAX_TABLE_LINE_SIZE (1024*16) #define MAX_EXPR_KEYLEN 1024 #define MAX_DISTRICT_LEN 64 diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 35633dc..64e9d47 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -4,6 +4,7 @@ #include "Maat_command.h" #include "Maat_limits.h" #include "Maat_table.h" +#include "Maat_table_runtime.h" #include #include @@ -218,54 +219,7 @@ struct _stream_para_t pthread_mutex_t fuzzy_mutex; unsigned char query_point[8]; }; -struct similar_runtime -{ - GIE_handle_t* gie_handle; - MESA_lqueue_head update_q; -}; -struct plugin_runtime -{ - dynamic_array_t *cache_lines; - long long cache_line_num; - long long acc_line_num; - long long cache_size; - MESA_htable_handle key2ex_hash; -}; -struct expr_runtime -{ - long long expr_rule_cnt; //expr_type=0,1,3 - long long regex_rule_cnt; //expr_type=2 -}; -struct ip_runtime -{ - long long ipv4_rule_cnt; - long long ipv6_rule_cnt; - -}; -struct group_runtime -{ - long long not_flag_group; -}; -struct Maat_table_runtime -{ - enum MAAT_TABLE_TYPE table_type; - long origin_rule_num; - union - { - struct similar_runtime similar; //for digest and similarity - struct plugin_runtime plugin; - struct expr_runtime expr; - struct ip_runtime ip; - struct group_runtime group; - void * other; - }; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; -}; struct rule_tag { char* tag_name; @@ -280,9 +234,10 @@ struct Maat_scanner size_t gie_update_q_size; size_t to_update_group_cnt; size_t to_update_compile_cnt; - - struct Maat_table_runtime* table_rt[MAX_TABLE_NUM]; + struct Maat_table_runtime_manager* table_rt_mgr; + size_t max_table_num; + MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation* MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions; MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_inner* @@ -344,7 +299,8 @@ struct _Maat_feather_t struct Maat_scanner *scanner; struct Maat_scanner *update_tmp_scanner; MESA_lqueue_head garbage_q; - int table_cnt; + struct Maat_table_manager* table_mgr; + int DEFERRED_LOAD_ON; int REDIS_MODE_ON; enum data_source input_mode; @@ -360,8 +316,7 @@ struct _Maat_feather_t int cumulative_update_off; int stat_on; int perf_on; - struct Maat_table_desc *p_table_info[MAX_TABLE_NUM]; - MESA_htable_handle map_tablename2id; + void* logger; long long maat_version; long long last_full_version; @@ -394,8 +349,6 @@ struct _Maat_feather_t int foreign_cont_linger; //internal states long long new_version; - int active_plugin_table_num; - int is_last_plugin_table_updating; //for scanner independent stat>>>> int backgroud_update_enabled; diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h index e60d887..27112d2 100644 --- a/src/inc_internal/Maat_table.h +++ b/src/inc_internal/Maat_table.h @@ -20,6 +20,7 @@ enum USER_REGION_ENCODE enum MAAT_TABLE_TYPE { + TABLE_TYPE_INVALID=-1, TABLE_TYPE_EXPR=0, TABLE_TYPE_IP, TABLE_TYPE_IP_PLUS, @@ -50,6 +51,21 @@ struct compile_table_desc struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; }; +struct expr_table_desc +{ + enum MAAT_CHARSET src_charset; + enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; + int src_charset_in_dst; + int do_charset_merge; + int cross_cache_size; + int quick_expr_switch;//obsolete since 20190401 + long long iconv_err_cnt; +}; +struct virtual_table_desc +{ + int real_table_id; + char real_table_name[MAX_TABLE_NAME_LEN]; +}; struct plugin_table_callback_desc { Maat_start_callback_t *start; @@ -79,26 +95,7 @@ struct plugin_table_desc struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; struct plugin_table_ex_data_desc ex_desc; }; -struct expr_table_desc -{ - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cross_cache_size; - int quick_expr_switch;//obsolete since 20190401 - long long iconv_err_cnt; -}; -struct ip_table_desc -{ - int ipv4_rule_cnt; - int ipv6_rule_cnt; -}; -struct virtual_table_desc -{ - int real_table_id; - char real_table_name[MAX_TABLE_NAME_LEN]; -}; + struct Maat_table_desc { int table_id; @@ -110,17 +107,52 @@ struct Maat_table_desc { struct compile_table_desc compile; struct expr_table_desc expr; - struct ip_table_desc ip; struct plugin_table_desc plugin; struct virtual_table_desc virtual_table;; - void* others;//group, interval and digest don't have sperate description info. + void* others;//group, ip, interval and digest don't have sperate description info. }; //for stat>>>>>>>> unsigned long long udpate_err_cnt; unsigned long long unmatch_tag_cnt; int stat_line_id; }; -int Maat_table_read_table_info(struct Maat_table_desc** p_table_info, size_t n_table, const char* table_info_path, void* logger); -MESA_htable_handle Maat_table_build(struct Maat_table_desc** p_table_info, size_t n_table, char* compile_tn, size_t n_ctn, char* group_tn, size_t n_gtn, void* logger); -void Maat_table_clear(struct Maat_table_desc** p_table_info, size_t n_table); +struct Maat_table_manager; +struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger); +void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr); +size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr); +size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr); + +struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id); +struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id); + +int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name); +int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, + int table_id, + Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para + Maat_update_callback_t *update,//table line ,u_para + Maat_finish_callback_t *finish,//u_para + void* u_para); +int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id); +enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id); + +int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, + Maat_rule_EX_new_func_t *new_func, + Maat_rule_EX_free_func_t* free_func, + Maat_rule_EX_dup_func_t* dup_func, + long argl, void *argp); +struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx); +int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp); +void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type); +void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr); + +int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr); +struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name); +void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name); diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h new file mode 100644 index 0000000..caeae5d --- /dev/null +++ b/src/inc_internal/Maat_table_runtime.h @@ -0,0 +1,69 @@ +#include "Maat_table.h" +#include "gram_index_engine.h" +#include "alignment_int64.h" +#include "dynamic_array.h" +#include +#include + +struct similar_runtime +{ + GIE_handle_t* gie_handle; + MESA_lqueue_head update_q; +}; + +struct plugin_runtime +{ + dynamic_array_t *cache_lines; + long long cache_line_num; + long long acc_line_num; + long long cache_size; + MESA_htable_handle key2ex_hash; +}; +struct expr_runtime +{ + long long expr_rule_cnt; //expr_type=0,1,3 + long long regex_rule_cnt; //expr_type=2 +}; +struct ip_runtime +{ + long long ipv4_rule_cnt; + long long ipv6_rule_cnt; + +}; +struct group_runtime +{ + long long not_flag_group; +}; +struct Maat_table_runtime +{ + enum MAAT_TABLE_TYPE table_type; + long origin_rule_num; + union + { + struct similar_runtime similar; //for digest and similarity + struct plugin_runtime plugin; + struct expr_runtime expr; + struct ip_runtime ip; + struct group_runtime group; + void * other; + }; + mcore_long_t scan_cnt; + mcore_long_t scan_cpu_time; //nano + mcore_long_t input_bytes; + mcore_long_t stream_num; + mcore_long_t hit_cnt; +}; +struct Maat_table_runtime_manager; +struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num); +void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr); +struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id); +long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt); +const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line); + +int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger); +MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key); +void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag); +void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id); +int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt); + + diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 167048d..358b826 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -72,5 +72,6 @@ char* md5_file(const char* filename, char* md5string); int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); const char** charset_get_all_name(void); const char* charset_get_name(enum MAAT_CHARSET charset); +int lqueue_destroy_cb(void *data, long data_len, void *arg);