diff --git a/inc/Maat_command.h b/inc/Maat_command.h index 816dd63..5e0c6e1 100644 --- a/inc/Maat_command.h +++ b/inc/Maat_command.h @@ -109,6 +109,7 @@ struct Maat_region_t struct Maat_group_t { const char* table_name; + const char* virtual_table_name; int group_id; //If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number. int parent_id; int not_flag; diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 71eb650..3a93a6e 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -30,7 +30,9 @@ enum MAAT_CHARSET CHARSET_UNICODE_NCR_DEC, //SGML Numeric character reference,decimal base, e.g. "ا" CHARSET_UNICODE_NCR_HEX, //SGML Numeric character reference,hexdecimal base, e.g. "ا" CHARSET_URL_ENCODE_GB2312, //URL encode with GB2312, e.g. the chinese word "china" was encoded to %D6%D0%B9%FA - CHARSET_URL_ENCODE_UTF8 //11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD + CHARSET_URL_ENCODE_UTF8, //11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD + CHARSET_WINDOWS1251, + __CHARSET_MAX }; enum MAAT_ACTION { @@ -158,8 +160,7 @@ enum MAAT_INIT_OPT //This option also disables background update. MAAT_OPT_ENABLE_UPDATE, //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed. MAAT_OPT_ACCEPT_TAGS, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]} - MAAT_OPT_FOREIGN_CONT_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files - MAAT_OPT_FOREIGN_CONT_LINGER //VALUE is interger *, SIZE=sizeof(int). Greater than 0: delete after VALUE seconds; 0: delete foreign content right after the notification callbacks; Less than 0: NEVER delete. Default: 0. + MAAT_OPT_FOREIGN_CONT_DIR //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files }; //return -1 if failed, return 0 on success; int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 483ddf3..2e79378 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,14 +1,14 @@ cmake_minimum_required(VERSION 3.5) set(MAAT_FRAME_MAJOR_VERSION 2) -set(MAAT_FRAME_MINOR_VERSION 7) +set(MAAT_FRAME_MINOR_VERSION 8) set(MAAT_FRAME_PATCH_VERSION 0) set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION}) message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) +set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(/opt/MESA/include/MESA/) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index e332f45..7f0a637 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -16,32 +16,7 @@ #include "rulescan.h" #include "json2iris.h" -struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather, int table_id, enum MAAT_TABLE_TYPE expect_type) -{ - struct Maat_table_desc *p_table=NULL; - if(table_id>MAX_TABLE_NUM) - { - return NULL; - } - if(_feather->p_table_info[table_id]==NULL) - { - return NULL; - } - p_table=_feather->p_table_info[table_id]; - if(p_table==NULL) - { - return NULL; - } - if(p_table->table_type!=expect_type) - { - if((expect_type==TABLE_TYPE_EXPR && p_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| - (expect_type==TABLE_TYPE_IP && p_table->table_type!=TABLE_TYPE_IP_PLUS)) - { - return NULL; - } - } - return p_table; -} + inline void INC_SCANNER_REF(Maat_scanner*scanner,int thread_num) { alignment_int64_array_add(scanner->ref_cnt, thread_num, 1); @@ -158,12 +133,49 @@ static int compare_compile_inner(const void *a, const void *b) return (rb->compile_id-ra->compile_id); } } - -int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) +struct scan_region_hit_wraper { + void* elem_array; + size_t elem_size; + size_t n_elem; + size_t group_offset; + int virtual_table_id; + int is_last_region; +}; +void scan_region_hit_wraper_build_with_rulescan(struct scan_region_hit_wraper* region_hit, scan_result_t* rulescan_rslt, size_t n_rslt, int is_last_region, int virual_table_id) +{ + memset(region_hit, 0, sizeof(struct scan_region_hit_wraper)); + region_hit->elem_array=rulescan_rslt; + region_hit->n_elem=n_rslt; + region_hit->elem_size=sizeof(scan_result_t); + region_hit->group_offset=offsetof(scan_result_t, tag); + region_hit->is_last_region=is_last_region; + region_hit->virtual_table_id=virual_table_id; + return; +} +void scan_region_hit_wraper_build_with_GIE(struct scan_region_hit_wraper* region_hit, GIE_result_t* GIE_rslt, size_t n_rslt, int is_last_region, int virual_table_id) +{ + memset(region_hit, 0, sizeof(struct scan_region_hit_wraper)); + region_hit->elem_array=GIE_rslt; + region_hit->n_elem=n_rslt; + region_hit->elem_size=sizeof(GIE_result_t); + region_hit->group_offset=offsetof(GIE_result_t, tag); + region_hit->is_last_region=is_last_region; + region_hit->virtual_table_id=virual_table_id; + return; +} + +int region_compile(_Maat_feather_t*feather, struct _INNER_scan_status_t *_mid, const struct scan_region_hit_wraper* region_hit_wraper, struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) +{ + int is_last_region=region_hit_wraper->is_last_region; + void* region_hit=region_hit_wraper->elem_array; + size_t region_type_size=region_hit_wraper->elem_size; + size_t group_offset=region_hit_wraper->group_offset; + size_t region_hit_num=region_hit_wraper->n_elem; int scan_ret=0, result_cnt=0; - int ret=0, i=0, j=0; + int ret=0; + size_t i=0, j=0; size_t r_in_c_cnt=0; unsigned char has_not_flag=0; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; @@ -186,27 +198,33 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int { if(_mid->cur_hit_group_cntcur_hit_groups, _mid->cur_hit_group_cnt, (void*)group_rule->top_groups[j]); + dynamic_array_write(_mid->cur_hit_groups, _mid->cur_hit_group_cnt, (void*)TO_RELATION_ID(region_hit_wraper->virtual_table_id,group_rule->top_groups[j])); _mid->cur_hit_group_cnt++; } ret=insert_set_id(&(_mid->all_hit_group_array), &(_mid->all_hit_group_array_sz), _mid->all_hit_group_cnt, - group_rule->top_groups[j]); + TO_RELATION_ID(region_hit_wraper->virtual_table_id, group_rule->top_groups[j])); _mid->all_hit_group_cnt+=ret; } } - scan_ret=bool_matcher_match(bm, thread_num, + if(bm) + { + scan_ret=bool_matcher_match(bm, thread_num, _mid->all_hit_group_array, _mid->all_hit_group_cnt, (void **)relation_array, MAX_SCANNER_HIT_NUM); - + } + else + { + scan_ret=0; + } if(scan_ret>1) { qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**), compare_compile_inner); } - for(i=0;ilogger, RLOG_LV_INFO, maat_module , + "Maat initial with JSON file %s, formating..", + maat_json_fn); + if(strlen(feather->decrypt_key)&&strlen(feather->decrypt_algo)) + { + ret=decrypt_open(maat_json_fn, feather->decrypt_key, feather->decrypt_algo, (unsigned char**)&json_buff, &buff_sz, err_str, err_str_sz); + } + if(json_buff==NULL)//decryption failed or no decryption. + { + ret=load_file_to_memory(maat_json_fn, (unsigned char**)&json_buff, &buff_sz); + } + ret=json2iris(json_buff, + maat_json_fn, + feather->compile_tn, feather->group_tn, + NULL, + feather->json_ctx.iris_file, + sizeof(feather->json_ctx.iris_file), + strlen(feather->decrypt_key)?feather->decrypt_key:NULL, + strlen(feather->decrypt_algo)?feather->decrypt_algo:NULL, + feather->logger); + free(json_buff); + json_buff=NULL; + if(ret<0) + { + return -1; + } + strncpy(feather->json_ctx.json_file, maat_json_fn, sizeof(feather->json_ctx.json_file)); + + ret=stat(maat_json_fn, &fstat_buf); + feather->json_ctx.last_md5_time=fstat_buf.st_ctim; + + md5_file(feather->json_ctx.json_file, feather->json_ctx.effective_json_md5); + MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO, maat_module, + "JSON file %s md5: %s, generate index file %s OK.", + feather->json_ctx.json_file, + feather->json_ctx.effective_json_md5, + feather->json_ctx.iris_file); + feather->input_mode=SOURCE_JSON_FILE; + + return 0; +} Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger) { if(max_thread_num<=0) @@ -500,41 +566,14 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* return NULL; } _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); - feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM, table_info_path, max_thread_num, logger); - if(feather->table_cnt==0) + feather->table_mgr=Maat_table_manager_create(table_info_path, logger); + if(feather->table_mgr==NULL) { - free(feather); - return NULL; - } - feather->map_tablename2id=map_create(); - int i=0,j=0,ret=0; - for(i=0;ip_table_info[i]!=NULL) - { - if(feather->p_table_info[i]->table_type==TABLE_TYPE_GROUP) - { - feather->GROUP_MODE_ON=1; - strncpy(feather->group_tn,feather->p_table_info[i]->table_name[0],sizeof(feather->group_tn)); - } - if(feather->p_table_info[i]->table_type==TABLE_TYPE_COMPILE) - { - strncpy(feather->compile_tn,feather->p_table_info[i]->table_name[0],sizeof(feather->compile_tn)); - } - for(j=0;jp_table_info[i]->conj_cnt;j++) - { - ret=map_register(feather->map_tablename2id,feather->p_table_info[i]->table_name[j],feather->p_table_info[i]->table_id); - if(ret<0) - { - MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module , - "Duplicate table name %s of table id %d" - ,feather->p_table_info[i]->table_name[j] - ,feather->p_table_info[i]->table_id); - continue; - } - } - } + goto failed; } + Maat_table_get_compile_table_name(feather->table_mgr, feather->compile_tn, sizeof(feather->compile_tn)); + Maat_table_get_group_table_name(feather->table_mgr, feather->group_tn, sizeof(feather->group_tn)); + feather->logger=logger; feather->scan_thread_num=max_thread_num; feather->garbage_q=MESA_lqueue_create(0,0); @@ -552,17 +591,20 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->base_rgn_seq=0; feather->AUTO_NUMBERING_ON=1; feather->backgroud_update_enabled=1; - feather->foreign_cont_linger=0; + snprintf(feather->decrypt_algo, sizeof(feather->decrypt_algo), "aes-256-cbc"); snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path); pthread_mutex_init(&(feather->background_update_mutex),NULL); snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path); return feather; +failed: + free(feather); + return NULL; } int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct stat attrib; int intval=0,ret=-1; + char err_str[MAX_TABLE_NAME_LEN]; switch(type) { case MAAT_OPT_ENABLE_UPDATE: @@ -637,27 +679,15 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo break; case MAAT_OPT_JSON_FILE_PATH: assert(_feather->input_mode==SOURCE_NONE); - ret=json2iris((const char*)value, - _feather->compile_tn,_feather->group_tn, - NULL, - _feather->json_ctx.iris_file, - sizeof(_feather->json_ctx.iris_file), - _feather->logger); + ret=load_maat_json_file(_feather, (const char *)value, err_str, sizeof(err_str)); if(ret<0) { + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, + "Load maat json file %s failed: %s.", + (const char*)value, err_str); return -1; } - memcpy(_feather->json_ctx.json_file, value, size); - stat(_feather->json_ctx.json_file, &attrib); - _feather->json_ctx.last_md5_time=attrib.st_ctime; - md5_file(_feather->json_ctx.json_file, _feather->json_ctx.effective_json_md5); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat initial with JSON file %s md5: %s, generate index file %s OK.", - _feather->json_ctx.json_file, - _feather->json_ctx.effective_json_md5, - _feather->json_ctx.iris_file); - _feather->input_mode=SOURCE_JSON_FILE; break; case MAAT_OPT_STAT_ON: _feather->stat_on=1; @@ -670,10 +700,10 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo { return -1; } - memcpy(_feather->stat_file,(const char*)value,size); - MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module , - "Maat performance statistic output to %s." - ,(const char*)value); + memcpy(_feather->stat_file, (const char*)value, size); + MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module, + "Maat performance statistic output to %s.", + (const char*)value); _feather->stat_on=1; break; case MAAT_OPT_SCAN_DETAIL: @@ -681,9 +711,9 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo _feather->rule_scan_type=intval; break; case MAAT_OPT_INSTANCE_NAME: - snprintf(_feather->instance_name - ,sizeof(_feather->instance_name) - ,"%s", + snprintf(_feather->instance_name, + sizeof(_feather->instance_name), + "%s", (const char*)value); break; case MAAT_OPT_DECRYPT_KEY: @@ -760,9 +790,6 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo _feather->foreign_cont_dir[strlen(_feather->foreign_cont_dir)-1]='\0'; } break; - case MAAT_OPT_FOREIGN_CONT_LINGER: - _feather->foreign_cont_linger=*(int*)value; - break; default: return -1; } @@ -966,16 +993,7 @@ void Maat_burn_feather(Maat_feather_t feather) int Maat_table_register(Maat_feather_t feather,const char* table_name) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int table_id=-1,ret=0; - ret=map_str2int(_feather->map_tablename2id, table_name, &table_id); - if(ret>0) - { - return table_id; - } - else - { - return -1; - } + return Maat_table_get_id_by_name(_feather->table_mgr, table_name); } int Maat_table_callback_register(Maat_feather_t feather,short table_id, Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para @@ -984,52 +1002,39 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, void* u_para) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int idx=0,i=0; - struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; - struct plugin_table_desc *plugin_desc=&(p_table->plugin); - struct plugin_runtime* plugin_aux=NULL; - const char* lines=NULL; - if(p_table==NULL) - { - return -1; - } - if(p_table->table_type!=TABLE_TYPE_PLUGIN) - { - return -1; - } - //plugin table register blocks background update. + int i=0,ret=0; + pthread_mutex_lock(&(_feather->background_update_mutex)); - idx=plugin_desc->cb_plug_cnt; - if(idx==MAX_PLUGIN_PER_TABLE) + ret=Maat_table_add_callback_func(_feather->table_mgr, table_id, start, update, finish, u_para); + if(ret<0) { pthread_mutex_unlock(&(_feather->background_update_mutex)); return -1; } - plugin_desc->cb_plug_cnt++; - plugin_desc->cb_plug[idx].start=start; - plugin_desc->cb_plug[idx].update=update; - plugin_desc->cb_plug[idx].finish=finish; - plugin_desc->cb_plug[idx].u_para=u_para; - if(_feather->scanner==NULL) + if(!_feather->scanner) { pthread_mutex_unlock(&(_feather->background_update_mutex)); return 1; } - plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin); - if(plugin_aux->cache_line_num>0) + const char* line=NULL; + struct Maat_table_runtime* table_rt=NULL; + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + long long line_cnt=Maat_table_runtime_plugin_cached_line_count(table_rt); + + if(line_cnt>0) { if(start!=NULL) { start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); } - for(i=0;icache_line_num;i++) + for(i=0; icache_lines,i); - if(lines==NULL) + line=Maat_table_runtime_plugin_get_cached_line(table_rt, i); + if(line==NULL) { break; } - update(table_id,lines,u_para); + update(table_id,line,u_para); } if(finish!=NULL) { @@ -1067,39 +1072,31 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table long argl, void *argp) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - int table_id=-1,ret=0, idx=-1; - ret=map_str2int(_feather->map_tablename2id, compile_table_name, &table_id); - if(ret<0) + int idx=-1; + + if(new_func==NULL || free_func==NULL || dup_func==NULL) { return -1; } - struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; - if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL || dup_func==NULL) - { - return -1; - } - struct compile_table_desc* compile_desc=&(p_table->compile); pthread_mutex_lock(&(_feather->background_update_mutex)); - if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM) - { - ret=-1; - goto failed; - } - idx=compile_desc->ex_data_num; - compile_desc->ex_desc[idx].idx=idx; - compile_desc->ex_desc[idx].table_id=table_id; - compile_desc->ex_desc[idx].argl=argl; - compile_desc->ex_desc[idx].argp=argp; - compile_desc->ex_desc[idx].new_func=new_func; - compile_desc->ex_desc[idx].free_func=free_func; - compile_desc->ex_desc[idx].dup_func=dup_func; + idx=Maat_table_new_compile_rule_ex_index(_feather->table_mgr, compile_table_name, + new_func, + free_func, + dup_func, + argl, argp); - compile_desc->ex_data_num++; + if(idx<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + struct compile_ex_data_idx* compile_ex_desc=Maat_table_get_compile_rule_ex_desc(_feather->table_mgr, compile_table_name, idx); + if(_feather->scanner!=NULL) { - MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_desc->ex_desc+idx); + MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_ex_desc); } -failed: + pthread_mutex_unlock(&(_feather->background_update_mutex)); return idx; @@ -1127,129 +1124,7 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa pthread_rwlock_unlock(&(relation->rwlock)); return ad; } -struct wrap_plugin_EX_data -{ - MAAT_RULE_EX_DATA exdata; - const struct Maat_table_desc* ref_plugin_table; -}; -void wrap_plugin_EX_data_free(void *data) -{ - struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; - const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); - ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); - wrap_data->ref_plugin_table=NULL; - free(wrap_data); - return; -} -MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) -{ - MESA_htable_handle key2ex_hash=NULL; - unsigned int slot_size=1; - while(estimate_size!=0) - { - estimate_size=estimate_size>>1; - slot_size*=2; - } - if(slot_size==1) - { - slot_size=4096; - } - MESA_htable_create_args_t hargs; - memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=8; - hargs.hash_slot_size = slot_size; - hargs.max_elem_num = 0; - hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; - hargs.expire_time = 0; - hargs.key_comp = NULL; - hargs.key2index = NULL; //Not supported yet. - hargs.recursive = 1; - hargs.data_free = wrap_plugin_EX_data_free; - hargs.data_expire_with_condition = NULL; - key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); - MESA_htable_print_crtl(key2ex_hash, 0); - return key2ex_hash; -} - -int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, - MESA_htable_handle key2ex_hash, void *logger) -{ - size_t key_offset=0, key_len=0; - const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); - int ret=0; - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data del error: cannot find column %d of %s", - plugin_desc->key_column, line); - return -1; - } - ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data del error: no such key %.*s of %s", - key_len, line+key_offset, line); - return -1; - } - return 0; -} - -int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, - MESA_htable_handle key2ex_hash, void *logger) -{ - char* key=NULL; - size_t key_offset=0, key_len=0; - MAAT_RULE_EX_DATA exdata=NULL; - struct wrap_plugin_EX_data* wrap_data=NULL; - const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); - int ret=0; - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data add error: cannot find column %d of %s", - plugin_desc->key_column, line); - return -1; - } - key=ALLOC(char, key_len+1); - memcpy(key, line+key_offset, key_len); - plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, - plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); - wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); - wrap_data->exdata=exdata; - wrap_data->ref_plugin_table=plugin_table; - ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); - free(key); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Plugin EX data add error: duplicated key %.*s of %s", - key_len, line+key_offset, line); - wrap_plugin_EX_data_free(wrap_data); - return -1; - } - return 0; -} -MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table, - struct dynamic_array_t* lines, size_t line_cnt, void* logger) -{ - MESA_htable_handle key2ex_hash=NULL; - size_t i=0; - const char* line=NULL; - const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); - - key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); - - for(i=0; i< line_cnt; i++) - { - line=(const char*)dynamic_array_read(lines, i); - plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); - } - return key2ex_hash; -} int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, @@ -1259,39 +1134,35 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; - struct plugin_table_desc* plugin_desc=&(table_desc->plugin); - struct Maat_table_runtime* table_rt=NULL; + int idx=-1; + if(new_func==NULL || free_func==NULL || dup_func==NULL ) { assert(0); MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); return -1; } - if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata - || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) - { - assert(0); - MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, - "%s failed: key or valid flag column are not specified", __FUNCTION__); - return -1; - } - pthread_mutex_lock(&(_feather->background_update_mutex)); - plugin_desc->ex_desc.new_func=new_func; - plugin_desc->ex_desc.free_func=free_func; - plugin_desc->ex_desc.dup_func=dup_func; - plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. - plugin_desc->ex_desc.argl=argl; - plugin_desc->ex_desc.argp=argp; - plugin_desc->have_exdata=1; + idx=Maat_table_plugin_new_ex_index(_feather->table_mgr, table_id, + new_func, + free_func, + dup_func, + key2index_func, + argl, argp); + if(idx<0) + { + pthread_mutex_unlock(&(_feather->background_update_mutex)); + return -1; + } + struct Maat_table_desc *table_desc=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct Maat_table_runtime* table_rt=NULL; + + if(_feather->scanner!=NULL) { - table_rt=_feather->scanner->table_rt[table_id]; - assert(table_rt->plugin.key2ex_hash==NULL); - table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, - table_rt->plugin.cache_line_num, _feather->logger); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + Maat_table_runtime_plugin_new_ex_idx(table_rt, table_desc, _feather->logger); } pthread_mutex_unlock(&(_feather->background_update_mutex)); @@ -1300,28 +1171,16 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) { struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; - struct Maat_table_runtime *table_rt= NULL; - struct plugin_table_desc* plugin_desc=&(table_desc->plugin); - struct wrap_plugin_EX_data* wrap_data=NULL; + struct Maat_table_desc *table_desc=NULL; + struct Maat_table_runtime *table_rt=NULL; MAAT_RULE_EX_DATA exdata=NULL; - if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0) - { - assert(0); - return NULL; - } if(_feather->scanner==NULL) { return NULL; } - table_rt= _feather->scanner->table_rt[table_id]; - wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, - (const unsigned char*)key, strlen(key)); - if(wrap_data!=NULL) - { - plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata), - plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); - } + table_desc=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id); + exdata=Maat_table_runtime_plugin_get_ex_data(table_rt, table_desc, key); return exdata; } @@ -1332,6 +1191,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { int region_ret=0,compile_ret=0,hit_region_cnt=0; unsigned int sub_type=0; + int virtual_table_id=0; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _OUTER_scan_status_t* _mid=(struct _OUTER_scan_status_t*)(*mid); @@ -1355,7 +1215,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id clock_gettime(CLOCK_MONOTONIC,&start); } _mid=grab_mid(mid,_feather, thread_num, 0); - p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1369,11 +1229,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(expr_desc->do_charset_merge==1) { - sub_type=make_sub_type(table_id,CHARSET_NONE,0); + sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0); } else { - sub_type=make_sub_type(table_id,charset,0); + sub_type=make_sub_type(p_table->table_id, charset,0); } alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); scan_data_t scan_data; @@ -1381,7 +1241,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id scan_data.text_data.tlen=data_len; scan_data.text_data.toffset=0; - struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -1403,7 +1263,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id if(table_rt->expr.regex_rule_cnt>0) { scan_data.rule_type=RULETYPE_REG; - scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); + scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE,0); region_ret=rulescan_search(my_scanner->region, thread_num, &scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); if(region_ret>0) { @@ -1421,11 +1281,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1); } - _mid=grab_mid(mid,_feather,thread_num, 1); + _mid=grab_mid(mid, _feather, thread_num, 1); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - hit_region_cnt, + ®ion_hit_wraper, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1481,16 +1341,15 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id _compile_result_t compile_result[rule_num]; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct Maat_scanner* my_scanner=NULL; - intval_scan_data.rule_type=RULETYPE_INT; - intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); - intval_scan_data.int_data=intval; + Maat_table_desc* p_table=NULL; struct timespec start,end; if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&start); } - p_table=acqurie_table(_feather,table_id,TABLE_TYPE_INTERVAL); + int virutal_table_id=0; + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_INTERVAL, &virutal_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1501,11 +1360,15 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; } + intval_scan_data.rule_type=RULETYPE_INT; + intval_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0); + intval_scan_data.int_data=intval; + alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; @@ -1525,10 +1388,10 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); } _mid=grab_mid(mid, _feather, thread_num, 1); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, _mid->is_last_region, virutal_table_id); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - region_ret, + ®ion_hit_wraper, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1576,7 +1439,8 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { clock_gettime(CLOCK_MONOTONIC,&start); } - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP); + int virtual_table_id=0; + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_IP, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1587,7 +1451,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -1604,7 +1468,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); ip_scan_data.rule_type=RULETYPE_IPv4; - ip_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); + ip_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0); switch(addr->addrtype) { case ADDR_TYPE_IPV4: @@ -1647,11 +1511,10 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); } _mid=grab_mid(mid, _feather, thread_num, 1); - + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, region_ret, _mid->is_last_region, virtual_table_id); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - region_ret, + ®ion_hit_wraper, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -1694,8 +1557,9 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct Maat_scanner* scanner=NULL; struct Maat_table_desc *p_table=NULL; + int virtual_table_id=0; assert(thread_num<_feather->scan_thread_num); - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR); + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_EXPR, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1706,6 +1570,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1); scanner=_feather->scanner; sp->feather=_feather; + sp->p_real_table=p_table; + sp->virtual_table_id=virtual_table_id; sp->version=_feather->maat_version; sp->process_offset=0; sp->rs_stream_para=NULL; @@ -1713,14 +1579,14 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { return sp; } - struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; + + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); if(table_rt->origin_rule_num==0) { return sp; } INC_SCANNER_REF(scanner, thread_num); - sp->table_id=table_id; sp->thread_num=thread_num; sp->max_cross_size=expr_desc->cross_cache_size; sp->caching_size=0; @@ -1757,16 +1623,15 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para scan_result_t *region_result; _compile_result_t compile_result[rule_num];//dynamic array scan_data_t region_scan_data; - Maat_table_desc* p_table=NULL; - struct timespec start,end; + struct timespec start,end; if(data==NULL||data_len<=0||scanner==NULL) { return 0; - } - struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; + } + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); if(sp->feather->perf_on==1) { - clock_gettime(CLOCK_MONOTONIC,&start); + clock_gettime(CLOCK_MONOTONIC, &start); } _mid=grab_mid(mid, sp->feather, sp->thread_num,0); @@ -1774,11 +1639,10 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para { return 0; } - p_table=sp->feather->p_table_info[sp->table_id]; - //table rule num is already judged in Maat_stream_scan_string_start + //table rule num is already in Maat_stream_scan_string_start - if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) + if(sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) { sp->feather->scan_err_cnt++; return -1; @@ -1788,11 +1652,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para *detail_ret=0; if(sp->do_merge==1) { - sub_type=make_sub_type(sp->table_id,CHARSET_NONE,0); + sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE, 0); } else { - sub_type=make_sub_type(sp->table_id,charset,0); + sub_type=make_sub_type(sp->p_real_table->table_id, charset, 0); } if(sp->max_cross_size>0&&sp->caching_size>0) { @@ -1844,7 +1708,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->do_regex==1) { region_scan_data.rule_type=RULETYPE_REG; - region_scan_data.sub_type=make_sub_type(sp->table_id,CHARSET_NONE,0); + region_scan_data.sub_type=make_sub_type(sp->p_real_table->table_id, CHARSET_NONE,0); region_ret=rulescan_searchstream(sp->rs_stream_para, ®ion_scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt); if(region_ret<0) { @@ -1856,7 +1720,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para hit_region_cnt+=region_ret; } } - if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS) + if(hit_region_cnt>0&&sp->p_real_table->table_type==TABLE_TYPE_EXPR_PLUS) { hit_region_cnt=match_district(_mid, region_result, hit_region_cnt, scanner); } @@ -1867,10 +1731,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1); } _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_rulescan(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id); + compile_ret=region_compile(sp->feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), - hit_region_cnt, + ®ion_hit_wraper, result,compile_result,rule_num, sp->thread_num); assert(_mid->is_last_region<2); @@ -1936,7 +1801,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) struct Maat_table_runtime* table_rt=NULL; if(scanner!=NULL) { - table_rt=scanner->table_rt[sp->table_id]; + table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, sp->p_real_table->table_id); alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); } @@ -1977,7 +1842,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, struct Maat_scanner* scanner=NULL; sfh_instance_t * tmp_fuzzy_handle=NULL; struct Maat_table_desc *p_table=NULL; - p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST); + int virtual_table_id=0; + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_DIGEST, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -1986,13 +1852,15 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1); scanner=_feather->scanner; sp->feather=_feather; + sp->p_real_table=p_table; + sp->virtual_table_id=virtual_table_id; sp->version=_feather->maat_version; sp->process_offset=0; if(scanner==NULL) { return sp; } - struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table_id); tmp_fuzzy_handle=SFH_instance(total_len); if(tmp_fuzzy_handle==NULL) { @@ -2001,7 +1869,6 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, } INC_SCANNER_REF(scanner, thread_num); - sp->table_id=table_id; sp->thread_num=thread_num; sp->total_len=total_len; sp->fuzzy_hash_handle=tmp_fuzzy_handle; @@ -2040,7 +1907,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); int do_query=0; - GIE_result_t query_result[MAX_SCANNER_HIT_NUM]; int hit_region_cnt=0,compile_ret=0; _compile_result_t compile_result[rule_num];//dynamic array if(data==NULL||data_len<=0) @@ -2051,7 +1917,9 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return 0; } - struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + GIE_result_t* region_result=sp->feather->scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num; + + struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id); GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; unsigned long long digest_len=0; char* digest_buff=NULL; @@ -2084,7 +1952,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int if(GIE_handle!=NULL) { - hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), query_result, MAX_SCANNER_HIT_NUM); + hit_region_cnt=GIE_query(GIE_handle, digest_buff,(int)strlen(digest_buff), region_result, MAX_SCANNER_HIT_NUM); } free(digest_buff); @@ -2102,10 +1970,10 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1); } _mid=grab_mid(mid,sp->feather, sp->thread_num,1); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, sp->virtual_table_id); compile_ret=region_compile(sp->feather,_mid->inner, - _mid->is_last_region, - query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), - hit_region_cnt, + ®ion_hit_wraper, result,compile_result,rule_num, sp->thread_num); assert(_mid->is_last_region<2); @@ -2134,7 +2002,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct Maat_scanner* scanner=sp->feather->scanner; - struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + struct Maat_table_runtime *table_rt=Maat_table_runtime_get(sp->feather->scanner->table_rt_mgr, sp->p_real_table->table_id); alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); if(scanner!=NULL) { @@ -2227,7 +2095,6 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { int hit_region_cnt=0,compile_ret=0; struct _OUTER_scan_status_t* _mid=NULL; - GIE_result_t region_result[MAX_SCANNER_HIT_NUM]; _compile_result_t compile_result[rule_num]; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct Maat_scanner* my_scanner=NULL; @@ -2237,7 +2104,8 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { clock_gettime(CLOCK_MONOTONIC,&start); } - p_table=acqurie_table(_feather,table_id,TABLE_TYPE_SIMILARITY); + int virtual_table_id=0; + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_SIMILARITY, &virtual_table_id); if(p_table==NULL) { _feather->scan_err_cnt++; @@ -2248,7 +2116,9 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { return 0; } - struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + GIE_result_t* region_result=my_scanner->gie_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; + + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(my_scanner->table_rt_mgr, p_table->table_id); if(table_rt->origin_rule_num==0) { return 0; @@ -2257,7 +2127,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id INC_SCANNER_REF(my_scanner,thread_num); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); - hit_region_cnt=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM); + hit_region_cnt=GIE_query(gie_handle, data, data_len, region_result, MAX_SCANNER_HIT_NUM); if(hit_region_cnt<0) { DEC_SCANNER_REF(my_scanner, thread_num); @@ -2268,10 +2138,10 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); + struct scan_region_hit_wraper region_hit_wraper; + scan_region_hit_wraper_build_with_GIE(®ion_hit_wraper, region_result, hit_region_cnt, _mid->is_last_region, virtual_table_id); compile_ret=region_compile(_feather,_mid->inner, - _mid->is_last_region, - region_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), - hit_region_cnt, + ®ion_hit_wraper, result,compile_result,rule_num, thread_num); assert(_mid->is_last_region<2); @@ -2320,7 +2190,7 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu } break; case MAAT_STATE_LAST_UPDATING_TABLE: - *int_val=_feather->is_last_plugin_table_updating; + *int_val=Maat_table_manager_is_last_plugin_table_updating(_feather->table_mgr); break; case MAAT_STATE_IN_UPDATING: if(size!=sizeof(int)) diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index a86d303..a51b4de 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -143,9 +143,11 @@ long long redis_server_time(redisContext* ctx) long long server_time=0; redisReply* data_reply=NULL; data_reply=_wrap_redisCommand(ctx,"TIME"); - assert(data_reply->type==REDIS_REPLY_ARRAY); - server_time=atoll(data_reply->element[0]->str); - freeReplyObject(data_reply); + if(data_reply->type==REDIS_REPLY_ARRAY) + { + server_time=atoll(data_reply->element[0]->str); + freeReplyObject(data_reply); + } return server_time; } enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p) @@ -245,11 +247,17 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz) { if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD; - snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d", p_group->group_id, + const char* vt_name="null"; + if(p_group->virtual_table_name!=NULL) + { + vt_name=p_group->virtual_table_name; + } + snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d\t%s", p_group->group_id, p_group->parent_id, op, p_group->not_flag, - p_group->parent_type); + p_group->parent_type, + vt_name); return; } void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz) @@ -274,65 +282,65 @@ void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, s switch(p->region_type) { case REGION_IP: - ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1" - ,p->region_id - ,group_id - ,p->ip_rule.addr_type - ,p->ip_rule.src_ip - ,p->ip_rule.mask_src_ip - ,p->ip_rule.src_port - ,p->ip_rule.mask_src_port - ,p->ip_rule.dst_ip - ,p->ip_rule.mask_dst_ip - ,p->ip_rule.dst_port - ,p->ip_rule.mask_dst_port - ,p->ip_rule.protocol - ,p->ip_rule.direction); + ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1", + p->region_id, + group_id, + p->ip_rule.addr_type, + p->ip_rule.src_ip, + p->ip_rule.mask_src_ip, + p->ip_rule.src_port, + p->ip_rule.mask_src_port, + p->ip_rule.dst_ip, + p->ip_rule.mask_dst_ip, + p->ip_rule.dst_port, + p->ip_rule.mask_dst_port, + p->ip_rule.protocol, + p->ip_rule.direction); break; case REGION_EXPR: if(p->expr_rule.district==NULL) { - ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1" - ,p->region_id - ,group_id - ,p->expr_rule.keywords - ,p->expr_rule.expr_type - ,p->expr_rule.match_method - ,p->expr_rule.hex_bin); + ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1", + p->region_id, + group_id, + p->expr_rule.keywords, + p->expr_rule.expr_type, + p->expr_rule.match_method, + p->expr_rule.hex_bin); } else //expr_plus { - ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1" - ,p->region_id - ,group_id - ,p->expr_rule.keywords - ,p->expr_rule.district - ,p->expr_rule.expr_type - ,p->expr_rule.match_method - ,p->expr_rule.hex_bin); + ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1", + p->region_id, + group_id, + p->expr_rule.district, + p->expr_rule.keywords, + p->expr_rule.expr_type, + p->expr_rule.match_method, + p->expr_rule.hex_bin); } break; case REGION_INTERVAL: - ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1" - ,p->region_id - ,group_id - ,p->interval_rule.low_boundary - ,p->interval_rule.up_boundary); + ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1", + p->region_id, + group_id, + p->interval_rule.low_boundary, + p->interval_rule.up_boundary); break; case REGION_DIGEST: - ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1" - ,p->region_id - ,group_id - ,p->digest_rule.orgin_len - ,p->digest_rule.digest_string - ,p->digest_rule.confidence_degree); + ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1", + p->region_id, + group_id, + p->digest_rule.orgin_len, + p->digest_rule.digest_string, + p->digest_rule.confidence_degree); break; case REGION_SIMILARITY: - ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1" - ,p->region_id - ,group_id - ,p->similarity_rule.target - ,p->similarity_rule.threshold); + ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1", + p->region_id, + group_id, + p->similarity_rule.target, + p->similarity_rule.threshold); break; default: assert(0); @@ -915,7 +923,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ continue; } region_cmd=&(group_cmd->regions[j]); - region_cmd->table_name=_maat_strdup(feather->p_table_info[region_inner->table_id]->table_name[0]); + region_cmd->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, region_inner->table_id)); region_cmd->region_id=region_inner->region_id; //NOTICE: region_type only avilable when OP_ADD, region_cmd->region_type=REGION_EXPR; @@ -1374,7 +1382,7 @@ error_out: int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) { - int i=0,j=0,ret=0; + int i=0, j=0; const char *table_name=NULL; int table_id=0; struct Maat_group_t* p_group=NULL; @@ -1399,8 +1407,8 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) { p_region=&(p_group->regions[j]); table_name=p_region->table_name; - ret=map_str2int(feather->map_tablename2id, table_name, &table_id); - if(ret<0) + table_id=Maat_table_get_id_by_name(feather->table_mgr, table_name); + if(table_id<0) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module ,"Unknown table %s of Maat_cmd_t[%d]->group[%d]->region[%d]." @@ -1409,7 +1417,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) return -1; } table_type=type_region2table(p_region); - if(table_type!=feather->p_table_info[table_id]->table_type) + if(table_type!=Maat_table_get_type_by_id(feather->table_mgr, table_id)) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module ,"Table %s not support region type %d of Maat_cmd_t[%d]->group[%d]->region[%d]." @@ -1419,7 +1427,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) return -1; } free((char*)p_region->table_name); - p_region->table_name=_maat_strdup(feather->p_table_info[table_id]->table_name[0]); + p_region->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, table_id)); } } return 0; @@ -1434,7 +1442,10 @@ void check_maat_expiration(redisContext *ctx, void *logger) long long server_time=0; server_time=redis_server_time(ctx); - + if(!server_time) + { + return; + } data_reply=_wrap_redisCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_expire_sset,server_time); if(data_reply->type!=REDIS_REPLY_ARRAY||data_reply->elements==0) { @@ -1473,7 +1484,10 @@ void cleanup_update_status(redisContext *ctx, void *logger) long long server_time=0, version_upper_bound=0,version_lower_bound=0,version_num=0,entry_num=0; server_time=redis_server_time(ctx); - + if(!server_time) + { + return; + } reply=_wrap_redisCommand(ctx,"MULTI"); freeReplyObject(reply); redisAppendCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME); @@ -1599,10 +1613,9 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p) } void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n_foreign, const char* dir, void* logger) { - int ret=0, i=0; + int i=0; const char* p_foreign=NULL; int foreign_key_size=0; - struct stat file_info; p_rule->n_foreign=n_foreign; p_rule->f_keys=ALLOC(struct foreign_key, n_foreign); for(i=0; if_keys[i].key=(char*)calloc(sizeof(char),foreign_key_size+1); memcpy(p_rule->f_keys[i].key, p_foreign, foreign_key_size); - p_rule->f_keys[i].filename=get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[i].key, dir); - ret=stat(p_rule->f_keys[i].filename, &file_info); - if(ret==0) - { - p_rule->f_keys[i].is_existed=1; - } } if(i!=n_foreign) { @@ -1651,7 +1658,7 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n } int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger) { - int ret=0, table_id=0, i=0; + int i=0; int rule_with_foreign_key=0; struct Maat_table_desc* p_table=NULL; struct plugin_table_desc* plugin_desc=NULL; @@ -1661,14 +1668,13 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, { continue; } - ret=map_str2int(feather->map_tablename2id, rule_list[i].table_name, &table_id); - if(ret<0) + p_table=Maat_table_get_desc_by_name(feather->table_mgr, rule_list[i].table_name); + if(!p_table||p_table->table_type!=TABLE_TYPE_PLUGIN) { continue; } - p_table=feather->p_table_info[table_id]; plugin_desc= &(p_table->plugin); - if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0) + if(plugin_desc->n_foreign==0) { continue; } @@ -1722,25 +1728,44 @@ void _get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int redisReply* reply=NULL; struct serial_rule_t*p=NULL; FILE* fp=NULL; + struct stat file_info; + for(i=0;iop==MAAT_OP_DEL||p->n_foreign==0) + if(p->n_foreign==0) { continue; } - for(j=0; jn_foreign; j++) + if(p->op==MAAT_OP_DEL) { - if(p->f_keys[j].is_existed==1) + for(j=0; jn_foreign; j++) + { + ret=stat(p->f_keys[i].filename, &file_info); + if(ret==0) + { + continue; + } + snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key); + ret=redisAppendCommand(ctx, redis_cmd); + track[key_num].rule_idx=i; + track[key_num].foreign_idx=j; + key_num++; + assert(ret==REDIS_OK); } - snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key); - ret=redisAppendCommand(ctx, redis_cmd); - track[key_num].rule_idx=i; - track[key_num].foreign_idx=j; - key_num++; - assert(ret==REDIS_OK); } } for(i=0;ilogger; @@ -1889,17 +1915,18 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m { continue; } - ret=map_str2int(feather->map_tablename2id,rule_list[i].table_name,&table_id); - if(ret<0)//Unrecognized table. + table_id=Maat_table_get_id_by_name(feather->table_mgr, rule_list[i].table_name); + if(table_id<0)//Unrecognized table. { continue; } - table_type=feather->p_table_info[table_id]->table_type; + table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id); if(rule_list[i].op==MAAT_OP_DEL) { if(table_type==TABLE_TYPE_PLUGIN) { - plugin_desc=&(feather->p_table_info[table_id]->plugin); + table_desc=Maat_table_get_by_id(feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + plugin_desc=&(table_desc->plugin); valid_column=plugin_desc->valid_flag_column; } else @@ -1919,33 +1946,6 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m rewrite_table_line_with_foreign(rule_list+i); } update(rule_list[i].table_name,rule_list[i].table_line,u_para); - if(rule_list[i].n_foreign&&rule_list[i].op==MAAT_OP_DEL) - { - - for(j=0; jforeign_cont_linger==0) - { - ret=system_cmd_rm(rule_list[i].f_keys[j].filename); - if(ret==-1) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, - "Foreign content file %s remove failed.", - rule_list[i].f_keys[j].filename); - } - } - else if(feather->foreign_cont_linger>0) - { - garbage_bagging_with_timeout(GARBAGE_FOREIGN_FILE, rule_list[i].f_keys[j].filename, feather->foreign_cont_linger, feather->garbage_q); - rule_list[i].f_keys[j].filename=NULL;//transfer owner to garbage collection. - } - else - { - //Less than 0, don't delete. - } - } - - } } finish(u_para); @@ -2102,11 +2102,15 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru return -1; } server_time=redis_server_time(write_ctx); + if(!server_time) + { + return -1; + } s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),line_num); for(i=0;imap_tablename2id, line_rule[i]->table_name, &table_id); - if(ret<0) + table_id=Maat_table_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name); + if(table_id<0) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command ,"Command set line id %d failed: unknown table %s." @@ -2115,9 +2119,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru ret=-1; goto error_out; } - p_table=_feather->p_table_info[table_id]; - plugin_desc=&(p_table->plugin); - if(TABLE_TYPE_PLUGIN!=p_table->table_type) + p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + if(!p_table) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command ,"Command set line id %d failed: table %s is not a plugin table." @@ -2126,6 +2129,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru ret=-1; goto error_out; } + plugin_desc=&(p_table->plugin); if(op==MAAT_OP_ADD) { ret=get_valid_flag_offset(line_rule[i]->table_line @@ -2361,7 +2365,7 @@ int Maat_cmd_commit(Maat_feather_t feather) { _Maat_feather_t* _feather=(_Maat_feather_t*)feather; - int ret=0,i=0; + int ret=-1, i=0; int new_region_num=0,new_group_num=0; int serial_rule_num=0,serial_rule_idx=0; UNUSED int transaction_success=1; @@ -2373,7 +2377,7 @@ int Maat_cmd_commit(Maat_feather_t feather) struct serial_rule_t* s_rule=NULL; if(_feather->input_mode!=SOURCE_REDIS) { - return -1; + return ret; } if(_feather->cmd_q_cnt==0) { @@ -2391,14 +2395,16 @@ int Maat_cmd_commit(Maat_feather_t feather) p=p->next; } _feather->server_time=redis_server_time(write_ctx); - + if(!_feather->server_time) + { + goto error_out; + } if(_feather->AUTO_NUMBERING_ON==1) { data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num); if(data_reply->type!=REDIS_REPLY_INTEGER) { freeReplyObject(data_reply); - ret=-1; goto error_out; } _feather->base_rgn_seq=data_reply->integer-new_region_num; @@ -2408,7 +2414,6 @@ int Maat_cmd_commit(Maat_feather_t feather) if(data_reply->type!=REDIS_REPLY_INTEGER) { freeReplyObject(data_reply); - ret=-1; goto error_out; } _feather->base_grp_seq=data_reply->integer-new_group_num; @@ -2611,6 +2616,10 @@ static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAA { redisContext* write_ctx=get_redis_ctx_for_write(_feather); _feather->server_time=redis_server_time(write_ctx); + if(!_feather->server_time) + { + return -1; + } struct serial_rule_t s_rule; set_serial_rule(&s_rule, op, id, 0, table_name, line, 0); int transaction_success=0; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 0dd5571..3d93ad8 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -32,11 +33,26 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_7_20190629=1; - -const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", - "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; +int MAAT_FRAME_VERSION_2_8_20200113=1; +int is_valid_table_name(const char* str) +{ + size_t i=0, integer_cnt=0; + for(i=0; i='0'&&str[i]<='9') + { + integer_cnt++; + } + } + if(strlen(str)==0 || + integer_cnt==strlen(str) || + 0==strcasecmp(str, "null")) + { + return 0; + } + return 1; +} int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type) { switch(expr_type) @@ -67,8 +83,8 @@ int is_valid_match_method(enum MAAT_MATCH_METHOD match_method) iconv_t maat_iconv_open(struct Maat_scanner* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from) { - const char *from_s=CHARSET_STRING[from]; - const char *to_s=CHARSET_STRING[to]; + const char *from_s=charset_get_name(from); + const char *to_s=charset_get_name(to); iconv_t cd; if(from==CHARSET_GBK&&to==CHARSET_BIG5) { @@ -219,6 +235,7 @@ int universal_charset_convert(struct Maat_scanner* scanner,enum MAAT_CHARSET fro case CHARSET_BIG5: case CHARSET_UNICODE: case CHARSET_UTF8: + case CHARSET_WINDOWS1251: ret=iconv_convert(scanner,from,to,src,srclen,dst,dstlen); return ret; break; @@ -481,11 +498,7 @@ error_out: cJSON_Delete(json); return ret; } -int lqueue_destroy_cb(void *data, long data_len, void *arg) -{ - assert(0); - return 0; -} + void * HASH_fetch_by_id(MESA_htable_handle hash,int id) { return MESA_htable_search(hash,(unsigned char*)&(id),sizeof(id)); @@ -523,323 +536,6 @@ void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_ } -int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map) -{ - int j=0,ret[4]={0}; - char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; - char *token=NULL,*sub_token=NULL,*saveptr; - struct expr_table_desc* p=&(table->expr); - sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) - ,table->table_name[0] - ,table_type - ,src_charset - ,dst_charset - ,merge - ,&(p->cross_cache_size) - ,quick_str_scan); - memset(ret,0,sizeof(ret)); - ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); - ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset)); - ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge)); - if(strlen(quick_str_scan)>0) - { - ret[3]=map_str2int(string2int_map,str_tolower(quick_str_scan),&(p->quick_expr_switch)); - } - memset(quick_str_scan,0,sizeof(quick_str_scan)); - - for(j=0;j<4;j++) - { - if(ret[j]<0) - { - return -1; - } - } - j=0; - for (token = dst_charset; ; token= NULL) - { - sub_token= strtok_r(token,"/", &saveptr); - if (sub_token == NULL) - break; - ret[3]=map_str2int(string2int_map,str_tolower(sub_token),(int*)&(p->dst_charset[j])); - if(ret[3]>0) - { - if(p->dst_charset[j]==p->src_charset) - { - p->src_charset_in_dst=TRUE; - } - j++; - } - else - { - return -1; - } - - } - return 0; -} -Maat_table_desc* table_info_new(int max_thread_num) -{ - struct Maat_table_desc*p=ALLOC(struct Maat_table_desc, 1); - p->conj_cnt=1; - return p; -} -void table_info_free(struct Maat_table_desc*p) -{ - free(p); - return; -} -int _read_integer_arrary(char* string, int *array, int size) -{ - char *token=NULL,*sub_token=NULL,*saveptr; - int i=0; - for (token = string, i=0; iplugin); - copy_line=_maat_strdup(line); - ret=get_column_pos(copy_line, COLUMN_PLUGIN_DESCR_JSON, &offset, &len); - if(i<0) - { - goto error_out; - } - if(offset+lenvalid_flag_column)); - if(ret==0||ret==EOF) - { - plugin_desc->valid_flag_column=-1; - } - free(copy_line); - return 0; - } - json=cJSON_Parse(plug_info); - if(!json) - { - goto error_out; - } - tmp=cJSON_GetObjectItem(json, "key"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->key_column=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "valid"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->valid_flag_column=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "tag"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->rule_tag_column=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "estimate_size"); - if(tmp!=NULL) - { - assert(tmp->type==cJSON_Number); - plugin_desc->estimate_size=tmp->valueint; - } - tmp=cJSON_GetObjectItem(json, "foreign"); - if(tmp!=NULL) - { - if(tmp->type==cJSON_String) - { - plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM); - } - else if(tmp->type==cJSON_Array) - { - plugin_desc->n_foreign= cJSON_GetArraySize(tmp); - for(i=0;in_foreign; i++) - { - array_item=cJSON_GetArrayItem(tmp, i); - assert(array_item->type==cJSON_Number); - plugin_desc->foreign_columns[i]=array_item->valueint; - } - } - } - cJSON_Delete(json); - - free(copy_line); - return 0; -error_out: - free(copy_line); - return -1; -} -int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) -{ - FILE*fp=NULL; - char line[MAX_TABLE_LINE_SIZE]; - int i=0,ret=0,table_cnt=0; - char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; - MESA_htable_handle string2int_map=NULL;; - struct Maat_table_desc*p=NULL; - struct Maat_table_desc*conj_table=NULL; - fp=fopen(table_info_path,"r"); - if(fp==NULL) - { - fprintf(stderr,"Maat read table info %s error.\n",table_info_path); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s failed: %s.\n", table_info_path, strerror(errno)); - return 0; - } - - string2int_map=map_create(); - map_register(string2int_map,"expr", TABLE_TYPE_EXPR); - map_register(string2int_map,"ip", TABLE_TYPE_IP); - map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS); - map_register(string2int_map,"compile", TABLE_TYPE_COMPILE); - map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); - map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL); - map_register(string2int_map,"digest", TABLE_TYPE_DIGEST); - map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS); - map_register(string2int_map,"group", TABLE_TYPE_GROUP); - map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY); - map_register(string2int_map,"quickoff",0); - map_register(string2int_map,"quickon",1); - map_register(string2int_map,"escape",USER_REGION_ENCODE_ESCAPE); -// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet - - for(i=0;i0) - { - map_register(string2int_map,CHARSET_STRING[i], i); - } - else - { - break; - } - } - - map_register(string2int_map,"yes", 1); - map_register(string2int_map,"no", 0); - - - i=0; - while(NULL!=fgets(line,sizeof(line),fp)) - { - i++; - - if(line[0]=='#'||line[0]==' '||line[0]=='\t'||strlen(line)<4) - { - continue; - } - p=table_info_new(max_thread_num); - - ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) - ,p->table_name[0] - ,table_type_str - ,not_care); - if(ret<3) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error: not enough column.",table_info_path,i); - continue; - } - ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type)); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:invalid table type.",table_info_path,i); - goto invalid_table; - } - switch(p->table_type) - { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - ret=read_expr_table_info(line, p, string2int_map); - if(ret<0) - { - fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal column.",table_info_path,i); - goto invalid_table; - } - break; - case TABLE_TYPE_PLUGIN: - ret=read_plugin_table_description(line, p); - if(ret<0) - { - fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal plugin info.",table_info_path,i); - goto invalid_table; - } - break; - case TABLE_TYPE_COMPILE: - ret=sscanf(not_care,"%[a-z0-9]",tmp_str); - if(ret>0) - { - ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); - } - if(ret!=1) - { - p->compile.user_region_encoding=USER_REGION_ENCODE_NONE; - } - default: - break; - } - - if(p->table_id>=num) - { - fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,num); - - goto invalid_table; - } - if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table; - { - conj_table=p_table_info[p->table_id]; - if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error:reach tableid %d conjunction upper limit." - ,table_info_path,i,p->table_id); - goto invalid_table; - } - memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN); - conj_table->conj_cnt++; - MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, - "Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)." - ,table_info_path,i,p->table_name[0] - ,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt); - //use goto to free the conjunctioned table_info - goto invalid_table; - } - - p_table_info[p->table_id]=p; - table_cnt++; - continue; -invalid_table: - table_info_free(p); - p=NULL; - } - fclose(fp); - map_destroy(string2int_map); - return table_cnt; -} struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Maat_scanner *scanner) { int ret=0; @@ -941,8 +637,9 @@ void make_group_set(struct Maat_compile_group_relation* relation, struct bool_ex { continue; } - a_set->items[j].item_id=group->group_id; - a_set->items[j].not_flag=relation->not_flag[j]; + //high 32 bit is virtual table id, low 32 bit is group id. + a_set->items[j].item_id=TO_RELATION_ID(relation->virtual_table_id[i], group->group_id); + a_set->items[j].not_flag=relation->not_flag[i]; if(a_set->items[j].not_flag) { *has_not=1; @@ -1257,127 +954,10 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) op_expr->rule_type=p_rule->rule_type; return; } -GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, - short cfds_lvl,struct Maat_group_inner* tag) -{ - GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); - int digest_len=0; - rule->id=id; - rule->operation=op; - if(digest!=NULL) - { - digest_len=strlen(digest); - rule->sfh=(char*)calloc(sizeof(char),digest_len+1); - memcpy(rule->sfh,digest,digest_len); - - } - rule->sfh_length=digest_len; - rule->cfds_lvl=cfds_lvl; - rule->tag=(void*)tag; - return rule; -} -void destroy_digest_rule(GIE_digest_t*rule) -{ - if(rule->sfh!=NULL) - { - free(rule->sfh); - rule->sfh=NULL; - } - free(rule); - rule=NULL; - return; -} - - -struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num) -{ - - struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); - table_rt->table_type=table_desc->table_type; - switch(table_desc->table_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - table_rt->similar.update_q=MESA_lqueue_create(0,0); - break; - case TABLE_TYPE_PLUGIN: - table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); - if(table_desc->plugin.have_exdata) - { - table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, - table_desc->plugin.ex_desc.key2index_func); - } - break; - default: - break; - } - - table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); - table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); - table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); - table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); - table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); - return table_rt; -} -void table_runtime_free(struct Maat_table_runtime* p) -{ - long q_cnt=0,data_size=0; - int i=0; - UNUSED int q_ret=0; - - GIE_digest_t* digest_rule=NULL; - if(p==NULL) - { - return; - } - switch(p->table_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - if(p->similar.gie_handle!=NULL) - { - GIE_destory(p->similar.gie_handle); - } - if(p->similar.update_q!=NULL) - { - q_cnt=MESA_lqueue_get_count(p->similar.update_q); - for(i=0;isimilar.update_q,&digest_rule,&data_size); - assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); - destroy_digest_rule(digest_rule); - } - MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); - } - break; - case TABLE_TYPE_PLUGIN: - dynamic_array_destroy(p->plugin.cache_lines, free); - p->plugin.cache_lines=NULL; - if(p->plugin.key2ex_hash!=NULL) - { - MESA_htable_destroy(p->plugin.key2ex_hash, NULL); - } - default: - break; - } - - alignment_int64_array_free(p->scan_cnt); - alignment_int64_array_free(p->scan_cpu_time); - alignment_int64_array_free(p->input_bytes); - alignment_int64_array_free(p->stream_num); - alignment_int64_array_free(p->hit_cnt); - free(p); - return; -} struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *feather) { int scan_thread_num=feather->scan_thread_num; -// int rs_scan_type=feather->rule_scan_type; - struct Maat_table_desc ** pp_table_desc=feather->p_table_info; - struct Maat_table_runtime* table_rt=NULL; - int i=0; UNUSED int ret=0; MESA_htable_create_args_t hargs; @@ -1434,8 +1014,8 @@ struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *f scanner->region_update_q=MESA_lqueue_create(0, 0); scanner->region=rulescan_initialize(scan_thread_num); - //For best performance test: - //1.Do NOT set this option,rulescan return no hit detail as default; + //For best scan performance: + //1.Do NOT set this option, rulescan return no hit detail as default; //2.Set necessary STR rule to QUICK; if(feather->rule_scan_type==1) { @@ -1449,16 +1029,9 @@ struct Maat_scanner* create_maat_scanner(unsigned int version,_Maat_feather_t *f scanner->tomb_ref=feather->garbage_q; scanner->logger_ref=feather->logger; scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); - - for(i=0;iscan_thread_num); - scanner->table_rt[i]=table_rt; - } + scanner->gie_rslt_buff=ALLOC(GIE_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); + scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num); + scanner->max_table_num=Maat_table_manager_get_size(feather->table_mgr); return scanner; } @@ -1496,6 +1069,8 @@ void destroy_maat_scanner(struct Maat_scanner*scanner) MESA_lqueue_destroy(scanner->region_update_q, lqueue_destroy_cb, NULL); free(scanner->region_rslt_buff); scanner->region_rslt_buff=NULL; + free(scanner->gie_rslt_buff); + scanner->gie_rslt_buff=NULL; alignment_int64_array_free(scanner->ref_cnt); scanner->ref_cnt=NULL; for(i=0;itable_rt[i]); - } + Maat_table_rt_manager_destroy(scanner->table_rt_mgr); + scanner->table_rt_mgr=NULL; + igraph_destroy(&scanner->group_graph); free(scanner); return; @@ -1554,9 +1128,8 @@ struct _region_stat_t int ipv6_rule_cnt; }; }; -void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, int size) +void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, size_t size) { - assert(op_expr->table_idp_expr->operation==0)//add { @@ -1596,7 +1169,7 @@ void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_sta void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner* maat_scanner) { - long i=0,data_size=0; + long data_size=0, i=0; unsigned int j=0; int ret=0; unsigned int failed_ids[MAX_FAILED_NUM]; @@ -1607,7 +1180,8 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, const long q_cnt=MESA_lqueue_get_count(expr_queue); struct timespec start,end; unsigned long long update_interval=0; - struct _region_stat_t region_counter[MAX_TABLE_NUM]; + size_t max_table_num=maat_scanner->max_table_num; + struct _region_stat_t region_counter[max_table_num]; memset(region_counter, 0, sizeof(region_counter)); struct Maat_table_runtime* table_rt=NULL; if(q_cnt==0) @@ -1636,7 +1210,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, } } - count_rs_region(op_expr,region_counter,MAX_TABLE_NUM); + count_rs_region(op_expr, region_counter, max_table_num); destroy_op_expr(op_expr); op_expr=NULL; } @@ -1663,9 +1237,9 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, ,(double)update_interval ,update_interval); //update scanner's region cnt; - for(i=0;itable_rt[i]; + table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, i); if(table_rt==NULL) { continue; @@ -1703,52 +1277,6 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, free(to_update_expr); } -void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct Maat_scanner* maat_scanner,int table_id) -{ - long i=0,data_size=0; - int ret=0; - GIE_digest_t* digest_rule=NULL; - GIE_digest_t** update_array=NULL; - UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; - const long q_cnt=MESA_lqueue_get_count(update_q); - if(q_cnt==0) - { - return; - } - struct Maat_table_runtime* table_rt=maat_scanner->table_rt[table_id]; - update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); - for(i=0;ioperation==GIE_INSERT_OPT) - { - table_rt->origin_rule_num++; - } - else - { - table_rt->origin_rule_num--; - } - destroy_digest_rule(update_array[i]); - update_array[i]=NULL; - } - free(update_array); - update_array=NULL; - return; -} struct region_group_relation { int region_id; @@ -1894,7 +1422,7 @@ unsigned int del_region_from_group(struct Maat_group_inner* group,int region_id, return i; } -int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int not_flag) +int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag) { int i=0,ret=-1; int write_pos=-1; @@ -1918,7 +1446,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa } else { - if(p->group_id==a_rule_group->group_id)//duplicate group + if(p->group_id==a_rule_group->group_id && relation->virtual_table_id[i]==virual_table_id)//duplicate group { ret=-1; goto error_out; @@ -1945,6 +1473,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa { relation->not_flag[write_pos]=0; } + relation->virtual_table_id[write_pos]=virual_table_id; relation->group_cnt++; a_rule_group->ref_by_parent_cnt++; ret=1; @@ -1953,7 +1482,7 @@ error_out: return ret; } -struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relation*relation, int group_id) +struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relation*relation, int group_id, int virual_table_id) { int i=0; struct Maat_group_inner* group_rule=NULL; @@ -1965,14 +1494,15 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati { continue; } - if(group_rule->group_id==group_id) + if(group_rule->group_id==group_id && relation->virtual_table_id[i]==virual_table_id) { dynamic_array_write(relation->groups,i,NULL); if(relation->not_flag[i]==1) { relation->not_group_cnt--; relation->not_flag[i]=0; - } + } + relation->virtual_table_id[i]=0; relation->group_cnt--; break; } @@ -2221,10 +1751,11 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "Table %s region cfg %d charset convert from %s to %s failed.",table->table_name - ,db_rule->region_id - ,CHARSET_STRING[expr_desc->src_charset] - ,CHARSET_STRING[dst_charset]); + "Table %s region cfg %d charset convert from %s to %s failed.", + table->table_name, + db_rule->region_id, + charset_get_name(expr_desc->src_charset), + charset_get_name(dst_charset)); free(region_string); op_expr->convert_failed++; expr_desc->iconv_err_cnt++; @@ -2366,34 +1897,25 @@ int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule* intval_ MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule* db_digest_rule,struct Maat_scanner *scanner,void* logger) +int add_digest_rule(struct Maat_table_desc* table, struct db_digest_rule* db_rule, struct Maat_scanner *scanner,void* logger) { struct Maat_group_inner* group_rule=NULL; - GIE_digest_t* digest_rule=NULL; struct Maat_group_inner* u_para=NULL; - struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime * table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); int expr_id=0,district_id=-1; - group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_digest_rule->group_id, 0, scanner); + group_rule=create_group_rule(db_rule->group_id, 0, scanner); } expr_id=scanner->exprid_generator++; - u_para=add_region_to_group(group_rule, table->table_id, db_digest_rule->region_id, district_id, expr_id, TABLE_TYPE_DIGEST, scanner); + u_para=add_region_to_group(group_rule, table->table_id, db_rule->region_id, district_id, expr_id, TABLE_TYPE_DIGEST, scanner); if(u_para==NULL) { return -1; } - if(table->table_type==TABLE_TYPE_SIMILARITY) - { - db_digest_rule->digest_string=str_unescape(db_digest_rule->digest_string); - } - digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT - ,db_digest_rule->digest_string - ,db_digest_rule->confidence_degree - ,group_rule); - MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); + Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, group_rule); scanner->gie_update_q_size++; return 0; } @@ -2403,8 +1925,8 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; int expr_num=0; struct Maat_group_inner* group_rule=NULL; + struct Maat_table_runtime* table_rt=NULL; struct op_expr_t* op_expr=NULL; - GIE_digest_t* digest_rule=NULL; group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); if(group_rule==NULL) { @@ -2443,11 +1965,8 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int case TABLE_TYPE_SIMILARITY: case TABLE_TYPE_DIGEST: assert(expr_num==1); - digest_rule=create_digest_rule(expr_id[0], GIE_DELETE_OPT //del digest - ,NULL - ,0 - ,NULL); - MESA_lqueue_join_tail(maat_scanner->table_rt[table->table_id]->similar.update_q,&digest_rule, sizeof(void*)); + table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id); + Maat_table_runtime_digest_del(table_rt, expr_id[0]); maat_scanner->gie_update_q_size++; break; default: @@ -2494,17 +2013,17 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro } else { - group_rule->has_compile_neighbors=1; + group_rule->ref_by_compile_cnt++; compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); if(compile_rule==NULL) { compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner); } - ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->not_flag); + ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->virtual_table_id, db_group_rule->not_flag); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "update error,add group: %s %d to compile rule %d error, compile rule is full or duplicate group.", + "update error, add group: %s %d to compile rule %d error, compile rule is full or duplicate group.", table->table_name[table->updating_name], db_group_rule->group_id, db_group_rule->parent_id); @@ -2582,7 +2101,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro db_group_rule->parent_id); return 0; } - group_rule=del_group_from_compile(relation, db_group_rule->group_id); + group_rule=del_group_from_compile(relation, db_group_rule->group_id, db_group_rule->virtual_table_id); if(group_rule==NULL) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, @@ -2596,6 +2115,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro { destroy_compile_group_relation(relation, scanner); } + group_rule->ref_by_compile_cnt--; } destroy_group_rule(group_rule, DESTROY_GROUP_BY_PARENT, scanner); scanner->to_update_group_cnt++; @@ -2648,18 +2168,20 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_ scanner->to_update_compile_cnt++; return 1; } -void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger) +void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner, struct Maat_table_manager* table_mgr, void* logger) { struct db_group_rule_t db_group_rule; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); int ret=0; + char virtual_table_name[MAX_TABLE_NAME_LEN]={0}; memset(&db_group_rule, 0, sizeof(db_group_rule)); - ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d", &(db_group_rule.group_id), + ret=sscanf(table_line,"%d\t%d\t%d\t%d\t%d\t%s", &(db_group_rule.group_id), &(db_group_rule.parent_id), &(db_group_rule.is_valid), &(db_group_rule.not_flag), - &(db_group_rule.parent_type)); - if(ret!=3&&ret!=4&&ret!=5) + &(db_group_rule.parent_type), + virtual_table_name); + if(ret!=3&&ret!=4&&ret!=5&&ret!=6) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , "update error, invalid format of group table %s:%s", @@ -2673,17 +2195,33 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru } if(db_group_rule.parent_type==PARENT_TYPE_GROUP && db_group_rule.not_flag) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module , "update error, invalid format of group table %s:%s not operation is forbidden for non-compile parent.", table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; return; } + if(is_valid_table_name(virtual_table_name)) + { + db_group_rule.virtual_table_id=Maat_table_get_id_by_name(table_mgr, virtual_table_name); + if(db_group_rule.virtual_table_id<0) + { + //This happens when one data source (e.g. redis) is consumed by multiple Maat instance. + //Maat ignores unrealated groups. + MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_module, + "group table load abandon, unknown virtual table name: %s of group table %s:%s.", + virtual_table_name, + table->table_name[table->updating_name], table_line); + table->udpate_err_cnt++; + return; + } + } if(db_group_rule.is_valid==FALSE) { ret=del_group_rule(table, &db_group_rule, scanner, logger); //leave no trace when compatible_group_update calling - if(table->table_type==TABLE_TYPE_GROUP&&ret==1) + assert(table->table_type==TABLE_TYPE_GROUP); + if(ret==1) { table_rt->origin_rule_num--; assert(table_rt->origin_rule_num>=0); @@ -2698,7 +2236,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru ret=add_group_rule(table,&db_group_rule, scanner, logger); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + MESA_handle_runtime_log(logger,RLOG_LV_FATAL, maat_module, "duplicate config of group table %s group_id %d compile_id %d.", table->table_name[0], db_group_rule.group_id, db_group_rule.parent_id); @@ -2720,19 +2258,11 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru return; } -void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct Maat_scanner *scanner,void* logger) -{ - char virtual_group_line[256]; - snprintf(virtual_group_line,sizeof(virtual_group_line), - "%d\t%d\t%d",region_id,compile_id,is_valid); - update_group_rule(table, virtual_group_line,scanner,logger); - return; -} -void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) +void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger) { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); switch(table->table_type) { case TABLE_TYPE_EXPR: @@ -2745,8 +2275,8 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc ,&(maat_str_rule->is_valid)); if(ret!=7) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config: invalid format of expr table %s:%s",table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config: invalid format of expr table %s:%s", table->table_name[table->updating_name], table_line); free(maat_str_rule); maat_str_rule=NULL; table->udpate_err_cnt++; @@ -2764,8 +2294,8 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc ,&(maat_str_rule->is_valid)); if(ret!=8) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config: invalid format of expr_plus table %s:%s",table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config: invalid format of expr_plus table %s:%s", table->table_name[table->updating_name], table_line); free(maat_str_rule); maat_str_rule=NULL; table->udpate_err_cnt++; @@ -2791,30 +2321,30 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc maat_str_rule->is_case_sensitive=TRUE; break; default: - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config %d:update error,invalid hexbin value of expr table %s:%s" - ,maat_str_rule->region_id - ,table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config %d:update error,invalid hexbin value of expr table %s:%s", + maat_str_rule->region_id, + table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; goto error_out; } if(!is_valid_match_method(maat_str_rule->match_method)) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config %d:update error,invalid match method=%d in expr table %s:%s" - ,maat_str_rule->region_id - ,maat_str_rule->match_method - ,table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config %d:update error,invalid match method=%d in expr table %s:%s", + maat_str_rule->region_id, + maat_str_rule->match_method, + table->table_name[table->updating_name],table_line); table->udpate_err_cnt++; goto error_out; } if(!is_valid_expr_type(maat_str_rule->expr_type)) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config %d:update error,invalid expr type=%d in expr table %s:%s" - ,maat_str_rule->region_id - ,maat_str_rule->expr_type - ,table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config %d:update error,invalid expr type=%d in expr table %s:%s", + maat_str_rule->region_id, + maat_str_rule->expr_type, + table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; goto error_out; } @@ -2828,16 +2358,6 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table - ,maat_str_rule->region_id - ,maat_str_rule->group_id - ,maat_str_rule->is_valid - ,scanner - ,logger); - maat_str_rule->group_id=maat_str_rule->region_id; - } if(maat_str_rule->is_valid==FALSE) { @@ -2849,9 +2369,8 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc { rule_type=RULETYPE_STR; } - ret=del_region_rule(table - ,maat_str_rule->region_id,maat_str_rule->group_id,rule_type - ,scanner, logger); + ret=del_region_rule(table, maat_str_rule->region_id, maat_str_rule->group_id, rule_type, + scanner, logger); if(ret<0) { table->udpate_err_cnt++; @@ -2866,18 +2385,18 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc if(maat_str_rule->expr_type==EXPR_TYPE_AND &&maat_str_rule->match_method!=MATCH_METHOD_SUB) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "table %s region cfg %d is EXPR_TYPE_AND,but match method is not MATCH_METHOD_SUB,force fixed.", - table->table_name[table->updating_name],maat_str_rule->region_id); + table->table_name[table->updating_name], maat_str_rule->region_id); maat_str_rule->match_method=MATCH_METHOD_SUB; } - ret=add_expr_rule(table, maat_str_rule,scanner, logger); + ret=add_expr_rule(table, maat_str_rule, scanner, logger); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "duplicate config of expr table %s region_id=%d" - ,table->table_name[table->updating_name],maat_str_rule->region_id); + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, + "duplicate config of expr table %s region_id=%d", + table->table_name[table->updating_name], maat_str_rule->region_id); table->udpate_err_cnt++; } else @@ -3026,12 +2545,12 @@ int ip_format2range(int ip_type, enum MAAT_IP_FORMAT format, const char* ip1, co } return 0; } -void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger, int group_mode_on) +void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); char src_ip1[40]={0}, src_ip2[40]={0}, dst_ip1[40]={0}, dst_ip2[40]={0}; char saddr_format[16]={0}, sport_format[16]={0}, daddr_format[16]={0}, dport_format[16]={0}; - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); unsigned short src_port1=0, src_port2=0, dst_port1=0, dst_port2=0; int protocol=0,direction=0; int ret=0; @@ -3199,16 +2718,6 @@ void update_ip_rule(struct Maat_table_desc* table, const char* table_line, struc table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table, - ip_rule->region_id, - ip_rule->group_id, - ip_rule->is_valid, - scanner, - logger); - ip_rule->group_id=ip_rule->region_id; - } if(ip_rule->is_valid==FALSE) { ret=del_region_rule(table, @@ -3245,10 +2754,10 @@ error_out: ip_rule=NULL; } -void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) +void update_intval_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1); - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); int ret=0; ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) ,&(intval_rule->group_id) @@ -3274,16 +2783,6 @@ void update_intval_rule(struct Maat_table_desc* table,const char* table_line,str table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table - ,intval_rule->region_id - ,intval_rule->group_id - ,intval_rule->is_valid - ,scanner - ,logger); - intval_rule->group_id=intval_rule->region_id; - } if(intval_rule->is_valid==FALSE) { @@ -3324,7 +2823,7 @@ error_out: void update_compile_rule(struct Maat_table_desc* table,const char* table_line ,struct Maat_scanner *scanner, const struct rule_tag* tags, int n_tags,void* logger) { struct compile_table_desc* compile_desc=&(table->compile); - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); struct Maat_compile_rule *p_compile=NULL; struct Maat_rule_head m_rule_tmp; @@ -3407,9 +2906,9 @@ error_out: return; } -void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner *scanner,void* logger,int group_mode_on) +void update_digest_rule(struct Maat_table_desc* table, const char* table_line, struct Maat_scanner *scanner, void* logger) { - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); struct db_digest_rule* digest_rule=ALLOC(struct db_digest_rule, 1); int ret=0; char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'}; @@ -3454,16 +2953,6 @@ void update_digest_rule(struct Maat_table_desc* table,const char* table_line,str table->udpate_err_cnt++; goto error_out; } - if(group_mode_on==FALSE)//for compatible old version - { - compatible_group_udpate(table - ,digest_rule->region_id - ,digest_rule->group_id - ,digest_rule->is_valid - ,scanner - ,logger); - digest_rule->group_id=digest_rule->region_id; - } if(digest_rule->is_valid==FALSE) { @@ -3620,7 +3109,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma int i=0, ret=1, matched_tag=1; unsigned int len=strlen(table_line)+1; struct plugin_table_desc* plugin_desc=&(table->plugin); - struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id); char *p=NULL; char* copy=NULL; size_t is_valid_offset=0, valid_len=0; @@ -3705,35 +3194,56 @@ void vector_print(igraph_vector_t *v) { } printf("\n"); } - +static size_t effective_vertices_count(igraph_vector_t *vids) +{ + size_t i=0; + int tmp_vid=0; + for(i=0; i<(size_t)igraph_vector_size(vids); i++) + { + tmp_vid=(int) VECTOR(*vids)[i]; + if(tmp_vid<0) + { + break; + } + } + return i; +} void walk_group_hash(const uchar * key, uint size, void * data, void * user) { struct Maat_group_inner* group_rule=(struct Maat_group_inner*)data; struct Maat_group_inner* parent_group=NULL; struct Maat_scanner* scanner=(struct Maat_scanner*)user; int tmp_vid=0; - igraph_vector_t vids; - igraph_vector_init(&vids, 0); - - igraph_dfs(&(scanner->group_graph), group_rule->vertex_id, IGRAPH_OUT, - 0, &vids, NULL, NULL, NULL, NULL, NULL, NULL); - - - long int i=0; - long long* temp_group_ids=ALLOC(long long, igraph_vector_size(&vids)); - size_t top_group_cnt=0; - for(i=0; iref_by_compile_cnt==group_rule->ref_by_parent_cnt) { - tmp_vid=(int) VECTOR(vids)[i]; - if(tmp_vid<0) + //fast path, group is only referenced by compile rules. + top_group_cnt=1; + temp_group_ids=ALLOC(long long, top_group_cnt); + temp_group_ids[0]=group_rule->group_id; + } + else + { + igraph_vector_t *vids=&(scanner->dfs_vids); + igraph_dfs(&(scanner->group_graph), group_rule->vertex_id, IGRAPH_OUT, + 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); + + temp_group_ids=ALLOC(long long, effective_vertices_count(vids)); + + for(i=0; i<(size_t)igraph_vector_size(vids); i++) { - break; - } - parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->vertex_id2group, tmp_vid); - if(parent_group->has_compile_neighbors)//including itself - { - temp_group_ids[top_group_cnt]=parent_group->group_id; - top_group_cnt++; + tmp_vid=(int) VECTOR(*vids)[i]; + if(tmp_vid<0) + { + break; + } + parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->vertex_id2group, tmp_vid); + if(parent_group->ref_by_compile_cnt>0)//including itself + { + temp_group_ids[top_group_cnt]=parent_group->group_id; + top_group_cnt++; + } } } pthread_mutex_lock(&(group_rule->mutex)); @@ -3747,7 +3257,6 @@ void walk_group_hash(const uchar * key, uint size, void * data, void * user) scanner->most_popular_sub_group=group_rule->group_id; } pthread_mutex_unlock(&(group_rule->mutex)); - igraph_vector_destroy(&vids); free(temp_group_ids); temp_group_ids=NULL; return; @@ -3755,7 +3264,10 @@ void walk_group_hash(const uchar * key, uint size, void * data, void * user) void find_group_paths(struct Maat_scanner* scanner) { + scanner->group_graph_vcount=igraph_vcount(&scanner->group_graph); + igraph_vector_init(&(scanner->dfs_vids), scanner->group_graph_vcount); MESA_htable_iterate(scanner->group_hash, walk_group_hash, scanner); + igraph_vector_destroy(&scanner->dfs_vids); return; } void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, int scan_thread_num, void* logger) @@ -3763,11 +3275,7 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, struct bool_matcher *tmp1=NULL,*tmp2=NULL; MESA_htable_handle tmp_map=NULL; struct Maat_table_runtime* table_rt=NULL; - int i=0; - long q_cnt; - GIE_create_para_t para; - para.gram_value=7; - para.position_accuracy=10; + int i=0, ret=0; igraph_bool_t is_dag; igraph_is_dag(&(scanner->group_graph), &is_dag); if(!is_dag) @@ -3801,9 +3309,9 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, scanner->region_update_q, logger, scanner); - for(i=0;imax_table_num; i++) { - table_rt=scanner->table_rt[i]; + table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i); if(table_rt==NULL) { continue; @@ -3812,30 +3320,13 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, { case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); - if(q_cnt==0) + + ret=Maat_table_runtime_digest_batch_udpate(table_rt); + if(ret<0) { - continue; + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "GIE_update error."); } - if(table_rt->similar.gie_handle==NULL) - { - if(table_rt->table_type==TABLE_TYPE_SIMILARITY) - { - para.ED_reexamine=1; - para.format=GIE_INPUT_FORMAT_PLAIN; - } - else - { - para.ED_reexamine=0; - para.format=GIE_INPUT_FORMAT_SFH; - } - table_rt->similar.gie_handle=GIE_create(¶); - } - digest_batch_update(table_rt->similar.gie_handle, - table_rt->similar.update_q, - logger, - scanner, - i); break; case TABLE_TYPE_PLUGIN: break; @@ -3859,12 +3350,9 @@ void do_scanner_update(struct Maat_scanner* scanner, MESA_lqueue_head garbage_q, } -void maat_start_cb(long long new_version,int update_type,void*u_para) +void maat_start_cb(long long new_version, int update_type, void*u_para) { - struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct Maat_table_desc* p_table=NULL; - struct plugin_table_desc* plugin_desc=NULL; - int i=0,j=0; + struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; feather->new_version=new_version; if(update_type==CM_UPDATE_TYPE_FULL) @@ -3882,26 +3370,7 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) feather->maat_version,new_version); feather->maat_version=new_version; } - feather->active_plugin_table_num=0; - for(i=0;ip_table_info[i]; - plugin_desc=&(p_table->plugin); - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0) - { - continue; - } - - feather->active_plugin_table_num++; - - for(j=0;jcb_plug_cnt;j++) - { - if(plugin_desc->cb_plug[j].start!=NULL) - { - plugin_desc->cb_plug[j].start(update_type,plugin_desc->cb_plug[j].u_para); - } - } - } + Maat_table_manager_all_plugin_cb_start(feather->table_mgr, update_type); return; } long long scanner_rule_num(struct Maat_scanner *scanner) @@ -3909,9 +3378,9 @@ long long scanner_rule_num(struct Maat_scanner *scanner) long long total=0; struct Maat_table_runtime* table_rt=NULL; int i=0; - for(i=0;imax_table_num; i++) { - table_rt=scanner->table_rt[i]; + table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, i); if(table_rt!=NULL) { total+=table_rt->origin_rule_num; @@ -3922,42 +3391,9 @@ long long scanner_rule_num(struct Maat_scanner *scanner) void maat_finish_cb(void* u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct Maat_table_desc* p_table=NULL; - struct plugin_table_desc* plugin_desc=NULL; long expr_wait_q_cnt=0; - int i=0, j=0; - - int call_plugin_table_cnt=0; - for(i=0;ip_table_info[i]; - if(p_table==NULL) - { - continue; - } - switch(p_table->table_type) - { - case TABLE_TYPE_PLUGIN: - plugin_desc=&(p_table->plugin); - call_plugin_table_cnt++; - if(call_plugin_table_cnt==feather->active_plugin_table_num) - { - feather->is_last_plugin_table_updating=1; - } - for(j=0;jcb_plug_cnt;j++) - { - if(plugin_desc->cb_plug[j].finish!=NULL) - { - plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para); - } - } - feather->is_last_plugin_table_updating=0; - break; - default: - break; - } - } + Maat_table_manager_all_plugin_cb_finish(feather->table_mgr); if(feather->update_tmp_scanner!=NULL) { @@ -4001,14 +3437,11 @@ void maat_finish_cb(void* u_para) feather->maat_version); } feather->new_version=-1; - feather->active_plugin_table_num=0; return; } int maat_update_cb(const char* table_name,const char* line,void *u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - int ret=-1,i=0; - int table_id=-1; Maat_scanner* scanner=NULL; struct Maat_table_desc* p_table=NULL; if(feather->update_tmp_scanner!=NULL) @@ -4020,47 +3453,39 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) scanner=feather->scanner; } // MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line); - ret=map_str2int(feather->map_tablename2id,table_name,&table_id); - if(ret<0) + p_table=Maat_table_get_desc_by_name(feather->table_mgr, table_name); + if(!p_table) { - MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s",table_name); + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s", table_name); return -1; } - p_table=feather->p_table_info[table_id]; - for(i=0;iconj_cnt;i++) - { - if(0==memcmp(p_table->table_name[i],table_name,strlen(table_name))) - { - p_table->updating_name=i; - } - } - assert(i<=p_table->conj_cnt); + Maat_table_set_updating_name(p_table, table_name); - switch(feather->p_table_info[table_id]->table_type) + switch(p_table->table_type) { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - update_expr_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + update_expr_rule(p_table, line, scanner, feather->logger); break; case TABLE_TYPE_IP: case TABLE_TYPE_IP_PLUS: - update_ip_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + update_ip_rule(p_table, line, scanner, feather->logger); break; case TABLE_TYPE_INTERVAL: - update_intval_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + update_intval_rule(p_table, line, scanner,feather->logger); break; case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - update_digest_rule(feather->p_table_info[table_id], line, scanner,feather->logger,feather->GROUP_MODE_ON); + update_digest_rule(p_table, line, scanner,feather->logger); break; case TABLE_TYPE_COMPILE: - update_compile_rule(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + update_compile_rule(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); break; case TABLE_TYPE_GROUP: - update_group_rule(feather->p_table_info[table_id], line, scanner,feather->logger); + update_group_rule(p_table, line, scanner, feather->table_mgr, feather->logger); break; case TABLE_TYPE_PLUGIN: - update_plugin_table(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); + update_plugin_table(p_table, line, scanner, feather->accept_tags, feather->n_tags, feather->logger); default: break; @@ -4075,7 +3500,8 @@ void *thread_rule_monitor(void *arg) int scan_dir_cnt=0; int ret=0; char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0}; - char tmp_dir[MAX_TABLE_NAME_LEN]={0}; + char err_str[MAX_TABLE_NAME_LEN]={0}; + struct stat attrib; size_t total_wait_rule_cnt=0; @@ -4132,31 +3558,24 @@ void *thread_rule_monitor(void *arg) break; case SOURCE_JSON_FILE: memset(md5_tmp, 0, sizeof(md5_tmp)); - memset(tmp_dir, 0, sizeof(tmp_dir)); stat(feather->json_ctx.json_file, &attrib); - if(attrib.st_ctime!=feather->json_ctx.last_md5_time) + if(memcmp(&attrib.st_ctim, &(feather->json_ctx.last_md5_time), sizeof(attrib.st_ctim))) { - feather->json_ctx.last_md5_time=attrib.st_ctime; + feather->json_ctx.last_md5_time=attrib.st_ctim; md5_file(feather->json_ctx.json_file, md5_tmp); if(0!=strcmp(md5_tmp,feather->json_ctx.effective_json_md5)) { - ret=json2iris(feather->json_ctx.json_file, - feather->compile_tn, feather->group_tn, - NULL, - tmp_dir, - sizeof(tmp_dir), - feather->logger); + ret=load_maat_json_file(feather, feather->json_ctx.json_file, err_str, sizeof(err_str)); if(ret<0) { MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module , - "Maat re-initiate with JSON file %s failed, md5: %s", + "Maat re-initiate with JSON file %s (md5=%s)failed: %s", feather->json_ctx.json_file, - md5_tmp); + md5_tmp, + err_str); } else { - strcpy(feather->json_ctx.effective_json_md5, md5_tmp); - strcpy(feather->json_ctx.iris_file, tmp_dir); config_monitor_traverse(0, feather->json_ctx.iris_file, maat_start_cb, @@ -4234,21 +3653,12 @@ void *thread_rule_monitor(void *arg) } } - MESA_htable_destroy(feather->map_tablename2id,free); + Maat_table_manager_destroy(feather->table_mgr); destroy_maat_scanner(feather->scanner); garbage_bury(feather->garbage_q,0,feather->logger); assert(0==MESA_lqueue_get_count(feather->garbage_q)); MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); - int i=0; - for(i=0;ip_table_info[i]==NULL) - { - continue; - } - table_info_free(feather->p_table_info[i]); - feather->p_table_info[i]=NULL; - } + alignment_int64_array_free(feather->thread_call_cnt); alignment_int64_array_free(feather->inner_mid_cnt); alignment_int64_array_free(feather->outer_mid_cnt); @@ -4267,6 +3677,7 @@ void *thread_rule_monitor(void *arg) feather->mr_ctx.write_ctx=NULL; } } + int i=0; for(i=0; in_tags; i++) { free(feather->accept_tags[i].tag_name); diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index f0573eb..bcf3bfb 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -1,4 +1,5 @@ #include "Maat_rule_internal.h" +#include "Maat_table.h" #include "alignment_int64.h" #include #include @@ -121,10 +122,11 @@ void maat_stat_init(struct _Maat_feather_t* feather) FS_STYLE_COLUMN, FS_CALC_SPEED, "hit_rate"); - feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,"Sum"); - for(i=0;itotal_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, "Sum"); + size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr); + for(i=0; i<(int)max_table_num; i++) { - p_table=feather->p_table_info[i]; + p_table=Maat_table_get_by_id_raw(feather->table_mgr, i); if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN ||p_table->table_type==TABLE_TYPE_GROUP ||p_table->table_type==TABLE_TYPE_COMPILE) @@ -185,7 +187,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,Maat_table_manager_get_count(feather->table_mgr)); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt); @@ -197,7 +200,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value); feather->update_err_cnt=0; feather->iconv_err_cnt=0; - for(i=0;itable_mgr); + for(i=0; i<(int)max_table_num; i++) { table_stream_num=0; table_scan_cnt=0; @@ -205,12 +209,13 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_scan_cpu_time=0; table_hit_cnt=0; table_regex_ipv6_num=0; - p_table=feather->p_table_info[i]; + + p_table=Maat_table_get_by_id_raw(feather->table_mgr, i); if(p_table==NULL) { continue; } - table_rt=feather->scanner->table_rt[i]; + table_rt=Maat_table_runtime_get(feather->scanner->table_rt_mgr, i); switch(p_table->table_type) { case TABLE_TYPE_PLUGIN: diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp new file mode 100644 index 0000000..32b2c39 --- /dev/null +++ b/src/entry/Maat_table.cpp @@ -0,0 +1,784 @@ +#include "Maat_table.h" +#include "map_str2int.h" +#include "Maat_utils.h" +#include "cJSON.h" + +#include +#include +#include +#include +#include + +#define MAX_TABLE_NUM 256 + +struct Maat_table_manager +{ + struct Maat_table_desc* p_table_info[MAX_TABLE_NUM]; + size_t table_cnt; + MESA_htable_handle map_tablename2id; + int active_plugin_table_num; + int is_last_plugin_table_updating; +}; + +int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map) +{ + int j=0,ret[4]={0}; + char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; + char *token=NULL,*sub_token=NULL,*saveptr; + struct expr_table_desc* p=&(table->expr); + sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) + ,table->table_name[0] + ,table_type + ,src_charset + ,dst_charset + ,merge + ,&(p->cross_cache_size) + ,quick_str_scan); + memset(ret,0,sizeof(ret)); + ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); + ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset)); + ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge)); + if(strlen(quick_str_scan)>0) + { + ret[3]=map_str2int(string2int_map,str_tolower(quick_str_scan),&(p->quick_expr_switch)); + } + memset(quick_str_scan,0,sizeof(quick_str_scan)); + + for(j=0;j<4;j++) + { + if(ret[j]<0) + { + return -1; + } + } + j=0; + for (token = dst_charset; ; token= NULL) + { + sub_token= strtok_r(token,"/", &saveptr); + if (sub_token == NULL) + break; + ret[3]=map_str2int(string2int_map,str_tolower(sub_token),(int*)&(p->dst_charset[j])); + if(ret[3]>0) + { + if(p->dst_charset[j]==p->src_charset) + { + p->src_charset_in_dst=TRUE; + } + j++; + } + else + { + return -1; + } + + } + return 0; +} +int read_virtual_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map) +{ + int ret=0; + char table_type[16]; + ret=sscanf(line, "%d\t%s\t%s\t%s", &(table->table_id), + table->table_name[0], + table_type, + table->virtual_table.real_table_name); + if(ret!=4) + { + return -1; + } + ret=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); + if(ret<0) + { + return -1; + } + return 0; +} +Maat_table_desc* table_info_new(void) +{ + struct Maat_table_desc*p=ALLOC(struct Maat_table_desc, 1); + p->conj_cnt=1; + return p; +} +void table_info_free(struct Maat_table_desc*p) +{ + free(p); + return; +} +int _read_integer_arrary(char* string, int *array, int size) +{ + char *token=NULL,*sub_token=NULL,*saveptr; + int i=0; + for (token = string, i=0; iplugin); + copy_line=_maat_strdup(line); + ret=get_column_pos(copy_line, COLUMN_PLUGIN_DESCR_JSON, &offset, &len); + if(i<0) + { + goto error_out; + } + if(offset+lenvalid_flag_column)); + if(ret==0||ret==EOF) + { + plugin_desc->valid_flag_column=-1; + } + free(copy_line); + return 0; + } + json=cJSON_Parse(plug_info); + if(!json) + { + goto error_out; + } + tmp=cJSON_GetObjectItem(json, "key"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->key_column=tmp->valueint; + } + tmp=cJSON_GetObjectItem(json, "valid"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->valid_flag_column=tmp->valueint; + } + tmp=cJSON_GetObjectItem(json, "tag"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->rule_tag_column=tmp->valueint; + } + tmp=cJSON_GetObjectItem(json, "estimate_size"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->estimate_size=tmp->valueint; + } + tmp=cJSON_GetObjectItem(json, "foreign"); + if(tmp!=NULL) + { + if(tmp->type==cJSON_String) + { + plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM); + } + else if(tmp->type==cJSON_Array) + { + plugin_desc->n_foreign= cJSON_GetArraySize(tmp); + for(i=0;in_foreign; i++) + { + array_item=cJSON_GetArrayItem(tmp, i); + assert(array_item->type==cJSON_Number); + plugin_desc->foreign_columns[i]=array_item->valueint; + } + } + } + cJSON_Delete(json); + + free(copy_line); + return 0; +error_out: + free(copy_line); + return -1; +} + +static int Maat_table_build_map(struct Maat_table_manager* table_mgr, void* logger) +{ + struct Maat_table_desc** p_table_info=table_mgr->p_table_info; + size_t n_table=MAX_TABLE_NUM; + + MESA_htable_handle map_tablename2id=map_create(); + size_t i=0; + int j=0, ret=0; + for(i=0;itable_type) + { + case TABLE_TYPE_VIRTUAL: + ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id)); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Undefined real table %s, virtual table %s of table id %d.", + p_table_info[i]->virtual_table.real_table_name, + p_table_info[i]->table_name[j], + p_table_info[i]->table_id); + goto failed; + } + break; + default: + break; + } + + + for(j=0; jconj_cnt; j++) + { + ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Duplicate table %s of table id %d", + p_table_info[i]->table_name[j], + p_table_info[i]->table_id); + continue; + } + } + + } + table_mgr->map_tablename2id=map_tablename2id; + return 0; +failed: + map_destroy(map_tablename2id); + return -1; +} + +void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr) +{ + size_t i=0; + for(i=0;ip_table_info[i]==NULL) + { + continue; + } + table_info_free(table_mgr->p_table_info[i]); + table_mgr->p_table_info[i]=NULL; + } + MESA_htable_destroy(table_mgr->map_tablename2id, free); + free(table_mgr); + return; +} + +struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger) +{ + struct Maat_table_manager* table_mgr=NULL; + FILE*fp=NULL; + char line[MAX_TABLE_LINE_SIZE]; + int i=0, ret=0; + char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; + MESA_htable_handle string2int_map=NULL;; + struct Maat_table_desc*p=NULL; + struct Maat_table_desc*conj_table=NULL; + fp=fopen(table_info_path,"r"); + if(fp==NULL) + { + fprintf(stderr,"Maat read table info %s error.\n",table_info_path); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s failed: %s.\n", table_info_path, strerror(errno)); + return NULL; + } + table_mgr=ALLOC(struct Maat_table_manager, 1); + struct Maat_table_desc** p_table_info=table_mgr->p_table_info; + size_t n_table=MAX_TABLE_NUM; + + string2int_map=map_create(); + map_register(string2int_map,"expr", TABLE_TYPE_EXPR); + map_register(string2int_map,"ip", TABLE_TYPE_IP); + map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS); + map_register(string2int_map,"compile", TABLE_TYPE_COMPILE); + map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN); + map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL); + map_register(string2int_map,"digest", TABLE_TYPE_DIGEST); + map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS); + map_register(string2int_map,"group", TABLE_TYPE_GROUP); + map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY); + map_register(string2int_map,"virtual", TABLE_TYPE_VIRTUAL); + map_register(string2int_map,"quickoff", 0); + map_register(string2int_map,"quickon", 1); + map_register(string2int_map,"escape", USER_REGION_ENCODE_ESCAPE); +// map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet + + const char** charset_name_list=charset_get_all_name(); + for(i=0;i0) + { + map_register(string2int_map, charset_name_list[i], i); + } + else + { + break; + } + } + + map_register(string2int_map,"yes", 1); + map_register(string2int_map,"no", 0); + + + i=0; + while(NULL!=fgets(line,sizeof(line),fp)) + { + i++; + + if(line[0]=='#'||line[0]==' '||line[0]=='\t'||strlen(line)<4) + { + continue; + } + p=table_info_new(); + + ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) + ,p->table_name[0] + ,table_type_str + ,not_care); + if(ret<3) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error: not enough column.",table_info_path,i); + continue; + } + ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type)); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:invalid table type.",table_info_path,i); + goto invalid_table; + } + switch(p->table_type) + { + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + ret=read_expr_table_info(line, p, string2int_map); + if(ret<0) + { + fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal column.",table_info_path,i); + goto invalid_table; + } + break; + case TABLE_TYPE_PLUGIN: + ret=read_plugin_table_description(line, p); + if(ret<0) + { + fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal plugin info.",table_info_path,i); + goto invalid_table; + } + break; + case TABLE_TYPE_VIRTUAL: + ret=read_virtual_table_info(line, p, string2int_map); + if(ret<0) + { + fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n",table_info_path,i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error:illegal virtual info.",table_info_path,i); + goto invalid_table; + } + break; + case TABLE_TYPE_COMPILE: + ret=sscanf(not_care,"%[a-z0-9]",tmp_str); + if(ret>0) + { + ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); + } + if(ret!=1) + { + p->compile.user_region_encoding=USER_REGION_ENCODE_NONE; + } + default: + break; + } + + if((unsigned int)p->table_id>=n_table) + { + fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %zu.\n",table_info_path,i,p->table_id,n_table); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + "Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,n_table); + + goto invalid_table; + } + if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table; + { + conj_table=p_table_info[p->table_id]; + if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Maat read table info %s line %d error:reach tableid %d conjunction upper limit." + ,table_info_path,i,p->table_id); + goto invalid_table; + } + memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN); + conj_table->conj_cnt++; + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, + "Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)." + ,table_info_path,i,p->table_name[0] + ,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt); + //use goto to free the conjunctioned table_info + goto invalid_table; + } + + p_table_info[p->table_id]=p; + table_mgr->table_cnt++; + continue; +invalid_table: + table_info_free(p); + p=NULL; + } + fclose(fp); + ret=Maat_table_build_map(table_mgr, logger); + if(ret<0) + { + return NULL; + } + map_destroy(string2int_map); + return table_mgr; +} +size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr) +{ + return MAX_TABLE_NUM; +} +size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr) +{ + return table_mgr->table_cnt; +} +int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +{ + int i=0; + for(i=0; i< MAX_TABLE_NUM; i++) + { + if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_COMPILE) + { + strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); + return 1; + } + } + return 0; +} +int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz) +{ + int i=0; + for(i=0; i< MAX_TABLE_NUM; i++) + { + if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP) + { + strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz); + return 1; + } + } + return 0; +} +const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM) + { + return NULL; + } + if(table_mgr->p_table_info[table_id]) + { + return table_mgr->p_table_info[table_id]->table_name[0]; + } + return NULL; +} +enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM) + { + return TABLE_TYPE_INVALID; + } + if(table_mgr->p_table_info[table_id]) + { + return table_mgr->p_table_info[table_id]->table_type; + } + return TABLE_TYPE_INVALID; + +} +struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id) +{ + if(table_id>MAX_TABLE_NUM) + { + return NULL; + } + + return table_mgr->p_table_info[table_id]; +} + +struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id) +{ + + struct Maat_table_desc **p_table_info=table_mgr->p_table_info; + size_t n_table=MAX_TABLE_NUM; + + struct Maat_table_desc *p_table=NULL, *p_real_table=NULL; + if((unsigned int) table_id>n_table) + { + return NULL; + } + if(p_table_info[table_id]==NULL) + { + return NULL; + } + p_table=p_table_info[table_id]; + if(p_table==NULL) + { + return NULL; + } + if(p_table->table_type==TABLE_TYPE_VIRTUAL) + { + p_real_table=p_table_info[p_table->virtual_table.real_table_id]; + *virutal_table_id=table_id; + } + else + { + p_real_table=p_table; + if(virutal_table_id) *virutal_table_id=0; + } + if(p_real_table->table_type!=expect_type) + { + if((expect_type==TABLE_TYPE_EXPR && p_real_table->table_type!=TABLE_TYPE_EXPR_PLUS)|| + (expect_type==TABLE_TYPE_IP && p_real_table->table_type!=TABLE_TYPE_IP_PLUS)) + { + return NULL; + } + } + return p_real_table; +} +int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name) +{ + int table_id=-1,ret=0; + ret=map_str2int(table_mgr->map_tablename2id, table_name, &table_id); + if(ret>0) + { + return table_id; + } + else + { + return -1; + } +} +int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, + int table_id, + Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para + Maat_update_callback_t *update,//table line ,u_para + Maat_finish_callback_t *finish,//u_para + void* u_para) +{ + int idx=0; + struct Maat_table_desc *p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct plugin_table_desc *plugin_desc=&(p_table->plugin); + if(p_table==NULL) + { + return -1; + } + + idx=plugin_desc->cb_plug_cnt; + if(idx==MAX_PLUGIN_PER_TABLE) + { + return -1; + } + plugin_desc->cb_plug_cnt++; + plugin_desc->cb_plug[idx].start=start; + plugin_desc->cb_plug[idx].update=update; + plugin_desc->cb_plug[idx].finish=finish; + plugin_desc->cb_plug[idx].u_para=u_para; + return 1; +} + +struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx) +{ + int table_id=-1; + struct Maat_table_desc *p_table=NULL; + + table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name); + if(table_id<0) + { + return NULL; + } + p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + if(!p_table) + { + return NULL; + } + if(idxcompile.ex_data_num) + { + return p_table->compile.ex_desc+idx; + } + return NULL; + +} +int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, + Maat_rule_EX_new_func_t *new_func, + Maat_rule_EX_free_func_t* free_func, + Maat_rule_EX_dup_func_t* dup_func, + long argl, void *argp) +{ + int table_id=-1; + struct Maat_table_desc *p_table=NULL; + table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name); + if(table_id<0) + { + return -1; + } + p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL); + if(!p_table) + { + return -1; + } + int idx=-1; + + struct compile_table_desc* compile_desc=&(p_table->compile); + if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM) + { + return -1; + } + idx=compile_desc->ex_data_num; + compile_desc->ex_desc[idx].idx=idx; + compile_desc->ex_desc[idx].table_id=table_id; + compile_desc->ex_desc[idx].argl=argl; + compile_desc->ex_desc[idx].argp=argp; + compile_desc->ex_desc[idx].new_func=new_func; + compile_desc->ex_desc[idx].free_func=free_func; + compile_desc->ex_desc[idx].dup_func=dup_func; + + compile_desc->ex_data_num++; + + return idx; +} +int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp) + +{ + struct Maat_table_desc *table_desc=NULL;; + table_desc=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL); + struct plugin_table_desc* plugin_desc=&(table_desc->plugin); + + if(plugin_desc->have_exdata + || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) + { + return -1; + } + plugin_desc->ex_desc.new_func=new_func; + plugin_desc->ex_desc.free_func=free_func; + plugin_desc->ex_desc.dup_func=dup_func; + plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. + plugin_desc->ex_desc.argl=argl; + plugin_desc->ex_desc.argp=argp; + plugin_desc->have_exdata=1; + return 0; +} +void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type) +{ + table_mgr->active_plugin_table_num=0; + int i=0, j=0; + struct Maat_table_desc* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; + + for(i=0; ip_table_info[i]; + plugin_desc=&(p_table->plugin); + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0) + { + continue; + } + + table_mgr->active_plugin_table_num++; + + for(j=0;jcb_plug_cnt;j++) + { + if(plugin_desc->cb_plug[j].start!=NULL) + { + plugin_desc->cb_plug[j].start(update_type, plugin_desc->cb_plug[j].u_para); + } + } + } + +} +void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr) +{ + int i=0, j=0; + struct Maat_table_desc* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; + + int call_plugin_table_cnt=0; + for(i=0;ip_table_info[i]; + if(p_table==NULL) + { + continue; + } + switch(p_table->table_type) + { + case TABLE_TYPE_PLUGIN: + plugin_desc=&(p_table->plugin); + call_plugin_table_cnt++; + if(call_plugin_table_cnt==table_mgr->active_plugin_table_num) + { + table_mgr->is_last_plugin_table_updating=1; + } + for(j=0;jcb_plug_cnt;j++) + { + if(plugin_desc->cb_plug[j].finish!=NULL) + { + plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para); + } + } + table_mgr->is_last_plugin_table_updating=0; + break; + default: + break; + } + } + table_mgr->active_plugin_table_num=0; + return; +} +int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr) +{ + return table_mgr->is_last_plugin_table_updating; +} +struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name) +{ + struct Maat_table_desc * p_table=NULL; + int table_id=0; + table_id=Maat_table_get_id_by_name(table_mgr, table_name); + if(table_id<0) + { + return NULL; + } + p_table=table_mgr->p_table_info[table_id]; + return p_table; +} +void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name) +{ + int i=0; + for(i=0; iconj_cnt; i++) + { + if(0==strcmp(p_table->table_name[i], table_name)) + { + p_table->updating_name=i; + } + } + assert(i<=p_table->conj_cnt); +} + diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp new file mode 100644 index 0000000..5b9dbc0 --- /dev/null +++ b/src/entry/Maat_table_runtime.cpp @@ -0,0 +1,427 @@ +#include "Maat_table_runtime.h" +#include "Maat_rule.h" +#include "Maat_utils.h" + +#include +#include +#include + +struct wrap_plugin_EX_data +{ + MAAT_RULE_EX_DATA exdata; + const struct Maat_table_desc* ref_plugin_table; +}; +void wrap_plugin_EX_data_free(void *data) +{ + struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; + const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); + ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); + wrap_data->ref_plugin_table=NULL; + free(wrap_data); + return; +} +MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) +{ + MESA_htable_handle key2ex_hash=NULL; + unsigned int slot_size=1; + while(estimate_size!=0) + { + estimate_size=estimate_size>>1; + slot_size*=2; + } + if(slot_size==1) + { + slot_size=4096; + } + + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=8; + hargs.hash_slot_size = slot_size; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = NULL; //Not supported yet. + hargs.recursive = 1; + hargs.data_free = wrap_plugin_EX_data_free; + hargs.data_expire_with_condition = NULL; + key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(key2ex_hash, 0); + return key2ex_hash; +} + +int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + size_t key_offset=0, key_len=0; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: no such key %.*s of %s", + key_len, line+key_offset, line); + return -1; + } + return 0; +} + +int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + char* key=NULL; + size_t key_offset=0, key_len=0; + MAAT_RULE_EX_DATA exdata=NULL; + struct wrap_plugin_EX_data* wrap_data=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + key=ALLOC(char, key_len+1); + memcpy(key, line+key_offset, key_len); + plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); + wrap_data->exdata=exdata; + wrap_data->ref_plugin_table=plugin_table; + ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); + free(key); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: duplicated key %.*s of %s", + key_len, line+key_offset, line); + wrap_plugin_EX_data_free(wrap_data); + return -1; + } + return 0; +} + +struct Maat_table_runtime_manager +{ + struct Maat_table_runtime** table_rt; + size_t n_table_rt; +}; +static GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, + short cfds_lvl, void* tag) +{ + GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); + int digest_len=0; + rule->id=id; + rule->operation=op; + if(digest!=NULL) + { + digest_len=strlen(digest); + rule->sfh=(char*)calloc(sizeof(char),digest_len+1); + memcpy(rule->sfh,digest,digest_len); + + } + rule->sfh_length=digest_len; + rule->cfds_lvl=cfds_lvl; + rule->tag=tag; + return rule; +} +static void destroy_digest_rule(GIE_digest_t*rule) +{ + if(rule->sfh!=NULL) + { + free(rule->sfh); + rule->sfh=NULL; + } + free(rule); + rule=NULL; + return; +} + +static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num) +{ + + struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); + table_rt->table_type=table_desc->table_type; + switch(table_desc->table_type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + table_rt->similar.update_q=MESA_lqueue_create(0,0); + break; + case TABLE_TYPE_PLUGIN: + table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); + if(table_desc->plugin.have_exdata) + { + table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, + table_desc->plugin.ex_desc.key2index_func); + } + break; + default: + break; + } + + table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); + table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); + table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); + table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); + table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); + return table_rt; +} +static void table_runtime_free(struct Maat_table_runtime* p) +{ + long q_cnt=0,data_size=0; + int i=0; + UNUSED int q_ret=0; + + GIE_digest_t* digest_rule=NULL; + if(p==NULL) + { + return; + } + switch(p->table_type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + if(p->similar.gie_handle!=NULL) + { + GIE_destory(p->similar.gie_handle); + } + if(p->similar.update_q!=NULL) + { + q_cnt=MESA_lqueue_get_count(p->similar.update_q); + for(i=0;isimilar.update_q,&digest_rule,&data_size); + assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); + destroy_digest_rule(digest_rule); + } + MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); + } + break; + case TABLE_TYPE_PLUGIN: + dynamic_array_destroy(p->plugin.cache_lines, free); + p->plugin.cache_lines=NULL; + if(p->plugin.key2ex_hash!=NULL) + { + MESA_htable_destroy(p->plugin.key2ex_hash, NULL); + } + default: + break; + } + + alignment_int64_array_free(p->scan_cnt); + alignment_int64_array_free(p->scan_cpu_time); + alignment_int64_array_free(p->input_bytes); + alignment_int64_array_free(p->stream_num); + alignment_int64_array_free(p->hit_cnt); + free(p); + return; +} + +struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num) +{ + const struct Maat_table_desc* table_desc=NULL; + struct Maat_table_runtime* table_rt=NULL; + struct Maat_table_runtime_manager* table_rt_mgr=ALLOC(struct Maat_table_runtime_manager, 1); + size_t i=0; + table_rt_mgr->n_table_rt=Maat_table_manager_get_size(table_manager); + table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt); + for(i=0; in_table_rt; i++) + { + table_desc=Maat_table_get_by_id_raw(table_manager, i); + if(!table_desc) + { + continue; + } + table_rt=table_runtime_new(table_desc, max_thread_num); + table_rt_mgr->table_rt[i]=table_rt; + } + return table_rt_mgr; +} +void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr) +{ + size_t i=0; + for(i=0; in_table_rt; i++) + { + table_runtime_free(table_rt_mgr->table_rt[i]); + table_rt_mgr->table_rt[i]=NULL; + } + free(table_rt_mgr->table_rt); + table_rt_mgr->table_rt=NULL; + free(table_rt_mgr); +} +struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id) +{ + assert(table_id<(int)table_rt_mgr->n_table_rt); + return table_rt_mgr->table_rt[table_id]; +} +long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt) +{ + struct plugin_runtime* plugin_rt=&(table_rt->plugin); + return plugin_rt->cache_line_num; +} +const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line) +{ + const char* line=NULL; + struct plugin_runtime* plugin_rt=&(table_rt->plugin); + + line=(const char*)dynamic_array_read(plugin_rt->cache_lines, Nth_line); + return line; +} +MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table, + struct dynamic_array_t* lines, size_t line_cnt, void* logger) +{ + MESA_htable_handle key2ex_hash=NULL; + size_t i=0; + const char* line=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + + key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); + + for(i=0; i< line_cnt; i++) + { + line=(const char*)dynamic_array_read(lines, i); + plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); + } + return key2ex_hash; +} +int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger) +{ + assert(table_rt->plugin.key2ex_hash==NULL); + if(table_rt->plugin.key2ex_hash) + { + return -1; + } + table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, + table_rt->plugin.cache_line_num, logger); + return 0; +} +MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key) +{ + struct plugin_table_desc* plugin_desc=NULL; + struct wrap_plugin_EX_data* wrap_data=NULL; + MAAT_RULE_EX_DATA exdata=NULL; + + plugin_desc=&(table_desc->plugin); + if(!plugin_desc->have_exdata) + { + assert(0); + return NULL; + } + wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, + (const unsigned char*)key, strlen(key)); + if(wrap_data!=NULL) + { + plugin_desc->ex_desc.dup_func(table_desc->table_id, &(exdata), &(wrap_data->exdata), + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + } + return exdata; + +} + +void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag) +{ + GIE_digest_t* digest_rule=NULL; + char *dup_digest=_maat_strdup(digest); + + if(table_rt->table_type==TABLE_TYPE_SIMILARITY) + { + dup_digest=str_unescape(dup_digest); + } + digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT, + dup_digest, + confidence_degree, + tag); + MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); + return; +} +void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id) +{ + GIE_digest_t* digest_rule=NULL; + digest_rule=create_digest_rule(expr_id, GIE_DELETE_OPT //del digest + ,NULL + ,0 + ,NULL); + MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*)); + return; +} +int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) +{ + long i=0,data_size=0; + int ret=0; + GIE_digest_t* digest_rule=NULL; + GIE_digest_t** update_array=NULL; + UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; + + + GIE_create_para_t para; + para.gram_value=7; + para.position_accuracy=10; + + const long q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); + if(q_cnt==0) + { + return 0; + } + if(table_rt->similar.gie_handle==NULL) + { + if(table_rt->table_type==TABLE_TYPE_SIMILARITY) + { + para.ED_reexamine=1; + para.format=GIE_INPUT_FORMAT_PLAIN; + } + else + { + para.ED_reexamine=0; + para.format=GIE_INPUT_FORMAT_SFH; + } + table_rt->similar.gie_handle=GIE_create(¶); + } + + update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); + for(i=0;isimilar.update_q, &digest_rule, &data_size); + assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); + update_array[i]=digest_rule; + digest_rule=NULL; + } + ret=GIE_update(table_rt->similar.gie_handle, update_array, (int)q_cnt); + for(i=0;ioperation==GIE_INSERT_OPT) + { + table_rt->origin_rule_num++; + } + else + { + table_rt->origin_rule_num--; + } + destroy_digest_rule(update_array[i]); + update_array[i]=NULL; + } + free(update_array); + update_array=NULL; + + if(ret!=(int)q_cnt) + { + return -1; + } + return q_cnt; +} + + diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 36411fa..8462ba7 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -1,8 +1,10 @@ #include -#include #include #include +#include #include +#include +#include #include "Maat_utils.h" pid_t gettid() @@ -164,6 +166,17 @@ char* str_unescape(char* s) s[j]='\0'; return s; } +size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t n_src) +{ + if(*n_destcfg_path,"r"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error,open %s failed.",index->cfg_path); - return -1; - } + char* table_file_buff=NULL; + size_t file_sz=0, file_offset=0; + if(strlen(index->encryp_algorithm)>0) { if(key==NULL||strlen((const char*)key)==0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, no key to decrypt %s.",index->cfg_path); - fclose(fp); return -1; } - decrypt_len=decrypt_open(fp, key,index->encryp_algorithm, &decrypt_buff,logger); - if(decrypt_len==0) + ret=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, &file_sz, error_string, sizeof(error_string)); + if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, %s decrypt failed.",index->cfg_path); - fclose(fp); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, decrypt %s failed: %s", + index->cfg_path, error_string); return -1; } - read_nxt_line_from_buff(decrypt_buff, decrypt_len, &decrypt_offset, line, sizeof(line)); - sscanf(line,"%d\n",&cfg_num); - do_decrypt=1; } else { - fscanf(fp,"%d\n",&cfg_num); + ret=load_file_to_memory(index->cfg_path, (unsigned char**)&table_file_buff, &file_sz); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, open %s failed: %s", + index->cfg_path, error_string); + return -1; + } + } + read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line)); + sscanf(line, "%d\n", &cfg_num); + if(cfg_num!=index->cfg_num) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,"file %s config num not matched",index->cfg_path); - fclose(fp); + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor, "file %s config num not matched", index->cfg_path); return -1; } for(i=0;iiris_table_map, (const unsigned char*)table_name,strlen(table_name)); if(table_info==NULL) { - table_info=(struct iris_table_t*)calloc(sizeof(struct iris_table_t),1); + table_info=ALLOC(struct iris_table_t, 1); table_info->line_count=0; - memcpy(table_info->table_name,table_name,MIN(sizeof(table_info->table_name)-1, strlen(table_name))); - snprintf(table_info->table_path,sizeof(table_info->table_path),"%s/%s.local",p_iris->tmp_iris_dir,table_info->table_name); - MESA_htable_add(p_iris->iris_table_map,(const unsigned char*)table_info->table_name,strlen(table_info->table_name),table_info); + table_info->table_type=table_type; + memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name)-1, strlen(table_name))); + snprintf(table_info->table_path,sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name); + MESA_htable_add(p_iris->iris_table_map, (const unsigned char*)table_info->table_name, strlen(table_info->table_name), table_info); } return table_info; } - +void free_iris_table_info(void* p) +{ + struct iris_table_t* table=(struct iris_table_t*)p; + free(table->buff); + table->buff=NULL; + free(table); +} static int get_group_seq(struct iris_description_t* iris_cfg) { redisReply* data_reply=NULL; @@ -108,7 +122,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg) return sequence; } -int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) +int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_key, const char* encrypt_algo, const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger) { memset(iris_cfg,0,sizeof(struct iris_description_t)); snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file); @@ -131,7 +145,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, iris_cfg->group_name_map=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(iris_cfg->group_name_map, 0); - + + hargs.data_free = free_iris_table_info; iris_cfg->iris_table_map=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(iris_cfg->iris_table_map, 0); @@ -172,8 +187,14 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn, map_register(iris_cfg->str2int_map, "hexbin",1); map_register(iris_cfg->str2int_map, "case plain",2); - iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn); - iris_cfg->group_table=query_table_info(iris_cfg, group_tn); + iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE); + iris_cfg->group_table=query_table_info(iris_cfg, group_tn, TABLE_TYPE_GROUP); + + if(encrypt_key && encrypt_algo) + { + iris_cfg->encrypt_key=_maat_strdup(encrypt_key); + iris_cfg->encrypt_algo=_maat_strdup(encrypt_algo); + } return 0; } @@ -188,6 +209,8 @@ void clear_iris_descriptor(struct iris_description_t *iris_cfg) MESA_htable_destroy(iris_cfg->iris_table_map, NULL); } map_destroy(iris_cfg->str2int_map); + free(iris_cfg->encrypt_algo); + free(iris_cfg->encrypt_key); return; } int create_tmp_dir(struct iris_description_t *p) @@ -231,14 +254,13 @@ int set_file_rulenum(const char* path,int rulenum,void* logger) fclose(fp); return 0; } -int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_command_t*cmd,int cmd_cnt,const char* path,void* logger) +int direct_write_rule(cJSON* json, MESA_htable_handle str2int, struct traslate_command_t*cmd, int cmd_cnt, struct iris_table_t* table, void* logger) { int i=0,ret=-1; cJSON* item=NULL; cJSON dummy; char *p=NULL; int int_value=0; - FILE* fp=NULL; for(i=0;ivaluestring)+1,1); - memcpy(cmd[i].json_value,item->valuestring,strlen(item->valuestring)); + cmd[i].json_value=ALLOC(char, strlen(item->valuestring)+1); + memcpy(cmd[i].json_value, item->valuestring, strlen(item->valuestring)); break; default://impossible ,already checked assert(0); @@ -289,30 +313,16 @@ int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_com } } } - fp=fopen(path,"a"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s error %s.",path,strerror(errno)); - goto error_out; - } for(i=0;iwrite_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, cmd[i].json_value, strlen(cmd[i].json_value)); + table->write_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1); } - fprintf(fp,"\n"); - fclose(fp); - - for(i=0;iwrite_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1); + table->line_count++; + ret=0; -error_out: +error_out: for(i=0;istr2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -516,11 +527,11 @@ int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, co json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger) +int write_expr_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -534,7 +545,7 @@ int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const c json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - if(table_type==TABLE_TYPE_EXPR_PLUS) + if(table->table_type==TABLE_TYPE_EXPR_PLUS) { json_cmd[cmd_cnt].json_string="district"; json_cmd[cmd_cnt].json_type=cJSON_String; @@ -564,10 +575,10 @@ int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const c json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_intval_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -593,10 +604,10 @@ int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -626,10 +637,10 @@ int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, con json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) +int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -655,28 +666,27 @@ int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, co json_cmd[cmd_cnt].json_type=cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); + return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger); } -int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger) +int write_plugin_line(cJSON* plug_table_json, int sequence, iris_description_t* p_iris, void* logger) { - cJSON* item=NULL,*table_content=NULL,*each_line=NULL; + cJSON* item=NULL,*table_content=NULL, *each_line=NULL; struct iris_table_t* table_info=NULL; - const char* table_name=NULL,*line_content=NULL; - int ret=0,i=0,line_cnt=0; - FILE*fp=NULL; + const char* table_name=NULL, *line_content=NULL; + int i=0, line_cnt=0; - item=cJSON_GetObjectItem(plug_table_json,"table_name"); + item=cJSON_GetObjectItem(plug_table_json, "table_name"); if(item==NULL||item->type!=cJSON_String) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "The %d plugin_table's table_name not defined or format error.",sequence); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, + "The %d plugin_table's table_name not defined or format error.", sequence); return -1; } - table_name= item->valuestring; - table_info=query_table_info(p_iris, table_name); - table_content=cJSON_GetObjectItem(plug_table_json,"table_content"); + table_name=item->valuestring; + table_info=query_table_info(p_iris, table_name, TABLE_TYPE_PLUGIN); + table_content=cJSON_GetObjectItem(plug_table_json, "table_content"); if(table_content==NULL||table_content->type!=cJSON_Array) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -685,21 +695,6 @@ int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_ return -1; } line_cnt=cJSON_GetArraySize(table_content); - if(table_info->line_count==0) - { - ret=set_file_rulenum(table_info->table_path,0,logger); - if(ret<0) - { - return -1; - } - } - fp=fopen(table_info->table_path,"a"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s error %s.",table_info->table_path,strerror(errno)); - return -1; - } for(i=0;ivaluestring; - fprintf(fp,"%s\n",line_content); + table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), line_content, strlen(line_content)); + table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), "\n", 1); table_info->line_count++; } - fclose(fp); - set_file_rulenum(table_info->table_path,table_info->line_count,logger); return 0; } -int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_description_t* p_iris,void* logger) +int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_description_t* p_iris, void* logger) { cJSON* item=NULL,*table_content=NULL; int ret=0; @@ -727,15 +721,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_EXPR; struct iris_table_t* table_info=NULL; - item=cJSON_GetObjectItem(region_json,"table_name"); - if(item==NULL||item->type!=cJSON_String) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d's region table_name not defined or format error.",compile_id); - return -1; - } - table_name=item->valuestring; - table_info=query_table_info( p_iris, table_name); + item=cJSON_GetObjectItem(region_json,"table_type"); if(item==NULL||item->type!=cJSON_String) { @@ -745,7 +731,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri return -1; } table_type_str=item->valuestring; - ret=map_str2int(p_iris->str2int_map,table_type_str,(int*)&(table_type)); + ret=map_str2int(p_iris->str2int_map, table_type_str, (int*)&(table_type)); if(ret!=1) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -753,6 +739,16 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri ,compile_id,table_name,table_type_str); return -1; } + item=cJSON_GetObjectItem(region_json,"table_name"); + if(item==NULL||item->type!=cJSON_String) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d's region table_name not defined or format error.",compile_id); + return -1; + } + table_name=item->valuestring; + table_info=query_table_info(p_iris, table_name, table_type); + table_content=cJSON_GetObjectItem(region_json,"table_content"); if(table_content==NULL||table_content->type!=cJSON_Object) { @@ -761,14 +757,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri ,compile_id,table_name); return -1; } - if(table_info->line_count==0) - { - ret=set_file_rulenum(table_info->table_path,0,logger); - if(ret<0) - { - return -1; - } - } + region_id=get_region_seq(p_iris); cJSON_AddNumberToObject(table_content, "region_id", region_id); cJSON_AddNumberToObject(table_content, "group_id", group_id); @@ -778,32 +767,27 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - ret=write_expr_line(table_content, p_iris, table_info->table_path,table_type, logger); + ret=write_expr_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_IP: - ret=write_ip_line(table_content, p_iris, table_info->table_path, logger); + ret=write_ip_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_IP_PLUS: - write_ip_plus_line(table_content, p_iris, table_info->table_path, logger); + write_ip_plus_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_INTERVAL: - ret=write_intval_line(table_content, p_iris, table_info->table_path, logger); + ret=write_intval_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_DIGEST: - ret=write_digest_line(table_content, p_iris, table_info->table_path, logger); + ret=write_digest_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_SIMILARITY: - write_similar_line(table_content, p_iris,table_info->table_path, logger); + ret=write_similar_line(table_content, p_iris, table_info, logger); break; default: assert(0); break; } - if(ret>=0) - { - table_info->line_count++; - set_file_rulenum(table_info->table_path,table_info->line_count,logger); - } return ret; } @@ -860,18 +844,10 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * } else { - table_info=query_table_info(p_iris,item->valuestring); + table_info=query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE); } - if(table_info->line_count==0) - { - ret=set_file_rulenum(table_info->table_path, 0,logger); - if(ret<0) - { - return -1; - } - } - ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info->table_path,logger); + ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info, logger); if(ret<0) { return -1; @@ -884,66 +860,73 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * return -1; } compile_id=item->valueint; - table_info->line_count++; - set_file_rulenum(table_info->table_path,table_info->line_count,logger); return compile_id; } -int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, struct iris_description_t *p_iris, void * logger) +int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, const char* virtual_table, struct iris_description_t *p_iris, void * logger) { - FILE*fp=NULL; - int ret=0; - - if(p_iris->group_table->line_count==0) - { - ret=set_file_rulenum(p_iris->group_table->table_path,0,logger); - if(ret<0) - { - return -1; - } - } - fp=fopen(p_iris->group_table->table_path,"a"); - if(fp==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "fopen %s error %s.",p_iris->group_table->table_path,strerror(errno)); - return -1; - } - fprintf(fp,"%d\t%d\t1\t%d\t%d\n",group_id, parent_id, group_not_flag, parent_type); - fclose(fp); - p_iris->group_table->line_count++; - ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger); + char buff[1024*4]; + struct iris_table_t* table=p_iris->group_table; + snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%d\t%s\n", group_id, parent_id, group_not_flag, parent_type, virtual_table); + table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); + table->line_count++; return 0; } void table_idx_write_cb(const uchar * key, uint size, void * data, void * user) { - struct iris_table_t* p_table=(struct iris_table_t*)data; - FILE* fp=(FILE*)user; - fprintf(fp,"%s\t%d\t%s\n",p_table->table_name,p_table->line_count,p_table->table_path); + struct iris_description_t *p_iris=(struct iris_description_t *)user; + struct iris_table_t* table=(struct iris_table_t*)data; + FILE* table_fp=NULL; + char line_cnt_str[32], err_str[256]; + snprintf(line_cnt_str, sizeof(line_cnt_str), "%010d\n", table->line_count); + + int ret=0; + size_t table_file_sz=strlen(line_cnt_str)+table->write_pos; + unsigned char* buff=ALLOC(unsigned char, table_file_sz); + unsigned char* encrypt_buff=NULL; + size_t encrypt_buff_sz=0; + memcpy(buff, line_cnt_str, strlen(line_cnt_str)); + memcpy(buff+strlen(line_cnt_str), table->buff, table->write_pos); + table_fp=fopen(table->table_path, "w"); + if(p_iris->encrypt_key) + { + ret=crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key, p_iris->encrypt_algo, 1, err_str, sizeof(err_str)); + assert(ret==0); + fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp); + fprintf(p_iris->idx_fp,"%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo); + } + else + { + fwrite(buff, table_file_sz, 1, table_fp); + fprintf(p_iris->idx_fp,"%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); + } + fclose(table_fp); + free(buff); + buff=NULL; + + } int write_index_file(struct iris_description_t *p_iris,void* logger) { - FILE*fp=NULL; - fp=fopen(p_iris->index_path,"w"); - if(fp==NULL) + p_iris->idx_fp=fopen(p_iris->index_path,"w"); + if(p_iris->idx_fp==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "index file %s fopen error %s.",p_iris->index_path,strerror(errno)); + "index file %s fopen error %s.",p_iris->index_path, strerror(errno)); return -1; } - MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, fp); - fclose(fp); + MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, p_iris); + fclose(p_iris->idx_fp); + p_iris->idx_fp=NULL; return 0; } int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger) { const char* _str_parent_type[2]={"compile", "group"}; - int i=0; - int sub_group_cnt=0, region_cnt=0; int ret=0; int group_not_flag=0; cJSON *region_json=NULL, *item=NULL; cJSON *sub_groups=NULL, *region_rule=NULL; - const char* group_name=NULL; + const char* group_name=NULL, *virtual_table=NULL; struct group_info_t *group_info=NULL; struct group_info_t untitled_group; @@ -956,6 +939,16 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac { group_name=item->valuestring; } + + item=cJSON_GetObjectItem(group_json, "virtual_table"); + if(item==NULL||item->type!=cJSON_String) + { + virtual_table="null"; + } + else + { + virtual_table=item->valuestring; + } item=cJSON_GetObjectItem(group_json,"not_flag"); if(item==NULL||item->type!=cJSON_Number) { @@ -982,10 +975,10 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac { group_info=ALLOC(struct group_info_t, 1); group_info->group_id=get_group_seq(p_iris); - MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); + MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info); } } - ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, p_iris, logger); + ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, virtual_table, p_iris, logger); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -995,10 +988,8 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac region_json=cJSON_GetObjectItem(group_json,"regions"); if(region_json!=NULL) { - region_cnt=cJSON_GetArraySize(region_json); - for(i=0; igroup_id, p_iris, logger); if(ret<0) { @@ -1012,10 +1003,9 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac if(sub_groups!=NULL) { //recursively - sub_group_cnt=cJSON_GetArraySize(sub_groups); - for(i=0; igroup_id, PARENT_TYPE_GROUP, tracking_compile_id, p_iris, logger); if(ret<0) { @@ -1027,18 +1017,16 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac } int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) { - int i=0,j=0; - int compile_id=-1, compile_cnt=0, group_cnt=0, plug_table_cnt=0; + int i=0; + int compile_id=-1, compile_cnt=0, group_cnt=0; int ret=0; cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL; cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL; plug_tables=cJSON_GetObjectItem(json,"plugin_table"); if(NULL!=plug_tables) { - plug_table_cnt=cJSON_GetArraySize(plug_tables); - for(i=0;ivaluestring; } - tmp_obj=cJSON_GetObjectItem(json,"group_table"); + tmp_obj=cJSON_GetObjectItem(json, "group_table"); if(tmp_obj) { group_tn=tmp_obj->valuestring; } - ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger); + ret=set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger); if(ret<0) { goto error_out; @@ -1162,31 +1116,24 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r ret=create_tmp_dir(&iris_cfg); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "create tmp folder %s error",iris_cfg.tmp_iris_dir); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json, + "create tmp folder %s error", iris_cfg.tmp_iris_dir); goto error_out; } - ret=write_iris(json,&iris_cfg,logger); + ret=write_iris(json, &iris_cfg, logger); if(ret<0) { goto error_out; } - memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir,MIN(strlen(iris_cfg.tmp_iris_index_dir)+1,(unsigned int)buf_len)); + memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len)); cJSON_Delete(json); - fclose(json_fp); - free(json_buff); clear_iris_descriptor(&iris_cfg); return 0; error_out: cJSON_Delete(json); - if(json_fp!=NULL) - { - fclose(json_fp); - } - free(json_buff); clear_iris_descriptor(&iris_cfg); return -1; } diff --git a/src/inc_internal/Maat_limits.h b/src/inc_internal/Maat_limits.h new file mode 100644 index 0000000..f7a6abf --- /dev/null +++ b/src/inc_internal/Maat_limits.h @@ -0,0 +1,11 @@ +#define MAX_TABLE_LINE_SIZE (1024*16) +#define MAX_EXPR_KEYLEN 1024 +#define MAX_DISTRICT_LEN 64 + +#define MAX_SCANNER_HIT_NUM 4096 +#define MAX_GROUP_CACHE 128 + +#define MAX_FAILED_NUM 128 + +#define MAX_MAAT_STAT_NUM 64 + diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index dcb31a0..3aed79e 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -1,9 +1,10 @@ -#ifndef H_MAAT_RULE_INTERNAL_H_INCLUDE -#define H_MAAT_RULE_INTERNAL_H_INCLUDE +#pragma once #include "Maat_rule.h" #include "Maat_command.h" -#include "Maat_table_description.h" +#include "Maat_limits.h" +#include "Maat_table.h" +#include "Maat_table_runtime.h" #include #include @@ -20,24 +21,13 @@ #include #include -extern const char *maat_module; #define mr_region_id_var "SEQUENCE_REGION" #define mr_group_id_var "SEQUENCE_GROUP" -#define MAX_TABLE_NUM 256 -#define MAX_TABLE_LINE_SIZE (1024*16) -#define MAX_EXPR_KEYLEN 1024 -#define MAX_DISTRICT_LEN 64 -#define MAX_SCANNER_HIT_NUM 64 -#define MAX_GROUP_CACHE 128 - -#define MAX_FAILED_NUM 128 - -#define MAX_MAAT_STAT_NUM 64 typedef void* rule_scanner_t; @@ -114,6 +104,7 @@ struct db_group_rule_t int is_valid; int not_flag; int parent_type; //PARENT_TYPE_**, 0:compile, 1: group. + int virtual_table_id; }; struct op_expr_t { @@ -141,6 +132,7 @@ struct Maat_compile_group_relation long long magic_num; struct Maat_compile_rule *compile; dynamic_array_t *groups; //element is struct Maat_group_inner* + int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR]; char not_flag[MAX_ITEMS_PER_BOOL_EXPR]; int compile_id;//equal to compile->m_rule.config_id int group_boundary; @@ -159,9 +151,9 @@ struct Maat_group_inner int ref_by_parent_cnt; int ref_by_children_cnt; char* group_name; - int has_compile_neighbors; + int ref_by_compile_cnt; int vertex_id; - int top_group_cnt; + size_t top_group_cnt; long long* top_groups; dynamic_array_t *regions; pthread_mutex_t mutex; @@ -208,11 +200,12 @@ struct iconv_handle_t struct _stream_para_t { struct _Maat_feather_t* feather; + const struct Maat_table_desc* p_real_table; + int virtual_table_id; int version; int thread_num; int max_cross_size; int caching_size; - unsigned short table_id; char do_merge; char do_expr:4; char do_regex:4; @@ -225,54 +218,7 @@ struct _stream_para_t pthread_mutex_t fuzzy_mutex; unsigned char query_point[8]; }; -struct similar_runtime -{ - GIE_handle_t* gie_handle; - MESA_lqueue_head update_q; -}; -struct plugin_runtime -{ - dynamic_array_t *cache_lines; - long long cache_line_num; - long long acc_line_num; - long long cache_size; - MESA_htable_handle key2ex_hash; -}; -struct expr_runtime -{ - long long expr_rule_cnt; //expr_type=0,1,3 - long long regex_rule_cnt; //expr_type=2 -}; -struct ip_runtime -{ - long long ipv4_rule_cnt; - long long ipv6_rule_cnt; - -}; -struct group_runtime -{ - long long not_flag_group; -}; -struct Maat_table_runtime -{ - enum MAAT_TABLE_TYPE table_type; - long origin_rule_num; - union - { - struct similar_runtime similar; //for digest and similarity - struct plugin_runtime plugin; - struct expr_runtime expr; - struct ip_runtime ip; - struct group_runtime group; - void * other; - }; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; -}; struct rule_tag { char* tag_name; @@ -287,9 +233,10 @@ struct Maat_scanner size_t gie_update_q_size; size_t to_update_group_cnt; size_t to_update_compile_cnt; - - struct Maat_table_runtime* table_rt[MAX_TABLE_NUM]; + struct Maat_table_runtime_manager* table_rt_mgr; + size_t max_table_num; + MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation* MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions; MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_inner* @@ -301,9 +248,12 @@ struct Maat_scanner MESA_htable_handle vertex_id2group; igraph_t group_graph; + igraph_integer_t group_graph_vcount; + igraph_vector_t dfs_vids; + int grp_vertex_id_generator; int most_popular_sub_group; - long long max_presented_top_group_cnt; + unsigned long long max_presented_top_group_cnt; unsigned int district_num; unsigned int cfg_num; @@ -312,6 +262,7 @@ struct Maat_scanner MESA_lqueue_head region_update_q; struct bool_matcher * bool_matcher_expr_compiler; scan_result_t *region_rslt_buff; + GIE_result_t* gie_rslt_buff; void* logger_ref; MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q @@ -335,7 +286,7 @@ struct source_json_ctx char json_file[MAX_TABLE_NAME_LEN]; char iris_file[MAX_TABLE_NAME_LEN]; char effective_json_md5[MD5_DIGEST_LENGTH*2+1]; - time_t last_md5_time; + struct timespec last_md5_time; }; struct source_redis_ctx { @@ -351,9 +302,9 @@ struct _Maat_feather_t struct Maat_scanner *scanner; struct Maat_scanner *update_tmp_scanner; MESA_lqueue_head garbage_q; - int table_cnt; + struct Maat_table_manager* table_mgr; + int DEFERRED_LOAD_ON; - int GROUP_MODE_ON; int REDIS_MODE_ON; enum data_source input_mode; union @@ -368,8 +319,7 @@ struct _Maat_feather_t int cumulative_update_off; int stat_on; int perf_on; - struct Maat_table_desc *p_table_info[MAX_TABLE_NUM]; - MESA_htable_handle map_tablename2id; + void* logger; long long maat_version; long long last_full_version; @@ -382,7 +332,8 @@ struct _Maat_feather_t char compile_tn[MAX_TABLE_NAME_LEN]; char group_tn[MAX_TABLE_NAME_LEN]; pthread_mutex_t background_update_mutex; - unsigned char decrypt_key[MAX_TABLE_NAME_LEN]; + char decrypt_key[MAX_TABLE_NAME_LEN]; + char decrypt_algo[MAX_TABLE_NAME_LEN]; pthread_t cfg_mon_t; int AUTO_NUMBERING_ON; @@ -399,11 +350,8 @@ struct _Maat_feather_t int n_tags; char foreign_cont_dir[MAX_TABLE_NAME_LEN]; - int foreign_cont_linger; //internal states long long new_version; - int active_plugin_table_num; - int is_last_plugin_table_updating; //for scanner independent stat>>>> int backgroud_update_enabled; @@ -448,7 +396,6 @@ struct _maat_garbage_t struct foreign_key { int column; - int is_existed; char* key; char* filename; }; @@ -469,7 +416,6 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not); -int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); @@ -504,17 +450,14 @@ void empty_serial_rules(struct serial_rule_t* rule); int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger); long long redis_server_time(redisContext* ctx); redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger); -char* md5_file(const char* filename, char* md5string); +int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz); void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, _Maat_feather_t* feather); - -#endif - diff --git a/src/inc_internal/Maat_table.h b/src/inc_internal/Maat_table.h new file mode 100644 index 0000000..d056b2d --- /dev/null +++ b/src/inc_internal/Maat_table.h @@ -0,0 +1,158 @@ +#pragma once +#include +#include "Maat_rule.h" +#include "Maat_limits.h" + +#define MAX_COMPILE_EX_DATA_NUM 2 +#define MAX_FOREIGN_CLMN_NUM 8 +#define MAX_PLUGIN_PER_TABLE 32 +#define MAX_CHARSET_NUM __CHARSET_MAX +#define MAX_CONJUNCTION_TABLE_NUM 8 +#define MAX_TABLE_NAME_LEN 256 + + +enum USER_REGION_ENCODE +{ + USER_REGION_ENCODE_NONE=0, + USER_REGION_ENCODE_ESCAPE, + USER_REGION_ENCODE_BASE64 +}; + +enum MAAT_TABLE_TYPE +{ + TABLE_TYPE_INVALID=-1, + TABLE_TYPE_EXPR=0, + TABLE_TYPE_IP, + TABLE_TYPE_IP_PLUS, + TABLE_TYPE_INTERVAL, + TABLE_TYPE_DIGEST, + TABLE_TYPE_EXPR_PLUS, + TABLE_TYPE_SIMILARITY, + TABLE_TYPE_VIRTUAL, + TABLE_TYPE_GROUP, + TABLE_TYPE_COMPILE, + TABLE_TYPE_PLUGIN +}; + +struct compile_ex_data_idx +{ + Maat_rule_EX_new_func_t *new_func; + Maat_rule_EX_free_func_t* free_func; + Maat_rule_EX_dup_func_t* dup_func; + long argl; + void *argp; + int idx; + int table_id; +}; +struct compile_table_desc +{ + enum USER_REGION_ENCODE user_region_encoding; + int ex_data_num; + struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; +}; + +struct expr_table_desc +{ + enum MAAT_CHARSET src_charset; + enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; + int src_charset_in_dst; + int do_charset_merge; + int cross_cache_size; + int quick_expr_switch;//obsolete since 20190401 + long long iconv_err_cnt; +}; +struct virtual_table_desc +{ + int real_table_id; + char real_table_name[MAX_TABLE_NAME_LEN]; +}; +struct plugin_table_callback_desc +{ + Maat_start_callback_t *start; + Maat_update_callback_t *update; + Maat_finish_callback_t *finish; + void* u_para; +}; +struct plugin_table_ex_data_desc +{ + Maat_plugin_EX_new_func_t* new_func; + Maat_plugin_EX_free_func_t* free_func; + Maat_plugin_EX_dup_func_t* dup_func; + Maat_plugin_EX_key2index_func_t* key2index_func; + long argl; + void *argp; +}; +struct plugin_table_desc +{ + int key_column; + int valid_flag_column; + int rule_tag_column; + int n_foreign; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; + int cb_plug_cnt; + int have_exdata; + long long estimate_size; + struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; + struct plugin_table_ex_data_desc ex_desc; +}; + +struct Maat_table_desc +{ + int table_id; + int conj_cnt; + int updating_name; + char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; + enum MAAT_TABLE_TYPE table_type; + union + { + struct compile_table_desc compile; + struct expr_table_desc expr; + struct plugin_table_desc plugin; + struct virtual_table_desc virtual_table;; + void* others;//group, ip, interval and digest don't have sperate description info. + }; +//for stat>>>>>>>> + unsigned long long udpate_err_cnt; + unsigned long long unmatch_tag_cnt; + int stat_line_id; +}; +struct Maat_table_manager; +struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger); +void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr); +size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr); +size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr); + +struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id); +struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id); + +int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name); +int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr, + int table_id, + Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para + Maat_update_callback_t *update,//table line ,u_para + Maat_finish_callback_t *finish,//u_para + void* u_para); +int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz); +const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id); +enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id); + +int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name, + Maat_rule_EX_new_func_t *new_func, + Maat_rule_EX_free_func_t* free_func, + Maat_rule_EX_dup_func_t* dup_func, + long argl, void *argp); +struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx); +int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp); +void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type); +void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr); + +int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr); +struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name); +void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name); + diff --git a/src/inc_internal/Maat_table_description.h b/src/inc_internal/Maat_table_description.h deleted file mode 100644 index b924c34..0000000 --- a/src/inc_internal/Maat_table_description.h +++ /dev/null @@ -1,114 +0,0 @@ -#pragma once -#include "Maat_rule.h" -#define MAX_COMPILE_EX_DATA_NUM 2 -#define MAX_FOREIGN_CLMN_NUM 8 -#define MAX_PLUGIN_PER_TABLE 32 -#define MAX_CHARSET_NUM 16 -#define MAX_CONJUNCTION_TABLE_NUM 8 -#define MAX_TABLE_NAME_LEN 256 - -enum USER_REGION_ENCODE -{ - USER_REGION_ENCODE_NONE=0, - USER_REGION_ENCODE_ESCAPE, - USER_REGION_ENCODE_BASE64 -}; - -enum MAAT_TABLE_TYPE -{ - TABLE_TYPE_EXPR=0, - TABLE_TYPE_IP, - TABLE_TYPE_IP_PLUS, - TABLE_TYPE_INTERVAL, - TABLE_TYPE_DIGEST, - TABLE_TYPE_EXPR_PLUS, - TABLE_TYPE_SIMILARITY, - TABLE_TYPE_GROUP, - TABLE_TYPE_COMPILE, - TABLE_TYPE_PLUGIN - -}; - -struct compile_ex_data_idx -{ - Maat_rule_EX_new_func_t *new_func; - Maat_rule_EX_free_func_t* free_func; - Maat_rule_EX_dup_func_t* dup_func; - long argl; - void *argp; - int idx; - int table_id; -}; -struct compile_table_desc -{ - enum USER_REGION_ENCODE user_region_encoding; - int ex_data_num; - struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; -}; - -struct plugin_table_callback_desc -{ - Maat_start_callback_t *start; - Maat_update_callback_t *update; - Maat_finish_callback_t *finish; - void* u_para; -}; -struct plugin_table_ex_data_desc -{ - Maat_plugin_EX_new_func_t* new_func; - Maat_plugin_EX_free_func_t* free_func; - Maat_plugin_EX_dup_func_t* dup_func; - Maat_plugin_EX_key2index_func_t* key2index_func; - long argl; - void *argp; -}; -struct plugin_table_desc -{ - int key_column; - int valid_flag_column; - int rule_tag_column; - int n_foreign; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; - int cb_plug_cnt; - int have_exdata; - long long estimate_size; - struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; - struct plugin_table_ex_data_desc ex_desc; -}; -struct expr_table_desc -{ - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cross_cache_size; - int quick_expr_switch;//obsolete since 20190401 - long long iconv_err_cnt; -}; -struct ip_table_desc -{ - int ipv4_rule_cnt; - int ipv6_rule_cnt; -}; - -struct Maat_table_desc -{ - int table_id; - int conj_cnt; - int updating_name; - char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; - enum MAAT_TABLE_TYPE table_type; - union - { - struct compile_table_desc compile; - struct expr_table_desc expr; - struct ip_table_desc ip; - struct plugin_table_desc plugin; - void* others;//group, interval and digest don't have sperate description info. - }; -//for stat>>>>>>>> - unsigned long long udpate_err_cnt; - unsigned long long unmatch_tag_cnt; - int stat_line_id; -}; - diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h new file mode 100644 index 0000000..caeae5d --- /dev/null +++ b/src/inc_internal/Maat_table_runtime.h @@ -0,0 +1,69 @@ +#include "Maat_table.h" +#include "gram_index_engine.h" +#include "alignment_int64.h" +#include "dynamic_array.h" +#include +#include + +struct similar_runtime +{ + GIE_handle_t* gie_handle; + MESA_lqueue_head update_q; +}; + +struct plugin_runtime +{ + dynamic_array_t *cache_lines; + long long cache_line_num; + long long acc_line_num; + long long cache_size; + MESA_htable_handle key2ex_hash; +}; +struct expr_runtime +{ + long long expr_rule_cnt; //expr_type=0,1,3 + long long regex_rule_cnt; //expr_type=2 +}; +struct ip_runtime +{ + long long ipv4_rule_cnt; + long long ipv6_rule_cnt; + +}; +struct group_runtime +{ + long long not_flag_group; +}; +struct Maat_table_runtime +{ + enum MAAT_TABLE_TYPE table_type; + long origin_rule_num; + union + { + struct similar_runtime similar; //for digest and similarity + struct plugin_runtime plugin; + struct expr_runtime expr; + struct ip_runtime ip; + struct group_runtime group; + void * other; + }; + mcore_long_t scan_cnt; + mcore_long_t scan_cpu_time; //nano + mcore_long_t input_bytes; + mcore_long_t stream_num; + mcore_long_t hit_cnt; +}; +struct Maat_table_runtime_manager; +struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num); +void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr); +struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id); +long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt); +const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line); + +int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger); +MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key); +void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag); +void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id); +int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt); + + diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index e8d7114..ac4054b 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -1,6 +1,8 @@ #pragma once +#include "Maat_rule.h" #include #include +#include #include #include //fstat #include //fstat @@ -45,7 +47,7 @@ #define UNUSED __attribute__((unused)) const char* module_name_str(const char*name); #define maat_module (module_name_str("MAAT_Frame")) - +#define TO_RELATION_ID(vid, gid) ((unsigned long long)vid<<32|gid) char* _maat_strdup(const char* s); char* str_unescape(char* s); inline void ipv6_ntoh(unsigned int *v6_addr) @@ -62,12 +64,23 @@ char* str_tolower(char* string); char *strtok_r_esc(char *s, const char delim, char **save_ptr); char *str_unescape_and(char*s); char* str_unescape(char* s); +size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t n_src); + pid_t gettid(void); int system_cmd_mkdir(const char* path); int system_cmd_rm(const char* src_file); -int system_cmd_mv(const char* src_file,const char*dst_file); -int system_cmd_cp(const char* src_file,const char*dst_file); +int system_cmd_mv(const char* src_file, const char*dst_file); +int system_cmd_cp(const char* src_file, const char*dst_file); +int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password); + char* md5_file(const char* filename, char* md5string); int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); +const char** charset_get_all_name(void); +const char* charset_get_name(enum MAAT_CHARSET charset); +int lqueue_destroy_cb(void *data, long data_len, void *arg); +int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz); +int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz); +//do_encrypt: 1 for encryption, 0 for decryption. +int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz); diff --git a/src/inc_internal/config_monitor.h b/src/inc_internal/config_monitor.h index faabf50..6167ce4 100644 --- a/src/inc_internal/config_monitor.h +++ b/src/inc_internal/config_monitor.h @@ -9,7 +9,7 @@ void config_monitor_traverse(long long version,const char*idx_dir, int (*update)(const char*, const char*, void*),//table name ,line ,u_para void (*finish)(void*),//u_para void* u_para, - const unsigned char* dec_key, + const char* dec_key, void* logger); #endif diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index f61f6e5..247a191 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -1,6 +1,6 @@ #ifndef H_MAAT_JSON2IRIS_H_INCLUDE #define H_MAAT_JSON2IRIS_H_INCLUDE -int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger); -int set_file_rulenum(const char* path,int rulenum,void* logger); +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger); +int set_file_rulenum(const char* path, int rulenum, void* logger); #endif diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 02e8590..db47175 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,6 +7,9 @@ target_link_libraries(test_igraph igraph-static) add_executable(test_maatframe test_maatframe.cpp) target_link_libraries(test_maatframe maat_frame_shared gtest) +add_executable(perf_test_maatframe perf_test_maatframe.cpp) +target_link_libraries(perf_test_maatframe maat_frame_shared gtest) + configure_file(table_info.conf table_info.conf COPYONLY) configure_file(t2_tableinfo.conf t2_tableinfo.conf COPYONLY) configure_file(maat_json.json maat_json.json COPYONLY) diff --git a/test/maat_json.json b/test/maat_json.json index ee03144..1c690f9 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -196,7 +196,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_REGION", + "table_name": "HTTP_SIGNATURE", "table_type": "expr_plus", "table_content": { "district": "HTTP\\bURL", @@ -298,7 +298,7 @@ "is_valid": "yes", "groups": [ { - "group_name": "Untitled", + "group_name": "TakeMeHome", "regions": [ { "table_name": "KEYWORDS_TABLE", @@ -1027,7 +1027,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_REGION", + "table_name": "HTTP_SIGNATURE", "table_type": "expr_plus", "table_content": { "district": "Content-Type", @@ -1141,8 +1141,167 @@ ], "not_flag" : 0 } - ] - } + ] + }, + { + "compile_id": 160, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "VirtualWithPhysical", + "is_valid": "yes", + "groups": [ + { + "group_name":"TakeMeHome", + "virtual_table":"HTTP_RESPONSE_KEYWORDS", + "not_flag" : 0 + }, + { + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "string", + "table_content": { + "keywords": "https://blog.csdn.net/littlefang/article/details/8213058", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 161, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "virtual_table_test_temp", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig1", + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "User-Agent", + "keywords": "Chrome/78.0.3904.108", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "group_name":"vt_grp_http_sig2", + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "Cookie", + "keywords": "uid=12345678", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + }, + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "Cookie", + "keywords": "sessionid=888888", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 162, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "VirtualWithVirtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig1", + "virtual_table":"HTTP_REQUEST_HEADER", + "not_flag":0 + }, + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_RESPONSE_HEADER", + "not_flag":0 + } + ] + }, + { + "compile_id": 163, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "OneGroupInTwoVirtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_REQUEST_HEADER", + "not_flag":0 + }, + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_RESPONSE_HEADER", + "not_flag":0 + } + ] + }, + { + "compile_id": 164, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "CharsetWindows1251", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "string", + "table_content": { + "keywords": ">ЗАО\\b«Севергазвтоматика\\bАйС»<", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + } ], "plugin_table": [ { diff --git a/test/perf_test_maatframe.cpp b/test/perf_test_maatframe.cpp new file mode 100644 index 0000000..f139e39 --- /dev/null +++ b/test/perf_test_maatframe.cpp @@ -0,0 +1,203 @@ +#include "Maat_rule.h" +#include "stream_fuzzy_hash.h" +#include "Maat_command.h" +#include +#include +int test_add_expr_command_copy(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords) +{ + struct Maat_cmd_t* cmd=NULL; + struct Maat_rule_t rule; + char huge_serv_def[1024*2]; + memset(huge_serv_def,'s',sizeof(huge_serv_def)); + struct Maat_region_t region; + int group_num=1,ret=0; + memset(&rule,0,sizeof(rule)); + rule.config_id=config_id; + strcpy(rule.service_defined,"maat_command"); + //MUST acqire by function, because Maat_cmd_t has some hidden members. + cmd=Maat_create_cmd(&rule, group_num); + cmd->expire_after=timeout; + cmd->label_id=label_id; + memset(®ion,0,sizeof(region)); + region.region_type=REGION_EXPR; + region.table_name=region_table; + region.expr_rule.district=NULL; + region.expr_rule.keywords=keywords; + region.expr_rule.expr_type=EXPR_TYPE_AND; + region.expr_rule.match_method=MATCH_METHOD_SUB; + region.expr_rule.hex_bin=UNCASE_PLAIN; + Maat_cmd_set_opt(cmd, MAAT_RULE_SERV_DEFINE, huge_serv_def, sizeof(huge_serv_def)); + Maat_add_region2cmd(cmd, 0, ®ion); + //use pipeline model. + ret=Maat_cmd_append(feather, cmd, MAAT_OP_ADD); + if(ret<0) + { + printf("Add Maat command %d failed.\n",rule.config_id); + Maat_free_cmd(cmd); + return 0; + } + //cmd has been saved in feather, so free cmd before commit is allowed. + Maat_free_cmd(cmd); + return 0; + +} +void wait_for_cmd_effective_copy(Maat_feather_t feather, long long version_before) +{ + long long version_after=version_before; + int is_updating=1; + long long wating_us=0, sleep_us=1000*100; + while(is_updating||version_before==version_after) + { + Maat_read_state(feather,MAAT_STATE_IN_UPDATING, &is_updating, sizeof(is_updating)); + Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after)); + + usleep(sleep_us);//waiting for commands go into effect + wating_us+=sleep_us; + } +// printf("wait for %lld ms\n", wating_us/1000); +} + +class MaatCMDPerfTest : public testing::Test +{ + +protected: + + + static void SetUpTestCase() + { + const char* test_maat_redis_ip="127.0.0.1"; + unsigned short test_maat_redis_port=6379; + int g_iThreadNum=4; + const char* table_info_path="./table_info.conf"; + int scan_interval_ms=500; + int effective_interval_ms=0; + + logger=MESA_create_runtime_log_handle("test_maat_redis.log",0); + + _shared_feather=Maat_feather(g_iThreadNum, table_info_path, logger); + Maat_set_feather_opt(_shared_feather,MAAT_OPT_INSTANCE_NAME,"perf", strlen("perf")+1); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_IP, test_maat_redis_ip, strlen(test_maat_redis_ip)+1); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_PORT, &test_maat_redis_port, sizeof(test_maat_redis_port)); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); + //Set a short intevral for testing. + Maat_set_feather_opt(_shared_feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms)); + + const char* foregin_dir="./foreign_files/"; + Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR, foregin_dir, strlen(foregin_dir)+1); + + Maat_cmd_flushDB(_shared_feather); + Maat_initiate_feather(_shared_feather); + } + static void TearDownTestCase() + { + Maat_burn_feather(_shared_feather); + MESA_destroy_runtime_log_handle(logger); + + } + // Some expensive resource shared by all tests. + static Maat_feather_t _shared_feather; + static void *logger; +}; +Maat_feather_t MaatCMDPerfTest::_shared_feather; + +void* MaatCMDPerfTest::logger; + +//Following tests must be coded/tested at last, for they stalled the maat update thread and interrupt other tests. +TEST_F(MaatCMDPerfTest, SetExpr200K) +{ + const int CMD_EXPR_NUM=200*1000; + const char* table_name="HTTP_URL"; + + const char* keywords1="Hiredis"; + const char* keywords2="C Client"; + char escape_buff1[256],escape_buff2[256]; + char keywords[256]; + + int label_id=5210, config_id=0,ret=0, output_id_cnt=0; + Maat_feather_t feather=MaatCMDPerfTest::_shared_feather; + long long version_before=0; + ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before)); + + Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1); + Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2); + snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2); + + config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", CMD_EXPR_NUM); + int i=0; + for(i=0; i=0); + wait_for_cmd_effective_copy(feather, version_before); + struct Maat_cmd_t* cmd=NULL; + struct Maat_rule_t rule; + memset(&rule,0,sizeof(rule)); + int *output_ids=(int*)malloc(sizeof(int)*CMD_EXPR_NUM); + output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, CMD_EXPR_NUM); + EXPECT_EQ(output_id_cnt, CMD_EXPR_NUM); + for(i=0; i0) + { + pass_flag=1; + break; + } + } + EXPECT_EQ(pass_flag, 1); + EXPECT_EQ(result[0].config_id, 164); + + Maat_stream_scan_string_end(&sp); + free(hit_detail); + fclose(fp); + Maat_clean_status(&mid); + return; + +} + + TEST(NOTLogic, ScanNotAtLast) { const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-144."; @@ -1462,6 +1517,126 @@ TEST(ScanResult, LongerServiceDefine) free(buff); return; } +TEST(VirtualTable, VirtualWithPhysical) +{ +#define TestVirtualTable1 + int ret=0, table_id=0; + const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me."; + const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "HTTP_URL"); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_url, strlen(http_url), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_KEYWORDS"); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_content, strlen(http_content), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 160); + + const char* should_not_hit="2018-10-05 is a keywords of table KEYWORDS_TABLE. Should not hit."; + + mid=NULL; + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, should_not_hit, strlen(should_not_hit), + result, NULL, 4, &mid, 0); + EXPECT_EQ(ret, -2); + Maat_clean_status(&mid); + + return; +} +TEST(VirtualTable, VirtualWithVirtual) +{ +#define TestVirtualTable2 + int ret=0, table_id=0; + const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; + const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "User-Agent", strlen("User-Agent")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_req_hdr_ua, strlen(http_req_hdr_ua), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 162); + + Maat_clean_status(&mid); + + return; +} +TEST(VirtualTable, OneGroupInTwoVirtual) +{ +#define TestVirtualTable3 + int ret=0, table_id=0; + const char* http_resp_hdr_cookie="sessionid=888888;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 163); + + Maat_clean_status(&mid); + + return; +} + class MaatFileTest : public testing::Test { @@ -1589,8 +1764,6 @@ protected: const char* foregin_dir="./foreign_files/"; Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR,foregin_dir, strlen(foregin_dir)+1); - linger_timeout=2; - Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_LINGER,&linger_timeout, sizeof(linger_timeout)); Maat_cmd_flushDB(_shared_feather); Maat_initiate_feather(_shared_feather); @@ -2174,6 +2347,123 @@ TEST_F(MaatCmdTest, RefGroup) EXPECT_EQ(result[0].config_id, compile1.config_id); Maat_clean_status(&mid); +} +#define MaatCmdTest_VirtualTable +TEST_F(MaatCmdTest, VirtualTable) +{ + Maat_feather_t feather=MaatCmdTest::_shared_feather; + const char* group_table_name="GROUP"; + const char* compile_table_name="COMPILE"; + const char* region_table_name="HTTP_SIGNATURE"; + + struct Maat_rule_t compile1; + struct Maat_group_t group1, group2; + struct Maat_region_t region1, region2; + + memset(&compile1, 0, sizeof(compile1)); + compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 2); + + + //group1->compile1 + memset(&group1, 0, sizeof(group1)); + group1.group_id=Maat_cmd_get_new_group_id(feather); + group1.table_name=group_table_name; + group1.virtual_table_name="HTTP_REQUEST_HEADER"; + group1.parent_id=compile1.config_id; + group1.parent_type=PARENT_TYPE_COMPILE; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1); + + + /*region1->group1->compile1 + */ + memset(®ion1, 0, sizeof(region1)); + region1.region_id=Maat_cmd_get_new_region_id(feather); + region1.region_type=REGION_EXPR; + region1.table_name=region_table_name; + region1.expr_rule.district="User-Agent"; + region1.expr_rule.keywords="AppleWebKit"; + region1.expr_rule.expr_type=EXPR_TYPE_STRING; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion1, group1.group_id); + + + //group2->compile1 + memset(&group2, 0, sizeof(group2)); + group2.group_id=Maat_cmd_get_new_group_id(feather); + group2.table_name=group_table_name; + group2.virtual_table_name="HTTP_RESPONSE_HEADER"; + group2.parent_id=compile1.config_id; + group2.parent_type=PARENT_TYPE_COMPILE; + Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2); + + //region2->group2 + memset(®ion2, 0, sizeof(region2)); + region2.region_id=Maat_cmd_get_new_region_id(feather); + region2.region_type=REGION_EXPR; + region2.table_name=region_table_name; + region2.expr_rule.district="Cookie"; + region2.expr_rule.keywords="uid=12345678;"; + region2.expr_rule.expr_type=EXPR_TYPE_STRING; + Maat_command_raw_set_region(feather, MAAT_OP_ADD, ®ion2, group2.group_id); + + sleep(1); + + int ret=0, table_id=0; + const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; + const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(feather, "HTTP_REQUEST_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, "User-Agent", strlen("User-Agent")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, http_req_hdr_ua, strlen(http_req_hdr_ua), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile1.config_id); + + Maat_clean_status(&mid); + + //Delete group1 + Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1); + Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 2); + Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1); + + sleep(1); + + table_id=Maat_table_register(feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, compile1.config_id); + Maat_clean_status(&mid); + return; + } TEST_F(MaatCmdTest, SetLines) @@ -2211,97 +2501,7 @@ TEST_F(MaatCmdTest, SetLines) return; } -/* -//Following tests must be coded/tested at last, for they stalled the maat update thread and interrupt other tests. -TEST_F(MaatCmdTest, SetExprOneMillion) -{ - const int CMD_EXPR_NUM=1000*1000; - const char* table_name="HTTP_URL"; - - const char* keywords1="Hiredis"; - const char* keywords2="C Client"; - char escape_buff1[256],escape_buff2[256]; - char keywords[256]; - int label_id=5210, config_id=0,ret=0, output_id_cnt=0; - Maat_feather_t feather=MaatCmdTest::_shared_feather; - long long version_before=0; - ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before)); - - Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1); - Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2); - snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2); - - config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", CMD_EXPR_NUM); - int i=0; - for(i=0; i=0); - wait_for_cmd_effective(feather, version_before); - struct Maat_cmd_t* cmd=NULL; - struct Maat_rule_t rule; - memset(&rule,0,sizeof(rule)); - int *output_ids=(int*)malloc(sizeof(int)*CMD_EXPR_NUM); - output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, CMD_EXPR_NUM); - EXPECT_EQ(output_id_cnt, CMD_EXPR_NUM); - for(i=0; i + + + + + + + + + + + + + + + ѻ + + + + + +
+ +
+ + + + + +
+ + + +
+ + + + + + +
+ :
+ +#include #include /* for printf */ #include /* for exit */ #include #include +#include +void debug_maat_result_print(const char* table_name, int scan_ret, struct Maat_rule_t* result) +{ + printf("Scan table %s ", table_name); + if(scan_ret==-1) + { + printf("error.\n"); + } + else if(scan_ret==-2) + { + printf("hits group, but not compile.\n"); + } + else if(scan_ret==0) + { + printf("not hit."); + } + else + { + printf("hits "); + for(int i=0; i0) { - printf("Read %s failed, invalid maat json.\n", arg_value[ARG_INPUT_JSON]); + ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, arg_value[ARG_INPUT_JSON], strlen(arg_value[ARG_INPUT_JSON])+1); + if(ret!=0) + { + printf("Read %s failed, invalid maat json.\n", arg_value[ARG_INPUT_JSON]); + ret=-1; + goto clean_up; + } + } + else if(strlen(arg_value[ARG_INPUT_FULL_INDEX])) + { + ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, arg_value[ARG_INPUT_FULL_INDEX], strlen(arg_value[ARG_INPUT_FULL_INDEX])+1); + if(ret!=0) + { + printf("Set %s failed, invalid maat json.\n", arg_value[ARG_INPUT_FULL_INDEX]); + ret=-1; + goto clean_up; + } + } + else + { + printf("Error: One of --%s and --%s should be specified.\n", long_options[ARG_INPUT_JSON].name, long_options[ARG_INPUT_FULL_INDEX].name); ret=-1; goto clean_up; } @@ -165,6 +221,10 @@ int main(int argc, char ** argv) debug_maat_str_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_DISTRICT], file_buff, file_size); free(file_buff); } + if(strlen(arg_value[ARG_SCAN_IPv4])>0) + { + debug_maat_ip_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_IPv4]); + } clean_up: Maat_burn_feather(feather); MESA_destroy_runtime_log_handle(g_logger); diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 60a648d..41fe97e 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -1,4 +1,5 @@ #include "Maat_rule.h" +#include "Maat_utils.h" #include "Maat_command.h" #include "Maat_rule_internal.h" #include "cJSON.h" @@ -238,7 +239,7 @@ int main(int argc, char * argv[]) unsigned long json_file_size=0,read_size=0; long long desired_version=0; char* json_buff=NULL; - + size_t json_buff_sz=0; while((oc=getopt(argc,argv,"h:p:n:d:v:f:j:t:"))!=-1) { switch(oc) @@ -325,8 +326,13 @@ int main(int argc, char * argv[]) read_rule_from_redis(ctx,desired_version,dump_dir, NULL); } else if(model==WORK_MODE_JSON) - { - ret=json2iris(json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL); + { + ret=load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz); + if(ret<0) + { + printf("open %s failed.\n", json_file); + } + ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL); if(ret<0) { printf("Invalid json format.\n"); @@ -336,6 +342,10 @@ int main(int argc, char * argv[]) printf("Serialize %s to %d lines, write temp file to %s .\n",json_file, total_line_cnt, tmp_iris_path); struct serial_rule_t * s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),total_line_cnt); server_time=redis_server_time(ctx); + if(!server_time) + { + printf("Get Redis Time failed.\n"); + } if(timeout>0) { absolute_expire_time=server_time+timeout;