diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index bea7e76..30a25c9 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -6,7 +6,7 @@ * to reside in the heart) of the departed would reach the paradise of afterlife * successfully. * Author: zhengchao@iie.ac.cn, MESA -* Version 2018-11-06 Maat Rule Extra Data. +* Version 2018-12-07 Plugin Extra Data. * NOTE: MUST compile with G++ * All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018 ********************************************************* @@ -165,8 +165,9 @@ enum MAAT_INIT_OPT int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); enum MAAT_STATE_OPT { - MAAT_STATE_VERSION=1, //Get current maat version. VALUE is long long, SIZE=sizeof(long long). - MAAT_STATE_LAST_UPDATING_TABLE //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no + MAAT_STATE_VERSION=1, //Get current maat version, if maat is in update progress, the updating version is returned. VALUE is long long, SIZE=sizeof(long long). + MAAT_STATE_LAST_UPDATING_TABLE, //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no + MAAT_STATE_IN_UPDATING }; int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size); @@ -257,6 +258,21 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table //returned data is duplicated by dup_func of Maat_rule_get_ex_new_index, caller is responsible to free the data. MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx); +//Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table. +typedef void* MAAT_PLUGIN_EX_DATA; +typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_free_func_t(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp); +typedef int Maat_plugin_EX_key2index_func_t(const char* key); + +int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp); +//Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to free the data. +MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key); enum MAAT_RULE_OPT { diff --git a/inc/gram_index_engine.h b/inc/gram_index_engine.h index a69e924..452dc4d 100644 --- a/inc/gram_index_engine.h +++ b/inc/gram_index_engine.h @@ -4,12 +4,16 @@ #ifdef __cplusplus extern "C" { #endif - -#define GIE_INSERT_OPT 0 -#define GIE_DELETE_OPT 1 -#define GIE_INPUT_FORMAT_SFH 1 -#define GIE_INPUT_FORMAT_PLAIN 0 - +enum GIE_operation +{ + GIE_INSERT_OPT, + GIE_DELETE_OPT +}; +enum GIE_INPUT_FORMAT +{ + GIE_INPUT_FORMAT_PLAIN, + GIE_INPUT_FORMAT_SFH +}; typedef struct { @@ -21,7 +25,7 @@ typedef struct { unsigned int id; unsigned int sfh_length;//size of fuzzy_hash - short operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed; + enum GIE_operation operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed; short cfds_lvl; char * sfh; void * tag; @@ -41,9 +45,9 @@ typedef struct unsigned int gram_value; //unsigned int htable_num; unsigned int position_accuracy; - short format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string + enum GIE_INPUT_FORMAT format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string //else id format==PALIN, means the input string is common string - short ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result + int ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result }GIE_create_para_t; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f9846ec..20b6197 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,8 +1,8 @@ cmake_minimum_required(VERSION 3.5) set(MAAT_FRAME_MAJOR_VERSION 2) -set(MAAT_FRAME_MINOR_VERSION 4) -set(MAAT_FRAME_PATCH_VERSION 20181115) +set(MAAT_FRAME_MINOR_VERSION 5) +set(MAAT_FRAME_PATCH_VERSION 0) set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION}) message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index b353063..a8ccf77 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -16,9 +16,9 @@ #include "rulescan.h" #include "json2iris.h" -struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type) +struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type) { - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; if(table_id>MAX_TABLE_NUM) { return NULL; @@ -113,7 +113,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int int ret=0,i=0; int r_in_c_cnt=0; int shortcut_avilable_cnt=0; - void* bool_matcher=feather->scanner->expr_compiler; + void* bool_matcher=feather->scanner->bool_macher_expr_compiler; struct _Maat_group_inner_t* group_rule=NULL; struct _Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; struct _Maat_compile_inner_t* _mi_rule=NULL; @@ -443,7 +443,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* return NULL; } _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); - feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger); + feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger); feather->map_tablename2id=map_create(); int i=0,j=0,ret=0; for(i=0;ip_table_info[table_id]; + struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; + struct plugin_table_desc *plugin_desc=&(p_table->plugin); + struct plugin_runtime* plugin_aux=NULL; const char* lines=NULL; if(p_table==NULL) { @@ -930,26 +932,32 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, } //plugin table register blocks background update. pthread_mutex_lock(&(_feather->backgroud_update_mutex)); - idx=p_table->cb_info->cb_plug_cnt; + idx=plugin_desc->cb_plug_cnt; if(idx==MAX_PLUGIN_PER_TABLE) { pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); return -1; } - p_table->cb_info->cb_plug_cnt++; - p_table->cb_info->cb_plug[idx].start=start; - p_table->cb_info->cb_plug[idx].update=update; - p_table->cb_info->cb_plug[idx].finish=finish; - p_table->cb_info->cb_plug[idx].u_para=u_para; - if(p_table->cb_info->cache_line_num>0) + plugin_desc->cb_plug_cnt++; + plugin_desc->cb_plug[idx].start=start; + plugin_desc->cb_plug[idx].update=update; + plugin_desc->cb_plug[idx].finish=finish; + plugin_desc->cb_plug[idx].u_para=u_para; + if(_feather->scanner==NULL) + { + pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); + return 1; + } + plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin); + if(plugin_aux->cache_line_num>0) { if(start!=NULL) { start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); } - for(i=0;icb_info->cache_line_num;i++) + for(i=0;icache_line_num;i++) { - lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i); + lines=(const char*)dynamic_array_read(plugin_aux->cache_lines,i); if(lines==NULL) { break; @@ -996,31 +1004,31 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table { return -1; } - struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; - if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL) + struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; + if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL || dup_func==NULL) { return -1; } - + struct compile_table_desc* compile_desc=&(p_table->compile); pthread_mutex_lock(&(_feather->backgroud_update_mutex)); - if(p_table->ex_data_num==MAX_COMPILE_EX_DATA_NUM) + if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM) { ret=-1; goto failed; } - idx=p_table->ex_data_num; - p_table->ex_desc[idx].idx=idx; - p_table->ex_desc[idx].table_id=table_id; - p_table->ex_desc[idx].argl=argl; - p_table->ex_desc[idx].argp=argp; - p_table->ex_desc[idx].new_func=new_func; - p_table->ex_desc[idx].free_func=free_func; - p_table->ex_desc[idx].dup_func=dup_func; + idx=compile_desc->ex_data_num; + compile_desc->ex_desc[idx].idx=idx; + compile_desc->ex_desc[idx].table_id=table_id; + compile_desc->ex_desc[idx].argl=argl; + compile_desc->ex_desc[idx].argp=argp; + compile_desc->ex_desc[idx].new_func=new_func; + compile_desc->ex_desc[idx].free_func=free_func; + compile_desc->ex_desc[idx].dup_func=dup_func; - p_table->ex_data_num++; + compile_desc->ex_data_num++; if(_feather->scanner!=NULL) { - MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, p_table->ex_desc+idx); + MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_desc->ex_desc+idx); } failed: pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); @@ -1031,7 +1039,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_compile_inner_t *compile_inner=NULL; - const struct compile_ex_data_idx* ex_desc=NULL; + const struct compile_table_desc* compile_desc=NULL; + const struct compile_ex_data_idx* ex_desc=NULL; MAAT_RULE_EX_DATA ad=NULL; compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); @@ -1040,12 +1049,213 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa return NULL; } pthread_rwlock_rdlock(&(compile_inner->rwlock)); - assert(idxref_table->ex_data_num); - ex_desc=compile_inner->ref_table->ex_desc+idx; + compile_desc=&(compile_inner->ref_table->compile); + assert(idxex_data_num); + ex_desc=compile_desc->ex_desc+idx; ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp); pthread_rwlock_unlock(&(compile_inner->rwlock)); return ad; } +struct wrap_plugin_EX_data +{ + MAAT_RULE_EX_DATA exdata; + const struct Maat_table_desc* ref_plugin_table; +}; +void wrap_plugin_EX_data_free(void *data) +{ + struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; + const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); + ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); + wrap_data->ref_plugin_table=NULL; + free(wrap_data); + return; +} +MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) +{ + MESA_htable_handle key2ex_hash=NULL; + unsigned int slot_size=1; + while(estimate_size!=0) + { + estimate_size=estimate_size>>1; + slot_size*=2; + } + if(slot_size==1) + { + slot_size=4096; + } + + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=0; + hargs.hash_slot_size = slot_size; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = NULL; //Not supported yet. + hargs.recursive = 1; + hargs.data_free = wrap_plugin_EX_data_free; + hargs.data_expire_with_condition = NULL; + key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(key2ex_hash, 0); + return key2ex_hash; +} + +int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + size_t key_offset=0, key_len=0; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: no such key %.*s of %s", + key_len, line+key_offset, line); + return -1; + } + return 0; +} + +int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + char* key=NULL; + size_t key_offset=0, key_len=0; + MAAT_RULE_EX_DATA exdata=NULL; + struct wrap_plugin_EX_data* wrap_data=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + key=ALLOC(char, key_len+1); + memcpy(key, line+key_offset, key_len); + plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); + wrap_data->exdata=exdata; + wrap_data->ref_plugin_table=plugin_table; + ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); + free(key); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: duplicated key %.*s of %s", + key_len, line+key_offset, line); + wrap_plugin_EX_data_free(wrap_data); + return -1; + } + return 0; +} +MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table, + struct dynamic_array_t* lines, size_t line_cnt, void* logger) +{ + MESA_htable_handle key2ex_hash=NULL; + size_t i=0; + const char* line=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + + key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); + + for(i=0; i< line_cnt; i++) + { + line=(const char*)dynamic_array_read(lines, i); + plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); + } + return key2ex_hash; +} +int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp) + +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; + struct plugin_table_desc* plugin_desc=&(table_desc->plugin); + struct Maat_table_runtime* table_rt=NULL; + if(new_func==NULL || free_func==NULL || dup_func==NULL ) + { + assert(0); + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); + return -1; + } + if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata + || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) + { + assert(0); + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, + "%s failed: key or valid flag column are not specified", __FUNCTION__); + return -1; + } + plugin_desc->ex_desc.new_func=new_func; + plugin_desc->ex_desc.free_func=free_func; + plugin_desc->ex_desc.dup_func=dup_func; + plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. + plugin_desc->ex_desc.argl=argl; + plugin_desc->ex_desc.argp=argp; + plugin_desc->have_exdata=1; + if(_feather->scanner==NULL) + { + return 0; + } + + pthread_mutex_lock(&(_feather->backgroud_update_mutex)); + table_rt=_feather->scanner->table_rt[table_id]; + assert(table_rt->plugin.key2ex_hash==NULL); + table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, + table_rt->plugin.cache_line_num, _feather->logger); + pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); + + return 0; +} +MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; + struct Maat_table_runtime *table_rt= NULL; + struct plugin_table_desc* plugin_desc=&(table_desc->plugin); + struct wrap_plugin_EX_data* wrap_data=NULL; + MAAT_RULE_EX_DATA exdata=NULL; + if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0) + { + assert(0); + return NULL; + } + if(_feather->scanner==NULL) + { + return NULL; + } + table_rt= _feather->scanner->table_rt[table_id]; + pthread_rwlock_rdlock(&table_rt->plugin.rwlock); + wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, + (const unsigned char*)key, strlen(key)); + if(wrap_data!=NULL) + { + plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata), + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + } + pthread_rwlock_unlock(&table_rt->plugin.rwlock); + return exdata; +} + int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num @@ -1058,7 +1268,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num];//dynamic array - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; + struct expr_table_desc* expr_desc=NULL; struct timespec start,end; _Maat_scanner_t* my_scanner=NULL; if(data==NULL||data_len<=0) @@ -1076,17 +1287,13 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id _feather->scan_err_cnt++; return -1; } - - if(p_table->cfg_num==0) - { - return 0; - } + expr_desc=&(p_table->expr); if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) { _feather->scan_err_cnt++; return -1; } - if(p_table->do_charset_merge==1) + if(expr_desc->do_charset_merge==1) { sub_type=make_sub_type(table_id,CHARSET_NONE,0); } @@ -1104,9 +1311,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { return 0; } - struct _region_stat_t * region_stat=NULL; - region_stat=&(my_scanner->region_counter[p_table->table_id]); - if(region_stat->cfg_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } @@ -1114,7 +1320,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; INC_SCANNER_REF(my_scanner, thread_num); - if(region_stat->expr_rule_cnt>0) + if(table_rt->expr.expr_rule_cnt>0) { scan_data.rule_type=RULETYPE_STR; scan_data.sub_type=sub_type; @@ -1124,7 +1330,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id hit_region_cnt+=region_ret; } } - if(region_stat->regex_rule_cnt>0) + if(table_rt->expr.regex_rule_cnt>0) { scan_data.rule_type=RULETYPE_REG; scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); @@ -1141,7 +1347,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(hit_region_cnt>0) { - alignment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid,_feather,thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1162,17 +1368,16 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id hit_detail,detail_num); } } - DEC_SCANNER_REF(my_scanner, thread_num); if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(p_table,data_len,&start, &end,thread_num); + maat_stat_table(table_rt, data_len, &start, &end, thread_num); } else { - maat_stat_table(p_table,data_len,NULL, NULL,thread_num); + maat_stat_table(table_rt, data_len, NULL, NULL, thread_num); } - + DEC_SCANNER_REF(my_scanner, thread_num); if(compile_ret==0&&hit_region_cnt>0) { return -2; @@ -1206,7 +1411,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id intval_scan_data.rule_type=RULETYPE_INT; intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); intval_scan_data.int_data=intval; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; struct timespec start,end; if(_feather->perf_on==1) { @@ -1218,18 +1423,13 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id _feather->scan_err_cnt++; return -1; } - if(p_table->cfg_num==0) - { - return 0; - } my_scanner=_feather->scanner; if(my_scanner==NULL) { return 0; } - struct _region_stat_t * region_stat=NULL; - region_stat=&(my_scanner->region_counter[p_table->table_id]); - if(region_stat->cfg_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } @@ -1247,7 +1447,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1262,16 +1462,17 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } } - DEC_SCANNER_REF(my_scanner,thread_num); if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(p_table,0,&start, &end,thread_num); + maat_stat_table(table_rt, 0, &start, &end, thread_num); } else { - maat_stat_table(p_table,0,NULL, NULL,thread_num); + maat_stat_table(table_rt, 0, NULL, NULL, thread_num); } + DEC_SCANNER_REF(my_scanner,thread_num); + if(compile_ret==0&®ion_ret>0) { return -2; @@ -1290,7 +1491,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id scan_data_t ip_scan_data; scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num]; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* my_scanner=NULL; @@ -1305,26 +1506,22 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id _feather->scan_err_cnt++; return -1; } - if(p_table->cfg_num==0) - { - return 0; - } my_scanner=_feather->scanner; if(my_scanner==NULL) { return 0; } - struct _region_stat_t * region_stat=NULL; - region_stat=&(my_scanner->region_counter[p_table->table_id]); - if(region_stat->cfg_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } - if(region_stat->ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4) + + if(table_rt->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4) { return 0; } - if(region_stat->ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) + if(table_rt->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) { return 0; } @@ -1369,7 +1566,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, @@ -1388,11 +1585,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(p_table,0,&start, &end,thread_num); + maat_stat_table(table_rt, 0, &start, &end, thread_num); } else { - maat_stat_table(p_table,0,NULL, NULL,thread_num); + maat_stat_table(table_rt, 0, NULL, NULL, thread_num); } if(compile_ret==0&®ion_ret>0) { @@ -1417,7 +1614,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* scanner=NULL; - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; assert(thread_num<_feather->scan_thread_num); p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR); if(p_table==NULL) @@ -1425,12 +1622,14 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, _feather->scan_err_cnt++; return NULL; } - if(p_table->quick_expr_switch==1) + + struct expr_table_desc* expr_desc=&(p_table->expr); + if(expr_desc->quick_expr_switch==1) { _feather->scan_err_cnt++; return NULL; } - struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); + struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1); scanner=_feather->scanner; sp->feather=_feather; sp->version=_feather->maat_version; @@ -1440,29 +1639,32 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { return sp; } - struct _region_stat_t * region_stat=NULL; - region_stat=&(scanner->region_counter[p_table->table_id]); + struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) + { + return 0; + } INC_SCANNER_REF(scanner, thread_num); sp->table_id=table_id; sp->thread_num=thread_num; - sp->max_cross_size=p_table->cross_cache_size; + sp->max_cross_size=expr_desc->cross_cache_size; sp->caching_size=0; sp->scan_buff=NULL; sp->last_cache=NULL; - if(p_table->do_charset_merge==1) + if(expr_desc->do_charset_merge==1) { sp->do_merge=1; } - if(region_stat->expr_rule_cnt>0) + if(table_rt->expr.expr_rule_cnt>0) { sp->do_expr=1; } - if(region_stat->regex_rule_cnt>0) + if(table_rt->expr.regex_rule_cnt>0) { sp->do_regex=1; } - alignment_int64_array_add(p_table->stream_num,thread_num,1); + alignment_int64_array_add(table_rt->stream_num,thread_num,1); sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); return sp; } @@ -1481,7 +1683,8 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para scan_result_t *region_result; _compile_result_t compile_result[rule_num];//dynamic array scan_data_t region_scan_data; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; + struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; struct timespec start,end; if(data==NULL||data_len<=0) { @@ -1528,7 +1731,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para free(sp->scan_buff); sp->scan_buff=NULL; } - sp->scan_buff=(char*)malloc(sp->caching_size+data_len); + sp->scan_buff=ALLOC(char, sp->caching_size+data_len); memcpy(sp->scan_buff,sp->last_cache,sp->caching_size); memcpy(sp->scan_buff+sp->caching_size,data,data_len); region_scan_data.text_data.text=sp->scan_buff; @@ -1543,7 +1746,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->last_cache==NULL&&sp->max_cross_size>0) { assert(sp->caching_size==0); - sp->last_cache=(char*)malloc(sizeof(char)*sp->max_cross_size); + sp->last_cache=ALLOC(char, sp->max_cross_size); } if(sp->max_cross_size>0) { @@ -1589,7 +1792,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } if(hit_region_cnt>0) { - alignment_int64_array_add(p_table->hit_cnt, sp->thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1628,11 +1831,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num); + maat_stat_table(table_rt,data_len,&start, &end,sp->thread_num); } else { - maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num); + maat_stat_table(table_rt,data_len,NULL, NULL,sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -1657,8 +1860,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; - alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1); + struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; + alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); if(sp->rs_stream_para!=NULL) { if(scanner!=NULL&&sp->version>=sp->feather->last_full_version) @@ -1695,14 +1898,14 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* scanner=NULL; sfh_instance_t * tmp_fuzzy_handle=NULL; - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST); if(p_table==NULL) { _feather->scan_err_cnt++; return NULL; - } - struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); + } + struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1); scanner=_feather->scanner; sp->feather=_feather; sp->version=_feather->maat_version; @@ -1710,7 +1913,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, if(scanner==NULL) { return sp; - } + } + struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; tmp_fuzzy_handle=SFH_instance(total_len); if(tmp_fuzzy_handle==NULL) { @@ -1724,7 +1928,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sp->total_len=total_len; sp->fuzzy_hash_handle=tmp_fuzzy_handle; pthread_mutex_init(&(sp->fuzzy_mutex),NULL); - alignment_int64_array_add(p_table->stream_num,thread_num,1); + alignment_int64_array_add(table_rt->stream_num,thread_num,1); return sp; } @@ -1761,7 +1965,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int GIE_result_t query_result[MAX_SCANNER_HIT_NUM]; int hit_region_cnt=0,compile_ret=0; _compile_result_t compile_result[rule_num];//dynamic array - _Maat_table_info_t* p_table=NULL; if(data==NULL||data_len<=0) { return 0; @@ -1770,7 +1973,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return 0; } - GIE_handle_t* GIE_handle=sp->feather->scanner->gie_aux[sp->table_id].gie_handle; + struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; unsigned long long digest_len=0; char* digest_buff=NULL; struct _OUTER_scan_status_t* _mid=NULL; @@ -1779,7 +1983,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { clock_gettime(CLOCK_MONOTONIC,&start); } - p_table=sp->feather->p_table_info[sp->table_id]; alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); pthread_mutex_lock(&(sp->fuzzy_mutex)); sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset); @@ -1796,7 +1999,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { goto fast_out; } - digest_buff=(char*)malloc(sizeof(char)*digest_len); + digest_buff=ALLOC(char, digest_len); pthread_mutex_lock(&(sp->fuzzy_mutex)); SFH_digest(sp->fuzzy_hash_handle,digest_buff, digest_len); pthread_mutex_unlock(&(sp->fuzzy_mutex)); @@ -1816,7 +2019,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int } if(hit_region_cnt>0) { - alignment_int64_array_add(p_table->hit_cnt, sp->thread_num, 1); + alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1); _mid=grab_mid(mid,sp->feather, sp->thread_num,1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1834,11 +2037,11 @@ fast_out: if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num); + maat_stat_table(table_rt, data_len, &start, &end, sp->thread_num); } else { - maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num); + maat_stat_table(table_rt, data_len, NULL, NULL, sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -1850,8 +2053,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; - alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1); + struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); if(scanner!=NULL) { if(sp->version==sp->feather->maat_version) @@ -1946,7 +2149,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id _compile_result_t compile_result[rule_num]; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* my_scanner=NULL; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; struct timespec start,end; if(_feather->perf_on==1) { @@ -1958,25 +2161,20 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id _feather->scan_err_cnt++; return -1; } - if(p_table->cfg_num==0) - { - return 0; - } my_scanner=_feather->scanner; if(my_scanner==NULL) { return 0; } - GIE_handle_t* gie_handle=my_scanner->gie_aux[table_id].gie_handle; - struct _region_stat_t * region_stat=NULL; - region_stat=&(my_scanner->region_counter[p_table->table_id]); - if(region_stat->cfg_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } + GIE_handle_t* gie_handle=table_rt->similar.gie_handle; + INC_SCANNER_REF(my_scanner,thread_num); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); - INC_SCANNER_REF(my_scanner,thread_num); region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM); if(region_ret<0) { @@ -1986,7 +2184,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -2005,11 +2203,11 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(p_table,0,&start, &end,thread_num); + maat_stat_table(table_rt,0,&start, &end,thread_num); } else { - maat_stat_table(p_table,0,NULL, NULL,thread_num); + maat_stat_table(table_rt,0,NULL, NULL,thread_num); } if(compile_ret==0&®ion_ret>0) { @@ -2062,6 +2260,20 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu case MAAT_STATE_LAST_UPDATING_TABLE: *int_val=_feather->is_last_plugin_table_updating; break; + case MAAT_STATE_IN_UPDATING: + if(size!=sizeof(int)) + { + return -1; + } + if(0==pthread_mutex_trylock(&(_feather->backgroud_update_mutex))) + { + *int_val=0; + pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); + } + else + { + *int_val=1; + } default: return -1; break; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index bf6f8b4..bb43b09 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -182,61 +182,51 @@ enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p) } int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) { - unsigned int offset=0; - unsigned int i=0,j=0; - switch(type) + size_t offset=0, len=0; + unsigned int column_seq=0, ret=0; + switch(type) { case TABLE_TYPE_EXPR: - offset=7; + column_seq=7; break; case TABLE_TYPE_IP: - offset=14; + column_seq=14; break; case TABLE_TYPE_COMPILE: - offset=8; + column_seq=8; break; case TABLE_TYPE_PLUGIN: if(valid_column_seq<0) { return -1; } - offset=(unsigned int)valid_column_seq; + column_seq=(unsigned int)valid_column_seq; break; case TABLE_TYPE_INTERVAL: - offset=5; + column_seq=5; break; case TABLE_TYPE_DIGEST: - offset=6; + column_seq=6; break; case TABLE_TYPE_SIMILARITY: - offset=5; + column_seq=5; break; case TABLE_TYPE_EXPR_PLUS: - offset=8; + column_seq=8; break; case TABLE_TYPE_GROUP: - offset=3; + column_seq=3; break; default: assert(0); } - for(i=0;i=strlen(line)||(line[i]!='1'&&line[i]!='0'))// 0 is also a valid value for some non-MAAT producer. + + ret=get_column_pos(line, column_seq, &offset, &len); + if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer. { return -1; } - return i; + return offset; } int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) { @@ -1074,28 +1064,34 @@ long long _exec_serial_rule_begin(redisContext* ctx,int rule_num, int renew_rule } return ret; } -redisReply* _exec_serial_rule_end(redisContext* ctx,long long maat_redis_version, long long server_time, int renew_allowed) +redisReply* _exec_serial_rule_end(redisContext* ctx,long long maat_redis_version, long long server_time, int renew_allowed, struct expected_reply_t* expect_reply, unsigned int *cnt) { redisReply* data_reply=NULL; if(renew_allowed==1) { redlock_unlock(ctx, rm_expire_lock); + expect_reply[*cnt].srule_seq=-1; + (*cnt)++; } if(maat_redis_version>0) { data_reply=_wrap_redisCommand(ctx,"ZADD %s NX %d %d",rm_version_sset,server_time,maat_redis_version); freeReplyObject(data_reply); + expect_reply[*cnt].srule_seq=-1; + (*cnt)++; data_reply=_wrap_redisCommand(ctx,"INCRBY MAAT_VERSION 1"); freeReplyObject(data_reply); + expect_reply[*cnt].srule_seq=-1; + expect_reply[*cnt].reply.type=REDIS_REPLY_INTEGER; + (*cnt)++; } data_reply=_wrap_redisCommand(ctx,"EXEC"); return data_reply; } -void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_t* s_rule, int rule_num, struct expected_reply_t* expect_reply, unsigned int *cnt, int offset,int renew_allowed) +void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_t* s_rule, unsigned int rule_num, struct expected_reply_t* expect_reply, unsigned int *cnt, int offset,int renew_allowed) { - int i=0; redisReply* data_reply=NULL; - int append_cmd_cnt=0; + unsigned int append_cmd_cnt=0, i=0; for(i=0;ielements-2<=multi_cmd_cnt); + assert(transaction_reply->elements==multi_cmd_cnt); for(i=0;ielement[i]; @@ -1272,11 +1268,11 @@ int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_r { continue; } - assert(jp_table_info[table_id]; - if(p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->n_foreign==0) + plugin_desc= &(p_table->plugin); + if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0) { continue; } - _get_foregin_keys(rule_list+i, p_table->foreign_columns, p_table->n_foreign, dir, logger); + _get_foregin_keys(rule_list+i, plugin_desc->foreign_columns, plugin_desc->n_foreign, dir, logger); rule_with_foreign_key++; } return rule_with_foreign_key; @@ -1736,12 +1734,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m const unsigned char* dec_key, _Maat_feather_t* feather) { - int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0; + int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0, valid_column=-1; int ret=0; struct serial_rule_t* rule_list=NULL; int update_type=CM_UPDATE_TYPE_INC; long long new_version=0; enum MAAT_TABLE_TYPE table_type; + const struct plugin_table_desc* plugin_desc=NULL; void* logger=feather->logger; if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write @@ -1830,7 +1829,16 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m table_type=feather->p_table_info[table_id]->table_type; if(rule_list[i].op==MAAT_OP_DEL) { - ret=invalidate_line(rule_list[i].table_line,table_type,feather->p_table_info[table_id]->valid_flag_column); + if(table_type==TABLE_TYPE_PLUGIN) + { + plugin_desc=&(feather->p_table_info[table_id]->plugin); + valid_column=plugin_desc->valid_flag_column; + } + else + { + valid_column=-1; + } + ret=invalidate_line(rule_list[i].table_line, table_type, valid_column); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invaid format %s ." @@ -2015,7 +2023,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru _Maat_feather_t* _feather=(_Maat_feather_t*)feather; int ret=0, table_id=0,success_cnt=0; struct serial_rule_t *s_rule=NULL; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; long long server_time=0,absolute_expire_time=0; const char* p_foreign=NULL; int foreign_key_size=0; @@ -2039,6 +2048,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru goto error_out; } p_table=_feather->p_table_info[table_id]; + plugin_desc=&(p_table->plugin); if(TABLE_TYPE_PLUGIN!=p_table->table_type) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command @@ -2052,7 +2062,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru { ret=get_valid_flag_offset(line_rule[i]->table_line , p_table->table_type - , p_table->valid_flag_column); + , plugin_desc->valid_flag_column); if(ret<0|| (op==MAAT_OP_ADD&&line_rule[i]->table_line[ret]!='1')) { @@ -2071,17 +2081,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru { absolute_expire_time=server_time+line_rule[i]->expire_after; } - if(p_table->n_foreign>0) + if(plugin_desc->n_foreign>0) { - for(j=0;jn_foreign;j++) + for(j=0;jn_foreign;j++) { - p_foreign=find_Nth_column(line_rule[i]->table_line, p_table->foreign_columns[j], &foreign_key_size); + p_foreign=find_Nth_column(line_rule[i]->table_line, plugin_desc->foreign_columns[j], &foreign_key_size); if(p_foreign==NULL) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL, maat_command , "Command set line %s %d failed: No %dth column." , line_rule[i]->table_name, line_rule[i]->rule_id - , p_table->foreign_columns[j]); + , plugin_desc->foreign_columns[j]); ret=-1; goto error_out; } @@ -2096,7 +2106,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru } } - set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id,line_rule[i]->table_name,line_rule[i]->table_line, absolute_expire_time); + set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name, + line_rule[i]->table_line, absolute_expire_time); } success_cnt=exec_serial_rule(write_ctx,s_rule, line_num,server_time,_feather->logger); if(success_cnt<0||success_cnt!=line_num)//error @@ -2126,6 +2137,11 @@ int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, { struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; redisContext* ctx=_feather->mr_ctx.write_ctx; + if(ctx==NULL) + { + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "%s failed: Redis is not connected.", __FUNCTION__); + return -1; + } const char *arg_vec[3]; size_t len_vec[3]; arg_vec[0] = "SET"; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 26e1171..3fe7c84 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -32,7 +32,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_4_20181202=1; +int MAAT_FRAME_VERSION_2_5_20181205=1; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; @@ -63,6 +63,8 @@ int is_valid_match_method(enum MAAT_MATCH_METHOD match_method) return 0; } } + + iconv_t maat_iconv_open(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from) { const char *from_s=CHARSET_STRING[from]; @@ -521,45 +523,14 @@ void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* s } -_Maat_table_info_t* create_table_info(int max_thread_num) -{ - struct _Maat_table_info_t*p=ALLOC(struct _Maat_table_info_t, 1); - - p->conj_cnt=1; - p->scan_cnt=alignment_int64_array_alloc(max_thread_num); - p->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); - p->input_bytes=alignment_int64_array_alloc(max_thread_num); - p->stream_num=alignment_int64_array_alloc(max_thread_num); - p->hit_cnt=alignment_int64_array_alloc(max_thread_num); - p->cross_cache_size=0; - p->quick_expr_switch=0; - return p; -} -void destroy_table_info(struct _Maat_table_info_t*p) -{ - alignment_int64_array_free(p->scan_cnt); - alignment_int64_array_free(p->scan_cpu_time); - alignment_int64_array_free(p->input_bytes); - alignment_int64_array_free(p->stream_num); - alignment_int64_array_free(p->hit_cnt); - if(p->cb_info!=NULL) - { - dynamic_array_destroy(p->cb_info->cache_lines, free); - p->cb_info->cache_lines=NULL; - free(p->cb_info); - p->cb_info=NULL; - } - free(p); - return; -} -int read_expr_table_info(const char* line, struct _Maat_table_info_t* p, MESA_htable_handle string2int_map) +int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map) { int j=0,ret[4]={0}; char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; char *token=NULL,*sub_token=NULL,*saveptr; - - sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(p->table_id) - ,p->table_name[0] + struct expr_table_desc* p=&(table->expr); + sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) + ,table->table_name[0] ,table_type ,src_charset ,dst_charset @@ -567,7 +538,7 @@ int read_expr_table_info(const char* line, struct _Maat_table_info_t* p, MESA_ht ,&(p->cross_cache_size) ,quick_str_scan); memset(ret,0,sizeof(ret)); - ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(p->table_type)); + ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset)); ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge)); if(strlen(quick_str_scan)>0) @@ -606,6 +577,17 @@ int read_expr_table_info(const char* line, struct _Maat_table_info_t* p, MESA_ht } return 0; } +Maat_table_desc* table_info_new(int max_thread_num) +{ + struct Maat_table_desc*p=ALLOC(struct Maat_table_desc, 1); + p->conj_cnt=1; + return p; +} +void table_info_free(struct Maat_table_desc*p) +{ + free(p); + return; +} int _read_integer_arrary(char* string, int *array, int size) { char *token=NULL,*sub_token=NULL,*saveptr; @@ -619,38 +601,29 @@ int _read_integer_arrary(char* string, int *array, int size) } return i; } -int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p) + +int read_plugin_table_description(const char* line, struct Maat_table_desc* p) { int i=0,ret=0; - - cJSON* json=NULL, *tmp=NULL; + size_t offset=0, len=0; + cJSON* json=NULL, *tmp=NULL, *array_item=NULL; char* copy_line=NULL, *plug_info=NULL; - char *token=NULL,*sub_token=NULL,*saveptr; - - p->valid_flag_column=-1; + struct plugin_table_desc* plugin_desc=&(p->plugin); copy_line=_maat_strdup(line); - for (token = copy_line, i=0; ; token= NULL, i++) - { - sub_token= strtok_r(token,"\t ", &saveptr); - if (sub_token == NULL) - break; - if(i==3) - { - break; - } - } - if(i<3) + ret=get_column_pos(copy_line, 4, &offset, &len); + if(i<0) { goto error_out; } - plug_info=sub_token; + copy_line[offset+len+1]='\0'; + plug_info=copy_line+offset; if(strlen(plug_info)<4)//For old version compatible. { - ret=sscanf(plug_info, "%d", &(p->valid_flag_column)); + ret=sscanf(plug_info, "%d", &(plugin_desc->valid_flag_column)); if(ret==0||ret==EOF) { - p->valid_flag_column=-1; + plugin_desc->valid_flag_column=-1; } free(copy_line); return 0; @@ -660,20 +633,47 @@ int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p) { goto error_out; } + tmp=cJSON_GetObjectItem(json, "key"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->key_column=tmp->valueint; + } tmp=cJSON_GetObjectItem(json, "valid"); if(tmp!=NULL) { - p->valid_flag_column=tmp->valueint; + assert(tmp->type==cJSON_Number); + plugin_desc->valid_flag_column=tmp->valueint; } tmp=cJSON_GetObjectItem(json, "tag"); if(tmp!=NULL) { - p->rule_tag_column=tmp->valueint; + assert(tmp->type==cJSON_Number); + plugin_desc->rule_tag_column=tmp->valueint; + } + tmp=cJSON_GetObjectItem(json, "estimate_size"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->estimate_size=tmp->valueint; } tmp=cJSON_GetObjectItem(json, "foreign"); if(tmp!=NULL) { - p->n_foreign=_read_integer_arrary(tmp->valuestring, p->foreign_columns, MAX_FOREIGN_CLMN_NUM); + if(tmp->type==cJSON_String) + { + plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM); + } + else if(tmp->type==cJSON_Array) + { + plugin_desc->n_foreign= cJSON_GetArraySize(tmp); + for(i=0;in_foreign; i++) + { + array_item=cJSON_GetArrayItem(tmp, i); + assert(array_item->type==cJSON_Number); + plugin_desc->foreign_columns[i]=array_item->valueint; + } + } } cJSON_Delete(json); @@ -683,15 +683,15 @@ error_out: free(copy_line); return -1; } -int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) +int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) { FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; int i=0,ret=0,table_cnt=0; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; MESA_htable_handle string2int_map=map_create(); - struct _Maat_table_info_t*p=NULL; - struct _Maat_table_info_t*conj_table=NULL; + struct Maat_table_desc*p=NULL; + struct Maat_table_desc*conj_table=NULL; map_register(string2int_map,"expr", TABLE_TYPE_EXPR); map_register(string2int_map,"ip", TABLE_TYPE_IP); @@ -738,9 +738,9 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* { continue; } - p=create_table_info(max_thread_num); + p=table_info_new(max_thread_num); - ret=sscanf(line,"%hu\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) + ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) ,p->table_name[0] ,table_type_str ,not_care); @@ -771,9 +771,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* } break; case TABLE_TYPE_PLUGIN: - p->cb_info=(struct _plugin_table_info*)calloc(sizeof(struct _plugin_table_info),1); - p->cb_info->cache_lines=dynamic_array_create(1024,1024); - ret=read_plugin_table_info(line, p); + ret=read_plugin_table_description(line, p); if(ret<0) { fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i); @@ -786,11 +784,11 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* ret=sscanf(not_care,"%[a-z0-9]",tmp_str); if(ret>0) { - ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->user_region_encoding)); + ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); } if(ret!=1) { - p->user_region_encoding=USER_REGION_ENCODE_NONE; + p->compile.user_region_encoding=USER_REGION_ENCODE_NONE; } default: break; @@ -828,7 +826,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_cnt++; continue; error_jump: - destroy_table_info(p); + table_info_free(p); p=NULL; } fclose(fp); @@ -988,7 +986,8 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id) } void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule) { - const struct _Maat_table_info_t* table=compile_rule->ref_table; + const struct Maat_table_desc* table=compile_rule->ref_table; + const struct compile_table_desc* compile_desc=&(table->compile); struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule; int i=0; compile_rule->compile_id=-1; @@ -999,9 +998,9 @@ void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule) if(db_compile_rule!=NULL) { - for(i=0; iex_data_num; i++) + for(i=0; iex_data_num; i++) { - rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i); + rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, compile_desc->ex_desc+i); compile_rule->ads[i]=NULL; } @@ -1152,7 +1151,7 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) op_expr->rule_type=p_rule->rule_type; return; } -GIE_digest_t* create_digest_rule(unsigned int id,short op,const char* digest, +GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, short cfds_lvl,struct _Maat_group_inner_t* tag) { GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); @@ -1182,13 +1181,99 @@ void destroy_digest_rule(GIE_digest_t*rule) rule=NULL; return; } + + +struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num) +{ + + struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); + table_rt->table_type=table_desc->table_type; + switch(table_desc->table_type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + table_rt->similar.update_q=MESA_lqueue_create(0,0); + break; + case TABLE_TYPE_PLUGIN: + table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); + if(table_desc->plugin.have_exdata) + { + table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, + table_desc->plugin.ex_desc.key2index_func); + pthread_rwlock_init(&(table_rt->plugin.rwlock), NULL); + } + break; + default: + break; + } + + table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); + table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); + table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); + table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); + table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); + return table_rt; +} +void table_runtime_free(struct Maat_table_runtime* p) +{ + long q_cnt=0,data_size=0; + int i=0; + UNUSED int q_ret=0; + + GIE_digest_t* digest_rule=NULL; + if(p==NULL) + { + return; + } + switch(p->table_type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + if(p->similar.gie_handle!=NULL) + { + GIE_destory(p->similar.gie_handle); + } + if(p->similar.update_q!=NULL) + { + q_cnt=MESA_lqueue_get_count(p->similar.update_q); + for(i=0;isimilar.update_q,&digest_rule,&data_size); + assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); + destroy_digest_rule(digest_rule); + } + MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); + } + break; + case TABLE_TYPE_PLUGIN: + dynamic_array_destroy(p->plugin.cache_lines, free); + p->plugin.cache_lines=NULL; + if(p->plugin.key2ex_hash!=NULL) + { + MESA_htable_destroy(p->plugin.key2ex_hash, NULL); + } + break; + default: + break; + } + + alignment_int64_array_free(p->scan_cnt); + alignment_int64_array_free(p->scan_cpu_time); + alignment_int64_array_free(p->input_bytes); + alignment_int64_array_free(p->stream_num); + alignment_int64_array_free(p->hit_cnt); + free(p); + return; +} + struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t *feather) { int scan_thread_num=feather->scan_thread_num; - MESA_lqueue_head tomb=feather->garbage_q; // int rs_scan_type=feather->rule_scan_type; - struct _Maat_table_info_t ** pp_table=feather->p_table_info; - + struct Maat_table_desc ** pp_table_desc=feather->p_table_info; + struct Maat_table_runtime* table_rt=NULL; + const struct expr_table_desc* expr_desc=NULL; int i=0,j=0; unsigned int sub_type=0; UNUSED int ret=0; @@ -1209,7 +1294,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t hargs.data_expire_with_condition = NULL; struct _Maat_scanner_t* scanner=NULL; - scanner=(struct _Maat_scanner_t*)calloc(sizeof(struct _Maat_scanner_t),1); + scanner=ALLOC(struct _Maat_scanner_t, 1); //Function Maat_cmd_append will access compile_hash in user thread. hargs.thread_safe=1; @@ -1232,7 +1317,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t scanner->max_thread_num=scan_thread_num; //optimized for CPU cache_alignment 64 scanner->ref_cnt=alignment_int64_array_alloc(scan_thread_num); - scanner->region_update_q=MESA_lqueue_create(0,0); + scanner->region_update_q=MESA_lqueue_create(0, 0); scanner->region=rulescan_initialize(scan_thread_num); //For best performance test: @@ -1247,41 +1332,34 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t rulescan_set_param(scanner->region,RULESCAN_DETAIL_RESULT,NULL,0); rulescan_set_param(scanner->region,RULESCAN_REGEX_GROUP,NULL,0); } - scanner->tomb_ref=tomb; - scanner->region_rslt_buff=(scan_result_t*)malloc(sizeof(scan_result_t)*MAX_SCANNER_HIT_NUM*scan_thread_num); + scanner->tomb_ref=feather->garbage_q; + scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); for(i=0;itable_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - scanner->gie_aux[i].table_type=pp_table[i]->table_type; - scanner->gie_aux[i].update_q=MESA_lqueue_create(0,0); - break; - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - if(pp_table[i]->quick_expr_switch==1) + table_rt=table_runtime_new(pp_table_desc[i], feather->scan_thread_num); + if(pp_table_desc[i]->table_type==TABLE_TYPE_EXPR||pp_table_desc[i]->table_type==TABLE_TYPE_EXPR_PLUS) + { + expr_desc=&(pp_table_desc[i]->expr); + if(expr_desc->quick_expr_switch==1) + { + for(j=0;jexpr.dst_charset[j]!=CHARSET_NONE;j++) { - for(j=0;jdst_charset[j]!=CHARSET_NONE;j++) + sub_type=make_sub_type(pp_table_desc[i]->table_id, expr_desc->dst_charset[j], expr_desc->do_charset_merge); + ret=rulescan_set_param(scanner->region,RULESCAN_QUICK_SCAN,&sub_type,sizeof(sub_type)); + assert(ret==1); + if(expr_desc->do_charset_merge==1) { - sub_type=make_sub_type(pp_table[i]->table_id, pp_table[i]->dst_charset[j], pp_table[i]->do_charset_merge); - ret=rulescan_set_param(scanner->region,RULESCAN_QUICK_SCAN,&sub_type,sizeof(sub_type)); - assert(ret==1); - if(pp_table[i]->do_charset_merge==1) - { - break; - } + break; } } - break; - default: - break; + } } + scanner->table_rt[i]=table_rt; } return scanner; } @@ -1294,7 +1372,6 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner) int i=0,j=0; UNUSED int q_ret=0; struct op_expr_t* op_expr=NULL; - GIE_digest_t* digest_rule=NULL; if(scanner==NULL) { return; @@ -1306,7 +1383,7 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner) map_destroy(scanner->district_map); scanner->district_map=NULL; assert(scanner->tmp_district_map==NULL); - destroy_bool_matcher((void*)scanner->expr_compiler); + destroy_bool_matcher((void*)scanner->bool_macher_expr_compiler); q_cnt=MESA_lqueue_get_count(scanner->region_update_q); for(i=0;iregion_update_q, lqueue_destroy_cb, NULL); free(scanner->region_rslt_buff); scanner->region_rslt_buff=NULL; - free(scanner->ref_cnt); + alignment_int64_array_free(scanner->ref_cnt); scanner->ref_cnt=NULL; for(i=0;igie_aux[i].gie_handle!=NULL) - { - GIE_destory(scanner->gie_aux[i].gie_handle); - } - if(scanner->gie_aux[i].update_q==NULL) - { - continue; - } - q_cnt=MESA_lqueue_get_count(scanner->gie_aux[i].update_q); - for(j=0;jgie_aux[i].update_q,&digest_rule,&data_size); - assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); - destroy_digest_rule(digest_rule); - } - MESA_lqueue_destroy(scanner->gie_aux[i].update_q, lqueue_destroy_cb, NULL); + table_runtime_free(scanner->table_rt[i]); } free(scanner); return; @@ -1377,6 +1438,20 @@ void destroy_ip_expr(boolean_expr_t*p) free(p); return; } +struct _region_stat_t +{ + int cfg_num; + union + { + int expr_rule_cnt; //expr_type=0,1,3 + int ipv4_rule_cnt; + }; + union + { + int regex_rule_cnt; //expr_type=2 + int ipv6_rule_cnt; + }; +}; void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, int size) { assert(op_expr->table_idp_expr,sizeof(boolean_expr_t)); //make a whole memory chunk - to_update_expr[i].rules=(scan_rule_t*)calloc(sizeof(scan_rule_t),op_expr->p_expr->rnum); - for(j=0;j<(int)op_expr->p_expr->rnum;j++) + to_update_expr[i].rules= ALLOC(scan_rule_t, op_expr->p_expr->rnum); + for(j=0;jp_expr->rnum;j++) { memcpy(&(to_update_expr[i].rules[j]),op_expr->p_rules[j],sizeof(scan_rule_t)); if(to_update_expr[i].rules[j].rule_type==RULETYPE_REG||to_update_expr[i].rules[j].rule_type==RULETYPE_STR) @@ -1456,6 +1533,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, ,to_update_expr[i].rules[j].string_rule.len); } } + count_rs_region(op_expr,region_counter,MAX_TABLE_NUM); destroy_op_expr(op_expr); op_expr=NULL; @@ -1485,16 +1563,32 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, //update scanner's region cnt; for(i=0;iregion_counter[i].cfg_num+=region_counter[i].cfg_num; - maat_scanner->region_counter[i].expr_rule_cnt+=region_counter[i].expr_rule_cnt; - maat_scanner->region_counter[i].regex_rule_cnt+=region_counter[i].regex_rule_cnt; - assert(maat_scanner->region_counter[i].cfg_num>=0); - assert(maat_scanner->region_counter[i].expr_rule_cnt>=0); - assert(maat_scanner->region_counter[i].regex_rule_cnt>=0); + table_rt=maat_scanner->table_rt[i]; + if(table_rt==NULL) + { + continue; + } + switch(table_rt->table_type) + { + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + table_rt->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt; + table_rt->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt; + assert(table_rt->expr.expr_rule_cnt>=0); + assert(table_rt->expr.regex_rule_cnt>=0); + break; + case TABLE_TYPE_IP: + table_rt->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt; + table_rt->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt; + break; + default: + break; + } + assert(table_rt->origin_rule_num>=0); } for(i=0;itable_rt[table_id]; update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); for(i=0;ioperation==GIE_INSERT_OPT) { - maat_scanner->region_counter[table_id].cfg_num++; + table_rt->origin_rule_num++; } else { - maat_scanner->region_counter[table_id].cfg_num--; + table_rt->origin_rule_num--; } destroy_digest_rule(update_array[i]); update_array[i]=NULL; @@ -1789,13 +1882,13 @@ int get_district_id(_Maat_scanner_t *scanner,const char* district_str) } return district_id; } -int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger) { unsigned int i=0,j=0; char* p=NULL,*saveptr=NULL,*region_string=NULL; int region_str_len=0,ret=0,k=0; int expr_id=0,district_id=-1; - + struct expr_table_desc* expr_desc=&(table->expr); scan_rule_t*p_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL; enum MAAT_CHARSET dst_charset=CHARSET_NONE; @@ -1841,7 +1934,6 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule sub_key_array[i]=str_unescape(sub_key_array[i]); } sub_expr_cnt=i; - table->expr_rule_cnt++; break; case EXPR_TYPE_OFFSET: for(i=0,p=db_rule->keywords;;i++,p=NULL) @@ -1875,7 +1967,6 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule sub_key_array[i]=str_unescape(sub_key_array[i]); } sub_expr_cnt=i; - table->expr_rule_cnt++; break; case EXPR_TYPE_REGEX://it's easy,no need to charset convert expr_id=scanner->exprid_generator++; @@ -1914,14 +2005,12 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*)); - table->regex_rule_cnt++; return 0;//yes,we returned. break; case EXPR_TYPE_STRING: sub_expr_cnt=1; sub_key_array[0]=db_rule->keywords; sub_key_array[0]=str_unescape(sub_key_array[0]); - table->expr_rule_cnt++; break; default: break; @@ -1940,7 +2029,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule { for(j=0;jdst_charset[j]; + dst_charset=expr_desc->dst_charset[j]; if(dst_charset==CHARSET_NONE) { break; @@ -1964,10 +2053,10 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } region_str_len=strlen(sub_key_array[k])*8+1; // 1 byte map to 8 bytes maximum, e.g. "ا" or "\u63221;" region_string=(char*)calloc(sizeof(char),region_str_len); - if(table->src_charset!=dst_charset)//need convert + if(expr_desc->src_charset!=dst_charset)//need convert { - ret=universal_charset_convert(scanner,table->src_charset, dst_charset, + ret=universal_charset_convert(scanner,expr_desc->src_charset, dst_charset, sub_key_array[k],strlen(sub_key_array[k]), region_string, ®ion_str_len); if(ret<0) @@ -1975,11 +2064,11 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , "Table %s region cfg %d charset convert from %s to %s failed.",table->table_name ,db_rule->region_id - ,CHARSET_STRING[table->src_charset] + ,CHARSET_STRING[expr_desc->src_charset] ,CHARSET_STRING[dst_charset]); free(region_string); op_expr->convert_failed++; - table->iconv_err_cnt++; + expr_desc->iconv_err_cnt++; break; } if(region_str_len==(int)strlen(sub_key_array[k])&& @@ -1993,7 +2082,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule memcpy(region_string,sub_key_array[k],strlen(sub_key_array[k])); region_str_len=strlen(sub_key_array[k]); } - p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,table->do_charset_merge) + p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge) ,db_rule->match_method ,db_rule->is_case_sensitive ,region_string @@ -2006,7 +2095,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } //if each sub string's convert take no effect and src charset is one of the dst. //if any sub expr convert failed - if((TRUE==table->src_charset_in_dst&&op_expr->no_effect_convert_cnt==sub_expr_cnt)|| + if((TRUE==expr_desc->src_charset_in_dst&&op_expr->no_effect_convert_cnt==sub_expr_cnt)|| op_expr->convert_failed>0) { scanner->dedup_expr_num++; @@ -2042,7 +2131,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule region_string=(char*)calloc(sizeof(char),region_str_len); region_str_len=hex2bin(sub_key_array[0], strlen(sub_key_array[0]),region_string,region_str_len); - p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,table->do_charset_merge) + p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge) ,db_rule->match_method ,db_rule->is_case_sensitive ,region_string @@ -2057,7 +2146,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } return 0; } -int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; scan_rule_t* p_rule=NULL; @@ -2089,7 +2178,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; scan_rule_t* p_rule=NULL; @@ -2120,11 +2209,12 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; GIE_digest_t* digest_rule=NULL; struct _Maat_group_inner_t* u_para=NULL; + struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id]; int expr_id=0,district_id=-1; group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); @@ -2147,11 +2237,11 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db ,db_digest_rule->digest_string ,db_digest_rule->confidence_degree ,group_rule); - MESA_lqueue_join_tail(scanner->gie_aux[table->table_id].update_q, &digest_rule, sizeof(void*)); + MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); scanner->gie_total_q_size++; return 0; } -int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger) +int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger) { int i=0; unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; @@ -2200,7 +2290,7 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id, ,NULL ,0 ,NULL); - MESA_lqueue_join_tail(maat_scanner->gie_aux[table->table_id].update_q,&digest_rule, sizeof(void*)); + MESA_lqueue_join_tail(maat_scanner->table_rt[table->table_id]->similar.update_q,&digest_rule, sizeof(void*)); maat_scanner->gie_total_q_size++; break; default: @@ -2213,9 +2303,9 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id, garbage_bagging(GARBAGE_GROUP_RULE, group_rule, maat_scanner->tomb_ref); } - return 1; + return 0; } -int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; struct _Maat_compile_inner_t*compile_rule=NULL; @@ -2250,7 +2340,7 @@ int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_g return 0; } -void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) +void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_compile_inner_t*compile_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL; @@ -2285,7 +2375,7 @@ void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_ } return; } -int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_compile_inner_t *compile_rule=NULL; struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head); @@ -2307,9 +2397,9 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* pthread_rwlock_wrlock(&(compile_rule->rwlock)); compile_rule->ref_table=table; compile_rule->db_c_rule=db_compile_rule; - for(i=0; iex_data_num; i++) + for(i=0; icompile.ex_data_num; i++) { - compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->ex_desc+i); + compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->compile.ex_desc+i); } compile_rule->is_valid=1; pthread_rwlock_unlock(&(compile_rule->rwlock)); @@ -2317,7 +2407,7 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* return 0; } -int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) +int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_compile_inner_t *compile_rule=NULL; compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); @@ -2340,9 +2430,10 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* } return 1; } -void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) +void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) { struct db_group_rule_t db_group_rule; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; ret=sscanf(table_line,"%d\t%d\t%d",&(db_group_rule.group_id) ,&(db_group_rule.compile_id) @@ -2361,7 +2452,7 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s //leave no trace when compatible_group_update calling if(table->table_type==TABLE_TYPE_GROUP) { - table->cfg_num--; + table_rt->origin_rule_num--; } } else @@ -2380,14 +2471,14 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s //no need to free db_group_rule,it was saved in scanner->compile_hash if(table->table_type==TABLE_TYPE_GROUP) { - table->cfg_num++; + table_rt->origin_rule_num++; } } } return; } -void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger) +void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger) { char virtual_group_line[256]; snprintf(virtual_group_line,sizeof(virtual_group_line), @@ -2395,10 +2486,12 @@ void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int update_group_rule(table, virtual_group_line,scanner,logger); return; } -void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; + const struct expr_table_desc* expr_desc=&(table->expr); + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; switch(table->table_type) { case TABLE_TYPE_EXPR: @@ -2517,13 +2610,13 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st ret=del_region_rule(table ,maat_str_rule->region_id,maat_str_rule->group_id,rule_type ,scanner, logger); - if(ret>0) + if(ret<0) { - table->cfg_num--; + table->udpate_err_cnt++; } else { - table->udpate_err_cnt++; + table_rt->origin_rule_num--; } } else @@ -2538,7 +2631,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st } if(maat_str_rule->expr_type==EXPR_TYPE_STRING - &&table->quick_expr_switch==1 + &&expr_desc->quick_expr_switch==1 &&maat_str_rule->match_method!=MATCH_METHOD_SUB) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , @@ -2557,18 +2650,19 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st } else { - table->cfg_num++; + table_rt->origin_rule_num++; } + } error_out: free(maat_str_rule); maat_str_rule=NULL; } -void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40]; - + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; unsigned short i_src_port,i_sport_mask,i_dst_port,i_dport_mask; int protocol=0,direction=0; int ret=0,rule_type=0; @@ -2677,21 +2771,13 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru ret=del_region_rule(table ,ip_rule->region_id,ip_rule->group_id,rule_type ,scanner, logger); - if(ret>0) + if(ret<0) { - table->cfg_num--; - if(ip_rule->addr_type==4) - { - table->ipv4_rule_cnt--; - } - else - { - table->ipv6_rule_cnt--; - } + table->udpate_err_cnt++; } else { - table->udpate_err_cnt++; + table_rt->origin_rule_num--; } } else @@ -2707,25 +2793,19 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru } else { - table->cfg_num++; - if(ip_rule->addr_type==4) - { - table->ipv4_rule_cnt++; - } - else - { - table->ipv6_rule_cnt++; - } + table_rt->origin_rule_num++; } + } error_out: free(ip_rule); ip_rule=NULL; } -void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { - struct db_intval_rule_t* intval_rule=(struct db_intval_rule_t*)calloc(sizeof(struct db_intval_rule_t),1); + struct db_intval_rule_t* intval_rule=ALLOC(struct db_intval_rule_t, 1); + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) ,&(intval_rule->group_id) @@ -2766,14 +2846,15 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line, ret=del_region_rule(table ,intval_rule->region_id,intval_rule->group_id,RULETYPE_INT ,scanner, logger); - if(ret>0) - { - table->cfg_num--; - } - else + if(ret<0) { table->udpate_err_cnt++; } + else + { + table_rt->origin_rule_num--; + } + } else { @@ -2787,17 +2868,21 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table->cfg_num++; + table_rt->origin_rule_num++; } + } error_out: free(intval_rule); intval_rule=NULL; } -void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) +void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) { - struct db_compile_rule_t *p_compile=(struct db_compile_rule_t*)calloc(sizeof(struct db_compile_rule_t ),1); + struct compile_table_desc* compile_desc=&(table->compile); + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + + struct db_compile_rule_t *p_compile=ALLOC(struct db_compile_rule_t, 1); struct _head_Maat_rule_t* p_m_rule=&(p_compile->m_rule_head); char user_region[MAX_TABLE_LINE_SIZE]={0}; char tag_str[MAX_TABLE_LINE_SIZE]={0}; @@ -2837,7 +2922,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line goto no_save; } } - switch(table->user_region_encoding) + switch(compile_desc->user_region_encoding) { case USER_REGION_ENCODE_ESCAPE: str_unescape(user_region); @@ -2846,16 +2931,13 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line break; } p_m_rule->serv_def_len=strlen(user_region)+1; - p_compile->service_defined=(char*)malloc(p_m_rule->serv_def_len*sizeof(char)); + p_compile->service_defined= ALLOC(char, p_m_rule->serv_def_len); memcpy(p_compile->service_defined,user_region,p_m_rule->serv_def_len); if(p_compile->is_valid==FALSE) { ret=del_compile_rule(table,p_compile,scanner, logger); - if(ret>0) - { - table->cfg_num--; - } + table_rt->origin_rule_num--; goto no_save; } else @@ -2869,11 +2951,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line table->udpate_err_cnt++; goto no_save; } - else - { - //no need to free p_compile,it was saved in scanner->compile_hash - table->cfg_num++; - } + table_rt->origin_rule_num++; } return; @@ -2885,9 +2963,10 @@ no_save: return; } -void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { - struct db_digest_rule_t* digest_rule=(struct db_digest_rule_t*)calloc(sizeof(struct db_digest_rule_t),1); + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; + struct db_digest_rule_t* digest_rule=ALLOC(struct db_digest_rule_t, 1); int ret=0; char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'}; if(table->table_type==TABLE_TYPE_DIGEST) @@ -2945,14 +3024,15 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line, { //digest rule is not build with rulescan, this rule type is useless in count_rs_region funciton. ret=del_region_rule(table,digest_rule->region_id,digest_rule->group_id,0 ,scanner, logger); - if(ret>0) - { - table->cfg_num--; - } - else + if(ret<0) { table->udpate_err_cnt++; } + else + { + table_rt->origin_rule_num--; + } + } else { @@ -2966,8 +3046,9 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table->cfg_num++; + table_rt->origin_rule_num++; } + } error_out: @@ -3086,25 +3167,26 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) q_cnt,bury_cnt); } } -void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line, const struct rule_tag* tags, int n_tags, void* logger) +void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int i=0, ret=1; unsigned int len=strlen(table_line)+1; - struct _plugin_table_info* p_table_cb=table->cb_info; + struct plugin_table_desc* plugin_desc=&(table->plugin); + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; char *p=NULL; char* copy=NULL; - + size_t is_valid_offset=0, valid_len=0; char *token=NULL,*sub_token=NULL,*saveptr; - if(table->rule_tag_column>0&&n_tags>0) + if(plugin_desc->rule_tag_column>0&&n_tags>0) { copy=_maat_strdup(table_line); - for (token = copy, i=0; irule_tag_column ; token= NULL, i++) + for (token = copy, i=0; irule_tag_column ; token= NULL, i++) { sub_token= strtok_r(token," \t", &saveptr); if (sub_token == NULL) break; } - if(i==table->rule_tag_column&&strlen(sub_token)>2) + if(i==plugin_desc->rule_tag_column&&strlen(sub_token)>2) { ret=compare_accept_tag(sub_token, tags, n_tags); if(ret<0) @@ -3127,28 +3209,48 @@ void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line } } - p_table_cb->acc_line_num++; - - if(p_table_cb->cb_plug_cnt>0) + table_rt->plugin.acc_line_num++; + if(plugin_desc->have_exdata || plugin_desc->cb_plug_cnt>0) { - for(i=0;icb_plug_cnt;i++) + + if(plugin_desc->have_exdata) { - p_table_cb->cb_plug[i].update(table->table_id,table_line,p_table_cb->cb_plug[i].u_para); + ret=get_column_pos(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len); + + pthread_rwlock_wrlock(&(table_rt->plugin.rwlock)); + if(atoi(table_line+is_valid_offset)==1) + { + plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger); + } + else + { + plugin_EX_data_free(table, table_line, table_rt->plugin.key2ex_hash, logger); + } + pthread_rwlock_unlock(&(table_rt->plugin.rwlock)); } + if(plugin_desc->cb_plug_cnt>0) + { + for(i=0;icb_plug_cnt;i++) + { + plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para); + } + } + } else { - p=(char*)calloc(len,1); + p=ALLOC(char, len); memcpy(p,table_line,len); - p_table_cb->cache_size+=len; - dynamic_array_write(p_table_cb->cache_lines,p_table_cb->cache_line_num,p); - p_table_cb->cache_line_num++; + table_rt->plugin.cache_size+=len; + dynamic_array_write(table_rt->plugin.cache_lines,table_rt->plugin.cache_line_num,p); + table_rt->plugin.cache_line_num++; } } void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger) { void *tmp1=NULL,*tmp2=NULL; MESA_htable_handle tmp_map=NULL; + struct Maat_table_runtime* table_rt=NULL; int i=0; long q_cnt; GIE_create_para_t para; @@ -3157,10 +3259,10 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ tmp1=create_bool_matcher(scanner->compile_hash, scan_thread_num, logger); - tmp2=scanner->expr_compiler; + tmp2=scanner->bool_macher_expr_compiler; //assume pinter = operation is thread safe - scanner->expr_compiler=tmp1; + scanner->bool_macher_expr_compiler=tmp1; if(tmp2!=NULL) { garbage_bagging(GARBAGE_BOOL_MATCHER, tmp2, garbage_q); @@ -3174,34 +3276,46 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ ,scanner); for(i=0;igie_aux[i].update_q==NULL) + table_rt=scanner->table_rt[i]; + if(table_rt==NULL) { continue; } - q_cnt=MESA_lqueue_get_count(scanner->gie_aux[i].update_q); - if(q_cnt==0) + switch(table_rt->table_type) { - continue; + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); + if(q_cnt==0) + { + continue; + } + if(table_rt->similar.gie_handle==NULL) + { + if(table_rt->table_type==TABLE_TYPE_SIMILARITY) + { + para.ED_reexamine=1; + para.format=GIE_INPUT_FORMAT_PLAIN; + } + else + { + para.ED_reexamine=0; + para.format=GIE_INPUT_FORMAT_SFH; + } + table_rt->similar.gie_handle=GIE_create(¶); + } + digest_batch_update(table_rt->similar.gie_handle, + table_rt->similar.update_q, + logger, + scanner, + i); + break; + case TABLE_TYPE_PLUGIN: + break; + default: + break; } - if(scanner->gie_aux[i].gie_handle==NULL) - { - if(scanner->gie_aux[i].table_type==TABLE_TYPE_SIMILARITY) - { - para.ED_reexamine=1; - para.format=GIE_INPUT_FORMAT_PLAIN; - } - else - { - para.ED_reexamine=0; - para.format=GIE_INPUT_FORMAT_SFH; - } - scanner->gie_aux[i].gie_handle=GIE_create(¶); - } - digest_batch_update(scanner->gie_aux[i].gie_handle - ,scanner->gie_aux[i].update_q - ,logger - ,scanner - ,i); + } scanner->gie_total_q_size=0; if(scanner->tmp_district_map!=NULL) @@ -3216,26 +3330,12 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ return; } -void clear_plugin_table_info(struct _plugin_table_info *cb_info) -{ - int i=0; - void *line=NULL; - for(i=0;icache_line_num;i++) - { - line=dynamic_array_read(cb_info->cache_lines,i); - free(line); - dynamic_array_write(cb_info->cache_lines,i,NULL); - } - cb_info->cache_line_num=0; - cb_info->cache_size=0; - cb_info->acc_line_num=0; - return; -} + void maat_start_cb(long long new_version,int update_type,void*u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct _Maat_table_info_t* p_table=NULL; - struct _plugin_table_info* p_table_cb=NULL; + struct Maat_table_desc* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; int i=0,j=0; feather->new_version=new_version; @@ -3245,21 +3345,6 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, "Full config version %u -> %u update start", feather->maat_version,new_version); - for(i=0;ip_table_info[i]; - if(p_table==NULL) - { - continue; - } - p_table->cfg_num=0; - p_table->expr_rule_cnt=0; - p_table->regex_rule_cnt=0; - if(p_table->table_type==TABLE_TYPE_PLUGIN) - { - clear_plugin_table_info(p_table->cb_info); - } - } } else { @@ -3273,66 +3358,82 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) for(i=0;ip_table_info[i]; - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->cb_info->cb_plug_cnt==0) + plugin_desc=&(p_table->plugin); + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0) { continue; } - p_table_cb=p_table->cb_info; feather->active_plugin_table_num++; - for(j=0;jcb_plug_cnt;j++) + for(j=0;jcb_plug_cnt;j++) { - if(p_table_cb->cb_plug[j].start!=NULL) + if(plugin_desc->cb_plug[j].start!=NULL) { - p_table_cb->cb_plug[j].start(update_type,p_table_cb->cb_plug[j].u_para); + plugin_desc->cb_plug[j].start(update_type,plugin_desc->cb_plug[j].u_para); } } } return; } +long long scanner_rule_num(struct _Maat_scanner_t *scanner) +{ + long long total=0; + struct Maat_table_runtime* table_rt=NULL; + int i=0; + for(i=0;itable_rt[i]; + if(table_rt!=NULL) + { + total+=table_rt->origin_rule_num; + } + } + return total; +} void maat_finish_cb(void* u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct _Maat_table_info_t* p_table=NULL; - struct _plugin_table_info* p_table_cb=NULL; + struct Maat_table_desc* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; long expr_wait_q_cnt=0; - int i=0,j=0,total=0; - for(i=0;ip_table_info[i]; - if(p_table!=NULL) - { - total+=p_table->cfg_num; - } - } + int i=0, j=0; + int call_plugin_table_cnt=0; for(i=0;ip_table_info[i]; - - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->cb_info->cb_plug_cnt==0) + if(p_table==NULL) { continue; } - call_plugin_table_cnt++; - p_table_cb=p_table->cb_info; - if(call_plugin_table_cnt==feather->active_plugin_table_num) + switch(p_table->table_type) { - feather->is_last_plugin_table_updating=1; - } - for(j=0;jcb_plug_cnt;j++) - { - if(p_table_cb->cb_plug[j].finish!=NULL) - { - p_table_cb->cb_plug[j].finish(p_table_cb->cb_plug[j].u_para); - } - } - feather->is_last_plugin_table_updating=0; + case TABLE_TYPE_PLUGIN: + plugin_desc=&(p_table->plugin); + call_plugin_table_cnt++; + if(call_plugin_table_cnt==feather->active_plugin_table_num) + { + feather->is_last_plugin_table_updating=1; + } + for(j=0;jcb_plug_cnt;j++) + { + if(plugin_desc->cb_plug[j].finish!=NULL) + { + plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para); + } + } + feather->is_last_plugin_table_updating=0; + break; + default: + break; + } } + + if(feather->update_tmp_scanner!=NULL) { - feather->update_tmp_scanner->cfg_num=total; + feather->update_tmp_scanner->cfg_num=scanner_rule_num(feather->update_tmp_scanner); do_scanner_update(feather->update_tmp_scanner ,feather->garbage_q ,feather->scan_thread_num @@ -3343,7 +3444,7 @@ void maat_finish_cb(void* u_para) } else if(feather->scanner!=NULL) { - feather->scanner->cfg_num=total; + feather->scanner->cfg_num=scanner_rule_num(feather->scanner); feather->scanner->version=feather->maat_version; expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_total_q_size; @@ -3353,23 +3454,23 @@ void maat_finish_cb(void* u_para) ,feather->garbage_q ,feather->scan_thread_num ,feather->logger); - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Inc config version %u build complete, %d entries in total.", - feather->scanner->version,feather->scanner->cfg_num); + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module + ,"Inc config version %u build complete, %d entries in total." + ,feather->scanner->version,feather->scanner->cfg_num); feather->postpone_q_size=0; } else { - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module, "Postpone %d entries of version %u load to rulescan.", - feather->scanner->cfg_num,feather->scanner->version); + feather->scanner->cfg_num, feather->scanner->version); } } else { MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Version %d have no valid scan rules, plugin callback complete.", - feather->maat_version); + "Version %d have no valid scan rules, plugin callback complete.", + feather->maat_version); } feather->new_version=-1; feather->active_plugin_table_num=0; @@ -3381,7 +3482,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) int ret=-1,i=0; int table_id=-1; _Maat_scanner_t* scanner=NULL; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; if(feather->update_tmp_scanner!=NULL) { scanner=feather->update_tmp_scanner; @@ -3429,7 +3530,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) update_group_rule(feather->p_table_info[table_id], line, scanner,feather->logger); break; case TABLE_TYPE_PLUGIN: - update_plugin_table(feather->p_table_info[table_id], line, feather->accept_tags, feather->n_tags, feather->logger); + update_plugin_table(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); default: break; @@ -3610,7 +3711,7 @@ void *thread_rule_monitor(void *arg) { continue; } - destroy_table_info(feather->p_table_info[i]); + table_info_free(feather->p_table_info[i]); feather->p_table_info[i]=NULL; } alignment_int64_array_free(feather->thread_call_cnt); diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index ed7190b..de8fcc3 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -42,7 +42,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) { int value=0; int i=0,j=0,offset=0; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; char conj_table_name[(MAX_TABLE_NAME_LEN+1)*MAX_CONJUNCTION_TABLE_NUM]={0}; feather->stat_handle=FS_create_handle(); @@ -84,7 +84,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_CMD_LINE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_SPEED,"line_cmd/s"); feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule"); - feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex"); + feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"reg/v6"); feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream"); feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Bps"); if(feather->perf_on==1) @@ -149,20 +149,20 @@ void maat_stat_init(struct _Maat_feather_t* feather) FS_start(feather->stat_handle); return; } -void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num) +void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num) { - alignment_int64_array_add(p_table->scan_cnt,thread_num,1); - alignment_int64_array_add(p_table->input_bytes,thread_num,scan_len); + alignment_int64_array_add(p->scan_cnt,thread_num,1); + alignment_int64_array_add(p->input_bytes,thread_num,scan_len); if(start!=NULL&&end!=NULL) { - alignment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec); + alignment_int64_array_add(p->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec); } return; } void maat_stat_output(struct _Maat_feather_t* feather) { long value=0; - long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0; + long long total_cfg_num=0, table_regex_ipv6_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0; long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0; long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; long long outer_mid_cnt=0,inner_mid_cnt=0; @@ -171,7 +171,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0; int i=0; time_t now; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; + struct Maat_table_runtime* table_rt=NULL; time(&now); active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); @@ -201,86 +202,105 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_input_bytes=0; table_scan_cpu_time=0; table_hit_cnt=0; + table_regex_ipv6_num=0; p_table=feather->p_table_info[i]; if(p_table==NULL) { continue; } + table_rt=feather->scanner->table_rt[i]; switch(p_table->table_type) { case TABLE_TYPE_PLUGIN: - plugin_cache_num+=p_table->cb_info->cache_line_num; - plugin_acc_num+=p_table->cb_info->acc_line_num; + plugin_cache_num+=table_rt->plugin.cache_line_num; + plugin_acc_num+=table_rt->plugin.acc_line_num; break; case TABLE_TYPE_GROUP: - group_rule_num+=p_table->cfg_num; + group_rule_num+=table_rt->origin_rule_num; break; case TABLE_TYPE_COMPILE: - compile_rule_num+=p_table->cfg_num; + compile_rule_num+=table_rt->origin_rule_num; + break; + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + table_regex_ipv6_num=table_rt->expr.regex_rule_cnt; + total_iconv_error=p_table->expr.iconv_err_cnt; + break; + case TABLE_TYPE_IP: + table_regex_ipv6_num=table_rt->ip.ipv6_rule_cnt; break; default: - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_RULE_NUM], - FS_OP_SET, - p_table->cfg_num); - total_cfg_num+=p_table->cfg_num; - - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], - FS_OP_SET, - p_table->regex_rule_cnt); - total_regex_num+= p_table->regex_rule_cnt; - - table_stream_num=alignment_int64_array_sum(p_table->stream_num,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], - FS_OP_SET, - table_stream_num); - total_stream_cnt+= table_stream_num; - - table_scan_cnt=alignment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], - FS_OP_SET, - table_scan_cnt); - total_scan_cnt+=table_scan_cnt; - - table_input_bytes=alignment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], - FS_OP_SET, - table_input_bytes); - total_input_bytes+=table_input_bytes; - if(feather->perf_on==1) - { - table_scan_cpu_time=alignment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num); - table_scan_cpu_time/=1000; - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_CPU_TIME], - FS_OP_SET, - table_scan_cpu_time); - total_cpu_time+=table_scan_cpu_time; - } - - table_hit_cnt=alignment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_HIT_CNT], - FS_OP_SET, - table_hit_cnt); - total_update_error+=p_table->udpate_err_cnt; - total_iconv_error+=p_table->iconv_err_cnt; - //total hit count stat in region_compile break; } - } + if(p_table->table_type==TABLE_TYPE_PLUGIN|| + p_table->table_type==TABLE_TYPE_GROUP|| + p_table->table_type==TABLE_TYPE_COMPILE) + { + continue; + } + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_RULE_NUM], + FS_OP_SET, + table_rt->origin_rule_num); + total_cfg_num+=table_rt->origin_rule_num; + + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], + FS_OP_SET, + table_regex_ipv6_num); + total_regex_num+=table_regex_ipv6_num; + table_stream_num=alignment_int64_array_sum(table_rt->stream_num,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->stream_num,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], + FS_OP_ADD, + table_stream_num); + total_stream_cnt+= table_stream_num; + + + table_scan_cnt=alignment_int64_array_sum(table_rt->scan_cnt,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->scan_cnt,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], + FS_OP_ADD, + table_scan_cnt); + total_scan_cnt+=table_scan_cnt; + + table_input_bytes=alignment_int64_array_sum(table_rt->input_bytes,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->input_bytes,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], + FS_OP_ADD, + table_input_bytes); + total_input_bytes+=table_input_bytes; + if(feather->perf_on==1) + { + table_scan_cpu_time=alignment_int64_array_sum(table_rt->scan_cpu_time,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->scan_cpu_time,feather->scan_thread_num); + table_scan_cpu_time/=1000; + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_CPU_TIME], + FS_OP_ADD, + table_scan_cpu_time); + total_cpu_time+=table_scan_cpu_time; + } + + table_hit_cnt=alignment_int64_array_sum(table_rt->hit_cnt,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->hit_cnt,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_HIT_CNT], + FS_OP_ADD, + table_hit_cnt); + total_update_error+=p_table->udpate_err_cnt; + } FS_operate(feather->stat_handle, feather->total_stat_id, feather->fs_column_id[COLUMN_TABLE_RULE_NUM], @@ -314,7 +334,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_OP_SET, total_cpu_time); } - total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num); + total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt, feather->scan_thread_num); FS_operate(feather->stat_handle, feather->total_stat_id, feather->fs_column_id[COLUMN_TABLE_HIT_CNT], diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 1ceffab..36411fa 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -164,6 +164,30 @@ char* str_unescape(char* s) s[j]='\0'; return s; } +int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len) +{ + const char* seps=" \t"; + char* saveptr=NULL, *subtoken=NULL, *str=NULL; + char* dup_line=_maat_strdup(line); + int i=0, ret=-1; + for (str = dup_line; ; str = NULL) + { + subtoken = strtok_r(str, seps, &saveptr); + if (subtoken == NULL) + break; + if(i==column_seq-1) + { + *offset=subtoken-dup_line; + *len=strlen(subtoken); + ret=0; + break; + } + i++; + } + free(dup_line); + return ret; +} + #define MAX_SYSTEM_CMD_LEN 512 int system_cmd_mkdir(const char* path) diff --git a/src/entry/dynamic_array.cpp b/src/entry/dynamic_array.cpp index 26c414e..46b7159 100644 --- a/src/entry/dynamic_array.cpp +++ b/src/entry/dynamic_array.cpp @@ -2,11 +2,11 @@ #include #include int dynamic_array_VERSION_20141202=0; -struct dynamic_array_t* dynamic_array_create(int size,int step) +struct dynamic_array_t* dynamic_array_create(long long init_size, long long step) { struct dynamic_array_t* d_array=(struct dynamic_array_t*)calloc(sizeof(struct dynamic_array_t),1); - d_array->array=(void**)calloc(sizeof(void*),size); - d_array->size=size; + d_array->array=(void**)calloc(sizeof(void*),init_size); + d_array->size=init_size; d_array->enlarge_step=step; return d_array; } @@ -23,7 +23,7 @@ void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(vo free(d_array->array); free(d_array); } -void* dynamic_array_read(struct dynamic_array_t* d_array,int i) +void* dynamic_array_read(struct dynamic_array_t* d_array, long long i) { if(isize) { @@ -34,7 +34,7 @@ void* dynamic_array_read(struct dynamic_array_t* d_array,int i) return NULL; } } -void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data) +void dynamic_array_write(struct dynamic_array_t* d_array, long long i, void* data) { int new_size=0; if(isize) diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 60b474c..a81994e 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -12,6 +12,7 @@ #include "cJSON.h" #include "hiredis.h" #include "map_str2int.h" +#include "Maat_table_description.h" #include "Maat_rule_internal.h" #include "Maat_utils.h" diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 475eb7d..00b5529 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -3,6 +3,7 @@ #include "Maat_rule.h" #include "Maat_command.h" +#include "Maat_table_description.h" #include #include @@ -22,130 +23,24 @@ extern const char *maat_module; -#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) -#define atomic_inc(x) __sync_add_and_fetch((x),1) -#define atomic_dec(x) __sync_sub_and_fetch((x),1) -#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) -#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) -typedef int atomic_t; -#define ATOMIC_INIT(i) { (i) } -#define atomic_read(x) __sync_add_and_fetch((x),0) -#define atomic_set(x,y) __sync_lock_test_and_set((x),y) -#else -#include -#endif -#define TRUE 1 -#define FALSE 0 + #define MAX_TABLE_NUM 256 -#define MAX_CONJUNCTION_TABLE_NUM 8 -#define MAX_CHARSET_NUM 16 -#define MAX_TABLE_NAME_LEN 256 #define MAX_TABLE_LINE_SIZE (1024*16) #define MAX_EXPR_KEYLEN 1024 #define MAX_DISTRICT_LEN 64 -#define MAX_PLUGIN_PER_TABLE 32 -#define MAX_FOREIGN_CLMN_NUM 8 + #define MAX_SCANNER_HIT_NUM 64 -#define MAX_COMPILE_EX_DATA_NUM 2 #define MAX_GROUP_CACHE 128 #define MAX_FAILED_NUM 128 #define MAX_MAAT_STAT_NUM 64 -#ifndef MAX -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -#endif -#ifndef MIN -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -#endif - -#ifndef offsetof -#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) -#endif - -#ifndef container_of -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) -#endif typedef void* rule_scanner_t; -enum MAAT_TABLE_TYPE -{ - TABLE_TYPE_EXPR=0, - TABLE_TYPE_IP, - TABLE_TYPE_INTERVAL, - TABLE_TYPE_DIGEST, - TABLE_TYPE_EXPR_PLUS, - TABLE_TYPE_SIMILARITY, - TABLE_TYPE_GROUP, - TABLE_TYPE_COMPILE, - TABLE_TYPE_PLUGIN -}; -enum USER_REGION_ENCODE -{ - USER_REGION_ENCODE_NONE=0, - USER_REGION_ENCODE_ESCAPE, - USER_REGION_ENCODE_BASE64 -}; -struct compile_ex_data_idx -{ - Maat_rule_EX_new_func_t *new_func; - Maat_rule_EX_free_func_t* free_func; - Maat_rule_EX_dup_func_t* dup_func; - long argl; - void *argp; - int idx; - int table_id; -}; -struct _Maat_table_info_t -{ - unsigned short table_id; - unsigned short conj_cnt; - unsigned short updating_name; - char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; - enum MAAT_TABLE_TYPE table_type; - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cfg_num; - int cross_cache_size; - int quick_expr_switch; - union - { - int expr_rule_cnt; //expr_type=0,1,3 - int ipv4_rule_cnt; - }; - union - { - int regex_rule_cnt; //expr_type=2 - int ipv6_rule_cnt; - }; - struct _plugin_table_info *cb_info; - int valid_flag_column; //for plugin table - int rule_tag_column; //for plugin table; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table; - int n_foreign; - //for compile table - enum USER_REGION_ENCODE user_region_encoding; - int ex_data_num; - struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; -//for stat>>>>>>>> - unsigned long long udpate_err_cnt; - unsigned long long unmatch_tag_cnt; - unsigned long long iconv_err_cnt; - int stat_line_id; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; -}; struct db_str_rule_t { @@ -252,7 +147,7 @@ struct _Maat_compile_inner_t dynamic_array_t *groups; int is_valid; int compile_id;//equal to db_c_rule->m_rule.config_id - const struct _Maat_table_info_t* ref_table; + const struct Maat_table_desc* ref_table; int group_boundary; int group_cnt; MAAT_RULE_EX_DATA* ads; @@ -263,41 +158,6 @@ struct _compile_result_t int compile_id; universal_bool_expr_t group_set; }; -struct _callback_plugin -{ - Maat_start_callback_t *start; - Maat_update_callback_t *update; - Maat_finish_callback_t *finish; - void* u_para; -}; - -struct _plugin_table_info -{ - int cb_plug_cnt; - struct _callback_plugin cb_plug[MAX_PLUGIN_PER_TABLE]; - dynamic_array_t *cache_lines; - int cache_line_num; - int acc_line_num; - int update_type; - long cache_size; -}; -struct _region_stat_t -{ - int cfg_num; - union - { - int expr_rule_cnt; //expr_type=0,1,3 - int ipv4_rule_cnt; - }; - union - { - int regex_rule_cnt; //expr_type=2 - int ipv6_rule_cnt; - }; -}; - - - struct _INNER_scan_status_t { int cur_hit_cnt; @@ -349,12 +209,49 @@ struct _stream_para_t pthread_mutex_t fuzzy_mutex; unsigned char query_point[8]; }; -struct GIE_aux_t +struct similar_runtime { - enum MAAT_TABLE_TYPE table_type; GIE_handle_t* gie_handle; MESA_lqueue_head update_q; }; +struct plugin_runtime +{ + dynamic_array_t *cache_lines; + long long cache_line_num; + long long acc_line_num; + long long cache_size; + pthread_rwlock_t rwlock; + MESA_htable_handle key2ex_hash; +}; +struct expr_runtime +{ + long long expr_rule_cnt; //expr_type=0,1,3 + long long regex_rule_cnt; //expr_type=2 +}; +struct ip_runtime +{ + long long ipv4_rule_cnt; + long long ipv6_rule_cnt; + +}; +struct Maat_table_runtime +{ + enum MAAT_TABLE_TYPE table_type; + long origin_rule_num; + union + { + struct similar_runtime similar; //for digest and similarity + struct plugin_runtime plugin; + struct expr_runtime expr; + struct ip_runtime ip; + void * other; + }; + mcore_long_t scan_cnt; + mcore_long_t scan_cpu_time; //nano + mcore_long_t input_bytes; + mcore_long_t stream_num; + mcore_long_t hit_cnt; +}; struct rule_tag { char* tag_name; @@ -364,10 +261,15 @@ struct _Maat_scanner_t { long long version; time_t last_update_time; - long long *ref_cnt; //optimized for cache_alignment 64 + mcore_long_t ref_cnt; rule_scanner_t region; long gie_total_q_size; - struct GIE_aux_t gie_aux[MAX_TABLE_NUM]; + + struct Maat_table_runtime* table_rt[MAX_TABLE_NUM]; +/* + struct similar_runtime gie_aux[MAX_TABLE_NUM]; + struct plugin_runtime plugin_aux[MAX_TABLE_NUM]; +*/ MESA_htable_handle region_hash; MESA_htable_handle group_hash; MESA_htable_handle compile_hash; @@ -378,10 +280,10 @@ struct _Maat_scanner_t unsigned int exprid_generator; unsigned int dedup_expr_num; MESA_lqueue_head region_update_q; - void * expr_compiler; + void * bool_macher_expr_compiler; scan_result_t *region_rslt_buff; MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q - struct _region_stat_t region_counter[MAX_TABLE_NUM]; +// struct _region_stat_t region_counter[MAX_TABLE_NUM]; int max_thread_num; iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from] }; @@ -435,7 +337,7 @@ struct _Maat_feather_t int cumulative_update_off; int stat_on; int perf_on; - struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM]; + struct Maat_table_desc *p_table_info[MAX_TABLE_NUM]; MESA_htable_handle map_tablename2id; void* logger; long long maat_version; @@ -534,7 +436,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); -int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); +int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); @@ -546,7 +448,7 @@ int HASH_add_by_id(MESA_htable_handle hash,int id,void*data); int HASH_delete_by_id(MESA_htable_handle hash,int id); void maat_read_full_config(_Maat_feather_t* _feather); void maat_stat_init(struct _Maat_feather_t* feather); -void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num); +void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_output(struct _Maat_feather_t* feather); redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); @@ -559,11 +461,14 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p); void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len); MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); +MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index); +int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); +int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout); void empty_serial_rules(struct serial_rule_t* rule); -int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger); +int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger); long long redis_server_time(redisContext* ctx); redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger); char* md5_file(const char* filename, char* md5string); diff --git a/src/inc_internal/Maat_table_description.h b/src/inc_internal/Maat_table_description.h new file mode 100644 index 0000000..deb7629 --- /dev/null +++ b/src/inc_internal/Maat_table_description.h @@ -0,0 +1,113 @@ +#pragma once +#include "Maat_rule.h" +#define MAX_COMPILE_EX_DATA_NUM 2 +#define MAX_FOREIGN_CLMN_NUM 8 +#define MAX_PLUGIN_PER_TABLE 32 +#define MAX_CHARSET_NUM 16 +#define MAX_CONJUNCTION_TABLE_NUM 8 +#define MAX_TABLE_NAME_LEN 256 + +enum USER_REGION_ENCODE +{ + USER_REGION_ENCODE_NONE=0, + USER_REGION_ENCODE_ESCAPE, + USER_REGION_ENCODE_BASE64 +}; + +enum MAAT_TABLE_TYPE +{ + TABLE_TYPE_EXPR=0, + TABLE_TYPE_IP, + TABLE_TYPE_INTERVAL, + TABLE_TYPE_DIGEST, + TABLE_TYPE_EXPR_PLUS, + TABLE_TYPE_SIMILARITY, + TABLE_TYPE_GROUP, + TABLE_TYPE_COMPILE, + TABLE_TYPE_PLUGIN + +}; + +struct compile_ex_data_idx +{ + Maat_rule_EX_new_func_t *new_func; + Maat_rule_EX_free_func_t* free_func; + Maat_rule_EX_dup_func_t* dup_func; + long argl; + void *argp; + int idx; + int table_id; +}; +struct compile_table_desc +{ + enum USER_REGION_ENCODE user_region_encoding; + int ex_data_num; + struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; +}; + +struct plugin_table_callback_desc +{ + Maat_start_callback_t *start; + Maat_update_callback_t *update; + Maat_finish_callback_t *finish; + void* u_para; +}; +struct plugin_table_ex_data_desc +{ + Maat_plugin_EX_new_func_t* new_func; + Maat_plugin_EX_free_func_t* free_func; + Maat_plugin_EX_dup_func_t* dup_func; + Maat_plugin_EX_key2index_func_t* key2index_func; + long argl; + void *argp; +}; +struct plugin_table_desc +{ + int key_column; + int valid_flag_column; + int rule_tag_column; + int n_foreign; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; + int cb_plug_cnt; + int have_exdata; + long long estimate_size; + struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; + struct plugin_table_ex_data_desc ex_desc; +}; +struct expr_table_desc +{ + enum MAAT_CHARSET src_charset; + enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; + int src_charset_in_dst; + int do_charset_merge; + int cross_cache_size; + int quick_expr_switch; + long long iconv_err_cnt; +}; +struct ip_table_desc +{ + int ipv4_rule_cnt; + int ipv6_rule_cnt; +}; + +struct Maat_table_desc +{ + int table_id; + int conj_cnt; + int updating_name; + char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; + enum MAAT_TABLE_TYPE table_type; + union + { + struct compile_table_desc compile; + struct expr_table_desc expr; + struct ip_table_desc ip; + struct plugin_table_desc plugin; + void* others;//group, interval and digest don't have sperate description info. + }; +//for stat>>>>>>>> + unsigned long long udpate_err_cnt; + unsigned long long unmatch_tag_cnt; + int stat_line_id; +}; + diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 2843db1..e8d7114 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -9,9 +9,20 @@ #define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) #define FREE(p) {free(*p);*p=NULL;} -#define ATOMIC_INC(x) __atomic_fetch_add(x,1,__ATOMIC_RELAXED) -#define ATOMIC_READ(x) __atomic_fetch_add(x,0,__ATOMIC_RELAXED) - +#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) +#define atomic_inc(x) __sync_add_and_fetch((x),1) +#define atomic_dec(x) __sync_sub_and_fetch((x),1) +#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) +#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) + typedef int atomic_t; +#define ATOMIC_INIT(i) { (i) } +#define atomic_read(x) __sync_add_and_fetch((x),0) +#define atomic_set(x,y) __sync_lock_test_and_set((x),y) +#else +#include +#endif +#define TRUE 1 +#define FALSE 0 #ifndef MAX #define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -21,6 +32,16 @@ #define MIN(a, b) (((a) < (b)) ? (a) : (b)) #endif +#ifndef offsetof +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif + +#ifndef container_of +#define container_of(ptr, type, member) ({ \ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) );}) +#endif + #define UNUSED __attribute__((unused)) const char* module_name_str(const char*name); #define maat_module (module_name_str("MAAT_Frame")) @@ -47,4 +68,6 @@ int system_cmd_rm(const char* src_file); int system_cmd_mv(const char* src_file,const char*dst_file); int system_cmd_cp(const char* src_file,const char*dst_file); char* md5_file(const char* filename, char* md5string); +int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); + diff --git a/src/inc_internal/alignment_int64.h b/src/inc_internal/alignment_int64.h index a342205..a7a7d2a 100644 --- a/src/inc_internal/alignment_int64.h +++ b/src/inc_internal/alignment_int64.h @@ -43,6 +43,11 @@ inline long long alignment_int64_array_cnt(mcore_long_t array,int size) } return cnt; } +inline void alignment_int64_array_reset(mcore_long_t array,int size) +{ + memset(array, 0, CPU_CACHE_ALIGMENT*size); + return; +} inline void alignment_int64_array_free(mcore_long_t array) { free(array); diff --git a/src/inc_internal/dynamic_array.h b/src/inc_internal/dynamic_array.h index 7583c92..11b3643 100644 --- a/src/inc_internal/dynamic_array.h +++ b/src/inc_internal/dynamic_array.h @@ -3,11 +3,11 @@ struct dynamic_array_t { void ** array; - int size; - int enlarge_step; + long long size; + long long enlarge_step; }; -struct dynamic_array_t* dynamic_array_create(int size,int step); +struct dynamic_array_t* dynamic_array_create(long long init_size, long long step); void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(void *)); -void* dynamic_array_read(struct dynamic_array_t* d_array,int i); -void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data); +void* dynamic_array_read(struct dynamic_array_t* d_array,long long i); +void dynamic_array_write(struct dynamic_array_t* d_array, long long i,void* data); #endif //_DYNAMIC_ARRAY_H_INCLUDE_ diff --git a/test/maat_demo.cpp b/test/maat_demo.cpp index b536059..c4ffd4f 100644 --- a/test/maat_demo.cpp +++ b/test/maat_demo.cpp @@ -675,35 +675,36 @@ int test_table_conjunction(Maat_feather_t feather,const char* table_name,const c } return 0; } -#define TEST_CMD_LINE_NUM 4 +#define TEST_CMD_LINE_NUM 200*1000 void test_set_cmd_line(Maat_feather_t feather) { - const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM]; - struct Maat_line_t line_rule[TEST_CMD_LINE_NUM]; - char table_line[TEST_CMD_LINE_NUM][128]; + struct Maat_line_t **p_line=(struct Maat_line_t **)calloc(sizeof(struct Maat_line_t *), TEST_CMD_LINE_NUM); + struct Maat_line_t *line_rule=(struct Maat_line_t *)calloc(sizeof(struct Maat_line_t), TEST_CMD_LINE_NUM); int i=0; + const char* line="1\t192.168.0.1\t4444444444\t1"; memset(&line_rule,0,sizeof(line_rule)); for(i=0;ilock)); @@ -686,9 +699,9 @@ void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long -TEST(EX_DATA_INDEX, all) +TEST(EX_DATA_INDEX, MaatRuleEXData) { -#define EX_data_index +#define rule_EX_data_index int ret=0; int table_id=0, ex_data_counter=0; @@ -701,7 +714,7 @@ TEST(EX_DATA_INDEX, all) ASSERT_GT(table_id, 0); int ex_param_idx=Maat_rule_get_ex_new_index(g_feather, "COMPILE_ALIAS", - ex_param_new, ex_param_free, ex_param_dup, + compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, 0, &ex_data_counter); ASSERT_TRUE(ex_param_idx>=0); EXPECT_EQ(ex_data_counter, 1); @@ -716,7 +729,7 @@ TEST(EX_DATA_INDEX, all) struct rule_ex_param* param=(struct rule_ex_param*)ex_data; EXPECT_EQ(param->id, 7799); EXPECT_EQ(strcmp(param->name, expect_name),0); - ex_param_free(0, NULL, NULL, &ex_data, 0, NULL); + compile_ex_param_free(0, NULL, NULL, &ex_data, 0, NULL); Maat_clean_status(&mid); return; } @@ -1033,6 +1046,95 @@ int del_command(Maat_feather_t feather,int config_id) Maat_free_cmd(cmd); return 0; } +TEST_F(MaatCmdTest, SetIP) +{ + struct Maat_cmd_t* cmd=NULL; + struct Maat_rule_t rule; + int config_id=0,timeout=4; + long long version_before=0; + const char* region_table="IP_CONFIG"; + struct Maat_region_t region; + int group_num=1,ret=0; + memset(&rule,0,sizeof(rule)); + Maat_feather_t feather=MaatCmdTest::_shared_feather; + + //MUST acquire by Maat_cmd_incrby to guarantee a unique compile ID. + config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1); + rule.config_id=config_id; + + strcpy(rule.service_defined,"maat_command"); + //MUST acqire by function, because Maat_cmd_t has some hidden members. + cmd=Maat_create_cmd(&rule, group_num); + cmd->expire_after=timeout; + cmd->label_id=0; //no lable + memset(®ion,0,sizeof(region)); + region.region_type=REGION_IP; + region.table_name=region_table; + region.ip_rule.addr_type=ADDR_TYPE_IPv4; + region.ip_rule.direction=ADDR_DIR_DOUBLE; + region.ip_rule.src_ip="172.0.0.1"; + region.ip_rule.mask_src_ip="255.255.255.255"; + region.ip_rule.src_port=53331; + region.ip_rule.mask_src_port=0;//means any port should hit. + + region.ip_rule.dst_ip="172.0.0.2"; + region.ip_rule.mask_dst_ip="255.255.255.255"; + region.ip_rule.dst_port=80; + region.ip_rule.mask_dst_port=65535; + region.ip_rule.protocol=0;//means any protocol should hit. + Maat_add_region2cmd(cmd, 0, ®ion); + + ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before)); + EXPECT_EQ(ret, 0); + ret=Maat_cmd(feather, cmd, MAAT_OP_ADD); + EXPECT_GE(ret, 0); + Maat_free_cmd(cmd); + cmd=NULL; + + wait_for_cmd_effective(feather, version_before); + struct ipaddr ipv4_addr; + struct stream_tuple4_v4 v4_addr; + ipv4_addr.addrtype=ADDR_TYPE_IPV4; + inet_pton(AF_INET,region.ip_rule.src_ip,&(v4_addr.saddr)); + v4_addr.source=htons(region.ip_rule.src_port+1);//Not use the exactly port for testing port mask. + inet_pton(AF_INET,region.ip_rule.dst_ip,&(v4_addr.daddr)); + v4_addr.dest=htons(region.ip_rule.dst_port); + ipv4_addr.v4=&v4_addr; + + int table_id=0; + struct Maat_rule_t result; + memset(&result, 0, sizeof(result)); + scan_status_t mid=NULL; + table_id=Maat_table_register(feather,region_table); + ASSERT_GE(table_id, 0); + + ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result.config_id, config_id); + + Maat_clean_status(&mid); + + ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before)); + + //reset timeout. + cmd=Maat_create_cmd(&rule, 0); + cmd->expire_after=10; + ret=Maat_cmd(feather, cmd, MAAT_OP_RENEW_TIMEOUT); + EXPECT_EQ(ret ,1); + +// wait_for_cmd_effective(feather, version_before); + + Maat_free_cmd(cmd); + cmd=NULL; + sleep(timeout+1); + ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0); + EXPECT_EQ(ret, 1); + + Maat_clean_status(&mid); + + return; + +} TEST_F(MaatCmdTest, SetExpr) { @@ -1107,6 +1209,8 @@ TEST_F(MaatCmdTest, SetExpr) &mid, 0); EXPECT_EQ(ret, 0); } + + TEST_F(MaatCmdTest, SetLines) { const int TEST_CMD_LINE_NUM=4; @@ -1142,95 +1246,96 @@ TEST_F(MaatCmdTest, SetLines) return; } -TEST_F(MaatCmdTest, SetIP) +//Following tests must be coded/tested at last, for they stalled the maat update thread and interrupt other tests. +TEST_F(MaatCmdTest, SetExprOneMillion) { + const int CMD_EXPR_NUM=1000*1000; + const char* table_name="HTTP_URL"; + + const char* keywords1="Hiredis"; + const char* keywords2="C Client"; + char escape_buff1[256],escape_buff2[256]; + char keywords[256]; + + int label_id=5210, config_id=0,ret=0, output_id_cnt=0; + Maat_feather_t feather=MaatCmdTest::_shared_feather; + long long version_before=0; + ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before)); + + Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1); + Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2); + snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2); + + config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", CMD_EXPR_NUM); + int i=0; + for(i=0; i=0); + wait_for_cmd_effective(feather, version_before); struct Maat_cmd_t* cmd=NULL; struct Maat_rule_t rule; - int config_id=0,timeout=4; - long long version_before=0,version_after=0; - const char* region_table="IP_CONFIG"; - struct Maat_region_t region; - int group_num=1,ret=0; memset(&rule,0,sizeof(rule)); + int *output_ids=(int*)malloc(sizeof(int)*CMD_EXPR_NUM); + output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, CMD_EXPR_NUM); + EXPECT_EQ(output_id_cnt, CMD_EXPR_NUM); + for(i=0; i