From 895344d7d2f616b8c18dfc34138abe2c02692756 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 4 Dec 2018 13:38:55 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E9=87=8D=E6=9E=84table=5Finfo=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E7=BB=93=E6=9E=84=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_rule.h | 15 ++++ src/entry/Maat_api.cpp | 13 +++- src/entry/Maat_rule.cpp | 6 +- src/inc_internal/Maat_rule_internal.h | 102 +++++++++++++++++++++----- 4 files changed, 112 insertions(+), 24 deletions(-) diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index bea7e76..2e956a5 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -257,6 +257,21 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table //returned data is duplicated by dup_func of Maat_rule_get_ex_new_index, caller is responsible to free the data. MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx); +//Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table. +typedef void* MAAT_PLUGIN_EX_DATA; +typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp); +typedef int Maat_plugin_EX_key2index_func_t(const char* key); + +int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp); +//Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to free the data. +MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key, int thread_num); enum MAAT_RULE_OPT { diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index b353063..e42c2e6 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -113,7 +113,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int int ret=0,i=0; int r_in_c_cnt=0; int shortcut_avilable_cnt=0; - void* bool_matcher=feather->scanner->expr_compiler; + void* bool_matcher=feather->scanner->bool_macher_expr_compiler; struct _Maat_group_inner_t* group_rule=NULL; struct _Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; struct _Maat_compile_inner_t* _mi_rule=NULL; @@ -1046,6 +1046,17 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa pthread_rwlock_unlock(&(compile_inner->rwlock)); return ad; } +int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, + Maat_plugin_EX_new_func_t* new_func, + Maat_plugin_EX_free_func_t* free_func, + Maat_plugin_EX_dup_func_t* dup_func, + Maat_plugin_EX_key2index_func_t* key2index_func, + long argl, void *argp) + +{ + +} + int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 26e1171..f06d60a 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -1306,7 +1306,7 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner) map_destroy(scanner->district_map); scanner->district_map=NULL; assert(scanner->tmp_district_map==NULL); - destroy_bool_matcher((void*)scanner->expr_compiler); + destroy_bool_matcher((void*)scanner->bool_macher_expr_compiler); q_cnt=MESA_lqueue_get_count(scanner->region_update_q); for(i=0;icompile_hash, scan_thread_num, logger); - tmp2=scanner->expr_compiler; + tmp2=scanner->bool_macher_expr_compiler; //assume pinter = operation is thread safe - scanner->expr_compiler=tmp1; + scanner->bool_macher_expr_compiler=tmp1; if(tmp2!=NULL) { garbage_bagging(GARBAGE_BOOL_MATCHER, tmp2, garbage_q); diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 475eb7d..42cdb28 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -101,6 +101,69 @@ struct compile_ex_data_idx int idx; int table_id; }; +struct compile_table_descr +{ + enum USER_REGION_ENCODE user_region_encoding; + int ex_data_num; + struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; +}; + +struct plugin_table_callback_desc +{ + Maat_start_callback_t *start; + Maat_update_callback_t *update; + Maat_finish_callback_t *finish; + void* u_para; +}; +struct plugin_table_ex_data_desc +{ + Maat_plugin_EX_new_func_t* new_func; + Maat_plugin_EX_free_func_t* free_func; + Maat_plugin_EX_dup_func_t* dup_func; + Maat_plugin_EX_key2index_func_t* key2index_func; + long argl; + void *argp; +}; +struct _plugin_table_info +{ + int cb_plug_cnt; + struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; + dynamic_array_t *cache_lines; + int cache_line_num; + int acc_line_num; + int update_type; + long cache_size; +}; + +struct plugin_table_descr +{ + int key_column; + int valid_flag_column; + int rule_tag_column; + int n_foreign; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; + int cb_plug_cnt; + struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; + struct plugin_table_ex_data_desc ex_desc; + int acc_line_num; +}; +struct expr_table_descr +{ + enum MAAT_CHARSET src_charset; + enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; + int src_charset_in_dst; + int do_charset_merge; + int cross_cache_size; + int quick_expr_switch; + int expr_rule_cnt; //expr_type=0,1,3 + int regex_rule_cnt; //expr_type=2 +}; +struct ip_table_descr +{ + int ipv4_rule_cnt; + int ipv6_rule_cnt; +}; + struct _Maat_table_info_t { unsigned short table_id; @@ -108,6 +171,14 @@ struct _Maat_table_info_t unsigned short updating_name; char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; enum MAAT_TABLE_TYPE table_type; + union + { + struct compile_table_descr compile; + struct expr_table_descr expr; + struct ip_table_descr ip; + struct plugin_table_descr plugin; + void* others;//group, interval and digest don't have sperate description info. + }; enum MAAT_CHARSET src_charset; enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; int src_charset_in_dst; @@ -263,24 +334,7 @@ struct _compile_result_t int compile_id; universal_bool_expr_t group_set; }; -struct _callback_plugin -{ - Maat_start_callback_t *start; - Maat_update_callback_t *update; - Maat_finish_callback_t *finish; - void* u_para; -}; -struct _plugin_table_info -{ - int cb_plug_cnt; - struct _callback_plugin cb_plug[MAX_PLUGIN_PER_TABLE]; - dynamic_array_t *cache_lines; - int cache_line_num; - int acc_line_num; - int update_type; - long cache_size; -}; struct _region_stat_t { int cfg_num; @@ -349,12 +403,19 @@ struct _stream_para_t pthread_mutex_t fuzzy_mutex; unsigned char query_point[8]; }; -struct GIE_aux_t +struct digest_engine { enum MAAT_TABLE_TYPE table_type; GIE_handle_t* gie_handle; MESA_lqueue_head update_q; }; +struct plugin_engine +{ + dynamic_array_t *cache_lines; + int cache_line_num; + long cache_size; +}; + struct rule_tag { char* tag_name; @@ -367,7 +428,8 @@ struct _Maat_scanner_t long long *ref_cnt; //optimized for cache_alignment 64 rule_scanner_t region; long gie_total_q_size; - struct GIE_aux_t gie_aux[MAX_TABLE_NUM]; + struct digest_engine gie_aux[MAX_TABLE_NUM]; + struct plugin_engine plugin_aux[MAX_TABLE_NUM]; MESA_htable_handle region_hash; MESA_htable_handle group_hash; MESA_htable_handle compile_hash; @@ -378,7 +440,7 @@ struct _Maat_scanner_t unsigned int exprid_generator; unsigned int dedup_expr_num; MESA_lqueue_head region_update_q; - void * expr_compiler; + void * bool_macher_expr_compiler; scan_result_t *region_rslt_buff; MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q struct _region_stat_t region_counter[MAX_TABLE_NUM]; From 6971f4cb56ad013736bd4c9c975459da0364e732 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 4 Dec 2018 20:12:56 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E5=B0=86=E8=A7=84=E5=88=99=E6=95=B0?= =?UTF-8?q?=E9=87=8F=E5=92=8C=E6=89=AB=E6=8F=8F=E7=8A=B6=E6=80=81=E7=9A=84?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E5=BD=92=E9=9B=86=E5=88=B0table=20runtime?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E4=BD=93=E4=B8=AD=EF=BC=8C=E5=B1=9E=E4=BA=8E?= =?UTF-8?q?scanner=E7=9A=84=E4=B8=80=E9=83=A8=E5=88=86=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/gram_index_engine.h | 22 +- src/entry/Maat_api.cpp | 203 ++++---- src/entry/Maat_command.cpp | 35 +- src/entry/Maat_rule.cpp | 657 +++++++++++++++----------- src/entry/Maat_stat.cpp | 158 ++++--- src/inc_internal/Maat_rule_internal.h | 101 ++-- src/inc_internal/alignment_int64.h | 5 + test/test_maatframe.cpp | 2 +- 8 files changed, 660 insertions(+), 523 deletions(-) diff --git a/inc/gram_index_engine.h b/inc/gram_index_engine.h index a69e924..452dc4d 100644 --- a/inc/gram_index_engine.h +++ b/inc/gram_index_engine.h @@ -4,12 +4,16 @@ #ifdef __cplusplus extern "C" { #endif - -#define GIE_INSERT_OPT 0 -#define GIE_DELETE_OPT 1 -#define GIE_INPUT_FORMAT_SFH 1 -#define GIE_INPUT_FORMAT_PLAIN 0 - +enum GIE_operation +{ + GIE_INSERT_OPT, + GIE_DELETE_OPT +}; +enum GIE_INPUT_FORMAT +{ + GIE_INPUT_FORMAT_PLAIN, + GIE_INPUT_FORMAT_SFH +}; typedef struct { @@ -21,7 +25,7 @@ typedef struct { unsigned int id; unsigned int sfh_length;//size of fuzzy_hash - short operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed; + enum GIE_operation operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed; short cfds_lvl; char * sfh; void * tag; @@ -41,9 +45,9 @@ typedef struct unsigned int gram_value; //unsigned int htable_num; unsigned int position_accuracy; - short format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string + enum GIE_INPUT_FORMAT format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string //else id format==PALIN, means the input string is common string - short ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result + int ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result }GIE_create_para_t; diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index e42c2e6..dc24da0 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -919,6 +919,8 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; int idx=0,i=0; struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; + struct plugin_table_desc *plugin_desc=&(p_table->plugin); + struct plugin_runtime* plugin_aux=NULL; const char* lines=NULL; if(p_table==NULL) { @@ -928,28 +930,29 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, { return -1; } + plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin); //plugin table register blocks background update. pthread_mutex_lock(&(_feather->backgroud_update_mutex)); - idx=p_table->cb_info->cb_plug_cnt; + idx=plugin_desc->cb_plug_cnt; if(idx==MAX_PLUGIN_PER_TABLE) { pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); return -1; } - p_table->cb_info->cb_plug_cnt++; - p_table->cb_info->cb_plug[idx].start=start; - p_table->cb_info->cb_plug[idx].update=update; - p_table->cb_info->cb_plug[idx].finish=finish; - p_table->cb_info->cb_plug[idx].u_para=u_para; - if(p_table->cb_info->cache_line_num>0) + plugin_desc->cb_plug_cnt++; + plugin_desc->cb_plug[idx].start=start; + plugin_desc->cb_plug[idx].update=update; + plugin_desc->cb_plug[idx].finish=finish; + plugin_desc->cb_plug[idx].u_para=u_para; + if(plugin_aux->cache_line_num>0) { if(start!=NULL) { start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); } - for(i=0;icb_info->cache_line_num;i++) + for(i=0;icache_line_num;i++) { - lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i); + lines=(const char*)dynamic_array_read(plugin_aux->cache_lines,i); if(lines==NULL) { break; @@ -1001,26 +1004,26 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table { return -1; } - + struct compile_table_desc* compile_desc=&(p_table->compile); pthread_mutex_lock(&(_feather->backgroud_update_mutex)); - if(p_table->ex_data_num==MAX_COMPILE_EX_DATA_NUM) + if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM) { ret=-1; goto failed; } - idx=p_table->ex_data_num; - p_table->ex_desc[idx].idx=idx; - p_table->ex_desc[idx].table_id=table_id; - p_table->ex_desc[idx].argl=argl; - p_table->ex_desc[idx].argp=argp; - p_table->ex_desc[idx].new_func=new_func; - p_table->ex_desc[idx].free_func=free_func; - p_table->ex_desc[idx].dup_func=dup_func; + idx=compile_desc->ex_data_num; + compile_desc->ex_desc[idx].idx=idx; + compile_desc->ex_desc[idx].table_id=table_id; + compile_desc->ex_desc[idx].argl=argl; + compile_desc->ex_desc[idx].argp=argp; + compile_desc->ex_desc[idx].new_func=new_func; + compile_desc->ex_desc[idx].free_func=free_func; + compile_desc->ex_desc[idx].dup_func=dup_func; - p_table->ex_data_num++; + compile_desc->ex_data_num++; if(_feather->scanner!=NULL) { - MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, p_table->ex_desc+idx); + MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_desc->ex_desc+idx); } failed: pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); @@ -1031,7 +1034,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_compile_inner_t *compile_inner=NULL; - const struct compile_ex_data_idx* ex_desc=NULL; + const struct compile_table_desc* compile_desc=NULL; + const struct compile_ex_data_idx* ex_desc=NULL; MAAT_RULE_EX_DATA ad=NULL; compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); @@ -1040,8 +1044,9 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa return NULL; } pthread_rwlock_rdlock(&(compile_inner->rwlock)); - assert(idxref_table->ex_data_num); - ex_desc=compile_inner->ref_table->ex_desc+idx; + compile_desc=&(compile_inner->ref_table->compile); + assert(idxex_data_num); + ex_desc=compile_desc->ex_desc+idx; ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp); pthread_rwlock_unlock(&(compile_inner->rwlock)); return ad; @@ -1054,7 +1059,7 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, long argl, void *argp) { - + return 0; } int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id @@ -1070,6 +1075,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num];//dynamic array struct _Maat_table_info_t *p_table=NULL; + struct expr_table_desc* expr_desc=NULL; struct timespec start,end; _Maat_scanner_t* my_scanner=NULL; if(data==NULL||data_len<=0) @@ -1087,17 +1093,13 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id _feather->scan_err_cnt++; return -1; } - - if(p_table->cfg_num==0) - { - return 0; - } + expr_desc=&(p_table->expr); if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) { _feather->scan_err_cnt++; return -1; } - if(p_table->do_charset_merge==1) + if(expr_desc->do_charset_merge==1) { sub_type=make_sub_type(table_id,CHARSET_NONE,0); } @@ -1115,9 +1117,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { return 0; } - struct _region_stat_t * region_stat=NULL; - region_stat=&(my_scanner->region_counter[p_table->table_id]); - if(region_stat->cfg_num==0) + struct table_runtime* table_aux=my_scanner->table_rt[table_id]; + if(table_aux->origin_rule_num==0) { return 0; } @@ -1125,7 +1126,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; INC_SCANNER_REF(my_scanner, thread_num); - if(region_stat->expr_rule_cnt>0) + if(table_aux->expr.expr_rule_cnt>0) { scan_data.rule_type=RULETYPE_STR; scan_data.sub_type=sub_type; @@ -1135,7 +1136,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id hit_region_cnt+=region_ret; } } - if(region_stat->regex_rule_cnt>0) + if(table_aux->expr.regex_rule_cnt>0) { scan_data.rule_type=RULETYPE_REG; scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); @@ -1152,7 +1153,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(hit_region_cnt>0) { - alignment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); _mid=grab_mid(mid,_feather,thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1173,17 +1174,16 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id hit_detail,detail_num); } } - DEC_SCANNER_REF(my_scanner, thread_num); if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(p_table,data_len,&start, &end,thread_num); + maat_stat_table(table_aux, data_len, &start, &end, thread_num); } else { - maat_stat_table(p_table,data_len,NULL, NULL,thread_num); + maat_stat_table(table_aux, data_len, NULL, NULL, thread_num); } - + DEC_SCANNER_REF(my_scanner, thread_num); if(compile_ret==0&&hit_region_cnt>0) { return -2; @@ -1229,18 +1229,13 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id _feather->scan_err_cnt++; return -1; } - if(p_table->cfg_num==0) - { - return 0; - } my_scanner=_feather->scanner; if(my_scanner==NULL) { return 0; } - struct _region_stat_t * region_stat=NULL; - region_stat=&(my_scanner->region_counter[p_table->table_id]); - if(region_stat->cfg_num==0) + struct table_runtime* table_aux=my_scanner->table_rt[table_id]; + if(table_aux->origin_rule_num==0) { return 0; } @@ -1258,7 +1253,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1273,16 +1268,17 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } } - DEC_SCANNER_REF(my_scanner,thread_num); if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(p_table,0,&start, &end,thread_num); + maat_stat_table(table_aux, 0, &start, &end, thread_num); } else { - maat_stat_table(p_table,0,NULL, NULL,thread_num); + maat_stat_table(table_aux, 0, NULL, NULL, thread_num); } + DEC_SCANNER_REF(my_scanner,thread_num); + if(compile_ret==0&®ion_ret>0) { return -2; @@ -1316,26 +1312,22 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id _feather->scan_err_cnt++; return -1; } - if(p_table->cfg_num==0) - { - return 0; - } my_scanner=_feather->scanner; if(my_scanner==NULL) { return 0; } - struct _region_stat_t * region_stat=NULL; - region_stat=&(my_scanner->region_counter[p_table->table_id]); - if(region_stat->cfg_num==0) + struct table_runtime* table_aux=my_scanner->table_rt[table_id]; + if(table_aux->origin_rule_num==0) { return 0; } - if(region_stat->ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4) + + if(table_aux->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4) { return 0; } - if(region_stat->ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) + if(table_aux->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) { return 0; } @@ -1380,7 +1372,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, @@ -1399,11 +1391,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(p_table,0,&start, &end,thread_num); + maat_stat_table(table_aux, 0, &start, &end, thread_num); } else { - maat_stat_table(p_table,0,NULL, NULL,thread_num); + maat_stat_table(table_aux, 0, NULL, NULL, thread_num); } if(compile_ret==0&®ion_ret>0) { @@ -1436,12 +1428,14 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, _feather->scan_err_cnt++; return NULL; } - if(p_table->quick_expr_switch==1) + + struct expr_table_desc* expr_desc=&(p_table->expr); + if(expr_desc->quick_expr_switch==1) { _feather->scan_err_cnt++; return NULL; } - struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); + struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1); scanner=_feather->scanner; sp->feather=_feather; sp->version=_feather->maat_version; @@ -1451,29 +1445,32 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { return sp; } - struct _region_stat_t * region_stat=NULL; - region_stat=&(scanner->region_counter[p_table->table_id]); + struct table_runtime* table_aux=scanner->table_rt[table_id]; + if(table_aux->origin_rule_num==0) + { + return 0; + } INC_SCANNER_REF(scanner, thread_num); sp->table_id=table_id; sp->thread_num=thread_num; - sp->max_cross_size=p_table->cross_cache_size; + sp->max_cross_size=expr_desc->cross_cache_size; sp->caching_size=0; sp->scan_buff=NULL; sp->last_cache=NULL; - if(p_table->do_charset_merge==1) + if(expr_desc->do_charset_merge==1) { sp->do_merge=1; } - if(region_stat->expr_rule_cnt>0) + if(table_aux->expr.expr_rule_cnt>0) { sp->do_expr=1; } - if(region_stat->regex_rule_cnt>0) + if(table_aux->expr.regex_rule_cnt>0) { sp->do_regex=1; } - alignment_int64_array_add(p_table->stream_num,thread_num,1); + alignment_int64_array_add(table_aux->stream_num,thread_num,1); sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); return sp; } @@ -1493,6 +1490,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para _compile_result_t compile_result[rule_num];//dynamic array scan_data_t region_scan_data; _Maat_table_info_t* p_table=NULL; + struct table_runtime* table_aux=scanner->table_rt[sp->table_id]; struct timespec start,end; if(data==NULL||data_len<=0) { @@ -1539,7 +1537,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para free(sp->scan_buff); sp->scan_buff=NULL; } - sp->scan_buff=(char*)malloc(sp->caching_size+data_len); + sp->scan_buff=ALLOC(char, sp->caching_size+data_len); memcpy(sp->scan_buff,sp->last_cache,sp->caching_size); memcpy(sp->scan_buff+sp->caching_size,data,data_len); region_scan_data.text_data.text=sp->scan_buff; @@ -1554,7 +1552,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->last_cache==NULL&&sp->max_cross_size>0) { assert(sp->caching_size==0); - sp->last_cache=(char*)malloc(sizeof(char)*sp->max_cross_size); + sp->last_cache=ALLOC(char, sp->max_cross_size); } if(sp->max_cross_size>0) { @@ -1600,7 +1598,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } if(hit_region_cnt>0) { - alignment_int64_array_add(p_table->hit_cnt, sp->thread_num,1); + alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num,1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1639,11 +1637,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num); + maat_stat_table(table_aux,data_len,&start, &end,sp->thread_num); } else { - maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num); + maat_stat_table(table_aux,data_len,NULL, NULL,sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -1668,8 +1666,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; - alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1); + struct table_runtime* table_aux=scanner->table_rt[sp->table_id]; + alignment_int64_array_add(table_aux->stream_num, sp->thread_num, -1); if(sp->rs_stream_para!=NULL) { if(scanner!=NULL&&sp->version>=sp->feather->last_full_version) @@ -1712,8 +1710,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, { _feather->scan_err_cnt++; return NULL; - } - struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); + } + struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1); scanner=_feather->scanner; sp->feather=_feather; sp->version=_feather->maat_version; @@ -1721,7 +1719,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, if(scanner==NULL) { return sp; - } + } + struct table_runtime* table_aux=scanner->table_rt[table_id]; tmp_fuzzy_handle=SFH_instance(total_len); if(tmp_fuzzy_handle==NULL) { @@ -1735,7 +1734,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sp->total_len=total_len; sp->fuzzy_hash_handle=tmp_fuzzy_handle; pthread_mutex_init(&(sp->fuzzy_mutex),NULL); - alignment_int64_array_add(p_table->stream_num,thread_num,1); + alignment_int64_array_add(table_aux->stream_num,thread_num,1); return sp; } @@ -1772,7 +1771,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int GIE_result_t query_result[MAX_SCANNER_HIT_NUM]; int hit_region_cnt=0,compile_ret=0; _compile_result_t compile_result[rule_num];//dynamic array - _Maat_table_info_t* p_table=NULL; if(data==NULL||data_len<=0) { return 0; @@ -1781,7 +1779,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return 0; } - GIE_handle_t* GIE_handle=sp->feather->scanner->gie_aux[sp->table_id].gie_handle; + struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id]; + GIE_handle_t* GIE_handle=table_aux->similar.gie_handle; unsigned long long digest_len=0; char* digest_buff=NULL; struct _OUTER_scan_status_t* _mid=NULL; @@ -1790,7 +1789,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { clock_gettime(CLOCK_MONOTONIC,&start); } - p_table=sp->feather->p_table_info[sp->table_id]; alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); pthread_mutex_lock(&(sp->fuzzy_mutex)); sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset); @@ -1807,7 +1805,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { goto fast_out; } - digest_buff=(char*)malloc(sizeof(char)*digest_len); + digest_buff=ALLOC(char, digest_len); pthread_mutex_lock(&(sp->fuzzy_mutex)); SFH_digest(sp->fuzzy_hash_handle,digest_buff, digest_len); pthread_mutex_unlock(&(sp->fuzzy_mutex)); @@ -1827,7 +1825,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int } if(hit_region_cnt>0) { - alignment_int64_array_add(p_table->hit_cnt, sp->thread_num, 1); + alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num, 1); _mid=grab_mid(mid,sp->feather, sp->thread_num,1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1845,11 +1843,11 @@ fast_out: if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num); + maat_stat_table(table_aux, data_len, &start, &end, sp->thread_num); } else { - maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num); + maat_stat_table(table_aux, data_len, NULL, NULL, sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -1861,8 +1859,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; - alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1); + struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id]; + alignment_int64_array_add(table_aux->stream_num, sp->thread_num,-1); if(scanner!=NULL) { if(sp->version==sp->feather->maat_version) @@ -1969,25 +1967,20 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id _feather->scan_err_cnt++; return -1; } - if(p_table->cfg_num==0) - { - return 0; - } my_scanner=_feather->scanner; if(my_scanner==NULL) { return 0; } - GIE_handle_t* gie_handle=my_scanner->gie_aux[table_id].gie_handle; - struct _region_stat_t * region_stat=NULL; - region_stat=&(my_scanner->region_counter[p_table->table_id]); - if(region_stat->cfg_num==0) + struct table_runtime* table_aux=my_scanner->table_rt[table_id]; + if(table_aux->origin_rule_num==0) { return 0; } + GIE_handle_t* gie_handle=table_aux->similar.gie_handle; + INC_SCANNER_REF(my_scanner,thread_num); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); - INC_SCANNER_REF(my_scanner,thread_num); region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM); if(region_ret<0) { @@ -1997,7 +1990,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(p_table->hit_cnt, thread_num,1); + alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -2016,11 +2009,11 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(p_table,0,&start, &end,thread_num); + maat_stat_table(table_aux,0,&start, &end,thread_num); } else { - maat_stat_table(p_table,0,NULL, NULL,thread_num); + maat_stat_table(table_aux,0,NULL, NULL,thread_num); } if(compile_ret==0&®ion_ret>0) { diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index bf6f8b4..523e864 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1590,6 +1590,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int ret=0, table_id=0, i=0; int rule_with_foreign_key=0; struct _Maat_table_info_t* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; for(i=0; ip_table_info[table_id]; - if(p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->n_foreign==0) + plugin_desc= &(p_table->plugin); + if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0) { continue; } - _get_foregin_keys(rule_list+i, p_table->foreign_columns, p_table->n_foreign, dir, logger); + _get_foregin_keys(rule_list+i, plugin_desc->foreign_columns, plugin_desc->n_foreign, dir, logger); rule_with_foreign_key++; } return rule_with_foreign_key; @@ -1736,12 +1738,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m const unsigned char* dec_key, _Maat_feather_t* feather) { - int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0; + int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0, valid_column=-1; int ret=0; struct serial_rule_t* rule_list=NULL; int update_type=CM_UPDATE_TYPE_INC; long long new_version=0; enum MAAT_TABLE_TYPE table_type; + const struct plugin_table_desc* plugin_desc=NULL; void* logger=feather->logger; if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write @@ -1830,7 +1833,16 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m table_type=feather->p_table_info[table_id]->table_type; if(rule_list[i].op==MAAT_OP_DEL) { - ret=invalidate_line(rule_list[i].table_line,table_type,feather->p_table_info[table_id]->valid_flag_column); + if(table_type==TABLE_TYPE_PLUGIN) + { + plugin_desc=&(feather->p_table_info[table_id]->plugin); + valid_column=plugin_desc->valid_flag_column; + } + else + { + valid_column=-1; + } + ret=invalidate_line(rule_list[i].table_line, table_type, valid_column); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Invaid format %s ." @@ -2016,6 +2028,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru int ret=0, table_id=0,success_cnt=0; struct serial_rule_t *s_rule=NULL; struct _Maat_table_info_t* p_table=NULL; + struct plugin_table_desc* plugin_desc=NULL; long long server_time=0,absolute_expire_time=0; const char* p_foreign=NULL; int foreign_key_size=0; @@ -2039,6 +2052,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru goto error_out; } p_table=_feather->p_table_info[table_id]; + plugin_desc=&(p_table->plugin); if(TABLE_TYPE_PLUGIN!=p_table->table_type) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command @@ -2052,7 +2066,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru { ret=get_valid_flag_offset(line_rule[i]->table_line , p_table->table_type - , p_table->valid_flag_column); + , plugin_desc->valid_flag_column); if(ret<0|| (op==MAAT_OP_ADD&&line_rule[i]->table_line[ret]!='1')) { @@ -2071,17 +2085,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru { absolute_expire_time=server_time+line_rule[i]->expire_after; } - if(p_table->n_foreign>0) + if(plugin_desc->n_foreign>0) { - for(j=0;jn_foreign;j++) + for(j=0;jn_foreign;j++) { - p_foreign=find_Nth_column(line_rule[i]->table_line, p_table->foreign_columns[j], &foreign_key_size); + p_foreign=find_Nth_column(line_rule[i]->table_line, plugin_desc->foreign_columns[j], &foreign_key_size); if(p_foreign==NULL) { MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL, maat_command , "Command set line %s %d failed: No %dth column." , line_rule[i]->table_name, line_rule[i]->rule_id - , p_table->foreign_columns[j]); + , plugin_desc->foreign_columns[j]); ret=-1; goto error_out; } @@ -2096,7 +2110,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru } } - set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id,line_rule[i]->table_name,line_rule[i]->table_line, absolute_expire_time); + set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name, + line_rule[i]->table_line, absolute_expire_time); } success_cnt=exec_serial_rule(write_ctx,s_rule, line_num,server_time,_feather->logger); if(success_cnt<0||success_cnt!=line_num)//error diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index f06d60a..4ad3e79 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -521,45 +521,14 @@ void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* s } -_Maat_table_info_t* create_table_info(int max_thread_num) -{ - struct _Maat_table_info_t*p=ALLOC(struct _Maat_table_info_t, 1); - - p->conj_cnt=1; - p->scan_cnt=alignment_int64_array_alloc(max_thread_num); - p->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); - p->input_bytes=alignment_int64_array_alloc(max_thread_num); - p->stream_num=alignment_int64_array_alloc(max_thread_num); - p->hit_cnt=alignment_int64_array_alloc(max_thread_num); - p->cross_cache_size=0; - p->quick_expr_switch=0; - return p; -} -void destroy_table_info(struct _Maat_table_info_t*p) -{ - alignment_int64_array_free(p->scan_cnt); - alignment_int64_array_free(p->scan_cpu_time); - alignment_int64_array_free(p->input_bytes); - alignment_int64_array_free(p->stream_num); - alignment_int64_array_free(p->hit_cnt); - if(p->cb_info!=NULL) - { - dynamic_array_destroy(p->cb_info->cache_lines, free); - p->cb_info->cache_lines=NULL; - free(p->cb_info); - p->cb_info=NULL; - } - free(p); - return; -} -int read_expr_table_info(const char* line, struct _Maat_table_info_t* p, MESA_htable_handle string2int_map) +int read_expr_table_info(const char* line, struct _Maat_table_info_t* table, MESA_htable_handle string2int_map) { int j=0,ret[4]={0}; char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; char *token=NULL,*sub_token=NULL,*saveptr; - - sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(p->table_id) - ,p->table_name[0] + struct expr_table_desc* p=&(table->expr); + sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) + ,table->table_name[0] ,table_type ,src_charset ,dst_charset @@ -567,7 +536,7 @@ int read_expr_table_info(const char* line, struct _Maat_table_info_t* p, MESA_ht ,&(p->cross_cache_size) ,quick_str_scan); memset(ret,0,sizeof(ret)); - ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(p->table_type)); + ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset)); ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge)); if(strlen(quick_str_scan)>0) @@ -606,6 +575,17 @@ int read_expr_table_info(const char* line, struct _Maat_table_info_t* p, MESA_ht } return 0; } +_Maat_table_info_t* table_info_new(int max_thread_num) +{ + struct _Maat_table_info_t*p=ALLOC(struct _Maat_table_info_t, 1); + p->conj_cnt=1; + return p; +} +void table_info_free(struct _Maat_table_info_t*p) +{ + free(p); + return; +} int _read_integer_arrary(char* string, int *array, int size) { char *token=NULL,*sub_token=NULL,*saveptr; @@ -619,15 +599,16 @@ int _read_integer_arrary(char* string, int *array, int size) } return i; } + int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p) { int i=0,ret=0; - cJSON* json=NULL, *tmp=NULL; + cJSON* json=NULL, *tmp=NULL, *array_item=NULL; char* copy_line=NULL, *plug_info=NULL; char *token=NULL,*sub_token=NULL,*saveptr; - - p->valid_flag_column=-1; + struct plugin_table_desc* plugin_desc=&(p->plugin); + plugin_desc->valid_flag_column=-1; copy_line=_maat_strdup(line); for (token = copy_line, i=0; ; token= NULL, i++) { @@ -647,10 +628,10 @@ int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p) if(strlen(plug_info)<4)//For old version compatible. { - ret=sscanf(plug_info, "%d", &(p->valid_flag_column)); + ret=sscanf(plug_info, "%d", &(plugin_desc->valid_flag_column)); if(ret==0||ret==EOF) { - p->valid_flag_column=-1; + plugin_desc->valid_flag_column=-1; } free(copy_line); return 0; @@ -660,20 +641,41 @@ int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p) { goto error_out; } + tmp=cJSON_GetObjectItem(json, "key"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->key_column=tmp->valueint; + } tmp=cJSON_GetObjectItem(json, "valid"); if(tmp!=NULL) { - p->valid_flag_column=tmp->valueint; + assert(tmp->type==cJSON_Number); + plugin_desc->valid_flag_column=tmp->valueint; } tmp=cJSON_GetObjectItem(json, "tag"); if(tmp!=NULL) { - p->rule_tag_column=tmp->valueint; + assert(tmp->type==cJSON_Number); + plugin_desc->rule_tag_column=tmp->valueint; } tmp=cJSON_GetObjectItem(json, "foreign"); if(tmp!=NULL) { - p->n_foreign=_read_integer_arrary(tmp->valuestring, p->foreign_columns, MAX_FOREIGN_CLMN_NUM); + if(tmp->type==cJSON_String) + { + plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM); + } + else if(tmp->type==cJSON_Array) + { + plugin_desc->n_foreign= cJSON_GetArraySize(tmp); + for(i=0;in_foreign; i++) + { + array_item=cJSON_GetArrayItem(tmp, i); + assert(array_item->type==cJSON_Number); + plugin_desc->foreign_columns[i]=array_item->valueint; + } + } } cJSON_Delete(json); @@ -738,7 +740,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* { continue; } - p=create_table_info(max_thread_num); + p=table_info_new(max_thread_num); ret=sscanf(line,"%hu\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) ,p->table_name[0] @@ -771,8 +773,6 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* } break; case TABLE_TYPE_PLUGIN: - p->cb_info=(struct _plugin_table_info*)calloc(sizeof(struct _plugin_table_info),1); - p->cb_info->cache_lines=dynamic_array_create(1024,1024); ret=read_plugin_table_info(line, p); if(ret<0) { @@ -786,11 +786,11 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* ret=sscanf(not_care,"%[a-z0-9]",tmp_str); if(ret>0) { - ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->user_region_encoding)); + ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); } if(ret!=1) { - p->user_region_encoding=USER_REGION_ENCODE_NONE; + p->compile.user_region_encoding=USER_REGION_ENCODE_NONE; } default: break; @@ -828,7 +828,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_cnt++; continue; error_jump: - destroy_table_info(p); + table_info_free(p); p=NULL; } fclose(fp); @@ -989,6 +989,7 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id) void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule) { const struct _Maat_table_info_t* table=compile_rule->ref_table; + const struct compile_table_desc* compile_desc=&(table->compile); struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule; int i=0; compile_rule->compile_id=-1; @@ -999,9 +1000,9 @@ void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule) if(db_compile_rule!=NULL) { - for(i=0; iex_data_num; i++) + for(i=0; iex_data_num; i++) { - rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i); + rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, compile_desc->ex_desc+i); compile_rule->ads[i]=NULL; } @@ -1152,7 +1153,7 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule) op_expr->rule_type=p_rule->rule_type; return; } -GIE_digest_t* create_digest_rule(unsigned int id,short op,const char* digest, +GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, short cfds_lvl,struct _Maat_group_inner_t* tag) { GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); @@ -1182,13 +1183,92 @@ void destroy_digest_rule(GIE_digest_t*rule) rule=NULL; return; } + + +struct table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num) +{ + struct table_runtime* p= ALLOC(struct table_runtime, 1); + p->table_type=type; + switch(type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + p->similar.update_q=MESA_lqueue_create(0,0); + break; + case TABLE_TYPE_PLUGIN: + p->plugin.cache_lines=dynamic_array_create(1, 1024); + break; + default: + break; + } + + p->scan_cnt=alignment_int64_array_alloc(max_thread_num); + p->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); + p->input_bytes=alignment_int64_array_alloc(max_thread_num); + p->stream_num=alignment_int64_array_alloc(max_thread_num); + p->hit_cnt=alignment_int64_array_alloc(max_thread_num); + return p; +} +void table_runtime_free(struct table_runtime* p) +{ + long q_cnt=0,data_size=0; + int i=0; + UNUSED int q_ret=0; + + GIE_digest_t* digest_rule=NULL; + if(p==NULL) + { + return; + } + switch(p->table_type) + { + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + if(p->similar.gie_handle!=NULL) + { + GIE_destory(p->similar.gie_handle); + } + if(p->similar.update_q!=NULL) + { + q_cnt=MESA_lqueue_get_count(p->similar.update_q); + for(i=0;isimilar.update_q,&digest_rule,&data_size); + assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); + destroy_digest_rule(digest_rule); + } + MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL); + } + break; + case TABLE_TYPE_PLUGIN: + dynamic_array_destroy(p->plugin.cache_lines, free); + p->plugin.cache_lines=NULL; + if(p->plugin.ex_data_hash!=NULL) + { + MESA_htable_destroy(p->plugin.ex_data_hash, NULL); + } + break; + default: + break; + } + + alignment_int64_array_free(p->scan_cnt); + alignment_int64_array_free(p->scan_cpu_time); + alignment_int64_array_free(p->input_bytes); + alignment_int64_array_free(p->stream_num); + alignment_int64_array_free(p->hit_cnt); + free(p); + return; +} + struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t *feather) { int scan_thread_num=feather->scan_thread_num; - MESA_lqueue_head tomb=feather->garbage_q; // int rs_scan_type=feather->rule_scan_type; - struct _Maat_table_info_t ** pp_table=feather->p_table_info; - + struct _Maat_table_info_t ** pp_table_desc=feather->p_table_info; + struct table_runtime* table_aux=NULL; + const struct expr_table_desc* expr_desc=NULL; int i=0,j=0; unsigned int sub_type=0; UNUSED int ret=0; @@ -1209,7 +1289,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t hargs.data_expire_with_condition = NULL; struct _Maat_scanner_t* scanner=NULL; - scanner=(struct _Maat_scanner_t*)calloc(sizeof(struct _Maat_scanner_t),1); + scanner=ALLOC(struct _Maat_scanner_t, 1); //Function Maat_cmd_append will access compile_hash in user thread. hargs.thread_safe=1; @@ -1232,7 +1312,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t scanner->max_thread_num=scan_thread_num; //optimized for CPU cache_alignment 64 scanner->ref_cnt=alignment_int64_array_alloc(scan_thread_num); - scanner->region_update_q=MESA_lqueue_create(0,0); + scanner->region_update_q=MESA_lqueue_create(0, 0); scanner->region=rulescan_initialize(scan_thread_num); //For best performance test: @@ -1247,41 +1327,34 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t rulescan_set_param(scanner->region,RULESCAN_DETAIL_RESULT,NULL,0); rulescan_set_param(scanner->region,RULESCAN_REGEX_GROUP,NULL,0); } - scanner->tomb_ref=tomb; - scanner->region_rslt_buff=(scan_result_t*)malloc(sizeof(scan_result_t)*MAX_SCANNER_HIT_NUM*scan_thread_num); + scanner->tomb_ref=feather->garbage_q; + scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num); for(i=0;itable_type) - { - case TABLE_TYPE_DIGEST: - case TABLE_TYPE_SIMILARITY: - scanner->gie_aux[i].table_type=pp_table[i]->table_type; - scanner->gie_aux[i].update_q=MESA_lqueue_create(0,0); - break; - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - if(pp_table[i]->quick_expr_switch==1) + table_aux=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num); + if(pp_table_desc[i]->table_type==TABLE_TYPE_EXPR||pp_table_desc[i]->table_type==TABLE_TYPE_EXPR_PLUS) + { + expr_desc=&(pp_table_desc[i]->expr); + if(expr_desc->quick_expr_switch==1) + { + for(j=0;jexpr.dst_charset[j]!=CHARSET_NONE;j++) { - for(j=0;jdst_charset[j]!=CHARSET_NONE;j++) + sub_type=make_sub_type(pp_table_desc[i]->table_id, expr_desc->dst_charset[j], expr_desc->do_charset_merge); + ret=rulescan_set_param(scanner->region,RULESCAN_QUICK_SCAN,&sub_type,sizeof(sub_type)); + assert(ret==1); + if(expr_desc->do_charset_merge==1) { - sub_type=make_sub_type(pp_table[i]->table_id, pp_table[i]->dst_charset[j], pp_table[i]->do_charset_merge); - ret=rulescan_set_param(scanner->region,RULESCAN_QUICK_SCAN,&sub_type,sizeof(sub_type)); - assert(ret==1); - if(pp_table[i]->do_charset_merge==1) - { - break; - } + break; } } - break; - default: - break; + } } + scanner->table_rt[i]=table_aux; } return scanner; } @@ -1294,7 +1367,6 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner) int i=0,j=0; UNUSED int q_ret=0; struct op_expr_t* op_expr=NULL; - GIE_digest_t* digest_rule=NULL; if(scanner==NULL) { return; @@ -1318,7 +1390,7 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner) MESA_lqueue_destroy(scanner->region_update_q, lqueue_destroy_cb, NULL); free(scanner->region_rslt_buff); scanner->region_rslt_buff=NULL; - free(scanner->ref_cnt); + alignment_int64_array_free(scanner->ref_cnt); scanner->ref_cnt=NULL; for(i=0;igie_aux[i].gie_handle!=NULL) - { - GIE_destory(scanner->gie_aux[i].gie_handle); - } - if(scanner->gie_aux[i].update_q==NULL) - { - continue; - } - q_cnt=MESA_lqueue_get_count(scanner->gie_aux[i].update_q); - for(j=0;jgie_aux[i].update_q,&digest_rule,&data_size); - assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); - destroy_digest_rule(digest_rule); - } - MESA_lqueue_destroy(scanner->gie_aux[i].update_q, lqueue_destroy_cb, NULL); + table_runtime_free(scanner->table_rt[i]); } free(scanner); return; @@ -1377,6 +1433,20 @@ void destroy_ip_expr(boolean_expr_t*p) free(p); return; } +struct _region_stat_t +{ + int cfg_num; + union + { + int expr_rule_cnt; //expr_type=0,1,3 + int ipv4_rule_cnt; + }; + union + { + int regex_rule_cnt; //expr_type=2 + int ipv6_rule_cnt; + }; +}; void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, int size) { assert(op_expr->table_idp_expr,sizeof(boolean_expr_t)); //make a whole memory chunk - to_update_expr[i].rules=(scan_rule_t*)calloc(sizeof(scan_rule_t),op_expr->p_expr->rnum); - for(j=0;j<(int)op_expr->p_expr->rnum;j++) + to_update_expr[i].rules= ALLOC(scan_rule_t, op_expr->p_expr->rnum); + for(j=0;jp_expr->rnum;j++) { memcpy(&(to_update_expr[i].rules[j]),op_expr->p_rules[j],sizeof(scan_rule_t)); if(to_update_expr[i].rules[j].rule_type==RULETYPE_REG||to_update_expr[i].rules[j].rule_type==RULETYPE_STR) @@ -1456,6 +1528,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, ,to_update_expr[i].rules[j].string_rule.len); } } + count_rs_region(op_expr,region_counter,MAX_TABLE_NUM); destroy_op_expr(op_expr); op_expr=NULL; @@ -1485,16 +1558,32 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, //update scanner's region cnt; for(i=0;iregion_counter[i].cfg_num+=region_counter[i].cfg_num; - maat_scanner->region_counter[i].expr_rule_cnt+=region_counter[i].expr_rule_cnt; - maat_scanner->region_counter[i].regex_rule_cnt+=region_counter[i].regex_rule_cnt; - assert(maat_scanner->region_counter[i].cfg_num>=0); - assert(maat_scanner->region_counter[i].expr_rule_cnt>=0); - assert(maat_scanner->region_counter[i].regex_rule_cnt>=0); + table_aux=maat_scanner->table_rt[i]; + if(table_aux==NULL) + { + continue; + } + switch(table_aux->table_type) + { + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + table_aux->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt; + table_aux->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt; + assert(table_aux->expr.expr_rule_cnt>=0); + assert(table_aux->expr.regex_rule_cnt>=0); + break; + case TABLE_TYPE_IP: + table_aux->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt; + table_aux->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt; + break; + default: + break; + } + assert(table_aux->origin_rule_num>=0); } for(i=0;itable_rt[table_id]; update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); for(i=0;ioperation==GIE_INSERT_OPT) { - maat_scanner->region_counter[table_id].cfg_num++; + table_aux->origin_rule_num++; } else { - maat_scanner->region_counter[table_id].cfg_num--; + table_aux->origin_rule_num--; } destroy_digest_rule(update_array[i]); update_array[i]=NULL; @@ -1795,7 +1883,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule char* p=NULL,*saveptr=NULL,*region_string=NULL; int region_str_len=0,ret=0,k=0; int expr_id=0,district_id=-1; - + struct expr_table_desc* expr_desc=&(table->expr); scan_rule_t*p_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL; enum MAAT_CHARSET dst_charset=CHARSET_NONE; @@ -1841,7 +1929,6 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule sub_key_array[i]=str_unescape(sub_key_array[i]); } sub_expr_cnt=i; - table->expr_rule_cnt++; break; case EXPR_TYPE_OFFSET: for(i=0,p=db_rule->keywords;;i++,p=NULL) @@ -1875,7 +1962,6 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule sub_key_array[i]=str_unescape(sub_key_array[i]); } sub_expr_cnt=i; - table->expr_rule_cnt++; break; case EXPR_TYPE_REGEX://it's easy,no need to charset convert expr_id=scanner->exprid_generator++; @@ -1914,14 +2000,12 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } MESA_lqueue_join_tail(scanner->region_update_q,&op_expr, sizeof(void*)); - table->regex_rule_cnt++; return 0;//yes,we returned. break; case EXPR_TYPE_STRING: sub_expr_cnt=1; sub_key_array[0]=db_rule->keywords; sub_key_array[0]=str_unescape(sub_key_array[0]); - table->expr_rule_cnt++; break; default: break; @@ -1940,7 +2024,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule { for(j=0;jdst_charset[j]; + dst_charset=expr_desc->dst_charset[j]; if(dst_charset==CHARSET_NONE) { break; @@ -1964,10 +2048,10 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } region_str_len=strlen(sub_key_array[k])*8+1; // 1 byte map to 8 bytes maximum, e.g. "ا" or "\u63221;" region_string=(char*)calloc(sizeof(char),region_str_len); - if(table->src_charset!=dst_charset)//need convert + if(expr_desc->src_charset!=dst_charset)//need convert { - ret=universal_charset_convert(scanner,table->src_charset, dst_charset, + ret=universal_charset_convert(scanner,expr_desc->src_charset, dst_charset, sub_key_array[k],strlen(sub_key_array[k]), region_string, ®ion_str_len); if(ret<0) @@ -1975,11 +2059,11 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , "Table %s region cfg %d charset convert from %s to %s failed.",table->table_name ,db_rule->region_id - ,CHARSET_STRING[table->src_charset] + ,CHARSET_STRING[expr_desc->src_charset] ,CHARSET_STRING[dst_charset]); free(region_string); op_expr->convert_failed++; - table->iconv_err_cnt++; + expr_desc->iconv_err_cnt++; break; } if(region_str_len==(int)strlen(sub_key_array[k])&& @@ -1993,7 +2077,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule memcpy(region_string,sub_key_array[k],strlen(sub_key_array[k])); region_str_len=strlen(sub_key_array[k]); } - p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,table->do_charset_merge) + p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge) ,db_rule->match_method ,db_rule->is_case_sensitive ,region_string @@ -2006,7 +2090,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } //if each sub string's convert take no effect and src charset is one of the dst. //if any sub expr convert failed - if((TRUE==table->src_charset_in_dst&&op_expr->no_effect_convert_cnt==sub_expr_cnt)|| + if((TRUE==expr_desc->src_charset_in_dst&&op_expr->no_effect_convert_cnt==sub_expr_cnt)|| op_expr->convert_failed>0) { scanner->dedup_expr_num++; @@ -2042,7 +2126,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule region_string=(char*)calloc(sizeof(char),region_str_len); region_str_len=hex2bin(sub_key_array[0], strlen(sub_key_array[0]),region_string,region_str_len); - p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,table->do_charset_merge) + p_rule=create_rs_str_rule(make_sub_type(table->table_id,dst_charset,expr_desc->do_charset_merge) ,db_rule->match_method ,db_rule->is_case_sensitive ,region_string @@ -2125,6 +2209,7 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db struct _Maat_group_inner_t* group_rule=NULL; GIE_digest_t* digest_rule=NULL; struct _Maat_group_inner_t* u_para=NULL; + struct table_runtime * table_aux=scanner->table_rt[table->table_id]; int expr_id=0,district_id=-1; group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); @@ -2147,7 +2232,7 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db ,db_digest_rule->digest_string ,db_digest_rule->confidence_degree ,group_rule); - MESA_lqueue_join_tail(scanner->gie_aux[table->table_id].update_q, &digest_rule, sizeof(void*)); + MESA_lqueue_join_tail(table_aux->similar.update_q, &digest_rule, sizeof(void*)); scanner->gie_total_q_size++; return 0; } @@ -2200,7 +2285,7 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id, ,NULL ,0 ,NULL); - MESA_lqueue_join_tail(maat_scanner->gie_aux[table->table_id].update_q,&digest_rule, sizeof(void*)); + MESA_lqueue_join_tail(maat_scanner->table_rt[table->table_id]->similar.update_q,&digest_rule, sizeof(void*)); maat_scanner->gie_total_q_size++; break; default: @@ -2213,7 +2298,7 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id, garbage_bagging(GARBAGE_GROUP_RULE, group_rule, maat_scanner->tomb_ref); } - return 1; + return 0; } int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) { @@ -2307,9 +2392,9 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* pthread_rwlock_wrlock(&(compile_rule->rwlock)); compile_rule->ref_table=table; compile_rule->db_c_rule=db_compile_rule; - for(i=0; iex_data_num; i++) + for(i=0; icompile.ex_data_num; i++) { - compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->ex_desc+i); + compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->compile.ex_desc+i); } compile_rule->is_valid=1; pthread_rwlock_unlock(&(compile_rule->rwlock)); @@ -2343,6 +2428,7 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) { struct db_group_rule_t db_group_rule; + struct table_runtime* table_aux=scanner->table_rt[table->table_id]; int ret=0; ret=sscanf(table_line,"%d\t%d\t%d",&(db_group_rule.group_id) ,&(db_group_rule.compile_id) @@ -2361,7 +2447,7 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s //leave no trace when compatible_group_update calling if(table->table_type==TABLE_TYPE_GROUP) { - table->cfg_num--; + table_aux->origin_rule_num--; } } else @@ -2380,7 +2466,7 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s //no need to free db_group_rule,it was saved in scanner->compile_hash if(table->table_type==TABLE_TYPE_GROUP) { - table->cfg_num++; + table_aux->origin_rule_num++; } } } @@ -2399,6 +2485,8 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; + const struct expr_table_desc* expr_desc=&(table->expr); + struct table_runtime* table_aux=scanner->table_rt[table->table_id]; switch(table->table_type) { case TABLE_TYPE_EXPR: @@ -2517,13 +2605,13 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st ret=del_region_rule(table ,maat_str_rule->region_id,maat_str_rule->group_id,rule_type ,scanner, logger); - if(ret>0) + if(ret<0) { - table->cfg_num--; + table->udpate_err_cnt++; } else { - table->udpate_err_cnt++; + table_aux->origin_rule_num--; } } else @@ -2538,7 +2626,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st } if(maat_str_rule->expr_type==EXPR_TYPE_STRING - &&table->quick_expr_switch==1 + &&expr_desc->quick_expr_switch==1 &&maat_str_rule->match_method!=MATCH_METHOD_SUB) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , @@ -2557,8 +2645,9 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st } else { - table->cfg_num++; + table_aux->origin_rule_num++; } + } error_out: free(maat_str_rule); @@ -2568,7 +2657,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru { struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40]; - + struct table_runtime* table_aux=scanner->table_rt[table->table_id]; unsigned short i_src_port,i_sport_mask,i_dst_port,i_dport_mask; int protocol=0,direction=0; int ret=0,rule_type=0; @@ -2677,21 +2766,13 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru ret=del_region_rule(table ,ip_rule->region_id,ip_rule->group_id,rule_type ,scanner, logger); - if(ret>0) + if(ret<0) { - table->cfg_num--; - if(ip_rule->addr_type==4) - { - table->ipv4_rule_cnt--; - } - else - { - table->ipv6_rule_cnt--; - } + table->udpate_err_cnt++; } else { - table->udpate_err_cnt++; + table_aux->origin_rule_num--; } } else @@ -2707,16 +2788,9 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru } else { - table->cfg_num++; - if(ip_rule->addr_type==4) - { - table->ipv4_rule_cnt++; - } - else - { - table->ipv6_rule_cnt++; - } + table_aux->origin_rule_num++; } + } error_out: free(ip_rule); @@ -2725,7 +2799,8 @@ error_out: void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { - struct db_intval_rule_t* intval_rule=(struct db_intval_rule_t*)calloc(sizeof(struct db_intval_rule_t),1); + struct db_intval_rule_t* intval_rule=ALLOC(struct db_intval_rule_t, 1); + struct table_runtime* table_aux=scanner->table_rt[table->table_id]; int ret=0; ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) ,&(intval_rule->group_id) @@ -2766,14 +2841,15 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line, ret=del_region_rule(table ,intval_rule->region_id,intval_rule->group_id,RULETYPE_INT ,scanner, logger); - if(ret>0) - { - table->cfg_num--; - } - else + if(ret<0) { table->udpate_err_cnt++; } + else + { + table_aux->origin_rule_num--; + } + } else { @@ -2787,8 +2863,9 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table->cfg_num++; + table_aux->origin_rule_num++; } + } error_out: free(intval_rule); @@ -2797,7 +2874,10 @@ error_out: void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) { - struct db_compile_rule_t *p_compile=(struct db_compile_rule_t*)calloc(sizeof(struct db_compile_rule_t ),1); + struct compile_table_desc* compile_desc=&(table->compile); + struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + + struct db_compile_rule_t *p_compile=ALLOC(struct db_compile_rule_t, 1); struct _head_Maat_rule_t* p_m_rule=&(p_compile->m_rule_head); char user_region[MAX_TABLE_LINE_SIZE]={0}; char tag_str[MAX_TABLE_LINE_SIZE]={0}; @@ -2837,7 +2917,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line goto no_save; } } - switch(table->user_region_encoding) + switch(compile_desc->user_region_encoding) { case USER_REGION_ENCODE_ESCAPE: str_unescape(user_region); @@ -2846,16 +2926,13 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line break; } p_m_rule->serv_def_len=strlen(user_region)+1; - p_compile->service_defined=(char*)malloc(p_m_rule->serv_def_len*sizeof(char)); + p_compile->service_defined= ALLOC(char, p_m_rule->serv_def_len); memcpy(p_compile->service_defined,user_region,p_m_rule->serv_def_len); if(p_compile->is_valid==FALSE) { ret=del_compile_rule(table,p_compile,scanner, logger); - if(ret>0) - { - table->cfg_num--; - } + table_aux->origin_rule_num--; goto no_save; } else @@ -2869,11 +2946,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line table->udpate_err_cnt++; goto no_save; } - else - { - //no need to free p_compile,it was saved in scanner->compile_hash - table->cfg_num++; - } + table_aux->origin_rule_num++; } return; @@ -2887,7 +2960,8 @@ no_save: void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { - struct db_digest_rule_t* digest_rule=(struct db_digest_rule_t*)calloc(sizeof(struct db_digest_rule_t),1); + struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct db_digest_rule_t* digest_rule=ALLOC(struct db_digest_rule_t, 1); int ret=0; char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'}; if(table->table_type==TABLE_TYPE_DIGEST) @@ -2945,14 +3019,15 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line, { //digest rule is not build with rulescan, this rule type is useless in count_rs_region funciton. ret=del_region_rule(table,digest_rule->region_id,digest_rule->group_id,0 ,scanner, logger); - if(ret>0) - { - table->cfg_num--; - } - else + if(ret<0) { table->udpate_err_cnt++; } + else + { + table_aux->origin_rule_num--; + } + } else { @@ -2966,8 +3041,9 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table->cfg_num++; + table_aux->origin_rule_num++; } + } error_out: @@ -3086,25 +3162,26 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) q_cnt,bury_cnt); } } -void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line, const struct rule_tag* tags, int n_tags, void* logger) +void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int i=0, ret=1; unsigned int len=strlen(table_line)+1; - struct _plugin_table_info* p_table_cb=table->cb_info; + struct plugin_table_desc* plugin_desc=&(table->plugin); + struct table_runtime* table_aux=scanner->table_rt[table->table_id]; char *p=NULL; char* copy=NULL; char *token=NULL,*sub_token=NULL,*saveptr; - if(table->rule_tag_column>0&&n_tags>0) + if(plugin_desc->rule_tag_column>0&&n_tags>0) { copy=_maat_strdup(table_line); - for (token = copy, i=0; irule_tag_column ; token= NULL, i++) + for (token = copy, i=0; irule_tag_column ; token= NULL, i++) { sub_token= strtok_r(token," \t", &saveptr); if (sub_token == NULL) break; } - if(i==table->rule_tag_column&&strlen(sub_token)>2) + if(i==plugin_desc->rule_tag_column&&strlen(sub_token)>2) { ret=compare_accept_tag(sub_token, tags, n_tags); if(ret<0) @@ -3127,28 +3204,29 @@ void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line } } - p_table_cb->acc_line_num++; + table_aux->plugin.acc_line_num++; - if(p_table_cb->cb_plug_cnt>0) + if(plugin_desc->cb_plug_cnt>0) { - for(i=0;icb_plug_cnt;i++) + for(i=0;icb_plug_cnt;i++) { - p_table_cb->cb_plug[i].update(table->table_id,table_line,p_table_cb->cb_plug[i].u_para); + plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para); } } else { - p=(char*)calloc(len,1); + p=ALLOC(char, len); memcpy(p,table_line,len); - p_table_cb->cache_size+=len; - dynamic_array_write(p_table_cb->cache_lines,p_table_cb->cache_line_num,p); - p_table_cb->cache_line_num++; + table_aux->plugin.cache_size+=len; + dynamic_array_write(table_aux->plugin.cache_lines,table_aux->plugin.cache_line_num,p); + table_aux->plugin.cache_line_num++; } } void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger) { void *tmp1=NULL,*tmp2=NULL; MESA_htable_handle tmp_map=NULL; + struct table_runtime* table_aux=NULL; int i=0; long q_cnt; GIE_create_para_t para; @@ -3174,34 +3252,46 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ ,scanner); for(i=0;igie_aux[i].update_q==NULL) + table_aux=scanner->table_rt[i]; + if(table_aux==NULL) { continue; } - q_cnt=MESA_lqueue_get_count(scanner->gie_aux[i].update_q); - if(q_cnt==0) + switch(table_aux->table_type) { - continue; + case TABLE_TYPE_DIGEST: + case TABLE_TYPE_SIMILARITY: + q_cnt=MESA_lqueue_get_count(table_aux->similar.update_q); + if(q_cnt==0) + { + continue; + } + if(table_aux->similar.gie_handle==NULL) + { + if(table_aux->table_type==TABLE_TYPE_SIMILARITY) + { + para.ED_reexamine=1; + para.format=GIE_INPUT_FORMAT_PLAIN; + } + else + { + para.ED_reexamine=0; + para.format=GIE_INPUT_FORMAT_SFH; + } + table_aux->similar.gie_handle=GIE_create(¶); + } + digest_batch_update(table_aux->similar.gie_handle, + table_aux->similar.update_q, + logger, + scanner, + i); + break; + case TABLE_TYPE_PLUGIN: + break; + default: + break; } - if(scanner->gie_aux[i].gie_handle==NULL) - { - if(scanner->gie_aux[i].table_type==TABLE_TYPE_SIMILARITY) - { - para.ED_reexamine=1; - para.format=GIE_INPUT_FORMAT_PLAIN; - } - else - { - para.ED_reexamine=0; - para.format=GIE_INPUT_FORMAT_SFH; - } - scanner->gie_aux[i].gie_handle=GIE_create(¶); - } - digest_batch_update(scanner->gie_aux[i].gie_handle - ,scanner->gie_aux[i].update_q - ,logger - ,scanner - ,i); + } scanner->gie_total_q_size=0; if(scanner->tmp_district_map!=NULL) @@ -3235,7 +3325,7 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; struct _Maat_table_info_t* p_table=NULL; - struct _plugin_table_info* p_table_cb=NULL; + struct plugin_table_desc* plugin_desc=NULL; int i=0,j=0; feather->new_version=new_version; @@ -3245,21 +3335,6 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, "Full config version %u -> %u update start", feather->maat_version,new_version); - for(i=0;ip_table_info[i]; - if(p_table==NULL) - { - continue; - } - p_table->cfg_num=0; - p_table->expr_rule_cnt=0; - p_table->regex_rule_cnt=0; - if(p_table->table_type==TABLE_TYPE_PLUGIN) - { - clear_plugin_table_info(p_table->cb_info); - } - } } else { @@ -3273,66 +3348,82 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) for(i=0;ip_table_info[i]; - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->cb_info->cb_plug_cnt==0) + plugin_desc=&(p_table->plugin); + if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0) { continue; } - p_table_cb=p_table->cb_info; feather->active_plugin_table_num++; - for(j=0;jcb_plug_cnt;j++) + for(j=0;jcb_plug_cnt;j++) { - if(p_table_cb->cb_plug[j].start!=NULL) + if(plugin_desc->cb_plug[j].start!=NULL) { - p_table_cb->cb_plug[j].start(update_type,p_table_cb->cb_plug[j].u_para); + plugin_desc->cb_plug[j].start(update_type,plugin_desc->cb_plug[j].u_para); } } } return; } +long long scanner_rule_num(struct _Maat_scanner_t *scanner) +{ + long long total=0; + struct table_runtime* table_aux=NULL; + int i=0; + for(i=0;itable_rt[i]; + if(table_aux!=NULL) + { + total+=table_aux->origin_rule_num; + } + } + return total; +} void maat_finish_cb(void* u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; struct _Maat_table_info_t* p_table=NULL; - struct _plugin_table_info* p_table_cb=NULL; + struct plugin_table_desc* plugin_desc=NULL; long expr_wait_q_cnt=0; - int i=0,j=0,total=0; - for(i=0;ip_table_info[i]; - if(p_table!=NULL) - { - total+=p_table->cfg_num; - } - } + int i=0, j=0; + int call_plugin_table_cnt=0; for(i=0;ip_table_info[i]; - - if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->cb_info->cb_plug_cnt==0) + if(p_table==NULL) { continue; } - call_plugin_table_cnt++; - p_table_cb=p_table->cb_info; - if(call_plugin_table_cnt==feather->active_plugin_table_num) + switch(p_table->table_type) { - feather->is_last_plugin_table_updating=1; - } - for(j=0;jcb_plug_cnt;j++) - { - if(p_table_cb->cb_plug[j].finish!=NULL) - { - p_table_cb->cb_plug[j].finish(p_table_cb->cb_plug[j].u_para); - } - } - feather->is_last_plugin_table_updating=0; + case TABLE_TYPE_PLUGIN: + plugin_desc=&(p_table->plugin); + call_plugin_table_cnt++; + if(call_plugin_table_cnt==feather->active_plugin_table_num) + { + feather->is_last_plugin_table_updating=1; + } + for(j=0;jcb_plug_cnt;j++) + { + if(plugin_desc->cb_plug[j].finish!=NULL) + { + plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para); + } + } + feather->is_last_plugin_table_updating=0; + break; + default: + break; + } } + + if(feather->update_tmp_scanner!=NULL) { - feather->update_tmp_scanner->cfg_num=total; + feather->update_tmp_scanner->cfg_num=scanner_rule_num(feather->update_tmp_scanner); do_scanner_update(feather->update_tmp_scanner ,feather->garbage_q ,feather->scan_thread_num @@ -3343,7 +3434,7 @@ void maat_finish_cb(void* u_para) } else if(feather->scanner!=NULL) { - feather->scanner->cfg_num=total; + feather->scanner->cfg_num=scanner_rule_num(feather->scanner); feather->scanner->version=feather->maat_version; expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_total_q_size; @@ -3353,23 +3444,23 @@ void maat_finish_cb(void* u_para) ,feather->garbage_q ,feather->scan_thread_num ,feather->logger); - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Inc config version %u build complete, %d entries in total.", - feather->scanner->version,feather->scanner->cfg_num); + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module + ,"Inc config version %u build complete, %d entries in total." + ,feather->scanner->version,feather->scanner->cfg_num); feather->postpone_q_size=0; } else { - MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, + MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module, "Postpone %d entries of version %u load to rulescan.", - feather->scanner->cfg_num,feather->scanner->version); + feather->scanner->cfg_num, feather->scanner->version); } } else { MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, - "Version %d have no valid scan rules, plugin callback complete.", - feather->maat_version); + "Version %d have no valid scan rules, plugin callback complete.", + feather->maat_version); } feather->new_version=-1; feather->active_plugin_table_num=0; @@ -3429,7 +3520,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) update_group_rule(feather->p_table_info[table_id], line, scanner,feather->logger); break; case TABLE_TYPE_PLUGIN: - update_plugin_table(feather->p_table_info[table_id], line, feather->accept_tags, feather->n_tags, feather->logger); + update_plugin_table(feather->p_table_info[table_id], line, scanner, feather->accept_tags, feather->n_tags, feather->logger); default: break; @@ -3610,7 +3701,7 @@ void *thread_rule_monitor(void *arg) { continue; } - destroy_table_info(feather->p_table_info[i]); + table_info_free(feather->p_table_info[i]); feather->p_table_info[i]=NULL; } alignment_int64_array_free(feather->thread_call_cnt); diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index ed7190b..cc0fcbf 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -149,20 +149,20 @@ void maat_stat_init(struct _Maat_feather_t* feather) FS_start(feather->stat_handle); return; } -void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num) +void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num) { - alignment_int64_array_add(p_table->scan_cnt,thread_num,1); - alignment_int64_array_add(p_table->input_bytes,thread_num,scan_len); + alignment_int64_array_add(p->scan_cnt,thread_num,1); + alignment_int64_array_add(p->input_bytes,thread_num,scan_len); if(start!=NULL&&end!=NULL) { - alignment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec); + alignment_int64_array_add(p->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec); } return; } void maat_stat_output(struct _Maat_feather_t* feather) { long value=0; - long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0; + long long total_cfg_num=0, table_regex_ipv6_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0; long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0; long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; long long outer_mid_cnt=0,inner_mid_cnt=0; @@ -172,6 +172,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) int i=0; time_t now; struct _Maat_table_info_t* p_table=NULL; + struct table_runtime* table_aux=NULL; time(&now); active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); @@ -201,86 +202,105 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_input_bytes=0; table_scan_cpu_time=0; table_hit_cnt=0; + table_regex_ipv6_num=0; p_table=feather->p_table_info[i]; if(p_table==NULL) { continue; } + table_aux=feather->scanner->table_rt[i]; switch(p_table->table_type) { case TABLE_TYPE_PLUGIN: - plugin_cache_num+=p_table->cb_info->cache_line_num; - plugin_acc_num+=p_table->cb_info->acc_line_num; + plugin_cache_num+=table_aux->plugin.cache_line_num; + plugin_acc_num+=table_aux->plugin.acc_line_num; break; case TABLE_TYPE_GROUP: - group_rule_num+=p_table->cfg_num; + group_rule_num+=table_aux->origin_rule_num; break; case TABLE_TYPE_COMPILE: - compile_rule_num+=p_table->cfg_num; + compile_rule_num+=table_aux->origin_rule_num; + break; + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + table_regex_ipv6_num=table_aux->expr.regex_rule_cnt; + total_iconv_error=p_table->expr.iconv_err_cnt; + break; + case TABLE_TYPE_IP: + table_regex_ipv6_num=table_aux->ip.ipv6_rule_cnt; break; default: - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_RULE_NUM], - FS_OP_SET, - p_table->cfg_num); - total_cfg_num+=p_table->cfg_num; - - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], - FS_OP_SET, - p_table->regex_rule_cnt); - total_regex_num+= p_table->regex_rule_cnt; - - table_stream_num=alignment_int64_array_sum(p_table->stream_num,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], - FS_OP_SET, - table_stream_num); - total_stream_cnt+= table_stream_num; - - table_scan_cnt=alignment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], - FS_OP_SET, - table_scan_cnt); - total_scan_cnt+=table_scan_cnt; - - table_input_bytes=alignment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], - FS_OP_SET, - table_input_bytes); - total_input_bytes+=table_input_bytes; - if(feather->perf_on==1) - { - table_scan_cpu_time=alignment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num); - table_scan_cpu_time/=1000; - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_CPU_TIME], - FS_OP_SET, - table_scan_cpu_time); - total_cpu_time+=table_scan_cpu_time; - } - - table_hit_cnt=alignment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_HIT_CNT], - FS_OP_SET, - table_hit_cnt); - total_update_error+=p_table->udpate_err_cnt; - total_iconv_error+=p_table->iconv_err_cnt; - //total hit count stat in region_compile break; } - } + if(p_table->table_type==TABLE_TYPE_PLUGIN|| + p_table->table_type==TABLE_TYPE_GROUP|| + p_table->table_type==TABLE_TYPE_COMPILE) + { + continue; + } + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_RULE_NUM], + FS_OP_SET, + table_aux->origin_rule_num); + total_cfg_num+=table_aux->origin_rule_num; + + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], + FS_OP_SET, + table_regex_ipv6_num); + total_regex_num+=table_regex_ipv6_num; + table_stream_num=alignment_int64_array_sum(table_aux->stream_num,feather->scan_thread_num); + alignment_int64_array_reset(table_aux->stream_num,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], + FS_OP_ADD, + table_stream_num); + total_stream_cnt+= table_stream_num; + + + table_scan_cnt=alignment_int64_array_sum(table_aux->scan_cnt,feather->scan_thread_num); + alignment_int64_array_reset(table_aux->scan_cnt,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], + FS_OP_ADD, + table_scan_cnt); + total_scan_cnt+=table_scan_cnt; + + table_input_bytes=alignment_int64_array_sum(table_aux->input_bytes,feather->scan_thread_num); + alignment_int64_array_reset(table_aux->input_bytes,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], + FS_OP_ADD, + table_input_bytes); + total_input_bytes+=table_input_bytes; + if(feather->perf_on==1) + { + table_scan_cpu_time=alignment_int64_array_sum(table_aux->scan_cpu_time,feather->scan_thread_num); + alignment_int64_array_reset(table_aux->scan_cpu_time,feather->scan_thread_num); + table_scan_cpu_time/=1000; + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_CPU_TIME], + FS_OP_ADD, + table_scan_cpu_time); + total_cpu_time+=table_scan_cpu_time; + } + + table_hit_cnt=alignment_int64_array_sum(table_aux->hit_cnt,feather->scan_thread_num); + alignment_int64_array_reset(table_aux->hit_cnt,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_HIT_CNT], + FS_OP_ADD, + table_hit_cnt); + total_update_error+=p_table->udpate_err_cnt; + } FS_operate(feather->stat_handle, feather->total_stat_id, feather->fs_column_id[COLUMN_TABLE_RULE_NUM], @@ -314,7 +334,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_OP_SET, total_cpu_time); } - total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num); + total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt, feather->scan_thread_num); FS_operate(feather->stat_handle, feather->total_stat_id, feather->fs_column_id[COLUMN_TABLE_HIT_CNT], diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 42cdb28..f123134 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -101,7 +101,7 @@ struct compile_ex_data_idx int idx; int table_id; }; -struct compile_table_descr +struct compile_table_desc { enum USER_REGION_ENCODE user_region_encoding; int ex_data_num; @@ -135,7 +135,7 @@ struct _plugin_table_info long cache_size; }; -struct plugin_table_descr +struct plugin_table_desc { int key_column; int valid_flag_column; @@ -147,7 +147,7 @@ struct plugin_table_descr struct plugin_table_ex_data_desc ex_desc; int acc_line_num; }; -struct expr_table_descr +struct expr_table_desc { enum MAAT_CHARSET src_charset; enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; @@ -155,10 +155,9 @@ struct expr_table_descr int do_charset_merge; int cross_cache_size; int quick_expr_switch; - int expr_rule_cnt; //expr_type=0,1,3 - int regex_rule_cnt; //expr_type=2 + long long iconv_err_cnt; }; -struct ip_table_descr +struct ip_table_desc { int ipv4_rule_cnt; int ipv6_rule_cnt; @@ -173,17 +172,17 @@ struct _Maat_table_info_t enum MAAT_TABLE_TYPE table_type; union { - struct compile_table_descr compile; - struct expr_table_descr expr; - struct ip_table_descr ip; - struct plugin_table_descr plugin; + struct compile_table_desc compile; + struct expr_table_desc expr; + struct ip_table_desc ip; + struct plugin_table_desc plugin; void* others;//group, interval and digest don't have sperate description info. - }; + }; +/* enum MAAT_CHARSET src_charset; enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; int src_charset_in_dst; int do_charset_merge; - int cfg_num; int cross_cache_size; int quick_expr_switch; union @@ -206,16 +205,11 @@ struct _Maat_table_info_t enum USER_REGION_ENCODE user_region_encoding; int ex_data_num; struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; +*/ //for stat>>>>>>>> unsigned long long udpate_err_cnt; unsigned long long unmatch_tag_cnt; - unsigned long long iconv_err_cnt; int stat_line_id; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; }; struct db_str_rule_t @@ -334,24 +328,6 @@ struct _compile_result_t int compile_id; universal_bool_expr_t group_set; }; - -struct _region_stat_t -{ - int cfg_num; - union - { - int expr_rule_cnt; //expr_type=0,1,3 - int ipv4_rule_cnt; - }; - union - { - int regex_rule_cnt; //expr_type=2 - int ipv6_rule_cnt; - }; -}; - - - struct _INNER_scan_status_t { int cur_hit_cnt; @@ -403,19 +379,48 @@ struct _stream_para_t pthread_mutex_t fuzzy_mutex; unsigned char query_point[8]; }; -struct digest_engine +struct similar_runtime { - enum MAAT_TABLE_TYPE table_type; GIE_handle_t* gie_handle; MESA_lqueue_head update_q; }; -struct plugin_engine +struct plugin_runtime { dynamic_array_t *cache_lines; - int cache_line_num; - long cache_size; + long long cache_line_num; + long long acc_line_num; + long long cache_size; + MESA_htable_handle ex_data_hash; }; +struct expr_runtime +{ + long long expr_rule_cnt; //expr_type=0,1,3 + long long regex_rule_cnt; //expr_type=2 +}; +struct ip_runtime +{ + long long ipv4_rule_cnt; + long long ipv6_rule_cnt; +}; +struct table_runtime +{ + enum MAAT_TABLE_TYPE table_type; + long origin_rule_num; + union + { + struct similar_runtime similar; //for digest and similarity + struct plugin_runtime plugin; + struct expr_runtime expr; + struct ip_runtime ip; + void * other; + }; + mcore_long_t scan_cnt; + mcore_long_t scan_cpu_time; //nano + mcore_long_t input_bytes; + mcore_long_t stream_num; + mcore_long_t hit_cnt; +}; struct rule_tag { char* tag_name; @@ -425,11 +430,15 @@ struct _Maat_scanner_t { long long version; time_t last_update_time; - long long *ref_cnt; //optimized for cache_alignment 64 + mcore_long_t ref_cnt; rule_scanner_t region; long gie_total_q_size; - struct digest_engine gie_aux[MAX_TABLE_NUM]; - struct plugin_engine plugin_aux[MAX_TABLE_NUM]; + + struct table_runtime* table_rt[MAX_TABLE_NUM]; +/* + struct similar_runtime gie_aux[MAX_TABLE_NUM]; + struct plugin_runtime plugin_aux[MAX_TABLE_NUM]; +*/ MESA_htable_handle region_hash; MESA_htable_handle group_hash; MESA_htable_handle compile_hash; @@ -443,7 +452,7 @@ struct _Maat_scanner_t void * bool_macher_expr_compiler; scan_result_t *region_rslt_buff; MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q - struct _region_stat_t region_counter[MAX_TABLE_NUM]; +// struct _region_stat_t region_counter[MAX_TABLE_NUM]; int max_thread_num; iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from] }; @@ -608,7 +617,7 @@ int HASH_add_by_id(MESA_htable_handle hash,int id,void*data); int HASH_delete_by_id(MESA_htable_handle hash,int id); void maat_read_full_config(_Maat_feather_t* _feather); void maat_stat_init(struct _Maat_feather_t* feather); -void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num); +void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_output(struct _Maat_feather_t* feather); redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); diff --git a/src/inc_internal/alignment_int64.h b/src/inc_internal/alignment_int64.h index a342205..a7a7d2a 100644 --- a/src/inc_internal/alignment_int64.h +++ b/src/inc_internal/alignment_int64.h @@ -43,6 +43,11 @@ inline long long alignment_int64_array_cnt(mcore_long_t array,int size) } return cnt; } +inline void alignment_int64_array_reset(mcore_long_t array,int size) +{ + memset(array, 0, CPU_CACHE_ALIGMENT*size); + return; +} inline void alignment_int64_array_free(mcore_long_t array) { free(array); diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 9930379..41b6b5e 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -480,7 +480,7 @@ TEST(SimilarScan, Pure) const char* scan_data="mwss.xiu.youku.com/live/hls/v1/0000000000000000000000001526a0a8/714.ts?&token=98765"; const char* table_name="SIM_URL"; scan_status_t mid=NULL; - + memset(result, 0 , sizeof(result)); table_id=Maat_table_register(g_feather,table_name); ASSERT_GT(table_id, 0); From 7b5baacf623f672bfef550973830726f07cd48a3 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 4 Dec 2018 23:26:59 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E9=87=8D=E6=9E=84=E5=8F=96column=E7=9A=84?= =?UTF-8?q?=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_rule.h | 4 +- src/entry/Maat_api.cpp | 176 +++++++++++++++------- src/entry/Maat_command.cpp | 46 +++--- src/entry/Maat_rule.cpp | 206 +++++++++++++------------- src/entry/Maat_stat.cpp | 48 +++--- src/entry/Maat_utils.cpp | 24 +++ src/entry/dynamic_array.cpp | 10 +- src/inc_internal/Maat_rule_internal.h | 30 ++-- src/inc_internal/Maat_utils.h | 2 + src/inc_internal/dynamic_array.h | 10 +- test/table_info.conf | 2 +- 11 files changed, 310 insertions(+), 248 deletions(-) diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 2e956a5..83c35f8 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -259,8 +259,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa //Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table. typedef void* MAAT_PLUGIN_EX_DATA; -typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); -typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, size_t key_sz, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, size_t key_sz, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp); typedef int Maat_plugin_EX_key2index_func_t(const char* key); diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index dc24da0..18d22b8 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -16,9 +16,9 @@ #include "rulescan.h" #include "json2iris.h" -struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type) +struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type) { - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; if(table_id>MAX_TABLE_NUM) { return NULL; @@ -918,7 +918,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; int idx=0,i=0; - struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; + struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; struct plugin_table_desc *plugin_desc=&(p_table->plugin); struct plugin_runtime* plugin_aux=NULL; const char* lines=NULL; @@ -999,8 +999,8 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table { return -1; } - struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; - if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL) + struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; + if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL || dup_func==NULL) { return -1; } @@ -1051,6 +1051,47 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa pthread_rwlock_unlock(&(compile_inner->rwlock)); return ad; } +MESA_htable_handle plugin_EX_htable_new(const struct plugin_table_desc* plugin_desc, + struct dynamic_array_t* lines, size_t line_cnt) +{ + MESA_htable_handle key2ex_hash=NULL; + unsigned int slot_size=1; + const char* line=NULL; + size_t key_offset=0, key_len=0; + long long estimate_size=plugin_desc->estimate_size, i=0; + int ret=0; + MAAT_RULE_EX_DATA exdata=NULL; + while(estimate_size>0) + { + estimate_size=estimate_size<<1; + slot_size*=2; + } + if(slot_size==1) + { + slot_size=4096; + } + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=1; + hargs.hash_slot_size = slot_size; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = plugin_desc->ex_desc.key2index_func; + hargs.recursive = 1; + hargs.data_free = NULL ; + hargs.data_expire_with_condition = NULL; + key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(key2ex_hash, 0); + for(i=0; i< line_cnt; i++) + { + line=(const char*)dynamic_array_read(lines, i); + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + exdata=plugin_desc->ex_desc.new_func(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);; + MESA_htable_add(key2ex_hash, line+key_offset, key_len, const void * data) + } +} int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, @@ -1059,6 +1100,27 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, long argl, void *argp) { + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; + struct plugin_table_desc* plugin_desc=&(table_desc->plugin); + struct Maat_table_runtime* table_rt=_feather->scanner->table_rt[table_id]; + MESA_htable_handle key2ex_hash=NULL; + if(table_desc->table_type!=TABLE_TYPE_PLUGIN || new_func==NULL || free_func==NULL || dup_func==NULL) + { + assert(0); + return -1; + } + plugin_desc->ex_desc.new_func=new_func; + plugin_desc->ex_desc.free_func=free_func; + plugin_desc->ex_desc.dup_func=dup_func; + plugin_desc->ex_desc.key2index_func=key2index_func; + plugin_desc->ex_desc.argl=argl; + plugin_desc->ex_desc.argp=argp; + + pthread_mutex_lock(&(_feather->backgroud_update_mutex)); + table_rt->plugin.key2ex_hash=plugin_EX_htable_new(plugin_desc,table_rt->plugin.cache_lines, table_rt->plugin.cache_line_num); + pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); + return 0; } @@ -1074,7 +1136,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num];//dynamic array - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; struct expr_table_desc* expr_desc=NULL; struct timespec start,end; _Maat_scanner_t* my_scanner=NULL; @@ -1117,8 +1179,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { return 0; } - struct table_runtime* table_aux=my_scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } @@ -1126,7 +1188,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; INC_SCANNER_REF(my_scanner, thread_num); - if(table_aux->expr.expr_rule_cnt>0) + if(table_rt->expr.expr_rule_cnt>0) { scan_data.rule_type=RULETYPE_STR; scan_data.sub_type=sub_type; @@ -1136,7 +1198,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id hit_region_cnt+=region_ret; } } - if(table_aux->expr.regex_rule_cnt>0) + if(table_rt->expr.regex_rule_cnt>0) { scan_data.rule_type=RULETYPE_REG; scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); @@ -1153,7 +1215,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(hit_region_cnt>0) { - alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid,_feather,thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1177,11 +1239,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux, data_len, &start, &end, thread_num); + maat_stat_table(table_rt, data_len, &start, &end, thread_num); } else { - maat_stat_table(table_aux, data_len, NULL, NULL, thread_num); + maat_stat_table(table_rt, data_len, NULL, NULL, thread_num); } DEC_SCANNER_REF(my_scanner, thread_num); if(compile_ret==0&&hit_region_cnt>0) @@ -1217,7 +1279,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id intval_scan_data.rule_type=RULETYPE_INT; intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); intval_scan_data.int_data=intval; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; struct timespec start,end; if(_feather->perf_on==1) { @@ -1234,8 +1296,8 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id { return 0; } - struct table_runtime* table_aux=my_scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } @@ -1253,7 +1315,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1271,11 +1333,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux, 0, &start, &end, thread_num); + maat_stat_table(table_rt, 0, &start, &end, thread_num); } else { - maat_stat_table(table_aux, 0, NULL, NULL, thread_num); + maat_stat_table(table_rt, 0, NULL, NULL, thread_num); } DEC_SCANNER_REF(my_scanner,thread_num); @@ -1297,7 +1359,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id scan_data_t ip_scan_data; scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num]; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* my_scanner=NULL; @@ -1317,17 +1379,17 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { return 0; } - struct table_runtime* table_aux=my_scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } - if(table_aux->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4) + if(table_rt->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4) { return 0; } - if(table_aux->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) + if(table_rt->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) { return 0; } @@ -1372,7 +1434,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, @@ -1391,11 +1453,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux, 0, &start, &end, thread_num); + maat_stat_table(table_rt, 0, &start, &end, thread_num); } else { - maat_stat_table(table_aux, 0, NULL, NULL, thread_num); + maat_stat_table(table_rt, 0, NULL, NULL, thread_num); } if(compile_ret==0&®ion_ret>0) { @@ -1420,7 +1482,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* scanner=NULL; - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; assert(thread_num<_feather->scan_thread_num); p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR); if(p_table==NULL) @@ -1445,8 +1507,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { return sp; } - struct table_runtime* table_aux=scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } @@ -1462,15 +1524,15 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { sp->do_merge=1; } - if(table_aux->expr.expr_rule_cnt>0) + if(table_rt->expr.expr_rule_cnt>0) { sp->do_expr=1; } - if(table_aux->expr.regex_rule_cnt>0) + if(table_rt->expr.regex_rule_cnt>0) { sp->do_regex=1; } - alignment_int64_array_add(table_aux->stream_num,thread_num,1); + alignment_int64_array_add(table_rt->stream_num,thread_num,1); sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); return sp; } @@ -1489,8 +1551,8 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para scan_result_t *region_result; _compile_result_t compile_result[rule_num];//dynamic array scan_data_t region_scan_data; - _Maat_table_info_t* p_table=NULL; - struct table_runtime* table_aux=scanner->table_rt[sp->table_id]; + Maat_table_desc* p_table=NULL; + struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; struct timespec start,end; if(data==NULL||data_len<=0) { @@ -1598,7 +1660,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } if(hit_region_cnt>0) { - alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1637,11 +1699,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux,data_len,&start, &end,sp->thread_num); + maat_stat_table(table_rt,data_len,&start, &end,sp->thread_num); } else { - maat_stat_table(table_aux,data_len,NULL, NULL,sp->thread_num); + maat_stat_table(table_rt,data_len,NULL, NULL,sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -1666,8 +1728,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - struct table_runtime* table_aux=scanner->table_rt[sp->table_id]; - alignment_int64_array_add(table_aux->stream_num, sp->thread_num, -1); + struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; + alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); if(sp->rs_stream_para!=NULL) { if(scanner!=NULL&&sp->version>=sp->feather->last_full_version) @@ -1704,7 +1766,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* scanner=NULL; sfh_instance_t * tmp_fuzzy_handle=NULL; - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST); if(p_table==NULL) { @@ -1720,7 +1782,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, { return sp; } - struct table_runtime* table_aux=scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; tmp_fuzzy_handle=SFH_instance(total_len); if(tmp_fuzzy_handle==NULL) { @@ -1734,7 +1796,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sp->total_len=total_len; sp->fuzzy_hash_handle=tmp_fuzzy_handle; pthread_mutex_init(&(sp->fuzzy_mutex),NULL); - alignment_int64_array_add(table_aux->stream_num,thread_num,1); + alignment_int64_array_add(table_rt->stream_num,thread_num,1); return sp; } @@ -1779,8 +1841,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return 0; } - struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id]; - GIE_handle_t* GIE_handle=table_aux->similar.gie_handle; + struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; unsigned long long digest_len=0; char* digest_buff=NULL; struct _OUTER_scan_status_t* _mid=NULL; @@ -1825,7 +1887,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int } if(hit_region_cnt>0) { - alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num, 1); + alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1); _mid=grab_mid(mid,sp->feather, sp->thread_num,1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1843,11 +1905,11 @@ fast_out: if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux, data_len, &start, &end, sp->thread_num); + maat_stat_table(table_rt, data_len, &start, &end, sp->thread_num); } else { - maat_stat_table(table_aux, data_len, NULL, NULL, sp->thread_num); + maat_stat_table(table_rt, data_len, NULL, NULL, sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -1859,8 +1921,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id]; - alignment_int64_array_add(table_aux->stream_num, sp->thread_num,-1); + struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); if(scanner!=NULL) { if(sp->version==sp->feather->maat_version) @@ -1955,7 +2017,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id _compile_result_t compile_result[rule_num]; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* my_scanner=NULL; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; struct timespec start,end; if(_feather->perf_on==1) { @@ -1972,12 +2034,12 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { return 0; } - struct table_runtime* table_aux=my_scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } - GIE_handle_t* gie_handle=table_aux->similar.gie_handle; + GIE_handle_t* gie_handle=table_rt->similar.gie_handle; INC_SCANNER_REF(my_scanner,thread_num); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); @@ -1990,7 +2052,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -2009,11 +2071,11 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux,0,&start, &end,thread_num); + maat_stat_table(table_rt,0,&start, &end,thread_num); } else { - maat_stat_table(table_aux,0,NULL, NULL,thread_num); + maat_stat_table(table_rt,0,NULL, NULL,thread_num); } if(compile_ret==0&®ion_ret>0) { diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 523e864..c8e09a4 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -182,61 +182,51 @@ enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p) } int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) { - unsigned int offset=0; - unsigned int i=0,j=0; - switch(type) + size_t offset=0, len=0; + unsigned int column_seq=0, ret=0; + switch(type) { case TABLE_TYPE_EXPR: - offset=7; + column_seq=7; break; case TABLE_TYPE_IP: - offset=14; + column_seq=14; break; case TABLE_TYPE_COMPILE: - offset=8; + column_seq=8; break; case TABLE_TYPE_PLUGIN: if(valid_column_seq<0) { return -1; } - offset=(unsigned int)valid_column_seq; + column_seq=(unsigned int)valid_column_seq; break; case TABLE_TYPE_INTERVAL: - offset=5; + column_seq=5; break; case TABLE_TYPE_DIGEST: - offset=6; + column_seq=6; break; case TABLE_TYPE_SIMILARITY: - offset=5; + column_seq=5; break; case TABLE_TYPE_EXPR_PLUS: - offset=8; + column_seq=8; break; case TABLE_TYPE_GROUP: - offset=3; + column_seq=3; break; default: assert(0); } - for(i=0;i=strlen(line)||(line[i]!='1'&&line[i]!='0'))// 0 is also a valid value for some non-MAAT producer. + + ret=get_column_pos(const char* line, int column_seq, &offset, &len); + if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer. { return -1; } - return i; + return offset; } int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) { @@ -1589,7 +1579,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, { int ret=0, table_id=0, i=0; int rule_with_foreign_key=0; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; struct plugin_table_desc* plugin_desc=NULL; for(i=0; iconj_cnt=1; return p; } -void table_info_free(struct _Maat_table_info_t*p) +void table_info_free(struct Maat_table_desc*p) { free(p); return; @@ -600,7 +602,7 @@ int _read_integer_arrary(char* string, int *array, int size) return i; } -int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p) +int read_plugin_table_info(const char* line, struct Maat_table_desc* p) { int i=0,ret=0; @@ -659,6 +661,12 @@ int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p) assert(tmp->type==cJSON_Number); plugin_desc->rule_tag_column=tmp->valueint; } + tmp=cJSON_GetObjectItem(json, "estimate_size"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->estimate_size=tmp->valueint; + } tmp=cJSON_GetObjectItem(json, "foreign"); if(tmp!=NULL) { @@ -685,15 +693,15 @@ error_out: free(copy_line); return -1; } -int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) +int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) { FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; int i=0,ret=0,table_cnt=0; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; MESA_htable_handle string2int_map=map_create(); - struct _Maat_table_info_t*p=NULL; - struct _Maat_table_info_t*conj_table=NULL; + struct Maat_table_desc*p=NULL; + struct Maat_table_desc*conj_table=NULL; map_register(string2int_map,"expr", TABLE_TYPE_EXPR); map_register(string2int_map,"ip", TABLE_TYPE_IP); @@ -988,7 +996,7 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id) } void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule) { - const struct _Maat_table_info_t* table=compile_rule->ref_table; + const struct Maat_table_desc* table=compile_rule->ref_table; const struct compile_table_desc* compile_desc=&(table->compile); struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule; int i=0; @@ -1185,9 +1193,9 @@ void destroy_digest_rule(GIE_digest_t*rule) } -struct table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num) +struct Maat_table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num) { - struct table_runtime* p= ALLOC(struct table_runtime, 1); + struct Maat_table_runtime* p= ALLOC(struct Maat_table_runtime, 1); p->table_type=type; switch(type) { @@ -1209,7 +1217,7 @@ struct table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_threa p->hit_cnt=alignment_int64_array_alloc(max_thread_num); return p; } -void table_runtime_free(struct table_runtime* p) +void table_runtime_free(struct Maat_table_runtime* p) { long q_cnt=0,data_size=0; int i=0; @@ -1244,9 +1252,9 @@ void table_runtime_free(struct table_runtime* p) case TABLE_TYPE_PLUGIN: dynamic_array_destroy(p->plugin.cache_lines, free); p->plugin.cache_lines=NULL; - if(p->plugin.ex_data_hash!=NULL) + if(p->plugin.key2ex_hash!=NULL) { - MESA_htable_destroy(p->plugin.ex_data_hash, NULL); + MESA_htable_destroy(p->plugin.key2ex_hash, NULL); } break; default: @@ -1266,8 +1274,8 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t { int scan_thread_num=feather->scan_thread_num; // int rs_scan_type=feather->rule_scan_type; - struct _Maat_table_info_t ** pp_table_desc=feather->p_table_info; - struct table_runtime* table_aux=NULL; + struct Maat_table_desc ** pp_table_desc=feather->p_table_info; + struct Maat_table_runtime* table_rt=NULL; const struct expr_table_desc* expr_desc=NULL; int i=0,j=0; unsigned int sub_type=0; @@ -1336,7 +1344,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t { continue; } - table_aux=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num); + table_rt=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num); if(pp_table_desc[i]->table_type==TABLE_TYPE_EXPR||pp_table_desc[i]->table_type==TABLE_TYPE_EXPR_PLUS) { expr_desc=&(pp_table_desc[i]->expr); @@ -1354,7 +1362,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t } } } - scanner->table_rt[i]=table_aux; + scanner->table_rt[i]=table_rt; } return scanner; } @@ -1502,7 +1510,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, unsigned long long update_interval=0; struct _region_stat_t region_counter[MAX_TABLE_NUM]; memset(region_counter, 0, sizeof(region_counter)); - struct table_runtime* table_aux=NULL; + struct Maat_table_runtime* table_rt=NULL; if(q_cnt==0) { return; @@ -1558,28 +1566,28 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, //update scanner's region cnt; for(i=0;itable_rt[i]; - if(table_aux==NULL) + table_rt=maat_scanner->table_rt[i]; + if(table_rt==NULL) { continue; } - switch(table_aux->table_type) + switch(table_rt->table_type) { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - table_aux->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt; - table_aux->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt; - assert(table_aux->expr.expr_rule_cnt>=0); - assert(table_aux->expr.regex_rule_cnt>=0); + table_rt->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt; + table_rt->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt; + assert(table_rt->expr.expr_rule_cnt>=0); + assert(table_rt->expr.regex_rule_cnt>=0); break; case TABLE_TYPE_IP: - table_aux->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt; - table_aux->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt; + table_rt->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt; + table_rt->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt; break; default: break; } - assert(table_aux->origin_rule_num>=0); + assert(table_rt->origin_rule_num>=0); } for(i=0;itable_rt[table_id]; + struct Maat_table_runtime* table_rt=maat_scanner->table_rt[table_id]; update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); for(i=0;ioperation==GIE_INSERT_OPT) { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } destroy_digest_rule(update_array[i]); update_array[i]=NULL; @@ -1877,7 +1885,7 @@ int get_district_id(_Maat_scanner_t *scanner,const char* district_str) } return district_id; } -int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger) { unsigned int i=0,j=0; char* p=NULL,*saveptr=NULL,*region_string=NULL; @@ -2141,7 +2149,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } return 0; } -int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; scan_rule_t* p_rule=NULL; @@ -2173,7 +2181,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; scan_rule_t* p_rule=NULL; @@ -2204,12 +2212,12 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; GIE_digest_t* digest_rule=NULL; struct _Maat_group_inner_t* u_para=NULL; - struct table_runtime * table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id]; int expr_id=0,district_id=-1; group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); @@ -2232,11 +2240,11 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db ,db_digest_rule->digest_string ,db_digest_rule->confidence_degree ,group_rule); - MESA_lqueue_join_tail(table_aux->similar.update_q, &digest_rule, sizeof(void*)); + MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); scanner->gie_total_q_size++; return 0; } -int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger) +int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger) { int i=0; unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; @@ -2300,7 +2308,7 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id, } return 0; } -int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; struct _Maat_compile_inner_t*compile_rule=NULL; @@ -2335,7 +2343,7 @@ int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_g return 0; } -void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) +void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_compile_inner_t*compile_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL; @@ -2370,7 +2378,7 @@ void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_ } return; } -int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_compile_inner_t *compile_rule=NULL; struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head); @@ -2402,7 +2410,7 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* return 0; } -int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) +int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_compile_inner_t *compile_rule=NULL; compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); @@ -2425,10 +2433,10 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* } return 1; } -void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) +void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) { struct db_group_rule_t db_group_rule; - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; ret=sscanf(table_line,"%d\t%d\t%d",&(db_group_rule.group_id) ,&(db_group_rule.compile_id) @@ -2447,7 +2455,7 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s //leave no trace when compatible_group_update calling if(table->table_type==TABLE_TYPE_GROUP) { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } else @@ -2466,14 +2474,14 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s //no need to free db_group_rule,it was saved in scanner->compile_hash if(table->table_type==TABLE_TYPE_GROUP) { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } } return; } -void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger) +void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger) { char virtual_group_line[256]; snprintf(virtual_group_line,sizeof(virtual_group_line), @@ -2481,12 +2489,12 @@ void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int update_group_rule(table, virtual_group_line,scanner,logger); return; } -void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; const struct expr_table_desc* expr_desc=&(table->expr); - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; switch(table->table_type) { case TABLE_TYPE_EXPR: @@ -2611,7 +2619,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } else @@ -2645,7 +2653,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st } else { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } @@ -2653,11 +2661,11 @@ error_out: free(maat_str_rule); maat_str_rule=NULL; } -void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40]; - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; unsigned short i_src_port,i_sport_mask,i_dst_port,i_dport_mask; int protocol=0,direction=0; int ret=0,rule_type=0; @@ -2772,7 +2780,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } else @@ -2788,7 +2796,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru } else { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } @@ -2797,10 +2805,10 @@ error_out: ip_rule=NULL; } -void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { struct db_intval_rule_t* intval_rule=ALLOC(struct db_intval_rule_t, 1); - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) ,&(intval_rule->group_id) @@ -2847,7 +2855,7 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } @@ -2863,7 +2871,7 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } @@ -2872,10 +2880,10 @@ error_out: intval_rule=NULL; } -void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) +void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) { struct compile_table_desc* compile_desc=&(table->compile); - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct db_compile_rule_t *p_compile=ALLOC(struct db_compile_rule_t, 1); struct _head_Maat_rule_t* p_m_rule=&(p_compile->m_rule_head); @@ -2932,7 +2940,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line if(p_compile->is_valid==FALSE) { ret=del_compile_rule(table,p_compile,scanner, logger); - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; goto no_save; } else @@ -2946,7 +2954,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line table->udpate_err_cnt++; goto no_save; } - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } return; @@ -2958,9 +2966,9 @@ no_save: return; } -void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct db_digest_rule_t* digest_rule=ALLOC(struct db_digest_rule_t, 1); int ret=0; char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'}; @@ -3025,7 +3033,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } @@ -3041,7 +3049,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } @@ -3162,12 +3170,12 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) q_cnt,bury_cnt); } } -void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger) +void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int i=0, ret=1; unsigned int len=strlen(table_line)+1; struct plugin_table_desc* plugin_desc=&(table->plugin); - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; char *p=NULL; char* copy=NULL; @@ -3204,7 +3212,7 @@ void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line } } - table_aux->plugin.acc_line_num++; + table_rt->plugin.acc_line_num++; if(plugin_desc->cb_plug_cnt>0) { @@ -3217,16 +3225,16 @@ void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line { p=ALLOC(char, len); memcpy(p,table_line,len); - table_aux->plugin.cache_size+=len; - dynamic_array_write(table_aux->plugin.cache_lines,table_aux->plugin.cache_line_num,p); - table_aux->plugin.cache_line_num++; + table_rt->plugin.cache_size+=len; + dynamic_array_write(table_rt->plugin.cache_lines,table_rt->plugin.cache_line_num,p); + table_rt->plugin.cache_line_num++; } } void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger) { void *tmp1=NULL,*tmp2=NULL; MESA_htable_handle tmp_map=NULL; - struct table_runtime* table_aux=NULL; + struct Maat_table_runtime* table_rt=NULL; int i=0; long q_cnt; GIE_create_para_t para; @@ -3252,23 +3260,23 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ ,scanner); for(i=0;itable_rt[i]; - if(table_aux==NULL) + table_rt=scanner->table_rt[i]; + if(table_rt==NULL) { continue; } - switch(table_aux->table_type) + switch(table_rt->table_type) { case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - q_cnt=MESA_lqueue_get_count(table_aux->similar.update_q); + q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); if(q_cnt==0) { continue; } - if(table_aux->similar.gie_handle==NULL) + if(table_rt->similar.gie_handle==NULL) { - if(table_aux->table_type==TABLE_TYPE_SIMILARITY) + if(table_rt->table_type==TABLE_TYPE_SIMILARITY) { para.ED_reexamine=1; para.format=GIE_INPUT_FORMAT_PLAIN; @@ -3278,10 +3286,10 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ para.ED_reexamine=0; para.format=GIE_INPUT_FORMAT_SFH; } - table_aux->similar.gie_handle=GIE_create(¶); + table_rt->similar.gie_handle=GIE_create(¶); } - digest_batch_update(table_aux->similar.gie_handle, - table_aux->similar.update_q, + digest_batch_update(table_rt->similar.gie_handle, + table_rt->similar.update_q, logger, scanner, i); @@ -3306,25 +3314,11 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ return; } -void clear_plugin_table_info(struct _plugin_table_info *cb_info) -{ - int i=0; - void *line=NULL; - for(i=0;icache_line_num;i++) - { - line=dynamic_array_read(cb_info->cache_lines,i); - free(line); - dynamic_array_write(cb_info->cache_lines,i,NULL); - } - cb_info->cache_line_num=0; - cb_info->cache_size=0; - cb_info->acc_line_num=0; - return; -} + void maat_start_cb(long long new_version,int update_type,void*u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; struct plugin_table_desc* plugin_desc=NULL; int i=0,j=0; feather->new_version=new_version; @@ -3369,14 +3363,14 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) long long scanner_rule_num(struct _Maat_scanner_t *scanner) { long long total=0; - struct table_runtime* table_aux=NULL; + struct Maat_table_runtime* table_rt=NULL; int i=0; for(i=0;itable_rt[i]; - if(table_aux!=NULL) + table_rt=scanner->table_rt[i]; + if(table_rt!=NULL) { - total+=table_aux->origin_rule_num; + total+=table_rt->origin_rule_num; } } return total; @@ -3384,7 +3378,7 @@ long long scanner_rule_num(struct _Maat_scanner_t *scanner) void maat_finish_cb(void* u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; struct plugin_table_desc* plugin_desc=NULL; long expr_wait_q_cnt=0; int i=0, j=0; @@ -3472,7 +3466,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) int ret=-1,i=0; int table_id=-1; _Maat_scanner_t* scanner=NULL; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; if(feather->update_tmp_scanner!=NULL) { scanner=feather->update_tmp_scanner; diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index cc0fcbf..de8fcc3 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -42,7 +42,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) { int value=0; int i=0,j=0,offset=0; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; char conj_table_name[(MAX_TABLE_NAME_LEN+1)*MAX_CONJUNCTION_TABLE_NUM]={0}; feather->stat_handle=FS_create_handle(); @@ -84,7 +84,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_CMD_LINE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_SPEED,"line_cmd/s"); feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule"); - feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex"); + feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"reg/v6"); feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream"); feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Bps"); if(feather->perf_on==1) @@ -149,7 +149,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) FS_start(feather->stat_handle); return; } -void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num) +void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num) { alignment_int64_array_add(p->scan_cnt,thread_num,1); alignment_int64_array_add(p->input_bytes,thread_num,scan_len); @@ -171,8 +171,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0; int i=0; time_t now; - struct _Maat_table_info_t* p_table=NULL; - struct table_runtime* table_aux=NULL; + struct Maat_table_desc* p_table=NULL; + struct Maat_table_runtime* table_rt=NULL; time(&now); active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); @@ -208,26 +208,26 @@ void maat_stat_output(struct _Maat_feather_t* feather) { continue; } - table_aux=feather->scanner->table_rt[i]; + table_rt=feather->scanner->table_rt[i]; switch(p_table->table_type) { case TABLE_TYPE_PLUGIN: - plugin_cache_num+=table_aux->plugin.cache_line_num; - plugin_acc_num+=table_aux->plugin.acc_line_num; + plugin_cache_num+=table_rt->plugin.cache_line_num; + plugin_acc_num+=table_rt->plugin.acc_line_num; break; case TABLE_TYPE_GROUP: - group_rule_num+=table_aux->origin_rule_num; + group_rule_num+=table_rt->origin_rule_num; break; case TABLE_TYPE_COMPILE: - compile_rule_num+=table_aux->origin_rule_num; + compile_rule_num+=table_rt->origin_rule_num; break; case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - table_regex_ipv6_num=table_aux->expr.regex_rule_cnt; + table_regex_ipv6_num=table_rt->expr.regex_rule_cnt; total_iconv_error=p_table->expr.iconv_err_cnt; break; case TABLE_TYPE_IP: - table_regex_ipv6_num=table_aux->ip.ipv6_rule_cnt; + table_regex_ipv6_num=table_rt->ip.ipv6_rule_cnt; break; default: break; @@ -242,8 +242,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_RULE_NUM], FS_OP_SET, - table_aux->origin_rule_num); - total_cfg_num+=table_aux->origin_rule_num; + table_rt->origin_rule_num); + total_cfg_num+=table_rt->origin_rule_num; FS_operate(feather->stat_handle, p_table->stat_line_id, @@ -252,8 +252,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_regex_ipv6_num); total_regex_num+=table_regex_ipv6_num; - table_stream_num=alignment_int64_array_sum(table_aux->stream_num,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->stream_num,feather->scan_thread_num); + table_stream_num=alignment_int64_array_sum(table_rt->stream_num,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->stream_num,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], @@ -262,8 +262,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) total_stream_cnt+= table_stream_num; - table_scan_cnt=alignment_int64_array_sum(table_aux->scan_cnt,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->scan_cnt,feather->scan_thread_num); + table_scan_cnt=alignment_int64_array_sum(table_rt->scan_cnt,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->scan_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], @@ -271,8 +271,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_scan_cnt); total_scan_cnt+=table_scan_cnt; - table_input_bytes=alignment_int64_array_sum(table_aux->input_bytes,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->input_bytes,feather->scan_thread_num); + table_input_bytes=alignment_int64_array_sum(table_rt->input_bytes,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->input_bytes,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], @@ -281,8 +281,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) total_input_bytes+=table_input_bytes; if(feather->perf_on==1) { - table_scan_cpu_time=alignment_int64_array_sum(table_aux->scan_cpu_time,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->scan_cpu_time,feather->scan_thread_num); + table_scan_cpu_time=alignment_int64_array_sum(table_rt->scan_cpu_time,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->scan_cpu_time,feather->scan_thread_num); table_scan_cpu_time/=1000; FS_operate(feather->stat_handle, p_table->stat_line_id, @@ -292,8 +292,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) total_cpu_time+=table_scan_cpu_time; } - table_hit_cnt=alignment_int64_array_sum(table_aux->hit_cnt,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->hit_cnt,feather->scan_thread_num); + table_hit_cnt=alignment_int64_array_sum(table_rt->hit_cnt,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->hit_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_HIT_CNT], diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 1ceffab..81618ac 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -164,6 +164,30 @@ char* str_unescape(char* s) s[j]='\0'; return s; } +int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len) +{ + const char* seps=" \t"; + char* saveptr=NULL, *subtoken=NULL, *str=NULL; + char* dup_line=_maat_strdup(line); + int i=0, ret=-1; + for (str = dup_line; ; str = NULL) + { + subtoken = strtok_r(str, seps, &saveptr); + if (subtoken == NULL) + break; + if(i==column_seq-1) + { + *offset=subtoken-dup_line; + *len=strlen(subtoken); + ret=0 + break; + } + i++; + } + free(dup_line); + return ret; +} + #define MAX_SYSTEM_CMD_LEN 512 int system_cmd_mkdir(const char* path) diff --git a/src/entry/dynamic_array.cpp b/src/entry/dynamic_array.cpp index 26c414e..46b7159 100644 --- a/src/entry/dynamic_array.cpp +++ b/src/entry/dynamic_array.cpp @@ -2,11 +2,11 @@ #include #include int dynamic_array_VERSION_20141202=0; -struct dynamic_array_t* dynamic_array_create(int size,int step) +struct dynamic_array_t* dynamic_array_create(long long init_size, long long step) { struct dynamic_array_t* d_array=(struct dynamic_array_t*)calloc(sizeof(struct dynamic_array_t),1); - d_array->array=(void**)calloc(sizeof(void*),size); - d_array->size=size; + d_array->array=(void**)calloc(sizeof(void*),init_size); + d_array->size=init_size; d_array->enlarge_step=step; return d_array; } @@ -23,7 +23,7 @@ void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(vo free(d_array->array); free(d_array); } -void* dynamic_array_read(struct dynamic_array_t* d_array,int i) +void* dynamic_array_read(struct dynamic_array_t* d_array, long long i) { if(isize) { @@ -34,7 +34,7 @@ void* dynamic_array_read(struct dynamic_array_t* d_array,int i) return NULL; } } -void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data) +void dynamic_array_write(struct dynamic_array_t* d_array, long long i, void* data) { int new_size=0; if(isize) diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index f123134..2c78e6a 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -123,18 +123,8 @@ struct plugin_table_ex_data_desc Maat_plugin_EX_key2index_func_t* key2index_func; long argl; void *argp; + MESA_htable_handle key2ex_hash; }; -struct _plugin_table_info -{ - int cb_plug_cnt; - struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; - dynamic_array_t *cache_lines; - int cache_line_num; - int acc_line_num; - int update_type; - long cache_size; -}; - struct plugin_table_desc { int key_column; @@ -143,9 +133,9 @@ struct plugin_table_desc int n_foreign; int foreign_columns[MAX_FOREIGN_CLMN_NUM]; int cb_plug_cnt; + long long estimate_size; struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; struct plugin_table_ex_data_desc ex_desc; - int acc_line_num; }; struct expr_table_desc { @@ -163,7 +153,7 @@ struct ip_table_desc int ipv6_rule_cnt; }; -struct _Maat_table_info_t +struct Maat_table_desc { unsigned short table_id; unsigned short conj_cnt; @@ -317,7 +307,7 @@ struct _Maat_compile_inner_t dynamic_array_t *groups; int is_valid; int compile_id;//equal to db_c_rule->m_rule.config_id - const struct _Maat_table_info_t* ref_table; + const struct Maat_table_desc* ref_table; int group_boundary; int group_cnt; MAAT_RULE_EX_DATA* ads; @@ -390,7 +380,7 @@ struct plugin_runtime long long cache_line_num; long long acc_line_num; long long cache_size; - MESA_htable_handle ex_data_hash; + MESA_htable_handle key2ex_hash; }; struct expr_runtime { @@ -403,7 +393,7 @@ struct ip_runtime long long ipv6_rule_cnt; }; -struct table_runtime +struct Maat_table_runtime { enum MAAT_TABLE_TYPE table_type; long origin_rule_num; @@ -434,7 +424,7 @@ struct _Maat_scanner_t rule_scanner_t region; long gie_total_q_size; - struct table_runtime* table_rt[MAX_TABLE_NUM]; + struct Maat_table_runtime* table_rt[MAX_TABLE_NUM]; /* struct similar_runtime gie_aux[MAX_TABLE_NUM]; struct plugin_runtime plugin_aux[MAX_TABLE_NUM]; @@ -506,7 +496,7 @@ struct _Maat_feather_t int cumulative_update_off; int stat_on; int perf_on; - struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM]; + struct Maat_table_desc *p_table_info[MAX_TABLE_NUM]; MESA_htable_handle map_tablename2id; void* logger; long long maat_version; @@ -605,7 +595,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); -int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); +int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); @@ -617,7 +607,7 @@ int HASH_add_by_id(MESA_htable_handle hash,int id,void*data); int HASH_delete_by_id(MESA_htable_handle hash,int id); void maat_read_full_config(_Maat_feather_t* _feather); void maat_stat_init(struct _Maat_feather_t* feather); -void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num); +void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_output(struct _Maat_feather_t* feather); redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 2843db1..23e22ec 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -47,4 +47,6 @@ int system_cmd_rm(const char* src_file); int system_cmd_mv(const char* src_file,const char*dst_file); int system_cmd_cp(const char* src_file,const char*dst_file); char* md5_file(const char* filename, char* md5string); +int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); + diff --git a/src/inc_internal/dynamic_array.h b/src/inc_internal/dynamic_array.h index 7583c92..11b3643 100644 --- a/src/inc_internal/dynamic_array.h +++ b/src/inc_internal/dynamic_array.h @@ -3,11 +3,11 @@ struct dynamic_array_t { void ** array; - int size; - int enlarge_step; + long long size; + long long enlarge_step; }; -struct dynamic_array_t* dynamic_array_create(int size,int step); +struct dynamic_array_t* dynamic_array_create(long long init_size, long long step); void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(void *)); -void* dynamic_array_read(struct dynamic_array_t* d_array,int i); -void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data); +void* dynamic_array_read(struct dynamic_array_t* d_array,long long i); +void dynamic_array_write(struct dynamic_array_t* d_array, long long i,void* data); #endif //_DYNAMIC_ARRAY_H_INCLUDE_ diff --git a/test/table_info.conf b/test/table_info.conf index a920f3c..4b03043 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -28,5 +28,5 @@ 9 SIM_URL similar -- 10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff 11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} -- -12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":"6,8","tag":3} -- +12 TEST_FOREIGN_KEY plugin {"key":2,"valid":4,"foreign":[6,8],"tag":3, "estimate_size":4096} -- 13 COMPILE_ALIAS compile escape -- From 899a61e37116301fa7aef475bdf763a2bf74a9cd Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 5 Dec 2018 18:00:55 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=8D=95=E5=85=83?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_rule.h | 6 +- src/entry/Maat_api.cpp | 176 +++++++++++++++++++--- src/entry/Maat_command.cpp | 7 +- src/entry/Maat_rule.cpp | 94 +++++++----- src/entry/Maat_utils.cpp | 2 +- src/entry/json2iris.cpp | 1 + src/inc_internal/Maat_rule_internal.h | 172 +-------------------- src/inc_internal/Maat_table_description.h | 113 ++++++++++++++ src/inc_internal/Maat_utils.h | 27 +++- test/table_info.conf | 3 +- test/test_maatframe.cpp | 106 ++++++++++++- 11 files changed, 466 insertions(+), 241 deletions(-) create mode 100644 src/inc_internal/Maat_table_description.h diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 83c35f8..d5e4621 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -259,8 +259,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa //Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table. typedef void* MAAT_PLUGIN_EX_DATA; -typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, size_t key_sz, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); -typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, size_t key_sz, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_free_func_t(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp); typedef int Maat_plugin_EX_key2index_func_t(const char* key); @@ -271,7 +271,7 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_key2index_func_t* key2index_func, long argl, void *argp); //Data is duplicated by dup_func of Maat_plugin_EX_register, caller is responsible to free the data. -MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key, int thread_num); +MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key); enum MAAT_RULE_OPT { diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 18d22b8..803ee46 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -443,7 +443,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* return NULL; } _Maat_feather_t* feather=ALLOC(struct _Maat_feather_t, 1); - feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger); + feather->table_cnt=read_table_description(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger); feather->map_tablename2id=map_create(); int i=0,j=0,ret=0; for(i=0;iscanner->table_rt[table_id]->plugin); //plugin table register blocks background update. pthread_mutex_lock(&(_feather->backgroud_update_mutex)); idx=plugin_desc->cb_plug_cnt; @@ -944,6 +943,12 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, plugin_desc->cb_plug[idx].update=update; plugin_desc->cb_plug[idx].finish=finish; plugin_desc->cb_plug[idx].u_para=u_para; + if(_feather->scanner==NULL) + { + pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); + return 1; + } + plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin); if(plugin_aux->cache_line_num>0) { if(start!=NULL) @@ -1051,46 +1056,128 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa pthread_rwlock_unlock(&(compile_inner->rwlock)); return ad; } -MESA_htable_handle plugin_EX_htable_new(const struct plugin_table_desc* plugin_desc, - struct dynamic_array_t* lines, size_t line_cnt) +struct wrap_plugin_EX_data +{ + MAAT_RULE_EX_DATA exdata; + const struct Maat_table_desc* ref_plugin_table; +}; +void wrap_plugin_EX_data_free(void *data) +{ + struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data; + const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc); + ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp); + wrap_data->ref_plugin_table=NULL; + free(wrap_data); + return; +} +MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index) { MESA_htable_handle key2ex_hash=NULL; unsigned int slot_size=1; - const char* line=NULL; - size_t key_offset=0, key_len=0; - long long estimate_size=plugin_desc->estimate_size, i=0; - int ret=0; - MAAT_RULE_EX_DATA exdata=NULL; - while(estimate_size>0) + while(estimate_size!=0) { - estimate_size=estimate_size<<1; + estimate_size=estimate_size>>1; slot_size*=2; } if(slot_size==1) { slot_size=4096; } + MESA_htable_create_args_t hargs; memset(&hargs,0,sizeof(hargs)); - hargs.thread_safe=1; + hargs.thread_safe=0; hargs.hash_slot_size = slot_size; hargs.max_elem_num = 0; hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; hargs.expire_time = 0; hargs.key_comp = NULL; - hargs.key2index = plugin_desc->ex_desc.key2index_func; + hargs.key2index = NULL; //Not supported yet. hargs.recursive = 1; - hargs.data_free = NULL ; + hargs.data_free = wrap_plugin_EX_data_free; hargs.data_expire_with_condition = NULL; key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(key2ex_hash, 0); + return key2ex_hash; +} + +int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + size_t key_offset=0, key_len=0; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data del error: no such key %.*s of %s", + key_len, line+key_offset, line); + return -1; + } + return 0; +} + +int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, + MESA_htable_handle key2ex_hash, void *logger) +{ + char* key=NULL; + size_t key_offset=0, key_len=0; + MAAT_RULE_EX_DATA exdata=NULL; + struct wrap_plugin_EX_data* wrap_data=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + int ret=0; + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: cannot find column %d of %s", + plugin_desc->key_column, line); + return -1; + } + key=ALLOC(char, key_len+1); + memcpy(key, line+key_offset, key_len); + plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + wrap_data=ALLOC(struct wrap_plugin_EX_data, 1); + wrap_data->exdata=exdata; + wrap_data->ref_plugin_table=plugin_table; + ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data); + free(key); + if(ret<0) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Plugin EX data add error: duplicated key %.*s of %s", + key_len, line+key_offset, line); + wrap_plugin_EX_data_free(wrap_data); + return -1; + } + return 0; +} +MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table, + struct dynamic_array_t* lines, size_t line_cnt, void* logger) +{ + MESA_htable_handle key2ex_hash=NULL; + size_t i=0; + const char* line=NULL; + const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin); + + key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func); + for(i=0; i< line_cnt; i++) { line=(const char*)dynamic_array_read(lines, i); - ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); - exdata=plugin_desc->ex_desc.new_func(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);; - MESA_htable_add(key2ex_hash, line+key_offset, key_len, const void * data) + plugin_EX_data_new(plugin_table, line, key2ex_hash, logger); } + return key2ex_hash; } int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_new_func_t* new_func, @@ -1103,26 +1190,71 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; struct plugin_table_desc* plugin_desc=&(table_desc->plugin); - struct Maat_table_runtime* table_rt=_feather->scanner->table_rt[table_id]; - MESA_htable_handle key2ex_hash=NULL; - if(table_desc->table_type!=TABLE_TYPE_PLUGIN || new_func==NULL || free_func==NULL || dup_func==NULL) + struct Maat_table_runtime* table_rt=NULL; + if(new_func==NULL || free_func==NULL || dup_func==NULL ) { assert(0); + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, "%s failed: invalid paramter", __FUNCTION__); + return -1; + } + if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata + || plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0) + { + assert(0); + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_module, + "%s failed: key or valid flag column are not specified", __FUNCTION__); return -1; } plugin_desc->ex_desc.new_func=new_func; plugin_desc->ex_desc.free_func=free_func; plugin_desc->ex_desc.dup_func=dup_func; - plugin_desc->ex_desc.key2index_func=key2index_func; + plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used. plugin_desc->ex_desc.argl=argl; plugin_desc->ex_desc.argp=argp; + plugin_desc->have_exdata=1; + if(_feather->scanner==NULL) + { + return 0; + } pthread_mutex_lock(&(_feather->backgroud_update_mutex)); - table_rt->plugin.key2ex_hash=plugin_EX_htable_new(plugin_desc,table_rt->plugin.cache_lines, table_rt->plugin.cache_line_num); + table_rt=_feather->scanner->table_rt[table_id]; + assert(table_rt->plugin.key2ex_hash==NULL); + table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, + table_rt->plugin.cache_line_num, _feather->logger); pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); return 0; } +MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id, const char* key) +{ + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; + struct Maat_table_runtime *table_rt= NULL; + struct plugin_table_desc* plugin_desc=&(table_desc->plugin); + struct wrap_plugin_EX_data* wrap_data=NULL; + MAAT_RULE_EX_DATA exdata=NULL; + if(table_desc->table_type!=TABLE_TYPE_PLUGIN || plugin_desc->have_exdata==0) + { + assert(0); + return NULL; + } + if(_feather->scanner==NULL) + { + return NULL; + } + table_rt= _feather->scanner->table_rt[table_id]; + pthread_rwlock_rdlock(&table_rt->plugin.rwlock); + wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, + (const unsigned char*)key, strlen(key)); + if(wrap_data!=NULL) + { + plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata), + plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); + } + pthread_rwlock_unlock(&table_rt->plugin.rwlock); + return exdata; +} int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id ,enum MAAT_CHARSET charset,const char* data,int data_len diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index c8e09a4..8b6f188 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -221,7 +221,7 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_ assert(0); } - ret=get_column_pos(const char* line, int column_seq, &offset, &len); + ret=get_column_pos(line, column_seq, &offset, &len); if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer. { return -1; @@ -2131,6 +2131,11 @@ int Maat_cmd_set_file(Maat_feather_t feather,const char* key, const char* value, { struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather; redisContext* ctx=_feather->mr_ctx.write_ctx; + if(ctx==NULL) + { + MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command, "%s failed: Redis is not connected.", __FUNCTION__); + return -1; + } const char *arg_vec[3]; size_t len_vec[3]; arg_vec[0] = "SET"; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 1178912..e66a4f1 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -529,7 +529,7 @@ int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_h char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; char *token=NULL,*sub_token=NULL,*saveptr; struct expr_table_desc* p=&(table->expr); - sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) + sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) ,table->table_name[0] ,table_type ,src_charset @@ -602,31 +602,21 @@ int _read_integer_arrary(char* string, int *array, int size) return i; } -int read_plugin_table_info(const char* line, struct Maat_table_desc* p) +int read_plugin_table_description(const char* line, struct Maat_table_desc* p) { int i=0,ret=0; - + size_t offset=0, len=0; cJSON* json=NULL, *tmp=NULL, *array_item=NULL; char* copy_line=NULL, *plug_info=NULL; - char *token=NULL,*sub_token=NULL,*saveptr; struct plugin_table_desc* plugin_desc=&(p->plugin); - plugin_desc->valid_flag_column=-1; copy_line=_maat_strdup(line); - for (token = copy_line, i=0; ; token= NULL, i++) - { - sub_token= strtok_r(token,"\t ", &saveptr); - if (sub_token == NULL) - break; - if(i==3) - { - break; - } - } - if(i<3) + ret=get_column_pos(copy_line, 4, &offset, &len); + if(i<0) { goto error_out; } - plug_info=sub_token; + copy_line[offset+len+1]='\0'; + plug_info=copy_line+offset; if(strlen(plug_info)<4)//For old version compatible. { @@ -693,7 +683,7 @@ error_out: free(copy_line); return -1; } -int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) +int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) { FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; @@ -750,7 +740,7 @@ int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* ta } p=table_info_new(max_thread_num); - ret=sscanf(line,"%hu\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) + ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) ,p->table_name[0] ,table_type_str ,not_care); @@ -781,7 +771,7 @@ int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* ta } break; case TABLE_TYPE_PLUGIN: - ret=read_plugin_table_info(line, p); + ret=read_plugin_table_description(line, p); if(ret<0) { fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i); @@ -1193,29 +1183,36 @@ void destroy_digest_rule(GIE_digest_t*rule) } -struct Maat_table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num) +struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num) { - struct Maat_table_runtime* p= ALLOC(struct Maat_table_runtime, 1); - p->table_type=type; - switch(type) + + struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1); + table_rt->table_type=table_desc->table_type; + switch(table_desc->table_type) { case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - p->similar.update_q=MESA_lqueue_create(0,0); + table_rt->similar.update_q=MESA_lqueue_create(0,0); break; case TABLE_TYPE_PLUGIN: - p->plugin.cache_lines=dynamic_array_create(1, 1024); + table_rt->plugin.cache_lines=dynamic_array_create(1, 1024); + if(table_desc->plugin.have_exdata) + { + table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, + table_desc->plugin.ex_desc.key2index_func); + pthread_rwlock_init(&(table_rt->plugin.rwlock), NULL); + } break; default: break; } - p->scan_cnt=alignment_int64_array_alloc(max_thread_num); - p->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); - p->input_bytes=alignment_int64_array_alloc(max_thread_num); - p->stream_num=alignment_int64_array_alloc(max_thread_num); - p->hit_cnt=alignment_int64_array_alloc(max_thread_num); - return p; + table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); + table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); + table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); + table_rt->stream_num=alignment_int64_array_alloc(max_thread_num); + table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num); + return table_rt; } void table_runtime_free(struct Maat_table_runtime* p) { @@ -1344,7 +1341,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t { continue; } - table_rt=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num); + table_rt=table_runtime_new(pp_table_desc[i], feather->scan_thread_num); if(pp_table_desc[i]->table_type==TABLE_TYPE_EXPR||pp_table_desc[i]->table_type==TABLE_TYPE_EXPR_PLUS) { expr_desc=&(pp_table_desc[i]->expr); @@ -3178,7 +3175,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; char *p=NULL; char* copy=NULL; - + size_t is_valid_offset=0, valid_len=0; char *token=NULL,*sub_token=NULL,*saveptr; if(plugin_desc->rule_tag_column>0&&n_tags>0) { @@ -3213,13 +3210,32 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M } table_rt->plugin.acc_line_num++; - - if(plugin_desc->cb_plug_cnt>0) + if(plugin_desc->have_exdata || plugin_desc->cb_plug_cnt>0) { - for(i=0;icb_plug_cnt;i++) + + if(plugin_desc->have_exdata) { - plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para); + ret=get_column_pos(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len); + + pthread_rwlock_wrlock(&(table_rt->plugin.rwlock)); + if(atoi(table_line+is_valid_offset)==1) + { + plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger); + } + else + { + plugin_EX_data_free(table, table_line, table_rt->plugin.key2ex_hash, logger); + } + pthread_rwlock_unlock(&(table_rt->plugin.rwlock)); } + if(plugin_desc->cb_plug_cnt>0) + { + for(i=0;icb_plug_cnt;i++) + { + plugin_desc->cb_plug[i].update(table->table_id,table_line,plugin_desc->cb_plug[i].u_para); + } + } + } else { @@ -3227,7 +3243,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M memcpy(p,table_line,len); table_rt->plugin.cache_size+=len; dynamic_array_write(table_rt->plugin.cache_lines,table_rt->plugin.cache_line_num,p); - table_rt->plugin.cache_line_num++; + table_rt->plugin.cache_line_num++; } } void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger) diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 81618ac..36411fa 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -179,7 +179,7 @@ int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len { *offset=subtoken-dup_line; *len=strlen(subtoken); - ret=0 + ret=0; break; } i++; diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 60b474c..a81994e 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -12,6 +12,7 @@ #include "cJSON.h" #include "hiredis.h" #include "map_str2int.h" +#include "Maat_table_description.h" #include "Maat_rule_internal.h" #include "Maat_utils.h" diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 2c78e6a..dfe551f 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -3,6 +3,7 @@ #include "Maat_rule.h" #include "Maat_command.h" +#include "Maat_table_description.h" #include #include @@ -22,185 +23,24 @@ extern const char *maat_module; -#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) -#define atomic_inc(x) __sync_add_and_fetch((x),1) -#define atomic_dec(x) __sync_sub_and_fetch((x),1) -#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) -#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) -typedef int atomic_t; -#define ATOMIC_INIT(i) { (i) } -#define atomic_read(x) __sync_add_and_fetch((x),0) -#define atomic_set(x,y) __sync_lock_test_and_set((x),y) -#else -#include -#endif -#define TRUE 1 -#define FALSE 0 + #define MAX_TABLE_NUM 256 -#define MAX_CONJUNCTION_TABLE_NUM 8 -#define MAX_CHARSET_NUM 16 -#define MAX_TABLE_NAME_LEN 256 #define MAX_TABLE_LINE_SIZE (1024*16) #define MAX_EXPR_KEYLEN 1024 #define MAX_DISTRICT_LEN 64 -#define MAX_PLUGIN_PER_TABLE 32 -#define MAX_FOREIGN_CLMN_NUM 8 + #define MAX_SCANNER_HIT_NUM 64 -#define MAX_COMPILE_EX_DATA_NUM 2 #define MAX_GROUP_CACHE 128 #define MAX_FAILED_NUM 128 #define MAX_MAAT_STAT_NUM 64 -#ifndef MAX -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -#endif -#ifndef MIN -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -#endif - -#ifndef offsetof -#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) -#endif - -#ifndef container_of -#define container_of(ptr, type, member) ({ \ - const typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) -#endif typedef void* rule_scanner_t; -enum MAAT_TABLE_TYPE -{ - TABLE_TYPE_EXPR=0, - TABLE_TYPE_IP, - TABLE_TYPE_INTERVAL, - TABLE_TYPE_DIGEST, - TABLE_TYPE_EXPR_PLUS, - TABLE_TYPE_SIMILARITY, - TABLE_TYPE_GROUP, - TABLE_TYPE_COMPILE, - TABLE_TYPE_PLUGIN -}; -enum USER_REGION_ENCODE -{ - USER_REGION_ENCODE_NONE=0, - USER_REGION_ENCODE_ESCAPE, - USER_REGION_ENCODE_BASE64 -}; -struct compile_ex_data_idx -{ - Maat_rule_EX_new_func_t *new_func; - Maat_rule_EX_free_func_t* free_func; - Maat_rule_EX_dup_func_t* dup_func; - long argl; - void *argp; - int idx; - int table_id; -}; -struct compile_table_desc -{ - enum USER_REGION_ENCODE user_region_encoding; - int ex_data_num; - struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; -}; -struct plugin_table_callback_desc -{ - Maat_start_callback_t *start; - Maat_update_callback_t *update; - Maat_finish_callback_t *finish; - void* u_para; -}; -struct plugin_table_ex_data_desc -{ - Maat_plugin_EX_new_func_t* new_func; - Maat_plugin_EX_free_func_t* free_func; - Maat_plugin_EX_dup_func_t* dup_func; - Maat_plugin_EX_key2index_func_t* key2index_func; - long argl; - void *argp; - MESA_htable_handle key2ex_hash; -}; -struct plugin_table_desc -{ - int key_column; - int valid_flag_column; - int rule_tag_column; - int n_foreign; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; - int cb_plug_cnt; - long long estimate_size; - struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; - struct plugin_table_ex_data_desc ex_desc; -}; -struct expr_table_desc -{ - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cross_cache_size; - int quick_expr_switch; - long long iconv_err_cnt; -}; -struct ip_table_desc -{ - int ipv4_rule_cnt; - int ipv6_rule_cnt; -}; - -struct Maat_table_desc -{ - unsigned short table_id; - unsigned short conj_cnt; - unsigned short updating_name; - char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; - enum MAAT_TABLE_TYPE table_type; - union - { - struct compile_table_desc compile; - struct expr_table_desc expr; - struct ip_table_desc ip; - struct plugin_table_desc plugin; - void* others;//group, interval and digest don't have sperate description info. - }; -/* - enum MAAT_CHARSET src_charset; - enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; - int src_charset_in_dst; - int do_charset_merge; - int cross_cache_size; - int quick_expr_switch; - union - { - int expr_rule_cnt; //expr_type=0,1,3 - int ipv4_rule_cnt; - }; - union - { - int regex_rule_cnt; //expr_type=2 - int ipv6_rule_cnt; - }; - struct _plugin_table_info *cb_info; - int valid_flag_column; //for plugin table - int rule_tag_column; //for plugin table; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; //for plugin table; - int n_foreign; - - //for compile table - enum USER_REGION_ENCODE user_region_encoding; - int ex_data_num; - struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; -*/ -//for stat>>>>>>>> - unsigned long long udpate_err_cnt; - unsigned long long unmatch_tag_cnt; - int stat_line_id; -}; struct db_str_rule_t { @@ -380,6 +220,7 @@ struct plugin_runtime long long cache_line_num; long long acc_line_num; long long cache_size; + pthread_rwlock_t rwlock; MESA_htable_handle key2ex_hash; }; struct expr_runtime @@ -595,7 +436,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); -int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); +int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); @@ -620,6 +461,9 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p); void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len); MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); +MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index); +int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); +int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout); diff --git a/src/inc_internal/Maat_table_description.h b/src/inc_internal/Maat_table_description.h new file mode 100644 index 0000000..deb7629 --- /dev/null +++ b/src/inc_internal/Maat_table_description.h @@ -0,0 +1,113 @@ +#pragma once +#include "Maat_rule.h" +#define MAX_COMPILE_EX_DATA_NUM 2 +#define MAX_FOREIGN_CLMN_NUM 8 +#define MAX_PLUGIN_PER_TABLE 32 +#define MAX_CHARSET_NUM 16 +#define MAX_CONJUNCTION_TABLE_NUM 8 +#define MAX_TABLE_NAME_LEN 256 + +enum USER_REGION_ENCODE +{ + USER_REGION_ENCODE_NONE=0, + USER_REGION_ENCODE_ESCAPE, + USER_REGION_ENCODE_BASE64 +}; + +enum MAAT_TABLE_TYPE +{ + TABLE_TYPE_EXPR=0, + TABLE_TYPE_IP, + TABLE_TYPE_INTERVAL, + TABLE_TYPE_DIGEST, + TABLE_TYPE_EXPR_PLUS, + TABLE_TYPE_SIMILARITY, + TABLE_TYPE_GROUP, + TABLE_TYPE_COMPILE, + TABLE_TYPE_PLUGIN + +}; + +struct compile_ex_data_idx +{ + Maat_rule_EX_new_func_t *new_func; + Maat_rule_EX_free_func_t* free_func; + Maat_rule_EX_dup_func_t* dup_func; + long argl; + void *argp; + int idx; + int table_id; +}; +struct compile_table_desc +{ + enum USER_REGION_ENCODE user_region_encoding; + int ex_data_num; + struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; +}; + +struct plugin_table_callback_desc +{ + Maat_start_callback_t *start; + Maat_update_callback_t *update; + Maat_finish_callback_t *finish; + void* u_para; +}; +struct plugin_table_ex_data_desc +{ + Maat_plugin_EX_new_func_t* new_func; + Maat_plugin_EX_free_func_t* free_func; + Maat_plugin_EX_dup_func_t* dup_func; + Maat_plugin_EX_key2index_func_t* key2index_func; + long argl; + void *argp; +}; +struct plugin_table_desc +{ + int key_column; + int valid_flag_column; + int rule_tag_column; + int n_foreign; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; + int cb_plug_cnt; + int have_exdata; + long long estimate_size; + struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; + struct plugin_table_ex_data_desc ex_desc; +}; +struct expr_table_desc +{ + enum MAAT_CHARSET src_charset; + enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; + int src_charset_in_dst; + int do_charset_merge; + int cross_cache_size; + int quick_expr_switch; + long long iconv_err_cnt; +}; +struct ip_table_desc +{ + int ipv4_rule_cnt; + int ipv6_rule_cnt; +}; + +struct Maat_table_desc +{ + int table_id; + int conj_cnt; + int updating_name; + char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN]; + enum MAAT_TABLE_TYPE table_type; + union + { + struct compile_table_desc compile; + struct expr_table_desc expr; + struct ip_table_desc ip; + struct plugin_table_desc plugin; + void* others;//group, interval and digest don't have sperate description info. + }; +//for stat>>>>>>>> + unsigned long long udpate_err_cnt; + unsigned long long unmatch_tag_cnt; + int stat_line_id; +}; + diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 23e22ec..e8d7114 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -9,9 +9,20 @@ #define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) #define FREE(p) {free(*p);*p=NULL;} -#define ATOMIC_INC(x) __atomic_fetch_add(x,1,__ATOMIC_RELAXED) -#define ATOMIC_READ(x) __atomic_fetch_add(x,0,__ATOMIC_RELAXED) - +#if(__GNUC__ * 100 + __GNUC_MINOR__ * 10 + __GNUC_PATCHLEVEL__ >= 411) +#define atomic_inc(x) __sync_add_and_fetch((x),1) +#define atomic_dec(x) __sync_sub_and_fetch((x),1) +#define atomic_add(x,y) __sync_add_and_fetch((x),(y)) +#define atomic_sub(x,y) __sync_sub_and_fetch((x),(y)) + typedef int atomic_t; +#define ATOMIC_INIT(i) { (i) } +#define atomic_read(x) __sync_add_and_fetch((x),0) +#define atomic_set(x,y) __sync_lock_test_and_set((x),y) +#else +#include +#endif +#define TRUE 1 +#define FALSE 0 #ifndef MAX #define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -21,6 +32,16 @@ #define MIN(a, b) (((a) < (b)) ? (a) : (b)) #endif +#ifndef offsetof +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif + +#ifndef container_of +#define container_of(ptr, type, member) ({ \ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) );}) +#endif + #define UNUSED __attribute__((unused)) const char* module_name_str(const char*name); #define maat_module (module_name_str("MAAT_Frame")) diff --git a/test/table_info.conf b/test/table_info.conf index 4b03043..55b0825 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -28,5 +28,6 @@ 9 SIM_URL similar -- 10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff 11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} -- -12 TEST_FOREIGN_KEY plugin {"key":2,"valid":4,"foreign":[6,8],"tag":3, "estimate_size":4096} -- +12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":[6,8],"tag":3} -- 13 COMPILE_ALIAS compile escape -- +14 TEST_PLUGIN_EXDATA_TABLE plugin {"key":2,"valid":4,"tag":5,"estimate_size":1024} -- diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 41b6b5e..4a5c084 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -640,7 +640,7 @@ struct rule_ex_param pthread_mutex_t lock; }; -void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, +void compile_ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) { int *counter=(int*)argp; @@ -657,7 +657,7 @@ void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_l *ad=param; return; } -void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) +void compile_ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) { if(*ad==NULL) { @@ -674,7 +674,7 @@ void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_ free(param); return; } -void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp) +void compile_ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp) { struct rule_ex_param* from_param=*((struct rule_ex_param**)from); pthread_mutex_lock(&(from_param->lock)); @@ -686,9 +686,9 @@ void ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long -TEST(EX_DATA_INDEX, all) +TEST(EX_DATA_INDEX, MaatRuleEXData) { -#define EX_data_index +#define rule_EX_data_index int ret=0; int table_id=0, ex_data_counter=0; @@ -701,7 +701,7 @@ TEST(EX_DATA_INDEX, all) ASSERT_GT(table_id, 0); int ex_param_idx=Maat_rule_get_ex_new_index(g_feather, "COMPILE_ALIAS", - ex_param_new, ex_param_free, ex_param_dup, + compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, 0, &ex_data_counter); ASSERT_TRUE(ex_param_idx>=0); EXPECT_EQ(ex_data_counter, 1); @@ -716,7 +716,7 @@ TEST(EX_DATA_INDEX, all) struct rule_ex_param* param=(struct rule_ex_param*)ex_data; EXPECT_EQ(param->id, 7799); EXPECT_EQ(strcmp(param->name, expect_name),0); - ex_param_free(0, NULL, NULL, &ex_data, 0, NULL); + compile_ex_param_free(0, NULL, NULL, &ex_data, 0, NULL); Maat_clean_status(&mid); return; } @@ -1418,6 +1418,98 @@ TEST_F(MaatCmdTest, SetFile) return; } +struct user_info +{ + char name[256]; + char ip_addr[32]; + int id; + int ref_cnt; +}; +void plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + int *counter=(int *)argp; + struct user_info* u=(struct user_info*)calloc(sizeof(struct user_info), 1); + int valid=0, tag=0, ret; + ret=sscanf(table_line, "%d\t%s\t%s%d\t%d", &(u->id), u->ip_addr, u->name, &valid, &tag); + EXPECT_EQ(ret, 5); + u->ref_cnt=1; + *ad=u; + (*counter)++; + return; +} +void plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) +{ + struct user_info* u=(struct user_info*)(*ad); + u->ref_cnt--; + if(u->ref_cnt>0) return; + free(u); + *ad=NULL; +} +void plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) +{ + struct user_info* u=(struct user_info*)(*from); + u->ref_cnt++; + *to=u; +} + +TEST_F(MaatCmdTest, PluginEXData) +{ +#define plugin_EX_data_index + Maat_feather_t feather=MaatCmdTest::_shared_feather; + + int ret=0, i=0; + int table_id=0, ex_data_counter=0; + const char* table_name="TEST_PLUGIN_EXDATA_TABLE"; + + + const int TEST_CMD_LINE_NUM=4; + const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM]; + struct Maat_line_t line_rule[TEST_CMD_LINE_NUM]; + const char* table_line[TEST_CMD_LINE_NUM]={"1\t192.168.0.1\tmahuateng\t1\t0", + "2\t192.168.0.2\tliuqiangdong\t1\t0", + "3\t192.168.0.3\tmayun\t1\t0", + "4\t192.168.0.4\tliyanhong\t1\t0"}; + table_id=Maat_table_register(feather, table_name); + ASSERT_GT(table_id, 0); + memset(&line_rule,0,sizeof(line_rule)); + for(i=0;i=0); + EXPECT_EQ(ex_data_counter, TEST_CMD_LINE_NUM); + + struct user_info* uinfo=NULL; + uinfo=(struct user_info*)Maat_plugin_get_EX_data(feather, table_id, "192.168.0.2"); + ASSERT_TRUE(uinfo!=NULL); + EXPECT_EQ(0, strcmp(uinfo->name, "liuqiangdong")); + EXPECT_EQ(uinfo->id, 2); + plugin_EX_free_cb(table_id, (void**)&uinfo, 0, NULL); + + ret=Maat_cmd_set_lines(feather, p_line+1,1, MAAT_OP_DEL); + EXPECT_GT(ret, 0); + + usleep(WAIT_FOR_EFFECTIVE_US); + uinfo=(struct user_info*)Maat_plugin_get_EX_data(feather, table_id, "192.168.0.2"); + ASSERT_TRUE(uinfo==NULL); + + return; +} int main(int argc, char ** argv) { From 953393b448a7d6719bfac6af2c2c818c2d7bb289 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 5 Dec 2018 18:04:09 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/CMakeLists.txt | 4 ++-- src/entry/Maat_rule.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f9846ec..20b6197 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,8 +1,8 @@ cmake_minimum_required(VERSION 3.5) set(MAAT_FRAME_MAJOR_VERSION 2) -set(MAAT_FRAME_MINOR_VERSION 4) -set(MAAT_FRAME_PATCH_VERSION 20181115) +set(MAAT_FRAME_MINOR_VERSION 5) +set(MAAT_FRAME_PATCH_VERSION 0) set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION}) message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index e66a4f1..3fe7c84 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -32,7 +32,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_4_20181202=1; +int MAAT_FRAME_VERSION_2_5_20181205=1; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; From a92e7b4253c503b500bd8ac9ea74bd78fb45c7cb Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 6 Dec 2018 21:11:51 +0600 Subject: [PATCH 6/8] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E5=86=99=E5=85=A520=E4=B8=87lines=E5=92=8Crule=E7=9A=84?= =?UTF-8?q?=E6=80=A7=E8=83=BD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_rule.h | 5 +- src/entry/Maat_api.cpp | 14 ++ src/entry/Maat_command.cpp | 18 +- src/inc_internal/Maat_rule_internal.h | 2 +- test/maat_demo.cpp | 19 +- test/test_maatframe.cpp | 265 ++++++++++++++++++-------- tools/maat_redis_tool.cpp | 6 +- 7 files changed, 224 insertions(+), 105 deletions(-) diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index d5e4621..0163650 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -165,8 +165,9 @@ enum MAAT_INIT_OPT int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size); enum MAAT_STATE_OPT { - MAAT_STATE_VERSION=1, //Get current maat version. VALUE is long long, SIZE=sizeof(long long). - MAAT_STATE_LAST_UPDATING_TABLE //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no + MAAT_STATE_VERSION=1, //Get current maat version, if maat is in update progress, the updating version is returned. VALUE is long long, SIZE=sizeof(long long). + MAAT_STATE_LAST_UPDATING_TABLE, //Query at Maat_finish_callback_t to determine whether this table is the last one to update. VALUE is interger, SIZE=sizeof(int), 1:yes, 0: no + MAAT_STATE_IN_UPDATING }; int Maat_read_state(Maat_feather_t feather, enum MAAT_STATE_OPT type, void* value, int size); diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 803ee46..a8ccf77 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -2260,6 +2260,20 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu case MAAT_STATE_LAST_UPDATING_TABLE: *int_val=_feather->is_last_plugin_table_updating; break; + case MAAT_STATE_IN_UPDATING: + if(size!=sizeof(int)) + { + return -1; + } + if(0==pthread_mutex_trylock(&(_feather->backgroud_update_mutex))) + { + *int_val=0; + pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); + } + else + { + *int_val=1; + } default: return -1; break; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 8b6f188..c3f156d 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -1213,21 +1213,21 @@ void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_ return; } -int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger) +int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule, unsigned int serial_rule_num, long long server_time, void* logger) { - int max_redis_batch=1*1024,batch_cnt=0; - int success_cnt=0,j=0,renew_allowed=0,last_failed=-1; + unsigned int max_redis_batch=1*1024, batch_cnt=0; + int renew_allowed=0,last_failed=-1; redisReply*transaction_reply=NULL,*p=NULL; unsigned int i=0; - unsigned int multi_cmd_cnt=0; + unsigned int multi_cmd_cnt=0, success_cnt=0; const int MAX_REDIS_OP_PER_SRULE=8; unsigned int max_multi_cmd_num=MAX_REDIS_OP_PER_SRULE*serial_rule_num+2;// 2 for operation in _exec_serial_rule_end() struct expected_reply_t *expected_reply=(struct expected_reply_t*)calloc(sizeof(struct expected_reply_t), max_multi_cmd_num); long long new_version=0; int renew_num=0,ret=0; - for(i=0;i<(unsigned int)serial_rule_num;i++) + for(i=0;i