From 7b5baacf623f672bfef550973830726f07cd48a3 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Tue, 4 Dec 2018 23:26:59 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=E5=8F=96column=E7=9A=84?= =?UTF-8?q?=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/Maat_rule.h | 4 +- src/entry/Maat_api.cpp | 176 +++++++++++++++------- src/entry/Maat_command.cpp | 46 +++--- src/entry/Maat_rule.cpp | 206 +++++++++++++------------- src/entry/Maat_stat.cpp | 48 +++--- src/entry/Maat_utils.cpp | 24 +++ src/entry/dynamic_array.cpp | 10 +- src/inc_internal/Maat_rule_internal.h | 30 ++-- src/inc_internal/Maat_utils.h | 2 + src/inc_internal/dynamic_array.h | 10 +- test/table_info.conf | 2 +- 11 files changed, 310 insertions(+), 248 deletions(-) diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 2e956a5..83c35f8 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -259,8 +259,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa //Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table. typedef void* MAAT_PLUGIN_EX_DATA; -typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); -typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, size_t key_sz, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); +typedef void Maat_plugin_EX_free_func_t(int table_id, const char* key, size_t key_sz, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp); typedef void Maat_plugin_EX_dup_func_t(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp); typedef int Maat_plugin_EX_key2index_func_t(const char* key); diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index dc24da0..18d22b8 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -16,9 +16,9 @@ #include "rulescan.h" #include "json2iris.h" -struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type) +struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int table_id,enum MAAT_TABLE_TYPE expect_type) { - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; if(table_id>MAX_TABLE_NUM) { return NULL; @@ -918,7 +918,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; int idx=0,i=0; - struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; + struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; struct plugin_table_desc *plugin_desc=&(p_table->plugin); struct plugin_runtime* plugin_aux=NULL; const char* lines=NULL; @@ -999,8 +999,8 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table { return -1; } - struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; - if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL) + struct Maat_table_desc *p_table=_feather->p_table_info[table_id]; + if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL || dup_func==NULL) { return -1; } @@ -1051,6 +1051,47 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa pthread_rwlock_unlock(&(compile_inner->rwlock)); return ad; } +MESA_htable_handle plugin_EX_htable_new(const struct plugin_table_desc* plugin_desc, + struct dynamic_array_t* lines, size_t line_cnt) +{ + MESA_htable_handle key2ex_hash=NULL; + unsigned int slot_size=1; + const char* line=NULL; + size_t key_offset=0, key_len=0; + long long estimate_size=plugin_desc->estimate_size, i=0; + int ret=0; + MAAT_RULE_EX_DATA exdata=NULL; + while(estimate_size>0) + { + estimate_size=estimate_size<<1; + slot_size*=2; + } + if(slot_size==1) + { + slot_size=4096; + } + MESA_htable_create_args_t hargs; + memset(&hargs,0,sizeof(hargs)); + hargs.thread_safe=1; + hargs.hash_slot_size = slot_size; + hargs.max_elem_num = 0; + hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; + hargs.expire_time = 0; + hargs.key_comp = NULL; + hargs.key2index = plugin_desc->ex_desc.key2index_func; + hargs.recursive = 1; + hargs.data_free = NULL ; + hargs.data_expire_with_condition = NULL; + key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(key2ex_hash, 0); + for(i=0; i< line_cnt; i++) + { + line=(const char*)dynamic_array_read(lines, i); + ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len); + exdata=plugin_desc->ex_desc.new_func(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);; + MESA_htable_add(key2ex_hash, line+key_offset, key_len, const void * data) + } +} int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, Maat_plugin_EX_new_func_t* new_func, Maat_plugin_EX_free_func_t* free_func, @@ -1059,6 +1100,27 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id, long argl, void *argp) { + struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; + struct Maat_table_desc *table_desc=_feather->p_table_info[table_id]; + struct plugin_table_desc* plugin_desc=&(table_desc->plugin); + struct Maat_table_runtime* table_rt=_feather->scanner->table_rt[table_id]; + MESA_htable_handle key2ex_hash=NULL; + if(table_desc->table_type!=TABLE_TYPE_PLUGIN || new_func==NULL || free_func==NULL || dup_func==NULL) + { + assert(0); + return -1; + } + plugin_desc->ex_desc.new_func=new_func; + plugin_desc->ex_desc.free_func=free_func; + plugin_desc->ex_desc.dup_func=dup_func; + plugin_desc->ex_desc.key2index_func=key2index_func; + plugin_desc->ex_desc.argl=argl; + plugin_desc->ex_desc.argp=argp; + + pthread_mutex_lock(&(_feather->backgroud_update_mutex)); + table_rt->plugin.key2ex_hash=plugin_EX_htable_new(plugin_desc,table_rt->plugin.cache_lines, table_rt->plugin.cache_line_num); + pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); + return 0; } @@ -1074,7 +1136,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num];//dynamic array - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; struct expr_table_desc* expr_desc=NULL; struct timespec start,end; _Maat_scanner_t* my_scanner=NULL; @@ -1117,8 +1179,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id { return 0; } - struct table_runtime* table_aux=my_scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } @@ -1126,7 +1188,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; INC_SCANNER_REF(my_scanner, thread_num); - if(table_aux->expr.expr_rule_cnt>0) + if(table_rt->expr.expr_rule_cnt>0) { scan_data.rule_type=RULETYPE_STR; scan_data.sub_type=sub_type; @@ -1136,7 +1198,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id hit_region_cnt+=region_ret; } } - if(table_aux->expr.regex_rule_cnt>0) + if(table_rt->expr.regex_rule_cnt>0) { scan_data.rule_type=RULETYPE_REG; scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); @@ -1153,7 +1215,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id } if(hit_region_cnt>0) { - alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid,_feather,thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1177,11 +1239,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux, data_len, &start, &end, thread_num); + maat_stat_table(table_rt, data_len, &start, &end, thread_num); } else { - maat_stat_table(table_aux, data_len, NULL, NULL, thread_num); + maat_stat_table(table_rt, data_len, NULL, NULL, thread_num); } DEC_SCANNER_REF(my_scanner, thread_num); if(compile_ret==0&&hit_region_cnt>0) @@ -1217,7 +1279,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id intval_scan_data.rule_type=RULETYPE_INT; intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); intval_scan_data.int_data=intval; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; struct timespec start,end; if(_feather->perf_on==1) { @@ -1234,8 +1296,8 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id { return 0; } - struct table_runtime* table_aux=my_scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } @@ -1253,7 +1315,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -1271,11 +1333,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux, 0, &start, &end, thread_num); + maat_stat_table(table_rt, 0, &start, &end, thread_num); } else { - maat_stat_table(table_aux, 0, NULL, NULL, thread_num); + maat_stat_table(table_rt, 0, NULL, NULL, thread_num); } DEC_SCANNER_REF(my_scanner,thread_num); @@ -1297,7 +1359,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id scan_data_t ip_scan_data; scan_result_t *region_result=NULL; _compile_result_t compile_result[rule_num]; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* my_scanner=NULL; @@ -1317,17 +1379,17 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { return 0; } - struct table_runtime* table_aux=my_scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } - if(table_aux->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4) + if(table_rt->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4) { return 0; } - if(table_aux->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) + if(table_rt->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) { return 0; } @@ -1372,7 +1434,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, @@ -1391,11 +1453,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux, 0, &start, &end, thread_num); + maat_stat_table(table_rt, 0, &start, &end, thread_num); } else { - maat_stat_table(table_aux, 0, NULL, NULL, thread_num); + maat_stat_table(table_rt, 0, NULL, NULL, thread_num); } if(compile_ret==0&®ion_ret>0) { @@ -1420,7 +1482,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* scanner=NULL; - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; assert(thread_num<_feather->scan_thread_num); p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR); if(p_table==NULL) @@ -1445,8 +1507,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { return sp; } - struct table_runtime* table_aux=scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } @@ -1462,15 +1524,15 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id, { sp->do_merge=1; } - if(table_aux->expr.expr_rule_cnt>0) + if(table_rt->expr.expr_rule_cnt>0) { sp->do_expr=1; } - if(table_aux->expr.regex_rule_cnt>0) + if(table_rt->expr.regex_rule_cnt>0) { sp->do_regex=1; } - alignment_int64_array_add(table_aux->stream_num,thread_num,1); + alignment_int64_array_add(table_rt->stream_num,thread_num,1); sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); return sp; } @@ -1489,8 +1551,8 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para scan_result_t *region_result; _compile_result_t compile_result[rule_num];//dynamic array scan_data_t region_scan_data; - _Maat_table_info_t* p_table=NULL; - struct table_runtime* table_aux=scanner->table_rt[sp->table_id]; + Maat_table_desc* p_table=NULL; + struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; struct timespec start,end; if(data==NULL||data_len<=0) { @@ -1598,7 +1660,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para } if(hit_region_cnt>0) { - alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1637,11 +1699,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux,data_len,&start, &end,sp->thread_num); + maat_stat_table(table_rt,data_len,&start, &end,sp->thread_num); } else { - maat_stat_table(table_aux,data_len,NULL, NULL,sp->thread_num); + maat_stat_table(table_rt,data_len,NULL, NULL,sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -1666,8 +1728,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - struct table_runtime* table_aux=scanner->table_rt[sp->table_id]; - alignment_int64_array_add(table_aux->stream_num, sp->thread_num, -1); + struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; + alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); if(sp->rs_stream_para!=NULL) { if(scanner!=NULL&&sp->version>=sp->feather->last_full_version) @@ -1704,7 +1766,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* scanner=NULL; sfh_instance_t * tmp_fuzzy_handle=NULL; - struct _Maat_table_info_t *p_table=NULL; + struct Maat_table_desc *p_table=NULL; p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST); if(p_table==NULL) { @@ -1720,7 +1782,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, { return sp; } - struct table_runtime* table_aux=scanner->table_rt[table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table_id]; tmp_fuzzy_handle=SFH_instance(total_len); if(tmp_fuzzy_handle==NULL) { @@ -1734,7 +1796,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id, sp->total_len=total_len; sp->fuzzy_hash_handle=tmp_fuzzy_handle; pthread_mutex_init(&(sp->fuzzy_mutex),NULL); - alignment_int64_array_add(table_aux->stream_num,thread_num,1); + alignment_int64_array_add(table_rt->stream_num,thread_num,1); return sp; } @@ -1779,8 +1841,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int { return 0; } - struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id]; - GIE_handle_t* GIE_handle=table_aux->similar.gie_handle; + struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + GIE_handle_t* GIE_handle=table_rt->similar.gie_handle; unsigned long long digest_len=0; char* digest_buff=NULL; struct _OUTER_scan_status_t* _mid=NULL; @@ -1825,7 +1887,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int } if(hit_region_cnt>0) { - alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num, 1); + alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1); _mid=grab_mid(mid,sp->feather, sp->thread_num,1); compile_ret=region_compile(sp->feather,_mid->inner, _mid->is_last_region, @@ -1843,11 +1905,11 @@ fast_out: if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux, data_len, &start, &end, sp->thread_num); + maat_stat_table(table_rt, data_len, &start, &end, sp->thread_num); } else { - maat_stat_table(table_aux, data_len, NULL, NULL, sp->thread_num); + maat_stat_table(table_rt, data_len, NULL, NULL, sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -1859,8 +1921,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para) { struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _Maat_scanner_t* scanner=sp->feather->scanner; - struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id]; - alignment_int64_array_add(table_aux->stream_num, sp->thread_num,-1); + struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; + alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); if(scanner!=NULL) { if(sp->version==sp->feather->maat_version) @@ -1955,7 +2017,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id _compile_result_t compile_result[rule_num]; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_scanner_t* my_scanner=NULL; - _Maat_table_info_t* p_table=NULL; + Maat_table_desc* p_table=NULL; struct timespec start,end; if(_feather->perf_on==1) { @@ -1972,12 +2034,12 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id { return 0; } - struct table_runtime* table_aux=my_scanner->table_rt[table_id]; - if(table_aux->origin_rule_num==0) + struct Maat_table_runtime* table_rt=my_scanner->table_rt[table_id]; + if(table_rt->origin_rule_num==0) { return 0; } - GIE_handle_t* gie_handle=table_aux->similar.gie_handle; + GIE_handle_t* gie_handle=table_rt->similar.gie_handle; INC_SCANNER_REF(my_scanner,thread_num); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); @@ -1990,7 +2052,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id } else if(region_ret>0) { - alignment_int64_array_add(table_aux->hit_cnt, thread_num,1); + alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, _mid->is_last_region, @@ -2009,11 +2071,11 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_aux,0,&start, &end,thread_num); + maat_stat_table(table_rt,0,&start, &end,thread_num); } else { - maat_stat_table(table_aux,0,NULL, NULL,thread_num); + maat_stat_table(table_rt,0,NULL, NULL,thread_num); } if(compile_ret==0&®ion_ret>0) { diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index 523e864..c8e09a4 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -182,61 +182,51 @@ enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p) } int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) { - unsigned int offset=0; - unsigned int i=0,j=0; - switch(type) + size_t offset=0, len=0; + unsigned int column_seq=0, ret=0; + switch(type) { case TABLE_TYPE_EXPR: - offset=7; + column_seq=7; break; case TABLE_TYPE_IP: - offset=14; + column_seq=14; break; case TABLE_TYPE_COMPILE: - offset=8; + column_seq=8; break; case TABLE_TYPE_PLUGIN: if(valid_column_seq<0) { return -1; } - offset=(unsigned int)valid_column_seq; + column_seq=(unsigned int)valid_column_seq; break; case TABLE_TYPE_INTERVAL: - offset=5; + column_seq=5; break; case TABLE_TYPE_DIGEST: - offset=6; + column_seq=6; break; case TABLE_TYPE_SIMILARITY: - offset=5; + column_seq=5; break; case TABLE_TYPE_EXPR_PLUS: - offset=8; + column_seq=8; break; case TABLE_TYPE_GROUP: - offset=3; + column_seq=3; break; default: assert(0); } - for(i=0;i=strlen(line)||(line[i]!='1'&&line[i]!='0'))// 0 is also a valid value for some non-MAAT producer. + + ret=get_column_pos(const char* line, int column_seq, &offset, &len); + if(ret<0||offset>=strlen(line)||(line[offset]!='1'&&line[offset]!='0'))// 0 is also a valid value for some non-MAAT producer. { return -1; } - return i; + return offset; } int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq) { @@ -1589,7 +1579,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, { int ret=0, table_id=0, i=0; int rule_with_foreign_key=0; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; struct plugin_table_desc* plugin_desc=NULL; for(i=0; iconj_cnt=1; return p; } -void table_info_free(struct _Maat_table_info_t*p) +void table_info_free(struct Maat_table_desc*p) { free(p); return; @@ -600,7 +602,7 @@ int _read_integer_arrary(char* string, int *array, int size) return i; } -int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p) +int read_plugin_table_info(const char* line, struct Maat_table_desc* p) { int i=0,ret=0; @@ -659,6 +661,12 @@ int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p) assert(tmp->type==cJSON_Number); plugin_desc->rule_tag_column=tmp->valueint; } + tmp=cJSON_GetObjectItem(json, "estimate_size"); + if(tmp!=NULL) + { + assert(tmp->type==cJSON_Number); + plugin_desc->estimate_size=tmp->valueint; + } tmp=cJSON_GetObjectItem(json, "foreign"); if(tmp!=NULL) { @@ -685,15 +693,15 @@ error_out: free(copy_line); return -1; } -int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) +int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger) { FILE*fp=NULL; char line[MAX_TABLE_LINE_SIZE]; int i=0,ret=0,table_cnt=0; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; MESA_htable_handle string2int_map=map_create(); - struct _Maat_table_info_t*p=NULL; - struct _Maat_table_info_t*conj_table=NULL; + struct Maat_table_desc*p=NULL; + struct Maat_table_desc*conj_table=NULL; map_register(string2int_map,"expr", TABLE_TYPE_EXPR); map_register(string2int_map,"ip", TABLE_TYPE_IP); @@ -988,7 +996,7 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id) } void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule) { - const struct _Maat_table_info_t* table=compile_rule->ref_table; + const struct Maat_table_desc* table=compile_rule->ref_table; const struct compile_table_desc* compile_desc=&(table->compile); struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule; int i=0; @@ -1185,9 +1193,9 @@ void destroy_digest_rule(GIE_digest_t*rule) } -struct table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num) +struct Maat_table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_thread_num) { - struct table_runtime* p= ALLOC(struct table_runtime, 1); + struct Maat_table_runtime* p= ALLOC(struct Maat_table_runtime, 1); p->table_type=type; switch(type) { @@ -1209,7 +1217,7 @@ struct table_runtime* table_runtime_new(enum MAAT_TABLE_TYPE type, int max_threa p->hit_cnt=alignment_int64_array_alloc(max_thread_num); return p; } -void table_runtime_free(struct table_runtime* p) +void table_runtime_free(struct Maat_table_runtime* p) { long q_cnt=0,data_size=0; int i=0; @@ -1244,9 +1252,9 @@ void table_runtime_free(struct table_runtime* p) case TABLE_TYPE_PLUGIN: dynamic_array_destroy(p->plugin.cache_lines, free); p->plugin.cache_lines=NULL; - if(p->plugin.ex_data_hash!=NULL) + if(p->plugin.key2ex_hash!=NULL) { - MESA_htable_destroy(p->plugin.ex_data_hash, NULL); + MESA_htable_destroy(p->plugin.key2ex_hash, NULL); } break; default: @@ -1266,8 +1274,8 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t { int scan_thread_num=feather->scan_thread_num; // int rs_scan_type=feather->rule_scan_type; - struct _Maat_table_info_t ** pp_table_desc=feather->p_table_info; - struct table_runtime* table_aux=NULL; + struct Maat_table_desc ** pp_table_desc=feather->p_table_info; + struct Maat_table_runtime* table_rt=NULL; const struct expr_table_desc* expr_desc=NULL; int i=0,j=0; unsigned int sub_type=0; @@ -1336,7 +1344,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t { continue; } - table_aux=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num); + table_rt=table_runtime_new(pp_table_desc[i]->table_type, feather->scan_thread_num); if(pp_table_desc[i]->table_type==TABLE_TYPE_EXPR||pp_table_desc[i]->table_type==TABLE_TYPE_EXPR_PLUS) { expr_desc=&(pp_table_desc[i]->expr); @@ -1354,7 +1362,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t } } } - scanner->table_rt[i]=table_aux; + scanner->table_rt[i]=table_rt; } return scanner; } @@ -1502,7 +1510,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, unsigned long long update_interval=0; struct _region_stat_t region_counter[MAX_TABLE_NUM]; memset(region_counter, 0, sizeof(region_counter)); - struct table_runtime* table_aux=NULL; + struct Maat_table_runtime* table_rt=NULL; if(q_cnt==0) { return; @@ -1558,28 +1566,28 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, //update scanner's region cnt; for(i=0;itable_rt[i]; - if(table_aux==NULL) + table_rt=maat_scanner->table_rt[i]; + if(table_rt==NULL) { continue; } - switch(table_aux->table_type) + switch(table_rt->table_type) { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - table_aux->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt; - table_aux->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt; - assert(table_aux->expr.expr_rule_cnt>=0); - assert(table_aux->expr.regex_rule_cnt>=0); + table_rt->expr.expr_rule_cnt+=region_counter[i].expr_rule_cnt; + table_rt->expr.regex_rule_cnt+=region_counter[i].regex_rule_cnt; + assert(table_rt->expr.expr_rule_cnt>=0); + assert(table_rt->expr.regex_rule_cnt>=0); break; case TABLE_TYPE_IP: - table_aux->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt; - table_aux->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt; + table_rt->ip.ipv4_rule_cnt+=region_counter[i].ipv4_rule_cnt; + table_rt->ip.ipv6_rule_cnt+=region_counter[i].ipv6_rule_cnt; break; default: break; } - assert(table_aux->origin_rule_num>=0); + assert(table_rt->origin_rule_num>=0); } for(i=0;itable_rt[table_id]; + struct Maat_table_runtime* table_rt=maat_scanner->table_rt[table_id]; update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); for(i=0;ioperation==GIE_INSERT_OPT) { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } destroy_digest_rule(update_array[i]); update_array[i]=NULL; @@ -1877,7 +1885,7 @@ int get_district_id(_Maat_scanner_t *scanner,const char* district_str) } return district_id; } -int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger) { unsigned int i=0,j=0; char* p=NULL,*saveptr=NULL,*region_string=NULL; @@ -2141,7 +2149,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule } return 0; } -int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; scan_rule_t* p_rule=NULL; @@ -2173,7 +2181,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; scan_rule_t* p_rule=NULL; @@ -2204,12 +2212,12 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); return 0; } -int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; GIE_digest_t* digest_rule=NULL; struct _Maat_group_inner_t* u_para=NULL; - struct table_runtime * table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id]; int expr_id=0,district_id=-1; group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); @@ -2232,11 +2240,11 @@ int add_digest_rule(struct _Maat_table_info_t* table,struct db_digest_rule_t* db ,db_digest_rule->digest_string ,db_digest_rule->confidence_degree ,group_rule); - MESA_lqueue_join_tail(table_aux->similar.update_q, &digest_rule, sizeof(void*)); + MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*)); scanner->gie_total_q_size++; return 0; } -int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger) +int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger) { int i=0; unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; @@ -2300,7 +2308,7 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id, } return 0; } -int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_group_inner_t* group_rule=NULL; struct _Maat_compile_inner_t*compile_rule=NULL; @@ -2335,7 +2343,7 @@ int add_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_g return 0; } -void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) +void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_compile_inner_t*compile_rule=NULL; struct _Maat_group_inner_t* group_rule=NULL; @@ -2370,7 +2378,7 @@ void del_group_rule(struct _Maat_table_info_t* table,struct db_group_rule_t* db_ } return; } -int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) +int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_compile_inner_t *compile_rule=NULL; struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head); @@ -2402,7 +2410,7 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* return 0; } -int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) +int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) { struct _Maat_compile_inner_t *compile_rule=NULL; compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); @@ -2425,10 +2433,10 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t* } return 1; } -void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) +void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) { struct db_group_rule_t db_group_rule; - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; ret=sscanf(table_line,"%d\t%d\t%d",&(db_group_rule.group_id) ,&(db_group_rule.compile_id) @@ -2447,7 +2455,7 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s //leave no trace when compatible_group_update calling if(table->table_type==TABLE_TYPE_GROUP) { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } else @@ -2466,14 +2474,14 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s //no need to free db_group_rule,it was saved in scanner->compile_hash if(table->table_type==TABLE_TYPE_GROUP) { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } } return; } -void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger) +void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger) { char virtual_group_line[256]; snprintf(virtual_group_line,sizeof(virtual_group_line), @@ -2481,12 +2489,12 @@ void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int update_group_rule(table, virtual_group_line,scanner,logger); return; } -void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); int ret=0,db_hexbin=0,rule_type=0; const struct expr_table_desc* expr_desc=&(table->expr); - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; switch(table->table_type) { case TABLE_TYPE_EXPR: @@ -2611,7 +2619,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } else @@ -2645,7 +2653,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st } else { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } @@ -2653,11 +2661,11 @@ error_out: free(maat_str_rule); maat_str_rule=NULL; } -void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40]; - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; unsigned short i_src_port,i_sport_mask,i_dst_port,i_dport_mask; int protocol=0,direction=0; int ret=0,rule_type=0; @@ -2772,7 +2780,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } else @@ -2788,7 +2796,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru } else { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } @@ -2797,10 +2805,10 @@ error_out: ip_rule=NULL; } -void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { struct db_intval_rule_t* intval_rule=ALLOC(struct db_intval_rule_t, 1); - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; int ret=0; ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id) ,&(intval_rule->group_id) @@ -2847,7 +2855,7 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } @@ -2863,7 +2871,7 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } @@ -2872,10 +2880,10 @@ error_out: intval_rule=NULL; } -void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) +void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) { struct compile_table_desc* compile_desc=&(table->compile); - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct db_compile_rule_t *p_compile=ALLOC(struct db_compile_rule_t, 1); struct _head_Maat_rule_t* p_m_rule=&(p_compile->m_rule_head); @@ -2932,7 +2940,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line if(p_compile->is_valid==FALSE) { ret=del_compile_rule(table,p_compile,scanner, logger); - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; goto no_save; } else @@ -2946,7 +2954,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line table->udpate_err_cnt++; goto no_save; } - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } return; @@ -2958,9 +2966,9 @@ no_save: return; } -void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) +void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) { - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct db_digest_rule_t* digest_rule=ALLOC(struct db_digest_rule_t, 1); int ret=0; char digest_buff[MAX_TABLE_LINE_SIZE]={'\0'}; @@ -3025,7 +3033,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table_aux->origin_rule_num--; + table_rt->origin_rule_num--; } } @@ -3041,7 +3049,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line, } else { - table_aux->origin_rule_num++; + table_rt->origin_rule_num++; } } @@ -3162,12 +3170,12 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger) q_cnt,bury_cnt); } } -void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger) +void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger) { int i=0, ret=1; unsigned int len=strlen(table_line)+1; struct plugin_table_desc* plugin_desc=&(table->plugin); - struct table_runtime* table_aux=scanner->table_rt[table->table_id]; + struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; char *p=NULL; char* copy=NULL; @@ -3204,7 +3212,7 @@ void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line } } - table_aux->plugin.acc_line_num++; + table_rt->plugin.acc_line_num++; if(plugin_desc->cb_plug_cnt>0) { @@ -3217,16 +3225,16 @@ void update_plugin_table(struct _Maat_table_info_t* table,const char* table_line { p=ALLOC(char, len); memcpy(p,table_line,len); - table_aux->plugin.cache_size+=len; - dynamic_array_write(table_aux->plugin.cache_lines,table_aux->plugin.cache_line_num,p); - table_aux->plugin.cache_line_num++; + table_rt->plugin.cache_size+=len; + dynamic_array_write(table_rt->plugin.cache_lines,table_rt->plugin.cache_line_num,p); + table_rt->plugin.cache_line_num++; } } void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger) { void *tmp1=NULL,*tmp2=NULL; MESA_htable_handle tmp_map=NULL; - struct table_runtime* table_aux=NULL; + struct Maat_table_runtime* table_rt=NULL; int i=0; long q_cnt; GIE_create_para_t para; @@ -3252,23 +3260,23 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ ,scanner); for(i=0;itable_rt[i]; - if(table_aux==NULL) + table_rt=scanner->table_rt[i]; + if(table_rt==NULL) { continue; } - switch(table_aux->table_type) + switch(table_rt->table_type) { case TABLE_TYPE_DIGEST: case TABLE_TYPE_SIMILARITY: - q_cnt=MESA_lqueue_get_count(table_aux->similar.update_q); + q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q); if(q_cnt==0) { continue; } - if(table_aux->similar.gie_handle==NULL) + if(table_rt->similar.gie_handle==NULL) { - if(table_aux->table_type==TABLE_TYPE_SIMILARITY) + if(table_rt->table_type==TABLE_TYPE_SIMILARITY) { para.ED_reexamine=1; para.format=GIE_INPUT_FORMAT_PLAIN; @@ -3278,10 +3286,10 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ para.ED_reexamine=0; para.format=GIE_INPUT_FORMAT_SFH; } - table_aux->similar.gie_handle=GIE_create(¶); + table_rt->similar.gie_handle=GIE_create(¶); } - digest_batch_update(table_aux->similar.gie_handle, - table_aux->similar.update_q, + digest_batch_update(table_rt->similar.gie_handle, + table_rt->similar.update_q, logger, scanner, i); @@ -3306,25 +3314,11 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_ return; } -void clear_plugin_table_info(struct _plugin_table_info *cb_info) -{ - int i=0; - void *line=NULL; - for(i=0;icache_line_num;i++) - { - line=dynamic_array_read(cb_info->cache_lines,i); - free(line); - dynamic_array_write(cb_info->cache_lines,i,NULL); - } - cb_info->cache_line_num=0; - cb_info->cache_size=0; - cb_info->acc_line_num=0; - return; -} + void maat_start_cb(long long new_version,int update_type,void*u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; struct plugin_table_desc* plugin_desc=NULL; int i=0,j=0; feather->new_version=new_version; @@ -3369,14 +3363,14 @@ void maat_start_cb(long long new_version,int update_type,void*u_para) long long scanner_rule_num(struct _Maat_scanner_t *scanner) { long long total=0; - struct table_runtime* table_aux=NULL; + struct Maat_table_runtime* table_rt=NULL; int i=0; for(i=0;itable_rt[i]; - if(table_aux!=NULL) + table_rt=scanner->table_rt[i]; + if(table_rt!=NULL) { - total+=table_aux->origin_rule_num; + total+=table_rt->origin_rule_num; } } return total; @@ -3384,7 +3378,7 @@ long long scanner_rule_num(struct _Maat_scanner_t *scanner) void maat_finish_cb(void* u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; struct plugin_table_desc* plugin_desc=NULL; long expr_wait_q_cnt=0; int i=0, j=0; @@ -3472,7 +3466,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para) int ret=-1,i=0; int table_id=-1; _Maat_scanner_t* scanner=NULL; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; if(feather->update_tmp_scanner!=NULL) { scanner=feather->update_tmp_scanner; diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index cc0fcbf..de8fcc3 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -42,7 +42,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) { int value=0; int i=0,j=0,offset=0; - struct _Maat_table_info_t* p_table=NULL; + struct Maat_table_desc* p_table=NULL; char conj_table_name[(MAX_TABLE_NAME_LEN+1)*MAX_CONJUNCTION_TABLE_NUM]={0}; feather->stat_handle=FS_create_handle(); @@ -84,7 +84,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_CMD_LINE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_SPEED,"line_cmd/s"); feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule"); - feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex"); + feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"reg/v6"); feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream"); feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Bps"); if(feather->perf_on==1) @@ -149,7 +149,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) FS_start(feather->stat_handle); return; } -void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num) +void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num) { alignment_int64_array_add(p->scan_cnt,thread_num,1); alignment_int64_array_add(p->input_bytes,thread_num,scan_len); @@ -171,8 +171,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0; int i=0; time_t now; - struct _Maat_table_info_t* p_table=NULL; - struct table_runtime* table_aux=NULL; + struct Maat_table_desc* p_table=NULL; + struct Maat_table_runtime* table_rt=NULL; time(&now); active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); @@ -208,26 +208,26 @@ void maat_stat_output(struct _Maat_feather_t* feather) { continue; } - table_aux=feather->scanner->table_rt[i]; + table_rt=feather->scanner->table_rt[i]; switch(p_table->table_type) { case TABLE_TYPE_PLUGIN: - plugin_cache_num+=table_aux->plugin.cache_line_num; - plugin_acc_num+=table_aux->plugin.acc_line_num; + plugin_cache_num+=table_rt->plugin.cache_line_num; + plugin_acc_num+=table_rt->plugin.acc_line_num; break; case TABLE_TYPE_GROUP: - group_rule_num+=table_aux->origin_rule_num; + group_rule_num+=table_rt->origin_rule_num; break; case TABLE_TYPE_COMPILE: - compile_rule_num+=table_aux->origin_rule_num; + compile_rule_num+=table_rt->origin_rule_num; break; case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - table_regex_ipv6_num=table_aux->expr.regex_rule_cnt; + table_regex_ipv6_num=table_rt->expr.regex_rule_cnt; total_iconv_error=p_table->expr.iconv_err_cnt; break; case TABLE_TYPE_IP: - table_regex_ipv6_num=table_aux->ip.ipv6_rule_cnt; + table_regex_ipv6_num=table_rt->ip.ipv6_rule_cnt; break; default: break; @@ -242,8 +242,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_RULE_NUM], FS_OP_SET, - table_aux->origin_rule_num); - total_cfg_num+=table_aux->origin_rule_num; + table_rt->origin_rule_num); + total_cfg_num+=table_rt->origin_rule_num; FS_operate(feather->stat_handle, p_table->stat_line_id, @@ -252,8 +252,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_regex_ipv6_num); total_regex_num+=table_regex_ipv6_num; - table_stream_num=alignment_int64_array_sum(table_aux->stream_num,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->stream_num,feather->scan_thread_num); + table_stream_num=alignment_int64_array_sum(table_rt->stream_num,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->stream_num,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], @@ -262,8 +262,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) total_stream_cnt+= table_stream_num; - table_scan_cnt=alignment_int64_array_sum(table_aux->scan_cnt,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->scan_cnt,feather->scan_thread_num); + table_scan_cnt=alignment_int64_array_sum(table_rt->scan_cnt,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->scan_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], @@ -271,8 +271,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_scan_cnt); total_scan_cnt+=table_scan_cnt; - table_input_bytes=alignment_int64_array_sum(table_aux->input_bytes,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->input_bytes,feather->scan_thread_num); + table_input_bytes=alignment_int64_array_sum(table_rt->input_bytes,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->input_bytes,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], @@ -281,8 +281,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) total_input_bytes+=table_input_bytes; if(feather->perf_on==1) { - table_scan_cpu_time=alignment_int64_array_sum(table_aux->scan_cpu_time,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->scan_cpu_time,feather->scan_thread_num); + table_scan_cpu_time=alignment_int64_array_sum(table_rt->scan_cpu_time,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->scan_cpu_time,feather->scan_thread_num); table_scan_cpu_time/=1000; FS_operate(feather->stat_handle, p_table->stat_line_id, @@ -292,8 +292,8 @@ void maat_stat_output(struct _Maat_feather_t* feather) total_cpu_time+=table_scan_cpu_time; } - table_hit_cnt=alignment_int64_array_sum(table_aux->hit_cnt,feather->scan_thread_num); - alignment_int64_array_reset(table_aux->hit_cnt,feather->scan_thread_num); + table_hit_cnt=alignment_int64_array_sum(table_rt->hit_cnt,feather->scan_thread_num); + alignment_int64_array_reset(table_rt->hit_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, p_table->stat_line_id, feather->fs_column_id[COLUMN_TABLE_HIT_CNT], diff --git a/src/entry/Maat_utils.cpp b/src/entry/Maat_utils.cpp index 1ceffab..81618ac 100644 --- a/src/entry/Maat_utils.cpp +++ b/src/entry/Maat_utils.cpp @@ -164,6 +164,30 @@ char* str_unescape(char* s) s[j]='\0'; return s; } +int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len) +{ + const char* seps=" \t"; + char* saveptr=NULL, *subtoken=NULL, *str=NULL; + char* dup_line=_maat_strdup(line); + int i=0, ret=-1; + for (str = dup_line; ; str = NULL) + { + subtoken = strtok_r(str, seps, &saveptr); + if (subtoken == NULL) + break; + if(i==column_seq-1) + { + *offset=subtoken-dup_line; + *len=strlen(subtoken); + ret=0 + break; + } + i++; + } + free(dup_line); + return ret; +} + #define MAX_SYSTEM_CMD_LEN 512 int system_cmd_mkdir(const char* path) diff --git a/src/entry/dynamic_array.cpp b/src/entry/dynamic_array.cpp index 26c414e..46b7159 100644 --- a/src/entry/dynamic_array.cpp +++ b/src/entry/dynamic_array.cpp @@ -2,11 +2,11 @@ #include #include int dynamic_array_VERSION_20141202=0; -struct dynamic_array_t* dynamic_array_create(int size,int step) +struct dynamic_array_t* dynamic_array_create(long long init_size, long long step) { struct dynamic_array_t* d_array=(struct dynamic_array_t*)calloc(sizeof(struct dynamic_array_t),1); - d_array->array=(void**)calloc(sizeof(void*),size); - d_array->size=size; + d_array->array=(void**)calloc(sizeof(void*),init_size); + d_array->size=init_size; d_array->enlarge_step=step; return d_array; } @@ -23,7 +23,7 @@ void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(vo free(d_array->array); free(d_array); } -void* dynamic_array_read(struct dynamic_array_t* d_array,int i) +void* dynamic_array_read(struct dynamic_array_t* d_array, long long i) { if(isize) { @@ -34,7 +34,7 @@ void* dynamic_array_read(struct dynamic_array_t* d_array,int i) return NULL; } } -void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data) +void dynamic_array_write(struct dynamic_array_t* d_array, long long i, void* data) { int new_size=0; if(isize) diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index f123134..2c78e6a 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -123,18 +123,8 @@ struct plugin_table_ex_data_desc Maat_plugin_EX_key2index_func_t* key2index_func; long argl; void *argp; + MESA_htable_handle key2ex_hash; }; -struct _plugin_table_info -{ - int cb_plug_cnt; - struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; - dynamic_array_t *cache_lines; - int cache_line_num; - int acc_line_num; - int update_type; - long cache_size; -}; - struct plugin_table_desc { int key_column; @@ -143,9 +133,9 @@ struct plugin_table_desc int n_foreign; int foreign_columns[MAX_FOREIGN_CLMN_NUM]; int cb_plug_cnt; + long long estimate_size; struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE]; struct plugin_table_ex_data_desc ex_desc; - int acc_line_num; }; struct expr_table_desc { @@ -163,7 +153,7 @@ struct ip_table_desc int ipv6_rule_cnt; }; -struct _Maat_table_info_t +struct Maat_table_desc { unsigned short table_id; unsigned short conj_cnt; @@ -317,7 +307,7 @@ struct _Maat_compile_inner_t dynamic_array_t *groups; int is_valid; int compile_id;//equal to db_c_rule->m_rule.config_id - const struct _Maat_table_info_t* ref_table; + const struct Maat_table_desc* ref_table; int group_boundary; int group_cnt; MAAT_RULE_EX_DATA* ads; @@ -390,7 +380,7 @@ struct plugin_runtime long long cache_line_num; long long acc_line_num; long long cache_size; - MESA_htable_handle ex_data_hash; + MESA_htable_handle key2ex_hash; }; struct expr_runtime { @@ -403,7 +393,7 @@ struct ip_runtime long long ipv6_rule_cnt; }; -struct table_runtime +struct Maat_table_runtime { enum MAAT_TABLE_TYPE table_type; long origin_rule_num; @@ -434,7 +424,7 @@ struct _Maat_scanner_t rule_scanner_t region; long gie_total_q_size; - struct table_runtime* table_rt[MAX_TABLE_NUM]; + struct Maat_table_runtime* table_rt[MAX_TABLE_NUM]; /* struct similar_runtime gie_aux[MAX_TABLE_NUM]; struct plugin_runtime plugin_aux[MAX_TABLE_NUM]; @@ -506,7 +496,7 @@ struct _Maat_feather_t int cumulative_update_off; int stat_on; int perf_on; - struct _Maat_table_info_t *p_table_info[MAX_TABLE_NUM]; + struct Maat_table_desc *p_table_info[MAX_TABLE_NUM]; MESA_htable_handle map_tablename2id; void* logger; long long maat_version; @@ -605,7 +595,7 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); -int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); +int read_table_info(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); @@ -617,7 +607,7 @@ int HASH_add_by_id(MESA_htable_handle hash,int id,void*data); int HASH_delete_by_id(MESA_htable_handle hash,int id); void maat_read_full_config(_Maat_feather_t* _feather); void maat_stat_init(struct _Maat_feather_t* feather); -void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num); +void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_output(struct _Maat_feather_t* feather); redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); diff --git a/src/inc_internal/Maat_utils.h b/src/inc_internal/Maat_utils.h index 2843db1..23e22ec 100644 --- a/src/inc_internal/Maat_utils.h +++ b/src/inc_internal/Maat_utils.h @@ -47,4 +47,6 @@ int system_cmd_rm(const char* src_file); int system_cmd_mv(const char* src_file,const char*dst_file); int system_cmd_cp(const char* src_file,const char*dst_file); char* md5_file(const char* filename, char* md5string); +int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len); + diff --git a/src/inc_internal/dynamic_array.h b/src/inc_internal/dynamic_array.h index 7583c92..11b3643 100644 --- a/src/inc_internal/dynamic_array.h +++ b/src/inc_internal/dynamic_array.h @@ -3,11 +3,11 @@ struct dynamic_array_t { void ** array; - int size; - int enlarge_step; + long long size; + long long enlarge_step; }; -struct dynamic_array_t* dynamic_array_create(int size,int step); +struct dynamic_array_t* dynamic_array_create(long long init_size, long long step); void dynamic_array_destroy(struct dynamic_array_t* d_array,void (* free_data)(void *)); -void* dynamic_array_read(struct dynamic_array_t* d_array,int i); -void dynamic_array_write(struct dynamic_array_t* d_array,int i,void* data); +void* dynamic_array_read(struct dynamic_array_t* d_array,long long i); +void dynamic_array_write(struct dynamic_array_t* d_array, long long i,void* data); #endif //_DYNAMIC_ARRAY_H_INCLUDE_ diff --git a/test/table_info.conf b/test/table_info.conf index a920f3c..4b03043 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -28,5 +28,5 @@ 9 SIM_URL similar -- 10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff 11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} -- -12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":"6,8","tag":3} -- +12 TEST_FOREIGN_KEY plugin {"key":2,"valid":4,"foreign":[6,8],"tag":3, "estimate_size":4096} -- 13 COMPILE_ALIAS compile escape --