diff --git a/inc/Maat_rule.h b/inc/Maat_rule.h index 858c300..d3f50ef 100644 --- a/inc/Maat_rule.h +++ b/inc/Maat_rule.h @@ -163,7 +163,8 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, enum MAAT_SCAN_OPT { - MAAT_SET_SCAN_DISTRICT=1 //VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default. + MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default. + MAAT_SET_SCAN_LAST_REGION //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan cobination. }; //return 0 if success, return -1 when failed; int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size); diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 965651a..4587cd8 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -98,7 +98,7 @@ int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsi } return k; } -int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) +int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) { int scan_ret=0,result_cnt=0; @@ -135,6 +135,14 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,voi { //short cut for rules contains one group scan_ret=shortcut_avilable_cnt; + aligment_int64_array_add(feather->orphan_group_saving, thread_num, 1); + } + else if(shortcut_avilable_cnt==0&®ion_hit_num==1&&is_last_region==1) + { + //short cut for last scan and combination rules + //One groups may belong to a statisfy a compile rule, so only region_hit_num=1 could use the short cut + scan_ret=0; + aligment_int64_array_add(feather->last_region_saving, thread_num, 1); } else { @@ -457,6 +465,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->outer_mid_cnt=aligment_int64_array_alloc(max_thread_num); feather->inner_mid_cnt=aligment_int64_array_alloc(max_thread_num); feather->hit_cnt=aligment_int64_array_alloc(max_thread_num); + feather->orphan_group_saving=aligment_int64_array_alloc(max_thread_num); + feather->last_region_saving=aligment_int64_array_alloc(max_thread_num); feather->maat_version=0; feather->last_full_version=0; pthread_mutex_init(&(feather->plugin_table_reg_mutex),NULL); @@ -843,10 +853,16 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id aligment_int64_array_add(p_table->hit_cnt, thread_num,1); _mid=grab_mid(mid,_feather,thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, + _mid->is_last_region, region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), hit_region_cnt, result,compile_result,rule_num, thread_num); + assert(_mid->is_last_region<2); + if(_mid->is_last_region==1) + { + _mid->is_last_region=2; + } if(hit_detail!=NULL&&_feather->rule_scan_type!=0) { *detail_ret=fill_region_hit_detail(data,_mid->inner, @@ -943,10 +959,16 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id aligment_int64_array_add(p_table->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); compile_ret=region_compile(_feather,_mid->inner, + _mid->is_last_region, region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), region_ret, result,compile_result,rule_num, thread_num); + assert(_mid->is_last_region<2); + if(_mid->is_last_region==1) + { + _mid->is_last_region=2; + } } DEC_SCANNER_REF(my_scanner,thread_num); @@ -1058,11 +1080,18 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id { aligment_int64_array_add(p_table->hit_cnt, thread_num,1); _mid=grab_mid(mid, _feather, thread_num, 1); + compile_ret=region_compile(_feather,_mid->inner, + _mid->is_last_region, region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), region_ret, result,compile_result,rule_num, thread_num); + assert(_mid->is_last_region<2); + if(_mid->is_last_region==1) + { + _mid->is_last_region=2; + } } DEC_SCANNER_REF(my_scanner,thread_num); if(_feather->perf_on==1) @@ -1272,10 +1301,16 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para aligment_int64_array_add(p_table->hit_cnt, sp->thread_num,1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1); compile_ret=region_compile(sp->feather,_mid->inner, + _mid->is_last_region, region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), hit_region_cnt, result,compile_result,rule_num, sp->thread_num); + assert(_mid->is_last_region<2); + if(_mid->is_last_region==1) + { + _mid->is_last_region=2; + } if(hit_detail!=NULL&&sp->feather->rule_scan_type!=0) { if(sp->scan_buff!=NULL) @@ -1501,11 +1536,16 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int sp->feather->p_table_info[sp->table_id]->hit_cnt++; _mid=grab_mid(mid,sp->feather, sp->thread_num,1); compile_ret=region_compile(sp->feather,_mid->inner, + _mid->is_last_region, query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), hit_region_cnt, result,compile_result,rule_num, sp->thread_num); - + assert(_mid->is_last_region<2); + if(_mid->is_last_region==1) + { + _mid->is_last_region=2; + } } fast_out: if(sp->feather->perf_on==1) @@ -1576,6 +1616,9 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA } _mid->is_set_district=1; break; + case MAAT_SET_SCAN_LAST_REGION: + _mid->is_last_region=1; + break; default: _feather->scan_err_cnt++; return -1; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 0fab6ad..3e7396a 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -28,7 +28,7 @@ #include "mesa_fuzzy.h" #include "great_index_engine.h" -int MAAT_FRAME_VERSION_1_9_20170615=1; +int MAAT_FRAME_VERSION_1_9_20170624=1; const char *maat_module="MAAT Frame"; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", @@ -3080,6 +3080,8 @@ void *thread_rule_monitor(void *arg) aligment_int64_array_free(feather->inner_mid_cnt); aligment_int64_array_free(feather->outer_mid_cnt); aligment_int64_array_free(feather->hit_cnt); + aligment_int64_array_free(feather->orphan_group_saving); + aligment_int64_array_free(feather->last_region_saving); free(feather); return NULL; } diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index a3a8a76..d2e3e22 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -265,11 +265,11 @@ struct _Maat_table_info_t unsigned long long udpate_err_cnt; unsigned long long iconv_err_cnt; int stat_line_id; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; + mcore_long_t scan_cnt; + mcore_long_t scan_cpu_time; //nano + mcore_long_t input_bytes; + mcore_long_t stream_num; + mcore_long_t hit_cnt; }; struct _INNER_scan_status_t @@ -284,7 +284,8 @@ struct _OUTER_scan_status_t { struct _Maat_feather_t* feather; unsigned short thread_num; - unsigned short is_set_district; + unsigned char is_set_district; + unsigned char is_last_region; int district_id; struct _INNER_scan_status_t* inner; }; @@ -382,6 +383,8 @@ struct _Maat_feather_t mcore_long_t inner_mid_cnt; mcore_long_t hit_cnt; mcore_long_t thread_call_cnt;//size indicate by scan_thread_num, + mcore_long_t orphan_group_saving; + mcore_long_t last_region_saving; long long total_scan_bytes; long long total_scan_cnt; long long update_err_cnt;//sum of the same name variable in each table diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index ed528d7..393eed4 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -19,7 +19,9 @@ enum MAAT_FS_STATUS{ STATUS_UPDATE_ERR_CNT, STATUS_ICONV_ERR_CNT, STATUS_SCAN_ERR_CNT, - STATUS_ZOMBIE_RS_STREAM + STATUS_ZOMBIE_RS_STREAM, + STATUS_ORPHAN_GROUP_SAVING, + STATUS_LAST_REGION_SAVING }; enum MAAT_FS_COLUMN @@ -54,7 +56,7 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_cache"); feather->fs_status_id[STATUS_PLUGIN_ACC_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_acc"); - feather->fs_status_id[STATUS_GROUP_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"group_num"); + feather->fs_status_id[STATUS_GROUP_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"grp_num"); feather->fs_status_id[STATUS_COMPILE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"compile_num"); feather->fs_status_id[STATUS_POSTPONE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"postpone_num"); @@ -62,6 +64,8 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid"); feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid"); + feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"orphan_grp_sv"); + feather->fs_status_id[STATUS_LAST_REGION_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"last_rgn_sv"); feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes"); feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_times"); @@ -70,7 +74,6 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_SCAN_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_error"); feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"zombie_stream"); - feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule"); feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex"); feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream"); @@ -154,6 +157,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0; long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; long long outer_mid_cnt=0,inner_mid_cnt=0; + long long orphan_group_saving=0, last_region_saving=0; long long total_update_error=0,total_iconv_error=0; long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0; int i=0; @@ -163,11 +167,16 @@ void maat_stat_output(struct _Maat_feather_t* feather) active_thread_num=aligment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); outer_mid_cnt=aligment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); inner_mid_cnt=aligment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num); + orphan_group_saving=aligment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num); + last_region_saving=aligment_int64_array_sum(feather->last_region_saving,feather->scan_thread_num); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING], 0,FS_OP_SET,orphan_group_saving); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_LAST_REGION_SAVING], 0,FS_OP_SET,last_region_saving); value=MESA_lqueue_get_count(feather->garbage_q); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value); diff --git a/test/maat_test.cpp b/test/maat_test.cpp index e6b7603..5164aff 100644 --- a/test/maat_test.cpp +++ b/test/maat_test.cpp @@ -221,6 +221,8 @@ int test_ipv6_scan(Maat_feather_t feather,const char* table_name,scan_status_t* } else { + //for improving performance. + Maat_set_scan_status(feather, mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0); ret=Maat_scan_proto_addr(feather,table_id,&ipv6_addr,6,result,4, mid,0); printf("IPv6 addr Scan:%s\n",print_maat_result(result,ret));