From b5ad82afe9e5e2af6a00ba675cf9ae6f1b48b24e Mon Sep 17 00:00:00 2001 From: zhengchao Date: Sat, 29 Jun 2019 21:57:24 +0600 Subject: [PATCH] =?UTF-8?q?=E7=A7=BB=E9=99=A4compile=E5=BF=AB=E9=80=9F?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E7=9B=B8=E5=85=B3=E4=BB=A3=E7=A0=81=EF=BC=9A?= =?UTF-8?q?=E5=9C=A8=E6=94=AF=E6=8C=81=E5=AD=90=E5=88=86=E7=BB=84=E3=80=81?= =?UTF-8?q?=E2=80=9C=E9=9D=9E=E2=80=9D=E8=BF=90=E7=AE=97=E5=90=8E=EF=BC=8C?= =?UTF-8?q?=E8=AF=A5=E7=89=B9=E6=80=A7=E5=AF=B9=E6=80=A7=E8=83=BD=E6=8F=90?= =?UTF-8?q?=E5=8D=87=E6=9C=89=E9=99=90=EF=BC=8C=E5=8D=B4=E5=A4=A7=E5=B9=85?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BB=A3=E7=A0=81=E5=A4=8D=E6=9D=82=E5=BA=A6?= =?UTF-8?q?=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 33 ++++------------------- src/entry/Maat_rule.cpp | 39 ++------------------------- src/entry/Maat_stat.cpp | 6 +---- src/inc_internal/Maat_rule_internal.h | 2 -- 4 files changed, 8 insertions(+), 72 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 0f3ef01..e332f45 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -165,7 +165,6 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int int scan_ret=0, result_cnt=0; int ret=0, i=0, j=0; size_t r_in_c_cnt=0; - int shortcut_avilable_cnt=0; unsigned char has_not_flag=0; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; struct Maat_group_inner* group_rule=NULL; @@ -183,11 +182,6 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int } assert(group_rule->ref_by_children_cnt>=0); assert(group_rule->ref_by_parent_cnt>=0); - if(group_rule->compile_shortcut!=NULL&&group_rule->ref_by_parent_cnt==1&&shortcut_avilable_cntcompile_shortcut); - shortcut_avilable_cnt++; - } for(j=0; jtop_group_cnt; j++) { if(_mid->cur_hit_group_cntall_hit_group_cnt+=ret; } } - if((region_hit_num>0 &&shortcut_avilable_cnt==region_hit_num) || - shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM) - { - //short cut for rules contains one group - scan_ret=shortcut_avilable_cnt; - alignment_int64_array_add(feather->orphan_group_saving, thread_num, 1); - } - else if(0&&shortcut_avilable_cnt==0&®ion_hit_num==1&&_mid->all_hit_group_cnt==1&&is_last_region==1) - { - //This shortcut is NO longger valid after bool macher support NOT-logic. - //short cut for last scan and combination rules - //region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule. - //_mid->hit_group_cnt==1: With previously scan hitted group rules, one group may staisfy a compile rule - scan_ret=0; - } - else - { - scan_ret=bool_matcher_match(bm, thread_num, - _mid->all_hit_group_array, _mid->all_hit_group_cnt, - (void **)relation_array, MAX_SCANNER_HIT_NUM); - } + + scan_ret=bool_matcher_match(bm, thread_num, + _mid->all_hit_group_array, _mid->all_hit_group_cnt, + (void **)relation_array, MAX_SCANNER_HIT_NUM); + if(scan_ret>1) { qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**), @@ -567,7 +545,6 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num); feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num); feather->hit_cnt=alignment_int64_array_alloc(max_thread_num); - feather->orphan_group_saving=alignment_int64_array_alloc(max_thread_num); feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num); feather->maat_version=0; feather->last_full_version=0; diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 337748e..11cdf4b 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -32,7 +32,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_7_20190626=1; +int MAAT_FRAME_VERSION_2_7_20190629=1; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; @@ -849,7 +849,6 @@ struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Ma group->region_boundary=0; group->ref_by_parent_cnt=0; group->regions=dynamic_array_create(1,8); - group->compile_shortcut=NULL; group->table_id=table_id; group->group_name=NULL; group->vertex_id=scanner->grp_vertex_id_generator++; @@ -1948,27 +1947,6 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa } relation->group_cnt++; a_rule_group->ref_by_parent_cnt++; - //member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish. - if(a_rule_group->ref_by_parent_cnt==1&&relation->group_cnt==1) - { - a_rule_group->compile_shortcut=relation; - } - else - { - a_rule_group->compile_shortcut=NULL; - } - //update group's shortcut when compile has more than one group. - if(relation->group_cnt!=1) - { - for(i=0;igroup_boundary;i++) - { - p=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i); - if(p!=NULL) - { - p->compile_shortcut=NULL; - } - } - } ret=1; error_out: pthread_rwlock_unlock(&(relation->rwlock)); @@ -1994,13 +1972,7 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati { relation->not_group_cnt--; relation->not_flag[i]=0; - } - if(group_rule->compile_shortcut) - { - assert(group_rule->compile_shortcut==relation); - assert(group_rule->ref_by_parent_cnt==1&&relation->group_cnt==1); - group_rule->compile_shortcut=NULL; - } + } relation->group_cnt--; break; } @@ -2504,12 +2476,6 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro if(parent_group==NULL) { parent_group=create_group_rule(db_group_rule->parent_id, table->table_id, scanner); - } - //A sub group is not possile to have a compile shortcut. - if(group_rule->compile_shortcut) - { - assert(group_rule->ref_by_parent_cnt==1); - group_rule->compile_shortcut=NULL; } group_rule->ref_by_parent_cnt++; parent_group->ref_by_children_cnt++; @@ -4256,7 +4222,6 @@ void *thread_rule_monitor(void *arg) alignment_int64_array_free(feather->inner_mid_cnt); alignment_int64_array_free(feather->outer_mid_cnt); alignment_int64_array_free(feather->hit_cnt); - alignment_int64_array_free(feather->orphan_group_saving); alignment_int64_array_free(feather->not_grp_hit_cnt); if(feather->input_mode==SOURCE_REDIS) { diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index 112c612..f0573eb 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -21,7 +21,6 @@ enum MAAT_FS_STATUS{ STATUS_ICONV_ERR_CNT, STATUS_SCAN_ERR_CNT, STATUS_ZOMBIE_RS_STREAM, - STATUS_ORPHAN_GROUP_SAVING, STATUS_NOT_GROUP_HIT, STATUS_CMD_NUM, STATUS_CMD_Q_SIZE, @@ -73,7 +72,6 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid"); feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"z_stream"); - feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"shortcut_sv"); feather->fs_status_id[STATUS_NOT_GROUP_HIT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"nt_grp_hit"); feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes"); @@ -168,7 +166,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0; long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; long long outer_mid_cnt=0,inner_mid_cnt=0; - long long orphan_group_saving=0, not_grp_hit_cnt=0; + long long not_grp_hit_cnt=0; long long total_update_error=0,total_iconv_error=0; long long compile_rule_num=0, group_rule_num=0, not_group_rule_num=0, plugin_cache_num=0, plugin_acc_num=0; int i=0; @@ -183,7 +181,6 @@ void maat_stat_output(struct _Maat_feather_t* feather) active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num); - orphan_group_saving=alignment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num); not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt,feather->scan_thread_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); @@ -191,7 +188,6 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt); - FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING], 0,FS_OP_SET,orphan_group_saving); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_NUM], 0,FS_OP_SET,feather->cmd_acc_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0,FS_OP_SET,feather->cmd_q_cnt); diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index ce6e6a7..dcb31a0 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -164,7 +164,6 @@ struct Maat_group_inner int top_group_cnt; long long* top_groups; dynamic_array_t *regions; - struct Maat_compile_group_relation* compile_shortcut; pthread_mutex_t mutex; }; @@ -416,7 +415,6 @@ struct _Maat_feather_t mcore_long_t inner_mid_cnt; mcore_long_t hit_cnt; mcore_long_t thread_call_cnt;//size indicate by scan_thread_num, - mcore_long_t orphan_group_saving; mcore_long_t not_grp_hit_cnt; long long total_scan_bytes; long long total_scan_cnt;