移除compile快速路径相关代码:在支持子分组、“非”运算后,该特性对性能提升有限,却大幅增加代码复杂度。
This commit is contained in:
@@ -165,7 +165,6 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
|
||||
int scan_ret=0, result_cnt=0;
|
||||
int ret=0, i=0, j=0;
|
||||
size_t r_in_c_cnt=0;
|
||||
int shortcut_avilable_cnt=0;
|
||||
unsigned char has_not_flag=0;
|
||||
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
|
||||
struct Maat_group_inner* group_rule=NULL;
|
||||
@@ -183,11 +182,6 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
|
||||
}
|
||||
assert(group_rule->ref_by_children_cnt>=0);
|
||||
assert(group_rule->ref_by_parent_cnt>=0);
|
||||
if(group_rule->compile_shortcut!=NULL&&group_rule->ref_by_parent_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
|
||||
{
|
||||
relation_array[shortcut_avilable_cnt]=(struct Maat_compile_group_relation*)(group_rule->compile_shortcut);
|
||||
shortcut_avilable_cnt++;
|
||||
}
|
||||
for(j=0; j<group_rule->top_group_cnt; j++)
|
||||
{
|
||||
if(_mid->cur_hit_group_cnt<MAX_SCANNER_HIT_NUM)
|
||||
@@ -202,27 +196,11 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
|
||||
_mid->all_hit_group_cnt+=ret;
|
||||
}
|
||||
}
|
||||
if((region_hit_num>0 &&shortcut_avilable_cnt==region_hit_num) ||
|
||||
shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM)
|
||||
{
|
||||
//short cut for rules contains one group
|
||||
scan_ret=shortcut_avilable_cnt;
|
||||
alignment_int64_array_add(feather->orphan_group_saving, thread_num, 1);
|
||||
}
|
||||
else if(0&&shortcut_avilable_cnt==0&®ion_hit_num==1&&_mid->all_hit_group_cnt==1&&is_last_region==1)
|
||||
{
|
||||
//This shortcut is NO longger valid after bool macher support NOT-logic.
|
||||
//short cut for last scan and combination rules
|
||||
//region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule.
|
||||
//_mid->hit_group_cnt==1: With previously scan hitted group rules, one group may staisfy a compile rule
|
||||
scan_ret=0;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
scan_ret=bool_matcher_match(bm, thread_num,
|
||||
_mid->all_hit_group_array, _mid->all_hit_group_cnt,
|
||||
(void **)relation_array, MAX_SCANNER_HIT_NUM);
|
||||
}
|
||||
|
||||
if(scan_ret>1)
|
||||
{
|
||||
qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**),
|
||||
@@ -567,7 +545,6 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
|
||||
feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num);
|
||||
feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num);
|
||||
feather->hit_cnt=alignment_int64_array_alloc(max_thread_num);
|
||||
feather->orphan_group_saving=alignment_int64_array_alloc(max_thread_num);
|
||||
feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num);
|
||||
feather->maat_version=0;
|
||||
feather->last_full_version=0;
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
#include "stream_fuzzy_hash.h"
|
||||
#include "gram_index_engine.h"
|
||||
|
||||
int MAAT_FRAME_VERSION_2_7_20190626=1;
|
||||
int MAAT_FRAME_VERSION_2_7_20190629=1;
|
||||
|
||||
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
|
||||
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
|
||||
@@ -849,7 +849,6 @@ struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Ma
|
||||
group->region_boundary=0;
|
||||
group->ref_by_parent_cnt=0;
|
||||
group->regions=dynamic_array_create(1,8);
|
||||
group->compile_shortcut=NULL;
|
||||
group->table_id=table_id;
|
||||
group->group_name=NULL;
|
||||
group->vertex_id=scanner->grp_vertex_id_generator++;
|
||||
@@ -1948,27 +1947,6 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa
|
||||
}
|
||||
relation->group_cnt++;
|
||||
a_rule_group->ref_by_parent_cnt++;
|
||||
//member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish.
|
||||
if(a_rule_group->ref_by_parent_cnt==1&&relation->group_cnt==1)
|
||||
{
|
||||
a_rule_group->compile_shortcut=relation;
|
||||
}
|
||||
else
|
||||
{
|
||||
a_rule_group->compile_shortcut=NULL;
|
||||
}
|
||||
//update group's shortcut when compile has more than one group.
|
||||
if(relation->group_cnt!=1)
|
||||
{
|
||||
for(i=0;i<relation->group_boundary;i++)
|
||||
{
|
||||
p=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i);
|
||||
if(p!=NULL)
|
||||
{
|
||||
p->compile_shortcut=NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
ret=1;
|
||||
error_out:
|
||||
pthread_rwlock_unlock(&(relation->rwlock));
|
||||
@@ -1995,12 +1973,6 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati
|
||||
relation->not_group_cnt--;
|
||||
relation->not_flag[i]=0;
|
||||
}
|
||||
if(group_rule->compile_shortcut)
|
||||
{
|
||||
assert(group_rule->compile_shortcut==relation);
|
||||
assert(group_rule->ref_by_parent_cnt==1&&relation->group_cnt==1);
|
||||
group_rule->compile_shortcut=NULL;
|
||||
}
|
||||
relation->group_cnt--;
|
||||
break;
|
||||
}
|
||||
@@ -2505,12 +2477,6 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
|
||||
{
|
||||
parent_group=create_group_rule(db_group_rule->parent_id, table->table_id, scanner);
|
||||
}
|
||||
//A sub group is not possile to have a compile shortcut.
|
||||
if(group_rule->compile_shortcut)
|
||||
{
|
||||
assert(group_rule->ref_by_parent_cnt==1);
|
||||
group_rule->compile_shortcut=NULL;
|
||||
}
|
||||
group_rule->ref_by_parent_cnt++;
|
||||
parent_group->ref_by_children_cnt++;
|
||||
ret=igraph_get_eid(&scanner->group_graph, &edge_id, group_rule->vertex_id, parent_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
|
||||
@@ -4256,7 +4222,6 @@ void *thread_rule_monitor(void *arg)
|
||||
alignment_int64_array_free(feather->inner_mid_cnt);
|
||||
alignment_int64_array_free(feather->outer_mid_cnt);
|
||||
alignment_int64_array_free(feather->hit_cnt);
|
||||
alignment_int64_array_free(feather->orphan_group_saving);
|
||||
alignment_int64_array_free(feather->not_grp_hit_cnt);
|
||||
if(feather->input_mode==SOURCE_REDIS)
|
||||
{
|
||||
|
||||
@@ -21,7 +21,6 @@ enum MAAT_FS_STATUS{
|
||||
STATUS_ICONV_ERR_CNT,
|
||||
STATUS_SCAN_ERR_CNT,
|
||||
STATUS_ZOMBIE_RS_STREAM,
|
||||
STATUS_ORPHAN_GROUP_SAVING,
|
||||
STATUS_NOT_GROUP_HIT,
|
||||
STATUS_CMD_NUM,
|
||||
STATUS_CMD_Q_SIZE,
|
||||
@@ -73,7 +72,6 @@ void maat_stat_init(struct _Maat_feather_t* feather)
|
||||
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
|
||||
feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"z_stream");
|
||||
|
||||
feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"shortcut_sv");
|
||||
feather->fs_status_id[STATUS_NOT_GROUP_HIT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"nt_grp_hit");
|
||||
|
||||
feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes");
|
||||
@@ -168,7 +166,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
|
||||
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
|
||||
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
|
||||
long long outer_mid_cnt=0,inner_mid_cnt=0;
|
||||
long long orphan_group_saving=0, not_grp_hit_cnt=0;
|
||||
long long not_grp_hit_cnt=0;
|
||||
long long total_update_error=0,total_iconv_error=0;
|
||||
long long compile_rule_num=0, group_rule_num=0, not_group_rule_num=0, plugin_cache_num=0, plugin_acc_num=0;
|
||||
int i=0;
|
||||
@@ -183,7 +181,6 @@ void maat_stat_output(struct _Maat_feather_t* feather)
|
||||
active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
|
||||
outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
|
||||
inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num);
|
||||
orphan_group_saving=alignment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num);
|
||||
not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt,feather->scan_thread_num);
|
||||
|
||||
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
|
||||
@@ -191,7 +188,6 @@ void maat_stat_output(struct _Maat_feather_t* feather)
|
||||
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
|
||||
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt);
|
||||
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt);
|
||||
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING], 0,FS_OP_SET,orphan_group_saving);
|
||||
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt);
|
||||
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_NUM], 0,FS_OP_SET,feather->cmd_acc_num);
|
||||
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0,FS_OP_SET,feather->cmd_q_cnt);
|
||||
|
||||
@@ -164,7 +164,6 @@ struct Maat_group_inner
|
||||
int top_group_cnt;
|
||||
long long* top_groups;
|
||||
dynamic_array_t *regions;
|
||||
struct Maat_compile_group_relation* compile_shortcut;
|
||||
pthread_mutex_t mutex;
|
||||
};
|
||||
|
||||
@@ -416,7 +415,6 @@ struct _Maat_feather_t
|
||||
mcore_long_t inner_mid_cnt;
|
||||
mcore_long_t hit_cnt;
|
||||
mcore_long_t thread_call_cnt;//size indicate by scan_thread_num,
|
||||
mcore_long_t orphan_group_saving;
|
||||
mcore_long_t not_grp_hit_cnt;
|
||||
long long total_scan_bytes;
|
||||
long long total_scan_cnt;
|
||||
|
||||
Reference in New Issue
Block a user