gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

移除compile快速路径相关代码：在支持子分组、“非”运算后，该特性对性能提升有限，却大幅增加代码复杂度。

Browse Source
This commit is contained in:
zhengchao
2019-06-29 21:57:24 +06:00
parent 14a2691577
commit b5ad82afe9
4 changed files with 8 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/entry/Maat_api.cpp
View File

@@ -165,7 +165,6 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
int scan_ret=0, result_cnt=0; int scan_ret=0, result_cnt=0;
int ret=0, i=0, j=0; int ret=0, i=0, j=0;
size_t r_in_c_cnt=0; size_t r_in_c_cnt=0;
int shortcut_avilable_cnt=0;
unsigned char has_not_flag=0; unsigned char has_not_flag=0;
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
struct Maat_group_inner* group_rule=NULL; struct Maat_group_inner* group_rule=NULL;
@@ -183,11 +182,6 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
} }
assert(group_rule->ref_by_children_cnt>=0); assert(group_rule->ref_by_children_cnt>=0);
assert(group_rule->ref_by_parent_cnt>=0); assert(group_rule->ref_by_parent_cnt>=0);
if(group_rule->compile_shortcut!=NULL&&group_rule->ref_by_parent_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
{
relation_array[shortcut_avilable_cnt]=(struct Maat_compile_group_relation*)(group_rule->compile_shortcut);
shortcut_avilable_cnt++;
}
for(j=0; j<group_rule->top_group_cnt; j++) for(j=0; j<group_rule->top_group_cnt; j++)
{ {
if(_mid->cur_hit_group_cnt<MAX_SCANNER_HIT_NUM) if(_mid->cur_hit_group_cnt<MAX_SCANNER_HIT_NUM)
@@ -202,27 +196,11 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
_mid->all_hit_group_cnt+=ret; _mid->all_hit_group_cnt+=ret;
} }
} }
if((region_hit_num>0 &&shortcut_avilable_cnt==region_hit_num) ||
shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM)
{
//short cut for rules contains one group
scan_ret=shortcut_avilable_cnt;
alignment_int64_array_add(feather->orphan_group_saving, thread_num, 1);
}
else if(0&&shortcut_avilable_cnt==0&&region_hit_num==1&&_mid->all_hit_group_cnt==1&&is_last_region==1)
{
//This shortcut is NO longger valid after bool macher support NOT-logic.
//short cut for last scan and combination rules
//region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule.
//_mid->hit_group_cnt==1: With previously scan hitted group rules, one group may staisfy a compile rule
scan_ret=0;
}
else
{
scan_ret=bool_matcher_match(bm, thread_num, scan_ret=bool_matcher_match(bm, thread_num,
_mid->all_hit_group_array, _mid->all_hit_group_cnt, _mid->all_hit_group_array, _mid->all_hit_group_cnt,
(void **)relation_array, MAX_SCANNER_HIT_NUM); (void **)relation_array, MAX_SCANNER_HIT_NUM);
}
if(scan_ret>1) if(scan_ret>1)
{ {
qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**), qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation**),
@@ -567,7 +545,6 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num); feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num); feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather->hit_cnt=alignment_int64_array_alloc(max_thread_num); feather->hit_cnt=alignment_int64_array_alloc(max_thread_num);
feather->orphan_group_saving=alignment_int64_array_alloc(max_thread_num);
feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num); feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num);
feather->maat_version=0; feather->maat_version=0;
feather->last_full_version=0; feather->last_full_version=0;

37
src/entry/Maat_rule.cpp
View File

@@ -32,7 +32,7 @@
#include "stream_fuzzy_hash.h" #include "stream_fuzzy_hash.h"
#include "gram_index_engine.h" #include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_7_20190626=1; int MAAT_FRAME_VERSION_2_7_20190629=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
@@ -849,7 +849,6 @@ struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Ma
group->region_boundary=0; group->region_boundary=0;
group->ref_by_parent_cnt=0; group->ref_by_parent_cnt=0;
group->regions=dynamic_array_create(1,8); group->regions=dynamic_array_create(1,8);
group->compile_shortcut=NULL;
group->table_id=table_id; group->table_id=table_id;
group->group_name=NULL; group->group_name=NULL;
group->vertex_id=scanner->grp_vertex_id_generator++; group->vertex_id=scanner->grp_vertex_id_generator++;
@@ -1948,27 +1947,6 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa
} }
relation->group_cnt++; relation->group_cnt++;
a_rule_group->ref_by_parent_cnt++; a_rule_group->ref_by_parent_cnt++;
//member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish.
if(a_rule_group->ref_by_parent_cnt==1&&relation->group_cnt==1)
{
a_rule_group->compile_shortcut=relation;
}
else
{
a_rule_group->compile_shortcut=NULL;
}
//update group's shortcut when compile has more than one group.
if(relation->group_cnt!=1)
{
for(i=0;i<relation->group_boundary;i++)
{
p=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i);
if(p!=NULL)
{
p->compile_shortcut=NULL;
}
}
}
ret=1; ret=1;
error_out: error_out:
pthread_rwlock_unlock(&(relation->rwlock)); pthread_rwlock_unlock(&(relation->rwlock));
@@ -1995,12 +1973,6 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati
relation->not_group_cnt--; relation->not_group_cnt--;
relation->not_flag[i]=0; relation->not_flag[i]=0;
} }
if(group_rule->compile_shortcut)
{
assert(group_rule->compile_shortcut==relation);
assert(group_rule->ref_by_parent_cnt==1&&relation->group_cnt==1);
group_rule->compile_shortcut=NULL;
}
relation->group_cnt--; relation->group_cnt--;
break; break;
} }
@@ -2505,12 +2477,6 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
{ {
parent_group=create_group_rule(db_group_rule->parent_id, table->table_id, scanner); parent_group=create_group_rule(db_group_rule->parent_id, table->table_id, scanner);
} }
//A sub group is not possile to have a compile shortcut.
if(group_rule->compile_shortcut)
{
assert(group_rule->ref_by_parent_cnt==1);
group_rule->compile_shortcut=NULL;
}
group_rule->ref_by_parent_cnt++; group_rule->ref_by_parent_cnt++;
parent_group->ref_by_children_cnt++; parent_group->ref_by_children_cnt++;
ret=igraph_get_eid(&scanner->group_graph, &edge_id, group_rule->vertex_id, parent_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); ret=igraph_get_eid(&scanner->group_graph, &edge_id, group_rule->vertex_id, parent_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
@@ -4256,7 +4222,6 @@ void *thread_rule_monitor(void *arg)
alignment_int64_array_free(feather->inner_mid_cnt); alignment_int64_array_free(feather->inner_mid_cnt);
alignment_int64_array_free(feather->outer_mid_cnt); alignment_int64_array_free(feather->outer_mid_cnt);
alignment_int64_array_free(feather->hit_cnt); alignment_int64_array_free(feather->hit_cnt);
alignment_int64_array_free(feather->orphan_group_saving);
alignment_int64_array_free(feather->not_grp_hit_cnt); alignment_int64_array_free(feather->not_grp_hit_cnt);
if(feather->input_mode==SOURCE_REDIS) if(feather->input_mode==SOURCE_REDIS)
{ {

6
src/entry/Maat_stat.cpp
View File

@@ -21,7 +21,6 @@ enum MAAT_FS_STATUS{
STATUS_ICONV_ERR_CNT, STATUS_ICONV_ERR_CNT,
STATUS_SCAN_ERR_CNT, STATUS_SCAN_ERR_CNT,
STATUS_ZOMBIE_RS_STREAM, STATUS_ZOMBIE_RS_STREAM,
STATUS_ORPHAN_GROUP_SAVING,
STATUS_NOT_GROUP_HIT, STATUS_NOT_GROUP_HIT,
STATUS_CMD_NUM, STATUS_CMD_NUM,
STATUS_CMD_Q_SIZE, STATUS_CMD_Q_SIZE,
@@ -73,7 +72,6 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid"); feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"z_stream"); feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"z_stream");
feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"shortcut_sv");
feather->fs_status_id[STATUS_NOT_GROUP_HIT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"nt_grp_hit"); feather->fs_status_id[STATUS_NOT_GROUP_HIT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"nt_grp_hit");
feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes"); feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes");
@@ -168,7 +166,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0; long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0; long long outer_mid_cnt=0,inner_mid_cnt=0;
long long orphan_group_saving=0, not_grp_hit_cnt=0; long long not_grp_hit_cnt=0;
long long total_update_error=0,total_iconv_error=0; long long total_update_error=0,total_iconv_error=0;
long long compile_rule_num=0, group_rule_num=0, not_group_rule_num=0, plugin_cache_num=0, plugin_acc_num=0; long long compile_rule_num=0, group_rule_num=0, not_group_rule_num=0, plugin_cache_num=0, plugin_acc_num=0;
int i=0; int i=0;
@@ -183,7 +181,6 @@ void maat_stat_output(struct _Maat_feather_t* feather)
active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num); inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num);
orphan_group_saving=alignment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num);
not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt,feather->scan_thread_num); not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
@@ -191,7 +188,6 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING], 0,FS_OP_SET,orphan_group_saving);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_NUM], 0,FS_OP_SET,feather->cmd_acc_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_NUM], 0,FS_OP_SET,feather->cmd_acc_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0,FS_OP_SET,feather->cmd_q_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0,FS_OP_SET,feather->cmd_q_cnt);

2
src/inc_internal/Maat_rule_internal.h
View File

@@ -164,7 +164,6 @@ struct Maat_group_inner
int top_group_cnt; int top_group_cnt;
long long* top_groups; long long* top_groups;
dynamic_array_t *regions; dynamic_array_t *regions;
struct Maat_compile_group_relation* compile_shortcut;
pthread_mutex_t mutex; pthread_mutex_t mutex;
}; };
@@ -416,7 +415,6 @@ struct _Maat_feather_t
mcore_long_t inner_mid_cnt; mcore_long_t inner_mid_cnt;
mcore_long_t hit_cnt; mcore_long_t hit_cnt;
mcore_long_t thread_call_cnt;//size indicate by scan_thread_num, mcore_long_t thread_call_cnt;//size indicate by scan_thread_num,
mcore_long_t orphan_group_saving;
mcore_long_t not_grp_hit_cnt; mcore_long_t not_grp_hit_cnt;
long long total_scan_bytes; long long total_scan_bytes;
long long total_scan_cnt; long long total_scan_cnt;