gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

可以提示MAAT组合扫描中最后一个域，可以提升与表达式运算性能。

Browse Source
This commit is contained in:
zhengchao
2017-06-24 21:38:11 +08:00
parent 8c4139c1bd
commit 255f0392be
6 changed files with 73 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
inc/Maat_rule.h
View File

@@ -163,7 +163,8 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
enum MAAT_SCAN_OPT
{
MAAT_SET_SCAN_DISTRICT=1 //VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default.
MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default.
MAAT_SET_SCAN_LAST_REGION //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan cobination.
};
//return 0 if success, return -1 when failed;
int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCAN_OPT type,const void* value,int size);

47
src/entry/Maat_api.cpp
View File

@@ -98,7 +98,7 @@ int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsi
}
return k;
}
int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
{
int scan_ret=0,result_cnt=0;
@@ -135,6 +135,14 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,voi
{
//short cut for rules contains one group
scan_ret=shortcut_avilable_cnt;
aligment_int64_array_add(feather->orphan_group_saving, thread_num, 1);
}
else if(shortcut_avilable_cnt==0&&region_hit_num==1&&is_last_region==1)
{
//short cut for last scan and combination rules
//One groups may belong to a statisfy a compile rule, so only region_hit_num=1 could use the short cut
scan_ret=0;
aligment_int64_array_add(feather->last_region_saving, thread_num, 1);
}
else
{
@@ -457,6 +465,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->outer_mid_cnt=aligment_int64_array_alloc(max_thread_num);
feather->inner_mid_cnt=aligment_int64_array_alloc(max_thread_num);
feather->hit_cnt=aligment_int64_array_alloc(max_thread_num);
feather->orphan_group_saving=aligment_int64_array_alloc(max_thread_num);
feather->last_region_saving=aligment_int64_array_alloc(max_thread_num);
feather->maat_version=0;
feather->last_full_version=0;
pthread_mutex_init(&(feather->plugin_table_reg_mutex),NULL);
@@ -843,10 +853,16 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
aligment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
hit_region_cnt,
result,compile_result,rule_num,
thread_num);
assert(_mid->is_last_region<2);
if(_mid->is_last_region==1)
{
_mid->is_last_region=2;
}
if(hit_detail!=NULL&&_feather->rule_scan_type!=0)
{
*detail_ret=fill_region_hit_detail(data,_mid->inner,
@@ -943,10 +959,16 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
aligment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
region_ret,
result,compile_result,rule_num,
thread_num);
assert(_mid->is_last_region<2);
if(_mid->is_last_region==1)
{
_mid->is_last_region=2;
}
}
DEC_SCANNER_REF(my_scanner,thread_num);
@@ -1058,11 +1080,18 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{
aligment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
region_ret,
result,compile_result,rule_num,
thread_num);
assert(_mid->is_last_region<2);
if(_mid->is_last_region==1)
{
_mid->is_last_region=2;
}
}
DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1)
@@ -1272,10 +1301,16 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
aligment_int64_array_add(p_table->hit_cnt, sp->thread_num,1);
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
hit_region_cnt,
result,compile_result,rule_num,
sp->thread_num);
assert(_mid->is_last_region<2);
if(_mid->is_last_region==1)
{
_mid->is_last_region=2;
}
if(hit_detail!=NULL&&sp->feather->rule_scan_type!=0)
{
if(sp->scan_buff!=NULL)
@@ -1501,11 +1536,16 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
sp->feather->p_table_info[sp->table_id]->hit_cnt++;
_mid=grab_mid(mid,sp->feather, sp->thread_num,1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag),
hit_region_cnt,
result,compile_result,rule_num,
sp->thread_num);
assert(_mid->is_last_region<2);
if(_mid->is_last_region==1)
{
_mid->is_last_region=2;
}
}
fast_out:
if(sp->feather->perf_on==1)
@@ -1576,6 +1616,9 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA
}
_mid->is_set_district=1;
break;
case MAAT_SET_SCAN_LAST_REGION:
_mid->is_last_region=1;
break;
default:
_feather->scan_err_cnt++;
return -1;

4
src/entry/Maat_rule.cpp
View File

@@ -28,7 +28,7 @@
#include "mesa_fuzzy.h"
#include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_9_20170615=1;
int MAAT_FRAME_VERSION_1_9_20170624=1;
const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
@@ -3080,6 +3080,8 @@ void *thread_rule_monitor(void *arg)
aligment_int64_array_free(feather->inner_mid_cnt);
aligment_int64_array_free(feather->outer_mid_cnt);
aligment_int64_array_free(feather->hit_cnt);
aligment_int64_array_free(feather->orphan_group_saving);
aligment_int64_array_free(feather->last_region_saving);
free(feather);
return NULL;
}

5
src/entry/Maat_rule_internal.h
View File

@@ -284,7 +284,8 @@ struct _OUTER_scan_status_t
{
struct _Maat_feather_t* feather;
unsigned short thread_num;
unsigned short is_set_district;
unsigned char is_set_district;
unsigned char is_last_region;
int district_id;
struct _INNER_scan_status_t* inner;
};
@@ -382,6 +383,8 @@ struct _Maat_feather_t
mcore_long_t inner_mid_cnt;
mcore_long_t hit_cnt;
mcore_long_t thread_call_cnt;//size indicate by scan_thread_num,
mcore_long_t orphan_group_saving;
mcore_long_t last_region_saving;
long long total_scan_bytes;
long long total_scan_cnt;
long long update_err_cnt;//sum of the same name variable in each table

15
src/entry/Maat_stat.cpp
View File

@@ -19,7 +19,9 @@ enum MAAT_FS_STATUS{
STATUS_UPDATE_ERR_CNT,
STATUS_ICONV_ERR_CNT,
STATUS_SCAN_ERR_CNT,
STATUS_ZOMBIE_RS_STREAM
STATUS_ZOMBIE_RS_STREAM,
STATUS_ORPHAN_GROUP_SAVING,
STATUS_LAST_REGION_SAVING
};
enum MAAT_FS_COLUMN
@@ -54,7 +56,7 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_cache");
feather->fs_status_id[STATUS_PLUGIN_ACC_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_acc");
feather->fs_status_id[STATUS_GROUP_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"group_num");
feather->fs_status_id[STATUS_GROUP_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"grp_num");
feather->fs_status_id[STATUS_COMPILE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"compile_num");
feather->fs_status_id[STATUS_POSTPONE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"postpone_num");
@@ -62,6 +64,8 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid");
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"orphan_grp_sv");
feather->fs_status_id[STATUS_LAST_REGION_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"last_rgn_sv");
feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes");
feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_times");
@@ -70,7 +74,6 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_SCAN_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_error");
feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"zombie_stream");
feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule");
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex");
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream");
@@ -154,6 +157,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0;
long long orphan_group_saving=0, last_region_saving=0;
long long total_update_error=0,total_iconv_error=0;
long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0;
int i=0;
@@ -163,11 +167,16 @@ void maat_stat_output(struct _Maat_feather_t* feather)
active_thread_num=aligment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=aligment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
inner_mid_cnt=aligment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num);
orphan_group_saving=aligment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num);
last_region_saving=aligment_int64_array_sum(feather->last_region_saving,feather->scan_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING], 0,FS_OP_SET,orphan_group_saving);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_LAST_REGION_SAVING], 0,FS_OP_SET,last_region_saving);
value=MESA_lqueue_get_count(feather->garbage_q);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);

2
test/maat_test.cpp
View File

@@ -221,6 +221,8 @@ int test_ipv6_scan(Maat_feather_t feather,const char* table_name,scan_status_t*
}
else
{
//for improving performance.
Maat_set_scan_status(feather, mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0);
ret=Maat_scan_proto_addr(feather,table_id,&ipv6_addr,6,result,4, mid,0);
printf("IPv6 addr Scan:%s\n",print_maat_result(result,ret));