gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

将规则数量和扫描状态的统计归集到table runtime结构体中，属于scanner的一部分。

Browse Source
This commit is contained in:
zhengchao
2018-12-04 20:12:56 +08:00
parent 895344d7d2
commit 6971f4cb56
8 changed files with 660 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
inc/gram_index_engine.h
View File

@@ -4,12 +4,16 @@
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
enum GIE_operation
#define GIE_INSERT_OPT 0 {
#define GIE_DELETE_OPT 1 GIE_INSERT_OPT,
#define GIE_INPUT_FORMAT_SFH 1 GIE_DELETE_OPT
#define GIE_INPUT_FORMAT_PLAIN 0 };
enum GIE_INPUT_FORMAT
{
GIE_INPUT_FORMAT_PLAIN,
GIE_INPUT_FORMAT_SFH
};
typedef struct typedef struct
{ {
@@ -21,7 +25,7 @@ typedef struct
{ {
unsigned int id; unsigned int id;
unsigned int sfh_length;//size of fuzzy_hash unsigned int sfh_length;//size of fuzzy_hash
short operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed; enum GIE_operation operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed;
short cfds_lvl; short cfds_lvl;
char * sfh; char * sfh;
void * tag; void * tag;
@@ -41,9 +45,9 @@ typedef struct
unsigned int gram_value; unsigned int gram_value;
//unsigned int htable_num; //unsigned int htable_num;
unsigned int position_accuracy; unsigned int position_accuracy;
short format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string enum GIE_INPUT_FORMAT format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string
//else id format==PALIN, means the input string is common string //else id format==PALIN, means the input string is common string
short ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result int ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result
}GIE_create_para_t; }GIE_create_para_t;

203
src/entry/Maat_api.cpp
View File

@@ -919,6 +919,8 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int idx=0,i=0; int idx=0,i=0;
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id]; struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
struct plugin_table_desc *plugin_desc=&(p_table->plugin);
struct plugin_runtime* plugin_aux=NULL;
const char* lines=NULL; const char* lines=NULL;
if(p_table==NULL) if(p_table==NULL)
{ {
@@ -928,28 +930,29 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
{ {
return -1; return -1;
} }
plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin);
//plugin table register blocks background update. //plugin table register blocks background update.
pthread_mutex_lock(&(_feather->backgroud_update_mutex)); pthread_mutex_lock(&(_feather->backgroud_update_mutex));
idx=p_table->cb_info->cb_plug_cnt; idx=plugin_desc->cb_plug_cnt;
if(idx==MAX_PLUGIN_PER_TABLE) if(idx==MAX_PLUGIN_PER_TABLE)
{ {
pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return -1; return -1;
} }
p_table->cb_info->cb_plug_cnt++; plugin_desc->cb_plug_cnt++;
p_table->cb_info->cb_plug[idx].start=start; plugin_desc->cb_plug[idx].start=start;
p_table->cb_info->cb_plug[idx].update=update; plugin_desc->cb_plug[idx].update=update;
p_table->cb_info->cb_plug[idx].finish=finish; plugin_desc->cb_plug[idx].finish=finish;
p_table->cb_info->cb_plug[idx].u_para=u_para; plugin_desc->cb_plug[idx].u_para=u_para;
if(p_table->cb_info->cache_line_num>0) if(plugin_aux->cache_line_num>0)
{ {
if(start!=NULL) if(start!=NULL)
{ {
start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); start(MAAT_RULE_UPDATE_TYPE_FULL,u_para);
} }
for(i=0;i<p_table->cb_info->cache_line_num;i++) for(i=0;i<plugin_aux->cache_line_num;i++)
{ {
lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i); lines=(const char*)dynamic_array_read(plugin_aux->cache_lines,i);
if(lines==NULL) if(lines==NULL)
{ {
break; break;
@@ -1001,26 +1004,26 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
{ {
return -1; return -1;
} }
struct compile_table_desc* compile_desc=&(p_table->compile);
pthread_mutex_lock(&(_feather->backgroud_update_mutex)); pthread_mutex_lock(&(_feather->backgroud_update_mutex));
if(p_table->ex_data_num==MAX_COMPILE_EX_DATA_NUM) if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
{ {
ret=-1; ret=-1;
goto failed; goto failed;
} }
idx=p_table->ex_data_num; idx=compile_desc->ex_data_num;
p_table->ex_desc[idx].idx=idx; compile_desc->ex_desc[idx].idx=idx;
p_table->ex_desc[idx].table_id=table_id; compile_desc->ex_desc[idx].table_id=table_id;
p_table->ex_desc[idx].argl=argl; compile_desc->ex_desc[idx].argl=argl;
p_table->ex_desc[idx].argp=argp; compile_desc->ex_desc[idx].argp=argp;
p_table->ex_desc[idx].new_func=new_func; compile_desc->ex_desc[idx].new_func=new_func;
p_table->ex_desc[idx].free_func=free_func; compile_desc->ex_desc[idx].free_func=free_func;
p_table->ex_desc[idx].dup_func=dup_func; compile_desc->ex_desc[idx].dup_func=dup_func;
p_table->ex_data_num++; compile_desc->ex_data_num++;
if(_feather->scanner!=NULL) if(_feather->scanner!=NULL)
{ {
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, p_table->ex_desc+idx); MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_desc->ex_desc+idx);
} }
failed: failed:
pthread_mutex_unlock(&(_feather->backgroud_update_mutex)); pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
@@ -1031,7 +1034,8 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
{ {
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct _Maat_compile_inner_t *compile_inner=NULL; struct _Maat_compile_inner_t *compile_inner=NULL;
const struct compile_ex_data_idx* ex_desc=NULL; const struct compile_table_desc* compile_desc=NULL;
const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL; MAAT_RULE_EX_DATA ad=NULL;
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
@@ -1040,8 +1044,9 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
return NULL; return NULL;
} }
pthread_rwlock_rdlock(&(compile_inner->rwlock)); pthread_rwlock_rdlock(&(compile_inner->rwlock));
assert(idx<compile_inner->ref_table->ex_data_num); compile_desc=&(compile_inner->ref_table->compile);
ex_desc=compile_inner->ref_table->ex_desc+idx; assert(idx<compile_desc->ex_data_num);
ex_desc=compile_desc->ex_desc+idx;
ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp); ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp);
pthread_rwlock_unlock(&(compile_inner->rwlock)); pthread_rwlock_unlock(&(compile_inner->rwlock));
return ad; return ad;
@@ -1054,7 +1059,7 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
long argl, void *argp) long argl, void *argp)
{ {
return 0;
} }
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
@@ -1070,6 +1075,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
scan_result_t *region_result=NULL; scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];//dynamic array _compile_result_t compile_result[rule_num];//dynamic array
struct _Maat_table_info_t *p_table=NULL; struct _Maat_table_info_t *p_table=NULL;
struct expr_table_desc* expr_desc=NULL;
struct timespec start,end; struct timespec start,end;
_Maat_scanner_t* my_scanner=NULL; _Maat_scanner_t* my_scanner=NULL;
if(data==NULL||data_len<=0) if(data==NULL||data_len<=0)
@@ -1087,17 +1093,13 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
return -1; return -1;
} }
expr_desc=&(p_table->expr);
if(p_table->cfg_num==0)
{
return 0;
}
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{ {
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
return -1; return -1;
} }
if(p_table->do_charset_merge==1) if(expr_desc->do_charset_merge==1)
{ {
sub_type=make_sub_type(table_id,CHARSET_NONE,0); sub_type=make_sub_type(table_id,CHARSET_NONE,0);
} }
@@ -1115,9 +1117,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{ {
return 0; return 0;
} }
struct _region_stat_t * region_stat=NULL; struct table_runtime* table_aux=my_scanner->table_rt[table_id];
region_stat=&(my_scanner->region_counter[p_table->table_id]); if(table_aux->origin_rule_num==0)
if(region_stat->cfg_num==0)
{ {
return 0; return 0;
} }
@@ -1125,7 +1126,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
INC_SCANNER_REF(my_scanner, thread_num); INC_SCANNER_REF(my_scanner, thread_num);
if(region_stat->expr_rule_cnt>0) if(table_aux->expr.expr_rule_cnt>0)
{ {
scan_data.rule_type=RULETYPE_STR; scan_data.rule_type=RULETYPE_STR;
scan_data.sub_type=sub_type; scan_data.sub_type=sub_type;
@@ -1135,7 +1136,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
hit_region_cnt+=region_ret; hit_region_cnt+=region_ret;
} }
} }
if(region_stat->regex_rule_cnt>0) if(table_aux->expr.regex_rule_cnt>0)
{ {
scan_data.rule_type=RULETYPE_REG; scan_data.rule_type=RULETYPE_REG;
scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0);
@@ -1152,7 +1153,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
alignment_int64_array_add(p_table->hit_cnt, thread_num,1); alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
_mid=grab_mid(mid,_feather,thread_num, 1); _mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -1173,17 +1174,16 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
hit_detail,detail_num); hit_detail,detail_num);
} }
} }
DEC_SCANNER_REF(my_scanner, thread_num);
if(_feather->perf_on==1) if(_feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,data_len,&start, &end,thread_num); maat_stat_table(table_aux, data_len, &start, &end, thread_num);
} }
else else
{ {
maat_stat_table(p_table,data_len,NULL, NULL,thread_num); maat_stat_table(table_aux, data_len, NULL, NULL, thread_num);
} }
DEC_SCANNER_REF(my_scanner, thread_num);
if(compile_ret==0&&hit_region_cnt>0) if(compile_ret==0&&hit_region_cnt>0)
{ {
return -2; return -2;
@@ -1229,18 +1229,13 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
return -1; return -1;
} }
if(p_table->cfg_num==0)
{
return 0;
}
my_scanner=_feather->scanner; my_scanner=_feather->scanner;
if(my_scanner==NULL) if(my_scanner==NULL)
{ {
return 0; return 0;
} }
struct _region_stat_t * region_stat=NULL; struct table_runtime* table_aux=my_scanner->table_rt[table_id];
region_stat=&(my_scanner->region_counter[p_table->table_id]); if(table_aux->origin_rule_num==0)
if(region_stat->cfg_num==0)
{ {
return 0; return 0;
} }
@@ -1258,7 +1253,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
alignment_int64_array_add(p_table->hit_cnt, thread_num,1); alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -1273,16 +1268,17 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
} }
} }
DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1) if(_feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end,thread_num); maat_stat_table(table_aux, 0, &start, &end, thread_num);
} }
else else
{ {
maat_stat_table(p_table,0,NULL, NULL,thread_num); maat_stat_table(table_aux, 0, NULL, NULL, thread_num);
} }
DEC_SCANNER_REF(my_scanner,thread_num);
if(compile_ret==0&&region_ret>0) if(compile_ret==0&&region_ret>0)
{ {
return -2; return -2;
@@ -1316,26 +1312,22 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
return -1; return -1;
} }
if(p_table->cfg_num==0)
{
return 0;
}
my_scanner=_feather->scanner; my_scanner=_feather->scanner;
if(my_scanner==NULL) if(my_scanner==NULL)
{ {
return 0; return 0;
} }
struct _region_stat_t * region_stat=NULL; struct table_runtime* table_aux=my_scanner->table_rt[table_id];
region_stat=&(my_scanner->region_counter[p_table->table_id]); if(table_aux->origin_rule_num==0)
if(region_stat->cfg_num==0)
{ {
return 0; return 0;
} }
if(region_stat->ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
if(table_aux->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
{ {
return 0; return 0;
} }
if(region_stat->ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6) if(table_aux->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
{ {
return 0; return 0;
} }
@@ -1380,7 +1372,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
alignment_int64_array_add(p_table->hit_cnt, thread_num,1); alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
@@ -1399,11 +1391,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
if(_feather->perf_on==1) if(_feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end,thread_num); maat_stat_table(table_aux, 0, &start, &end, thread_num);
} }
else else
{ {
maat_stat_table(p_table,0,NULL, NULL,thread_num); maat_stat_table(table_aux, 0, NULL, NULL, thread_num);
} }
if(compile_ret==0&&region_ret>0) if(compile_ret==0&&region_ret>0)
{ {
@@ -1436,12 +1428,14 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
return NULL; return NULL;
} }
if(p_table->quick_expr_switch==1)
struct expr_table_desc* expr_desc=&(p_table->expr);
if(expr_desc->quick_expr_switch==1)
{ {
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
return NULL; return NULL;
} }
struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1);
scanner=_feather->scanner; scanner=_feather->scanner;
sp->feather=_feather; sp->feather=_feather;
sp->version=_feather->maat_version; sp->version=_feather->maat_version;
@@ -1451,29 +1445,32 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{ {
return sp; return sp;
} }
struct _region_stat_t * region_stat=NULL; struct table_runtime* table_aux=scanner->table_rt[table_id];
region_stat=&(scanner->region_counter[p_table->table_id]); if(table_aux->origin_rule_num==0)
{
return 0;
}
INC_SCANNER_REF(scanner, thread_num); INC_SCANNER_REF(scanner, thread_num);
sp->table_id=table_id; sp->table_id=table_id;
sp->thread_num=thread_num; sp->thread_num=thread_num;
sp->max_cross_size=p_table->cross_cache_size; sp->max_cross_size=expr_desc->cross_cache_size;
sp->caching_size=0; sp->caching_size=0;
sp->scan_buff=NULL; sp->scan_buff=NULL;
sp->last_cache=NULL; sp->last_cache=NULL;
if(p_table->do_charset_merge==1) if(expr_desc->do_charset_merge==1)
{ {
sp->do_merge=1; sp->do_merge=1;
} }
if(region_stat->expr_rule_cnt>0) if(table_aux->expr.expr_rule_cnt>0)
{ {
sp->do_expr=1; sp->do_expr=1;
} }
if(region_stat->regex_rule_cnt>0) if(table_aux->expr.regex_rule_cnt>0)
{ {
sp->do_regex=1; sp->do_regex=1;
} }
alignment_int64_array_add(p_table->stream_num,thread_num,1); alignment_int64_array_add(table_aux->stream_num,thread_num,1);
sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num);
return sp; return sp;
} }
@@ -1493,6 +1490,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
_compile_result_t compile_result[rule_num];//dynamic array _compile_result_t compile_result[rule_num];//dynamic array
scan_data_t region_scan_data; scan_data_t region_scan_data;
_Maat_table_info_t* p_table=NULL; _Maat_table_info_t* p_table=NULL;
struct table_runtime* table_aux=scanner->table_rt[sp->table_id];
struct timespec start,end; struct timespec start,end;
if(data==NULL||data_len<=0) if(data==NULL||data_len<=0)
{ {
@@ -1539,7 +1537,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
free(sp->scan_buff); free(sp->scan_buff);
sp->scan_buff=NULL; sp->scan_buff=NULL;
} }
sp->scan_buff=(char*)malloc(sp->caching_size+data_len); sp->scan_buff=ALLOC(char, sp->caching_size+data_len);
memcpy(sp->scan_buff,sp->last_cache,sp->caching_size); memcpy(sp->scan_buff,sp->last_cache,sp->caching_size);
memcpy(sp->scan_buff+sp->caching_size,data,data_len); memcpy(sp->scan_buff+sp->caching_size,data,data_len);
region_scan_data.text_data.text=sp->scan_buff; region_scan_data.text_data.text=sp->scan_buff;
@@ -1554,7 +1552,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->last_cache==NULL&&sp->max_cross_size>0) if(sp->last_cache==NULL&&sp->max_cross_size>0)
{ {
assert(sp->caching_size==0); assert(sp->caching_size==0);
sp->last_cache=(char*)malloc(sizeof(char)*sp->max_cross_size); sp->last_cache=ALLOC(char, sp->max_cross_size);
} }
if(sp->max_cross_size>0) if(sp->max_cross_size>0)
{ {
@@ -1600,7 +1598,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
alignment_int64_array_add(p_table->hit_cnt, sp->thread_num,1); alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num,1);
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -1639,11 +1637,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->feather->perf_on==1) if(sp->feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num); maat_stat_table(table_aux,data_len,&start, &end,sp->thread_num);
} }
else else
{ {
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num); maat_stat_table(table_aux,data_len,NULL, NULL,sp->thread_num);
} }
if(compile_ret==0&&hit_region_cnt>0) if(compile_ret==0&&hit_region_cnt>0)
{ {
@@ -1668,8 +1666,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
{ {
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; struct table_runtime* table_aux=scanner->table_rt[sp->table_id];
alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1); alignment_int64_array_add(table_aux->stream_num, sp->thread_num, -1);
if(sp->rs_stream_para!=NULL) if(sp->rs_stream_para!=NULL)
{ {
if(scanner!=NULL&&sp->version>=sp->feather->last_full_version) if(scanner!=NULL&&sp->version>=sp->feather->last_full_version)
@@ -1712,8 +1710,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
{ {
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
return NULL; return NULL;
} }
struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1); struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1);
scanner=_feather->scanner; scanner=_feather->scanner;
sp->feather=_feather; sp->feather=_feather;
sp->version=_feather->maat_version; sp->version=_feather->maat_version;
@@ -1721,7 +1719,8 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
if(scanner==NULL) if(scanner==NULL)
{ {
return sp; return sp;
} }
struct table_runtime* table_aux=scanner->table_rt[table_id];
tmp_fuzzy_handle=SFH_instance(total_len); tmp_fuzzy_handle=SFH_instance(total_len);
if(tmp_fuzzy_handle==NULL) if(tmp_fuzzy_handle==NULL)
{ {
@@ -1735,7 +1734,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sp->total_len=total_len; sp->total_len=total_len;
sp->fuzzy_hash_handle=tmp_fuzzy_handle; sp->fuzzy_hash_handle=tmp_fuzzy_handle;
pthread_mutex_init(&(sp->fuzzy_mutex),NULL); pthread_mutex_init(&(sp->fuzzy_mutex),NULL);
alignment_int64_array_add(p_table->stream_num,thread_num,1); alignment_int64_array_add(table_aux->stream_num,thread_num,1);
return sp; return sp;
} }
@@ -1772,7 +1771,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
GIE_result_t query_result[MAX_SCANNER_HIT_NUM]; GIE_result_t query_result[MAX_SCANNER_HIT_NUM];
int hit_region_cnt=0,compile_ret=0; int hit_region_cnt=0,compile_ret=0;
_compile_result_t compile_result[rule_num];//dynamic array _compile_result_t compile_result[rule_num];//dynamic array
_Maat_table_info_t* p_table=NULL;
if(data==NULL||data_len<=0) if(data==NULL||data_len<=0)
{ {
return 0; return 0;
@@ -1781,7 +1779,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{ {
return 0; return 0;
} }
GIE_handle_t* GIE_handle=sp->feather->scanner->gie_aux[sp->table_id].gie_handle; struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id];
GIE_handle_t* GIE_handle=table_aux->similar.gie_handle;
unsigned long long digest_len=0; unsigned long long digest_len=0;
char* digest_buff=NULL; char* digest_buff=NULL;
struct _OUTER_scan_status_t* _mid=NULL; struct _OUTER_scan_status_t* _mid=NULL;
@@ -1790,7 +1789,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{ {
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC,&start);
} }
p_table=sp->feather->p_table_info[sp->table_id];
alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
pthread_mutex_lock(&(sp->fuzzy_mutex)); pthread_mutex_lock(&(sp->fuzzy_mutex));
sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset); sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
@@ -1807,7 +1805,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{ {
goto fast_out; goto fast_out;
} }
digest_buff=(char*)malloc(sizeof(char)*digest_len); digest_buff=ALLOC(char, digest_len);
pthread_mutex_lock(&(sp->fuzzy_mutex)); pthread_mutex_lock(&(sp->fuzzy_mutex));
SFH_digest(sp->fuzzy_hash_handle,digest_buff, digest_len); SFH_digest(sp->fuzzy_hash_handle,digest_buff, digest_len);
pthread_mutex_unlock(&(sp->fuzzy_mutex)); pthread_mutex_unlock(&(sp->fuzzy_mutex));
@@ -1827,7 +1825,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
alignment_int64_array_add(p_table->hit_cnt, sp->thread_num, 1); alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num, 1);
_mid=grab_mid(mid,sp->feather, sp->thread_num,1); _mid=grab_mid(mid,sp->feather, sp->thread_num,1);
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -1845,11 +1843,11 @@ fast_out:
if(sp->feather->perf_on==1) if(sp->feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num); maat_stat_table(table_aux, data_len, &start, &end, sp->thread_num);
} }
else else
{ {
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num); maat_stat_table(table_aux, data_len, NULL, NULL, sp->thread_num);
} }
if(compile_ret==0&&hit_region_cnt>0) if(compile_ret==0&&hit_region_cnt>0)
{ {
@@ -1861,8 +1859,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{ {
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id];
alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1); alignment_int64_array_add(table_aux->stream_num, sp->thread_num,-1);
if(scanner!=NULL) if(scanner!=NULL)
{ {
if(sp->version==sp->feather->maat_version) if(sp->version==sp->feather->maat_version)
@@ -1969,25 +1967,20 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
return -1; return -1;
} }
if(p_table->cfg_num==0)
{
return 0;
}
my_scanner=_feather->scanner; my_scanner=_feather->scanner;
if(my_scanner==NULL) if(my_scanner==NULL)
{ {
return 0; return 0;
} }
GIE_handle_t* gie_handle=my_scanner->gie_aux[table_id].gie_handle; struct table_runtime* table_aux=my_scanner->table_rt[table_id];
struct _region_stat_t * region_stat=NULL; if(table_aux->origin_rule_num==0)
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
{ {
return 0; return 0;
} }
GIE_handle_t* gie_handle=table_aux->similar.gie_handle;
INC_SCANNER_REF(my_scanner,thread_num);
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
INC_SCANNER_REF(my_scanner,thread_num);
region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM); region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM);
if(region_ret<0) if(region_ret<0)
{ {
@@ -1997,7 +1990,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
alignment_int64_array_add(p_table->hit_cnt, thread_num,1); alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -2016,11 +2009,11 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
if(_feather->perf_on==1) if(_feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end,thread_num); maat_stat_table(table_aux,0,&start, &end,thread_num);
} }
else else
{ {
maat_stat_table(p_table,0,NULL, NULL,thread_num); maat_stat_table(table_aux,0,NULL, NULL,thread_num);
} }
if(compile_ret==0&&region_ret>0) if(compile_ret==0&&region_ret>0)
{ {

35
src/entry/Maat_command.cpp
View File

@@ -1590,6 +1590,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
int ret=0, table_id=0, i=0; int ret=0, table_id=0, i=0;
int rule_with_foreign_key=0; int rule_with_foreign_key=0;
struct _Maat_table_info_t* p_table=NULL; struct _Maat_table_info_t* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
for(i=0; i<rule_num; i++) for(i=0; i<rule_num; i++)
{ {
if(rule_list[i].table_line==NULL) if(rule_list[i].table_line==NULL)
@@ -1602,11 +1603,12 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
continue; continue;
} }
p_table=feather->p_table_info[table_id]; p_table=feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->n_foreign==0) plugin_desc= &(p_table->plugin);
if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0)
{ {
continue; continue;
} }
_get_foregin_keys(rule_list+i, p_table->foreign_columns, p_table->n_foreign, dir, logger); _get_foregin_keys(rule_list+i, plugin_desc->foreign_columns, plugin_desc->n_foreign, dir, logger);
rule_with_foreign_key++; rule_with_foreign_key++;
} }
return rule_with_foreign_key; return rule_with_foreign_key;
@@ -1736,12 +1738,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
const unsigned char* dec_key, const unsigned char* dec_key,
_Maat_feather_t* feather) _Maat_feather_t* feather)
{ {
int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0; int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0, valid_column=-1;
int ret=0; int ret=0;
struct serial_rule_t* rule_list=NULL; struct serial_rule_t* rule_list=NULL;
int update_type=CM_UPDATE_TYPE_INC; int update_type=CM_UPDATE_TYPE_INC;
long long new_version=0; long long new_version=0;
enum MAAT_TABLE_TYPE table_type; enum MAAT_TABLE_TYPE table_type;
const struct plugin_table_desc* plugin_desc=NULL;
void* logger=feather->logger; void* logger=feather->logger;
if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write
@@ -1830,7 +1833,16 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
} }
table_type=feather->p_table_info[table_id]->table_type; table_type=feather->p_table_info[table_id]->table_type;
if(rule_list[i].op==MAAT_OP_DEL) if(rule_list[i].op==MAAT_OP_DEL)
{ {
if(table_type==TABLE_TYPE_PLUGIN)
{
plugin_desc=&(feather->p_table_info[table_id]->plugin);
valid_column=plugin_desc->valid_flag_column;
}
else
{
valid_column=-1;
}
ret=invalidate_line(rule_list[i].table_line, table_type, valid_column); ret=invalidate_line(rule_list[i].table_line, table_type, valid_column);
if(ret<0) if(ret<0)
{ {
@@ -2016,6 +2028,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
_Maat_feather_t* _feather=(_Maat_feather_t*)feather; _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=0, table_id=0,success_cnt=0; int ret=0, table_id=0,success_cnt=0;
struct serial_rule_t *s_rule=NULL; struct serial_rule_t *s_rule=NULL;
struct _Maat_table_info_t* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL; struct plugin_table_desc* plugin_desc=NULL;
long long server_time=0,absolute_expire_time=0; long long server_time=0,absolute_expire_time=0;
const char* p_foreign=NULL; const char* p_foreign=NULL;
@@ -2039,6 +2052,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
ret=-1; ret=-1;
goto error_out; goto error_out;
} }
p_table=_feather->p_table_info[table_id];
plugin_desc=&(p_table->plugin); plugin_desc=&(p_table->plugin);
if(TABLE_TYPE_PLUGIN!=p_table->table_type) if(TABLE_TYPE_PLUGIN!=p_table->table_type)
{ {
@@ -2052,7 +2066,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
if(op==MAAT_OP_ADD) if(op==MAAT_OP_ADD)
{ {
ret=get_valid_flag_offset(line_rule[i]->table_line ret=get_valid_flag_offset(line_rule[i]->table_line
, p_table->table_type , p_table->table_type
, plugin_desc->valid_flag_column); , plugin_desc->valid_flag_column);
if(ret<0|| if(ret<0||
(op==MAAT_OP_ADD&&line_rule[i]->table_line[ret]!='1')) (op==MAAT_OP_ADD&&line_rule[i]->table_line[ret]!='1'))
@@ -2071,17 +2085,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
if(line_rule[i]->expire_after>0) if(line_rule[i]->expire_after>0)
{ {
absolute_expire_time=server_time+line_rule[i]->expire_after; absolute_expire_time=server_time+line_rule[i]->expire_after;
} }
if(plugin_desc->n_foreign>0) if(plugin_desc->n_foreign>0)
{ {
for(j=0;j<plugin_desc->n_foreign;j++) for(j=0;j<plugin_desc->n_foreign;j++)
{ {
p_foreign=find_Nth_column(line_rule[i]->table_line, plugin_desc->foreign_columns[j], &foreign_key_size); p_foreign=find_Nth_column(line_rule[i]->table_line, plugin_desc->foreign_columns[j], &foreign_key_size);
if(p_foreign==NULL) if(p_foreign==NULL)
{ {
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL, maat_command MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL, maat_command
, "Command set line %s %d failed: No %dth column." , "Command set line %s %d failed: No %dth column."
, line_rule[i]->table_name, line_rule[i]->rule_id , line_rule[i]->table_name, line_rule[i]->rule_id
, plugin_desc->foreign_columns[j]); , plugin_desc->foreign_columns[j]);
ret=-1; ret=-1;
goto error_out; goto error_out;
@@ -2096,7 +2110,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
} }
} }
} }
set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name,
line_rule[i]->table_line, absolute_expire_time); line_rule[i]->table_line, absolute_expire_time);
} }
success_cnt=exec_serial_rule(write_ctx,s_rule, line_num,server_time,_feather->logger); success_cnt=exec_serial_rule(write_ctx,s_rule, line_num,server_time,_feather->logger);

657
src/entry/Maat_rule.cpp
View File

File diff suppressed because it is too large Load Diff

158
src/entry/Maat_stat.cpp
View File

@@ -149,20 +149,20 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_start(feather->stat_handle); FS_start(feather->stat_handle);
return; return;
} }
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num) void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
{ {
alignment_int64_array_add(p_table->scan_cnt,thread_num,1); alignment_int64_array_add(p->scan_cnt,thread_num,1);
alignment_int64_array_add(p_table->input_bytes,thread_num,scan_len); alignment_int64_array_add(p->input_bytes,thread_num,scan_len);
if(start!=NULL&&end!=NULL) if(start!=NULL&&end!=NULL)
{ {
alignment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec); alignment_int64_array_add(p->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
} }
return; return;
} }
void maat_stat_output(struct _Maat_feather_t* feather) void maat_stat_output(struct _Maat_feather_t* feather)
{ {
long value=0; long value=0;
long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0; long long total_cfg_num=0, table_regex_ipv6_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0;
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0; long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0; long long outer_mid_cnt=0,inner_mid_cnt=0;
@@ -172,6 +172,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
int i=0; int i=0;
time_t now; time_t now;
struct _Maat_table_info_t* p_table=NULL; struct _Maat_table_info_t* p_table=NULL;
struct table_runtime* table_aux=NULL;
time(&now); time(&now);
active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
@@ -201,86 +202,105 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_input_bytes=0; table_input_bytes=0;
table_scan_cpu_time=0; table_scan_cpu_time=0;
table_hit_cnt=0; table_hit_cnt=0;
table_regex_ipv6_num=0;
p_table=feather->p_table_info[i]; p_table=feather->p_table_info[i];
if(p_table==NULL) if(p_table==NULL)
{ {
continue; continue;
} }
table_aux=feather->scanner->table_rt[i];
switch(p_table->table_type) switch(p_table->table_type)
{ {
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
plugin_cache_num+=p_table->cb_info->cache_line_num; plugin_cache_num+=table_aux->plugin.cache_line_num;
plugin_acc_num+=p_table->cb_info->acc_line_num; plugin_acc_num+=table_aux->plugin.acc_line_num;
break; break;
case TABLE_TYPE_GROUP: case TABLE_TYPE_GROUP:
group_rule_num+=p_table->cfg_num; group_rule_num+=table_aux->origin_rule_num;
break; break;
case TABLE_TYPE_COMPILE: case TABLE_TYPE_COMPILE:
compile_rule_num+=p_table->cfg_num; compile_rule_num+=table_aux->origin_rule_num;
break;
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
table_regex_ipv6_num=table_aux->expr.regex_rule_cnt;
total_iconv_error=p_table->expr.iconv_err_cnt;
break;
case TABLE_TYPE_IP:
table_regex_ipv6_num=table_aux->ip.ipv6_rule_cnt;
break; break;
default: default:
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
p_table->cfg_num);
total_cfg_num+=p_table->cfg_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
p_table->regex_rule_cnt);
total_regex_num+= p_table->regex_rule_cnt;
table_stream_num=alignment_int64_array_sum(p_table->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
table_stream_num);
total_stream_cnt+= table_stream_num;
table_scan_cnt=alignment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_SET,
table_scan_cnt);
total_scan_cnt+=table_scan_cnt;
table_input_bytes=alignment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_SET,
table_input_bytes);
total_input_bytes+=table_input_bytes;
if(feather->perf_on==1)
{
table_scan_cpu_time=alignment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_SET,
table_scan_cpu_time);
total_cpu_time+=table_scan_cpu_time;
}
table_hit_cnt=alignment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_SET,
table_hit_cnt);
total_update_error+=p_table->udpate_err_cnt;
total_iconv_error+=p_table->iconv_err_cnt;
//total hit count stat in region_compile
break; break;
} }
} if(p_table->table_type==TABLE_TYPE_PLUGIN||
p_table->table_type==TABLE_TYPE_GROUP||
p_table->table_type==TABLE_TYPE_COMPILE)
{
continue;
}
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
table_aux->origin_rule_num);
total_cfg_num+=table_aux->origin_rule_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
table_regex_ipv6_num);
total_regex_num+=table_regex_ipv6_num;
table_stream_num=alignment_int64_array_sum(table_aux->stream_num,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_ADD,
table_stream_num);
total_stream_cnt+= table_stream_num;
table_scan_cnt=alignment_int64_array_sum(table_aux->scan_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_ADD,
table_scan_cnt);
total_scan_cnt+=table_scan_cnt;
table_input_bytes=alignment_int64_array_sum(table_aux->input_bytes,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_ADD,
table_input_bytes);
total_input_bytes+=table_input_bytes;
if(feather->perf_on==1)
{
table_scan_cpu_time=alignment_int64_array_sum(table_aux->scan_cpu_time,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_ADD,
table_scan_cpu_time);
total_cpu_time+=table_scan_cpu_time;
}
table_hit_cnt=alignment_int64_array_sum(table_aux->hit_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_ADD,
table_hit_cnt);
total_update_error+=p_table->udpate_err_cnt;
}
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
feather->total_stat_id, feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM], feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
@@ -314,7 +334,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_OP_SET, FS_OP_SET,
total_cpu_time); total_cpu_time);
} }
total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num); total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt, feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
feather->total_stat_id, feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT], feather->fs_column_id[COLUMN_TABLE_HIT_CNT],

101
src/inc_internal/Maat_rule_internal.h
View File

@@ -101,7 +101,7 @@ struct compile_ex_data_idx
int idx; int idx;
int table_id; int table_id;
}; };
struct compile_table_descr struct compile_table_desc
{ {
enum USER_REGION_ENCODE user_region_encoding; enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num; int ex_data_num;
@@ -135,7 +135,7 @@ struct _plugin_table_info
long cache_size; long cache_size;
}; };
struct plugin_table_descr struct plugin_table_desc
{ {
int key_column; int key_column;
int valid_flag_column; int valid_flag_column;
@@ -147,7 +147,7 @@ struct plugin_table_descr
struct plugin_table_ex_data_desc ex_desc; struct plugin_table_ex_data_desc ex_desc;
int acc_line_num; int acc_line_num;
}; };
struct expr_table_descr struct expr_table_desc
{ {
enum MAAT_CHARSET src_charset; enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
@@ -155,10 +155,9 @@ struct expr_table_descr
int do_charset_merge; int do_charset_merge;
int cross_cache_size; int cross_cache_size;
int quick_expr_switch; int quick_expr_switch;
int expr_rule_cnt; //expr_type=0,1,3 long long iconv_err_cnt;
int regex_rule_cnt; //expr_type=2
}; };
struct ip_table_descr struct ip_table_desc
{ {
int ipv4_rule_cnt; int ipv4_rule_cnt;
int ipv6_rule_cnt; int ipv6_rule_cnt;
@@ -173,17 +172,17 @@ struct _Maat_table_info_t
enum MAAT_TABLE_TYPE table_type; enum MAAT_TABLE_TYPE table_type;
union union
{ {
struct compile_table_descr compile; struct compile_table_desc compile;
struct expr_table_descr expr; struct expr_table_desc expr;
struct ip_table_descr ip; struct ip_table_desc ip;
struct plugin_table_descr plugin; struct plugin_table_desc plugin;
void* others;//group, interval and digest don't have sperate description info. void* others;//group, interval and digest don't have sperate description info.
}; };
/*
enum MAAT_CHARSET src_charset; enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM]; enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst; int src_charset_in_dst;
int do_charset_merge; int do_charset_merge;
int cfg_num;
int cross_cache_size; int cross_cache_size;
int quick_expr_switch; int quick_expr_switch;
union union
@@ -206,16 +205,11 @@ struct _Maat_table_info_t
enum USER_REGION_ENCODE user_region_encoding; enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num; int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM]; struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
*/
//for stat>>>>>>>> //for stat>>>>>>>>
unsigned long long udpate_err_cnt; unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt; unsigned long long unmatch_tag_cnt;
unsigned long long iconv_err_cnt;
int stat_line_id; int stat_line_id;
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
}; };
struct db_str_rule_t struct db_str_rule_t
@@ -334,24 +328,6 @@ struct _compile_result_t
int compile_id; int compile_id;
universal_bool_expr_t group_set; universal_bool_expr_t group_set;
}; };
struct _region_stat_t
{
int cfg_num;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
};
struct _INNER_scan_status_t struct _INNER_scan_status_t
{ {
int cur_hit_cnt; int cur_hit_cnt;
@@ -403,19 +379,48 @@ struct _stream_para_t
pthread_mutex_t fuzzy_mutex; pthread_mutex_t fuzzy_mutex;
unsigned char query_point[8]; unsigned char query_point[8];
}; };
struct digest_engine struct similar_runtime
{ {
enum MAAT_TABLE_TYPE table_type;
GIE_handle_t* gie_handle; GIE_handle_t* gie_handle;
MESA_lqueue_head update_q; MESA_lqueue_head update_q;
}; };
struct plugin_engine struct plugin_runtime
{ {
dynamic_array_t *cache_lines; dynamic_array_t *cache_lines;
int cache_line_num; long long cache_line_num;
long cache_size; long long acc_line_num;
long long cache_size;
MESA_htable_handle ex_data_hash;
}; };
struct expr_runtime
{
long long expr_rule_cnt; //expr_type=0,1,3
long long regex_rule_cnt; //expr_type=2
};
struct ip_runtime
{
long long ipv4_rule_cnt;
long long ipv6_rule_cnt;
};
struct table_runtime
{
enum MAAT_TABLE_TYPE table_type;
long origin_rule_num;
union
{
struct similar_runtime similar; //for digest and similarity
struct plugin_runtime plugin;
struct expr_runtime expr;
struct ip_runtime ip;
void * other;
};
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct rule_tag struct rule_tag
{ {
char* tag_name; char* tag_name;
@@ -425,11 +430,15 @@ struct _Maat_scanner_t
{ {
long long version; long long version;
time_t last_update_time; time_t last_update_time;
long long *ref_cnt; //optimized for cache_alignment 64 mcore_long_t ref_cnt;
rule_scanner_t region; rule_scanner_t region;
long gie_total_q_size; long gie_total_q_size;
struct digest_engine gie_aux[MAX_TABLE_NUM];
struct plugin_engine plugin_aux[MAX_TABLE_NUM]; struct table_runtime* table_rt[MAX_TABLE_NUM];
/*
struct similar_runtime gie_aux[MAX_TABLE_NUM];
struct plugin_runtime plugin_aux[MAX_TABLE_NUM];
*/
MESA_htable_handle region_hash; MESA_htable_handle region_hash;
MESA_htable_handle group_hash; MESA_htable_handle group_hash;
MESA_htable_handle compile_hash; MESA_htable_handle compile_hash;
@@ -443,7 +452,7 @@ struct _Maat_scanner_t
void * bool_macher_expr_compiler; void * bool_macher_expr_compiler;
scan_result_t *region_rslt_buff; scan_result_t *region_rslt_buff;
MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q
struct _region_stat_t region_counter[MAX_TABLE_NUM]; // struct _region_stat_t region_counter[MAX_TABLE_NUM];
int max_thread_num; int max_thread_num;
iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from] iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from]
}; };
@@ -608,7 +617,7 @@ int HASH_add_by_id(MESA_htable_handle hash,int id,void*data);
int HASH_delete_by_id(MESA_htable_handle hash,int id); int HASH_delete_by_id(MESA_htable_handle hash,int id);
void maat_read_full_config(_Maat_feather_t* _feather); void maat_read_full_config(_Maat_feather_t* _feather);
void maat_stat_init(struct _Maat_feather_t* feather); void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_output(struct _Maat_feather_t* feather); void maat_stat_output(struct _Maat_feather_t* feather);
redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);

5
src/inc_internal/alignment_int64.h
View File

@@ -43,6 +43,11 @@ inline long long alignment_int64_array_cnt(mcore_long_t array,int size)
} }
return cnt; return cnt;
} }
inline void alignment_int64_array_reset(mcore_long_t array,int size)
{
memset(array, 0, CPU_CACHE_ALIGMENT*size);
return;
}
inline void alignment_int64_array_free(mcore_long_t array) inline void alignment_int64_array_free(mcore_long_t array)
{ {
free(array); free(array);

2
test/test_maatframe.cpp
View File

@@ -480,7 +480,7 @@ TEST(SimilarScan, Pure)
const char* scan_data="mwss.xiu.youku.com/live/hls/v1/0000000000000000000000001526a0a8/714.ts?&token=98765"; const char* scan_data="mwss.xiu.youku.com/live/hls/v1/0000000000000000000000001526a0a8/714.ts?&token=98765";
const char* table_name="SIM_URL"; const char* table_name="SIM_URL";
scan_status_t mid=NULL; scan_status_t mid=NULL;
memset(result, 0 , sizeof(result));
table_id=Maat_table_register(g_feather,table_name); table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);