gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

将规则数量和扫描状态的统计归集到table runtime结构体中，属于scanner的一部分。

Browse Source
This commit is contained in:
zhengchao
2018-12-04 20:12:56 +08:00
parent 895344d7d2
commit 6971f4cb56
8 changed files with 660 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
inc/gram_index_engine.h
View File

@@ -4,12 +4,16 @@
#ifdef __cplusplus
extern "C" {
#endif
#define GIE_INSERT_OPT 0
#define GIE_DELETE_OPT 1
#define GIE_INPUT_FORMAT_SFH 1
#define GIE_INPUT_FORMAT_PLAIN 0
enum GIE_operation
{
GIE_INSERT_OPT,
GIE_DELETE_OPT
};
enum GIE_INPUT_FORMAT
{
GIE_INPUT_FORMAT_PLAIN,
GIE_INPUT_FORMAT_SFH
};
typedef struct
{
@@ -21,7 +25,7 @@ typedef struct
{
unsigned int id;
unsigned int sfh_length;//size of fuzzy_hash
short operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed;
enum GIE_operation operation;//GIE_INSERT_OPT or GIE_DELETE_OPT.if operation is GIE_DELETE_OPT, only id is needed;
short cfds_lvl;
char * sfh;
void * tag;
@@ -41,9 +45,9 @@ typedef struct
unsigned int gram_value;
//unsigned int htable_num;
unsigned int position_accuracy;
short format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string
enum GIE_INPUT_FORMAT format; //if format==GIE_INPUT_FORMAT_SFH, means the input string is a GIE_INPUT_FORMAT_SFH string
//else id format==PALIN, means the input string is common string
short ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result
int ED_reexamine;//if ED_reexamine==1, calculate edit distance to verify the final result
}GIE_create_para_t;

197
src/entry/Maat_api.cpp
View File

@@ -919,6 +919,8 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
int idx=0,i=0;
struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
struct plugin_table_desc *plugin_desc=&(p_table->plugin);
struct plugin_runtime* plugin_aux=NULL;
const char* lines=NULL;
if(p_table==NULL)
{
@@ -928,28 +930,29 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
{
return -1;
}
plugin_aux = &(_feather->scanner->table_rt[table_id]->plugin);
//plugin table register blocks background update.
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
idx=p_table->cb_info->cb_plug_cnt;
idx=plugin_desc->cb_plug_cnt;
if(idx==MAX_PLUGIN_PER_TABLE)
{
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
return -1;
}
p_table->cb_info->cb_plug_cnt++;
p_table->cb_info->cb_plug[idx].start=start;
p_table->cb_info->cb_plug[idx].update=update;
p_table->cb_info->cb_plug[idx].finish=finish;
p_table->cb_info->cb_plug[idx].u_para=u_para;
if(p_table->cb_info->cache_line_num>0)
plugin_desc->cb_plug_cnt++;
plugin_desc->cb_plug[idx].start=start;
plugin_desc->cb_plug[idx].update=update;
plugin_desc->cb_plug[idx].finish=finish;
plugin_desc->cb_plug[idx].u_para=u_para;
if(plugin_aux->cache_line_num>0)
{
if(start!=NULL)
{
start(MAAT_RULE_UPDATE_TYPE_FULL,u_para);
}
for(i=0;i<p_table->cb_info->cache_line_num;i++)
for(i=0;i<plugin_aux->cache_line_num;i++)
{
lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i);
lines=(const char*)dynamic_array_read(plugin_aux->cache_lines,i);
if(lines==NULL)
{
break;
@@ -1001,26 +1004,26 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
{
return -1;
}
struct compile_table_desc* compile_desc=&(p_table->compile);
pthread_mutex_lock(&(_feather->backgroud_update_mutex));
if(p_table->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
{
ret=-1;
goto failed;
}
idx=p_table->ex_data_num;
p_table->ex_desc[idx].idx=idx;
p_table->ex_desc[idx].table_id=table_id;
p_table->ex_desc[idx].argl=argl;
p_table->ex_desc[idx].argp=argp;
p_table->ex_desc[idx].new_func=new_func;
p_table->ex_desc[idx].free_func=free_func;
p_table->ex_desc[idx].dup_func=dup_func;
idx=compile_desc->ex_data_num;
compile_desc->ex_desc[idx].idx=idx;
compile_desc->ex_desc[idx].table_id=table_id;
compile_desc->ex_desc[idx].argl=argl;
compile_desc->ex_desc[idx].argp=argp;
compile_desc->ex_desc[idx].new_func=new_func;
compile_desc->ex_desc[idx].free_func=free_func;
compile_desc->ex_desc[idx].dup_func=dup_func;
p_table->ex_data_num++;
compile_desc->ex_data_num++;
if(_feather->scanner!=NULL)
{
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, p_table->ex_desc+idx);
MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, compile_desc->ex_desc+idx);
}
failed:
pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
@@ -1031,6 +1034,7 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct _Maat_compile_inner_t *compile_inner=NULL;
const struct compile_table_desc* compile_desc=NULL;
const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL;
@@ -1040,8 +1044,9 @@ MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maa
return NULL;
}
pthread_rwlock_rdlock(&(compile_inner->rwlock));
assert(idx<compile_inner->ref_table->ex_data_num);
ex_desc=compile_inner->ref_table->ex_desc+idx;
compile_desc=&(compile_inner->ref_table->compile);
assert(idx<compile_desc->ex_data_num);
ex_desc=compile_desc->ex_desc+idx;
ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp);
pthread_rwlock_unlock(&(compile_inner->rwlock));
return ad;
@@ -1054,7 +1059,7 @@ int Maat_plugin_EX_register(Maat_feather_t feather, int table_id,
long argl, void *argp)
{
return 0;
}
int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
@@ -1070,6 +1075,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];//dynamic array
struct _Maat_table_info_t *p_table=NULL;
struct expr_table_desc* expr_desc=NULL;
struct timespec start,end;
_Maat_scanner_t* my_scanner=NULL;
if(data==NULL||data_len<=0)
@@ -1087,17 +1093,13 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
{
return 0;
}
expr_desc=&(p_table->expr);
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{
_feather->scan_err_cnt++;
return -1;
}
if(p_table->do_charset_merge==1)
if(expr_desc->do_charset_merge==1)
{
sub_type=make_sub_type(table_id,CHARSET_NONE,0);
}
@@ -1115,9 +1117,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
return 0;
}
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
struct table_runtime* table_aux=my_scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
{
return 0;
}
@@ -1125,7 +1126,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
INC_SCANNER_REF(my_scanner, thread_num);
if(region_stat->expr_rule_cnt>0)
if(table_aux->expr.expr_rule_cnt>0)
{
scan_data.rule_type=RULETYPE_STR;
scan_data.sub_type=sub_type;
@@ -1135,7 +1136,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
hit_region_cnt+=region_ret;
}
}
if(region_stat->regex_rule_cnt>0)
if(table_aux->expr.regex_rule_cnt>0)
{
scan_data.rule_type=RULETYPE_REG;
scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0);
@@ -1152,7 +1153,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
}
if(hit_region_cnt>0)
{
alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
_mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -1173,17 +1174,16 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
hit_detail,detail_num);
}
}
DEC_SCANNER_REF(my_scanner, thread_num);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,data_len,&start, &end,thread_num);
maat_stat_table(table_aux, data_len, &start, &end, thread_num);
}
else
{
maat_stat_table(p_table,data_len,NULL, NULL,thread_num);
maat_stat_table(table_aux, data_len, NULL, NULL, thread_num);
}
DEC_SCANNER_REF(my_scanner, thread_num);
if(compile_ret==0&&hit_region_cnt>0)
{
return -2;
@@ -1229,18 +1229,13 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
{
return 0;
}
my_scanner=_feather->scanner;
if(my_scanner==NULL)
{
return 0;
}
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
struct table_runtime* table_aux=my_scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
{
return 0;
}
@@ -1258,7 +1253,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
}
else if(region_ret>0)
{
alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -1273,16 +1268,17 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
}
}
DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end,thread_num);
maat_stat_table(table_aux, 0, &start, &end, thread_num);
}
else
{
maat_stat_table(p_table,0,NULL, NULL,thread_num);
maat_stat_table(table_aux, 0, NULL, NULL, thread_num);
}
DEC_SCANNER_REF(my_scanner,thread_num);
if(compile_ret==0&&region_ret>0)
{
return -2;
@@ -1316,26 +1312,22 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
{
return 0;
}
my_scanner=_feather->scanner;
if(my_scanner==NULL)
{
return 0;
}
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
struct table_runtime* table_aux=my_scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
{
return 0;
}
if(region_stat->ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
if(table_aux->ip.ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
{
return 0;
}
if(region_stat->ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
if(table_aux->ip.ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
{
return 0;
}
@@ -1380,7 +1372,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
}
else if(region_ret>0)
{
alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
@@ -1399,11 +1391,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end,thread_num);
maat_stat_table(table_aux, 0, &start, &end, thread_num);
}
else
{
maat_stat_table(p_table,0,NULL, NULL,thread_num);
maat_stat_table(table_aux, 0, NULL, NULL, thread_num);
}
if(compile_ret==0&&region_ret>0)
{
@@ -1436,12 +1428,14 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
_feather->scan_err_cnt++;
return NULL;
}
if(p_table->quick_expr_switch==1)
struct expr_table_desc* expr_desc=&(p_table->expr);
if(expr_desc->quick_expr_switch==1)
{
_feather->scan_err_cnt++;
return NULL;
}
struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1);
struct _stream_para_t* sp=ALLOC(struct _stream_para_t ,1);
scanner=_feather->scanner;
sp->feather=_feather;
sp->version=_feather->maat_version;
@@ -1451,29 +1445,32 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{
return sp;
}
struct _region_stat_t * region_stat=NULL;
region_stat=&(scanner->region_counter[p_table->table_id]);
struct table_runtime* table_aux=scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
{
return 0;
}
INC_SCANNER_REF(scanner, thread_num);
sp->table_id=table_id;
sp->thread_num=thread_num;
sp->max_cross_size=p_table->cross_cache_size;
sp->max_cross_size=expr_desc->cross_cache_size;
sp->caching_size=0;
sp->scan_buff=NULL;
sp->last_cache=NULL;
if(p_table->do_charset_merge==1)
if(expr_desc->do_charset_merge==1)
{
sp->do_merge=1;
}
if(region_stat->expr_rule_cnt>0)
if(table_aux->expr.expr_rule_cnt>0)
{
sp->do_expr=1;
}
if(region_stat->regex_rule_cnt>0)
if(table_aux->expr.regex_rule_cnt>0)
{
sp->do_regex=1;
}
alignment_int64_array_add(p_table->stream_num,thread_num,1);
alignment_int64_array_add(table_aux->stream_num,thread_num,1);
sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num);
return sp;
}
@@ -1493,6 +1490,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
_compile_result_t compile_result[rule_num];//dynamic array
scan_data_t region_scan_data;
_Maat_table_info_t* p_table=NULL;
struct table_runtime* table_aux=scanner->table_rt[sp->table_id];
struct timespec start,end;
if(data==NULL||data_len<=0)
{
@@ -1539,7 +1537,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
free(sp->scan_buff);
sp->scan_buff=NULL;
}
sp->scan_buff=(char*)malloc(sp->caching_size+data_len);
sp->scan_buff=ALLOC(char, sp->caching_size+data_len);
memcpy(sp->scan_buff,sp->last_cache,sp->caching_size);
memcpy(sp->scan_buff+sp->caching_size,data,data_len);
region_scan_data.text_data.text=sp->scan_buff;
@@ -1554,7 +1552,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->last_cache==NULL&&sp->max_cross_size>0)
{
assert(sp->caching_size==0);
sp->last_cache=(char*)malloc(sizeof(char)*sp->max_cross_size);
sp->last_cache=ALLOC(char, sp->max_cross_size);
}
if(sp->max_cross_size>0)
{
@@ -1600,7 +1598,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
}
if(hit_region_cnt>0)
{
alignment_int64_array_add(p_table->hit_cnt, sp->thread_num,1);
alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num,1);
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
@@ -1639,11 +1637,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num);
maat_stat_table(table_aux,data_len,&start, &end,sp->thread_num);
}
else
{
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num);
maat_stat_table(table_aux,data_len,NULL, NULL,sp->thread_num);
}
if(compile_ret==0&&hit_region_cnt>0)
{
@@ -1668,8 +1666,8 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1);
struct table_runtime* table_aux=scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_aux->stream_num, sp->thread_num, -1);
if(sp->rs_stream_para!=NULL)
{
if(scanner!=NULL&&sp->version>=sp->feather->last_full_version)
@@ -1713,7 +1711,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
_feather->scan_err_cnt++;
return NULL;
}
struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1);
struct _stream_para_t* sp=ALLOC(struct _stream_para_t, 1);
scanner=_feather->scanner;
sp->feather=_feather;
sp->version=_feather->maat_version;
@@ -1722,6 +1720,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
{
return sp;
}
struct table_runtime* table_aux=scanner->table_rt[table_id];
tmp_fuzzy_handle=SFH_instance(total_len);
if(tmp_fuzzy_handle==NULL)
{
@@ -1735,7 +1734,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sp->total_len=total_len;
sp->fuzzy_hash_handle=tmp_fuzzy_handle;
pthread_mutex_init(&(sp->fuzzy_mutex),NULL);
alignment_int64_array_add(p_table->stream_num,thread_num,1);
alignment_int64_array_add(table_aux->stream_num,thread_num,1);
return sp;
}
@@ -1772,7 +1771,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
GIE_result_t query_result[MAX_SCANNER_HIT_NUM];
int hit_region_cnt=0,compile_ret=0;
_compile_result_t compile_result[rule_num];//dynamic array
_Maat_table_info_t* p_table=NULL;
if(data==NULL||data_len<=0)
{
return 0;
@@ -1781,7 +1779,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{
return 0;
}
GIE_handle_t* GIE_handle=sp->feather->scanner->gie_aux[sp->table_id].gie_handle;
struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id];
GIE_handle_t* GIE_handle=table_aux->similar.gie_handle;
unsigned long long digest_len=0;
char* digest_buff=NULL;
struct _OUTER_scan_status_t* _mid=NULL;
@@ -1790,7 +1789,6 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{
clock_gettime(CLOCK_MONOTONIC,&start);
}
p_table=sp->feather->p_table_info[sp->table_id];
alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
pthread_mutex_lock(&(sp->fuzzy_mutex));
sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
@@ -1807,7 +1805,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{
goto fast_out;
}
digest_buff=(char*)malloc(sizeof(char)*digest_len);
digest_buff=ALLOC(char, digest_len);
pthread_mutex_lock(&(sp->fuzzy_mutex));
SFH_digest(sp->fuzzy_hash_handle,digest_buff, digest_len);
pthread_mutex_unlock(&(sp->fuzzy_mutex));
@@ -1827,7 +1825,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
}
if(hit_region_cnt>0)
{
alignment_int64_array_add(p_table->hit_cnt, sp->thread_num, 1);
alignment_int64_array_add(table_aux->hit_cnt, sp->thread_num, 1);
_mid=grab_mid(mid,sp->feather, sp->thread_num,1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
@@ -1845,11 +1843,11 @@ fast_out:
if(sp->feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num);
maat_stat_table(table_aux, data_len, &start, &end, sp->thread_num);
}
else
{
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num);
maat_stat_table(table_aux, data_len, NULL, NULL, sp->thread_num);
}
if(compile_ret==0&&hit_region_cnt>0)
{
@@ -1861,8 +1859,8 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1);
struct table_runtime *table_aux=sp->feather->scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_aux->stream_num, sp->thread_num,-1);
if(scanner!=NULL)
{
if(sp->version==sp->feather->maat_version)
@@ -1969,25 +1967,20 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
{
return 0;
}
my_scanner=_feather->scanner;
if(my_scanner==NULL)
{
return 0;
}
GIE_handle_t* gie_handle=my_scanner->gie_aux[table_id].gie_handle;
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
struct table_runtime* table_aux=my_scanner->table_rt[table_id];
if(table_aux->origin_rule_num==0)
{
return 0;
}
GIE_handle_t* gie_handle=table_aux->similar.gie_handle;
INC_SCANNER_REF(my_scanner,thread_num);
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
INC_SCANNER_REF(my_scanner,thread_num);
region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM);
if(region_ret<0)
{
@@ -1997,7 +1990,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
}
else if(region_ret>0)
{
alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
alignment_int64_array_add(table_aux->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -2016,11 +2009,11 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end,thread_num);
maat_stat_table(table_aux,0,&start, &end,thread_num);
}
else
{
maat_stat_table(p_table,0,NULL, NULL,thread_num);
maat_stat_table(table_aux,0,NULL, NULL,thread_num);
}
if(compile_ret==0&&region_ret>0)
{

35
src/entry/Maat_command.cpp
View File

@@ -1590,6 +1590,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
int ret=0, table_id=0, i=0;
int rule_with_foreign_key=0;
struct _Maat_table_info_t* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
for(i=0; i<rule_num; i++)
{
if(rule_list[i].table_line==NULL)
@@ -1602,11 +1603,12 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
continue;
}
p_table=feather->p_table_info[table_id];
if(p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->n_foreign==0)
plugin_desc= &(p_table->plugin);
if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0)
{
continue;
}
_get_foregin_keys(rule_list+i, p_table->foreign_columns, p_table->n_foreign, dir, logger);
_get_foregin_keys(rule_list+i, plugin_desc->foreign_columns, plugin_desc->n_foreign, dir, logger);
rule_with_foreign_key++;
}
return rule_with_foreign_key;
@@ -1736,12 +1738,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
const unsigned char* dec_key,
_Maat_feather_t* feather)
{
int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0;
int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0, valid_column=-1;
int ret=0;
struct serial_rule_t* rule_list=NULL;
int update_type=CM_UPDATE_TYPE_INC;
long long new_version=0;
enum MAAT_TABLE_TYPE table_type;
const struct plugin_table_desc* plugin_desc=NULL;
void* logger=feather->logger;
if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write
@@ -1830,7 +1833,16 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
}
table_type=feather->p_table_info[table_id]->table_type;
if(rule_list[i].op==MAAT_OP_DEL)
{
{
if(table_type==TABLE_TYPE_PLUGIN)
{
plugin_desc=&(feather->p_table_info[table_id]->plugin);
valid_column=plugin_desc->valid_flag_column;
}
else
{
valid_column=-1;
}
ret=invalidate_line(rule_list[i].table_line, table_type, valid_column);
if(ret<0)
{
@@ -2016,6 +2028,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
int ret=0, table_id=0,success_cnt=0;
struct serial_rule_t *s_rule=NULL;
struct _Maat_table_info_t* p_table=NULL;
struct plugin_table_desc* plugin_desc=NULL;
long long server_time=0,absolute_expire_time=0;
const char* p_foreign=NULL;
@@ -2039,6 +2052,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
ret=-1;
goto error_out;
}
p_table=_feather->p_table_info[table_id];
plugin_desc=&(p_table->plugin);
if(TABLE_TYPE_PLUGIN!=p_table->table_type)
{
@@ -2052,7 +2066,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
if(op==MAAT_OP_ADD)
{
ret=get_valid_flag_offset(line_rule[i]->table_line
, p_table->table_type
, p_table->table_type
, plugin_desc->valid_flag_column);
if(ret<0||
(op==MAAT_OP_ADD&&line_rule[i]->table_line[ret]!='1'))
@@ -2071,17 +2085,17 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
if(line_rule[i]->expire_after>0)
{
absolute_expire_time=server_time+line_rule[i]->expire_after;
}
}
if(plugin_desc->n_foreign>0)
{
{
for(j=0;j<plugin_desc->n_foreign;j++)
{
{
p_foreign=find_Nth_column(line_rule[i]->table_line, plugin_desc->foreign_columns[j], &foreign_key_size);
if(p_foreign==NULL)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL, maat_command
, "Command set line %s %d failed: No %dth column."
, line_rule[i]->table_name, line_rule[i]->rule_id
, line_rule[i]->table_name, line_rule[i]->rule_id
, plugin_desc->foreign_columns[j]);
ret=-1;
goto error_out;
@@ -2096,7 +2110,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
}
}
}
}
set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name,
line_rule[i]->table_line, absolute_expire_time);
}
success_cnt=exec_serial_rule(write_ctx,s_rule, line_num,server_time,_feather->logger);

601
src/entry/Maat_rule.cpp
View File

File diff suppressed because it is too large Load Diff

78
src/entry/Maat_stat.cpp
View File

@@ -149,20 +149,20 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_start(feather->stat_handle);
return;
}
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
{
alignment_int64_array_add(p_table->scan_cnt,thread_num,1);
alignment_int64_array_add(p_table->input_bytes,thread_num,scan_len);
alignment_int64_array_add(p->scan_cnt,thread_num,1);
alignment_int64_array_add(p->input_bytes,thread_num,scan_len);
if(start!=NULL&&end!=NULL)
{
alignment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
alignment_int64_array_add(p->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
}
return;
}
void maat_stat_output(struct _Maat_feather_t* feather)
{
long value=0;
long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0;
long long total_cfg_num=0, table_regex_ipv6_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0;
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0;
@@ -172,6 +172,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
int i=0;
time_t now;
struct _Maat_table_info_t* p_table=NULL;
struct table_runtime* table_aux=NULL;
time(&now);
active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
@@ -201,86 +202,105 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_input_bytes=0;
table_scan_cpu_time=0;
table_hit_cnt=0;
table_regex_ipv6_num=0;
p_table=feather->p_table_info[i];
if(p_table==NULL)
{
continue;
}
table_aux=feather->scanner->table_rt[i];
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
plugin_cache_num+=p_table->cb_info->cache_line_num;
plugin_acc_num+=p_table->cb_info->acc_line_num;
plugin_cache_num+=table_aux->plugin.cache_line_num;
plugin_acc_num+=table_aux->plugin.acc_line_num;
break;
case TABLE_TYPE_GROUP:
group_rule_num+=p_table->cfg_num;
group_rule_num+=table_aux->origin_rule_num;
break;
case TABLE_TYPE_COMPILE:
compile_rule_num+=p_table->cfg_num;
compile_rule_num+=table_aux->origin_rule_num;
break;
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
table_regex_ipv6_num=table_aux->expr.regex_rule_cnt;
total_iconv_error=p_table->expr.iconv_err_cnt;
break;
case TABLE_TYPE_IP:
table_regex_ipv6_num=table_aux->ip.ipv6_rule_cnt;
break;
default:
break;
}
if(p_table->table_type==TABLE_TYPE_PLUGIN||
p_table->table_type==TABLE_TYPE_GROUP||
p_table->table_type==TABLE_TYPE_COMPILE)
{
continue;
}
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
p_table->cfg_num);
total_cfg_num+=p_table->cfg_num;
table_aux->origin_rule_num);
total_cfg_num+=table_aux->origin_rule_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
p_table->regex_rule_cnt);
total_regex_num+= p_table->regex_rule_cnt;
table_regex_ipv6_num);
total_regex_num+=table_regex_ipv6_num;
table_stream_num=alignment_int64_array_sum(p_table->stream_num,feather->scan_thread_num);
table_stream_num=alignment_int64_array_sum(table_aux->stream_num,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
FS_OP_ADD,
table_stream_num);
total_stream_cnt+= table_stream_num;
table_scan_cnt=alignment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num);
table_scan_cnt=alignment_int64_array_sum(table_aux->scan_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_SET,
FS_OP_ADD,
table_scan_cnt);
total_scan_cnt+=table_scan_cnt;
table_input_bytes=alignment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num);
table_input_bytes=alignment_int64_array_sum(table_aux->input_bytes,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_SET,
FS_OP_ADD,
table_input_bytes);
total_input_bytes+=table_input_bytes;
if(feather->perf_on==1)
{
table_scan_cpu_time=alignment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time=alignment_int64_array_sum(table_aux->scan_cpu_time,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_SET,
FS_OP_ADD,
table_scan_cpu_time);
total_cpu_time+=table_scan_cpu_time;
}
table_hit_cnt=alignment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num);
table_hit_cnt=alignment_int64_array_sum(table_aux->hit_cnt,feather->scan_thread_num);
alignment_int64_array_reset(table_aux->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_SET,
FS_OP_ADD,
table_hit_cnt);
total_update_error+=p_table->udpate_err_cnt;
total_iconv_error+=p_table->iconv_err_cnt;
//total hit count stat in region_compile
break;
}
}
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
@@ -314,7 +334,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_OP_SET,
total_cpu_time);
}
total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num);
total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt, feather->scan_thread_num);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],

99
src/inc_internal/Maat_rule_internal.h
View File

@@ -101,7 +101,7 @@ struct compile_ex_data_idx
int idx;
int table_id;
};
struct compile_table_descr
struct compile_table_desc
{
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
@@ -135,7 +135,7 @@ struct _plugin_table_info
long cache_size;
};
struct plugin_table_descr
struct plugin_table_desc
{
int key_column;
int valid_flag_column;
@@ -147,7 +147,7 @@ struct plugin_table_descr
struct plugin_table_ex_data_desc ex_desc;
int acc_line_num;
};
struct expr_table_descr
struct expr_table_desc
{
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
@@ -155,10 +155,9 @@ struct expr_table_descr
int do_charset_merge;
int cross_cache_size;
int quick_expr_switch;
int expr_rule_cnt; //expr_type=0,1,3
int regex_rule_cnt; //expr_type=2
long long iconv_err_cnt;
};
struct ip_table_descr
struct ip_table_desc
{
int ipv4_rule_cnt;
int ipv6_rule_cnt;
@@ -173,17 +172,17 @@ struct _Maat_table_info_t
enum MAAT_TABLE_TYPE table_type;
union
{
struct compile_table_descr compile;
struct expr_table_descr expr;
struct ip_table_descr ip;
struct plugin_table_descr plugin;
struct compile_table_desc compile;
struct expr_table_desc expr;
struct ip_table_desc ip;
struct plugin_table_desc plugin;
void* others;//group, interval and digest don't have sperate description info.
};
/*
enum MAAT_CHARSET src_charset;
enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
int src_charset_in_dst;
int do_charset_merge;
int cfg_num;
int cross_cache_size;
int quick_expr_switch;
union
@@ -206,16 +205,11 @@ struct _Maat_table_info_t
enum USER_REGION_ENCODE user_region_encoding;
int ex_data_num;
struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
*/
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long unmatch_tag_cnt;
unsigned long long iconv_err_cnt;
int stat_line_id;
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct db_str_rule_t
@@ -334,24 +328,6 @@ struct _compile_result_t
int compile_id;
universal_bool_expr_t group_set;
};
struct _region_stat_t
{
int cfg_num;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
};
struct _INNER_scan_status_t
{
int cur_hit_cnt;
@@ -403,19 +379,48 @@ struct _stream_para_t
pthread_mutex_t fuzzy_mutex;
unsigned char query_point[8];
};
struct digest_engine
struct similar_runtime
{
enum MAAT_TABLE_TYPE table_type;
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
struct plugin_engine
struct plugin_runtime
{
dynamic_array_t *cache_lines;
int cache_line_num;
long cache_size;
long long cache_line_num;
long long acc_line_num;
long long cache_size;
MESA_htable_handle ex_data_hash;
};
struct expr_runtime
{
long long expr_rule_cnt; //expr_type=0,1,3
long long regex_rule_cnt; //expr_type=2
};
struct ip_runtime
{
long long ipv4_rule_cnt;
long long ipv6_rule_cnt;
};
struct table_runtime
{
enum MAAT_TABLE_TYPE table_type;
long origin_rule_num;
union
{
struct similar_runtime similar; //for digest and similarity
struct plugin_runtime plugin;
struct expr_runtime expr;
struct ip_runtime ip;
void * other;
};
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct rule_tag
{
char* tag_name;
@@ -425,11 +430,15 @@ struct _Maat_scanner_t
{
long long version;
time_t last_update_time;
long long *ref_cnt; //optimized for cache_alignment 64
mcore_long_t ref_cnt;
rule_scanner_t region;
long gie_total_q_size;
struct digest_engine gie_aux[MAX_TABLE_NUM];
struct plugin_engine plugin_aux[MAX_TABLE_NUM];
struct table_runtime* table_rt[MAX_TABLE_NUM];
/*
struct similar_runtime gie_aux[MAX_TABLE_NUM];
struct plugin_runtime plugin_aux[MAX_TABLE_NUM];
*/
MESA_htable_handle region_hash;
MESA_htable_handle group_hash;
MESA_htable_handle compile_hash;
@@ -443,7 +452,7 @@ struct _Maat_scanner_t
void * bool_macher_expr_compiler;
scan_result_t *region_rslt_buff;
MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q
struct _region_stat_t region_counter[MAX_TABLE_NUM];
// struct _region_stat_t region_counter[MAX_TABLE_NUM];
int max_thread_num;
iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from]
};
@@ -608,7 +617,7 @@ int HASH_add_by_id(MESA_htable_handle hash,int id,void*data);
int HASH_delete_by_id(MESA_htable_handle hash,int id);
void maat_read_full_config(_Maat_feather_t* _feather);
void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_table(struct table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_output(struct _Maat_feather_t* feather);
redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);

5
src/inc_internal/alignment_int64.h
View File

@@ -43,6 +43,11 @@ inline long long alignment_int64_array_cnt(mcore_long_t array,int size)
}
return cnt;
}
inline void alignment_int64_array_reset(mcore_long_t array,int size)
{
memset(array, 0, CPU_CACHE_ALIGMENT*size);
return;
}
inline void alignment_int64_array_free(mcore_long_t array)
{
free(array);

2
test/test_maatframe.cpp
View File

@@ -480,7 +480,7 @@ TEST(SimilarScan, Pure)
const char* scan_data="mwss.xiu.youku.com/live/hls/v1/0000000000000000000000001526a0a8/714.ts?&token=98765";
const char* table_name="SIM_URL";
scan_status_t mid=NULL;
memset(result, 0 , sizeof(result));
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);