gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复域表配置计数bug：配置更新时，table->cfg_num与rulescan中实际的配置数可能不一致，导致扫描中rulescan产生调用无效subtype的错误日志。

Browse Source
This commit is contained in:
zhengchao
2016-12-28 18:06:34 +08:00
parent 621ae259fa
commit b17870141e
3 changed files with 138 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/entry/Maat_api.cpp
View File

@@ -741,6 +741,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
_compile_result_t compile_result[rule_num];//dynamic array _compile_result_t compile_result[rule_num];//dynamic array
struct _Maat_table_info_t *p_table=NULL; struct _Maat_table_info_t *p_table=NULL;
struct timespec start,end; struct timespec start,end;
_Maat_scanner_t* my_scanner=NULL;
if(data==NULL||data_len<=0) if(data==NULL||data_len<=0)
{ {
return 0; return 0;
@@ -756,6 +757,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++; _feather->scan_err_cnt++;
return -1; return -1;
} }
if(p_table->cfg_num==0) if(p_table->cfg_num==0)
{ {
return 0; return 0;
@@ -778,17 +780,22 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
scan_data.text_data.text=data; scan_data.text_data.text=data;
scan_data.text_data.tlen=data_len; scan_data.text_data.tlen=data_len;
scan_data.text_data.toffset=0; scan_data.text_data.toffset=0;
_Maat_scanner_t* my_scanner=NULL;
my_scanner=_feather->scanner; my_scanner=_feather->scanner;
if(my_scanner==NULL) if(my_scanner==NULL)
{ {
return 0; return 0;
} }
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
{
return 0;
}
assert(thread_num<_feather->scan_thread_num); assert(thread_num<_feather->scan_thread_num);
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
INC_SCANNER_REF(my_scanner, thread_num); INC_SCANNER_REF(my_scanner, thread_num);
if(p_table->expr_rule_cnt>0) if(region_stat->expr_rule_cnt>0)
{ {
scan_data.rule_type=RULETYPE_STR; scan_data.rule_type=RULETYPE_STR;
scan_data.sub_type=sub_type; scan_data.sub_type=sub_type;
@@ -798,7 +805,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
hit_region_cnt+=region_ret; hit_region_cnt+=region_ret;
} }
} }
if(p_table->regex_rule_cnt>0) if(region_stat->regex_rule_cnt>0)
{ {
scan_data.rule_type=RULETYPE_REG; scan_data.rule_type=RULETYPE_REG;
scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0); scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE,0);
@@ -895,6 +902,12 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
{ {
return 0; return 0;
} }
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
{
return 0;
}
aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
@@ -965,19 +978,25 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{ {
return 0; return 0;
} }
if(p_table->ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
{
return 0;
}
if(p_table->ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
{
return 0;
}
my_scanner=_feather->scanner; my_scanner=_feather->scanner;
if(my_scanner==NULL) if(my_scanner==NULL)
{ {
return 0; return 0;
} }
struct _region_stat_t * region_stat=NULL;
region_stat=&(my_scanner->region_counter[p_table->table_id]);
if(region_stat->cfg_num==0)
{
return 0;
}
if(region_stat->ipv4_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV4)
{
return 0;
}
if(region_stat->ipv6_rule_cnt==0&&addr->addrtype==ADDR_TYPE_IPV6)
{
return 0;
}
aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
ip_scan_data.rule_type=RULETYPE_IPv4; ip_scan_data.rule_type=RULETYPE_IPv4;
@@ -1083,7 +1102,8 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{ {
return sp; return sp;
} }
struct _region_stat_t * region_stat=NULL;
region_stat=&(scanner->region_counter[p_table->table_id]);
INC_SCANNER_REF(scanner, thread_num); INC_SCANNER_REF(scanner, thread_num);
sp->table_id=table_id; sp->table_id=table_id;
@@ -1096,11 +1116,11 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{ {
sp->do_merge=1; sp->do_merge=1;
} }
if(p_table->expr_rule_cnt>0) if(region_stat->expr_rule_cnt>0)
{ {
sp->do_expr=1; sp->do_expr=1;
} }
if(p_table->regex_rule_cnt>0) if(region_stat->regex_rule_cnt>0)
{ {
sp->do_regex=1; sp->do_regex=1;
} }
@@ -1144,10 +1164,9 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
return 0; return 0;
} }
p_table=sp->feather->p_table_info[sp->table_id]; p_table=sp->feather->p_table_info[sp->table_id];
if(p_table->cfg_num==0) //table rule num is already judged in Maat_stream_scan_string_start
{
return 0;
}
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{ {
sp->feather->scan_err_cnt++; sp->feather->scan_err_cnt++;

99
src/entry/Maat_rule.cpp
View File

@@ -27,7 +27,7 @@
#include "mesa_fuzzy.h" #include "mesa_fuzzy.h"
#include "great_index_engine.h" #include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_8_20161226=1; int MAAT_FRAME_VERSION_1_8_20161228=1;
const char *maat_module="MAAT Frame"; const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
@@ -848,7 +848,7 @@ void destroy_rs_intval_rule(scan_rule_t* p)
free(p); free(p);
} }
struct op_expr_t* create_op_expr(unsigned int expr_id,int operation,void* u_para) struct op_expr_t* create_op_expr(unsigned int expr_id,int operation,void* u_para,int table_id)
{ {
struct op_expr_t* op_expr=NULL; struct op_expr_t* op_expr=NULL;
op_expr=(struct op_expr_t*)calloc(sizeof(struct op_expr_t),1); op_expr=(struct op_expr_t*)calloc(sizeof(struct op_expr_t),1);
@@ -860,6 +860,7 @@ struct op_expr_t* create_op_expr(unsigned int expr_id,int operation,void* u_para
op_expr->p_expr->rnum=0; op_expr->p_expr->rnum=0;
op_expr->p_expr->rules=NULL; op_expr->p_expr->rules=NULL;
op_expr->p_expr->tag=u_para; op_expr->p_expr->tag=u_para;
op_expr->table_id=table_id;
return op_expr; return op_expr;
} }
void destroy_op_expr(struct op_expr_t* op_expr) void destroy_op_expr(struct op_expr_t* op_expr)
@@ -1115,9 +1116,47 @@ void destroy_ip_expr(boolean_expr_t*p)
free(p); free(p);
return; return;
} }
void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_stat, int size)
{
assert(op_expr->table_id<size);
int op=0;
if(op_expr->p_expr->operation==0)//add
{
op=1;
}
else if(op_expr->p_expr->operation==1)//delete
{
op=-1;
}
else
{
assert(0);
}
region_stat[op_expr->table_id].cfg_num+=op;
switch(op_expr->p_rules[0]->rule_type)
{
case RULETYPE_STR:
region_stat[op_expr->table_id].expr_rule_cnt+=op;
break;
case RULETYPE_REG:
region_stat[op_expr->table_id].regex_rule_cnt+=op;
break;
case RULETYPE_INT:
break;
case RULETYPE_IPv4:
region_stat[op_expr->table_id].ipv4_rule_cnt+=op;
break;
case RULETYPE_IPv6:
region_stat[op_expr->table_id].ipv6_rule_cnt+=op;
break;
default:
assert(0);
break;
}
return;
}
void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct _Maat_scanner_t* maat_scanner)
void rulescan_batch_update(rule_scanner_t scanner,MESA_lqueue_head expr_queue,void*logger)
{ {
long i=0,data_size=0; long i=0,data_size=0;
int j=0,ret=0; int j=0,ret=0;
@@ -1129,10 +1168,12 @@ void rulescan_batch_update(rule_scanner_t scanner,MESA_lqueue_head expr_queue,vo
const long q_cnt=MESA_lqueue_get_count(expr_queue); const long q_cnt=MESA_lqueue_get_count(expr_queue);
struct timespec start,end; struct timespec start,end;
unsigned long long update_interval=0; unsigned long long update_interval=0;
struct _region_stat_t region_counter[MAX_TABLE_NUM];
if(q_cnt==0) if(q_cnt==0)
{ {
return; return;
} }
memset(region_counter,0,sizeof(region_counter));
boolean_expr_t* to_update_expr=(boolean_expr_t*)calloc(sizeof(boolean_expr_t),q_cnt); boolean_expr_t* to_update_expr=(boolean_expr_t*)calloc(sizeof(boolean_expr_t),q_cnt);
struct op_expr_t* op_expr=NULL; struct op_expr_t* op_expr=NULL;
for(i=0;i<q_cnt;i++) for(i=0;i<q_cnt;i++)
@@ -1154,13 +1195,14 @@ void rulescan_batch_update(rule_scanner_t scanner,MESA_lqueue_head expr_queue,vo
,to_update_expr[i].rules[j].string_rule.len); ,to_update_expr[i].rules[j].string_rule.len);
} }
} }
count_rs_region(op_expr,region_counter,MAX_TABLE_NUM);
destroy_op_expr(op_expr); destroy_op_expr(op_expr);
op_expr=NULL; op_expr=NULL;
} }
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"scanner %p rulescan_update %ld rules.",scanner,q_cnt); "rs_handle %p rulescan_update %ld rules.",rs_handle,q_cnt);
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC,&start);
ret=rulescan_update(scanner, to_update_expr,q_cnt, failed_ids,MAX_FAILED_NUM); ret=rulescan_update(rs_handle, to_update_expr,q_cnt, failed_ids,MAX_FAILED_NUM);
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
if(ret!=1) if(ret!=1)
{ {
@@ -1174,11 +1216,20 @@ void rulescan_batch_update(rule_scanner_t scanner,MESA_lqueue_head expr_queue,vo
} }
update_interval=(end.tv_sec-start.tv_sec)*1000000000+end.tv_nsec-start.tv_nsec; update_interval=(end.tv_sec-start.tv_sec)*1000000000+end.tv_nsec-start.tv_nsec;
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"scanner %p rulescan_update with %2.2e (%llu) ns." "rs_handle %p rulescan_update with %2.2e (%llu) ns."
,scanner ,rs_handle
,(double)update_interval ,(double)update_interval
,update_interval); ,update_interval);
//update scanner's region cnt;
for(i=0;i<MAX_TABLE_NUM;i++)
{
maat_scanner->region_counter[i].cfg_num+=region_counter[i].cfg_num;
maat_scanner->region_counter[i].expr_rule_cnt+=region_counter[i].expr_rule_cnt;
maat_scanner->region_counter[i].regex_rule_cnt+=region_counter[i].regex_rule_cnt;
assert(maat_scanner->region_counter[i].cfg_num>=0);
assert(maat_scanner->region_counter[i].expr_rule_cnt>=0);
assert(maat_scanner->region_counter[i].regex_rule_cnt>=0);
}
for(i=0;i<q_cnt;i++) for(i=0;i<q_cnt;i++)
{ {
for(j=0;j<(int)to_update_expr[i].rnum;j++) for(j=0;j<(int)to_update_expr[i].rnum;j++)
@@ -1193,7 +1244,7 @@ void rulescan_batch_update(rule_scanner_t scanner,MESA_lqueue_head expr_queue,vo
free(to_update_expr); free(to_update_expr);
} }
void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger) void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct _Maat_scanner_t* maat_scanner,int table_id)
{ {
long i=0,data_size=0; long i=0,data_size=0;
int ret=0; int ret=0;
@@ -1205,6 +1256,8 @@ void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*log
{ {
return; return;
} }
struct _region_stat_t region_counter[MAX_TABLE_NUM];
memset(region_counter,0,sizeof(region_counter));
update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt); update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt);
for(i=0;i<q_cnt;i++) for(i=0;i<q_cnt;i++)
{ {
@@ -1223,6 +1276,14 @@ void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*log
} }
for(i=0;i<q_cnt;i++) for(i=0;i<q_cnt;i++)
{ {
if(update_array[i]->operation==GIE_INSERT_OPT)
{
maat_scanner->region_counter[table_id].cfg_num++;
}
else
{
maat_scanner->region_counter[table_id].cfg_num--;
}
destroy_digest_rule(update_array[i]); destroy_digest_rule(update_array[i]);
update_array[i]=NULL; update_array[i]=NULL;
} }
@@ -1505,7 +1566,8 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
} }
op_expr=create_op_expr(expr_id op_expr=create_op_expr(expr_id
,0 ,0
,u_para); ,u_para
,table->table_id);
for(i=0,p=db_rule->keywords;;i++,p=NULL) for(i=0,p=db_rule->keywords;;i++,p=NULL)
{ {
if(i>=MAAT_MAX_EXPR_ITEM_NUM) if(i>=MAAT_MAX_EXPR_ITEM_NUM)
@@ -1572,6 +1634,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
op_expr=create_op_expr(expr_id op_expr=create_op_expr(expr_id
,0 //add ,0 //add
,u_para ,u_para
,table->table_id
); );
for(k=0;k<sub_expr_cnt;k++) for(k=0;k<sub_expr_cnt;k++)
{ {
@@ -1648,6 +1711,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
op_expr=create_op_expr(expr_id op_expr=create_op_expr(expr_id
,0 //add ,0 //add
,u_para ,u_para
,table->table_id
); );
for(k=0;k<sub_expr_cnt;k++) for(k=0;k<sub_expr_cnt;k++)
{ {
@@ -1694,6 +1758,7 @@ int add_ip_rule(struct _Maat_table_info_t* table,struct db_ip_rule_t* db_ip_rule
op_expr=create_op_expr(expr_id op_expr=create_op_expr(expr_id
,0 ,0
,u_para ,u_para
,table->table_id
); );
p_rule=create_rs_ip_rule(make_sub_type(table->table_id,CHARSET_NONE,0) p_rule=create_rs_ip_rule(make_sub_type(table->table_id,CHARSET_NONE,0)
,db_ip_rule); ,db_ip_rule);
@@ -1724,6 +1789,7 @@ int add_intval_rule(struct _Maat_table_info_t* table,struct db_intval_rule_t* in
op_expr=create_op_expr(expr_id op_expr=create_op_expr(expr_id
,0 ,0
,u_para ,u_para
,table->table_id
); );
p_rule=create_rs_intval_rule(make_sub_type(table->table_id,CHARSET_NONE,0) p_rule=create_rs_intval_rule(make_sub_type(table->table_id,CHARSET_NONE,0)
,intval_rule); ,intval_rule);
@@ -1794,7 +1860,7 @@ int del_region_rule(struct _Maat_table_info_t* table,int region_id,int group_id,
case TABLE_TYPE_INTVAL: case TABLE_TYPE_INTVAL:
for(i=0;i<expr_num;i++) for(i=0;i<expr_num;i++)
{ {
op_expr=create_op_expr(expr_id[i],1,NULL);//del expr op_expr=create_op_expr(expr_id[i],1,NULL,table->table_id);//del expr
MESA_lqueue_join_tail(maat_scanner->region_update_q,&op_expr, sizeof(void*)); MESA_lqueue_join_tail(maat_scanner->region_update_q,&op_expr, sizeof(void*));
} }
break; break;
@@ -2638,7 +2704,8 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
scanner->dedup_expr_num=0; scanner->dedup_expr_num=0;
rulescan_batch_update(scanner->region, rulescan_batch_update(scanner->region,
scanner->region_update_q, scanner->region_update_q,
logger); logger
,scanner);
for(i=0;i<MAX_TABLE_NUM;i++) for(i=0;i<MAX_TABLE_NUM;i++)
{ {
if(scanner->digest_update_q[i]==NULL) if(scanner->digest_update_q[i]==NULL)
@@ -2655,7 +2722,11 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
{ {
scanner->digest_handle[i]=GIE_create(&para); scanner->digest_handle[i]=GIE_create(&para);
} }
digest_batch_update(scanner->digest_handle[i], scanner->digest_update_q[i],logger); digest_batch_update(scanner->digest_handle[i]
,scanner->digest_update_q[i]
,logger
,scanner
,i);
pthread_rwlock_unlock(&(scanner->digest_rwlock[i])); pthread_rwlock_unlock(&(scanner->digest_rwlock[i]));
} }
if(scanner->tmp_district_map!=NULL) if(scanner->tmp_district_map!=NULL)

16
src/entry/Maat_rule_internal.h
View File

@@ -170,6 +170,7 @@ struct op_expr_t
scan_rule_t* p_rules[MAAT_MAX_EXPR_ITEM_NUM]; scan_rule_t* p_rules[MAAT_MAX_EXPR_ITEM_NUM];
int convert_failed; int convert_failed;
int no_effect_convert_cnt; int no_effect_convert_cnt;
int table_id;
}; };
struct _Maat_region_rule_t struct _Maat_region_rule_t
@@ -221,6 +222,20 @@ struct _plugin_table_info
long cache_size; long cache_size;
pthread_mutex_t plugin_mutex; pthread_mutex_t plugin_mutex;
}; };
struct _region_stat_t
{
int cfg_num;
union
{
int expr_rule_cnt; //expr_type=0,1,3
int ipv4_rule_cnt;
};
union
{
int regex_rule_cnt; //expr_type=2
int ipv6_rule_cnt;
};
};
struct _Maat_table_info_t struct _Maat_table_info_t
{ {
unsigned short table_id; unsigned short table_id;
@@ -328,6 +343,7 @@ struct _Maat_scanner_t
void * expr_compiler; void * expr_compiler;
scan_result_t *region_rslt_buff; scan_result_t *region_rslt_buff;
MESA_lqueue_head tomb_ref;//reference of feather->garbage_q MESA_lqueue_head tomb_ref;//reference of feather->garbage_q
struct _region_stat_t region_counter[MAX_TABLE_NUM];
int max_thread_num; int max_thread_num;
iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from] iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from]
}; };