gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

在状态输出文件中，增加更新错误、转码错误和扫描错误计数。

Browse Source
This commit is contained in:
zhengchao
2016-10-08 11:40:57 +08:00
parent e51e8f1fe0
commit 2a0c7738e3
4 changed files with 75 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/entry/Maat_api.cpp
View File

@@ -750,6 +750,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
@@ -758,6 +759,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
}
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{
_feather->scan_err_cnt++;
return -1;
}
if(p_table->do_charset_merge==1)
@@ -878,6 +880,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
p_table=acqurie_table(_feather,table_id,TABLE_TYPE_INTVAL);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
@@ -898,6 +901,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
if(region_ret<0)
{
DEC_SCANNER_REF(my_scanner, thread_num);
_feather->scan_err_cnt++;
return -1;
}
else if(region_ret>0)
@@ -951,6 +955,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
return -1;
}
if(p_table->cfg_num==0)
@@ -994,6 +999,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
ip_scan_data.ipv6_data.proto=proto;
break;
default:
_feather->scan_err_cnt++;
return -1;
break;
}
@@ -1005,6 +1011,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
if(region_ret<0)
{
DEC_SCANNER_REF(my_scanner,thread_num);
_feather->scan_err_cnt++;
return -1;
}
else if(region_ret>0)
@@ -1055,10 +1062,12 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_EXPR);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
return NULL;
}
if(p_table->quick_expr_switch==1)
{
_feather->scan_err_cnt++;
return NULL;
}
struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1);
@@ -1137,6 +1146,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
}
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{
sp->feather->scan_err_cnt++;
return -1;
}
aligment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
@@ -1189,6 +1199,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
region_ret=rulescan_searchstream(sp->rs_stream_para, &region_scan_data, region_result, MAX_SCANNER_HIT_NUM);
if(region_ret<0)
{
sp->feather->scan_err_cnt++;
return -1;
}
else if(region_ret>0)
@@ -1203,6 +1214,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
region_ret=rulescan_searchstream(sp->rs_stream_para, &region_scan_data, region_result+hit_region_cnt, MAX_SCANNER_HIT_NUM-hit_region_cnt);
if(region_ret<0)
{
sp->feather->scan_err_cnt++;
return -1;
}
else if(region_ret>0)
@@ -1315,11 +1327,13 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST);
if(p_table==NULL)
{
_feather->scan_err_cnt++;
return NULL;
}
tmp_fuzzy_handle=fuzzy_create_handle(total_len);
if(tmp_fuzzy_handle==NULL)
{
_feather->scan_err_cnt++;
return NULL;
}
struct _stream_para_t* sp=(struct _stream_para_t*)calloc(sizeof(struct _stream_para_t),1);
@@ -1423,6 +1437,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
digest_buff=NULL;
if(hit_region_cnt<0)//error occurs
{
sp->feather->scan_err_cnt++;
return -1;
}
if(hit_region_cnt>0)
@@ -1489,6 +1504,7 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA
case MAAT_SET_SCAN_DISTRICT:
if(value==NULL||size<=0)
{
_feather->scan_err_cnt++;
return -1;
}
map_ret=map_unNullstr2int(_feather->scanner->district_map,(const char*)value,size,&(_mid->district_id));
@@ -1499,6 +1515,7 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA
_mid->is_set_district=1;
break;
default:
_feather->scan_err_cnt++;
return -1;
break;
}

38
src/entry/Maat_rule.cpp
View File

@@ -26,7 +26,7 @@
#include "mesa_fuzzy.h"
#include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_8_20160922=1;
int MAAT_FRAME_VERSION_1_8_20161008=1;
const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
@@ -1570,6 +1570,7 @@ int add_expr_rule(struct _Maat_table_info_t* table,struct db_str_rule_t* db_rule
,CHARSET_STRING[dst_charset]);
free(region_string);
op_expr->convert_failed++;
table->iconv_err_cnt++;
break;
}
if(region_str_len==(int)strlen(sub_key_array[k])&&
@@ -1973,6 +1974,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
"update error,invalid format of expr table %s:%s",table->table_name[table->updating_name],table_line);
free(maat_str_rule);
maat_str_rule=NULL;
table->udpate_err_cnt++;
return;
}
break;
@@ -1991,6 +1993,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
"update error,invalid format of expr_plus table %s:%s",table->table_name[table->updating_name],table_line);
free(maat_str_rule);
maat_str_rule=NULL;
table->udpate_err_cnt++;
return;
}
break;
@@ -2016,6 +2019,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid hexbin value of expr table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
goto error_out;
}
ret=sync_region(scanner->region_hash
@@ -2024,6 +2028,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
,maat_str_rule->is_valid,logger);
if(ret<0)
{
table->udpate_err_cnt++;
goto error_out;
}
if(group_mode_on==FALSE)//for compatible old version
@@ -2044,6 +2049,10 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
{
table->cfg_num--;
}
else
{
table->udpate_err_cnt++;
}
}
else
{
@@ -2054,7 +2063,7 @@ void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,st
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"duplicate config of expr table %s region_id=%d"
,table->table_name[table->updating_name],maat_str_rule->region_id);
table->udpate_err_cnt++;
}
else
{
@@ -2096,6 +2105,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of ip table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
goto error_out;
}
if(ip_rule->addr_type==4)
@@ -2147,6 +2157,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of ip table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
goto error_out;
}
}
@@ -2156,6 +2167,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru
,ip_rule->is_valid,logger);
if(ret<0)
{
table->udpate_err_cnt++;
goto error_out;
}
if(group_mode_on==FALSE)//for compatible old version
@@ -2183,6 +2195,10 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru
table->ipv6_rule_cnt--;
}
}
else
{
table->udpate_err_cnt++;
}
}
else
{
@@ -2193,7 +2209,7 @@ void update_ip_rule(struct _Maat_table_info_t* table,const char* table_line,stru
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"duplicate config of ip table %s config_id=%d"
,table->table_name[table->updating_name],ip_rule->region_id);
table->udpate_err_cnt++;
}
else
{
@@ -2228,6 +2244,7 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of interval table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
goto error_out;
}
ret=sync_region(scanner->region_hash
@@ -2236,6 +2253,7 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,
,intval_rule->is_valid,logger);
if(ret<0)
{
table->udpate_err_cnt++;
goto error_out;
}
if(group_mode_on==FALSE)//for compatible old version
@@ -2256,6 +2274,10 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,
{
table->cfg_num--;
}
else
{
table->udpate_err_cnt++;
}
}
else
{
@@ -2265,6 +2287,7 @@ void update_intval_rule(struct _Maat_table_info_t* table,const char* table_line,
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"duplicate config of intval table %s config_id=%d"
,table->table_name[table->updating_name],intval_rule->region_id);
table->udpate_err_cnt++;
}
else
{
@@ -2299,6 +2322,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line
,table->table_name[table->updating_name],table_line);
free(p_compile);
p_compile=NULL;
table->udpate_err_cnt++;
return;
}
@@ -2329,6 +2353,7 @@ void update_compile_rule(struct _Maat_table_info_t* table,const char* table_line
p_compile->service_defined=NULL;
free(p_compile);
p_compile=NULL;
table->udpate_err_cnt++;
}
else
@@ -2358,6 +2383,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of digest table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
goto error_out;
}
ret=sync_region(scanner->region_hash
@@ -2366,6 +2392,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,
,digest_rule->is_valid,logger);
if(ret<0)
{
table->udpate_err_cnt++;
goto error_out;
}
if(group_mode_on==FALSE)//for compatible old version
@@ -2386,6 +2413,10 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,
{
table->cfg_num--;
}
else
{
table->udpate_err_cnt++;
}
}
else
{
@@ -2395,6 +2426,7 @@ void update_digest_rule(struct _Maat_table_info_t* table,const char* table_line,
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"duplicate config of intval table %s config_id=%d"
,table->table_name[table->updating_name],digest_rule->region_id);
table->udpate_err_cnt++;
}
else
{

5
src/entry/Maat_rule_internal.h
View File

@@ -247,6 +247,8 @@ struct _Maat_table_info_t
};
struct _plugin_table_info *cb_info;
//for stat>>>>>>>>
unsigned long long udpate_err_cnt;
unsigned long long iconv_err_cnt;
int stat_line_id;
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
@@ -360,6 +362,9 @@ struct _Maat_feather_t
mcore_long_t thread_call_cnt;//size indicate by scan_thread_num,
long long total_scan_bytes;
long long total_scan_cnt;
long long update_err_cnt;//sum of the same name variable in each table
long long iconv_err_cnt;//sum of the same name variable in each table
long long scan_err_cnt;
};
struct _maat_garbage_t
{

20
src/entry/Maat_stat.cpp
View File

@@ -11,6 +11,9 @@ enum MAAT_FS_STATUS{
STATUS_GARBAGE_QSIZE,
STATUS_TOTAL_SCAN_LEN,
STATUS_TOTAL_SCAN_CNT,
STATUS_UPDATE_ERR_CNT,
STATUS_ICONV_ERR_CNT,
STATUS_SCAN_ERR_CNT
};
enum MAAT_FS_COLUMN
@@ -46,7 +49,9 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num");
feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes");
feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_times");
feather->fs_status_id[STATUS_UPDATE_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"update_error");
feather->fs_status_id[STATUS_ICONV_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"iconv_error");
feather->fs_status_id[STATUS_SCAN_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_error");
feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule");
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex");
@@ -131,6 +136,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0;
long long total_update_error=0,total_iconv_error=0;
int i=0;
time_t now;
struct _Maat_table_info_t* p_table=NULL;
@@ -146,7 +152,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
value=MESA_lqueue_get_count(feather->garbage_q);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);
feather->update_err_cnt=0;
feather->iconv_err_cnt=0;
for(i=0;i<MAX_TABLE_NUM;i++)
{
table_stream_num=0;
@@ -216,9 +223,12 @@ void maat_stat_output(struct _Maat_feather_t* feather)
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_SET,
table_hit_cnt);
total_update_error+=p_table->udpate_err_cnt;
total_iconv_error+=p_table->iconv_err_cnt;
//total hit count stat in region_compile
}
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
@@ -262,6 +272,12 @@ void maat_stat_output(struct _Maat_feather_t* feather)
feather->total_scan_cnt=total_scan_cnt;
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_LEN], 0,FS_OP_SET,feather->total_scan_bytes);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_CNT], 0,FS_OP_SET,feather->total_scan_cnt);
feather->update_err_cnt=total_update_error;
feather->iconv_err_cnt=total_iconv_error;
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_UPDATE_ERR_CNT], 0,FS_OP_SET,feather->update_err_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ICONV_ERR_CNT], 0,FS_OP_SET,feather->iconv_err_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_SCAN_ERR_CNT], 0,FS_OP_SET,feather->scan_err_cnt);
FS_passive_output(feather->stat_handle);
return;
}