gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1、修复postpone机制中最小更新间隔时间判断错误的bug；2、maat_stat中增加compile、group、plugin、postpone的统计。

Browse Source
This commit is contained in:
zhengchao
2016-12-15 12:05:48 +08:00
parent fb4f1b7a12
commit 220b3ec04c
4 changed files with 119 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/entry/Maat_api.cpp
View File

@@ -704,13 +704,13 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
p_table->cb_info->cb_plug[idx].update=update;
p_table->cb_info->cb_plug[idx].finish=finish;
p_table->cb_info->cb_plug[idx].u_para=u_para;
if(p_table->cb_info->line_num>0)
if(p_table->cb_info->cache_line_num>0)
{
if(start!=NULL)
{
start(MAAT_RULE_UPDATE_TYPE_FULL,u_para);
}
for(i=0;i<p_table->cb_info->line_num;i++)
for(i=0;i<p_table->cb_info->cache_line_num;i++)
{
lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i);
if(lines==NULL)

25
src/entry/Maat_rule.cpp
View File

@@ -27,7 +27,7 @@
#include "mesa_fuzzy.h"
#include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_8_20161123=1;
int MAAT_FRAME_VERSION_1_8_20161215=1;
const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
@@ -1922,8 +1922,11 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
if(db_group_rule.is_valid==FALSE)
{
del_group_rule(table, &db_group_rule,scanner,logger);
//leave no trace for virtual group rule
// table->cfg_num--;
//leave no trace when compatible_group_update calling
if(table->table_type==TABLE_TYPE_GROUP)
{
table->cfg_num--;
}
}
else
{
@@ -1939,7 +1942,10 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
else
{
//no need to free db_group_rule,it was saved in scanner->compile_hash
// table->cfg_num++;
if(table->table_type==TABLE_TYPE_GROUP)
{
table->cfg_num++;
}
}
}
@@ -2543,6 +2549,7 @@ void plugin_table_callback(struct _Maat_table_info_t* table,const char* table_li
struct _plugin_table_info* p_table_cb=table->cb_info;
char *p=NULL;
pthread_mutex_lock(&(p_table_cb->plugin_mutex));
p_table_cb->acc_line_num++;
if(p_table_cb->cb_plug_cnt>0)
{
for(i=0;i<p_table_cb->cb_plug_cnt;i++)
@@ -2555,8 +2562,8 @@ void plugin_table_callback(struct _Maat_table_info_t* table,const char* table_li
p=(char*)calloc(len,1);
memcpy(p,table_line,len);
p_table_cb->cache_size+=len;
dynamic_array_write(p_table_cb->cache_lines,p_table_cb->line_num,p);
p_table_cb->line_num++;
dynamic_array_write(p_table_cb->cache_lines,p_table_cb->cache_line_num,p);
p_table_cb->cache_line_num++;
}
pthread_mutex_unlock(&(p_table_cb->plugin_mutex));
}
@@ -2825,12 +2832,14 @@ void *thread_rule_monitor(void *arg)
if(feather->scanner!=NULL)
{
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
if(expr_wait_q_cnt>0&&time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms*1000)
feather->postpone_q_size=expr_wait_q_cnt;
if(expr_wait_q_cnt>0&&time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms/1000)
{
do_scanner_update(feather->scanner
,feather->garbage_q
,feather->scan_thread_num
,feather->logger);
feather->postpone_q_size=0;
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
"Actual udpate config version %u %d entries load to rulescan after postpone.",
feather->scanner->version,feather->scanner->cfg_num);
@@ -2861,7 +2870,7 @@ void *thread_rule_monitor(void *arg)
if(feather->p_table_info[i]->table_type==TABLE_TYPE_PLUGIN)
{
d_array=feather->p_table_info[i]->cb_info->cache_lines;
for(j=0;j<feather->p_table_info[i]->cb_info->line_num;j++)
for(j=0;j<feather->p_table_info[i]->cb_info->cache_line_num;j++)
{
lines=(char*)dynamic_array_read(d_array, j);
free(lines);

5
src/entry/Maat_rule_internal.h
View File

@@ -215,7 +215,8 @@ struct _plugin_table_info
int cb_plug_cnt;
struct _callback_plugin cb_plug[MAX_PLUGING_NUM];
dynamic_array_t *cache_lines;
int line_num;
int cache_line_num;
int acc_line_num;
int update_type;
int is_called_started;
long cache_size;
@@ -368,6 +369,8 @@ struct _Maat_feather_t
long long iconv_err_cnt;//sum of the same name variable in each table
long long scan_err_cnt;
long long zombie_rs_stream;
long long postpone_q_size;
long long compile_rule_num;
};
struct _maat_garbage_t
{

45
src/entry/Maat_stat.cpp
View File

@@ -6,6 +6,11 @@ enum MAAT_FS_STATUS{
STATUS_VERSION=0,
STATUS_THRED_NUM,
STATUS_TABLE_NUM,
STATUS_PLUGIN_CACHE_NUM,
STATUS_PLUGIN_ACC_NUM,
STATUS_GROUP_RULE_NUM,
STATUS_COMPILE_RULE_NUM,
STATUS_POSTPONE_QSIZE,
STATUS_OUTER_MID_NUM,
STATUS_INNER_MID_NUM,
STATUS_GARBAGE_QSIZE,
@@ -45,9 +50,19 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version");
feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"active_thread");
feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num");
feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_cache");
feather->fs_status_id[STATUS_PLUGIN_ACC_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_acc");
feather->fs_status_id[STATUS_GROUP_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"group_num");
feather->fs_status_id[STATUS_COMPILE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"compile_num");
feather->fs_status_id[STATUS_POSTPONE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"postpone_num");
feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num");
feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid");
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num");
feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes");
feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_times");
feather->fs_status_id[STATUS_UPDATE_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"update_error");
@@ -140,6 +155,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0;
long long total_update_error=0,total_iconv_error=0;
long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0;
int i=0;
time_t now;
struct _Maat_table_info_t* p_table=NULL;
@@ -165,12 +181,23 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_scan_cpu_time=0;
table_hit_cnt=0;
p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
||p_table->table_type==TABLE_TYPE_GROUP
||p_table->table_type==TABLE_TYPE_COMPILE)
if(p_table==NULL)
{
continue;
}
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
plugin_cache_num+=p_table->cb_info->cache_line_num;
plugin_acc_num+=p_table->cb_info->acc_line_num;
break;
case TABLE_TYPE_GROUP:
group_rule_num+=p_table->cfg_num;
break;
case TABLE_TYPE_COMPILE:
compile_rule_num+=p_table->cfg_num;
break;
default:
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
@@ -229,7 +256,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
total_update_error+=p_table->udpate_err_cnt;
total_iconv_error+=p_table->iconv_err_cnt;
//total hit count stat in region_compile
break;
}
}
FS_operate(feather->stat_handle,
@@ -282,6 +310,13 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_SCAN_ERR_CNT], 0,FS_OP_SET,feather->scan_err_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM], 0,FS_OP_SET,feather->zombie_rs_stream);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM], 0,FS_OP_SET,plugin_cache_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_ACC_NUM], 0,FS_OP_SET,plugin_acc_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_RULE_NUM], 0,FS_OP_SET,group_rule_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_COMPILE_RULE_NUM], 0,FS_OP_SET,compile_rule_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_POSTPONE_QSIZE], 0,FS_OP_SET,feather->postpone_q_size);
FS_passive_output(feather->stat_handle);
return;
}