gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1、修复postpone机制中最小更新间隔时间判断错误的bug；2、maat_stat中增加compile、group、plugin、postpone的统计。

Browse Source
This commit is contained in:
zhengchao
2016-12-15 12:05:48 +08:00
parent fb4f1b7a12
commit 220b3ec04c
4 changed files with 119 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/entry/Maat_api.cpp
View File

@@ -704,13 +704,13 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
p_table->cb_info->cb_plug[idx].update=update; p_table->cb_info->cb_plug[idx].update=update;
p_table->cb_info->cb_plug[idx].finish=finish; p_table->cb_info->cb_plug[idx].finish=finish;
p_table->cb_info->cb_plug[idx].u_para=u_para; p_table->cb_info->cb_plug[idx].u_para=u_para;
if(p_table->cb_info->line_num>0) if(p_table->cb_info->cache_line_num>0)
{ {
if(start!=NULL) if(start!=NULL)
{ {
start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); start(MAAT_RULE_UPDATE_TYPE_FULL,u_para);
} }
for(i=0;i<p_table->cb_info->line_num;i++) for(i=0;i<p_table->cb_info->cache_line_num;i++)
{ {
lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i); lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i);
if(lines==NULL) if(lines==NULL)

25
src/entry/Maat_rule.cpp
View File

@@ -27,7 +27,7 @@
#include "mesa_fuzzy.h" #include "mesa_fuzzy.h"
#include "great_index_engine.h" #include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_8_20161123=1; int MAAT_FRAME_VERSION_1_8_20161215=1;
const char *maat_module="MAAT Frame"; const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
@@ -1922,8 +1922,11 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
if(db_group_rule.is_valid==FALSE) if(db_group_rule.is_valid==FALSE)
{ {
del_group_rule(table, &db_group_rule,scanner,logger); del_group_rule(table, &db_group_rule,scanner,logger);
//leave no trace for virtual group rule //leave no trace when compatible_group_update calling
// table->cfg_num--; if(table->table_type==TABLE_TYPE_GROUP)
{
table->cfg_num--;
}
} }
else else
{ {
@@ -1939,7 +1942,10 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
else else
{ {
//no need to free db_group_rule,it was saved in scanner->compile_hash //no need to free db_group_rule,it was saved in scanner->compile_hash
// table->cfg_num++; if(table->table_type==TABLE_TYPE_GROUP)
{
table->cfg_num++;
}
} }
} }
@@ -2543,6 +2549,7 @@ void plugin_table_callback(struct _Maat_table_info_t* table,const char* table_li
struct _plugin_table_info* p_table_cb=table->cb_info; struct _plugin_table_info* p_table_cb=table->cb_info;
char *p=NULL; char *p=NULL;
pthread_mutex_lock(&(p_table_cb->plugin_mutex)); pthread_mutex_lock(&(p_table_cb->plugin_mutex));
p_table_cb->acc_line_num++;
if(p_table_cb->cb_plug_cnt>0) if(p_table_cb->cb_plug_cnt>0)
{ {
for(i=0;i<p_table_cb->cb_plug_cnt;i++) for(i=0;i<p_table_cb->cb_plug_cnt;i++)
@@ -2555,8 +2562,8 @@ void plugin_table_callback(struct _Maat_table_info_t* table,const char* table_li
p=(char*)calloc(len,1); p=(char*)calloc(len,1);
memcpy(p,table_line,len); memcpy(p,table_line,len);
p_table_cb->cache_size+=len; p_table_cb->cache_size+=len;
dynamic_array_write(p_table_cb->cache_lines,p_table_cb->line_num,p); dynamic_array_write(p_table_cb->cache_lines,p_table_cb->cache_line_num,p);
p_table_cb->line_num++; p_table_cb->cache_line_num++;
} }
pthread_mutex_unlock(&(p_table_cb->plugin_mutex)); pthread_mutex_unlock(&(p_table_cb->plugin_mutex));
} }
@@ -2825,12 +2832,14 @@ void *thread_rule_monitor(void *arg)
if(feather->scanner!=NULL) if(feather->scanner!=NULL)
{ {
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
if(expr_wait_q_cnt>0&&time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms*1000) feather->postpone_q_size=expr_wait_q_cnt;
if(expr_wait_q_cnt>0&&time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms/1000)
{ {
do_scanner_update(feather->scanner do_scanner_update(feather->scanner
,feather->garbage_q ,feather->garbage_q
,feather->scan_thread_num ,feather->scan_thread_num
,feather->logger); ,feather->logger);
feather->postpone_q_size=0;
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
"Actual udpate config version %u %d entries load to rulescan after postpone.", "Actual udpate config version %u %d entries load to rulescan after postpone.",
feather->scanner->version,feather->scanner->cfg_num); feather->scanner->version,feather->scanner->cfg_num);
@@ -2861,7 +2870,7 @@ void *thread_rule_monitor(void *arg)
if(feather->p_table_info[i]->table_type==TABLE_TYPE_PLUGIN) if(feather->p_table_info[i]->table_type==TABLE_TYPE_PLUGIN)
{ {
d_array=feather->p_table_info[i]->cb_info->cache_lines; d_array=feather->p_table_info[i]->cb_info->cache_lines;
for(j=0;j<feather->p_table_info[i]->cb_info->line_num;j++) for(j=0;j<feather->p_table_info[i]->cb_info->cache_line_num;j++)
{ {
lines=(char*)dynamic_array_read(d_array, j); lines=(char*)dynamic_array_read(d_array, j);
free(lines); free(lines);

5
src/entry/Maat_rule_internal.h
View File

@@ -215,7 +215,8 @@ struct _plugin_table_info
int cb_plug_cnt; int cb_plug_cnt;
struct _callback_plugin cb_plug[MAX_PLUGING_NUM]; struct _callback_plugin cb_plug[MAX_PLUGING_NUM];
dynamic_array_t *cache_lines; dynamic_array_t *cache_lines;
int line_num; int cache_line_num;
int acc_line_num;
int update_type; int update_type;
int is_called_started; int is_called_started;
long cache_size; long cache_size;
@@ -368,6 +369,8 @@ struct _Maat_feather_t
long long iconv_err_cnt;//sum of the same name variable in each table long long iconv_err_cnt;//sum of the same name variable in each table
long long scan_err_cnt; long long scan_err_cnt;
long long zombie_rs_stream; long long zombie_rs_stream;
long long postpone_q_size;
long long compile_rule_num;
}; };
struct _maat_garbage_t struct _maat_garbage_t
{ {

157
src/entry/Maat_stat.cpp
View File

@@ -6,6 +6,11 @@ enum MAAT_FS_STATUS{
STATUS_VERSION=0, STATUS_VERSION=0,
STATUS_THRED_NUM, STATUS_THRED_NUM,
STATUS_TABLE_NUM, STATUS_TABLE_NUM,
STATUS_PLUGIN_CACHE_NUM,
STATUS_PLUGIN_ACC_NUM,
STATUS_GROUP_RULE_NUM,
STATUS_COMPILE_RULE_NUM,
STATUS_POSTPONE_QSIZE,
STATUS_OUTER_MID_NUM, STATUS_OUTER_MID_NUM,
STATUS_INNER_MID_NUM, STATUS_INNER_MID_NUM,
STATUS_GARBAGE_QSIZE, STATUS_GARBAGE_QSIZE,
@@ -45,9 +50,19 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version"); feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version");
feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"active_thread"); feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"active_thread");
feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num"); feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num");
feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_cache");
feather->fs_status_id[STATUS_PLUGIN_ACC_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_acc");
feather->fs_status_id[STATUS_GROUP_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"group_num");
feather->fs_status_id[STATUS_COMPILE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"compile_num");
feather->fs_status_id[STATUS_POSTPONE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"postpone_num");
feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num");
feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid"); feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid");
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid"); feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num");
feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes"); feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes");
feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_times"); feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_times");
feather->fs_status_id[STATUS_UPDATE_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"update_error"); feather->fs_status_id[STATUS_UPDATE_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"update_error");
@@ -140,6 +155,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0; long long outer_mid_cnt=0,inner_mid_cnt=0;
long long total_update_error=0,total_iconv_error=0; long long total_update_error=0,total_iconv_error=0;
long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0;
int i=0; int i=0;
time_t now; time_t now;
struct _Maat_table_info_t* p_table=NULL; struct _Maat_table_info_t* p_table=NULL;
@@ -165,71 +181,83 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_scan_cpu_time=0; table_scan_cpu_time=0;
table_hit_cnt=0; table_hit_cnt=0;
p_table=feather->p_table_info[i]; p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN if(p_table==NULL)
||p_table->table_type==TABLE_TYPE_GROUP
||p_table->table_type==TABLE_TYPE_COMPILE)
{ {
continue; continue;
} }
FS_operate(feather->stat_handle, switch(p_table->table_type)
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET,
p_table->cfg_num);
total_cfg_num+=p_table->cfg_num;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
p_table->regex_rule_cnt);
total_regex_num+= p_table->regex_rule_cnt;
table_stream_num=aligment_int64_array_sum(p_table->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
table_stream_num);
total_stream_cnt+= table_stream_num;
table_scan_cnt=aligment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_SET,
table_scan_cnt);
total_scan_cnt+=table_scan_cnt;
table_input_bytes=aligment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_SET,
table_input_bytes);
total_input_bytes+=table_input_bytes;
if(feather->perf_on==1)
{ {
table_scan_cpu_time=aligment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num); case TABLE_TYPE_PLUGIN:
table_scan_cpu_time/=1000; plugin_cache_num+=p_table->cb_info->cache_line_num;
FS_operate(feather->stat_handle, plugin_acc_num+=p_table->cb_info->acc_line_num;
p_table->stat_line_id, break;
feather->fs_column_id[COLUMN_TABLE_CPU_TIME], case TABLE_TYPE_GROUP:
FS_OP_SET, group_rule_num+=p_table->cfg_num;
table_scan_cpu_time); break;
total_cpu_time+=table_scan_cpu_time; case TABLE_TYPE_COMPILE:
} compile_rule_num+=p_table->cfg_num;
break;
table_hit_cnt=aligment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num); default:
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT], feather->fs_column_id[COLUMN_TABLE_RULE_NUM],
FS_OP_SET, FS_OP_SET,
table_hit_cnt); p_table->cfg_num);
total_update_error+=p_table->udpate_err_cnt; total_cfg_num+=p_table->cfg_num;
total_iconv_error+=p_table->iconv_err_cnt;
//total hit count stat in region_compile FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM],
FS_OP_SET,
p_table->regex_rule_cnt);
total_regex_num+= p_table->regex_rule_cnt;
table_stream_num=aligment_int64_array_sum(p_table->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
table_stream_num);
total_stream_cnt+= table_stream_num;
table_scan_cnt=aligment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_SET,
table_scan_cnt);
total_scan_cnt+=table_scan_cnt;
table_input_bytes=aligment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_SET,
table_input_bytes);
total_input_bytes+=table_input_bytes;
if(feather->perf_on==1)
{
table_scan_cpu_time=aligment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_SET,
table_scan_cpu_time);
total_cpu_time+=table_scan_cpu_time;
}
table_hit_cnt=aligment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_SET,
table_hit_cnt);
total_update_error+=p_table->udpate_err_cnt;
total_iconv_error+=p_table->iconv_err_cnt;
//total hit count stat in region_compile
break;
}
} }
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
@@ -282,6 +310,13 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_SCAN_ERR_CNT], 0,FS_OP_SET,feather->scan_err_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_SCAN_ERR_CNT], 0,FS_OP_SET,feather->scan_err_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM], 0,FS_OP_SET,feather->zombie_rs_stream); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM], 0,FS_OP_SET,feather->zombie_rs_stream);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM], 0,FS_OP_SET,plugin_cache_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_ACC_NUM], 0,FS_OP_SET,plugin_acc_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_RULE_NUM], 0,FS_OP_SET,group_rule_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_COMPILE_RULE_NUM], 0,FS_OP_SET,compile_rule_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_POSTPONE_QSIZE], 0,FS_OP_SET,feather->postpone_q_size);
FS_passive_output(feather->stat_handle); FS_passive_output(feather->stat_handle);
return; return;
} }