From 220b3ec04c623108693bcf8530fa7bb4b320a456 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 15 Dec 2016 12:05:48 +0800 Subject: [PATCH] =?UTF-8?q?1=E3=80=81=E4=BF=AE=E5=A4=8Dpostpone=E6=9C=BA?= =?UTF-8?q?=E5=88=B6=E4=B8=AD=E6=9C=80=E5=B0=8F=E6=9B=B4=E6=96=B0=E9=97=B4?= =?UTF-8?q?=E9=9A=94=E6=97=B6=E9=97=B4=E5=88=A4=E6=96=AD=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E7=9A=84bug=EF=BC=9B2=E3=80=81maat=5Fstat=E4=B8=AD=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0compile=E3=80=81group=E3=80=81plugin=E3=80=81postpone?= =?UTF-8?q?=E7=9A=84=E7=BB=9F=E8=AE=A1=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 4 +- src/entry/Maat_rule.cpp | 25 ++++-- src/entry/Maat_rule_internal.h | 5 +- src/entry/Maat_stat.cpp | 157 ++++++++++++++++++++------------- 4 files changed, 119 insertions(+), 72 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 8fe33f5..a2da470 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -704,13 +704,13 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id, p_table->cb_info->cb_plug[idx].update=update; p_table->cb_info->cb_plug[idx].finish=finish; p_table->cb_info->cb_plug[idx].u_para=u_para; - if(p_table->cb_info->line_num>0) + if(p_table->cb_info->cache_line_num>0) { if(start!=NULL) { start(MAAT_RULE_UPDATE_TYPE_FULL,u_para); } - for(i=0;icb_info->line_num;i++) + for(i=0;icb_info->cache_line_num;i++) { lines=(const char*)dynamic_array_read(p_table->cb_info->cache_lines,i); if(lines==NULL) diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index cc0dc0c..166b601 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -27,7 +27,7 @@ #include "mesa_fuzzy.h" #include "great_index_engine.h" -int MAAT_FRAME_VERSION_1_8_20161123=1; +int MAAT_FRAME_VERSION_1_8_20161215=1; const char *maat_module="MAAT Frame"; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", @@ -1922,8 +1922,11 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s if(db_group_rule.is_valid==FALSE) { del_group_rule(table, &db_group_rule,scanner,logger); - //leave no trace for virtual group rule -// table->cfg_num--; + //leave no trace when compatible_group_update calling + if(table->table_type==TABLE_TYPE_GROUP) + { + table->cfg_num--; + } } else { @@ -1939,7 +1942,10 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s else { //no need to free db_group_rule,it was saved in scanner->compile_hash -// table->cfg_num++; + if(table->table_type==TABLE_TYPE_GROUP) + { + table->cfg_num++; + } } } @@ -2543,6 +2549,7 @@ void plugin_table_callback(struct _Maat_table_info_t* table,const char* table_li struct _plugin_table_info* p_table_cb=table->cb_info; char *p=NULL; pthread_mutex_lock(&(p_table_cb->plugin_mutex)); + p_table_cb->acc_line_num++; if(p_table_cb->cb_plug_cnt>0) { for(i=0;icb_plug_cnt;i++) @@ -2555,8 +2562,8 @@ void plugin_table_callback(struct _Maat_table_info_t* table,const char* table_li p=(char*)calloc(len,1); memcpy(p,table_line,len); p_table_cb->cache_size+=len; - dynamic_array_write(p_table_cb->cache_lines,p_table_cb->line_num,p); - p_table_cb->line_num++; + dynamic_array_write(p_table_cb->cache_lines,p_table_cb->cache_line_num,p); + p_table_cb->cache_line_num++; } pthread_mutex_unlock(&(p_table_cb->plugin_mutex)); } @@ -2825,12 +2832,14 @@ void *thread_rule_monitor(void *arg) if(feather->scanner!=NULL) { expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); - if(expr_wait_q_cnt>0&&time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms*1000) + feather->postpone_q_size=expr_wait_q_cnt; + if(expr_wait_q_cnt>0&&time(NULL)-feather->scanner->last_update_time>feather->effect_interval_ms/1000) { do_scanner_update(feather->scanner ,feather->garbage_q ,feather->scan_thread_num ,feather->logger); + feather->postpone_q_size=0; MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, "Actual udpate config version %u %d entries load to rulescan after postpone.", feather->scanner->version,feather->scanner->cfg_num); @@ -2861,7 +2870,7 @@ void *thread_rule_monitor(void *arg) if(feather->p_table_info[i]->table_type==TABLE_TYPE_PLUGIN) { d_array=feather->p_table_info[i]->cb_info->cache_lines; - for(j=0;jp_table_info[i]->cb_info->line_num;j++) + for(j=0;jp_table_info[i]->cb_info->cache_line_num;j++) { lines=(char*)dynamic_array_read(d_array, j); free(lines); diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index 309df38..1809409 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -215,7 +215,8 @@ struct _plugin_table_info int cb_plug_cnt; struct _callback_plugin cb_plug[MAX_PLUGING_NUM]; dynamic_array_t *cache_lines; - int line_num; + int cache_line_num; + int acc_line_num; int update_type; int is_called_started; long cache_size; @@ -368,6 +369,8 @@ struct _Maat_feather_t long long iconv_err_cnt;//sum of the same name variable in each table long long scan_err_cnt; long long zombie_rs_stream; + long long postpone_q_size; + long long compile_rule_num; }; struct _maat_garbage_t { diff --git a/src/entry/Maat_stat.cpp b/src/entry/Maat_stat.cpp index 6a9b0e8..ed528d7 100644 --- a/src/entry/Maat_stat.cpp +++ b/src/entry/Maat_stat.cpp @@ -6,6 +6,11 @@ enum MAAT_FS_STATUS{ STATUS_VERSION=0, STATUS_THRED_NUM, STATUS_TABLE_NUM, + STATUS_PLUGIN_CACHE_NUM, + STATUS_PLUGIN_ACC_NUM, + STATUS_GROUP_RULE_NUM, + STATUS_COMPILE_RULE_NUM, + STATUS_POSTPONE_QSIZE, STATUS_OUTER_MID_NUM, STATUS_INNER_MID_NUM, STATUS_GARBAGE_QSIZE, @@ -45,9 +50,19 @@ void maat_stat_init(struct _Maat_feather_t* feather) feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version"); feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"active_thread"); feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num"); + + feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_cache"); + feather->fs_status_id[STATUS_PLUGIN_ACC_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_acc"); + + feather->fs_status_id[STATUS_GROUP_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"group_num"); + feather->fs_status_id[STATUS_COMPILE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"compile_num"); + + feather->fs_status_id[STATUS_POSTPONE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"postpone_num"); + feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num"); + feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid"); feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid"); - feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num"); + feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes"); feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_times"); feather->fs_status_id[STATUS_UPDATE_ERR_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"update_error"); @@ -140,6 +155,7 @@ void maat_stat_output(struct _Maat_feather_t* feather) long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0; long long outer_mid_cnt=0,inner_mid_cnt=0; long long total_update_error=0,total_iconv_error=0; + long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0; int i=0; time_t now; struct _Maat_table_info_t* p_table=NULL; @@ -165,71 +181,83 @@ void maat_stat_output(struct _Maat_feather_t* feather) table_scan_cpu_time=0; table_hit_cnt=0; p_table=feather->p_table_info[i]; - if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN - ||p_table->table_type==TABLE_TYPE_GROUP - ||p_table->table_type==TABLE_TYPE_COMPILE) + if(p_table==NULL) { continue; } - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_RULE_NUM], - FS_OP_SET, - p_table->cfg_num); - total_cfg_num+=p_table->cfg_num; - - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], - FS_OP_SET, - p_table->regex_rule_cnt); - total_regex_num+= p_table->regex_rule_cnt; - - table_stream_num=aligment_int64_array_sum(p_table->stream_num,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], - FS_OP_SET, - table_stream_num); - total_stream_cnt+= table_stream_num; - - table_scan_cnt=aligment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], - FS_OP_SET, - table_scan_cnt); - total_scan_cnt+=table_scan_cnt; - - table_input_bytes=aligment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], - FS_OP_SET, - table_input_bytes); - total_input_bytes+=table_input_bytes; - if(feather->perf_on==1) + switch(p_table->table_type) { - table_scan_cpu_time=aligment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num); - table_scan_cpu_time/=1000; - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_CPU_TIME], - FS_OP_SET, - table_scan_cpu_time); - total_cpu_time+=table_scan_cpu_time; - } - - table_hit_cnt=aligment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num); - FS_operate(feather->stat_handle, - p_table->stat_line_id, - feather->fs_column_id[COLUMN_TABLE_HIT_CNT], - FS_OP_SET, - table_hit_cnt); - total_update_error+=p_table->udpate_err_cnt; - total_iconv_error+=p_table->iconv_err_cnt; - //total hit count stat in region_compile + case TABLE_TYPE_PLUGIN: + plugin_cache_num+=p_table->cb_info->cache_line_num; + plugin_acc_num+=p_table->cb_info->acc_line_num; + break; + case TABLE_TYPE_GROUP: + group_rule_num+=p_table->cfg_num; + break; + case TABLE_TYPE_COMPILE: + compile_rule_num+=p_table->cfg_num; + break; + default: + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_RULE_NUM], + FS_OP_SET, + p_table->cfg_num); + total_cfg_num+=p_table->cfg_num; + + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_REGEX_NUM], + FS_OP_SET, + p_table->regex_rule_cnt); + total_regex_num+= p_table->regex_rule_cnt; + table_stream_num=aligment_int64_array_sum(p_table->stream_num,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], + FS_OP_SET, + table_stream_num); + total_stream_cnt+= table_stream_num; + + table_scan_cnt=aligment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], + FS_OP_SET, + table_scan_cnt); + total_scan_cnt+=table_scan_cnt; + + table_input_bytes=aligment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], + FS_OP_SET, + table_input_bytes); + total_input_bytes+=table_input_bytes; + if(feather->perf_on==1) + { + table_scan_cpu_time=aligment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num); + table_scan_cpu_time/=1000; + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_CPU_TIME], + FS_OP_SET, + table_scan_cpu_time); + total_cpu_time+=table_scan_cpu_time; + } + + table_hit_cnt=aligment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num); + FS_operate(feather->stat_handle, + p_table->stat_line_id, + feather->fs_column_id[COLUMN_TABLE_HIT_CNT], + FS_OP_SET, + table_hit_cnt); + total_update_error+=p_table->udpate_err_cnt; + total_iconv_error+=p_table->iconv_err_cnt; + //total hit count stat in region_compile + break; + } } FS_operate(feather->stat_handle, @@ -282,6 +310,13 @@ void maat_stat_output(struct _Maat_feather_t* feather) FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_SCAN_ERR_CNT], 0,FS_OP_SET,feather->scan_err_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM], 0,FS_OP_SET,feather->zombie_rs_stream); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM], 0,FS_OP_SET,plugin_cache_num); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_ACC_NUM], 0,FS_OP_SET,plugin_acc_num); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_RULE_NUM], 0,FS_OP_SET,group_rule_num); + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_COMPILE_RULE_NUM], 0,FS_OP_SET,compile_rule_num); + + FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_POSTPONE_QSIZE], 0,FS_OP_SET,feather->postpone_q_size); + FS_passive_output(feather->stat_handle); return; }