gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

解决性能统计时stream_num等出现的负值的问题，方法是使用aligment_int64_array替代线程不安全的long long。

Browse Source
This commit is contained in:
zhengchao
2016-05-09 22:02:27 +08:00
parent 8e1a2edd94
commit dbc7fcc3ff
5 changed files with 149 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/entry/Maat_api.cpp
View File

@@ -98,7 +98,7 @@ int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsi
} }
return k; return k;
} }
int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size) int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
{ {
int scan_ret=0,result_cnt=0; int scan_ret=0,result_cnt=0;
@@ -138,7 +138,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,voi
} }
else else
{ {
scan_ret=boolexpr_match(bool_matcher,_mid->thread_num, scan_ret=boolexpr_match(bool_matcher,thread_num,
_mid->hitted_group_id,_mid->hit_group_cnt, _mid->hitted_group_id,_mid->hit_group_cnt,
(void **)array_mi_rule, MAX_SCANNER_HIT_NUM); (void **)array_mi_rule, MAX_SCANNER_HIT_NUM);
} }
@@ -169,7 +169,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,voi
} }
if(result_cnt>0) if(result_cnt>0)
{ {
feather->hit_cnt++; aligment_int64_array_add(feather->hit_cnt,thread_num,1);
} }
return result_cnt; return result_cnt;
} }
@@ -344,24 +344,24 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
} }
return j; return j;
} }
struct _INNER_scan_status_t* _make_inner_status(int thread_num) struct _INNER_scan_status_t* _make_inner_status(void)
{ {
struct _INNER_scan_status_t* inner_mid=NULL; struct _INNER_scan_status_t* inner_mid=NULL;
inner_mid=(struct _INNER_scan_status_t*)calloc(sizeof(struct _INNER_scan_status_t),1); inner_mid=(struct _INNER_scan_status_t*)calloc(sizeof(struct _INNER_scan_status_t),1);
inner_mid->thread_num=thread_num;
inner_mid->cur_hit_cnt=0; inner_mid->cur_hit_cnt=0;
inner_mid->hit_group_cnt=0; inner_mid->hit_group_cnt=0;
inner_mid->hit_group_size=4; inner_mid->hit_group_size=4;
inner_mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*inner_mid->hit_group_size); inner_mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*inner_mid->hit_group_size);
return inner_mid; return inner_mid;
} }
struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather) struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather,int thread_num)
{ {
struct _OUTER_scan_status_t* outer_mid=NULL; struct _OUTER_scan_status_t* outer_mid=NULL;
outer_mid=(struct _OUTER_scan_status_t*)calloc(sizeof(struct _OUTER_scan_status_t),1); outer_mid=(struct _OUTER_scan_status_t*)calloc(sizeof(struct _OUTER_scan_status_t),1);
outer_mid->feather=feather; outer_mid->feather=feather;
outer_mid->district_id=-1; outer_mid->district_id=-1;
feather->outer_mid_cnt++; outer_mid->thread_num=(unsigned short)thread_num;
aligment_int64_array_add(feather->outer_mid_cnt, thread_num,1);
return outer_mid; return outer_mid;
} }
struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* feather,int thread_num,int is_hit_region) struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* feather,int thread_num,int is_hit_region)
@@ -375,13 +375,13 @@ struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* fe
{ {
if(_mid==NULL) if(_mid==NULL)
{ {
_mid=_make_outer_status(feather); _mid=_make_outer_status(feather,thread_num);
*raw_mid=_mid; *raw_mid=_mid;
} }
if(_mid->inner==NULL) if(_mid->inner==NULL)
{ {
_mid->inner=_make_inner_status(thread_num); _mid->inner=_make_inner_status();
feather->inner_mid_cnt++; aligment_int64_array_add(feather->inner_mid_cnt,thread_num,1);
} }
} }
return _mid; return _mid;
@@ -415,7 +415,7 @@ int detain_last_data(char* buff,int buff_size,int detained_len,const char* data,
Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger) Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void* logger)
{ {
_Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1); _Maat_feather_t* feather=(_Maat_feather_t*)calloc(sizeof(struct _Maat_feather_t),1);
feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,logger); feather->table_cnt=read_table_info(feather->p_table_info, MAX_TABLE_NUM,table_info_path,max_thread_num,logger);
feather->map_tablename2id=map_create(); feather->map_tablename2id=map_create();
int i=0; int i=0;
for(i=0;i<MAX_TABLE_NUM;i++) for(i=0;i<MAX_TABLE_NUM;i++)
@@ -436,6 +436,9 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->scan_interval_ms=1*1000; feather->scan_interval_ms=1*1000;
feather->rule_scan_type=2; feather->rule_scan_type=2;
feather->thread_call_cnt=aligment_int64_array_alloc(max_thread_num); feather->thread_call_cnt=aligment_int64_array_alloc(max_thread_num);
feather->outer_mid_cnt=aligment_int64_array_alloc(max_thread_num);
feather->inner_mid_cnt=aligment_int64_array_alloc(max_thread_num);
feather->hit_cnt=aligment_int64_array_alloc(max_thread_num);
return feather; return feather;
} }
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size) int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size)
@@ -788,12 +791,13 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
p_table->hit_cnt++; aligment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid,_feather,thread_num, 1); _mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
hit_region_cnt, hit_region_cnt,
result,compile_result,rule_num); result,compile_result,rule_num,
thread_num);
if(hit_detail!=NULL&&_feather->rule_scan_type!=0) if(hit_detail!=NULL&&_feather->rule_scan_type!=0)
{ {
*detail_ret=fill_region_hit_detail(data,_mid->inner, *detail_ret=fill_region_hit_detail(data,_mid->inner,
@@ -806,11 +810,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
if(_feather->perf_on==1) if(_feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,data_len,&start, &end); maat_stat_table(p_table,data_len,&start, &end,thread_num);
} }
else else
{ {
maat_stat_table(p_table,data_len,NULL, NULL); maat_stat_table(p_table,data_len,NULL, NULL,thread_num);
} }
if(compile_ret==0&&hit_region_cnt>0) if(compile_ret==0&&hit_region_cnt>0)
@@ -879,23 +883,24 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
p_table->hit_cnt++; aligment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
region_ret, region_ret,
result,compile_result,rule_num); result,compile_result,rule_num,
thread_num);
} }
DEC_SCANNER_REF(my_scanner,thread_num); DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1) if(_feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end); maat_stat_table(p_table,0,&start, &end,thread_num);
} }
else else
{ {
maat_stat_table(p_table,0,NULL, NULL); maat_stat_table(p_table,0,NULL, NULL,thread_num);
} }
if(compile_ret==0&&region_ret>0) if(compile_ret==0&&region_ret>0)
{ {
@@ -985,22 +990,23 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
p_table->hit_cnt++; aligment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
region_ret, region_ret,
result,compile_result,rule_num); result,compile_result,rule_num,
thread_num);
} }
DEC_SCANNER_REF(my_scanner,thread_num); DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1) if(_feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end); maat_stat_table(p_table,0,&start, &end,thread_num);
} }
else else
{ {
maat_stat_table(p_table,0,NULL, NULL); maat_stat_table(p_table,0,NULL, NULL,thread_num);
} }
if(compile_ret==0&&region_ret>0) if(compile_ret==0&&region_ret>0)
{ {
@@ -1062,7 +1068,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{ {
sp->do_regex=1; sp->do_regex=1;
} }
p_table->stream_num++; aligment_int64_array_add(p_table->stream_num,thread_num,1);
sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num);
return sp; return sp;
} }
@@ -1187,12 +1193,13 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
p_table->hit_cnt++; aligment_int64_array_add(p_table->hit_cnt, sp->thread_num,1);
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag), region_result,sizeof(scan_result_t),offsetof(scan_result_t, tag),
hit_region_cnt, hit_region_cnt,
result,compile_result,rule_num); result,compile_result,rule_num,
sp->thread_num);
if(hit_detail!=NULL&&sp->feather->rule_scan_type!=0) if(hit_detail!=NULL&&sp->feather->rule_scan_type!=0)
{ {
if(sp->scan_buff!=NULL) if(sp->scan_buff!=NULL)
@@ -1219,11 +1226,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
if(sp->feather->perf_on==1) if(sp->feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end); maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num);
} }
else else
{ {
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL); maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num);
} }
if(compile_ret==0&&hit_region_cnt>0) if(compile_ret==0&&hit_region_cnt>0)
{ {
@@ -1249,7 +1256,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
p_table->stream_num--; aligment_int64_array_add(p_table->stream_num,sp->thread_num,-1);
if(scanner!=NULL) if(scanner!=NULL)
{ {
if(sp->version==sp->feather->maat_version) if(sp->version==sp->feather->maat_version)
@@ -1307,8 +1314,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sp->total_len=total_len; sp->total_len=total_len;
sp->fuzzy_hash_handle=tmp_fuzzy_handle; sp->fuzzy_hash_handle=tmp_fuzzy_handle;
pthread_mutex_init(&(sp->fuzzy_mutex),NULL); pthread_mutex_init(&(sp->fuzzy_mutex),NULL);
p_table->stream_num++; aligment_int64_array_add(p_table->stream_num,thread_num,1);
return sp; return sp;
} }
@@ -1395,17 +1401,18 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag), query_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag),
hit_region_cnt, hit_region_cnt,
result,compile_result,rule_num); result,compile_result,rule_num,
sp->thread_num);
} }
if(sp->feather->perf_on==1) if(sp->feather->perf_on==1)
{ {
clock_gettime(CLOCK_MONOTONIC,&end); clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end); maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end,sp->thread_num);
} }
else else
{ {
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL); maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL,sp->thread_num);
} }
if(compile_ret==0&&hit_region_cnt>0) if(compile_ret==0&&hit_region_cnt>0)
{ {
@@ -1418,7 +1425,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
p_table->stream_num--; aligment_int64_array_add(p_table->stream_num,sp->thread_num,-1);
if(scanner!=NULL) if(scanner!=NULL)
{ {
if(sp->version==sp->feather->maat_version) if(sp->version==sp->feather->maat_version)
@@ -1443,7 +1450,7 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA
_mid=grab_mid(mid,_feather, 0, 0); _mid=grab_mid(mid,_feather, 0, 0);
if(_mid==NULL) if(_mid==NULL)
{ {
_mid=_make_outer_status(_feather); _mid=_make_outer_status(_feather,0);
*mid=_mid; *mid=_mid;
} }
switch(type) switch(type)
@@ -1470,12 +1477,12 @@ void Maat_clean_status(scan_status_t* mid)
return; return;
} }
_mid=(struct _OUTER_scan_status_t*)(*mid); _mid=(struct _OUTER_scan_status_t*)(*mid);
_mid->feather->outer_mid_cnt--; aligment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL) if(_mid->inner!=NULL)
{ {
free(_mid->inner->hitted_group_id); free(_mid->inner->hitted_group_id);
free(_mid->inner); free(_mid->inner);
_mid->feather->inner_mid_cnt--; aligment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1);
} }
_mid->feather=NULL; _mid->feather=NULL;
free(_mid); free(_mid);

39
src/entry/Maat_rule.cpp
View File

@@ -26,7 +26,7 @@
#include "mesa_fuzzy.h" #include "mesa_fuzzy.h"
#include "great_index_engine.h" #include "great_index_engine.h"
int MAAT_FRAME_VERSION_1_7_20160425=1; int MAAT_FRAME_VERSION_1_7_20160509=1;
const char *maat_module="MAAT Frame"; const char *maat_module="MAAT Frame";
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
@@ -377,7 +377,29 @@ int HASH_delete_by_id(MESA_htable_handle hash,int id)
ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL); ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL);
return ret; return ret;
} }
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,void* logger) _Maat_table_info_t* create_table_info(int max_thread_num)
{
struct _Maat_table_info_t*p=NULL;
p=(struct _Maat_table_info_t*)calloc(sizeof(struct _Maat_table_info_t),1);
p->scan_cnt=aligment_int64_array_alloc(max_thread_num);
p->scan_cpu_time=aligment_int64_array_alloc(max_thread_num);
p->input_bytes=aligment_int64_array_alloc(max_thread_num);
p->stream_num=aligment_int64_array_alloc(max_thread_num);
p->hit_cnt=aligment_int64_array_alloc(max_thread_num);
p->cross_cache_size=0;
return p;
}
void destroy_table_info(struct _Maat_table_info_t*p)
{
aligment_int64_array_free(p->scan_cnt);
aligment_int64_array_free(p->scan_cpu_time);
aligment_int64_array_free(p->input_bytes);
aligment_int64_array_free(p->stream_num);
aligment_int64_array_free(p->hit_cnt);
free(p);
return;
}
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
{ {
FILE*fp=NULL; FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE]; char line[MAX_TABLE_LINE_SIZE];
@@ -434,8 +456,8 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
{ {
continue; continue;
} }
p=(struct _Maat_table_info_t*)calloc(sizeof(struct _Maat_table_info_t),1); p=create_table_info(max_thread_num);
p->cross_cache_size=0;
sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d",&(p->table_id) sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d",&(p->table_id)
,p->table_name ,p->table_name
,table_type ,table_type
@@ -506,7 +528,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
table_cnt++; table_cnt++;
continue; continue;
error_jump: error_jump:
free(p); destroy_table_info(p);
p=NULL; p=NULL;
} }
fclose(fp); fclose(fp);
@@ -2613,10 +2635,13 @@ void *thread_rule_monitor(void *arg)
free(lines); free(lines);
} }
} }
free(feather->p_table_info[i]); destroy_table_info(feather->p_table_info[i]);
feather->p_table_info[i]=NULL; feather->p_table_info[i]=NULL;
} }
free(feather->thread_call_cnt); aligment_int64_array_free(feather->thread_call_cnt);
aligment_int64_array_free(feather->inner_mid_cnt);
aligment_int64_array_free(feather->outer_mid_cnt);
aligment_int64_array_free(feather->hit_cnt);
free(feather); free(feather);
return NULL; return NULL;
} }

29
src/entry/Maat_rule_internal.h
View File

@@ -12,7 +12,7 @@
#include "mesa_fuzzy.h" #include "mesa_fuzzy.h"
#include "great_index_engine.h" #include "great_index_engine.h"
#include "aligment_int64.h"
#include <pthread.h> #include <pthread.h>
#include <iconv.h> #include <iconv.h>
@@ -31,7 +31,6 @@ typedef int atomic_t;
#else #else
#include <alsa/iatomic.h> #include <alsa/iatomic.h>
#endif #endif
#define CPU_CACHE_ALIGMENT 64
#define TRUE 1 #define TRUE 1
#define FALSE 0 #define FALSE 0
@@ -241,16 +240,15 @@ struct _Maat_table_info_t
struct _plugin_table_info *cb_info; struct _plugin_table_info *cb_info;
//for stat>>>>>>>> //for stat>>>>>>>>
int stat_line_id; int stat_line_id;
long long scan_cnt; mcore_long_t scan_cnt;
long long scan_cpu_time; //nano mcore_long_t scan_cpu_time; //nano
long long input_bytes; mcore_long_t input_bytes;
long long stream_num; mcore_long_t stream_num;
long long hit_cnt; mcore_long_t hit_cnt;
}; };
struct _INNER_scan_status_t struct _INNER_scan_status_t
{ {
int thread_num;
int cur_hit_cnt; int cur_hit_cnt;
int hit_group_cnt; int hit_group_cnt;
int hit_group_size; int hit_group_size;
@@ -260,7 +258,8 @@ struct _INNER_scan_status_t
struct _OUTER_scan_status_t struct _OUTER_scan_status_t
{ {
struct _Maat_feather_t* feather; struct _Maat_feather_t* feather;
int is_set_district; unsigned short thread_num;
unsigned short is_set_district;
int district_id; int district_id;
struct _INNER_scan_status_t* inner; struct _INNER_scan_status_t* inner;
}; };
@@ -345,10 +344,10 @@ struct _Maat_feather_t
int total_stat_id; int total_stat_id;
int fs_status_id[MAX_MAAT_STAT_NUM]; int fs_status_id[MAX_MAAT_STAT_NUM];
int fs_column_id[MAX_MAAT_STAT_NUM]; int fs_column_id[MAX_MAAT_STAT_NUM];
long long outer_mid_cnt; mcore_long_t outer_mid_cnt;
long long inner_mid_cnt; mcore_long_t inner_mid_cnt;
long long hit_cnt; mcore_long_t hit_cnt;
long long *thread_call_cnt;//size indicate by scan_thread_num, mcore_long_t thread_call_cnt;//size indicate by scan_thread_num,
}; };
struct _maat_garbage_t struct _maat_garbage_t
{ {
@@ -367,7 +366,7 @@ struct _maat_garbage_t
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set); void make_group_set(const struct _Maat_compile_rule_t* compile_rule,universal_bool_expr_t* a_set);
int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,void*logger); int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(unsigned int new_version,int update_type,void*u_para); void maat_start_cb(unsigned int new_version,int update_type,void*u_para);
void maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para); void maat_finish_cb(void* u_para);
@@ -383,7 +382,7 @@ inline void ipv6_ntoh(unsigned int *v6_addr)
return; return;
} }
void maat_stat_init(struct _Maat_feather_t* feather); void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end); void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
void maat_stat_output(struct _Maat_feather_t* feather); void maat_stat_output(struct _Maat_feather_t* feather);

75
src/entry/Maat_stat.cpp
View File

@@ -94,37 +94,46 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_start(feather->stat_handle); FS_start(feather->stat_handle);
return; return;
} }
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end) void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
{ {
p_table->scan_cnt++; aligment_int64_array_add(p_table->scan_cnt,thread_num,1);
p_table->input_bytes+=scan_len; aligment_int64_array_add(p_table->input_bytes,thread_num,scan_len);
if(start!=NULL&&end!=NULL) if(start!=NULL&&end!=NULL)
{ {
p_table->scan_cpu_time+=(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec; aligment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
} }
return; return;
} }
void maat_stat_output(struct _Maat_feather_t* feather) void maat_stat_output(struct _Maat_feather_t* feather)
{ {
long value=0; long value=0;
long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0; long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0,total_hit_cnt=0;
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0; long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0;
int i=0; int i=0;
time_t now; time_t now;
struct _Maat_table_info_t* p_table=NULL; struct _Maat_table_info_t* p_table=NULL;
time(&now); time(&now);
active_thread_num=aligment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); active_thread_num=aligment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=aligment_int64_array_cnt(feather->outer_mid_cnt,feather->scan_thread_num);
inner_mid_cnt=aligment_int64_array_cnt(feather->inner_mid_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,feather->outer_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,feather->inner_mid_cnt); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt);
value=MESA_lqueue_get_count(feather->garbage_q); value=MESA_lqueue_get_count(feather->garbage_q);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);
for(i=0;i<MAX_TABLE_NUM;i++) for(i=0;i<MAX_TABLE_NUM;i++)
{ {
table_stream_num=0;
table_scan_cnt=0;
table_input_bytes=0;
table_scan_cpu_time=0;
table_hit_cnt=0;
p_table=feather->p_table_info[i]; p_table=feather->p_table_info[i];
if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
||p_table->table_type==TABLE_TYPE_GROUP ||p_table->table_type==TABLE_TYPE_GROUP
@@ -146,45 +155,49 @@ void maat_stat_output(struct _Maat_feather_t* feather)
p_table->regex_rule_cnt); p_table->regex_rule_cnt);
total_regex_num+= p_table->regex_rule_cnt; total_regex_num+= p_table->regex_rule_cnt;
table_stream_num=aligment_int64_array_sum(p_table->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET, FS_OP_SET,
p_table->stream_num); table_stream_num);
total_stream_cnt+= p_table->stream_num; total_stream_cnt+= table_stream_num;
table_scan_cnt=aligment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_ADD, FS_OP_SET,
p_table->scan_cnt); table_scan_cnt);
total_scan_cnt+=p_table->scan_cnt; total_scan_cnt+=table_scan_cnt;
p_table->scan_cnt=0;
table_input_bytes=aligment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_ADD, FS_OP_SET,
p_table->input_bytes); table_input_bytes);
total_input_bytes+=p_table->input_bytes; total_input_bytes+=table_input_bytes;
p_table->input_bytes=0;
if(feather->perf_on==1) if(feather->perf_on==1)
{ {
table_scan_cpu_time=aligment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME], feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_ADD, FS_OP_SET,
p_table->scan_cpu_time/1000); table_scan_cpu_time);
total_cpu_time+=p_table->scan_cpu_time; total_cpu_time+=table_scan_cpu_time;
p_table->scan_cpu_time=0;
} }
table_hit_cnt=aligment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT], feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_ADD, FS_OP_SET,
p_table->hit_cnt); table_hit_cnt);
//total hit count stat in region_compile //total hit count stat in region_compile
p_table->hit_cnt=0;
} }
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
feather->total_stat_id, feather->total_stat_id,
@@ -204,27 +217,27 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
feather->total_stat_id, feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_ADD, FS_OP_SET,
total_scan_cnt); total_scan_cnt);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
feather->total_stat_id, feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_ADD, FS_OP_SET,
total_input_bytes); total_input_bytes);
if(feather->perf_on==1) if(feather->perf_on==1)
{ {
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
feather->total_stat_id, feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME], feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_ADD, FS_OP_SET,
total_cpu_time/1000); total_cpu_time);
} }
total_hit_cnt=aligment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
feather->total_stat_id, feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT], feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_ADD, FS_OP_SET,
feather->hit_cnt); total_hit_cnt);
feather->hit_cnt=0;
FS_passive_output(feather->stat_handle); FS_passive_output(feather->stat_handle);
return; return;
} }

15
src/entry/aligment_int64.h
View File

@@ -2,13 +2,17 @@
#define H_ALIGMENT_INT64_H_INCLUDE #define H_ALIGMENT_INT64_H_INCLUDE
#include <stdlib.h> #include <stdlib.h>
#define CPU_CACHE_ALIGMENT 64
typedef long long* mcore_long_t;
inline long long *aligment_int64_array_alloc(int size) inline long long *aligment_int64_array_alloc(int size)
{ {
long long *ret=NULL; long long *ret=NULL;
ret=(long long*)calloc(CPU_CACHE_ALIGMENT,size); ret=(long long*)calloc(CPU_CACHE_ALIGMENT,size);
return ret; return ret;
} }
inline long long aligment_int64_array_sum(long long * array,int size) inline long long aligment_int64_array_sum(mcore_long_t array,int size)
{ {
long long sum=0; long long sum=0;
int offset=0,i=0; int offset=0,i=0;
@@ -19,13 +23,13 @@ inline long long aligment_int64_array_sum(long long * array,int size)
} }
return sum; return sum;
} }
inline long long aligment_int64_array_add(long long * array,int offset,long long op_val) inline long long aligment_int64_array_add(mcore_long_t array,int offset,long long op_val)
{ {
int idx=(CPU_CACHE_ALIGMENT/sizeof(long long))*offset; int idx=(CPU_CACHE_ALIGMENT/sizeof(long long))*offset;
array[idx]+=op_val; array[idx]+=op_val;
return array[idx]; return array[idx];
} }
inline int aligment_int64_array_cnt(long long * array,int size) inline long long aligment_int64_array_cnt(mcore_long_t array,int size)
{ {
int offset=0,i=0; int offset=0,i=0;
int cnt=0; int cnt=0;
@@ -39,5 +43,10 @@ inline int aligment_int64_array_cnt(long long * array,int size)
} }
return cnt; return cnt;
} }
inline void aligment_int64_array_free(mcore_long_t array)
{
free(array);
}
#endif #endif