gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

根据《gettimeofday() should never be used to measure time》的建议，用clock_gettime替换gettimeofday。增加active_thread的统计。

Browse Source
This commit is contained in:
zhengchao
2016-02-18 14:53:06 +08:00
parent a6ef5b16d1
commit ab8a55678c
6 changed files with 125 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/entry/Maat_api.cpp
View File

@@ -9,6 +9,7 @@
#include "Maat_rule.h"
#include "Maat_rule_internal.h"
#include "dynamic_array.h"
#include "aligment_int64.h"
#include "config_monitor.h"
#include "map_str2int.h"
#include "rulescan.h"
@@ -42,14 +43,13 @@ struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int t
}
inline void INC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
{
int offset=(CPU_CACHE_ALIGMENT/sizeof(int))*thread_num;
scanner->ref_cnt[offset]++;
aligment_int64_array_add(scanner->ref_cnt, thread_num, 1);
return;
}
inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
{
int offset=(CPU_CACHE_ALIGMENT/sizeof(int))*thread_num;
scanner->ref_cnt[offset]--;
aligment_int64_array_add(scanner->ref_cnt, thread_num, -1);
return;
}
@@ -434,6 +434,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->garbage_q=MESA_lqueue_create(0,0);
feather->effect_interval_ms=60*1000;
feather->scan_interval_ms=1*1000;
feather->thread_call_cnt=aligment_int64_array_alloc(max_thread_num);
return feather;
}
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size)
@@ -714,10 +715,10 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];//dynamic array
struct _Maat_table_info_t *p_table=NULL;
struct timeval start,end;
struct timespec start,end;
if(_feather->perf_on==1)
{
gettimeofday(&start,NULL);
clock_gettime(CLOCK_MONOTONIC,&start);
}
_mid=grab_mid(mid,_feather, thread_num, 0);
p_table=acqurie_table(_feather, table_id,TABLE_TYPE_EXPR);
@@ -741,7 +742,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
sub_type=make_sub_type(table_id,charset,0);
}
aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
scan_data_t scan_data;
scan_data.text_data.text=data;
scan_data.text_data.tlen=data_len;
@@ -800,9 +801,14 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
DEC_SCANNER_REF(my_scanner, thread_num);
if(_feather->perf_on==1)
{
gettimeofday(&end,NULL);
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,data_len,&start, &end);
}
else
{
maat_stat_table(p_table,data_len,NULL, NULL);
}
if(compile_ret==0&&hit_region_cnt>0)
{
return -2;
@@ -837,10 +843,10 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
intval_scan_data.int_data=intval;
_Maat_table_info_t* p_table=NULL;
struct timeval start,end;
struct timespec start,end;
if(_feather->perf_on==1)
{
gettimeofday(&start,NULL);
clock_gettime(CLOCK_MONOTONIC,&start);
}
p_table=acqurie_table(_feather,table_id,TABLE_TYPE_INTVAL);
if(p_table==NULL)
@@ -856,6 +862,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
{
return 0;
}
aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
@@ -879,9 +886,13 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1)
{
gettimeofday(&end,NULL);
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end);
}
else
{
maat_stat_table(p_table,0,NULL, NULL);
}
if(compile_ret==0&&region_ret>0)
{
return -2;
@@ -904,10 +915,10 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL;
struct timeval start,end;
struct timespec start,end;
if(_feather->perf_on==1)
{
gettimeofday(&start,NULL);
clock_gettime(CLOCK_MONOTONIC,&start);
}
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_IP);
if(p_table==NULL)
@@ -923,6 +934,8 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{
return 0;
}
aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
ip_scan_data.rule_type=RULETYPE_IPv4;
ip_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
switch(addr->addrtype)
@@ -970,9 +983,13 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
DEC_SCANNER_REF(my_scanner,thread_num);
if(_feather->perf_on==1)
{
gettimeofday(&end,NULL);
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(p_table,0,&start, &end);
}
else
{
maat_stat_table(p_table,0,NULL, NULL);
}
if(compile_ret==0&&region_ret>0)
{
return -2;
@@ -1053,10 +1070,10 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
_compile_result_t compile_result[rule_num];//dynamic array
scan_data_t region_scan_data;
_Maat_table_info_t* p_table=NULL;
struct timeval start,end;
struct timespec start,end;
if(sp->feather->perf_on==1)
{
gettimeofday(&start,NULL);
clock_gettime(CLOCK_MONOTONIC,&start);
}
_mid=grab_mid(mid, sp->feather, sp->thread_num,0);
if(data==NULL||data_len==0)
@@ -1080,6 +1097,8 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
{
return -1;
}
aligment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
region_result=scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num;
*detail_ret=0;
if(sp->do_merge==1)
@@ -1186,9 +1205,13 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
}
if(sp->feather->perf_on==1)
{
gettimeofday(&end,NULL);
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end);
}
else
{
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL);
}
if(compile_ret==0&&hit_region_cnt>0)
{
return -2;
@@ -1309,15 +1332,16 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
char* digest_buff=NULL;
struct _OUTER_scan_status_t* _mid=NULL;
pthread_rwlock_t *GIE_rwlock=&(sp->feather->scanner->digest_rwlock[sp->table_id]);
struct timeval start,end;
struct timespec start,end;
if(sp->feather->perf_on==1)
{
gettimeofday(&start,NULL);
clock_gettime(CLOCK_MONOTONIC,&start);
}
if(sp->acc_scan_len+(unsigned long long)data_len > sp->total_len)
{
return 0;
}
aligment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
pthread_mutex_lock(&(sp->fuzzy_mutex));
sp->acc_scan_len+=fuzzy_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
pthread_mutex_unlock(&(sp->fuzzy_mutex));
@@ -1363,10 +1387,13 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
}
if(sp->feather->perf_on==1)
{
gettimeofday(&end,NULL);
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,&start, &end);
}
else
{
maat_stat_table(sp->feather->p_table_info[sp->table_id],data_len,NULL, NULL);
}
if(compile_ret==0&&hit_region_cnt>0)
{
return -2;

22
src/entry/Maat_rule.cpp
View File

@@ -17,6 +17,7 @@
#include "Maat_rule_internal.h"
#include "json2iris.h"
#include "dynamic_array.h"
#include "aligment_int64.h"
#include "config_monitor.h"
#include "map_str2int.h"
@@ -218,17 +219,7 @@ int cnt_maskbits(struct in6_addr mask)
}
return bits_cnt;
}
int aligment_int_array_sum(int * array,int size)
{
int sum=0,i=0;
int offset=0;
for(i=0;i<size;i++)
{
offset=(CPU_CACHE_ALIGMENT/sizeof(int))*i;
sum+=array[offset];
}
return sum;
}
int lqueue_destroy_cb(void *data, long data_len, void *arg)
{
assert(0);
@@ -714,7 +705,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread
scanner->cfg_num=0;
scanner->max_thread_num=scan_thread_num;
//optimized for CPU cache_alignment 64
scanner->ref_cnt=(int*)calloc(CPU_CACHE_ALIGMENT,scan_thread_num);
scanner->ref_cnt=aligment_int64_array_alloc(scan_thread_num);
scanner->region_update_q=MESA_lqueue_create(0,0);
scanner->region=rulescan_initialize(scan_thread_num);
rulescan_set_param(scanner->region,rs_scan_type);
@@ -2109,7 +2100,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,void *logger)
long data_size=0;
const long q_cnt=MESA_lqueue_get_count(garbage_q);
int i=0,bury_cnt=0;
int ref_cnt=0;
long long ref_cnt=0;
int have_timeout=0;
time_t now=time(NULL);
for(i=0;i<q_cnt;i++)
@@ -2132,7 +2123,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,void *logger)
destroy_group_rule(bag->group_rule);
break;
case GARBAGE_SCANNER:
ref_cnt=aligment_int_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num);
ref_cnt=aligment_int64_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num);
if(ref_cnt==0)
{
@@ -2142,7 +2133,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,void *logger)
else
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"scanner %p version %d force destroyed,ref_cnt %d.",
"scanner %p version %d force destroyed,ref_cnt %lld.",
bag->scanner,bag->scanner->version,ref_cnt);
}
@@ -2459,6 +2450,7 @@ void *thread_rule_monitor(void *arg)
free(feather->p_table_info[i]);
feather->p_table_info[i]=NULL;
}
free(feather->thread_call_cnt);
free(feather);
return NULL;
}

7
src/entry/Maat_rule_internal.h
View File

@@ -234,7 +234,7 @@ struct _Maat_table_info_t
//for stat>>>>>>>>
int stat_line_id;
long long scan_cnt;
long long scan_cpu_time; //microseconds
long long scan_cpu_time; //nano
long long input_bytes;
long long stream_num;
long long hit_cnt;
@@ -292,7 +292,7 @@ struct _Maat_scanner_t
{
int version;
time_t last_update_time;
int *ref_cnt; //optimized for cache_alignment 64
long long *ref_cnt; //optimized for cache_alignment 64
rule_scanner_t region;
pthread_rwlock_t digest_rwlock[MAX_TABLE_NUM];
GIE_handle_t* digest_handle[MAX_TABLE_NUM];
@@ -340,6 +340,7 @@ struct _Maat_feather_t
long long outer_mid_cnt;
long long inner_mid_cnt;
long long hit_cnt;
long long *thread_call_cnt;//size indicate by scan_thread_num,
};
struct _maat_garbage_t
{
@@ -374,7 +375,7 @@ inline void ipv6_ntoh(unsigned int *v6_addr)
return;
}
void maat_stat_init(struct _Maat_feather_t* feather);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timeval* start, struct timeval* end);
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end);
void maat_stat_output(struct _Maat_feather_t* feather);

28
src/entry/Maat_stat.cpp
View File

@@ -1,7 +1,10 @@
#include "Maat_rule_internal.h"
#include "aligment_int64.h"
#include <time.h>
#include <MESA/field_stat.h>
enum MAAT_FS_STATUS{
STATUS_VERSION=0,
STATUS_THRED_NUM,
STATUS_TABLE_NUM,
STATUS_OUTER_MID_NUM,
STATUS_INNER_MID_NUM,
@@ -32,6 +35,7 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_set_para(feather->stat_handle, CREATE_THREAD, &value, sizeof(value));
feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version");
feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"active_thread");
feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num");
feather->fs_status_id[STATUS_OUTER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"outer_mid");
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
@@ -40,7 +44,6 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_column_id[COLUMN_TABLE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"rule");
feather->fs_column_id[COLUMN_TABLE_REGEX_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"regex");
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_CURRENT,"stream");
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Tps");
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Bps");
if(feather->perf_on==1)
{
@@ -54,6 +57,10 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_STYLE_COLUMN,
FS_CALC_SPEED,
"PROC_Bps");
}
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_COLUMN, FS_CALC_SPEED,"IN_Tps");
if(feather->perf_on==1)
{
FS_register_ratio(feather->stat_handle,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
@@ -87,22 +94,28 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_start(feather->stat_handle);
return;
}
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timeval* start, struct timeval* end)
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end)
{
p_table->scan_cnt++;
p_table->input_bytes+=scan_len;
p_table->scan_cpu_time+=(end->tv_sec-start->tv_sec)*1000000+end->tv_usec-start->tv_usec;
if(start!=NULL&&end!=NULL)
{
p_table->scan_cpu_time+=(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec;
}
return;
}
void maat_stat_output(struct _Maat_feather_t* feather)
{
long value=0;
long long total_cfg_num=0, total_input_bytes=0, total_regex_num=0;
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0;
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
int i=0;
time_t now;
struct _Maat_table_info_t* p_table=NULL;
time(&now);
active_thread_num=aligment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,feather->outer_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,feather->inner_mid_cnt);
@@ -161,7 +174,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_ADD,
p_table->scan_cpu_time);
p_table->scan_cpu_time/1000);
total_cpu_time+=p_table->scan_cpu_time;
p_table->scan_cpu_time=0;
}
@@ -204,7 +217,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_ADD,
total_cpu_time);
total_cpu_time/1000);
printf("%lld us %s",total_cpu_time/1000,ctime(&now));
}
FS_operate(feather->stat_handle,
feather->total_stat_id,

2
src/entry/Makefile
View File

@@ -6,7 +6,7 @@ CCC = g++
CFLAGS = -Wall -g -fPIC
CFLAGS += $(OPTFLAGS)
LDDICTATOR = -Wl,-wrap,malloc -Wl,-wrap,calloc -Wl,-wrap,free -Wl,-wrap,realloc
LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lm -lrulescan -lpcre -lMESA_field_stat
LDFLAGS = -lMESA_handle_logger -lMESA_htable -lpthread -lrt -lm -lrulescan -lpcre -lMESA_field_stat
#LDFLAGS += $(LDDICTATOR)
MAILLIB = ../lib

43
src/entry/aligment_int64.h Normal file
View File

@@ -0,0 +1,43 @@
#ifndef H_ALIGMENT_INT64_H_INCLUDE
#define H_ALIGMENT_INT64_H_INCLUDE
#include <stdlib.h>
inline long long *aligment_int64_array_alloc(int size)
{
long long *ret=NULL;
ret=(long long*)calloc(CPU_CACHE_ALIGMENT,size);
return ret;
}
inline long long aligment_int64_array_sum(long long * array,int size)
{
long long sum=0;
int offset=0,i=0;
for(i=0;i<size;i++)
{
offset=(CPU_CACHE_ALIGMENT/sizeof(long long))*i;
sum+=array[offset];
}
return sum;
}
inline long long aligment_int64_array_add(long long * array,int offset,long long op_val)
{
int idx=(CPU_CACHE_ALIGMENT/sizeof(long long))*offset;
array[idx]+=op_val;
return array[idx];
}
inline int aligment_int64_array_cnt(long long * array,int size)
{
int offset=0,i=0;
int cnt=0;
for(i=0;i<size;i++)
{
offset=(CPU_CACHE_ALIGMENT/sizeof(long long))*i;
if(array[offset]>0)
{
cnt++;
}
}
return cnt;
}
#endif