gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复digest scan时，hitcnt计数访存bug。

Browse Source
This commit is contained in:
zhengchao
2018-11-27 12:55:52 +08:00
parent 8a1fe9caf2
commit be7290d1e3
7 changed files with 110 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
src/entry/Maat_api.cpp
View File

@@ -10,7 +10,7 @@
#include "Maat_rule_internal.h" #include "Maat_rule_internal.h"
#include "Maat_utils.h" #include "Maat_utils.h"
#include "dynamic_array.h" #include "dynamic_array.h"
#include "aligment_int64.h" #include "alignment_int64.h"
#include "config_monitor.h" #include "config_monitor.h"
#include "map_str2int.h" #include "map_str2int.h"
#include "rulescan.h" #include "rulescan.h"
@@ -44,13 +44,13 @@ struct _Maat_table_info_t * acqurie_table(struct _Maat_feather_t* _feather,int t
} }
inline void INC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num) inline void INC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
{ {
aligment_int64_array_add(scanner->ref_cnt, thread_num, 1); alignment_int64_array_add(scanner->ref_cnt, thread_num, 1);
return; return;
} }
inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num) inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
{ {
aligment_int64_array_add(scanner->ref_cnt, thread_num, -1); alignment_int64_array_add(scanner->ref_cnt, thread_num, -1);
return; return;
} }
@@ -143,7 +143,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
{ {
//short cut for rules contains one group //short cut for rules contains one group
scan_ret=shortcut_avilable_cnt; scan_ret=shortcut_avilable_cnt;
aligment_int64_array_add(feather->orphan_group_saving, thread_num, 1); alignment_int64_array_add(feather->orphan_group_saving, thread_num, 1);
} }
else if(shortcut_avilable_cnt==0&&region_hit_num==1&&_mid->hit_group_cnt==1&&is_last_region==1) else if(shortcut_avilable_cnt==0&&region_hit_num==1&&_mid->hit_group_cnt==1&&is_last_region==1)
{ {
@@ -151,7 +151,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
//region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule. //region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule.
//_mid->hit_group_cnt==1: With pre scan hitted group rules, one group may staisfy a compile rule //_mid->hit_group_cnt==1: With pre scan hitted group rules, one group may staisfy a compile rule
scan_ret=0; scan_ret=0;
aligment_int64_array_add(feather->last_region_saving, thread_num, 1); alignment_int64_array_add(feather->last_region_saving, thread_num, 1);
} }
else else
{ {
@@ -187,7 +187,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
} }
if(result_cnt>0) if(result_cnt>0)
{ {
aligment_int64_array_add(feather->hit_cnt,thread_num,1); alignment_int64_array_add(feather->hit_cnt,thread_num,1);
} }
return result_cnt; return result_cnt;
} }
@@ -380,7 +380,7 @@ struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather,int thr
outer_mid->feather=feather; outer_mid->feather=feather;
outer_mid->district_id=-1; outer_mid->district_id=-1;
outer_mid->thread_num=(unsigned short)thread_num; outer_mid->thread_num=(unsigned short)thread_num;
aligment_int64_array_add(feather->outer_mid_cnt, thread_num,1); alignment_int64_array_add(feather->outer_mid_cnt, thread_num,1);
return outer_mid; return outer_mid;
} }
struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* feather,int thread_num,int is_hit_region) struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* feather,int thread_num,int is_hit_region)
@@ -400,7 +400,7 @@ struct _OUTER_scan_status_t* grab_mid(scan_status_t* raw_mid,_Maat_feather_t* fe
if(_mid->inner==NULL) if(_mid->inner==NULL)
{ {
_mid->inner=_make_inner_status(); _mid->inner=_make_inner_status();
aligment_int64_array_add(feather->inner_mid_cnt,thread_num,1); alignment_int64_array_add(feather->inner_mid_cnt,thread_num,1);
} }
} }
return _mid; return _mid;
@@ -479,12 +479,12 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->effect_interval_ms=60*1000; feather->effect_interval_ms=60*1000;
feather->scan_interval_ms=1*1000; feather->scan_interval_ms=1*1000;
feather->rule_scan_type=2; feather->rule_scan_type=2;
feather->thread_call_cnt=aligment_int64_array_alloc(max_thread_num); feather->thread_call_cnt=alignment_int64_array_alloc(max_thread_num);
feather->outer_mid_cnt=aligment_int64_array_alloc(max_thread_num); feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather->inner_mid_cnt=aligment_int64_array_alloc(max_thread_num); feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather->hit_cnt=aligment_int64_array_alloc(max_thread_num); feather->hit_cnt=alignment_int64_array_alloc(max_thread_num);
feather->orphan_group_saving=aligment_int64_array_alloc(max_thread_num); feather->orphan_group_saving=alignment_int64_array_alloc(max_thread_num);
feather->last_region_saving=aligment_int64_array_alloc(max_thread_num); feather->last_region_saving=alignment_int64_array_alloc(max_thread_num);
feather->maat_version=0; feather->maat_version=0;
feather->last_full_version=0; feather->last_full_version=0;
feather->base_grp_seq=0; feather->base_grp_seq=0;
@@ -1091,7 +1091,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{ {
sub_type=make_sub_type(table_id,charset,0); sub_type=make_sub_type(table_id,charset,0);
} }
aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
scan_data_t scan_data; scan_data_t scan_data;
scan_data.text_data.text=data; scan_data.text_data.text=data;
scan_data.text_data.tlen=data_len; scan_data.text_data.tlen=data_len;
@@ -1138,7 +1138,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
aligment_int64_array_add(p_table->hit_cnt, thread_num,1); alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid,_feather,thread_num, 1); _mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -1230,7 +1230,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
{ {
return 0; return 0;
} }
aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num; region_result=my_scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*thread_num;
@@ -1244,7 +1244,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
aligment_int64_array_add(p_table->hit_cnt, thread_num,1); alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -1325,7 +1325,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
{ {
return 0; return 0;
} }
aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
ip_scan_data.rule_type=RULETYPE_IPv4; ip_scan_data.rule_type=RULETYPE_IPv4;
ip_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); ip_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
@@ -1366,7 +1366,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
aligment_int64_array_add(p_table->hit_cnt, thread_num,1); alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
@@ -1459,7 +1459,7 @@ stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,
{ {
sp->do_regex=1; sp->do_regex=1;
} }
aligment_int64_array_add(p_table->stream_num,thread_num,1); alignment_int64_array_add(p_table->stream_num,thread_num,1);
sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num); sp->rs_stream_para=rulescan_startstream(_feather->scanner->region,thread_num);
return sp; return sp;
} }
@@ -1507,7 +1507,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
sp->feather->scan_err_cnt++; sp->feather->scan_err_cnt++;
return -1; return -1;
} }
aligment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
region_result=scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num; region_result=scanner->region_rslt_buff+MAX_SCANNER_HIT_NUM*sp->thread_num;
*detail_ret=0; *detail_ret=0;
if(sp->do_merge==1) if(sp->do_merge==1)
@@ -1586,7 +1586,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
aligment_int64_array_add(p_table->hit_cnt, sp->thread_num,1); alignment_int64_array_add(p_table->hit_cnt, sp->thread_num,1);
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1); _mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -1655,7 +1655,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
aligment_int64_array_add(p_table->stream_num,sp->thread_num,-1); alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1);
if(sp->rs_stream_para!=NULL) if(sp->rs_stream_para!=NULL)
{ {
if(scanner!=NULL&&sp->version>=sp->feather->last_full_version) if(scanner!=NULL&&sp->version>=sp->feather->last_full_version)
@@ -1720,7 +1720,7 @@ stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,
sp->total_len=total_len; sp->total_len=total_len;
sp->fuzzy_hash_handle=tmp_fuzzy_handle; sp->fuzzy_hash_handle=tmp_fuzzy_handle;
pthread_mutex_init(&(sp->fuzzy_mutex),NULL); pthread_mutex_init(&(sp->fuzzy_mutex),NULL);
aligment_int64_array_add(p_table->stream_num,thread_num,1); alignment_int64_array_add(p_table->stream_num,thread_num,1);
return sp; return sp;
} }
@@ -1757,6 +1757,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
GIE_result_t query_result[MAX_SCANNER_HIT_NUM]; GIE_result_t query_result[MAX_SCANNER_HIT_NUM];
int hit_region_cnt=0,compile_ret=0; int hit_region_cnt=0,compile_ret=0;
_compile_result_t compile_result[rule_num];//dynamic array _compile_result_t compile_result[rule_num];//dynamic array
_Maat_table_info_t* p_table=NULL;
if(data==NULL||data_len<=0) if(data==NULL||data_len<=0)
{ {
return 0; return 0;
@@ -1774,8 +1775,8 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
{ {
clock_gettime(CLOCK_MONOTONIC,&start); clock_gettime(CLOCK_MONOTONIC,&start);
} }
p_table=sp->feather->p_table_info[sp->table_id];
aligment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1); alignment_int64_array_add(sp->feather->thread_call_cnt, sp->thread_num, 1);
pthread_mutex_lock(&(sp->fuzzy_mutex)); pthread_mutex_lock(&(sp->fuzzy_mutex));
sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset); sp->process_offset+=SFH_feed(sp->fuzzy_hash_handle, data, (unsigned int)data_len,offset);
pthread_mutex_unlock(&(sp->fuzzy_mutex)); pthread_mutex_unlock(&(sp->fuzzy_mutex));
@@ -1811,7 +1812,7 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
} }
if(hit_region_cnt>0) if(hit_region_cnt>0)
{ {
sp->feather->p_table_info[sp->table_id]->hit_cnt++; alignment_int64_array_add(p_table->hit_cnt, sp->thread_num, 1);
_mid=grab_mid(mid,sp->feather, sp->thread_num,1); _mid=grab_mid(mid,sp->feather, sp->thread_num,1);
compile_ret=region_compile(sp->feather,_mid->inner, compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -1846,7 +1847,7 @@ void Maat_stream_scan_digest_end(stream_para_t* stream_para)
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id]; struct _Maat_table_info_t * p_table=sp->feather->p_table_info[sp->table_id];
aligment_int64_array_add(p_table->stream_num,sp->thread_num,-1); alignment_int64_array_add(p_table->stream_num,sp->thread_num,-1);
if(scanner!=NULL) if(scanner!=NULL)
{ {
if(sp->version==sp->feather->maat_version) if(sp->version==sp->feather->maat_version)
@@ -1969,7 +1970,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
{ {
return 0; return 0;
} }
aligment_int64_array_add(_feather->thread_call_cnt, thread_num, 1); alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
INC_SCANNER_REF(my_scanner,thread_num); INC_SCANNER_REF(my_scanner,thread_num);
region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM); region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM);
@@ -1981,7 +1982,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
} }
else if(region_ret>0) else if(region_ret>0)
{ {
aligment_int64_array_add(p_table->hit_cnt, thread_num,1); alignment_int64_array_add(p_table->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner, compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region, _mid->is_last_region,
@@ -2021,12 +2022,12 @@ void Maat_clean_status(scan_status_t* mid)
return; return;
} }
_mid=(struct _OUTER_scan_status_t*)(*mid); _mid=(struct _OUTER_scan_status_t*)(*mid);
aligment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1); alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL) if(_mid->inner!=NULL)
{ {
free(_mid->inner->hitted_group_id); free(_mid->inner->hitted_group_id);
free(_mid->inner); free(_mid->inner);
aligment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1); alignment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1);
} }
_mid->feather=NULL; _mid->feather=NULL;
free(_mid); free(_mid);

1
src/entry/Maat_command.cpp
View File

@@ -2415,6 +2415,7 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
dbsize=read_redis_integer(data_reply); dbsize=read_redis_integer(data_reply);
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply=_wrap_redisCommand(ctx,"MULTI");
freeReplyObject(data_reply); freeReplyObject(data_reply);
redisAppendCommand(ctx,"FLUSHDB"); redisAppendCommand(ctx,"FLUSHDB");

73
src/entry/Maat_rule.cpp
View File

@@ -23,7 +23,7 @@
#include "json2iris.h" #include "json2iris.h"
#include "cJSON.h" #include "cJSON.h"
#include "dynamic_array.h" #include "dynamic_array.h"
#include "aligment_int64.h" #include "alignment_int64.h"
#include "config_monitor.h" #include "config_monitor.h"
#include "map_str2int.h" #include "map_str2int.h"
@@ -32,7 +32,7 @@
#include "stream_fuzzy_hash.h" #include "stream_fuzzy_hash.h"
#include "gram_index_engine.h" #include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_4_20181120=1; int MAAT_FRAME_VERSION_2_4_20181126=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
@@ -334,7 +334,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger)
} }
array=cJSON_GetObjectItem(json, "tags"); array=cJSON_GetObjectItem(json, "tags");
n_tags=cJSON_GetArraySize(array); n_tags=cJSON_GetArraySize(array);
p=(struct rule_tag*)calloc(sizeof(struct rule_tag), n_tags); p=ALLOC(struct rule_tag, n_tags);
for(int i=0;i<n_tags;i++) for(int i=0;i<n_tags;i++)
{ {
tag=cJSON_GetArrayItem(array, i); tag=cJSON_GetArrayItem(array, i);
@@ -523,25 +523,25 @@ void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* s
_Maat_table_info_t* create_table_info(int max_thread_num) _Maat_table_info_t* create_table_info(int max_thread_num)
{ {
struct _Maat_table_info_t*p=NULL; struct _Maat_table_info_t*p=ALLOC(struct _Maat_table_info_t, 1);
p=(struct _Maat_table_info_t*)calloc(sizeof(struct _Maat_table_info_t),1);
p->conj_cnt=1; p->conj_cnt=1;
p->scan_cnt=aligment_int64_array_alloc(max_thread_num); p->scan_cnt=alignment_int64_array_alloc(max_thread_num);
p->scan_cpu_time=aligment_int64_array_alloc(max_thread_num); p->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
p->input_bytes=aligment_int64_array_alloc(max_thread_num); p->input_bytes=alignment_int64_array_alloc(max_thread_num);
p->stream_num=aligment_int64_array_alloc(max_thread_num); p->stream_num=alignment_int64_array_alloc(max_thread_num);
p->hit_cnt=aligment_int64_array_alloc(max_thread_num); p->hit_cnt=alignment_int64_array_alloc(max_thread_num);
p->cross_cache_size=0; p->cross_cache_size=0;
p->quick_expr_switch=0; p->quick_expr_switch=0;
return p; return p;
} }
void destroy_table_info(struct _Maat_table_info_t*p) void destroy_table_info(struct _Maat_table_info_t*p)
{ {
aligment_int64_array_free(p->scan_cnt); alignment_int64_array_free(p->scan_cnt);
aligment_int64_array_free(p->scan_cpu_time); alignment_int64_array_free(p->scan_cpu_time);
aligment_int64_array_free(p->input_bytes); alignment_int64_array_free(p->input_bytes);
aligment_int64_array_free(p->stream_num); alignment_int64_array_free(p->stream_num);
aligment_int64_array_free(p->hit_cnt); alignment_int64_array_free(p->hit_cnt);
if(p->cb_info!=NULL) if(p->cb_info!=NULL)
{ {
dynamic_array_destroy(p->cb_info->cache_lines, free); dynamic_array_destroy(p->cb_info->cache_lines, free);
@@ -688,7 +688,7 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
FILE*fp=NULL; FILE*fp=NULL;
char line[MAX_TABLE_LINE_SIZE]; char line[MAX_TABLE_LINE_SIZE];
int i=0,ret=0,table_cnt=0; int i=0,ret=0,table_cnt=0;
char table_type_str[16],not_care[1024], tmp_str[32]; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0};
MESA_htable_handle string2int_map=map_create(); MESA_htable_handle string2int_map=map_create();
struct _Maat_table_info_t*p=NULL; struct _Maat_table_info_t*p=NULL;
struct _Maat_table_info_t*conj_table=NULL; struct _Maat_table_info_t*conj_table=NULL;
@@ -996,18 +996,19 @@ void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule)
pthread_rwlock_wrlock(&(compile_rule->rwlock)); pthread_rwlock_wrlock(&(compile_rule->rwlock));
if(db_compile_rule!=NULL) if(db_compile_rule!=NULL)
{ {
for(i=0; i<table->ex_data_num; i++)
{
rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i);
compile_rule->ads[i]=NULL;
}
free(db_compile_rule->service_defined); free(db_compile_rule->service_defined);
free(db_compile_rule); free(db_compile_rule);
compile_rule->db_c_rule=NULL; compile_rule->db_c_rule=NULL;
} }
for(i=0; table!=NULL && i<table->ex_data_num; i++)
{
rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i);
compile_rule->ads[i]=NULL;
}
free(compile_rule->ads); free(compile_rule->ads);
pthread_rwlock_unlock(&(compile_rule->rwlock)); pthread_rwlock_unlock(&(compile_rule->rwlock));
@@ -1230,7 +1231,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
scanner->dedup_expr_num=0; scanner->dedup_expr_num=0;
scanner->max_thread_num=scan_thread_num; scanner->max_thread_num=scan_thread_num;
//optimized for CPU cache_alignment 64 //optimized for CPU cache_alignment 64
scanner->ref_cnt=aligment_int64_array_alloc(scan_thread_num); scanner->ref_cnt=alignment_int64_array_alloc(scan_thread_num);
scanner->region_update_q=MESA_lqueue_create(0,0); scanner->region_update_q=MESA_lqueue_create(0,0);
scanner->region=rulescan_initialize(scan_thread_num); scanner->region=rulescan_initialize(scan_thread_num);
@@ -2396,7 +2397,7 @@ void compatible_group_udpate(struct _Maat_table_info_t* table,int region_id,int
} }
void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) void update_expr_rule(struct _Maat_table_info_t* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
{ {
struct db_str_rule_t* maat_str_rule=(struct db_str_rule_t*)malloc(sizeof(struct db_str_rule_t)); struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1);
int ret=0,db_hexbin=0,rule_type=0; int ret=0,db_hexbin=0,rule_type=0;
switch(table->table_type) switch(table->table_type)
{ {
@@ -3033,7 +3034,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
destroy_group_rule(bag->group_rule); destroy_group_rule(bag->group_rule);
break; break;
case GARBAGE_SCANNER: case GARBAGE_SCANNER:
ref_cnt=aligment_int64_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num); ref_cnt=alignment_int64_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num);
if(ref_cnt==0) if(ref_cnt==0)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
@@ -3513,7 +3514,7 @@ void *thread_rule_monitor(void *arg)
old_scanner->version, feather->scanner->version); old_scanner->version, feather->scanner->version);
} }
assert(old_scanner->tomb_ref==feather->garbage_q); assert(old_scanner->tomb_ref==feather->garbage_q);
feather->zombie_rs_stream+=aligment_int64_array_sum(old_scanner->ref_cnt,old_scanner->max_thread_num); feather->zombie_rs_stream+=alignment_int64_array_sum(old_scanner->ref_cnt,old_scanner->max_thread_num);
garbage_bagging(GARBAGE_SCANNER, old_scanner, feather->garbage_q); garbage_bagging(GARBAGE_SCANNER, old_scanner, feather->garbage_q);
} }
feather->update_tmp_scanner=NULL; feather->update_tmp_scanner=NULL;
@@ -3549,8 +3550,6 @@ void *thread_rule_monitor(void *arg)
destroy_maat_scanner(feather->scanner); destroy_maat_scanner(feather->scanner);
garbage_bury(feather->garbage_q,0,feather->logger); garbage_bury(feather->garbage_q,0,feather->logger);
MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL); MESA_lqueue_destroy(feather->garbage_q,lqueue_destroy_cb,NULL);
FS_stop(&(feather->stat_handle));
int i=0; int i=0;
for(i=0;i<MAX_TABLE_NUM;i++) for(i=0;i<MAX_TABLE_NUM;i++)
{ {
@@ -3561,12 +3560,12 @@ void *thread_rule_monitor(void *arg)
destroy_table_info(feather->p_table_info[i]); destroy_table_info(feather->p_table_info[i]);
feather->p_table_info[i]=NULL; feather->p_table_info[i]=NULL;
} }
aligment_int64_array_free(feather->thread_call_cnt); alignment_int64_array_free(feather->thread_call_cnt);
aligment_int64_array_free(feather->inner_mid_cnt); alignment_int64_array_free(feather->inner_mid_cnt);
aligment_int64_array_free(feather->outer_mid_cnt); alignment_int64_array_free(feather->outer_mid_cnt);
aligment_int64_array_free(feather->hit_cnt); alignment_int64_array_free(feather->hit_cnt);
aligment_int64_array_free(feather->orphan_group_saving); alignment_int64_array_free(feather->orphan_group_saving);
aligment_int64_array_free(feather->last_region_saving); alignment_int64_array_free(feather->last_region_saving);
if(feather->REDIS_MODE_ON==1&&feather->redis_read_ctx!=NULL) if(feather->REDIS_MODE_ON==1&&feather->redis_read_ctx!=NULL)
{ {
pthread_mutex_lock(&(feather->redis_write_lock)); pthread_mutex_lock(&(feather->redis_write_lock));
@@ -3576,6 +3575,12 @@ void *thread_rule_monitor(void *arg)
feather->redis_write_ctx=NULL; feather->redis_write_ctx=NULL;
pthread_mutex_unlock(&(feather->redis_write_lock)); pthread_mutex_unlock(&(feather->redis_write_lock));
} }
for(i=0; i<feather->n_tags; i++)
{
free(feather->accept_tags[i].tag_name);
free(feather->accept_tags[i].tag_val);
}
free(feather->accept_tags);
if(feather->stat_on&& feather->stat_handle) if(feather->stat_on&& feather->stat_handle)
{ {
FS_stop(&(feather->stat_handle)); FS_stop(&(feather->stat_handle));

30
src/entry/Maat_stat.cpp
View File

@@ -1,5 +1,5 @@
#include "Maat_rule_internal.h" #include "Maat_rule_internal.h"
#include "aligment_int64.h" #include "alignment_int64.h"
#include <time.h> #include <time.h>
#include <MESA/field_stat.h> #include <MESA/field_stat.h>
enum MAAT_FS_STATUS{ enum MAAT_FS_STATUS{
@@ -149,11 +149,11 @@ void maat_stat_init(struct _Maat_feather_t* feather)
} }
void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num) void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
{ {
aligment_int64_array_add(p_table->scan_cnt,thread_num,1); alignment_int64_array_add(p_table->scan_cnt,thread_num,1);
aligment_int64_array_add(p_table->input_bytes,thread_num,scan_len); alignment_int64_array_add(p_table->input_bytes,thread_num,scan_len);
if(start!=NULL&&end!=NULL) if(start!=NULL&&end!=NULL)
{ {
aligment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec); alignment_int64_array_add(p_table->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
} }
return; return;
} }
@@ -171,11 +171,11 @@ void maat_stat_output(struct _Maat_feather_t* feather)
time_t now; time_t now;
struct _Maat_table_info_t* p_table=NULL; struct _Maat_table_info_t* p_table=NULL;
time(&now); time(&now);
active_thread_num=aligment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num); active_thread_num=alignment_int64_array_cnt(feather->thread_call_cnt, feather->scan_thread_num);
outer_mid_cnt=aligment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num); outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
inner_mid_cnt=aligment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num); inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num);
orphan_group_saving=aligment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num); orphan_group_saving=alignment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num);
last_region_saving=aligment_int64_array_sum(feather->last_region_saving,feather->scan_thread_num); last_region_saving=alignment_int64_array_sum(feather->last_region_saving,feather->scan_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num); FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num);
@@ -231,7 +231,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
p_table->regex_rule_cnt); p_table->regex_rule_cnt);
total_regex_num+= p_table->regex_rule_cnt; total_regex_num+= p_table->regex_rule_cnt;
table_stream_num=aligment_int64_array_sum(p_table->stream_num,feather->scan_thread_num); table_stream_num=alignment_int64_array_sum(p_table->stream_num,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM], feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
@@ -239,7 +239,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_stream_num); table_stream_num);
total_stream_cnt+= table_stream_num; total_stream_cnt+= table_stream_num;
table_scan_cnt=aligment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num); table_scan_cnt=alignment_int64_array_sum(p_table->scan_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT], feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
@@ -247,7 +247,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
table_scan_cnt); table_scan_cnt);
total_scan_cnt+=table_scan_cnt; total_scan_cnt+=table_scan_cnt;
table_input_bytes=aligment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num); table_input_bytes=alignment_int64_array_sum(p_table->input_bytes,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES], feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
@@ -256,7 +256,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
total_input_bytes+=table_input_bytes; total_input_bytes+=table_input_bytes;
if(feather->perf_on==1) if(feather->perf_on==1)
{ {
table_scan_cpu_time=aligment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num); table_scan_cpu_time=alignment_int64_array_sum(p_table->scan_cpu_time,feather->scan_thread_num);
table_scan_cpu_time/=1000; table_scan_cpu_time/=1000;
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
@@ -266,7 +266,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
total_cpu_time+=table_scan_cpu_time; total_cpu_time+=table_scan_cpu_time;
} }
table_hit_cnt=aligment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num); table_hit_cnt=alignment_int64_array_sum(p_table->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
p_table->stat_line_id, p_table->stat_line_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT], feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
@@ -312,7 +312,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_OP_SET, FS_OP_SET,
total_cpu_time); total_cpu_time);
} }
total_hit_cnt=aligment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num); total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, FS_operate(feather->stat_handle,
feather->total_stat_id, feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT], feather->fs_column_id[COLUMN_TABLE_HIT_CNT],

2
src/inc_internal/Maat_rule_internal.h
View File

@@ -14,7 +14,7 @@
#include "stream_fuzzy_hash.h" #include "stream_fuzzy_hash.h"
#include "gram_index_engine.h" #include "gram_index_engine.h"
#include "aligment_int64.h" #include "alignment_int64.h"
#include <pthread.h> #include <pthread.h>
#include <iconv.h> #include <iconv.h>

14
src/inc_internal/aligment_int64.h → src/inc_internal/alignment_int64.h
View File

@@ -1,18 +1,18 @@
#ifndef H_ALIGMENT_INT64_H_INCLUDE #ifndef H_ALIGNMENT_INT64_H_INCLUDE
#define H_ALIGMENT_INT64_H_INCLUDE #define H_ALIGNMENT_INT64_H_INCLUDE
#include <stdlib.h> #include <stdlib.h>
#define CPU_CACHE_ALIGMENT 64 #define CPU_CACHE_ALIGMENT 64
typedef long long* mcore_long_t; typedef long long* mcore_long_t;
inline long long *aligment_int64_array_alloc(int size) inline mcore_long_t alignment_int64_array_alloc(int size)
{ {
long long *ret=NULL; long long *ret=NULL;
ret=(long long*)calloc(CPU_CACHE_ALIGMENT,size); ret=(long long*)calloc(CPU_CACHE_ALIGMENT,size);
return ret; return ret;
} }
inline long long aligment_int64_array_sum(mcore_long_t array,int size) inline long long alignment_int64_array_sum(mcore_long_t array,int size)
{ {
long long sum=0; long long sum=0;
int offset=0,i=0; int offset=0,i=0;
@@ -23,13 +23,13 @@ inline long long aligment_int64_array_sum(mcore_long_t array,int size)
} }
return sum; return sum;
} }
inline long long aligment_int64_array_add(mcore_long_t array,int offset,long long op_val) inline long long alignment_int64_array_add(mcore_long_t array,int offset,long long op_val)
{ {
int idx=(CPU_CACHE_ALIGMENT/sizeof(long long))*offset; int idx=(CPU_CACHE_ALIGMENT/sizeof(long long))*offset;
array[idx]+=op_val; array[idx]+=op_val;
return array[idx]; return array[idx];
} }
inline long long aligment_int64_array_cnt(mcore_long_t array,int size) inline long long alignment_int64_array_cnt(mcore_long_t array,int size)
{ {
int offset=0,i=0; int offset=0,i=0;
int cnt=0; int cnt=0;
@@ -43,7 +43,7 @@ inline long long aligment_int64_array_cnt(mcore_long_t array,int size)
} }
return cnt; return cnt;
} }
inline void aligment_int64_array_free(mcore_long_t array) inline void alignment_int64_array_free(mcore_long_t array)
{ {
free(array); free(array);
} }

12
test/test_maatframe.cpp
View File

@@ -743,10 +743,8 @@ class MaatFileTest : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() static void SetUpTestCase()
{ {
void *logger=NULL;
const char* rule_folder="./ntcrule/full/index"; const char* rule_folder="./ntcrule/full/index";
logger=MESA_create_runtime_log_handle("test_maat_file.log",0); logger=MESA_create_runtime_log_handle("test_maat_file.log",0);
const char* table_info="./t2_tableinfo.conf"; const char* table_info="./t2_tableinfo.conf";
@@ -766,12 +764,16 @@ protected:
static void TearDownTestCase() static void TearDownTestCase()
{ {
Maat_burn_feather(_shared_feather_f); Maat_burn_feather(_shared_feather_f);
MESA_destroy_runtime_log_handle(logger);
} }
// Some expensive resource shared by all tests. // Some expensive resource shared by all tests.
static Maat_feather_t _shared_feather_f; static Maat_feather_t _shared_feather_f;
static void *logger;
}; };
Maat_feather_t MaatFileTest::_shared_feather_f; Maat_feather_t MaatFileTest::_shared_feather_f;
Maat_feather_t MaatFileTest::logger;
TEST_F(MaatFileTest, StreamFiles) TEST_F(MaatFileTest, StreamFiles)
{ {
#define StreamScan_StreamFiles #define StreamScan_StreamFiles
@@ -1107,6 +1109,7 @@ TEST_F(MaatCmdTest, SetIP)
int table_id=0; int table_id=0;
struct Maat_rule_t result; struct Maat_rule_t result;
memset(&result, 0, sizeof(result));
scan_status_t mid=NULL; scan_status_t mid=NULL;
table_id=Maat_table_register(feather,region_table); table_id=Maat_table_register(feather,region_table);
ASSERT_GE(table_id, 0); ASSERT_GE(table_id, 0);
@@ -1377,6 +1380,9 @@ int main(int argc, char ** argv)
Maat_initiate_feather(g_feather); Maat_initiate_feather(g_feather);
printf("Maat initiating, see %s\n",log_file); printf("Maat initiating, see %s\n",log_file);
return RUN_ALL_TESTS(); int ret=RUN_ALL_TESTS();
Maat_burn_feather(g_feather);
MESA_destroy_runtime_log_handle(g_logger);
return ret;
} }