gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

非逻辑编译通过。

Browse Source
This commit is contained in:
zhengchao
2019-01-05 17:11:20 +08:00
parent 475c93093a
commit b49016adb7
5 changed files with 224 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/CMakeLists.txt
View File

@@ -8,7 +8,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC) add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/UniversalBoolMatch.cpp) set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/) include_directories(/opt/MESA/include/MESA/)

99
src/entry/Maat_api.cpp
View File

@@ -5,7 +5,7 @@
#include <MESA/MESA_handle_logger.h> #include <MESA/MESA_handle_logger.h>
#include "rulescan.h" #include "rulescan.h"
#include "UniversalBoolMatch.h" #include "bool_matcher.h"
#include "Maat_rule.h" #include "Maat_rule.h"
#include "Maat_rule_internal.h" #include "Maat_rule_internal.h"
#include "Maat_utils.h" #include "Maat_utils.h"
@@ -42,12 +42,12 @@ struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int tabl
} }
return p_table; return p_table;
} }
inline void INC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num) inline void INC_SCANNER_REF(Maat_scanner_t*scanner,int thread_num)
{ {
alignment_int64_array_add(scanner->ref_cnt, thread_num, 1); alignment_int64_array_add(scanner->ref_cnt, thread_num, 1);
return; return;
} }
inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num) inline void DEC_SCANNER_REF(Maat_scanner_t*scanner,int thread_num)
{ {
alignment_int64_array_add(scanner->ref_cnt, thread_num, -1); alignment_int64_array_add(scanner->ref_cnt, thread_num, -1);
@@ -57,10 +57,10 @@ inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
//return 1 if insert a unique id //return 1 if insert a unique id
//return 0 if id is duplicated //return 0 if id is duplicated
//return -1 if set is full //return -1 if set is full
int insert_set_id(unsigned int **set,int* size,int cnt,unsigned int id) int insert_set_id(unsigned long long **set, size_t* size, size_t cnt, unsigned long long id)
{ {
int i=0; size_t i=0;
for(i=0;i<cnt;i++) for(i=0; i<cnt; i++)
{ {
if((*set)[i]==id) if((*set)[i]==id)
{ {
@@ -72,7 +72,7 @@ int insert_set_id(unsigned int **set,int* size,int cnt,unsigned int id)
if(cnt==*size) if(cnt==*size)
{ {
*size+=16; *size+=16;
*set=(unsigned int*)realloc(*set,(*size)*sizeof(unsigned int)); *set=(unsigned long long*)realloc(*set, (*size)*sizeof(unsigned long long));
} }
(*set)[cnt]=id; (*set)[cnt]=id;
return 1; return 1;
@@ -82,18 +82,20 @@ int insert_set_id(unsigned int **set,int* size,int cnt,unsigned int id)
return 0; return 0;
} }
} }
int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsigned int* hitted_id,int hit_cnt,int* region_pos,int size) size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit,
const unsigned long long* hitted_id, size_t hit_cnt, int* region_pos, size_t size)
{ {
int i=0,j=0; size_t i=0, j=0;
int k=0; size_t k=0;
for(i=0;i<hit_cnt;i++) for(i=0;i<hit_cnt;i++)
{ {
for(j=0;j<(int)(compile_hit->bool_item_num);j++) for(j=0; j<compile_hit->item_num; j++)
{ {
if(hitted_id[i]==compile_hit->bool_item_ids[j]) if(hitted_id[i]==compile_hit->items[j].item_id)
{ {
region_pos[k]=i; region_pos[k]=i;
k++; k++;
assert(k<size);
} }
} }
} }
@@ -118,24 +120,25 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
int scan_ret=0,result_cnt=0; int scan_ret=0,result_cnt=0;
int ret=0,i=0; int ret=0,i=0;
int r_in_c_cnt=0; size_t r_in_c_cnt=0;
int shortcut_avilable_cnt=0; int shortcut_avilable_cnt=0;
void* bool_matcher=feather->scanner->bool_macher_expr_compiler; unsigned char has_not_flag=0;
struct _Maat_group_inner_t* group_rule=NULL; struct bool_matcher* bm=feather->scanner->bool_macher_expr_compiler;
struct _Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM]; struct Maat_group_inner_t* group_rule=NULL;
struct _Maat_compile_inner_t* _mi_rule=NULL; struct Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct Maat_compile_inner_t* _mi_rule=NULL;
int region_pos[MAX_SCANNER_HIT_NUM]; int region_pos[MAX_SCANNER_HIT_NUM];
_mid->cur_hit_cnt=0; _mid->cur_hit_cnt=0;
for(i=0;i<region_hit_num;i++) for(i=0;i<region_hit_num;i++)
{ {
group_rule=*(struct _Maat_group_inner_t**)((char*)region_hit+region_type_size*i+group_offset); group_rule=*(struct Maat_group_inner_t**)((char*)region_hit+region_type_size*i+group_offset);
if(group_rule->group_id<0) if(group_rule->group_id<0)
{ {
continue; continue;
} }
if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM) if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
{ {
array_mi_rule[shortcut_avilable_cnt]=(struct _Maat_compile_inner_t*)(group_rule->compile_shortcut); array_mi_rule[shortcut_avilable_cnt]=(struct Maat_compile_inner_t*)(group_rule->compile_shortcut);
shortcut_avilable_cnt++; shortcut_avilable_cnt++;
} }
_mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id; _mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id;
@@ -143,7 +146,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
ret=insert_set_id(&(_mid->hitted_group_id), ret=insert_set_id(&(_mid->hitted_group_id),
&(_mid->hit_group_size), &(_mid->hit_group_size),
_mid->hit_group_cnt, _mid->hit_group_cnt,
(unsigned int)group_rule->group_id); group_rule->group_id);
_mid->hit_group_cnt+=ret; _mid->hit_group_cnt+=ret;
} }
if(shortcut_avilable_cnt==region_hit_num||shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM) if(shortcut_avilable_cnt==region_hit_num||shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM)
@@ -162,8 +165,8 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
} }
else else
{ {
scan_ret=boolexpr_match(bool_matcher,thread_num, scan_ret=bool_matcher_match(bm, thread_num,
_mid->hitted_group_id,_mid->hit_group_cnt, _mid->hitted_group_id, _mid->hit_group_cnt,
(void **)array_mi_rule, MAX_SCANNER_HIT_NUM); (void **)array_mi_rule, MAX_SCANNER_HIT_NUM);
} }
for(i=0;i<scan_ret&&result_cnt<size;i++) for(i=0;i<scan_ret&&result_cnt<size;i++)
@@ -178,8 +181,8 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
{ {
if(_mi_rule->is_valid==1) if(_mi_rule->is_valid==1)
{ {
make_group_set(_mi_rule,&(rs_result[result_cnt].group_set)); make_group_set(_mi_rule, &(rs_result[result_cnt].group_set), &has_not_flag);
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt, r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM); region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0)//compile config hitted becasue of new reigon if(r_in_c_cnt>0)//compile config hitted becasue of new reigon
{ {
@@ -201,7 +204,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
return result_cnt; return result_cnt;
} }
int exprid2region_id(struct _Maat_group_inner_t* group_rule,int expr_id,int* district_id) int exprid2region_id(struct Maat_group_inner_t* group_rule,int expr_id,int* district_id)
{ {
int i=0,region_id=-1; int i=0,region_id=-1;
struct _Maat_region_inner_t* region_rule=NULL; struct _Maat_region_inner_t* region_rule=NULL;
@@ -226,13 +229,13 @@ int exprid2region_id(struct _Maat_group_inner_t* group_rule,int expr_id,int* dis
} }
int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num) int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num)
{ {
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
int i=0; int i=0;
int district_id=-1,region_id=-1; int district_id=-1,region_id=-1;
int ret_region_num=region_hit_num; int ret_region_num=region_hit_num;
while(i<ret_region_num) while(i<ret_region_num)
{ {
group_rule=(struct _Maat_group_inner_t*)(region_hit[i].tag); group_rule=(struct Maat_group_inner_t*)(region_hit[i].tag);
region_id=exprid2region_id(group_rule, region_hit[i].expr_id,&district_id); region_id=exprid2region_id(group_rule, region_hit[i].expr_id,&district_id);
if(region_id>0&&district_id!=_mid->district_id) if(region_id>0&&district_id!=_mid->district_id)
{ {
@@ -328,19 +331,19 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
memset(r_in_c_flag,0,sizeof(r_in_c_flag)); memset(r_in_c_flag,0,sizeof(r_in_c_flag));
memset(region_pos,0,sizeof(region_pos)); memset(region_pos,0,sizeof(region_pos));
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
//for each hitted compile cfg,find its region_ids //for each hitted compile cfg,find its region_ids
for(i=0;i<compile_cnt&&i<detail_num;i++) for(i=0;i<compile_cnt&&i<detail_num;i++)
{ {
hit_detail[i].config_id=compile_hit[i].compile_id; hit_detail[i].config_id=compile_hit[i].compile_id;
r_in_c_cnt=pickup_hit_region_from_compile(&(compile_hit[i].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt, r_in_c_cnt=pickup_hit_region_from_compile(&(compile_hit[i].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM); region_pos, MAX_SCANNER_HIT_NUM);
assert(r_in_c_cnt>0);//previous hitted compile was elimited in region_compile assert(r_in_c_cnt>0);//previous hitted compile was elimited in region_compile
for(j=0,k=0;j<r_in_c_cnt&&k<MAAT_MAX_HIT_RULE_NUM;j++) for(j=0,k=0;j<r_in_c_cnt&&k<MAAT_MAX_HIT_RULE_NUM;j++)
{ {
pos=region_pos[j]; pos=region_pos[j];
r_in_c_flag[pos]=1; r_in_c_flag[pos]=1;
group_rule=(struct _Maat_group_inner_t*)(region_hit[pos].tag); group_rule=(struct Maat_group_inner_t*)(region_hit[pos].tag);
region_id=exprid2region_id(group_rule,region_hit[pos].expr_id,&district_id); region_id=exprid2region_id(group_rule,region_hit[pos].expr_id,&district_id);
if(region_id<0) if(region_id<0)
{ {
@@ -360,7 +363,7 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
{ {
if(r_in_c_flag[k]==0) if(r_in_c_flag[k]==0)
{ {
group_rule=(struct _Maat_group_inner_t*)(region_hit[k].tag); group_rule=(struct Maat_group_inner_t*)(region_hit[k].tag);
hit_detail[j].config_id=-2; hit_detail[j].config_id=-2;
hit_detail[j].hit_region_cnt=1; hit_detail[j].hit_region_cnt=1;
hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id); hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id);
@@ -375,14 +378,14 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
struct _INNER_scan_status_t* _make_inner_status(void) struct _INNER_scan_status_t* _make_inner_status(void)
{ {
struct _INNER_scan_status_t* inner_mid=NULL; struct _INNER_scan_status_t* inner_mid=NULL;
inner_mid=(struct _INNER_scan_status_t*)calloc(sizeof(struct _INNER_scan_status_t),1); inner_mid=ALLOC(struct _INNER_scan_status_t, 1);
inner_mid->cur_hit_cnt=0; inner_mid->cur_hit_cnt=0;
inner_mid->hit_group_cnt=0; inner_mid->hit_group_cnt=0;
inner_mid->hit_group_size=4; inner_mid->hit_group_size=4;
inner_mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*inner_mid->hit_group_size); inner_mid->hitted_group_id= ALLOC(unsigned long long, inner_mid->hit_group_size);
return inner_mid; return inner_mid;
} }
struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather,int thread_num) struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather, int thread_num)
{ {
struct _OUTER_scan_status_t* outer_mid=NULL; struct _OUTER_scan_status_t* outer_mid=NULL;
outer_mid=(struct _OUTER_scan_status_t*)calloc(sizeof(struct _OUTER_scan_status_t),1); outer_mid=(struct _OUTER_scan_status_t*)calloc(sizeof(struct _OUTER_scan_status_t),1);
@@ -991,7 +994,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user) void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
{ {
struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user; struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
struct _Maat_compile_inner_t *compile_inner=(struct _Maat_compile_inner_t *)data; struct Maat_compile_inner_t *compile_inner=(struct Maat_compile_inner_t *)data;
MAAT_RULE_EX_DATA ad=NULL; MAAT_RULE_EX_DATA ad=NULL;
if(compile_inner->ref_table==NULL || compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL) if(compile_inner->ref_table==NULL || compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL)
@@ -1053,12 +1056,12 @@ failed:
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx) MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
{ {
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct _Maat_compile_inner_t *compile_inner=NULL; struct Maat_compile_inner_t *compile_inner=NULL;
const struct compile_table_desc* compile_desc=NULL; const struct compile_table_desc* compile_desc=NULL;
const struct compile_ex_data_idx* ex_desc=NULL; const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL; MAAT_RULE_EX_DATA ad=NULL;
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL) if(compile_inner==NULL)
{ {
return NULL; return NULL;
@@ -1286,7 +1289,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
struct Maat_table_desc *p_table=NULL; struct Maat_table_desc *p_table=NULL;
struct expr_table_desc* expr_desc=NULL; struct expr_table_desc* expr_desc=NULL;
struct timespec start,end; struct timespec start,end;
_Maat_scanner_t* my_scanner=NULL; Maat_scanner_t* my_scanner=NULL;
if(data==NULL||data_len<=0) if(data==NULL||data_len<=0)
{ {
return 0; return 0;
@@ -1422,7 +1425,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
scan_result_t *region_result=NULL; scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num]; _compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL; struct Maat_scanner_t* my_scanner=NULL;
intval_scan_data.rule_type=RULETYPE_INT; intval_scan_data.rule_type=RULETYPE_INT;
intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0); intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
intval_scan_data.int_data=intval; intval_scan_data.int_data=intval;
@@ -1509,7 +1512,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
Maat_table_desc* p_table=NULL; Maat_table_desc* p_table=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL; struct Maat_scanner_t* my_scanner=NULL;
struct timespec start,end; struct timespec start,end;
if(_feather->perf_on==1) if(_feather->perf_on==1)
{ {
@@ -1627,7 +1630,7 @@ int Maat_scan_addr(Maat_feather_t feather,int table_id
stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num) stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num)
{ {
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* scanner=NULL; struct Maat_scanner_t* scanner=NULL;
struct Maat_table_desc *p_table=NULL; struct Maat_table_desc *p_table=NULL;
assert(thread_num<_feather->scan_thread_num); assert(thread_num<_feather->scan_thread_num);
@@ -1690,7 +1693,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
,int* detail_ret,scan_status_t* mid) ,int* detail_ret,scan_status_t* mid)
{ {
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct Maat_scanner_t* scanner=sp->feather->scanner;
int sub_type=0; int sub_type=0;
int region_ret=0,hit_region_cnt=0,compile_ret=0; int region_ret=0,hit_region_cnt=0,compile_ret=0;
@@ -1874,7 +1877,7 @@ int Maat_stream_scan_string(stream_para_t* stream_para
void Maat_stream_scan_string_end(stream_para_t* stream_para) void Maat_stream_scan_string_end(stream_para_t* stream_para)
{ {
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1); alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1);
if(sp->rs_stream_para!=NULL) if(sp->rs_stream_para!=NULL)
@@ -1911,7 +1914,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num) stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num)
{ {
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* scanner=NULL; struct Maat_scanner_t* scanner=NULL;
sfh_instance_t * tmp_fuzzy_handle=NULL; sfh_instance_t * tmp_fuzzy_handle=NULL;
struct Maat_table_desc *p_table=NULL; struct Maat_table_desc *p_table=NULL;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST); p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST);
@@ -2067,7 +2070,7 @@ fast_out:
void Maat_stream_scan_digest_end(stream_para_t* stream_para) void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{ {
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para); struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner; struct Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id]; struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1); alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1);
if(scanner!=NULL) if(scanner!=NULL)
@@ -2090,11 +2093,11 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
{ {
int ret=0; int ret=0;
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct _Maat_compile_inner_t *compile_inner=NULL; struct Maat_compile_inner_t *compile_inner=NULL;
switch(type) switch(type)
{ {
case MAAT_RULE_SERV_DEFINE: case MAAT_RULE_SERV_DEFINE:
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL) if(compile_inner==NULL)
{ {
ret=0; ret=0;
@@ -2163,7 +2166,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
GIE_result_t region_result[MAX_SCANNER_HIT_NUM]; GIE_result_t region_result[MAX_SCANNER_HIT_NUM];
_compile_result_t compile_result[rule_num]; _compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather; struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL; struct Maat_scanner_t* my_scanner=NULL;
Maat_table_desc* p_table=NULL; Maat_table_desc* p_table=NULL;
struct timespec start,end; struct timespec start,end;
if(_feather->perf_on==1) if(_feather->perf_on==1)

16
src/entry/Maat_command.cpp
View File

@@ -836,8 +836,8 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
struct Maat_group_t* group_cmd=NULL; struct Maat_group_t* group_cmd=NULL;
struct Maat_region_t* region_cmd=NULL; struct Maat_region_t* region_cmd=NULL;
struct _Maat_compile_inner_t *compile_inner=NULL; struct Maat_compile_inner_t *compile_inner=NULL;
struct _Maat_group_inner_t* group_inner=NULL; struct Maat_group_inner_t* group_inner=NULL;
struct _Maat_region_inner_t* region_inner=NULL; struct _Maat_region_inner_t* region_inner=NULL;
void* logger=feather->logger; void* logger=feather->logger;
@@ -848,7 +848,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
,"MAAT not ready."); ,"MAAT not ready.");
return -1; return -1;
} }
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id); compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
//Operation on compile_inner is thread safe, no immediate memory free when delete a compile rule or a scanner. //Operation on compile_inner is thread safe, no immediate memory free when delete a compile rule or a scanner.
//In another words, if the compile_inner is accessable from compile means, its was valid in at least 10 seconds (garbage bury). //In another words, if the compile_inner is accessable from compile means, its was valid in at least 10 seconds (garbage bury).
if(compile_inner==NULL) if(compile_inner==NULL)
@@ -864,7 +864,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
cmd->groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),cmd->group_num); cmd->groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),cmd->group_num);
for(i=0;i<compile_inner->group_boundary;i++) for(i=0;i<compile_inner->group_boundary;i++)
{ {
group_inner=(struct _Maat_group_inner_t*)dynamic_array_read(compile_inner->groups,i); group_inner=(struct Maat_group_inner_t*)dynamic_array_read(compile_inner->groups,i);
if(group_inner==NULL) if(group_inner==NULL)
{ {
continue; continue;
@@ -1337,10 +1337,10 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
struct Maat_region_t* p_region=NULL; struct Maat_region_t* p_region=NULL;
enum MAAT_TABLE_TYPE table_type; enum MAAT_TABLE_TYPE table_type;
struct _Maat_compile_inner_t *compile_rule=NULL; struct Maat_compile_inner_t *compile_rule=NULL;
if(feather->scanner!=NULL) if(feather->scanner!=NULL)
{ {
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id); compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
if(compile_rule!=NULL) if(compile_rule!=NULL)
{ {
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
@@ -2035,8 +2035,8 @@ int Maat_cmd_set_group(Maat_feather_t feather,int group_id, const struct Maat_re
if(_feather->AUTO_NUMBERING_ON==1) if(_feather->AUTO_NUMBERING_ON==1)
{ {
return -1; return -1;
} }
//struct _Maat_group_inner_t* group_inner=NULL; //struct Maat_group_inner_t* group_inner=NULL;
//group_inner=(struct Maat_group_inner_t*)HASH_fetch_by_id(_feather->scanner->group_hash, group_id); //group_inner=(struct Maat_group_inner_t*)HASH_fetch_by_id(_feather->scanner->group_hash, group_id);
//NOT implemented yet. //NOT implemented yet.
assert(0); assert(0);

271
src/entry/Maat_rule.cpp
View File

@@ -28,7 +28,7 @@
#include "map_str2int.h" #include "map_str2int.h"
#include "rulescan.h" #include "rulescan.h"
#include "UniversalBoolMatch.h" #include "bool_matcher.h"
#include "stream_fuzzy_hash.h" #include "stream_fuzzy_hash.h"
#include "gram_index_engine.h" #include "gram_index_engine.h"
@@ -65,7 +65,7 @@ int is_valid_match_method(enum MAAT_MATCH_METHOD match_method)
} }
iconv_t maat_iconv_open(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from) iconv_t maat_iconv_open(struct Maat_scanner_t* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from)
{ {
const char *from_s=CHARSET_STRING[from]; const char *from_s=CHARSET_STRING[from];
const char *to_s=CHARSET_STRING[to]; const char *to_s=CHARSET_STRING[to];
@@ -87,7 +87,7 @@ iconv_t maat_iconv_open(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET to,enu
return cd; return cd;
} }
int iconv_convert(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen) int iconv_convert(struct Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen)
{ {
size_t ret; size_t ret;
int copy_len=0; int copy_len=0;
@@ -206,7 +206,7 @@ int uni2ascii(const char* fmt,const char* src, const int srclen, char* dst, cons
} }
return j; return j;
} }
int universal_charset_convert(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen) int universal_charset_convert(struct Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen)
{ {
int ret=0; int ret=0;
char* tmp_buff=NULL; char* tmp_buff=NULL;
@@ -836,9 +836,9 @@ error_jump:
map_destroy(string2int_map); map_destroy(string2int_map);
return table_cnt; return table_cnt;
} }
struct _Maat_group_inner_t* create_group_rule(int group_id) struct Maat_group_inner_t* create_group_rule(int group_id)
{ {
struct _Maat_group_inner_t* group=(struct _Maat_group_inner_t*)malloc(sizeof(struct _Maat_group_inner_t)); struct Maat_group_inner_t* group=(struct Maat_group_inner_t*)malloc(sizeof(struct Maat_group_inner_t));
group->group_id=group_id; group->group_id=group_id;
group->region_cnt=0; group->region_cnt=0;
group->region_boundary=0; group->region_boundary=0;
@@ -850,7 +850,7 @@ struct _Maat_group_inner_t* create_group_rule(int group_id)
pthread_mutex_init(&(group->mutex), NULL); pthread_mutex_init(&(group->mutex), NULL);
return group; return group;
} }
void _destroy_group_rule(struct _Maat_group_inner_t* group) void _destroy_group_rule(struct Maat_group_inner_t* group)
{ {
dynamic_array_destroy(group->regions,free); dynamic_array_destroy(group->regions,free);
group->region_cnt=0; group->region_cnt=0;
@@ -865,7 +865,7 @@ void _destroy_group_rule(struct _Maat_group_inner_t* group)
free(group); free(group);
} }
void destroy_group_rule(struct _Maat_group_inner_t* group) void destroy_group_rule(struct Maat_group_inner_t* group)
{ {
if(group->ref_cnt>0||group->region_cnt>0) if(group->ref_cnt>0||group->region_cnt>0)
@@ -874,30 +874,42 @@ void destroy_group_rule(struct _Maat_group_inner_t* group)
} }
_destroy_group_rule(group); _destroy_group_rule(group);
} }
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set) void make_group_set(struct Maat_compile_inner_t* compile_rule, struct bool_expr* a_set, unsigned char *has_not)
{ {
int i=0,j=0; int i=0,j=0;
a_set->bool_expr_id=(void*)compile_rule; a_set->user_tag=compile_rule;
struct _Maat_group_inner_t*group=NULL; struct Maat_group_inner_t*group=NULL;
assert(compile_rule->group_cnt<=MAX_ITEMS_PER_BOOL_EXPR); assert(compile_rule->group_cnt<=MAX_ITEMS_PER_BOOL_EXPR);
for(i=0,j=0;i<compile_rule->group_boundary&&j<MAX_ITEMS_PER_BOOL_EXPR;i++) for(i=0,j=0;i<compile_rule->group_boundary&&j<MAX_ITEMS_PER_BOOL_EXPR;i++)
{ {
group=(struct _Maat_group_inner_t*)dynamic_array_read(compile_rule->groups,i); group=(struct Maat_group_inner_t*)dynamic_array_read(compile_rule->groups, i);
if(group==NULL) if(group==NULL)
{ {
continue; continue;
} }
a_set->bool_item_ids[j]=group->group_id; a_set->items[j].item_id=group->group_id;
a_set->items[j].not_flag=compile_rule->not_flag[j];
if(a_set->items[j].not_flag)
{
*has_not=1;
}
j++; j++;
} }
assert(j==compile_rule->group_cnt); assert(j==compile_rule->group_cnt);
a_set->bool_item_num=j; a_set->item_num=j;
} }
struct compile_walker
{
MESA_lqueue_head update_q;
long long compile_has_not_flag;
};
void walk_compile_hash(const uchar * key, uint size, void * data, void * user) void walk_compile_hash(const uchar * key, uint size, void * data, void * user)
{ {
universal_bool_expr_t* one_set=NULL; struct bool_expr* one_set=NULL;
struct _Maat_compile_inner_t* compile_rule=(struct _Maat_compile_inner_t*)data; struct Maat_compile_inner_t* compile_rule=(struct Maat_compile_inner_t*)data;
MESA_lqueue_head update_q=(MESA_lqueue_head)user; struct compile_walker* walker=(struct compile_walker*)user;
unsigned char has_not_flag=0;
MESA_lqueue_head update_q=walker->update_q;
if(compile_rule->db_c_rule==NULL) if(compile_rule->db_c_rule==NULL)
{ {
return; return;
@@ -907,67 +919,73 @@ void walk_compile_hash(const uchar * key, uint size, void * data, void * user)
||compile_rule->db_c_rule->declare_grp_num==0)//for compatible old version ||compile_rule->db_c_rule->declare_grp_num==0)//for compatible old version
&&compile_rule->group_cnt>0) &&compile_rule->group_cnt>0)
{ {
one_set=(universal_bool_expr_t*)malloc(sizeof(universal_bool_expr_t)); one_set=ALLOC(struct bool_expr, 1);
//reading compile rule is safe in update thread, mutex lock called when modified //reading compile rule is safe in update thread, mutex lock called when modified
make_group_set(compile_rule, one_set); make_group_set(compile_rule, one_set, &has_not_flag);
MESA_lqueue_join_tail(update_q,&one_set, sizeof(void*));//put the pointer into queue if(has_not_flag)
{
walker->compile_has_not_flag++;
}
MESA_lqueue_join_tail(update_q, &one_set, sizeof(one_set));//put the pointer into queue
} }
return; return;
} }
void* create_bool_matcher(MESA_htable_handle compile_hash,int thread_num,void* logger) struct bool_matcher* create_bool_matcher(MESA_htable_handle compile_hash, int thread_num, void* logger)
{ {
void* bool_matcher=NULL; struct bool_matcher* bm=NULL;
MESA_lqueue_head update_q=MESA_lqueue_create(0,0);; struct compile_walker walker={NULL, 0};
walker.update_q=MESA_lqueue_create(0,0);
MESA_lqueue_head update_q=walker.update_q;
long data_size=0; long data_size=0;
unsigned int mem_size=0; size_t mem_size=0;
UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK;
data_size=sizeof(void*); data_size=sizeof(void*);
universal_bool_expr_t* one_set=NULL; struct bool_expr* one_set=NULL;
universal_bool_expr_t* set_array=NULL; struct bool_expr* set_array=NULL;
int i=0; int i=0;
MESA_htable_iterate(compile_hash, walk_compile_hash, update_q); MESA_htable_iterate(compile_hash, walk_compile_hash, &walker);
const long q_cnt=MESA_lqueue_get_count(update_q); const long q_cnt=MESA_lqueue_get_count(update_q);
if(q_cnt==0) if(q_cnt==0)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
"No compile rule to build a bool matcher."); "No compile rule to build a bool matcher.");
MESA_lqueue_destroy(update_q,lqueue_destroy_cb,NULL); MESA_lqueue_destroy(update_q, lqueue_destroy_cb, NULL);
return NULL; return NULL;
} }
set_array=(universal_bool_expr_t*)malloc(sizeof(universal_bool_expr_t)*q_cnt); set_array=ALLOC(struct bool_expr, q_cnt);
for(i=0;i<q_cnt;i++) for(i=0; i<q_cnt; i++)
{ {
q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(update_q,&one_set,&data_size); q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(update_q, &one_set, &data_size);
assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK); assert(data_size==sizeof(struct bool_expr*) && q_ret==MESA_QUEUE_RET_OK);
memcpy(set_array+i,one_set,sizeof(universal_bool_expr_t)); set_array[i]=*one_set;
free(one_set); free(one_set);
one_set=NULL; one_set=NULL;
} }
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, MESA_handle_runtime_log(logger, RLOG_LV_INFO,maat_module,
"build bool matcher start contain %ld compile rule", "build bool matcher start: compile count %ld, NOT-logic count %lld",
q_cnt); q_cnt, walker.compile_has_not_flag);
bool_matcher=boolexpr_initialize(set_array, q_cnt, thread_num, &mem_size); bm=bool_matcher_new(set_array, q_cnt, thread_num, &mem_size);
if(bool_matcher!=NULL) if(bm!=NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
"build bool matcher use %u memory",mem_size); "build bool matcher use %zu memory", mem_size);
} }
else else
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module, MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module,
"build bool matcher failed!", "build bool matcher failed!",
q_cnt,mem_size); q_cnt);
} }
free(set_array); free(set_array);
set_array=NULL; set_array=NULL;
MESA_lqueue_destroy(update_q,lqueue_destroy_cb,NULL); MESA_lqueue_destroy(update_q, lqueue_destroy_cb, NULL);
return bool_matcher; return bm;
} }
void destroy_bool_matcher(void * bool_matcher) void destroy_bool_matcher(struct bool_matcher * bm)
{ {
boolexpr_destroy(bool_matcher); bool_matcher_free(bm);
return; return;
} }
@@ -975,9 +993,9 @@ void EMPTY_FREE(void*p)
{ {
return; return;
} }
struct _Maat_compile_inner_t * create_compile_rule(int compile_id) struct Maat_compile_inner_t * create_compile_rule(int compile_id)
{ {
struct _Maat_compile_inner_t* p=ALLOC(struct _Maat_compile_inner_t,1); struct Maat_compile_inner_t* p=ALLOC(struct Maat_compile_inner_t,1);
p->compile_id=compile_id; p->compile_id=compile_id;
p->group_cnt=0; p->group_cnt=0;
p->group_boundary=1; p->group_boundary=1;
@@ -986,7 +1004,7 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id)
pthread_rwlock_init(&(p->rwlock), NULL); pthread_rwlock_init(&(p->rwlock), NULL);
return p; return p;
} }
void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule) void _destroy_compile_rule(struct Maat_compile_inner_t * compile_rule)
{ {
const struct Maat_table_desc* table=compile_rule->ref_table; const struct Maat_table_desc* table=compile_rule->ref_table;
const struct compile_table_desc* compile_desc=&(table->compile); const struct compile_table_desc* compile_desc=&(table->compile);
@@ -1016,14 +1034,14 @@ void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule)
pthread_rwlock_destroy(&(compile_rule->rwlock)); pthread_rwlock_destroy(&(compile_rule->rwlock));
free(compile_rule); free(compile_rule);
} }
void destroy_compile_rule(struct _Maat_compile_inner_t * p) void destroy_compile_rule(struct Maat_compile_inner_t * p)
{ {
int i=0; int i=0;
UNUSED struct _Maat_compile_inner_t* p_group=NULL; UNUSED struct Maat_compile_inner_t* p_group=NULL;
assert(p->group_cnt==0); assert(p->group_cnt==0);
for(i=0;i<p->group_boundary;i++) for(i=0;i<p->group_boundary;i++)
{ {
p_group=(struct _Maat_compile_inner_t*)dynamic_array_read(p->groups,i); p_group=(struct Maat_compile_inner_t*)dynamic_array_read(p->groups,i);
assert(p_group==NULL); assert(p_group==NULL);
} }
_destroy_compile_rule(p); _destroy_compile_rule(p);
@@ -1154,7 +1172,7 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule)
return; return;
} }
GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest, GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest,
short cfds_lvl,struct _Maat_group_inner_t* tag) short cfds_lvl,struct Maat_group_inner_t* tag)
{ {
GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1); GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1);
int digest_len=0; int digest_len=0;
@@ -1269,7 +1287,7 @@ void table_runtime_free(struct Maat_table_runtime* p)
return; return;
} }
struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t *feather) struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t *feather)
{ {
int scan_thread_num=feather->scan_thread_num; int scan_thread_num=feather->scan_thread_num;
// int rs_scan_type=feather->rule_scan_type; // int rs_scan_type=feather->rule_scan_type;
@@ -1295,8 +1313,8 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
hargs.data_expire_with_condition = NULL; hargs.data_expire_with_condition = NULL;
struct _Maat_scanner_t* scanner=NULL; struct Maat_scanner_t* scanner=NULL;
scanner=ALLOC(struct _Maat_scanner_t, 1); scanner=ALLOC(struct Maat_scanner_t, 1);
//Function Maat_cmd_append will access compile_hash in user thread. //Function Maat_cmd_append will access compile_hash in user thread.
hargs.thread_safe=1; hargs.thread_safe=1;
@@ -1368,7 +1386,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
void destroy_maat_scanner(struct _Maat_scanner_t*scanner) void destroy_maat_scanner(struct Maat_scanner_t*scanner)
{ {
long q_cnt=0,data_size=0; long q_cnt=0,data_size=0;
int i=0,j=0; int i=0,j=0;
@@ -1385,7 +1403,7 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
map_destroy(scanner->district_map); map_destroy(scanner->district_map);
scanner->district_map=NULL; scanner->district_map=NULL;
assert(scanner->tmp_district_map==NULL); assert(scanner->tmp_district_map==NULL);
destroy_bool_matcher((void*)scanner->bool_macher_expr_compiler); destroy_bool_matcher(scanner->bool_macher_expr_compiler);
q_cnt=MESA_lqueue_get_count(scanner->region_update_q); q_cnt=MESA_lqueue_get_count(scanner->region_update_q);
for(i=0;i<q_cnt;i++) for(i=0;i<q_cnt;i++)
{ {
@@ -1494,7 +1512,7 @@ void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_sta
return; return;
} }
void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct _Maat_scanner_t* maat_scanner) void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner_t* maat_scanner)
{ {
long i=0,data_size=0; long i=0,data_size=0;
unsigned int j=0; unsigned int j=0;
@@ -1602,7 +1620,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
free(to_update_expr); free(to_update_expr);
} }
void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct _Maat_scanner_t* maat_scanner,int table_id) void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct Maat_scanner_t* maat_scanner,int table_id)
{ {
long i=0,data_size=0; long i=0,data_size=0;
int ret=0; int ret=0;
@@ -1648,7 +1666,7 @@ void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*log
update_array=NULL; update_array=NULL;
return; return;
} }
struct _Maat_group_inner_t* add_region_to_group(struct _Maat_group_inner_t* group,int table_id,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type) struct Maat_group_inner_t* add_region_to_group(struct Maat_group_inner_t* group,int table_id,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type)
{ {
int i=0; int i=0;
struct _Maat_region_inner_t* region_rule=NULL; struct _Maat_region_inner_t* region_rule=NULL;
@@ -1687,7 +1705,7 @@ struct _Maat_group_inner_t* add_region_to_group(struct _Maat_group_inner_t* grou
} }
return group; return group;
} }
void cancel_last_region_from_group(struct _Maat_group_inner_t* group,int region_id,int expr_id) void cancel_last_region_from_group(struct Maat_group_inner_t* group,int region_id,int expr_id)
{ {
struct _Maat_region_inner_t* region_rule=NULL; struct _Maat_region_inner_t* region_rule=NULL;
pthread_mutex_lock(&(group->mutex)); pthread_mutex_lock(&(group->mutex));
@@ -1708,7 +1726,7 @@ void cancel_last_region_from_group(struct _Maat_group_inner_t* group,int region_
pthread_mutex_unlock(&(group->mutex)); pthread_mutex_unlock(&(group->mutex));
return; return;
} }
unsigned int del_region_from_group(struct _Maat_group_inner_t* group,int region_id,unsigned int *output_expr_id,int output_size) unsigned int del_region_from_group(struct Maat_group_inner_t* group,int region_id,unsigned int *output_expr_id,int output_size)
{ {
int i=0,j=0; int i=0,j=0;
struct _Maat_region_inner_t* region_rule=NULL; struct _Maat_region_inner_t* region_rule=NULL;
@@ -1740,11 +1758,11 @@ unsigned int del_region_from_group(struct _Maat_group_inner_t* group,int region_
return j; return j;
} }
int add_group_to_compile(struct _Maat_compile_inner_t*a_compile_rule,struct _Maat_group_inner_t* a_rule_group, int not_flag) int add_group_to_compile(struct Maat_compile_inner_t*a_compile_rule,struct Maat_group_inner_t* a_rule_group, int not_flag)
{ {
int i=0,ret=-1; int i=0,ret=-1;
int write_pos=-1; int write_pos=-1;
struct _Maat_group_inner_t* p=NULL; struct Maat_group_inner_t* p=NULL;
pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); pthread_rwlock_wrlock(&(a_compile_rule->rwlock));
if(a_compile_rule->db_c_rule!=NULL if(a_compile_rule->db_c_rule!=NULL
&&a_compile_rule->group_cnt>=a_compile_rule->db_c_rule->declare_grp_num &&a_compile_rule->group_cnt>=a_compile_rule->db_c_rule->declare_grp_num
@@ -1756,7 +1774,7 @@ int add_group_to_compile(struct _Maat_compile_inner_t*a_compile_rule,struct _Maa
for(i=0;i<a_compile_rule->group_boundary;i++) for(i=0;i<a_compile_rule->group_boundary;i++)
{ {
p=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i); p=(struct Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
if(p==NULL) if(p==NULL)
{ {
write_pos=i; write_pos=i;
@@ -1806,7 +1824,7 @@ int add_group_to_compile(struct _Maat_compile_inner_t*a_compile_rule,struct _Maa
{ {
for(i=0;i<a_compile_rule->group_boundary;i++) for(i=0;i<a_compile_rule->group_boundary;i++)
{ {
p=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i); p=(struct Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
if(p!=NULL) if(p!=NULL)
{ {
p->compile_shortcut=NULL; p->compile_shortcut=NULL;
@@ -1819,14 +1837,14 @@ error_out:
return ret; return ret;
} }
struct _Maat_group_inner_t* del_group_from_compile(struct _Maat_compile_inner_t*a_compile_rule,int group_id) struct Maat_group_inner_t* del_group_from_compile(struct Maat_compile_inner_t*a_compile_rule,int group_id)
{ {
int i=0; int i=0;
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); pthread_rwlock_wrlock(&(a_compile_rule->rwlock));
for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++) for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++)
{ {
group_rule=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i); group_rule=(struct Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
if(group_rule==NULL) if(group_rule==NULL)
{ {
continue; continue;
@@ -1835,6 +1853,7 @@ struct _Maat_group_inner_t* del_group_from_compile(struct _Maat_compile_inner_t*
{ {
group_rule->ref_cnt--; group_rule->ref_cnt--;
dynamic_array_write(a_compile_rule->groups,i,NULL); dynamic_array_write(a_compile_rule->groups,i,NULL);
a_compile_rule->not_flag[i]=0;
a_compile_rule->group_cnt--; a_compile_rule->group_cnt--;
pthread_rwlock_unlock(&(a_compile_rule->rwlock)); pthread_rwlock_unlock(&(a_compile_rule->rwlock));
return group_rule; return group_rule;
@@ -1871,7 +1890,7 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n
} }
return 1; return 1;
} }
int get_district_id(_Maat_scanner_t *scanner,const char* district_str) int get_district_id(Maat_scanner_t *scanner,const char* district_str)
{ {
int map_ret=0,district_id=-1; int map_ret=0,district_id=-1;
map_ret=map_str2int(scanner->district_map, district_str,&district_id); map_ret=map_str2int(scanner->district_map, district_str,&district_id);
@@ -1891,7 +1910,7 @@ int get_district_id(_Maat_scanner_t *scanner,const char* district_str)
} }
return district_id; return district_id;
} }
int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger) int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct Maat_scanner_t *scanner,void* logger)
{ {
unsigned int i=0,j=0; unsigned int i=0,j=0;
char* p=NULL,*saveptr=NULL,*region_string=NULL; char* p=NULL,*saveptr=NULL,*region_string=NULL;
@@ -1899,7 +1918,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
int expr_id=0,district_id=-1; int expr_id=0,district_id=-1;
struct expr_table_desc* expr_desc=&(table->expr); struct expr_table_desc* expr_desc=&(table->expr);
scan_rule_t*p_rule=NULL; scan_rule_t*p_rule=NULL;
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
enum MAAT_CHARSET dst_charset=CHARSET_NONE; enum MAAT_CHARSET dst_charset=CHARSET_NONE;
char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM]; char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM];
int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1}; int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1};
@@ -1910,7 +1929,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
} }
int sub_expr_cnt=0; int sub_expr_cnt=0;
struct op_expr_t *op_expr=NULL; struct op_expr_t *op_expr=NULL;
struct _Maat_group_inner_t* u_para=NULL; struct Maat_group_inner_t* u_para=NULL;
if(table->table_type==TABLE_TYPE_EXPR_PLUS) if(table->table_type==TABLE_TYPE_EXPR_PLUS)
{ {
@@ -1918,7 +1937,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
str_unescape(db_rule->district); str_unescape(db_rule->district);
district_id=get_district_id(scanner, db_rule->district); district_id=get_district_id(scanner, db_rule->district);
} }
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(db_rule->group_id); group_rule=create_group_rule(db_rule->group_id);
@@ -2155,15 +2174,15 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
} }
return 0; return 0;
} }
int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger) int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
scan_rule_t* p_rule=NULL; scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL; struct op_expr_t* op_expr=NULL;
struct _Maat_group_inner_t* u_para=NULL; struct Maat_group_inner_t* u_para=NULL;
int expr_id=0,district_id=-1; int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id); group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(db_ip_rule->group_id); group_rule=create_group_rule(db_ip_rule->group_id);
@@ -2187,15 +2206,15 @@ int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,st
MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*));
return 0; return 0;
} }
int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger) int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intval_rule,struct Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
scan_rule_t* p_rule=NULL; scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL; struct op_expr_t* op_expr=NULL;
struct _Maat_group_inner_t* u_para=NULL; struct Maat_group_inner_t* u_para=NULL;
int expr_id=0,district_id=-1; int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id); group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(intval_rule->group_id); group_rule=create_group_rule(intval_rule->group_id);
@@ -2218,15 +2237,15 @@ int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intva
MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*)); MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*));
return 0; return 0;
} }
int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger) int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_digest_rule,struct Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
GIE_digest_t* digest_rule=NULL; GIE_digest_t* digest_rule=NULL;
struct _Maat_group_inner_t* u_para=NULL; struct Maat_group_inner_t* u_para=NULL;
struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id]; struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id];
int expr_id=0,district_id=-1; int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(db_digest_rule->group_id); group_rule=create_group_rule(db_digest_rule->group_id);
@@ -2250,15 +2269,15 @@ int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_di
scanner->gie_total_q_size++; scanner->gie_total_q_size++;
return 0; return 0;
} }
int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger) int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct Maat_scanner_t *maat_scanner,void* logger)
{ {
int i=0; int i=0;
unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0}; unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0};
int expr_num=0; int expr_num=0;
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
struct op_expr_t* op_expr=NULL; struct op_expr_t* op_expr=NULL;
GIE_digest_t* digest_rule=NULL; GIE_digest_t* digest_rule=NULL;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id); group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
@@ -2314,12 +2333,12 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int
} }
return 0; return 0;
} }
int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
struct _Maat_compile_inner_t*compile_rule=NULL; struct Maat_compile_inner_t*compile_rule=NULL;
int ret=0; int ret=0;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
group_rule=create_group_rule(db_group_rule->group_id); group_rule=create_group_rule(db_group_rule->group_id);
@@ -2329,7 +2348,7 @@ int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_grou
assert(ret>=0); assert(ret>=0);
} }
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
compile_rule=create_compile_rule(db_group_rule->compile_id); compile_rule=create_compile_rule(db_group_rule->compile_id);
@@ -2349,11 +2368,11 @@ int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_grou
return 0; return 0;
} }
void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger) void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_compile_inner_t*compile_rule=NULL; struct Maat_compile_inner_t*compile_rule=NULL;
struct _Maat_group_inner_t* group_rule=NULL; struct Maat_group_inner_t* group_rule=NULL;
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id); compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
@@ -2384,12 +2403,12 @@ void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_gro
} }
return; return;
} }
int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_compile_inner_t *compile_rule=NULL; struct Maat_compile_inner_t *compile_rule=NULL;
struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head); struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head);
int i=0; int i=0;
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
compile_rule=create_compile_rule(p_maat_rule_head->config_id); compile_rule=create_compile_rule(p_maat_rule_head->config_id);
@@ -2416,10 +2435,10 @@ int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_
return 0; return 0;
} }
int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger) int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct Maat_scanner_t *scanner,void* logger)
{ {
struct _Maat_compile_inner_t *compile_rule=NULL; struct Maat_compile_inner_t *compile_rule=NULL;
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
@@ -2439,7 +2458,7 @@ int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_
} }
return 1; return 1;
} }
void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger) void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger)
{ {
struct db_group_rule_t db_group_rule; struct db_group_rule_t db_group_rule;
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
@@ -2496,7 +2515,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru
return; return;
} }
void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger) void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct Maat_scanner_t *scanner,void* logger)
{ {
char virtual_group_line[256]; char virtual_group_line[256];
snprintf(virtual_group_line,sizeof(virtual_group_line), snprintf(virtual_group_line,sizeof(virtual_group_line),
@@ -2504,7 +2523,7 @@ void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int com
update_group_rule(table, virtual_group_line,scanner,logger); update_group_rule(table, virtual_group_line,scanner,logger);
return; return;
} }
void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on)
{ {
struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1); struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1);
int ret=0,db_hexbin=0,rule_type=0; int ret=0,db_hexbin=0,rule_type=0;
@@ -2676,7 +2695,7 @@ error_out:
free(maat_str_rule); free(maat_str_rule);
maat_str_rule=NULL; maat_str_rule=NULL;
} }
void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on)
{ {
struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1); struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1);
char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40]; char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40];
@@ -2820,7 +2839,7 @@ error_out:
ip_rule=NULL; ip_rule=NULL;
} }
void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on)
{ {
struct db_intval_rule_t* intval_rule=ALLOC(struct db_intval_rule_t, 1); struct db_intval_rule_t* intval_rule=ALLOC(struct db_intval_rule_t, 1);
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
@@ -2895,7 +2914,7 @@ error_out:
intval_rule=NULL; intval_rule=NULL;
} }
void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger)
{ {
struct compile_table_desc* compile_desc=&(table->compile); struct compile_table_desc* compile_desc=&(table->compile);
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
@@ -2981,7 +3000,7 @@ no_save:
return; return;
} }
void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on) void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on)
{ {
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct db_digest_rule_t* digest_rule=ALLOC(struct db_digest_rule_t, 1); struct db_digest_rule_t* digest_rule=ALLOC(struct db_digest_rule_t, 1);
@@ -3133,17 +3152,17 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
destroy_group_rule(bag->group_rule); destroy_group_rule(bag->group_rule);
break; break;
case GARBAGE_SCANNER: case GARBAGE_SCANNER:
ref_cnt=alignment_int64_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num); ref_cnt=alignment_int64_array_sum(bag->scanner->ref_cnt, bag->scanner->max_thread_num);
if(ref_cnt==0) if(ref_cnt==0)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"scanner %p version %d has no reference peacefully destroyed.",bag->scanner,bag->scanner->version); "scanner %p version %d has no reference peacefully destroyed.", bag->scanner, bag->scanner->version);
} }
else else
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module, MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"scanner %p version %d force destroyed,ref_cnt %lld.", "scanner %p version %d force destroyed, ref_cnt %lld.",
bag->scanner,bag->scanner->version,ref_cnt); bag->scanner, bag->scanner->version, ref_cnt);
} }
destroy_maat_scanner(bag->scanner); destroy_maat_scanner(bag->scanner);
break; break;
@@ -3185,7 +3204,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
q_cnt,bury_cnt); q_cnt,bury_cnt);
} }
} }
void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger) void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{ {
int i=0, ret=1; int i=0, ret=1;
unsigned int len=strlen(table_line)+1; unsigned int len=strlen(table_line)+1;
@@ -3270,9 +3289,9 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
table_rt->plugin.cache_line_num++; table_rt->plugin.cache_line_num++;
} }
} }
void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger) void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger)
{ {
void *tmp1=NULL,*tmp2=NULL; struct bool_matcher *tmp1=NULL,*tmp2=NULL;
MESA_htable_handle tmp_map=NULL; MESA_htable_handle tmp_map=NULL;
struct Maat_table_runtime* table_rt=NULL; struct Maat_table_runtime* table_rt=NULL;
int i=0; int i=0;
@@ -3281,8 +3300,8 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
para.gram_value=7; para.gram_value=7;
para.position_accuracy=10; para.position_accuracy=10;
tmp1=create_bool_matcher(scanner->compile_hash, tmp1=create_bool_matcher(scanner->compile_hash,
scan_thread_num, scan_thread_num,
logger); logger);
tmp2=scanner->bool_macher_expr_compiler; tmp2=scanner->bool_macher_expr_compiler;
//assume pinter = operation is thread safe //assume pinter = operation is thread safe
@@ -3296,8 +3315,8 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
scanner->dedup_expr_num=0; scanner->dedup_expr_num=0;
rulescan_batch_update(scanner->region, rulescan_batch_update(scanner->region,
scanner->region_update_q, scanner->region_update_q,
logger logger,
,scanner); scanner);
for(i=0;i<MAX_TABLE_NUM;i++) for(i=0;i<MAX_TABLE_NUM;i++)
{ {
table_rt=scanner->table_rt[i]; table_rt=scanner->table_rt[i];
@@ -3400,7 +3419,7 @@ void maat_start_cb(long long new_version,int update_type,void*u_para)
} }
return; return;
} }
long long scanner_rule_num(struct _Maat_scanner_t *scanner) long long scanner_rule_num(struct Maat_scanner_t *scanner)
{ {
long long total=0; long long total=0;
struct Maat_table_runtime* table_rt=NULL; struct Maat_table_runtime* table_rt=NULL;
@@ -3505,7 +3524,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
int ret=-1,i=0; int ret=-1,i=0;
int table_id=-1; int table_id=-1;
_Maat_scanner_t* scanner=NULL; Maat_scanner_t* scanner=NULL;
struct Maat_table_desc* p_table=NULL; struct Maat_table_desc* p_table=NULL;
if(feather->update_tmp_scanner!=NULL) if(feather->update_tmp_scanner!=NULL)
{ {
@@ -3564,7 +3583,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
void *thread_rule_monitor(void *arg) void *thread_rule_monitor(void *arg)
{ {
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg; struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg;
struct _Maat_scanner_t* old_scanner=NULL; struct Maat_scanner_t* old_scanner=NULL;
long expr_wait_q_cnt=0; long expr_wait_q_cnt=0;
int scan_dir_cnt=0; int scan_dir_cnt=0;
int ret=0; int ret=0;

38
src/inc_internal/Maat_rule_internal.h
View File

@@ -10,7 +10,7 @@
#include <MESA/field_stat2.h> #include <MESA/field_stat2.h>
#include <MESA/rulescan.h> #include <MESA/rulescan.h>
#include "dynamic_array.h" #include "dynamic_array.h"
#include "UniversalBoolMatch.h" #include "bool_matcher.h"
#include "hiredis.h" #include "hiredis.h"
#include "stream_fuzzy_hash.h" #include "stream_fuzzy_hash.h"
@@ -130,7 +130,7 @@ struct _Maat_region_inner_t
int expr_id_ub; int expr_id_ub;
enum MAAT_TABLE_TYPE table_type; enum MAAT_TABLE_TYPE table_type;
}; };
struct _Maat_group_inner_t struct Maat_group_inner_t
{ {
int group_id; int group_id;
int table_id; int table_id;
@@ -142,10 +142,10 @@ struct _Maat_group_inner_t
void* compile_shortcut; void* compile_shortcut;
pthread_mutex_t mutex; pthread_mutex_t mutex;
}; };
struct _Maat_compile_inner_t struct Maat_compile_inner_t
{ {
struct db_compile_rule_t *db_c_rule; struct db_compile_rule_t *db_c_rule;
dynamic_array_t *groups; //element is struct _Maat_group_inner_t* dynamic_array_t *groups; //element is struct Maat_group_inner_t*
char not_flag[MAX_ITEMS_PER_BOOL_EXPR]; char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
int is_valid; int is_valid;
int compile_id;//equal to db_c_rule->m_rule.config_id int compile_id;//equal to db_c_rule->m_rule.config_id
@@ -158,15 +158,15 @@ struct _Maat_compile_inner_t
struct _compile_result_t struct _compile_result_t
{ {
int compile_id; int compile_id;
universal_bool_expr_t group_set; struct bool_expr group_set;
}; };
struct _INNER_scan_status_t struct _INNER_scan_status_t
{ {
int cur_hit_cnt; size_t cur_hit_cnt;
int hit_group_cnt; size_t hit_group_cnt;
int hit_group_size; size_t hit_group_size;
unsigned int cur_hit_id[MAX_SCANNER_HIT_NUM]; unsigned long long cur_hit_id[MAX_SCANNER_HIT_NUM];
unsigned int *hitted_group_id; unsigned long long *hitted_group_id;
}; };
struct _OUTER_scan_status_t struct _OUTER_scan_status_t
{ {
@@ -264,7 +264,7 @@ struct rule_tag
char* tag_name; char* tag_name;
char* tag_val; char* tag_val;
}; };
struct _Maat_scanner_t struct Maat_scanner_t
{ {
long long version; long long version;
time_t last_update_time; time_t last_update_time;
@@ -284,7 +284,7 @@ struct _Maat_scanner_t
unsigned int exprid_generator; unsigned int exprid_generator;
unsigned int dedup_expr_num; unsigned int dedup_expr_num;
MESA_lqueue_head region_update_q; MESA_lqueue_head region_update_q;
void * bool_macher_expr_compiler; struct bool_matcher * bool_macher_expr_compiler;
scan_result_t *region_rslt_buff; scan_result_t *region_rslt_buff;
MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q
@@ -321,8 +321,8 @@ struct source_redis_ctx
}; };
struct _Maat_feather_t struct _Maat_feather_t
{ {
struct _Maat_scanner_t *scanner; struct Maat_scanner_t *scanner;
struct _Maat_scanner_t *update_tmp_scanner; struct Maat_scanner_t *update_tmp_scanner;
MESA_lqueue_head garbage_q; MESA_lqueue_head garbage_q;
int table_cnt; int table_cnt;
int DEFERRED_LOAD_ON; int DEFERRED_LOAD_ON;
@@ -409,10 +409,10 @@ struct _maat_garbage_t
int ok_times; int ok_times;
union union
{ {
struct _Maat_scanner_t* scanner; struct Maat_scanner_t* scanner;
struct _Maat_group_inner_t* group_rule; struct Maat_group_inner_t* group_rule;
struct _Maat_compile_inner_t* compile_rule; struct Maat_compile_inner_t* compile_rule;
void* bool_matcher; struct bool_matcher* bool_matcher;
void * raw; void * raw;
MESA_htable_handle str2int_map; MESA_htable_handle str2int_map;
char* filename; char* filename;
@@ -441,7 +441,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger);
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set); void make_group_set(struct Maat_compile_inner_t* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para); void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para);