gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature-compile-with-NOT-logic' into 'master'

Browse Source 
Feature compile with not logic

Closes #4

See merge request MESA_framework/maat!21
This commit is contained in:
郑超
2019-01-16 17:57:03 +08:00
parent 004e79d5d5 73ac91b6b9
commit 9e88e3274f
14 changed files with 1372 additions and 469 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
inc/Maat_rule.h
View File

@@ -159,8 +159,8 @@ enum MAAT_INIT_OPT
MAAT_OPT_ENABLE_UPDATE, //VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled. DEFAULT: Backgroud update is enabled. Runtime setting is allowed.
MAAT_OPT_ACCEPT_TAGS, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]}
MAAT_OPT_FOREIGN_CONT_DIR, //VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files
MAAT_OPT_FOREIGN_CONT_LINGER //VALUE is interger *, SIZE=sizeof(int). Greater than 0: delete after VALUE seconds; 0: delete foreign content right after the notification callbacks; Less than 0: NEVER delete. Default: 0.
};
MAAT_OPT_FOREIGN_CONT_LINGER //VALUE is interger *, SIZE=sizeof(int). Greater than 0: delete after VALUE seconds; 0: delete foreign content right after the notification callbacks; Less than 0: NEVER delete. Default: 0.
};
//return -1 if failed, return 0 on success;
int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const void* value,int size);
enum MAAT_STATE_OPT
@@ -258,6 +258,12 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
//returned data is duplicated by dup_func of Maat_rule_get_ex_new_index, caller is responsible to free the data.
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx);
//Helper function for parsing space or tab seperated line.
//Nth_column: the Nth column is numberd from 1.
//Return 0 if success.
int Maat_helper_read_column(const char* line, int Nth_column, size_t *column_offset, size_t *column_len);
//Following functions are similar to Maat_rule_get_ex_data, except they are effective on plugin table.
typedef void* MAAT_PLUGIN_EX_DATA;
typedef void Maat_plugin_EX_new_func_t(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp);

4
src/CMakeLists.txt
View File

@@ -1,14 +1,14 @@
cmake_minimum_required(VERSION 3.5)
set(MAAT_FRAME_MAJOR_VERSION 2)
set(MAAT_FRAME_MINOR_VERSION 5)
set(MAAT_FRAME_MINOR_VERSION 6)
set(MAAT_FRAME_PATCH_VERSION 0)
set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION})
message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
add_definitions(-fPIC)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/UniversalBoolMatch.cpp)
set(MAAT_SRC entry/cJSON.c entry/config_monitor.cpp entry/dynamic_array.cpp entry/gram_index_engine.c entry/interval_index.c entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp entry/Maat_command.cpp entry/Maat_rule.cpp entry/Maat_stat.cpp entry/map_str2int.cpp entry/rbtree.c entry/stream_fuzzy_hash.c entry/bool_matcher.cpp)
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
include_directories(/opt/MESA/include/MESA/)

266
src/entry/Maat_api.cpp
View File

@@ -5,7 +5,7 @@
#include <MESA/MESA_handle_logger.h>
#include "rulescan.h"
#include "UniversalBoolMatch.h"
#include "bool_matcher.h"
#include "Maat_rule.h"
#include "Maat_rule_internal.h"
#include "Maat_utils.h"
@@ -42,25 +42,59 @@ struct Maat_table_desc * acqurie_table(struct _Maat_feather_t* _feather,int tabl
}
return p_table;
}
inline void INC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
inline void INC_SCANNER_REF(Maat_scanner_t*scanner,int thread_num)
{
alignment_int64_array_add(scanner->ref_cnt, thread_num, 1);
return;
}
inline void DEC_SCANNER_REF(_Maat_scanner_t*scanner,int thread_num)
inline void DEC_SCANNER_REF(Maat_scanner_t*scanner,int thread_num)
{
alignment_int64_array_add(scanner->ref_cnt, thread_num, -1);
return;
}
void Maat_clean_status(scan_status_t* mid)
{
struct _OUTER_scan_status_t* _mid=NULL;
if(*mid==NULL)
{
return;
}
_mid=(struct _OUTER_scan_status_t*)(*mid);
alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL)
{
free(_mid->inner->hitted_group_id);
free(_mid->inner);
alignment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1);
}
_mid->feather=NULL;
free(_mid);
*mid=NULL;
return;
}
inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid)
{
if( _mid &&
_mid->is_last_region==1 &&
_mid->inner &&
_mid->inner->not_grp_compile_hitted_flag)
{
return 1;
}
else
{
return 0;
}
}
//return 1 if insert a unique id
//return 0 if id is duplicated
//return -1 if set is full
int insert_set_id(unsigned int **set,int* size,int cnt,unsigned int id)
int insert_set_id(unsigned long long **set, size_t* size, size_t cnt, unsigned long long id)
{
int i=0;
for(i=0;i<cnt;i++)
size_t i=0;
for(i=0; i<cnt; i++)
{
if((*set)[i]==id)
{
@@ -72,7 +106,7 @@ int insert_set_id(unsigned int **set,int* size,int cnt,unsigned int id)
if(cnt==*size)
{
*size+=16;
*set=(unsigned int*)realloc(*set,(*size)*sizeof(unsigned int));
*set=(unsigned long long*)realloc(*set, (*size)*sizeof(unsigned long long));
}
(*set)[cnt]=id;
return 1;
@@ -82,18 +116,20 @@ int insert_set_id(unsigned int **set,int* size,int cnt,unsigned int id)
return 0;
}
}
int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsigned int* hitted_id,int hit_cnt,int* region_pos,int size)
size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit,
const unsigned long long* hitted_id, size_t hit_cnt, int* region_pos, size_t size)
{
int i=0,j=0;
int k=0;
size_t i=0, j=0;
size_t k=0;
for(i=0;i<hit_cnt;i++)
{
for(j=0;j<(int)(compile_hit->bool_item_num);j++)
for(j=0; j<compile_hit->item_num; j++)
{
if(hitted_id[i]==compile_hit->bool_item_ids[j])
if(hitted_id[i]==compile_hit->items[j].item_id)
{
region_pos[k]=i;
k++;
assert(k<size);
}
}
}
@@ -105,30 +141,38 @@ void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* ru
memcpy(rule->service_defined, srv_def, MIN(srv_def_len,MAX_SERVICE_DEFINE_LEN));
return;
}
static int compare_compile_id(const void *a, const void *b)
{
struct Maat_rule_t *ra=(struct Maat_rule_t *)a;
struct Maat_rule_t *rb=(struct Maat_rule_t *)b;
return (rb->config_id-ra->config_id);
}
int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
{
int scan_ret=0,result_cnt=0;
int ret=0,i=0;
int r_in_c_cnt=0;
size_t r_in_c_cnt=0;
int shortcut_avilable_cnt=0;
void* bool_matcher=feather->scanner->bool_macher_expr_compiler;
struct _Maat_group_inner_t* group_rule=NULL;
struct _Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct _Maat_compile_inner_t* _mi_rule=NULL;
unsigned char has_not_flag=0;
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
struct Maat_group_inner_t* group_rule=NULL;
struct Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct Maat_compile_inner_t* _mi_rule=NULL;
int region_pos[MAX_SCANNER_HIT_NUM];
_mid->cur_hit_cnt=0;
for(i=0;i<region_hit_num;i++)
{
group_rule=*(struct _Maat_group_inner_t**)((char*)region_hit+region_type_size*i+group_offset);
group_rule=*(struct Maat_group_inner_t**)((char*)region_hit+region_type_size*i+group_offset);
if(group_rule->group_id<0)
{
continue;
}
if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
{
array_mi_rule[shortcut_avilable_cnt]=(struct _Maat_compile_inner_t*)(group_rule->compile_shortcut);
array_mi_rule[shortcut_avilable_cnt]=(struct Maat_compile_inner_t*)(group_rule->compile_shortcut);
shortcut_avilable_cnt++;
}
_mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id;
@@ -136,27 +180,28 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
ret=insert_set_id(&(_mid->hitted_group_id),
&(_mid->hit_group_size),
_mid->hit_group_cnt,
(unsigned int)group_rule->group_id);
group_rule->group_id);
_mid->hit_group_cnt+=ret;
}
if(shortcut_avilable_cnt==region_hit_num||shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM)
if((region_hit_num>0 &&shortcut_avilable_cnt==region_hit_num) ||
shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM)
{
//short cut for rules contains one group
scan_ret=shortcut_avilable_cnt;
alignment_int64_array_add(feather->orphan_group_saving, thread_num, 1);
}
else if(shortcut_avilable_cnt==0&&region_hit_num==1&&_mid->hit_group_cnt==1&&is_last_region==1)
else if(0&&shortcut_avilable_cnt==0&&region_hit_num==1&&_mid->hit_group_cnt==1&&is_last_region==1)
{
//This shortcut is NO longger valid after bool macher support NOT-logic.
//short cut for last scan and combination rules
//region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule.
//_mid->hit_group_cnt==1: With pre scan hitted group rules, one group may staisfy a compile rule
scan_ret=0;
alignment_int64_array_add(feather->last_region_saving, thread_num, 1);
}
else
{
scan_ret=boolexpr_match(bool_matcher,thread_num,
_mid->hitted_group_id,_mid->hit_group_cnt,
scan_ret=bool_matcher_match(bm, thread_num,
_mid->hitted_group_id, _mid->hit_group_cnt,
(void **)array_mi_rule, MAX_SCANNER_HIT_NUM);
}
for(i=0;i<scan_ret&&result_cnt<size;i++)
@@ -170,29 +215,44 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
if(0==pthread_rwlock_tryrdlock(&(_mi_rule->rwlock)))
{
if(_mi_rule->is_valid==1)
{
make_group_set(_mi_rule,&(rs_result[result_cnt].group_set));
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0)//compile config hitted becasue of new reigon
{
if(_mi_rule->not_group_cnt>0 && !is_last_region)
{
fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head),
_mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len);
rs_result[result_cnt].compile_id=_mi_rule->compile_id;
result_cnt++;
_mid->not_grp_compile_hitted_flag=1;
}
}
else
{
make_group_set(_mi_rule, &(rs_result[result_cnt].group_set), &has_not_flag);
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0 || //compile config hitted becasue of new reigon
_mid->cur_hit_cnt==0) //or ever hit a compile that refer a NOT-logic group
{
fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head),
_mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len);
rs_result[result_cnt].compile_id=_mi_rule->compile_id;
result_cnt++;
}
}
}
pthread_rwlock_unlock(&(_mi_rule->rwlock));
}
}
if(result_cnt>0)
{
qsort(result, result_cnt, sizeof(struct Maat_rule_t),
compare_compile_id);
alignment_int64_array_add(feather->hit_cnt,thread_num,1);
}
if(region_hit_num==0&&result_cnt>0)
{
alignment_int64_array_add(feather->not_grp_hit_cnt, thread_num, 1);
}
return result_cnt;
}
int exprid2region_id(struct _Maat_group_inner_t* group_rule,int expr_id,int* district_id)
int exprid2region_id(struct Maat_group_inner_t* group_rule,int expr_id,int* district_id)
{
int i=0,region_id=-1;
struct _Maat_region_inner_t* region_rule=NULL;
@@ -217,13 +277,13 @@ int exprid2region_id(struct _Maat_group_inner_t* group_rule,int expr_id,int* dis
}
int match_district(struct _OUTER_scan_status_t *_mid,scan_result_t *region_hit,int region_hit_num)
{
struct _Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
int i=0;
int district_id=-1,region_id=-1;
int ret_region_num=region_hit_num;
while(i<ret_region_num)
{
group_rule=(struct _Maat_group_inner_t*)(region_hit[i].tag);
group_rule=(struct Maat_group_inner_t*)(region_hit[i].tag);
region_id=exprid2region_id(group_rule, region_hit[i].expr_id,&district_id);
if(region_id>0&&district_id!=_mid->district_id)
{
@@ -319,19 +379,19 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
memset(r_in_c_flag,0,sizeof(r_in_c_flag));
memset(region_pos,0,sizeof(region_pos));
struct _Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
//for each hitted compile cfg,find its region_ids
for(i=0;i<compile_cnt&&i<detail_num;i++)
{
hit_detail[i].config_id=compile_hit[i].compile_id;
r_in_c_cnt=pickup_hit_region_from_compile(&(compile_hit[i].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt,
r_in_c_cnt=pickup_hit_region_from_compile(&(compile_hit[i].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
assert(r_in_c_cnt>0);//previous hitted compile was elimited in region_compile
for(j=0,k=0;j<r_in_c_cnt&&k<MAAT_MAX_HIT_RULE_NUM;j++)
{
pos=region_pos[j];
r_in_c_flag[pos]=1;
group_rule=(struct _Maat_group_inner_t*)(region_hit[pos].tag);
group_rule=(struct Maat_group_inner_t*)(region_hit[pos].tag);
region_id=exprid2region_id(group_rule,region_hit[pos].expr_id,&district_id);
if(region_id<0)
{
@@ -351,7 +411,7 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
{
if(r_in_c_flag[k]==0)
{
group_rule=(struct _Maat_group_inner_t*)(region_hit[k].tag);
group_rule=(struct Maat_group_inner_t*)(region_hit[k].tag);
hit_detail[j].config_id=-2;
hit_detail[j].hit_region_cnt=1;
hit_detail[j].region_pos[0].region_id=exprid2region_id(group_rule,region_hit[k].expr_id,&district_id);
@@ -366,14 +426,14 @@ int fill_region_hit_detail(const char* scan_buff,const _INNER_scan_status_t* _mi
struct _INNER_scan_status_t* _make_inner_status(void)
{
struct _INNER_scan_status_t* inner_mid=NULL;
inner_mid=(struct _INNER_scan_status_t*)calloc(sizeof(struct _INNER_scan_status_t),1);
inner_mid=ALLOC(struct _INNER_scan_status_t, 1);
inner_mid->cur_hit_cnt=0;
inner_mid->hit_group_cnt=0;
inner_mid->hit_group_size=4;
inner_mid->hitted_group_id=(unsigned int*)malloc(sizeof(unsigned int)*inner_mid->hit_group_size);
inner_mid->hitted_group_id= ALLOC(unsigned long long, inner_mid->hit_group_size);
return inner_mid;
}
struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather,int thread_num)
struct _OUTER_scan_status_t* _make_outer_status(_Maat_feather_t *feather, int thread_num)
{
struct _OUTER_scan_status_t* outer_mid=NULL;
outer_mid=(struct _OUTER_scan_status_t*)calloc(sizeof(struct _OUTER_scan_status_t),1);
@@ -484,7 +544,7 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
feather->inner_mid_cnt=alignment_int64_array_alloc(max_thread_num);
feather->hit_cnt=alignment_int64_array_alloc(max_thread_num);
feather->orphan_group_saving=alignment_int64_array_alloc(max_thread_num);
feather->last_region_saving=alignment_int64_array_alloc(max_thread_num);
feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num);
feather->maat_version=0;
feather->last_full_version=0;
feather->base_grp_seq=0;
@@ -592,7 +652,7 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
md5_file(_feather->json_ctx.json_file, _feather->json_ctx.effective_json_md5);
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Maat initial with JSON file %s md5: %s,generate index file %s OK.",
"Maat initial with JSON file %s md5: %s, generate index file %s OK.",
_feather->json_ctx.json_file,
_feather->json_ctx.effective_json_md5,
_feather->json_ctx.iris_file);
@@ -689,6 +749,7 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
{
return -1;
}
_feather->accept_tags_raw=_maat_strdup((const char*) value);
break;
case MAAT_OPT_FOREIGN_CONT_DIR:
memset(_feather->foreign_cont_dir, 0, sizeof(_feather->foreign_cont_dir));
@@ -809,10 +870,14 @@ int Maat_initiate_feather(Maat_feather_t feather)
}
if(_feather->cumulative_update_off==1)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Update with cumulative version OFF.");
}
if(_feather->n_tags>0)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
"Accept tags: %s", _feather->accept_tags_raw);
}
if(_feather->stat_on==1)
{
maat_stat_init(_feather);
@@ -977,7 +1042,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
{
struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
struct _Maat_compile_inner_t *compile_inner=(struct _Maat_compile_inner_t *)data;
struct Maat_compile_inner_t *compile_inner=(struct Maat_compile_inner_t *)data;
MAAT_RULE_EX_DATA ad=NULL;
if(compile_inner->ref_table==NULL || compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL)
@@ -1039,12 +1104,12 @@ failed:
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
{
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct _Maat_compile_inner_t *compile_inner=NULL;
struct Maat_compile_inner_t *compile_inner=NULL;
const struct compile_table_desc* compile_desc=NULL;
const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL;
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL)
{
return NULL;
@@ -1272,7 +1337,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
struct Maat_table_desc *p_table=NULL;
struct expr_table_desc* expr_desc=NULL;
struct timespec start,end;
_Maat_scanner_t* my_scanner=NULL;
Maat_scanner_t* my_scanner=NULL;
if(data==NULL||data_len<=0)
{
return 0;
@@ -1346,9 +1411,12 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
hit_region_cnt=match_district(_mid,region_result,hit_region_cnt);
}
if(hit_region_cnt>0)
if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
if(hit_region_cnt>0)
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1);
}
_mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -1361,7 +1429,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
_mid->is_last_region=2;
}
if(hit_detail!=NULL&&_feather->rule_scan_type!=0)
if(hit_region_cnt>0&&hit_detail!=NULL&&_feather->rule_scan_type!=0)
{
*detail_ret=fill_region_hit_detail(data,_mid->inner,
region_result,hit_region_cnt,
@@ -1408,7 +1476,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
scan_result_t *region_result=NULL;
_compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL;
struct Maat_scanner_t* my_scanner=NULL;
intval_scan_data.rule_type=RULETYPE_INT;
intval_scan_data.sub_type=make_sub_type(table_id,CHARSET_NONE, 0);
intval_scan_data.int_data=intval;
@@ -1446,9 +1514,12 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
else if(region_ret>0)
else if(region_ret>0 || scan_status_should_compile_NOT(_mid))
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
if(region_ret>0)
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
}
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -1495,7 +1566,7 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
Maat_table_desc* p_table=NULL;
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL;
struct Maat_scanner_t* my_scanner=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
{
@@ -1565,9 +1636,12 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
else if(region_ret>0)
else if(region_ret>0 || scan_status_should_compile_NOT(_mid) )
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
if(region_ret>0)
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
}
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
@@ -1613,7 +1687,7 @@ int Maat_scan_addr(Maat_feather_t feather,int table_id
stream_para_t Maat_stream_scan_string_start(Maat_feather_t feather,int table_id,int thread_num)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* scanner=NULL;
struct Maat_scanner_t* scanner=NULL;
struct Maat_table_desc *p_table=NULL;
assert(thread_num<_feather->scan_thread_num);
@@ -1676,7 +1750,7 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
,int* detail_ret,scan_status_t* mid)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_scanner_t* scanner=sp->feather->scanner;
int sub_type=0;
int region_ret=0,hit_region_cnt=0,compile_ret=0;
@@ -1791,9 +1865,12 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para
{
hit_region_cnt=match_district(_mid,region_result,hit_region_cnt);
}
if(hit_region_cnt>0)
if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1);
if(hit_region_cnt>0)
{
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num,1);
}
_mid=grab_mid(mid, sp->feather,sp->thread_num, 1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
@@ -1860,7 +1937,7 @@ int Maat_stream_scan_string(stream_para_t* stream_para
void Maat_stream_scan_string_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_table_runtime* table_rt=scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num, -1);
if(sp->rs_stream_para!=NULL)
@@ -1897,7 +1974,7 @@ void Maat_stream_scan_string_end(stream_para_t* stream_para)
stream_para_t Maat_stream_scan_digest_start(Maat_feather_t feather,int table_id,unsigned long long total_len,int thread_num)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* scanner=NULL;
struct Maat_scanner_t* scanner=NULL;
sfh_instance_t * tmp_fuzzy_handle=NULL;
struct Maat_table_desc *p_table=NULL;
p_table=acqurie_table(_feather, table_id, TABLE_TYPE_DIGEST);
@@ -2018,9 +2095,12 @@ int Maat_stream_scan_digest(stream_para_t * stream_para, const char * data, int
compile_ret=-1;
goto fast_out;
}
if(hit_region_cnt>0)
if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1);
if(hit_region_cnt>0)
{
alignment_int64_array_add(table_rt->hit_cnt, sp->thread_num, 1);
}
_mid=grab_mid(mid,sp->feather, sp->thread_num,1);
compile_ret=region_compile(sp->feather,_mid->inner,
_mid->is_last_region,
@@ -2053,7 +2133,7 @@ fast_out:
void Maat_stream_scan_digest_end(stream_para_t* stream_para)
{
struct _stream_para_t* sp=(struct _stream_para_t*)(*stream_para);
struct _Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_scanner_t* scanner=sp->feather->scanner;
struct Maat_table_runtime *table_rt=sp->feather->scanner->table_rt[sp->table_id];
alignment_int64_array_add(table_rt->stream_num, sp->thread_num,-1);
if(scanner!=NULL)
@@ -2076,11 +2156,11 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
{
int ret=0;
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct _Maat_compile_inner_t *compile_inner=NULL;
struct Maat_compile_inner_t *compile_inner=NULL;
switch(type)
{
case MAAT_RULE_SERV_DEFINE:
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL)
{
ret=0;
@@ -2130,6 +2210,7 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA
_mid->is_set_district=1;
break;
case MAAT_SET_SCAN_LAST_REGION:
assert(_mid->is_last_region==0);
_mid->is_last_region=1;
break;
default:
@@ -2144,12 +2225,12 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
,struct Maat_rule_t*result,int rule_num
,scan_status_t* mid,int thread_num)
{
int region_ret=0,compile_ret=0;
int hit_region_cnt=0,compile_ret=0;
struct _OUTER_scan_status_t* _mid=NULL;
GIE_result_t region_result[MAX_SCANNER_HIT_NUM];
_compile_result_t compile_result[rule_num];
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
struct _Maat_scanner_t* my_scanner=NULL;
struct Maat_scanner_t* my_scanner=NULL;
Maat_table_desc* p_table=NULL;
struct timespec start,end;
if(_feather->perf_on==1)
@@ -2176,21 +2257,21 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
INC_SCANNER_REF(my_scanner,thread_num);
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);
region_ret=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM);
if(region_ret<0)
hit_region_cnt=GIE_query(gie_handle, data, data_len,region_result, MAX_SCANNER_HIT_NUM);
if(hit_region_cnt<0)
{
DEC_SCANNER_REF(my_scanner, thread_num);
_feather->scan_err_cnt++;
return -1;
}
else if(region_ret>0)
else if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
region_result,sizeof(GIE_result_t),offsetof(GIE_result_t, tag),
region_ret,
hit_region_cnt,
result,compile_result,rule_num,
thread_num);
assert(_mid->is_last_region<2);
@@ -2210,33 +2291,13 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
{
maat_stat_table(table_rt,0,NULL, NULL,thread_num);
}
if(compile_ret==0&&region_ret>0)
if(compile_ret==0&&hit_region_cnt>0)
{
return -2;
}
return compile_ret;
}
void Maat_clean_status(scan_status_t* mid)
{
struct _OUTER_scan_status_t* _mid=NULL;
if(*mid==NULL)
{
return;
}
_mid=(struct _OUTER_scan_status_t*)(*mid);
alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL)
{
free(_mid->inner->hitted_group_id);
free(_mid->inner);
alignment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1);
}
_mid->feather=NULL;
free(_mid);
*mid=NULL;
return;
}
int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* value,int size)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
@@ -2282,3 +2343,8 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu
return 0;
}
int Maat_helper_read_column(const char* line, int Nth_column, size_t *column_offset, size_t *column_len)
{
return get_column_pos(line, Nth_column, column_offset, column_len);
}

273
src/entry/Maat_command.cpp
View File

@@ -14,15 +14,15 @@
#define maat_redis_monitor (module_name_str("MAAT_REDIS_MONITOR"))
#define maat_command (module_name_str("MAAT_COMMAND"))
const time_t MAAT_REDIS_RECONNECT_INTERVAL=5;
const char* rm_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"};
const char* rm_status_sset="MAAT_UPDATE_STATUS";
const char* rm_expire_sset="MAAT_EXPIRE_TIMER";
const char* rm_label_sset="MAAT_LABEL_INDEX";
const char* rm_version_sset="MAAT_VERSION_TIMER";
const char* rm_expire_lock="EXPIRE_OP_LOCK";
const long rm_expire_lock_timeout=300*1000;
const char* mr_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"};
const char* mr_status_sset="MAAT_UPDATE_STATUS";
const char* mr_expire_sset="MAAT_EXPIRE_TIMER";
const char* mr_label_sset="MAAT_LABEL_INDEX";
const char* mr_version_sset="MAAT_VERSION_TIMER";
const char* mr_expire_lock="EXPIRE_OP_LOCK";
const long mr_expire_lock_timeout=300*1000;
const static int MAAT_REDIS_SYNC_TIME=30*60;
const char* rm_op_str[]={"DEL","ADD","RENEW_TIMEOUT"};
const char* mr_op_str[]={"DEL","ADD","RENEW_TIMEOUT"};
const char* foreign_source_prefix="redis://";
const char* foreign_key_prefix="__FILE_";
@@ -356,13 +356,13 @@ int get_inc_key_list(long long instance_version, long long target_version, redis
//Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version.
//The elements are considered to be ordered from low to high scores(instance_version).
reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld",rm_status_sset,instance_version,target_version);
reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld",mr_status_sset,instance_version,target_version);
if(reply==NULL)
{
__redis_strerror_r(errno,err_buff,sizeof(err_buff)-1);
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
"GET %s failed %s.",rm_status_sset,err_buff);
"GET %s failed %s.",mr_status_sset,err_buff);
return -1;
}
assert(reply->type==REDIS_REPLY_ARRAY);
@@ -373,11 +373,11 @@ int get_inc_key_list(long long instance_version, long long target_version, redis
return 0;
}
tmp_reply=_wrap_redisCommand(c, "ZSCORE %s %s",rm_status_sset,reply->element[0]->str);
tmp_reply=_wrap_redisCommand(c, "ZSCORE %s %s",mr_status_sset,reply->element[0]->str);
if(tmp_reply->type!=REDIS_REPLY_STRING)
{
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
"ZSCORE %s %s failed Version: %lld->%lld",rm_status_sset,reply->element[0]->str,instance_version, target_version);
"ZSCORE %s %s failed Version: %lld->%lld",mr_status_sset,reply->element[0]->str,instance_version, target_version);
freeReplyObject(tmp_reply);
freeReplyObject(reply);
return -1;
@@ -393,8 +393,6 @@ int get_inc_key_list(long long instance_version, long long target_version, redis
{
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
"Noncontinuous VERSION Redis: %lld MAAT: %lld.",nearest_rule_version,instance_version);
freeReplyObject(reply);
return -1;
}
s_rule=(struct serial_rule_t*)calloc(reply->elements,sizeof(struct serial_rule_t));
for(i=0, j=0;i<reply->elements;i++)
@@ -588,9 +586,9 @@ int get_rm_key_list(redisContext *c, long long instance_version, long long desir
}while(rule_num==0&&target_version<=redis_version&&cumulative_off==1);
if(rule_num==0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,"Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s"
,rm_status_sset,instance_version,target_version-1,cumulative_off==1?"OFF":"ON");
goto FULL_UPDATE;
MESA_handle_runtime_log(logger, RLOG_LV_DEBUG, maat_redis_monitor,"Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s"
,mr_status_sset,instance_version,target_version-1,cumulative_off==1?"OFF":"ON");
return 0;
}
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_redis_monitor,
"Inc Update form instance_version %lld to %lld (%d entries).",instance_version,target_version,rule_num);
@@ -707,7 +705,7 @@ int _get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int ru
redisReply* reply=NULL;
for(i=0;i<rule_num;i++)
{
snprintf(redis_cmd,sizeof(redis_cmd),"GET %s:%s,%ld",rm_key_prefix[rule_list[i].op]
snprintf(redis_cmd,sizeof(redis_cmd),"GET %s:%s,%ld",mr_key_prefix[rule_list[i].op]
,rule_list[i].table_name
,rule_list[i].rule_id);
ret=redisAppendCommand(c, redis_cmd);
@@ -730,7 +728,7 @@ int _get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int ru
else
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
,"Redis GET %s:%s,%d failed",rm_key_prefix[rule_list[i].op]
,"Redis GET %s:%s,%d failed",mr_key_prefix[rule_list[i].op]
,rule_list[i].table_name
,rule_list[i].rule_id);
error_happened=1;
@@ -747,7 +745,7 @@ int _get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int ru
for(i=0;i<failed_cnt;i++)
{
idx=retry_ids[i];
snprintf(redis_cmd,sizeof(redis_cmd),"GET %s:%s,%ld",rm_key_prefix[MAAT_OP_DEL]
snprintf(redis_cmd,sizeof(redis_cmd),"GET %s:%s,%ld",mr_key_prefix[MAAT_OP_DEL]
,rule_list[idx].table_name
,rule_list[idx].rule_id);
ret=redisAppendCommand(c, redis_cmd);
@@ -838,8 +836,8 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
struct Maat_group_t* group_cmd=NULL;
struct Maat_region_t* region_cmd=NULL;
struct _Maat_compile_inner_t *compile_inner=NULL;
struct _Maat_group_inner_t* group_inner=NULL;
struct Maat_compile_inner_t *compile_inner=NULL;
struct Maat_group_inner_t* group_inner=NULL;
struct _Maat_region_inner_t* region_inner=NULL;
void* logger=feather->logger;
@@ -850,7 +848,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
,"MAAT not ready.");
return -1;
}
compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
compile_inner=(struct Maat_compile_inner_t *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
//Operation on compile_inner is thread safe, no immediate memory free when delete a compile rule or a scanner.
//In another words, if the compile_inner is accessable from compile means, its was valid in at least 10 seconds (garbage bury).
if(compile_inner==NULL)
@@ -866,7 +864,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
cmd->groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),cmd->group_num);
for(i=0;i<compile_inner->group_boundary;i++)
{
group_inner=(struct _Maat_group_inner_t*)dynamic_array_read(compile_inner->groups,i);
group_inner=(struct Maat_group_inner_t*)dynamic_array_read(compile_inner->groups,i);
if(group_inner==NULL)
{
continue;
@@ -992,19 +990,7 @@ int mr_transaction_success(redisReply* data_reply)
return 1;
}
}
int mr_operation_success(redisReply* actual_reply, redisReply* expected_reply)
{
if(expected_reply->type!=actual_reply->type)
{
return 0;
}
if(expected_reply->type==REDIS_REPLY_INTEGER&&expected_reply->integer!=actual_reply->integer)
{
return 0;
}
return 1;
}
int redlock_try_lock(redisContext *ctx, const char* lock_name, long long expire)
{
redisReply* reply=NULL;
@@ -1028,11 +1014,47 @@ void redlock_unlock(redisContext * ctx, const char * lock_name)
freeReplyObject(reply);
}
struct expected_reply_t
#define POSSIBLE_REDIS_REPLY_SIZE 2
struct expected_reply
{
int srule_seq;
redisReply reply;
int possible_reply_num;
redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE];
};
void expected_reply_add(struct expected_reply* expected, int srule_seq, int type, long long integer)
{
int i=expected->possible_reply_num;
assert(i<POSSIBLE_REDIS_REPLY_SIZE);
expected->srule_seq=srule_seq;
expected->possible_replies[i].type=type;
expected->possible_replies[i].integer=integer;
expected->possible_reply_num++;
}
int mr_operation_success(redisReply* actual_reply, struct expected_reply* expected)
{
int i=0;
if(expected->possible_replies[0].type!=actual_reply->type)
{
return 0;
}
for(i=0; i< expected->possible_reply_num; i++)
{
if(expected->possible_replies[i].type==REDIS_REPLY_INTEGER &&
expected->possible_replies[i].type==actual_reply->type &&
expected->possible_replies[i].integer==actual_reply->integer)
{
return 1;
}
if(expected->possible_replies[i].type==REDIS_REPLY_STATUS &&
expected->possible_replies[i].type==actual_reply->type &&
0==strcasecmp(actual_reply->str, "OK"))
{
return 1;
}
}
return 0;
}
long long _exec_serial_rule_begin(redisContext* ctx,int rule_num, int renew_rule_num,int *renew_allowed, long long *maat_redis_version)
{
@@ -1040,7 +1062,7 @@ long long _exec_serial_rule_begin(redisContext* ctx,int rule_num, int renew_rule
redisReply* data_reply=NULL;
if(renew_rule_num>0)
{
while(0==redlock_try_lock(ctx, rm_expire_lock, rm_expire_lock_timeout))
while(0==redlock_try_lock(ctx, mr_expire_lock, mr_expire_lock_timeout))
{
usleep(1000);
}
@@ -1064,31 +1086,30 @@ long long _exec_serial_rule_begin(redisContext* ctx,int rule_num, int renew_rule
}
return ret;
}
redisReply* _exec_serial_rule_end(redisContext* ctx,long long maat_redis_version, long long server_time, int renew_allowed, struct expected_reply_t* expect_reply, unsigned int *cnt)
redisReply* _exec_serial_rule_end(redisContext* ctx,long long maat_redis_version, long long server_time, int renew_allowed, struct expected_reply* expect_reply, unsigned int *cnt)
{
redisReply* data_reply=NULL;
if(renew_allowed==1)
{
redlock_unlock(ctx, rm_expire_lock);
redlock_unlock(ctx, mr_expire_lock);
expect_reply[*cnt].srule_seq=-1;
(*cnt)++;
}
if(maat_redis_version>0)
{
data_reply=_wrap_redisCommand(ctx,"ZADD %s NX %d %d",rm_version_sset,server_time,maat_redis_version);
data_reply=_wrap_redisCommand(ctx,"ZADD %s NX %d %d",mr_version_sset,server_time,maat_redis_version);
freeReplyObject(data_reply);
expect_reply[*cnt].srule_seq=-1;
expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
data_reply=_wrap_redisCommand(ctx,"INCRBY MAAT_VERSION 1");
freeReplyObject(data_reply);
expect_reply[*cnt].srule_seq=-1;
expect_reply[*cnt].reply.type=REDIS_REPLY_INTEGER;
freeReplyObject(data_reply);
expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
}
data_reply=_wrap_redisCommand(ctx,"EXEC");
return data_reply;
}
void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_t* s_rule, unsigned int rule_num, struct expected_reply_t* expect_reply, unsigned int *cnt, int offset,int renew_allowed)
void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_t* s_rule, unsigned int rule_num, struct expected_reply* expect_reply, unsigned int *cnt, int offset,int renew_allowed)
{
redisReply* data_reply=NULL;
unsigned int append_cmd_cnt=0, i=0;
@@ -1097,98 +1118,99 @@ void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_
switch(s_rule[i].op)
{
case MAAT_OP_ADD:
redisAppendCommand(ctx,"SET %s:%s,%d %s",rm_key_prefix[MAAT_OP_ADD]
,s_rule[i].table_name
,s_rule[i].rule_id
,s_rule[i].table_line);
expect_reply[*cnt].srule_seq=i+offset;
expect_reply[*cnt].reply.type=REDIS_REPLY_STATUS;
redisAppendCommand(ctx,"SET %s:%s,%d %s",
mr_key_prefix[MAAT_OP_ADD],
s_rule[i].table_name,
s_rule[i].rule_id,
s_rule[i].table_line);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_STATUS, 0);
(*cnt)++;
append_cmd_cnt++;
//NX: Don't update already exisiting elements. Always add new elements.
redisAppendCommand(ctx,"ZADD %s NX %lld ADD,%s,%d",rm_status_sset
,version
,s_rule[i].table_name
,s_rule[i].rule_id);
expect_reply[*cnt].srule_seq=i+offset;
expect_reply[*cnt].reply.type=REDIS_REPLY_INTEGER;
expect_reply[*cnt].reply.integer=1;
//Allowing add duplicated members for rule id recycling.
redisAppendCommand(ctx,"ZADD %s %lld ADD,%s,%d",
mr_status_sset,
version,
s_rule[i].table_name,
s_rule[i].rule_id);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
append_cmd_cnt++;
if(s_rule[i].timeout>0)
{
redisAppendCommand(ctx,"ZADD %s NX %lld %s,%d",rm_expire_sset
,s_rule[i].timeout
,s_rule[i].table_name
,s_rule[i].rule_id);
expect_reply[*cnt].srule_seq=i+offset;
expect_reply[*cnt].reply.type=REDIS_REPLY_INTEGER;
expect_reply[*cnt].reply.integer=1;
redisAppendCommand(ctx,"ZADD %s %lld %s,%d",
mr_expire_sset,
s_rule[i].timeout,
s_rule[i].table_name,
s_rule[i].rule_id);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
append_cmd_cnt++;
}
if(s_rule[i].label_id>0)
{
redisAppendCommand(ctx,"ZADD %s NX %d %s,%d",
rm_label_sset,
redisAppendCommand(ctx,"ZADD %s %d %s,%d",
mr_label_sset,
s_rule[i].label_id,
s_rule[i].table_name,
s_rule[i].rule_id);
expect_reply[*cnt].srule_seq=i+offset;
expect_reply[*cnt].reply.type=REDIS_REPLY_INTEGER;
expect_reply[*cnt].reply.integer=1;
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
append_cmd_cnt++;
}
break;
case MAAT_OP_DEL:
redisAppendCommand(ctx,"RENAME %s:%s,%d %s:%s,%d"
,rm_key_prefix[MAAT_OP_ADD]
,s_rule[i].table_name
,s_rule[i].rule_id
,rm_key_prefix[MAAT_OP_DEL]
,s_rule[i].table_name
,s_rule[i].rule_id
redisAppendCommand(ctx,"RENAME %s:%s,%d %s:%s,%d",
mr_key_prefix[MAAT_OP_ADD],
s_rule[i].table_name,
s_rule[i].rule_id,
mr_key_prefix[MAAT_OP_DEL],
s_rule[i].table_name,
s_rule[i].rule_id
);
expect_reply[*cnt].srule_seq=i+offset;
expect_reply[*cnt].reply.type=REDIS_REPLY_STATUS;
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_STATUS, 0);
(*cnt)++;
append_cmd_cnt++;
redisAppendCommand(ctx,"EXPIRE %s:%s,%d %d",rm_key_prefix[MAAT_OP_DEL]
,s_rule[i].table_name
,s_rule[i].rule_id
,MAAT_REDIS_SYNC_TIME);
expect_reply[*cnt].srule_seq=i+offset;
expect_reply[*cnt].reply.type=REDIS_REPLY_INTEGER;
expect_reply[*cnt].reply.integer=1;
redisAppendCommand(ctx,"EXPIRE %s:%s,%d %d",
mr_key_prefix[MAAT_OP_DEL],
s_rule[i].table_name,
s_rule[i].rule_id,
MAAT_REDIS_SYNC_TIME);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1);
(*cnt)++;
append_cmd_cnt++;
//NX: Don't update already exisiting elements. Always add new elements.
redisAppendCommand(ctx,"ZADD %s NX %d DEL,%s,%d",rm_status_sset
,version
,s_rule[i].table_name
,s_rule[i].rule_id);
expect_reply[*cnt].srule_seq=i+offset;
expect_reply[*cnt].reply.type=REDIS_REPLY_INTEGER;
expect_reply[*cnt].reply.integer=1;
redisAppendCommand(ctx,"ZADD %s %d DEL,%s,%d",
mr_status_sset,
version,
s_rule[i].table_name,
s_rule[i].rule_id);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1);
expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
append_cmd_cnt++;
// Try to remove from expiration sorted set, no matter wheather it exists or not.
redisAppendCommand(ctx,"ZREM %s %s,%d",rm_expire_sset,
redisAppendCommand(ctx,"ZREM %s %s,%d",
mr_expire_sset,
s_rule[i].table_name,
s_rule[i].rule_id);
expect_reply[*cnt].srule_seq=-1;
expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
append_cmd_cnt++;
redisAppendCommand(ctx,"ZREM %s %s,%d",rm_label_sset,
// Try to remove from label sorted set, no matter wheather it exists or not.
redisAppendCommand(ctx,"ZREM %s %s,%d",
mr_label_sset,
s_rule[i].table_name,
s_rule[i].rule_id);
expect_reply[*cnt].srule_seq=-1;
expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
append_cmd_cnt++;
break;
@@ -1198,13 +1220,12 @@ void _exec_serial_rule(redisContext* ctx, long long version, struct serial_rule_
continue;
}
//s_rule[i].timeout>0 was checked by caller.
redisAppendCommand(ctx,"ZADD %s %lld %s,%d",rm_expire_sset
,s_rule[i].timeout
,s_rule[i].table_name
,s_rule[i].rule_id);
expect_reply[*cnt].srule_seq=i+offset;
expect_reply[*cnt].reply.type=REDIS_REPLY_INTEGER;
expect_reply[*cnt].reply.integer=0;
redisAppendCommand(ctx,"ZADD %s %lld %s,%d",
mr_expire_sset,
s_rule[i].timeout,
s_rule[i].table_name,
s_rule[i].rule_id);
expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
(*cnt)++;
append_cmd_cnt++;
@@ -1233,7 +1254,7 @@ int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule, unsigned in
const int MAX_REDIS_OP_PER_SRULE=8;
unsigned int max_multi_cmd_num=MAX_REDIS_OP_PER_SRULE*serial_rule_num+2;// 2 for operation in _exec_serial_rule_end()
struct expected_reply_t *expected_reply=(struct expected_reply_t*)calloc(sizeof(struct expected_reply_t), max_multi_cmd_num);
struct expected_reply *expected_reply=(struct expected_reply*)calloc(sizeof(struct expected_reply), max_multi_cmd_num);
long long new_version=0;
int renew_num=0,ret=0;
for(i=0;i<serial_rule_num;i++)
@@ -1267,14 +1288,14 @@ int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule, unsigned in
//failed is acceptable
//or transaciton is success
//or continuation of last failed
if(expected_reply[i].srule_seq==-1||1==mr_operation_success(p,&(expected_reply[i].reply))||last_failed==expected_reply[i].srule_seq)
if(expected_reply[i].srule_seq==-1||1==mr_operation_success(p, expected_reply+i)||last_failed==expected_reply[i].srule_seq)
{
continue;
}
rule_seq=expected_reply[i].srule_seq;
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_command
,"%s %s %d failed, rule id maybe conflict or not exist."
,rm_op_str[s_rule[rule_seq].op]
,mr_op_str[s_rule[rule_seq].op]
,s_rule[rule_seq].table_name,s_rule[rule_seq].rule_id);
success_cnt--;
last_failed=rule_seq;
@@ -1294,7 +1315,7 @@ error_out:
if(s_rule[i].op==MAAT_OP_RENEW_TIMEOUT)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_command
,"%s %s %d is not allowed due to lock contention.",rm_op_str[MAAT_OP_RENEW_TIMEOUT]
,"%s %s %d is not allowed due to lock contention.",mr_op_str[MAAT_OP_RENEW_TIMEOUT]
, s_rule[i].table_name,s_rule[i].rule_id);
}
}
@@ -1316,10 +1337,10 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
struct Maat_region_t* p_region=NULL;
enum MAAT_TABLE_TYPE table_type;
struct _Maat_compile_inner_t *compile_rule=NULL;
struct Maat_compile_inner_t *compile_rule=NULL;
if(feather->scanner!=NULL)
{
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
if(compile_rule!=NULL)
{
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module
@@ -1370,7 +1391,7 @@ void check_maat_expiration(redisContext *ctx, void *logger)
server_time=redis_server_time(ctx);
data_reply=_wrap_redisCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",rm_expire_sset,server_time);
data_reply=_wrap_redisCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_expire_sset,server_time);
if(data_reply->type!=REDIS_REPLY_ARRAY||data_reply->elements==0)
{
freeReplyObject(data_reply);
@@ -1411,9 +1432,9 @@ void cleanup_update_status(redisContext *ctx, void *logger)
reply=_wrap_redisCommand(ctx,"MULTI");
freeReplyObject(reply);
redisAppendCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",rm_version_sset,server_time-MAAT_REDIS_SYNC_TIME);
redisAppendCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME);
append_cmd_cnt++;
redisAppendCommand(ctx, "ZREMRANGEBYSCORE %s -inf %lld",rm_version_sset,server_time-MAAT_REDIS_SYNC_TIME);
redisAppendCommand(ctx, "ZREMRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME);
append_cmd_cnt++;
//consume reply "OK" and "QUEUED".
for(i=0;i<append_cmd_cnt;i++)
@@ -1442,7 +1463,7 @@ void cleanup_update_status(redisContext *ctx, void *logger)
freeReplyObject(reply);
//To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally.
reply=_wrap_redisCommand(ctx,"ZREMRANGEBYSCORE %s %lld %lld",rm_status_sset,version_lower_bound,version_upper_bound);
reply=_wrap_redisCommand(ctx,"ZREMRANGEBYSCORE %s %lld %lld",mr_status_sset,version_lower_bound,version_upper_bound);
entry_num=read_redis_integer(reply);
freeReplyObject(reply);
@@ -1749,11 +1770,11 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx* m
if(mr_ctx->write_ctx!=NULL&&mr_ctx->write_ctx->err==0)//authorized to write
{
//For thread safe, deliberately use redis_read_ctx but not redis_write_ctx.
if(1==redlock_try_lock(mr_ctx->read_ctx, rm_expire_lock, rm_expire_lock_timeout))
if(1==redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout))
{
check_maat_expiration(mr_ctx->read_ctx, logger);
cleanup_update_status(mr_ctx->read_ctx, logger);
redlock_unlock(mr_ctx->read_ctx, rm_expire_lock);
redlock_unlock(mr_ctx->read_ctx, mr_expire_lock);
}
}
if(mr_ctx->read_ctx==NULL||mr_ctx->read_ctx->err)
@@ -2014,8 +2035,8 @@ int Maat_cmd_set_group(Maat_feather_t feather,int group_id, const struct Maat_re
if(_feather->AUTO_NUMBERING_ON==1)
{
return -1;
}
//struct _Maat_group_inner_t* group_inner=NULL;
}
//struct Maat_group_inner_t* group_inner=NULL;
//group_inner=(struct Maat_group_inner_t*)HASH_fetch_by_id(_feather->scanner->group_hash, group_id);
//NOT implemented yet.
assert(0);
@@ -2299,7 +2320,7 @@ int Maat_cmd_commit(Maat_feather_t feather)
int ret=0,i=0;
int new_region_num=0,new_group_num=0;
int serial_rule_num=0,serial_rule_idx=0;
int serial_rule_num=0,serial_rule_idx=0;
UNUSED int transaction_success=1;
struct _Maat_cmd_inner_t* p=NULL,*n=NULL;
@@ -2357,9 +2378,9 @@ int Maat_cmd_commit(Maat_feather_t feather)
serial_rule_idx+=build_serial_rule(_feather,p,s_rule+serial_rule_idx, serial_rule_num-serial_rule_idx);
p=p->next;
}
assert(serial_rule_idx==serial_rule_num);
transection_success=0;
transection_success=exec_serial_rule(write_ctx, s_rule,serial_rule_num,_feather->server_time,_feather->logger);
assert(serial_rule_idx==serial_rule_num);
transaction_success=0;
transaction_success=exec_serial_rule(write_ctx, s_rule,serial_rule_num,_feather->server_time,_feather->logger);
assert(transaction_success==serial_rule_num);
ret=_feather->cmd_q_cnt;
_feather->cmd_acc_num+=_feather->cmd_q_cnt;
@@ -2433,7 +2454,7 @@ int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_ke
return -1;
}
data_reply=_wrap_redisCommand(write_ctx,"ZRANGEBYSCORE %s %d %d",
data_reply=_wrap_redisCommand(write_ctx,"ZRANGEBYSCORE %s %d %d",
mr_label_sset,
label_id,
label_id);

369
src/entry/Maat_rule.cpp
View File

@@ -28,11 +28,11 @@
#include "map_str2int.h"
#include "rulescan.h"
#include "UniversalBoolMatch.h"
#include "bool_matcher.h"
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_5_20181216=1;
int MAAT_FRAME_VERSION_2_6_20190110=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
@@ -65,7 +65,7 @@ int is_valid_match_method(enum MAAT_MATCH_METHOD match_method)
}
iconv_t maat_iconv_open(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from)
iconv_t maat_iconv_open(struct Maat_scanner_t* scanner,enum MAAT_CHARSET to,enum MAAT_CHARSET from)
{
const char *from_s=CHARSET_STRING[from];
const char *to_s=CHARSET_STRING[to];
@@ -87,7 +87,7 @@ iconv_t maat_iconv_open(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET to,enu
return cd;
}
int iconv_convert(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen)
int iconv_convert(struct Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen)
{
size_t ret;
int copy_len=0;
@@ -206,7 +206,7 @@ int uni2ascii(const char* fmt,const char* src, const int srclen, char* dst, cons
}
return j;
}
int universal_charset_convert(struct _Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen)
int universal_charset_convert(struct Maat_scanner_t* scanner,enum MAAT_CHARSET from,enum MAAT_CHARSET to,char *src,int srclen,char *dst,int *dstlen)
{
int ret=0;
char* tmp_buff=NULL;
@@ -836,9 +836,9 @@ error_jump:
map_destroy(string2int_map);
return table_cnt;
}
struct _Maat_group_inner_t* create_group_rule(int group_id)
struct Maat_group_inner_t* create_group_rule(int group_id)
{
struct _Maat_group_inner_t* group=(struct _Maat_group_inner_t*)malloc(sizeof(struct _Maat_group_inner_t));
struct Maat_group_inner_t* group=(struct Maat_group_inner_t*)malloc(sizeof(struct Maat_group_inner_t));
group->group_id=group_id;
group->region_cnt=0;
group->region_boundary=0;
@@ -850,7 +850,7 @@ struct _Maat_group_inner_t* create_group_rule(int group_id)
pthread_mutex_init(&(group->mutex), NULL);
return group;
}
void _destroy_group_rule(struct _Maat_group_inner_t* group)
void _destroy_group_rule(struct Maat_group_inner_t* group)
{
dynamic_array_destroy(group->regions,free);
group->region_cnt=0;
@@ -865,7 +865,7 @@ void _destroy_group_rule(struct _Maat_group_inner_t* group)
free(group);
}
void destroy_group_rule(struct _Maat_group_inner_t* group)
void destroy_group_rule(struct Maat_group_inner_t* group)
{
if(group->ref_cnt>0||group->region_cnt>0)
@@ -874,30 +874,42 @@ void destroy_group_rule(struct _Maat_group_inner_t* group)
}
_destroy_group_rule(group);
}
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set)
void make_group_set(struct Maat_compile_inner_t* compile_rule, struct bool_expr* a_set, unsigned char *has_not)
{
int i=0,j=0;
a_set->bool_expr_id=(void*)compile_rule;
struct _Maat_group_inner_t*group=NULL;
a_set->user_tag=compile_rule;
struct Maat_group_inner_t*group=NULL;
assert(compile_rule->group_cnt<=MAX_ITEMS_PER_BOOL_EXPR);
for(i=0,j=0;i<compile_rule->group_boundary&&j<MAX_ITEMS_PER_BOOL_EXPR;i++)
{
group=(struct _Maat_group_inner_t*)dynamic_array_read(compile_rule->groups,i);
group=(struct Maat_group_inner_t*)dynamic_array_read(compile_rule->groups, i);
if(group==NULL)
{
continue;
}
a_set->bool_item_ids[j]=group->group_id;
a_set->items[j].item_id=group->group_id;
a_set->items[j].not_flag=compile_rule->not_flag[j];
if(a_set->items[j].not_flag)
{
*has_not=1;
}
j++;
}
assert(j==compile_rule->group_cnt);
a_set->bool_item_num=j;
a_set->item_num=j;
}
struct compile_walker
{
MESA_lqueue_head update_q;
long long compile_has_not_flag;
};
void walk_compile_hash(const uchar * key, uint size, void * data, void * user)
{
universal_bool_expr_t* one_set=NULL;
struct _Maat_compile_inner_t* compile_rule=(struct _Maat_compile_inner_t*)data;
MESA_lqueue_head update_q=(MESA_lqueue_head)user;
struct bool_expr* one_set=NULL;
struct Maat_compile_inner_t* compile_rule=(struct Maat_compile_inner_t*)data;
struct compile_walker* walker=(struct compile_walker*)user;
unsigned char has_not_flag=0;
MESA_lqueue_head update_q=walker->update_q;
if(compile_rule->db_c_rule==NULL)
{
return;
@@ -905,69 +917,76 @@ void walk_compile_hash(const uchar * key, uint size, void * data, void * user)
//make sure compile rule's each group has loadded.
if((compile_rule->group_cnt==compile_rule->db_c_rule->declare_grp_num
||compile_rule->db_c_rule->declare_grp_num==0)//for compatible old version
&&compile_rule->group_cnt>0)
&&compile_rule->group_cnt>0
&&compile_rule->group_cnt!=compile_rule->not_group_cnt)
{
one_set=(universal_bool_expr_t*)malloc(sizeof(universal_bool_expr_t));
one_set=ALLOC(struct bool_expr, 1);
//reading compile rule is safe in update thread, mutex lock called when modified
make_group_set(compile_rule, one_set);
MESA_lqueue_join_tail(update_q,&one_set, sizeof(void*));//put the pointer into queue
make_group_set(compile_rule, one_set, &has_not_flag);
if(has_not_flag)
{
walker->compile_has_not_flag++;
}
MESA_lqueue_join_tail(update_q, &one_set, sizeof(one_set));//put the pointer into queue
}
return;
}
void* create_bool_matcher(MESA_htable_handle compile_hash,int thread_num,void* logger)
struct bool_matcher* create_bool_matcher(MESA_htable_handle compile_hash, int thread_num, void* logger)
{
void* bool_matcher=NULL;
MESA_lqueue_head update_q=MESA_lqueue_create(0,0);;
struct bool_matcher* bm=NULL;
struct compile_walker walker={NULL, 0};
walker.update_q=MESA_lqueue_create(0,0);
MESA_lqueue_head update_q=walker.update_q;
long data_size=0;
unsigned int mem_size=0;
size_t mem_size=0;
UNUSED MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK;
data_size=sizeof(void*);
universal_bool_expr_t* one_set=NULL;
universal_bool_expr_t* set_array=NULL;
struct bool_expr* one_set=NULL;
struct bool_expr* set_array=NULL;
int i=0;
MESA_htable_iterate(compile_hash, walk_compile_hash, update_q);
MESA_htable_iterate(compile_hash, walk_compile_hash, &walker);
const long q_cnt=MESA_lqueue_get_count(update_q);
if(q_cnt==0)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
"No compile rule to build a bool matcher.");
MESA_lqueue_destroy(update_q,lqueue_destroy_cb,NULL);
MESA_lqueue_destroy(update_q, lqueue_destroy_cb, NULL);
return NULL;
}
set_array=(universal_bool_expr_t*)malloc(sizeof(universal_bool_expr_t)*q_cnt);
for(i=0;i<q_cnt;i++)
set_array=ALLOC(struct bool_expr, q_cnt);
for(i=0; i<q_cnt; i++)
{
q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(update_q,&one_set,&data_size);
assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
memcpy(set_array+i,one_set,sizeof(universal_bool_expr_t));
q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(update_q, &one_set, &data_size);
assert(data_size==sizeof(struct bool_expr*) && q_ret==MESA_QUEUE_RET_OK);
set_array[i]=*one_set;
free(one_set);
one_set=NULL;
}
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"build bool matcher start contain %ld compile rule",
q_cnt);
bool_matcher=boolexpr_initialize(set_array, q_cnt, thread_num, &mem_size);
if(bool_matcher!=NULL)
MESA_handle_runtime_log(logger, RLOG_LV_INFO,maat_module,
"build bool matcher start: compile count %ld, NOT-logic count %lld",
q_cnt, walker.compile_has_not_flag);
bm=bool_matcher_new(set_array, q_cnt, thread_num, &mem_size);
if(bm!=NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"build bool matcher use %u memory",mem_size);
MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
"build bool matcher use %zu memory", mem_size);
}
else
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module,
"build bool matcher failed!",
q_cnt,mem_size);
q_cnt);
}
free(set_array);
set_array=NULL;
MESA_lqueue_destroy(update_q,lqueue_destroy_cb,NULL);
return bool_matcher;
MESA_lqueue_destroy(update_q, lqueue_destroy_cb, NULL);
return bm;
}
void destroy_bool_matcher(void * bool_matcher)
void destroy_bool_matcher(struct bool_matcher * bm)
{
boolexpr_destroy(bool_matcher);
bool_matcher_free(bm);
return;
}
@@ -975,10 +994,9 @@ void EMPTY_FREE(void*p)
{
return;
}
struct _Maat_compile_inner_t * create_compile_rule(int compile_id)
struct Maat_compile_inner_t * create_compile_rule(int compile_id)
{
struct _Maat_compile_inner_t* p=NULL;
p=(struct _Maat_compile_inner_t*)calloc(sizeof(struct _Maat_compile_inner_t),1);
struct Maat_compile_inner_t* p=ALLOC(struct Maat_compile_inner_t,1);
p->compile_id=compile_id;
p->group_cnt=0;
p->group_boundary=1;
@@ -987,7 +1005,7 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id)
pthread_rwlock_init(&(p->rwlock), NULL);
return p;
}
void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule)
void _destroy_compile_rule(struct Maat_compile_inner_t * compile_rule)
{
const struct Maat_table_desc* table=compile_rule->ref_table;
const struct compile_table_desc* compile_desc=&(table->compile);
@@ -1017,14 +1035,14 @@ void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule)
pthread_rwlock_destroy(&(compile_rule->rwlock));
free(compile_rule);
}
void destroy_compile_rule(struct _Maat_compile_inner_t * p)
void destroy_compile_rule(struct Maat_compile_inner_t * p)
{
int i=0;
UNUSED struct _Maat_compile_inner_t* p_group=NULL;
UNUSED struct Maat_compile_inner_t* p_group=NULL;
assert(p->group_cnt==0);
for(i=0;i<p->group_boundary;i++)
{
p_group=(struct _Maat_compile_inner_t*)dynamic_array_read(p->groups,i);
p_group=(struct Maat_compile_inner_t*)dynamic_array_read(p->groups,i);
assert(p_group==NULL);
}
_destroy_compile_rule(p);
@@ -1155,7 +1173,7 @@ void op_expr_add_rule(struct op_expr_t* op_expr,scan_rule_t* p_rule)
return;
}
GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest,
short cfds_lvl,struct _Maat_group_inner_t* tag)
short cfds_lvl,struct Maat_group_inner_t* tag)
{
GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1);
int digest_len=0;
@@ -1270,7 +1288,7 @@ void table_runtime_free(struct Maat_table_runtime* p)
return;
}
struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t *feather)
struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t *feather)
{
int scan_thread_num=feather->scan_thread_num;
// int rs_scan_type=feather->rule_scan_type;
@@ -1296,8 +1314,8 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
hargs.data_expire_with_condition = NULL;
struct _Maat_scanner_t* scanner=NULL;
scanner=ALLOC(struct _Maat_scanner_t, 1);
struct Maat_scanner_t* scanner=NULL;
scanner=ALLOC(struct Maat_scanner_t, 1);
//Function Maat_cmd_append will access compile_hash in user thread.
hargs.thread_safe=1;
@@ -1369,7 +1387,7 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
void destroy_maat_scanner(struct Maat_scanner_t*scanner)
{
long q_cnt=0,data_size=0;
int i=0,j=0;
@@ -1386,7 +1404,7 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
map_destroy(scanner->district_map);
scanner->district_map=NULL;
assert(scanner->tmp_district_map==NULL);
destroy_bool_matcher((void*)scanner->bool_macher_expr_compiler);
destroy_bool_matcher(scanner->bool_matcher_expr_compiler);
q_cnt=MESA_lqueue_get_count(scanner->region_update_q);
for(i=0;i<q_cnt;i++)
{
@@ -1495,7 +1513,7 @@ void count_rs_region(struct op_expr_t* op_expr,struct _region_stat_t* region_sta
return;
}
void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct _Maat_scanner_t* maat_scanner)
void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,void*logger,struct Maat_scanner_t* maat_scanner)
{
long i=0,data_size=0;
unsigned int j=0;
@@ -1603,7 +1621,7 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
free(to_update_expr);
}
void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct _Maat_scanner_t* maat_scanner,int table_id)
void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*logger,struct Maat_scanner_t* maat_scanner,int table_id)
{
long i=0,data_size=0;
int ret=0;
@@ -1649,7 +1667,7 @@ void digest_batch_update(GIE_handle_t* handle,MESA_lqueue_head update_q,void*log
update_array=NULL;
return;
}
struct _Maat_group_inner_t* add_region_to_group(struct _Maat_group_inner_t* group,int table_id,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type)
struct Maat_group_inner_t* add_region_to_group(struct Maat_group_inner_t* group,int table_id,int region_id,int district_id,int expr_id,enum MAAT_TABLE_TYPE region_type)
{
int i=0;
struct _Maat_region_inner_t* region_rule=NULL;
@@ -1688,7 +1706,7 @@ struct _Maat_group_inner_t* add_region_to_group(struct _Maat_group_inner_t* grou
}
return group;
}
void cancel_last_region_from_group(struct _Maat_group_inner_t* group,int region_id,int expr_id)
void cancel_last_region_from_group(struct Maat_group_inner_t* group,int region_id,int expr_id)
{
struct _Maat_region_inner_t* region_rule=NULL;
pthread_mutex_lock(&(group->mutex));
@@ -1709,7 +1727,7 @@ void cancel_last_region_from_group(struct _Maat_group_inner_t* group,int region_
pthread_mutex_unlock(&(group->mutex));
return;
}
unsigned int del_region_from_group(struct _Maat_group_inner_t* group,int region_id,unsigned int *output_expr_id,int output_size)
unsigned int del_region_from_group(struct Maat_group_inner_t* group,int region_id,unsigned int *output_expr_id,int output_size)
{
int i=0,j=0;
struct _Maat_region_inner_t* region_rule=NULL;
@@ -1741,11 +1759,11 @@ unsigned int del_region_from_group(struct _Maat_group_inner_t* group,int region_
return j;
}
int add_group_to_compile(struct _Maat_compile_inner_t*a_compile_rule,struct _Maat_group_inner_t* a_rule_group)
int add_group_to_compile(struct Maat_compile_inner_t*a_compile_rule,struct Maat_group_inner_t* a_rule_group, int not_flag)
{
int i=0,ret=-1;
int write_pos=-1;
struct _Maat_group_inner_t* p=NULL;
struct Maat_group_inner_t* p=NULL;
pthread_rwlock_wrlock(&(a_compile_rule->rwlock));
if(a_compile_rule->db_c_rule!=NULL
&&a_compile_rule->group_cnt>=a_compile_rule->db_c_rule->declare_grp_num
@@ -1757,7 +1775,7 @@ int add_group_to_compile(struct _Maat_compile_inner_t*a_compile_rule,struct _Maa
for(i=0;i<a_compile_rule->group_boundary;i++)
{
p=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
p=(struct Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
if(p==NULL)
{
write_pos=i;
@@ -1781,7 +1799,16 @@ int add_group_to_compile(struct _Maat_compile_inner_t*a_compile_rule,struct _Maa
write_pos=a_compile_rule->group_boundary;
a_compile_rule->group_boundary++;
}
dynamic_array_write(a_compile_rule->groups,write_pos, a_rule_group);
dynamic_array_write(a_compile_rule->groups, write_pos, a_rule_group);
if(not_flag)
{
a_compile_rule->not_flag[write_pos]=1;
a_compile_rule->not_group_cnt++;
}
else
{
a_compile_rule->not_flag[write_pos]=0;
}
a_compile_rule->group_cnt++;
a_rule_group->ref_cnt++;
//member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish.
@@ -1799,7 +1826,7 @@ int add_group_to_compile(struct _Maat_compile_inner_t*a_compile_rule,struct _Maa
{
for(i=0;i<a_compile_rule->group_boundary;i++)
{
p=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
p=(struct Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
if(p!=NULL)
{
p->compile_shortcut=NULL;
@@ -1812,14 +1839,14 @@ error_out:
return ret;
}
struct _Maat_group_inner_t* del_group_from_compile(struct _Maat_compile_inner_t*a_compile_rule,int group_id)
struct Maat_group_inner_t* del_group_from_compile(struct Maat_compile_inner_t*a_compile_rule,int group_id)
{
int i=0;
struct _Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
pthread_rwlock_wrlock(&(a_compile_rule->rwlock));
for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++)
{
group_rule=(struct _Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
group_rule=(struct Maat_group_inner_t*)dynamic_array_read(a_compile_rule->groups,i);
if(group_rule==NULL)
{
continue;
@@ -1828,6 +1855,11 @@ struct _Maat_group_inner_t* del_group_from_compile(struct _Maat_compile_inner_t*
{
group_rule->ref_cnt--;
dynamic_array_write(a_compile_rule->groups,i,NULL);
if(a_compile_rule->not_flag[i]==1)
{
a_compile_rule->not_group_cnt--;
a_compile_rule->not_flag[i]=0;
}
a_compile_rule->group_cnt--;
pthread_rwlock_unlock(&(a_compile_rule->rwlock));
return group_rule;
@@ -1849,7 +1881,6 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n
"region id %d of table %s is not unique.",region_id,table_name);
return -1;
}
}
else
{
@@ -1865,7 +1896,7 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n
}
return 1;
}
int get_district_id(_Maat_scanner_t *scanner,const char* district_str)
int get_district_id(Maat_scanner_t *scanner,const char* district_str)
{
int map_ret=0,district_id=-1;
map_ret=map_str2int(scanner->district_map, district_str,&district_id);
@@ -1885,7 +1916,7 @@ int get_district_id(_Maat_scanner_t *scanner,const char* district_str)
}
return district_id;
}
int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,struct Maat_scanner_t *scanner,void* logger)
{
unsigned int i=0,j=0;
char* p=NULL,*saveptr=NULL,*region_string=NULL;
@@ -1893,7 +1924,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
int expr_id=0,district_id=-1;
struct expr_table_desc* expr_desc=&(table->expr);
scan_rule_t*p_rule=NULL;
struct _Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
enum MAAT_CHARSET dst_charset=CHARSET_NONE;
char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM];
int key_left_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1},key_right_offset[MAAT_MAX_EXPR_ITEM_NUM]={-1};
@@ -1904,7 +1935,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
}
int sub_expr_cnt=0;
struct op_expr_t *op_expr=NULL;
struct _Maat_group_inner_t* u_para=NULL;
struct Maat_group_inner_t* u_para=NULL;
if(table->table_type==TABLE_TYPE_EXPR_PLUS)
{
@@ -1912,7 +1943,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
str_unescape(db_rule->district);
district_id=get_district_id(scanner, db_rule->district);
}
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id);
group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(db_rule->group_id);
@@ -2149,15 +2180,15 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
}
return 0;
}
int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,struct Maat_scanner_t *scanner,void* logger)
{
struct _Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL;
struct _Maat_group_inner_t* u_para=NULL;
struct Maat_group_inner_t* u_para=NULL;
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id);
group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(db_ip_rule->group_id);
@@ -2181,15 +2212,15 @@ int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,st
MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*));
return 0;
}
int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intval_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intval_rule,struct Maat_scanner_t *scanner,void* logger)
{
struct _Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL;
struct _Maat_group_inner_t* u_para=NULL;
struct Maat_group_inner_t* u_para=NULL;
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id);
group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(intval_rule->group_id);
@@ -2212,15 +2243,15 @@ int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule_t* intva
MESA_lqueue_join_tail(scanner->region_update_q, &op_expr, sizeof(void*));
return 0;
}
int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_digest_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_digest_rule,struct Maat_scanner_t *scanner,void* logger)
{
struct _Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
GIE_digest_t* digest_rule=NULL;
struct _Maat_group_inner_t* u_para=NULL;
struct Maat_group_inner_t* u_para=NULL;
struct Maat_table_runtime * table_rt=scanner->table_rt[table->table_id];
int expr_id=0,district_id=-1;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id);
group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(db_digest_rule->group_id);
@@ -2244,15 +2275,15 @@ int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule_t* db_di
scanner->gie_total_q_size++;
return 0;
}
int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct _Maat_scanner_t *maat_scanner,void* logger)
int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int rule_type,struct Maat_scanner_t *maat_scanner,void* logger)
{
int i=0;
unsigned int expr_id[MAAT_MAX_EXPR_ITEM_NUM*MAX_CHARSET_NUM]={0};
int expr_num=0;
struct _Maat_group_inner_t* group_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
struct op_expr_t* op_expr=NULL;
GIE_digest_t* digest_rule=NULL;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id);
group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(maat_scanner->group_hash, group_id);
if(group_rule==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
@@ -2308,29 +2339,29 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int
}
return 0;
}
int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct Maat_scanner_t *scanner,void* logger)
{
struct _Maat_group_inner_t* group_rule=NULL;
struct _Maat_compile_inner_t*compile_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
struct Maat_compile_inner_t*compile_rule=NULL;
int ret=0;
group_rule=(struct _Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
group_rule=(struct Maat_group_inner_t*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(db_group_rule->group_id);
group_rule->table_id=table->table_id;
ret=HASH_add_by_id(scanner->group_hash, db_group_rule->group_id,group_rule);
ret=HASH_add_by_id(scanner->group_hash, db_group_rule->group_id, group_rule);
assert(ret>=0);
}
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id);
compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id);
if(compile_rule==NULL)
{
compile_rule=create_compile_rule(db_group_rule->compile_id);
ret=HASH_add_by_id(scanner->compile_hash,db_group_rule->compile_id, compile_rule);
ret=HASH_add_by_id(scanner->compile_hash, db_group_rule->compile_id, compile_rule);
assert(ret>=0);
}
ret=add_group_to_compile(compile_rule,group_rule);
ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->not_flag);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module,
@@ -2343,11 +2374,11 @@ int add_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_grou
return 0;
}
void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct _Maat_scanner_t *scanner,void* logger)
void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_group_rule,struct Maat_scanner_t *scanner,void* logger)
{
struct _Maat_compile_inner_t*compile_rule=NULL;
struct _Maat_group_inner_t* group_rule=NULL;
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id);
struct Maat_compile_inner_t*compile_rule=NULL;
struct Maat_group_inner_t* group_rule=NULL;
compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->compile_id);
if(compile_rule==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
@@ -2366,7 +2397,7 @@ void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_gro
,db_group_rule->compile_id);
return;
}
if(compile_rule->group_cnt==0&&compile_rule->db_c_rule==NULL)
if(compile_rule->group_cnt==0&&compile_rule->is_valid==0)
{
HASH_delete_by_id(scanner->compile_hash, db_group_rule->compile_id);
garbage_bagging(GARBAGE_COMPILE_RULE, compile_rule, scanner->tomb_ref);
@@ -2378,12 +2409,12 @@ void del_group_rule(struct Maat_table_desc* table,struct db_group_rule_t* db_gro
}
return;
}
int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger)
int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct Maat_scanner_t *scanner,void* logger)
{
struct _Maat_compile_inner_t *compile_rule=NULL;
struct Maat_compile_inner_t *compile_rule=NULL;
struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head);
int i=0;
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
if(compile_rule==NULL)
{
compile_rule=create_compile_rule(p_maat_rule_head->config_id);
@@ -2410,10 +2441,10 @@ int add_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_
return 0;
}
int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct _Maat_scanner_t *scanner,void* logger)
int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_compile_rule,struct Maat_scanner_t *scanner,void* logger)
{
struct _Maat_compile_inner_t *compile_rule=NULL;
compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id);
struct Maat_compile_inner_t *compile_rule=NULL;
compile_rule=(struct Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id);
if(compile_rule==NULL)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
@@ -2433,40 +2464,47 @@ int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule_t* db_
}
return 1;
}
void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger)
void update_group_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger)
{
struct db_group_rule_t db_group_rule;
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
int ret=0;
ret=sscanf(table_line,"%d\t%d\t%d",&(db_group_rule.group_id)
,&(db_group_rule.compile_id)
,&(db_group_rule.is_valid));
if(ret!=3)
memset(&db_group_rule, 0, sizeof(db_group_rule));
ret=sscanf(table_line,"%d\t%d\t%d\t%d", &(db_group_rule.group_id),
&(db_group_rule.compile_id),
&(db_group_rule.is_valid),
&(db_group_rule.not_flag));
if((ret!=3&&ret!=4) ||
(db_group_rule.not_flag!=0&&db_group_rule.not_flag!=1))
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"update error,invalid format of group table %s:%s"
,table->table_name[table->updating_name], table_line);
"update error,invalid format of group table %s:%s",
table->table_name[table->updating_name], table_line);
table->udpate_err_cnt++;
return;
}
if(db_group_rule.is_valid==FALSE)
{
del_group_rule(table, &db_group_rule,scanner,logger);
del_group_rule(table, &db_group_rule, scanner, logger);
//leave no trace when compatible_group_update calling
if(table->table_type==TABLE_TYPE_GROUP)
{
table_rt->origin_rule_num--;
if(db_group_rule.not_flag)
{
table_rt->group.not_flag_group--;
}
}
}
else
{
ret=add_group_rule(table,&db_group_rule, scanner,logger);
ret=add_group_rule(table,&db_group_rule, scanner, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"duplicate config of group table %s group_id %d compile_id %d.",table->table_name[0]
,db_group_rule.group_id
,db_group_rule.compile_id);
"duplicate config of group table %s group_id %d compile_id %d.", table->table_name[0],
db_group_rule.group_id,
db_group_rule.compile_id);
}
else
@@ -2475,13 +2513,17 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru
if(table->table_type==TABLE_TYPE_GROUP)
{
table_rt->origin_rule_num++;
if(db_group_rule.not_flag)
{
table_rt->group.not_flag_group++;
}
}
}
}
return;
}
void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct _Maat_scanner_t *scanner,void* logger)
void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int compile_id,int is_valid,struct Maat_scanner_t *scanner,void* logger)
{
char virtual_group_line[256];
snprintf(virtual_group_line,sizeof(virtual_group_line),
@@ -2489,7 +2531,7 @@ void compatible_group_udpate(struct Maat_table_desc* table,int region_id,int com
update_group_rule(table, virtual_group_line,scanner,logger);
return;
}
void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on)
{
struct db_str_rule_t* maat_str_rule=ALLOC(struct db_str_rule_t, 1);
int ret=0,db_hexbin=0,rule_type=0;
@@ -2661,7 +2703,7 @@ error_out:
free(maat_str_rule);
maat_str_rule=NULL;
}
void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
void update_ip_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on)
{
struct db_ip_rule_t* ip_rule=(struct db_ip_rule_t*)calloc(sizeof(struct db_ip_rule_t),1);
char src_ip[40],mask_src_ip[40],dst_ip[40],mask_dst_ip[40];
@@ -2805,7 +2847,7 @@ error_out:
ip_rule=NULL;
}
void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
void update_intval_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on)
{
struct db_intval_rule_t* intval_rule=ALLOC(struct db_intval_rule_t, 1);
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
@@ -2880,7 +2922,7 @@ error_out:
intval_rule=NULL;
}
void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger)
void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger)
{
struct compile_table_desc* compile_desc=&(table->compile);
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
@@ -2966,7 +3008,7 @@ no_save:
return;
}
void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct _Maat_scanner_t *scanner,void* logger,int group_mode_on)
void update_digest_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner,void* logger,int group_mode_on)
{
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct db_digest_rule_t* digest_rule=ALLOC(struct db_digest_rule_t, 1);
@@ -3118,17 +3160,17 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
destroy_group_rule(bag->group_rule);
break;
case GARBAGE_SCANNER:
ref_cnt=alignment_int64_array_sum(bag->scanner->ref_cnt,bag->scanner->max_thread_num);
ref_cnt=alignment_int64_array_sum(bag->scanner->ref_cnt, bag->scanner->max_thread_num);
if(ref_cnt==0)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"scanner %p version %d has no reference peacefully destroyed.",bag->scanner,bag->scanner->version);
"scanner %p version %d has no reference peacefully destroyed.", bag->scanner, bag->scanner->version);
}
else
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
"scanner %p version %d force destroyed,ref_cnt %lld.",
bag->scanner,bag->scanner->version,ref_cnt);
"scanner %p version %d force destroyed, ref_cnt %lld.",
bag->scanner, bag->scanner->version, ref_cnt);
}
destroy_maat_scanner(bag->scanner);
break;
@@ -3170,7 +3212,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
q_cnt,bury_cnt);
}
}
void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger)
void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Maat_scanner_t* scanner, const struct rule_tag* tags, int n_tags, void* logger)
{
int i=0, ret=1;
unsigned int len=strlen(table_line)+1;
@@ -3179,19 +3221,25 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
char *p=NULL;
char* copy=NULL;
size_t is_valid_offset=0, valid_len=0;
char *token=NULL,*sub_token=NULL,*saveptr;
size_t accept_tag_offset=0, accept_tag_len=0;
if(plugin_desc->rule_tag_column>0&&n_tags>0)
{
copy=_maat_strdup(table_line);
for (token = copy, i=0; i<plugin_desc->rule_tag_column ; token= NULL, i++)
ret=Maat_helper_read_column(table_line, plugin_desc->rule_tag_column, &accept_tag_offset, &accept_tag_len);
if(ret<0)
{
sub_token= strtok_r(token," \t", &saveptr);
if (sub_token == NULL)
break;
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error, could not locate tag in column %d of plugin table %s:%s",
plugin_desc->rule_tag_column,
table->table_name[table->updating_name],
table_line);
table->udpate_err_cnt++;
return;
}
if(i==plugin_desc->rule_tag_column&&strlen(sub_token)>2)
if(accept_tag_len>2)
{
ret=compare_accept_tag(sub_token, tags, n_tags);
copy=ALLOC(char, accept_tag_len+1);
memcpy(copy, table_line+accept_tag_offset, accept_tag_len);
ret=compare_accept_tag(copy, tags, n_tags);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
@@ -3203,9 +3251,9 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
{
table->unmatch_tag_cnt++;
}
free(copy);
copy=NULL;
}
free(copy);
copy=NULL;
if(ret!=1)
{
return;
@@ -3218,7 +3266,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
if(plugin_desc->have_exdata)
{
ret=get_column_pos(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len);
ret=Maat_helper_read_column(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len);
pthread_rwlock_wrlock(&(table_rt->plugin.rwlock));
if(atoi(table_line+is_valid_offset)==1)
@@ -3249,9 +3297,9 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,_M
table_rt->plugin.cache_line_num++;
}
}
void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger)
void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger)
{
void *tmp1=NULL,*tmp2=NULL;
struct bool_matcher *tmp1=NULL,*tmp2=NULL;
MESA_htable_handle tmp_map=NULL;
struct Maat_table_runtime* table_rt=NULL;
int i=0;
@@ -3260,12 +3308,12 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
para.gram_value=7;
para.position_accuracy=10;
tmp1=create_bool_matcher(scanner->compile_hash,
scan_thread_num,
logger);
tmp2=scanner->bool_macher_expr_compiler;
scan_thread_num,
logger);
tmp2=scanner->bool_matcher_expr_compiler;
//assume pinter = operation is thread safe
scanner->bool_macher_expr_compiler=tmp1;
scanner->bool_matcher_expr_compiler=tmp1;
if(tmp2!=NULL)
{
garbage_bagging(GARBAGE_BOOL_MATCHER, tmp2, garbage_q);
@@ -3275,8 +3323,8 @@ void do_scanner_update(struct _Maat_scanner_t* scanner,MESA_lqueue_head garbage_
scanner->dedup_expr_num=0;
rulescan_batch_update(scanner->region,
scanner->region_update_q,
logger
,scanner);
logger,
scanner);
for(i=0;i<MAX_TABLE_NUM;i++)
{
table_rt=scanner->table_rt[i];
@@ -3379,7 +3427,7 @@ void maat_start_cb(long long new_version,int update_type,void*u_para)
}
return;
}
long long scanner_rule_num(struct _Maat_scanner_t *scanner)
long long scanner_rule_num(struct Maat_scanner_t *scanner)
{
long long total=0;
struct Maat_table_runtime* table_rt=NULL;
@@ -3484,7 +3532,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
int ret=-1,i=0;
int table_id=-1;
_Maat_scanner_t* scanner=NULL;
Maat_scanner_t* scanner=NULL;
struct Maat_table_desc* p_table=NULL;
if(feather->update_tmp_scanner!=NULL)
{
@@ -3543,7 +3591,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
void *thread_rule_monitor(void *arg)
{
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg;
struct _Maat_scanner_t* old_scanner=NULL;
struct Maat_scanner_t* old_scanner=NULL;
long expr_wait_q_cnt=0;
int scan_dir_cnt=0;
int ret=0;
@@ -3725,7 +3773,7 @@ void *thread_rule_monitor(void *arg)
alignment_int64_array_free(feather->outer_mid_cnt);
alignment_int64_array_free(feather->hit_cnt);
alignment_int64_array_free(feather->orphan_group_saving);
alignment_int64_array_free(feather->last_region_saving);
alignment_int64_array_free(feather->not_grp_hit_cnt);
if(feather->input_mode==SOURCE_REDIS)
{
if(feather->mr_ctx.read_ctx)
@@ -3745,6 +3793,7 @@ void *thread_rule_monitor(void *arg)
free(feather->accept_tags[i].tag_val);
}
free(feather->accept_tags);
free(feather->accept_tags_raw);
if(feather->stat_on&& feather->stat_handle)
{
FS_stop(&(feather->stat_handle));

47
src/entry/Maat_stat.cpp
View File

@@ -8,7 +8,8 @@ enum MAAT_FS_STATUS{
STATUS_TABLE_NUM,
STATUS_PLUGIN_CACHE_NUM,
STATUS_PLUGIN_ACC_NUM,
STATUS_GROUP_RULE_NUM,
STATUS_GROUP_REF_NUM,
STATUS_GROUP_REF_NOT_NUM,
STATUS_COMPILE_RULE_NUM,
STATUS_POSTPONE_QSIZE,
STATUS_OUTER_MID_NUM,
@@ -21,7 +22,7 @@ enum MAAT_FS_STATUS{
STATUS_SCAN_ERR_CNT,
STATUS_ZOMBIE_RS_STREAM,
STATUS_ORPHAN_GROUP_SAVING,
STATUS_LAST_REGION_SAVING,
STATUS_NOT_GROUP_HIT,
STATUS_CMD_NUM,
STATUS_CMD_Q_SIZE,
STATUS_CMD_LINE_NUM
@@ -55,14 +56,15 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_set_para(feather->stat_handle, APP_NAME, feather->instance_name, strlen(feather->instance_name)+1);
feather->fs_status_id[STATUS_VERSION]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"version");
feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"thread");
feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"table_num");
feather->fs_status_id[STATUS_THRED_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"threads");
feather->fs_status_id[STATUS_TABLE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"tables");
feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_cache");
feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_cached");
feather->fs_status_id[STATUS_PLUGIN_ACC_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"plug_acc");
feather->fs_status_id[STATUS_GROUP_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"grp_num");
feather->fs_status_id[STATUS_COMPILE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"compile_num");
feather->fs_status_id[STATUS_GROUP_REF_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"group");
feather->fs_status_id[STATUS_GROUP_REF_NOT_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"not_grp");
feather->fs_status_id[STATUS_COMPILE_RULE_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"compile");
feather->fs_status_id[STATUS_POSTPONE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"postponed");
feather->fs_status_id[STATUS_GARBAGE_QSIZE]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"garbage_num");
@@ -71,8 +73,8 @@ void maat_stat_init(struct _Maat_feather_t* feather)
feather->fs_status_id[STATUS_INNER_MID_NUM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"inner_mid");
feather->fs_status_id[STATUS_ZOMBIE_RS_STREAM]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"z_stream");
feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"only_grp_sv");
feather->fs_status_id[STATUS_LAST_REGION_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"last_rgn_sv");
feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"shortcut_sv");
feather->fs_status_id[STATUS_NOT_GROUP_HIT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"nt_grp_hit");
feather->fs_status_id[STATUS_TOTAL_SCAN_LEN]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_bytes");
feather->fs_status_id[STATUS_TOTAL_SCAN_CNT]=FS_register(feather->stat_handle, FS_STYLE_STATUS, FS_CALC_CURRENT,"scan_times");
@@ -166,9 +168,9 @@ void maat_stat_output(struct _Maat_feather_t* feather)
long long total_scan_cnt=0, total_cpu_time=0,total_stream_cnt=0,active_thread_num=0;
long long table_stream_num=0,table_scan_cnt=0,table_input_bytes=0,table_scan_cpu_time=0,table_hit_cnt=0;
long long outer_mid_cnt=0,inner_mid_cnt=0;
long long orphan_group_saving=0, last_region_saving=0;
long long orphan_group_saving=0, not_grp_hit_cnt=0;
long long total_update_error=0,total_iconv_error=0;
long long compile_rule_num=0,group_rule_num=0,plugin_cache_num=0,plugin_acc_num=0;
long long compile_rule_num=0, group_rule_num=0, not_group_rule_num=0, plugin_cache_num=0, plugin_acc_num=0;
int i=0;
time_t now;
struct Maat_table_desc* p_table=NULL;
@@ -182,7 +184,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
outer_mid_cnt=alignment_int64_array_sum(feather->outer_mid_cnt,feather->scan_thread_num);
inner_mid_cnt=alignment_int64_array_sum(feather->inner_mid_cnt,feather->scan_thread_num);
orphan_group_saving=alignment_int64_array_sum(feather->orphan_group_saving,feather->scan_thread_num);
last_region_saving=alignment_int64_array_sum(feather->last_region_saving,feather->scan_thread_num);
not_grp_hit_cnt=alignment_int64_array_sum(feather->not_grp_hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num);
@@ -190,7 +192,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_ORPHAN_GROUP_SAVING], 0,FS_OP_SET,orphan_group_saving);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_LAST_REGION_SAVING], 0,FS_OP_SET,last_region_saving);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_NUM], 0,FS_OP_SET,feather->cmd_acc_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0,FS_OP_SET,feather->cmd_q_cnt);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_LINE_NUM], 0,FS_OP_SET,feather->line_cmd_acc_num);
@@ -221,6 +223,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
break;
case TABLE_TYPE_GROUP:
group_rule_num+=table_rt->origin_rule_num;
not_group_rule_num+=table_rt->group.not_flag_group;
break;
case TABLE_TYPE_COMPILE:
compile_rule_num+=table_rt->origin_rule_num;
@@ -318,34 +321,35 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_STREAM_NUM],
FS_OP_SET,
FS_OP_ADD,
total_stream_cnt);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_CNT],
FS_OP_SET,
FS_OP_ADD,
total_scan_cnt);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_SCAN_BYTES],
FS_OP_SET,
FS_OP_ADD,
total_input_bytes);
if(feather->perf_on==1)
{
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_CPU_TIME],
FS_OP_SET,
FS_OP_ADD,
total_cpu_time);
}
total_hit_cnt=alignment_int64_array_sum(feather->hit_cnt, feather->scan_thread_num);
alignment_int64_array_reset(feather->hit_cnt,feather->scan_thread_num);
FS_operate(feather->stat_handle,
feather->total_stat_id,
feather->fs_column_id[COLUMN_TABLE_HIT_CNT],
FS_OP_SET,
FS_OP_ADD,
total_hit_cnt);
feather->total_scan_bytes=total_input_bytes;
feather->total_scan_cnt=total_scan_cnt;
feather->total_scan_bytes+=total_input_bytes;
feather->total_scan_cnt+=total_scan_cnt;
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_LEN], 0,FS_OP_SET,feather->total_scan_bytes);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TOTAL_SCAN_CNT], 0,FS_OP_SET,feather->total_scan_cnt);
feather->update_err_cnt=total_update_error;
@@ -357,7 +361,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_CACHE_NUM], 0,FS_OP_SET,plugin_cache_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_PLUGIN_ACC_NUM], 0,FS_OP_SET,plugin_acc_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_RULE_NUM], 0,FS_OP_SET,group_rule_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_REF_NUM], 0,FS_OP_SET,group_rule_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GROUP_REF_NOT_NUM], 0,FS_OP_SET,not_group_rule_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_COMPILE_RULE_NUM], 0,FS_OP_SET,compile_rule_num);
FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_POSTPONE_QSIZE], 0,FS_OP_SET,feather->postpone_q_size);

296
src/entry/bool_matcher.cpp Normal file
View File

@@ -0,0 +1,296 @@
#include "bool_matcher.h"
#include <map>
#include <vector>
#include <algorithm>
using namespace std;
#include <stdlib.h>
#include <stdio.h>
static const unsigned int MAX_ARRAY_SIZE=65536;
struct thread_local_data_t
{
unsigned int mapped_ids[MAX_ARRAY_SIZE];
unsigned int used_cells[MAX_ARRAY_SIZE];
void * cached_results[MAX_ARRAY_SIZE];
unsigned char * multiexpr_bitmap;
unsigned int * singlexpr_bitmap;
};
struct bool_matcher
{
unsigned int max_thread_num;
unsigned int bool_expr_num;
unsigned int multi_expr_num;
void ** bool_expr_ids;
unsigned char * multi_expr_size;
unsigned char * multi_expr_mask;
unsigned int bool_item_id_num;
unsigned long long min_item_id;
unsigned long long max_item_id;
unsigned long long * bool_item_ids;
unsigned int * mapped_ptr;
unsigned int * mapped_ids;
unsigned int theta;
unsigned int L[65537];
thread_local_data_t * thread_data;
};
struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, unsigned int max_thread_num, size_t * mem_size)
{
if(exprs==NULL || expr_num==0 || max_thread_num==0) return NULL;
for(unsigned int i=0; i<expr_num; i++)
{
if(exprs[i].item_num==0 || exprs[i].item_num>MAX_ITEMS_PER_BOOL_EXPR)
{
return NULL;
}
}
int I=-1, J=(int)expr_num;
while(I<J)
{
I++;
while(I<J && exprs[I].item_num>1) I++;
if(I==J) break;
J--;
while(J>I && exprs[J].item_num==1) J--;
if(J==I) break;
swap(exprs[I], exprs[J]);
}
for(int k=0; k<(int)expr_num; k++)
{
if((k<I && exprs[k].item_num==1) || (k>=I && exprs[k].item_num>1))
{
printf("[%s:%d]: fatal error!\n", __FILE__, __LINE__);
return NULL;
}
}
unsigned int mem_bytes=0;
struct bool_matcher * matcher=new struct bool_matcher;
mem_bytes+=sizeof(bool_matcher);
matcher->max_thread_num=max_thread_num;
matcher->bool_expr_num=(unsigned int)expr_num;
matcher->multi_expr_num=I;
matcher->bool_expr_ids=new void *[expr_num];
mem_bytes+=(unsigned int)expr_num*sizeof(void *);
matcher->multi_expr_size=new unsigned char[matcher->multi_expr_num+1];
mem_bytes+=(matcher->multi_expr_num+1)*sizeof(unsigned char);
matcher->multi_expr_mask=new unsigned char[matcher->multi_expr_num+1];
mem_bytes+=(matcher->multi_expr_num+1)*sizeof(unsigned char);
matcher->thread_data=new thread_local_data_t[max_thread_num];
mem_bytes+=max_thread_num*sizeof(thread_local_data_t);
for(unsigned int i=0; i<max_thread_num; i++)
{
matcher->thread_data[i].multiexpr_bitmap=new unsigned char[matcher->multi_expr_num+1];
mem_bytes+=(matcher->multi_expr_num+1)*sizeof(unsigned char);
unsigned int size=(unsigned int)(expr_num-matcher->multi_expr_num);
size=(size>>5)+1;
matcher->thread_data[i].singlexpr_bitmap=new unsigned int[size];
mem_bytes+=size*sizeof(unsigned int);
}
map< unsigned long long, vector<unsigned int> > M;
unsigned int count=0;
for(unsigned int i=0; i<expr_num; i++)
{
matcher->bool_expr_ids[i]=exprs[i].user_tag;
if(i<matcher->multi_expr_num)
{
matcher->multi_expr_size[i]=(unsigned int)exprs[i].item_num;
}
count+=(unsigned int)exprs[i].item_num;
unsigned char mask=0;
for(unsigned int j=0; j<exprs[i].item_num; j++)
{
if(exprs[i].items[j].not_flag==1) mask|=(1U<<j);
M[exprs[i].items[j].item_id].push_back((i<<3)|j);
}
if(i<matcher->multi_expr_num) matcher->multi_expr_mask[i]=mask;
}
matcher->bool_item_id_num=(unsigned int)M.size();
matcher->bool_item_ids=new unsigned long long[M.size()];
matcher->mapped_ptr =new unsigned int[M.size()+1];
matcher->mapped_ids =new unsigned int[count];
mem_bytes+=((unsigned int)M.size()+1+count)*sizeof(unsigned int)+(unsigned int)M.size()*sizeof(unsigned long long);
matcher->mapped_ptr[0]=0;
map< unsigned long long, vector<unsigned int> >::const_iterator it=M.begin();
for(unsigned int k=0; k<M.size(); ++k, ++it)
{
matcher->bool_item_ids[k]=it->first;
copy(it->second.begin(), it->second.end(), matcher->mapped_ids+matcher->mapped_ptr[k]);
matcher->mapped_ptr[k+1]=matcher->mapped_ptr[k]+(unsigned int)it->second.size();
}
matcher->min_item_id=matcher->bool_item_ids[0];
matcher->max_item_id=matcher->bool_item_ids[M.size()-1];
for(unsigned int k=0; k<M.size(); ++k)
{
matcher->bool_item_ids[k]-=matcher->min_item_id;
}
const unsigned long long ONE=1;
unsigned int theta=0;
while((ONE<<(theta+16))<=matcher->bool_item_ids[M.size()-1]) theta++;
matcher->theta=theta;
matcher->L[0]=0;
for(unsigned int i=1; i<65536; i++)
{
matcher->L[i]=(unsigned int)(lower_bound(matcher->bool_item_ids, matcher->bool_item_ids+M.size(), i*(ONE<<theta))-matcher->bool_item_ids);
}
matcher->L[65536]=(unsigned int)M.size();
M.clear();
*mem_size=mem_bytes;
return matcher;
}
int bool_matcher_match(struct bool_matcher * matcher, unsigned int thread_id, unsigned long long * item_ids, size_t item_num, void ** result, size_t size)
{
if(matcher==NULL) return -1;
if(thread_id>=matcher->max_thread_num) return -1;
unsigned int * mapped_ids=matcher->thread_data[thread_id].mapped_ids;
unsigned int ids_num=0;
for(unsigned int i=0; i<item_num; i++)
{
if(item_ids[i]<matcher->min_item_id || item_ids[i]>matcher->max_item_id) continue;
unsigned long long id=item_ids[i]-matcher->min_item_id;
unsigned int k=(unsigned int)(id>>matcher->theta);
int l=matcher->L[k], h=(int)matcher->L[k+1]-1;
if(h<l) continue;
while(l<=h)
{
int m=(l+h)/2;
if(id<matcher->bool_item_ids[m]) h=m-1;
else l=m+1;
}
if(h<(int)matcher->L[k] || matcher->bool_item_ids[h]!=id) continue;
for(unsigned int j=matcher->mapped_ptr[h]; j<matcher->mapped_ptr[h+1]; j++)
{
if(ids_num==MAX_ARRAY_SIZE) return -1;
mapped_ids[ids_num++]=matcher->mapped_ids[j];
}
}
unsigned int * used_cells=matcher->thread_data[thread_id].used_cells;
unsigned int used_num=0;
for(unsigned int i=0; i<ids_num; i++)
{
if(used_num==MAX_ARRAY_SIZE) return -1;
used_cells[used_num++]=(mapped_ids[i]>>3);
}
unsigned char * m_bitmap=matcher->thread_data[thread_id].multiexpr_bitmap;
unsigned int * s_bitmap=matcher->thread_data[thread_id].singlexpr_bitmap;
unsigned char * m_mask=matcher->multi_expr_mask;
for(unsigned int i=0; i<used_num; i++)
{
if(used_cells[i]<matcher->multi_expr_num)
{
m_bitmap[used_cells[i]]=m_mask[used_cells[i]];
}
else
{
unsigned int j=used_cells[i]-matcher->multi_expr_num;
s_bitmap[j>>5]&=~(1U<<(j&31));
}
}
for(unsigned int i=0; i<ids_num; i++)
{
unsigned int x=(mapped_ids[i]>>3);
if(x<matcher->multi_expr_num)
{
unsigned int y=(mapped_ids[i]&7);
if(m_mask[x]&(1U<<y))
{
m_bitmap[x]&=~(1U<<y);
}
else
{
m_bitmap[x]|=(1U<<y);
}
}
else
{
unsigned int j=x-matcher->multi_expr_num;
s_bitmap[j>>5]|=(1U<<(j&31));
}
}
unsigned int r=0;
void ** cached_results=matcher->thread_data[thread_id].cached_results;
for(unsigned int i=0; i<used_num; i++)
{
unsigned int x=used_cells[i];
if(x<matcher->multi_expr_num)
{
if(m_bitmap[x]==(1U<<matcher->multi_expr_size[x])-1)
{
if(r<MAX_ARRAY_SIZE) cached_results[r++]=matcher->bool_expr_ids[x];
}
}
else
{
unsigned int j=used_cells[i]-matcher->multi_expr_num;
if((s_bitmap[j>>5]&(1U<<(j&31)))!=0)
{
if(r<MAX_ARRAY_SIZE) cached_results[r++]=matcher->bool_expr_ids[x];
}
}
}
sort(cached_results, cached_results+r);
int I=0;
for(unsigned int J=0; J<r; ++J)
{
if(I==0 || cached_results[J]!=result[I-1])
{
if(I==(int)size) return I;
result[I++]=cached_results[J];
}
}
return I;
}
void bool_matcher_free(struct bool_matcher * matcher)
{
if(matcher==NULL) return;
delete [] matcher->bool_expr_ids;
delete [] matcher->multi_expr_size;
delete [] matcher->multi_expr_mask;
delete [] matcher->bool_item_ids;
delete [] matcher->mapped_ptr;
delete [] matcher->mapped_ids;
for(unsigned int i=0; i<matcher->max_thread_num; i++)
{
delete [] matcher->thread_data[i].multiexpr_bitmap;
delete [] matcher->thread_data[i].singlexpr_bitmap;
}
delete [] matcher->thread_data;
delete matcher;
return;
}

18
src/entry/json2iris.cpp
View File

@@ -770,7 +770,7 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l
set_file_rulenum(table_info->table_path,table_info->line_count,logger);
return compile_id;
}
int write_group_rule(int compile_id ,int group_id,struct iris_description_t *p_iris,void * logger)
int write_group_rule(int compile_id ,int group_id, int group_not_flag, struct iris_description_t *p_iris,void * logger)
{
FILE*fp=NULL;
int ret=0;
@@ -790,7 +790,7 @@ int write_group_rule(int compile_id ,int group_id,struct iris_description_t *p_i
"fopen %s error %s.",p_iris->group_table->table_path,strerror(errno));
return -1;
}
fprintf(fp,"%d\t%d\t1\n",group_id,compile_id);
fprintf(fp,"%d\t%d\t1\t%d\n",group_id, compile_id, group_not_flag);
fclose(fp);
p_iris->group_table->line_count++;
ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger);
@@ -821,6 +821,7 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
int i=0,j=0,k=0;
int compile_id=-1,compile_cnt=0,group_cnt=0,region_cnt=0,plug_table_cnt=0;
int ret=0;
int group_not_flag=0;
cJSON *c_rules=NULL,*g_rules=NULL,*r_rules=NULL,*item=NULL,*plug_tables=NULL;
cJSON *compile_rule=NULL,*group_rule=NULL,*region_rule=NULL,*each_plug_table=NULL;
const char* group_name=NULL;
@@ -878,6 +879,15 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
for(j=0;j<group_cnt;j++)
{
group_rule=cJSON_GetArrayItem(g_rules,j);
item=cJSON_GetObjectItem(group_rule,"not_flag");
if(item==NULL||item->type!=cJSON_Number)
{
group_not_flag=0;
}
else
{
group_not_flag=item->valueint;
}
item=cJSON_GetObjectItem(group_rule,"group_name");
if(item==NULL||item->type!=cJSON_String)
{
@@ -890,7 +900,7 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name));
if(group_info!=NULL)//exist group name ,region already read
{
ret=write_group_rule(compile_id, group_info->group_id, p_iris, logger);
ret=write_group_rule(compile_id, group_info->group_id, group_not_flag, p_iris, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
@@ -942,7 +952,7 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
return -1;
}
}
ret=write_group_rule(compile_id, group_info->group_id, p_iris, logger);
ret=write_group_rule(compile_id, group_info->group_id, group_not_flag, p_iris, logger);
if(ret<0)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,

62
src/inc_internal/Maat_rule_internal.h
View File

@@ -10,7 +10,7 @@
#include <MESA/field_stat2.h>
#include <MESA/rulescan.h>
#include "dynamic_array.h"
#include "UniversalBoolMatch.h"
#include "bool_matcher.h"
#include "hiredis.h"
#include "stream_fuzzy_hash.h"
@@ -108,6 +108,7 @@ struct db_group_rule_t
int group_id;
int compile_id;
int is_valid;
int not_flag;
};
struct op_expr_t
{
@@ -129,7 +130,7 @@ struct _Maat_region_inner_t
int expr_id_ub;
enum MAAT_TABLE_TYPE table_type;
};
struct _Maat_group_inner_t
struct Maat_group_inner_t
{
int group_id;
int table_id;
@@ -141,30 +142,33 @@ struct _Maat_group_inner_t
void* compile_shortcut;
pthread_mutex_t mutex;
};
struct _Maat_compile_inner_t
struct Maat_compile_inner_t
{
struct db_compile_rule_t *db_c_rule;
dynamic_array_t *groups;
int is_valid;
dynamic_array_t *groups; //element is struct Maat_group_inner_t*
char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
char is_valid;
int compile_id;//equal to db_c_rule->m_rule.config_id
const struct Maat_table_desc* ref_table;
int group_boundary;
int group_cnt;
int not_group_cnt;
MAAT_RULE_EX_DATA* ads;
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
};
struct _compile_result_t
{
int compile_id;
universal_bool_expr_t group_set;
struct bool_expr group_set;
};
struct _INNER_scan_status_t
{
int cur_hit_cnt;
int hit_group_cnt;
int hit_group_size;
unsigned int cur_hit_id[MAX_SCANNER_HIT_NUM];
unsigned int *hitted_group_id;
size_t cur_hit_cnt;
size_t hit_group_cnt;
size_t hit_group_size;
unsigned long long cur_hit_id[MAX_SCANNER_HIT_NUM];
unsigned long long *hitted_group_id;
char not_grp_compile_hitted_flag;
};
struct _OUTER_scan_status_t
{
@@ -234,6 +238,10 @@ struct ip_runtime
long long ipv6_rule_cnt;
};
struct group_runtime
{
long long not_flag_group;
};
struct Maat_table_runtime
{
enum MAAT_TABLE_TYPE table_type;
@@ -244,6 +252,7 @@ struct Maat_table_runtime
struct plugin_runtime plugin;
struct expr_runtime expr;
struct ip_runtime ip;
struct group_runtime group;
void * other;
};
mcore_long_t scan_cnt;
@@ -257,7 +266,7 @@ struct rule_tag
char* tag_name;
char* tag_val;
};
struct _Maat_scanner_t
struct Maat_scanner_t
{
long long version;
time_t last_update_time;
@@ -266,10 +275,7 @@ struct _Maat_scanner_t
long gie_total_q_size;
struct Maat_table_runtime* table_rt[MAX_TABLE_NUM];
/*
struct similar_runtime gie_aux[MAX_TABLE_NUM];
struct plugin_runtime plugin_aux[MAX_TABLE_NUM];
*/
MESA_htable_handle region_hash;
MESA_htable_handle group_hash;
MESA_htable_handle compile_hash;
@@ -280,10 +286,10 @@ struct _Maat_scanner_t
unsigned int exprid_generator;
unsigned int dedup_expr_num;
MESA_lqueue_head region_update_q;
void * bool_macher_expr_compiler;
struct bool_matcher * bool_matcher_expr_compiler;
scan_result_t *region_rslt_buff;
MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q
// struct _region_stat_t region_counter[MAX_TABLE_NUM];
int max_thread_num;
iconv_t iconv_handle[MAX_CHARSET_NUM][MAX_CHARSET_NUM];//iconv_handle[to][from]
};
@@ -317,8 +323,8 @@ struct source_redis_ctx
};
struct _Maat_feather_t
{
struct _Maat_scanner_t *scanner;
struct _Maat_scanner_t *update_tmp_scanner;
struct Maat_scanner_t *scanner;
struct Maat_scanner_t *update_tmp_scanner;
MESA_lqueue_head garbage_q;
int table_cnt;
int DEFERRED_LOAD_ON;
@@ -363,8 +369,10 @@ struct _Maat_feather_t
long long base_rgn_seq,base_grp_seq,server_time;
long long load_version_from;
char* accept_tags_raw;
struct rule_tag *accept_tags;
int n_tags;
char foreign_cont_dir[MAX_TABLE_NAME_LEN];
int foreign_cont_linger;
//internal states
@@ -372,7 +380,7 @@ struct _Maat_feather_t
int active_plugin_table_num;
int is_last_plugin_table_updating;
//for stat>>>>
//for scanner independent stat>>>>
int backgroud_update_enabled;
screen_stat_handle_t stat_handle;
int total_stat_id;
@@ -383,7 +391,7 @@ struct _Maat_feather_t
mcore_long_t hit_cnt;
mcore_long_t thread_call_cnt;//size indicate by scan_thread_num,
mcore_long_t orphan_group_saving;
mcore_long_t last_region_saving;
mcore_long_t not_grp_hit_cnt;
long long total_scan_bytes;
long long total_scan_cnt;
long long update_err_cnt;//sum of the same name variable in each table
@@ -403,10 +411,10 @@ struct _maat_garbage_t
int ok_times;
union
{
struct _Maat_scanner_t* scanner;
struct _Maat_group_inner_t* group_rule;
struct _Maat_compile_inner_t* compile_rule;
void* bool_matcher;
struct Maat_scanner_t* scanner;
struct Maat_group_inner_t* group_rule;
struct Maat_compile_inner_t* compile_rule;
struct bool_matcher* bool_matcher;
void * raw;
MESA_htable_handle str2int_map;
char* filename;
@@ -435,7 +443,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger);
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set);
void make_group_set(struct Maat_compile_inner_t* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para);

56
src/inc_internal/bool_matcher.h Normal file
View File

@@ -0,0 +1,56 @@
/*
*
* Copyright (c) 2018
* String Algorithms Research Group
* Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
* National Engineering Laboratory for Information Security Technologies (NELIST)
* All rights reserved
*
* Written by: LIU YANBING (liuyanbing@iie.ac.cn)
* Last modification: 2018-12-31
*
* This code is the exclusive and proprietary property of IIE-CAS and NELIST.
* Usage for direct or indirect commercial advantage is not allowed without
* written permission from the authors.
*
*/
#ifndef INCLUDE_BOOL_MATCHER_H
#define INCLUDE_BOOL_MATCHER_H
#include <stddef.h>
#ifdef __cplusplus
extern "C"
{
#endif
#define MAX_ITEMS_PER_BOOL_EXPR 8
/*not_flag=0表示布尔项item_id必须出现；not_flag=1表示布尔项item_id不能出现*/
struct bool_item
{
unsigned long long item_id;
unsigned char not_flag;
};
/*注意：不支持布尔项全“非”的情形*/
struct bool_expr
{
void * user_tag;
size_t item_num;
struct bool_item items[MAX_ITEMS_PER_BOOL_EXPR];
};
struct bool_matcher;
/*注意：本函数调用会交换bool_exprs中元素的位置*/
struct bool_matcher * bool_matcher_new(struct bool_expr * exprs, size_t expr_num, unsigned int max_thread_num, size_t * mem_size);
int bool_matcher_match(struct bool_matcher * matcher, unsigned int thread_id, unsigned long long * item_ids, size_t item_num, void ** result, size_t size);
void bool_matcher_free(struct bool_matcher * matcher);
#ifdef __cplusplus
}
#endif
#endif

137
test/maat_json.json
View File

@@ -548,6 +548,143 @@
]
}
]
},
{
"compile_id": 142,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": ",IgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 143,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "NOT-logic-test1",
"is_valid": "yes",
"groups": [
{
"not_flag":0,
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "must-contained-string-of-rule-143",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"not_flag":1,
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "must-not-contained-string-of-rule-143",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 144,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "NOT-logic-test2",
"is_valid": "yes",
"groups": [
{
"not_flag":0,
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "must-contained-string-of-rule-144",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"not_flag":1,
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "string",
"table_content": {
"keywords": "must-not-contained-string-of-rule-144",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 145,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "NOT-logic-test3",
"is_valid": "yes",
"groups": [
{
"not_flag":0,
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "must-contained-string-of-rule-145",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"not_flag":1,
"group_name": "IP_group"
}
]
}
],
"plugin_table": [

13
test/reset_redis4maat.sh
View File

@@ -1,10 +1,11 @@
#!/bin/sh
host="127.0.0.1"
port="6379"
db="0"
echo "Reseting Redis For Maat..."
redis-cli -h $host -p $port GET MAAT_VERSION
redis-cli -h $host -p $port FLUSHALL
redis-cli -h $host -p $port SET MAAT_VERSION "0"
redis-cli -h $host -p $port SET MAAT_PRE_VER "0"
redis-cli -h $host -p $port SET SEQUENCE_REGION "1"
redis-cli -h $host -p $port SET SEQUENCE_GROUP "1"
redis-cli -h $host -p $port -n $db GET MAAT_VERSION
redis-cli -h $host -p $port -n $db FLUSHALL
redis-cli -h $host -p $port -n $db SET MAAT_VERSION "0"
redis-cli -h $host -p $port -n $db SET MAAT_PRE_VER "0"
redis-cli -h $host -p $port -n $db SET SEQUENCE_REGION "1"
redis-cli -h $host -p $port -n $db SET SEQUENCE_GROUP "1"

289
test/test_maatframe.cpp
View File

@@ -36,7 +36,7 @@ Maat_feather_t g_feather=NULL;
void *g_logger=NULL;
int g_iThreadNum=4;
const char* table_info_path="./table_info.conf";
int scan_interval_ms=1;
int scan_interval_ms=500;
int effective_interval_ms=0;
void wait_for_cmd_effective(Maat_feather_t feather, long long version_before)
{
@@ -104,7 +104,7 @@ const char* old_json="./json_update/old.json";
const char* new_json="./json_update/new.json";
const char* corrupted_json="./json_update/corrupted.json";
class MaatJSONTest : public testing::Test
class JSONUpdate : public testing::Test
{
protected:
@@ -126,21 +126,21 @@ protected:
static Maat_feather_t _shared_feather_j;
static void *logger;
};
Maat_feather_t MaatJSONTest::_shared_feather_j;
Maat_feather_t JSONUpdate::_shared_feather_j;
TEST_F(MaatJSONTest, OldCfg)
TEST_F(JSONUpdate, OldCfg)
{
scan_with_old_or_new_cfg(MaatJSONTest::_shared_feather_j, 1);
scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1);
}
TEST_F(MaatJSONTest, NewCfg)
TEST_F(JSONUpdate, NewCfg)
{
system_cmd_cp(corrupted_json, watched_json);
sleep(2);
scan_with_old_or_new_cfg(MaatJSONTest::_shared_feather_j, 1);
scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1);
system_cmd_cp(new_json, watched_json);
sleep(2);
scan_with_old_or_new_cfg(MaatJSONTest::_shared_feather_j, 0);
scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0);
}
void Maat_read_entry_start_cb(int update_type,void* u_para)
@@ -265,7 +265,6 @@ TEST(IPScan, IPv4)
return;
}
TEST(IPScan, IPv6)
{
int table_id=0,ret=0;
struct Maat_rule_t result[4];
@@ -290,6 +289,145 @@ TEST(IPScan, IPv6)
Maat_clean_status(&mid);
return;
}
TEST(NOTLogic, OneRegion)
{
const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-143.";
const char* string_should_not_hit="This string contains both must-contained-string-of-rule-143 and must-not-contained-string-of-rule-143.";
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* table_name="HTTP_URL";
scan_status_t mid=NULL;
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result,found_pos, 4, &mid, 0);
EXPECT_GE(ret, 1);
Maat_clean_status(&mid);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, -2);
Maat_clean_status(&mid);
}
TEST(NOTLogic, ScanNotAtLast)
{
const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-144.";
const char* string_should_not_hit="This string contains both must-contained-string-of-rule-144 and must-not-contained-string-of-rule-144.";
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* hit_table_name="HTTP_URL", *not_hit_table_name="KEYWORDS_TABLE";
scan_status_t mid=NULL;
table_id=Maat_table_register(g_feather,hit_table_name);
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result,found_pos, 4, &mid, 0);
EXPECT_GE(ret, -2);
table_id=Maat_table_register(g_feather,not_hit_table_name);
ASSERT_GT(table_id, 0);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, -2);
Maat_clean_status(&mid);
}
TEST(NOTLogic, ScanIrrelavantAtLast)
{
const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-144.";
const char* string_irrelevant="This string contiains nothing to hit.";
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* hit_table_name="HTTP_URL", *not_hit_table_name="KEYWORDS_TABLE";
scan_status_t mid=NULL;
table_id=Maat_table_register(g_feather,hit_table_name);
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result,found_pos, 4, &mid, 0);
EXPECT_GE(ret, -2);
table_id=Maat_table_register(g_feather, not_hit_table_name);
ASSERT_GT(table_id, 0);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_irrelevant, strlen(string_irrelevant),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, 1);
Maat_clean_status(&mid);
}
TEST(NOTLogic, ScanNotIP)
{
const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-145.";
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* hit_table_name="HTTP_URL", *not_hit_table_name="IP_CONFIG";
scan_status_t mid=NULL;
table_id=Maat_table_register(g_feather,hit_table_name);
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result,found_pos, 4, &mid, 0);
EXPECT_GE(ret, -2);
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
ipv4_addr.addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr));
v4_addr.source=htons(50001);
inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr));
v4_addr.dest=htons(80);
ipv4_addr.v4=&v4_addr;
table_id=Maat_table_register(g_feather, not_hit_table_name);
ASSERT_GT(table_id, 0);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid,0);
EXPECT_EQ(ret, -2);
Maat_clean_status(&mid);
}
TEST(Helper, ReadColumn)
{
const char* ip="192.168.0.1";
const char* tmp="something";
char line[256];
snprintf(line, sizeof(line), "1\t%s\t%s",ip, tmp);
size_t offset=0, len=0;
int ret=0;
ret=Maat_helper_read_column(line, 2, &offset, &len);
EXPECT_EQ(ret, 0);
EXPECT_EQ(0, strncmp(ip, line+offset, len));
ret=Maat_helper_read_column(line, 3, &offset, &len);
EXPECT_EQ(ret, 0);
EXPECT_EQ(0, strncmp(tmp, line+offset, len));
}
TEST(IntervalScan, Pure)
{
@@ -344,10 +482,30 @@ TEST(DigestScan, Pure)
Maat_clean_status(&mid);
return;
}
TEST(StringScan, EncodedURL)
TEST(StringScan, GBKEncodedURL)
{
const char* url_gb2312="www.baidu.com/?wd=C%23%D6%D0%B9%FA";
int table_id=0,ret=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* table_name="HTTP_URL";
scan_status_t mid=NULL;
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_gb2312, strlen(url_gb2312),
result,found_pos, 4,
&mid, 0);
EXPECT_GE(ret, 1);
Maat_clean_status(&mid);
return;
}
TEST(StringScan, UTF8EncodedURL)
{
const char* url_utf8="www.google.com/?q=C%23%E4%B8%AD%E5%9B%BD";
const char* url_gb2312="www.baidu.com/?wd=C%23%D6%D0%B9%FA";
const char* url_utf8_qs="googlevideo.com/qs?nh=%2CIgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE";
int table_id=0,ret=0;
struct Maat_rule_t result[4];
int found_pos[4];
@@ -361,16 +519,17 @@ TEST(StringScan, EncodedURL)
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_utf8, strlen(url_utf8),
result,found_pos, 4,
&mid, 0);
EXPECT_GE(ret, 1);
EXPECT_EQ(ret, 1);
Maat_clean_status(&mid);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_gb2312, strlen(url_gb2312),
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_utf8_qs, strlen(url_utf8_qs),
result,found_pos, 4,
&mid, 0);
EXPECT_GE(ret, 1);
EXPECT_GT(ret, 1);
Maat_clean_status(&mid);
return;
}
TEST(StringScan, UnicodeEscape)
{
const char* test_data_dir="./testdata_uni2ascii";
@@ -1001,7 +1160,6 @@ int test_add_expr_command(Maat_feather_t feather,const char* region_table,int co
int group_num=1,ret=0;
memset(&rule,0,sizeof(rule));
rule.config_id=config_id;
strcpy(rule.service_defined,"maat_command");
//MUST acqire by function, because Maat_cmd_t has some hidden members.
cmd=Maat_create_cmd(&rule, group_num);
@@ -1211,7 +1369,98 @@ TEST_F(MaatCmdTest, SetExpr)
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
&result,NULL, 1,
&mid, 0);
EXPECT_EQ(ret, 0);
Maat_clean_status(&mid);
}
TEST_F(MaatCmdTest, RuleIDRecycle)
{
const char* table_name="HTTP_URL";
const char* scan_data="Reuse rule ID is allowed.";
const char* keywords="Reuse&rule";
Maat_feather_t feather=MaatCmdTest::_shared_feather;
struct Maat_rule_t result;
scan_status_t mid=NULL;
int table_id=0;
table_id=Maat_table_register(feather,table_name);
ASSERT_GT(table_id, 0);
int rule_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
int label_id=5211, ret=0;
test_add_expr_command(feather,table_name,rule_id, 0, label_id, keywords);
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
&result, NULL, 1,
&mid, 0);
Maat_clean_status(&mid);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result.config_id, rule_id);
del_command(feather, rule_id);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
&result, NULL, 1,
&mid, 0);
Maat_clean_status(&mid);
EXPECT_EQ(ret, 0);
test_add_expr_command(feather,table_name,rule_id, 0, label_id, keywords);
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
memset(&result, 0, sizeof(result));
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
&result, NULL, 1,
&mid, 0);
Maat_clean_status(&mid);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result.config_id, rule_id);
return;
}
TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder)
{
const char* table_name="HTTP_URL";
const char* scan_data="This string will hit mulptiple rules.";
const char* keywords="string\\bwill\\bhit";
Maat_feather_t feather=MaatCmdTest::_shared_feather;
int table_id=0;
table_id=Maat_table_register(feather,table_name);
ASSERT_GT(table_id, 0);
int i=0, repeat_times=4;
struct Maat_rule_t result[8];
int expect_ruleid[8];
scan_status_t mid=NULL;
int rule_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", repeat_times);
int label_id=5211, ret=0;
for(i=0; i<repeat_times; i++)
{
//add in ascending order
expect_ruleid[i]=rule_id+1-repeat_times+i;
test_add_expr_command(feather,table_name,rule_id+1-repeat_times+i, 0, label_id, keywords);
}
ret=Maat_cmd_commit(feather);
EXPECT_TRUE(ret>=0);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
result, NULL, 8,
&mid, 0);
Maat_clean_status(&mid);
EXPECT_EQ(ret, repeat_times);
for(i=0;i<repeat_times;i++)
{
//return in ascending order
EXPECT_EQ(result[i].config_id, expect_ruleid[repeat_times-i-1]);
}
return;
}
@@ -1632,16 +1881,16 @@ int main(int argc, char ** argv)
int scan_detail=0;
::testing::InitGoogleTest(&argc, argv);
g_logger=MESA_create_runtime_log_handle(log_file,0);
g_logger=MESA_create_runtime_log_handle(log_file, 0);
g_feather=Maat_feather(g_iThreadNum, table_info_path, g_logger);
Maat_set_feather_opt(g_feather,MAAT_OPT_INSTANCE_NAME,"demo", strlen("demo")+1);
Maat_set_feather_opt(g_feather,MAAT_OPT_DECRYPT_KEY,decrypt_key, strlen(decrypt_key)+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_INSTANCE_NAME, "demo", strlen("demo")+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_DECRYPT_KEY, decrypt_key, strlen(decrypt_key)+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms));
Maat_set_feather_opt(g_feather, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms));
//Set a short intevral for testing.
Maat_set_feather_opt(g_feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms));
Maat_set_feather_opt(g_feather, MAAT_OPT_EFFECT_INVERVAL_MS, &effective_interval_ms, sizeof(effective_interval_ms));
Maat_set_feather_opt(g_feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_STAT_ON, NULL, 0);

1
tools/maat_redis_tool.cpp
View File

@@ -51,7 +51,6 @@ void read_rule_from_redis(redisContext * ctx, long long desire_version, const ch
char table_path[256],index_path[256];
FILE *table_fp=NULL, *index_fp=NULL;
rule_num=get_rm_key_list(ctx, 0, desire_version, &version, &rule_list, &update_type, logger,0);
if(desire_version!=0)
if(rule_num==0)
{
if(desire_version!=0)