gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重构编译配置表的加载和扫描逻辑，fix #12， 设置igraph_es_pairs_small最后一个参数为-1， fix #11。

Browse Source
This commit is contained in:
zhengchao
2019-06-12 02:11:41 +08:00
parent 28a66ce961
commit a238b357d7
4 changed files with 269 additions and 248 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
src/entry/Maat_api.cpp
View File

@@ -138,16 +138,16 @@ size_t pickup_hit_region_from_compile(struct bool_expr *compile_hit,
} }
return k; return k;
} }
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule* rule_head, const char* srv_def, int srv_def_len) void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len)
{ {
memcpy(rule, rule_head, sizeof(struct _head_Maat_rule)); memcpy(rule, rule_head, sizeof(struct Maat_rule_head));
memcpy(rule->service_defined, srv_def, MIN(srv_def_len,MAX_SERVICE_DEFINE_LEN)); memcpy(rule->service_defined, srv_def, MIN(srv_def_len,MAX_SERVICE_DEFINE_LEN));
return; return;
} }
static int compare_compile_inner(const void *a, const void *b) static int compare_compile_inner(const void *a, const void *b)
{ {
const struct Maat_compile_inner *ra=*(const struct Maat_compile_inner **)a; const struct Maat_compile_group_relation *ra=*(const struct Maat_compile_group_relation **)a;
const struct Maat_compile_inner *rb=*(const struct Maat_compile_inner **)b; const struct Maat_compile_group_relation *rb=*(const struct Maat_compile_group_relation **)b;
if(ra->group_cnt!=rb->group_cnt) if(ra->group_cnt!=rb->group_cnt)
{ {
@@ -169,8 +169,8 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
unsigned char has_not_flag=0; unsigned char has_not_flag=0;
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
struct Maat_group_inner* group_rule=NULL; struct Maat_group_inner* group_rule=NULL;
struct Maat_compile_inner* compile_arrary[MAX_SCANNER_HIT_NUM]; struct Maat_compile_group_relation* relation_arrary[MAX_SCANNER_HIT_NUM];
struct Maat_compile_inner* compile_rule=NULL; struct Maat_compile_group_relation* relation=NULL;
int region_pos[MAX_SCANNER_HIT_NUM]; int region_pos[MAX_SCANNER_HIT_NUM];
_mid->cur_hit_group_cnt=0; _mid->cur_hit_group_cnt=0;
@@ -183,7 +183,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
} }
if(group_rule->compile_shortcut!=NULL&&group_rule->parent_ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM) if(group_rule->compile_shortcut!=NULL&&group_rule->parent_ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
{ {
compile_arrary[shortcut_avilable_cnt]=(struct Maat_compile_inner*)(group_rule->compile_shortcut); relation_arrary[shortcut_avilable_cnt]=(struct Maat_compile_group_relation*)(group_rule->compile_shortcut);
shortcut_avilable_cnt++; shortcut_avilable_cnt++;
} }
for(j=0; j<group_rule->top_group_cnt; j++) for(j=0; j<group_rule->top_group_cnt; j++)
@@ -219,45 +219,45 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
{ {
scan_ret=bool_matcher_match(bm, thread_num, scan_ret=bool_matcher_match(bm, thread_num,
_mid->all_hit_group_array, _mid->all_hit_group_cnt, _mid->all_hit_group_array, _mid->all_hit_group_cnt,
(void **)compile_arrary, MAX_SCANNER_HIT_NUM); (void **)relation_arrary, MAX_SCANNER_HIT_NUM);
} }
if(scan_ret>1) if(scan_ret>1)
{ {
qsort(compile_arrary, scan_ret, sizeof(struct Maat_compile_inner**), qsort(relation_arrary, scan_ret, sizeof(struct Maat_compile_group_relation**),
compare_compile_inner); compare_compile_inner);
} }
for(i=0;i<scan_ret&&result_cnt<size;i++) for(i=0;i<scan_ret&&result_cnt<size;i++)
{ {
compile_rule=compile_arrary[i]; relation=relation_arrary[i];
if(compile_rule==NULL) if(relation==NULL)
{ {
continue; continue;
} }
if(0==pthread_rwlock_tryrdlock(&(compile_rule->rwlock))) if(0==pthread_rwlock_tryrdlock(&(relation->rwlock)))
{ {
if(compile_rule->is_valid==1) if(relation->compile)
{ {
if(compile_rule->not_group_cnt>0 && !is_last_region) if(relation->not_group_cnt>0 && !is_last_region)
{ {
_mid->not_grp_compile_hitted_flag=1; _mid->not_grp_compile_hitted_flag=1;
} }
else else
{ {
make_group_set(compile_rule, &(rs_result[result_cnt].group_set), &has_not_flag); make_group_set(relation, &(rs_result[result_cnt].group_set), &has_not_flag);
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_groups, _mid->cur_hit_group_cnt, r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_groups, _mid->cur_hit_group_cnt,
region_pos, MAX_SCANNER_HIT_NUM); region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0 || //compile config hitted becasue of new reigon if(r_in_c_cnt>0 || //compile config hitted becasue of new reigon
_mid->cur_hit_group_cnt==0) //or ever hit a compile that refer a NOT-logic group _mid->cur_hit_group_cnt==0) //or ever hit a compile that refer a NOT-logic group
{ {
fill_maat_rule(&(result[result_cnt]), &(compile_rule->db_c_rule->m_rule_head), fill_maat_rule(&(result[result_cnt]), &(relation->compile->head),
compile_rule->db_c_rule->service_defined ,compile_rule->db_c_rule->m_rule_head.serv_def_len); relation->compile->service_defined ,relation->compile->head.serv_def_len);
rs_result[result_cnt].compile_id=compile_rule->compile_id; rs_result[result_cnt].compile_id=relation->compile_id;
result_cnt++; result_cnt++;
} }
} }
} }
pthread_rwlock_unlock(&(compile_rule->rwlock)); pthread_rwlock_unlock(&(relation->rwlock));
} }
} }
if(result_cnt>0) if(result_cnt>0)
@@ -1068,19 +1068,21 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user) void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
{ {
struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user; struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
struct Maat_compile_inner *compile_inner=(struct Maat_compile_inner *)data; struct Maat_compile_group_relation *relation=(struct Maat_compile_group_relation *)data;
struct Maat_compile_rule* compile_rule=relation->compile;
MAAT_RULE_EX_DATA ad=NULL; MAAT_RULE_EX_DATA ad=NULL;
if(compile_inner->ref_table==NULL || compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL) if(compile_rule==NULL|| compile_rule->ref_table->table_id!=ex_desc->table_id)
{ {
return; return;
} }
pthread_rwlock_wrlock(&(compile_inner->rwlock));
ad=rule_ex_data_new(&(compile_inner->db_c_rule->m_rule_head), pthread_rwlock_wrlock(&(relation->rwlock));
compile_inner->db_c_rule->service_defined, ad=rule_ex_data_new(&(relation->compile->head),
relation->compile->service_defined,
ex_desc); ex_desc);
compile_inner->ads[ex_desc->idx]=ad; compile_rule->ads[ex_desc->idx]=ad;
pthread_rwlock_unlock(&(compile_inner->rwlock)); pthread_rwlock_unlock(&(relation->rwlock));
return; return;
} }
int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name, int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
@@ -1130,22 +1132,24 @@ failed:
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx) MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
{ {
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct Maat_compile_inner *compile_inner=NULL; struct Maat_compile_group_relation *relation=NULL;
struct Maat_compile_rule* compile=NULL;
const struct compile_table_desc* compile_desc=NULL; const struct compile_table_desc* compile_desc=NULL;
const struct compile_ex_data_idx* ex_desc=NULL; const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL; MAAT_RULE_EX_DATA ad=NULL;
compile_inner=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL) if(relation==NULL||relation->compile==NULL)
{ {
return NULL; return NULL;
} }
pthread_rwlock_rdlock(&(compile_inner->rwlock)); pthread_rwlock_rdlock(&(relation->rwlock));
compile_desc=&(compile_inner->ref_table->compile); compile=relation->compile;
compile_desc=&(compile->ref_table->compile);
assert(idx<compile_desc->ex_data_num); assert(idx<compile_desc->ex_data_num);
ex_desc=compile_desc->ex_desc+idx; ex_desc=compile_desc->ex_desc+idx;
ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp); ex_desc->dup_func(ex_desc->idx, &ad, compile->ads+idx, ex_desc->argl,ex_desc->argp);
pthread_rwlock_unlock(&(compile_inner->rwlock)); pthread_rwlock_unlock(&(relation->rwlock));
return ad; return ad;
} }
struct wrap_plugin_EX_data struct wrap_plugin_EX_data
@@ -1178,7 +1182,7 @@ MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_
MESA_htable_create_args_t hargs; MESA_htable_create_args_t hargs;
memset(&hargs,0,sizeof(hargs)); memset(&hargs,0,sizeof(hargs));
hargs.thread_safe=0; hargs.thread_safe=8;
hargs.hash_slot_size = slot_size; hargs.hash_slot_size = slot_size;
hargs.max_elem_num = 0; hargs.max_elem_num = 0;
hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO; hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
@@ -1336,7 +1340,6 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
return NULL; return NULL;
} }
table_rt= _feather->scanner->table_rt[table_id]; table_rt= _feather->scanner->table_rt[table_id];
pthread_rwlock_rdlock(&table_rt->plugin.rwlock);
wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash, wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash,
(const unsigned char*)key, strlen(key)); (const unsigned char*)key, strlen(key));
if(wrap_data!=NULL) if(wrap_data!=NULL)
@@ -1344,7 +1347,6 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id
plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata), plugin_desc->ex_desc.dup_func(table_id, &(exdata), &(wrap_data->exdata),
plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp); plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
} }
pthread_rwlock_unlock(&table_rt->plugin.rwlock);
return exdata; return exdata;
} }
@@ -2179,11 +2181,11 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
{ {
int ret=0; int ret=0;
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct Maat_compile_inner *compile_inner=NULL; struct Maat_compile_group_relation *compile_inner=NULL;
switch(type) switch(type)
{ {
case MAAT_RULE_SERV_DEFINE: case MAAT_RULE_SERV_DEFINE:
compile_inner=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); compile_inner=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL) if(compile_inner==NULL)
{ {
ret=0; ret=0;
@@ -2191,8 +2193,8 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
else else
{ {
pthread_rwlock_rdlock(&(compile_inner->rwlock)); pthread_rwlock_rdlock(&(compile_inner->rwlock));
ret=MIN(size,compile_inner->db_c_rule->m_rule_head.serv_def_len); ret=MIN(size,compile_inner->compile->head.serv_def_len);
memcpy(value,compile_inner->db_c_rule->service_defined,ret); memcpy(value,compile_inner->compile->service_defined,ret);
pthread_rwlock_unlock(&(compile_inner->rwlock)); pthread_rwlock_unlock(&(compile_inner->rwlock));
} }
break; break;

28
src/entry/Maat_command.cpp
View File

@@ -839,7 +839,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
struct Maat_group_t* group_cmd=NULL; struct Maat_group_t* group_cmd=NULL;
struct Maat_region_t* region_cmd=NULL; struct Maat_region_t* region_cmd=NULL;
struct Maat_compile_inner *compile_inner=NULL; struct Maat_compile_group_relation *relation=NULL;
struct Maat_group_inner* group_inner=NULL; struct Maat_group_inner* group_inner=NULL;
struct Maat_region_inner* region_inner=NULL; struct Maat_region_inner* region_inner=NULL;
void* logger=feather->logger; void* logger=feather->logger;
@@ -847,27 +847,27 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
int config_id=cmd->compile.config_id; int config_id=cmd->compile.config_id;
if(feather->scanner==NULL) if(feather->scanner==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_command MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command
,"MAAT not ready."); ,"MAAT not ready.");
return -1; return -1;
} }
compile_inner=(struct Maat_compile_inner *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id); relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
//Operation on compile_inner is thread safe, no immediate memory free when delete a compile rule or a scanner. //Operation on relation is thread safe, no immediate memory free when delete a compile rule or a scanner.
//In another words, if the compile_inner is accessable from compile means, its was valid in at least 10 seconds (garbage bury). //In another words, if the relation is accessable from compile means, its was valid in at least 10 seconds (garbage bury).
if(compile_inner==NULL) if(relation==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_command MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_command
,"config %d not exist." ,"config %d not exist."
,config_id); ,config_id);
return -1; return -1;
} }
pthread_rwlock_rdlock(&(compile_inner->rwlock)); pthread_rwlock_rdlock(&(relation->rwlock));
cmd->group_num=compile_inner->group_cnt; cmd->group_num=relation->group_cnt;
assert(cmd->groups==NULL); assert(cmd->groups==NULL);
cmd->groups=ALLOC(struct Maat_group_t, cmd->group_num); cmd->groups=ALLOC(struct Maat_group_t, cmd->group_num);
for(i=0;i<compile_inner->group_boundary;i++) for(i=0;i<relation->group_boundary;i++)
{ {
group_inner=(struct Maat_group_inner*)dynamic_array_read(compile_inner->groups,i); group_inner=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i);
if(group_inner==NULL) if(group_inner==NULL)
{ {
continue; continue;
@@ -896,7 +896,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
} }
grp_idx++; grp_idx++;
} }
pthread_rwlock_unlock(&(compile_inner->rwlock)); pthread_rwlock_unlock(&(relation->rwlock));
return 0; return 0;
} }
@@ -1361,10 +1361,10 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
struct Maat_region_t* p_region=NULL; struct Maat_region_t* p_region=NULL;
enum MAAT_TABLE_TYPE table_type; enum MAAT_TABLE_TYPE table_type;
struct Maat_compile_inner *compile_rule=NULL; struct Maat_compile_group_relation *compile_rule=NULL;
if(feather->scanner!=NULL) if(feather->scanner!=NULL)
{ {
compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id); compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
if(compile_rule!=NULL) if(compile_rule!=NULL)
{ {
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module

370
src/entry/Maat_rule.cpp
View File

@@ -32,7 +32,7 @@
#include "stream_fuzzy_hash.h" #include "stream_fuzzy_hash.h"
#include "gram_index_engine.h" #include "gram_index_engine.h"
int MAAT_FRAME_VERSION_2_7_20190608=1; int MAAT_FRAME_VERSION_2_7_20190612=1;
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""}; "unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
@@ -506,7 +506,7 @@ int HASH_delete_by_id(MESA_htable_handle hash,int id)
ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL); ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL);
return ret; return ret;
} }
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc) MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc)
{ {
MAAT_RULE_EX_DATA ad=NULL; MAAT_RULE_EX_DATA ad=NULL;
struct Maat_rule_t rule; struct Maat_rule_t rule;
@@ -514,7 +514,7 @@ MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule * rule_head, con
ex_desc->new_func(ex_desc->idx, &rule, srv_def, &ad, ex_desc->argl,ex_desc->argp); ex_desc->new_func(ex_desc->idx, &rule, srv_def, &ad, ex_desc->argl,ex_desc->argp);
return ad; return ad;
} }
void rule_ex_data_free(const struct _head_Maat_rule * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc) void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc)
{ {
struct Maat_rule_t rule; struct Maat_rule_t rule;
fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
@@ -903,28 +903,28 @@ void destroy_group_rule(struct Maat_group_inner* group_rule, int by_whom, struct
} }
} }
void make_group_set(struct Maat_compile_inner* compile_rule, struct bool_expr* a_set, unsigned char *has_not) void make_group_set(struct Maat_compile_group_relation* relation, struct bool_expr* a_set, unsigned char *has_not)
{ {
int i=0,j=0; int i=0,j=0;
a_set->user_tag=compile_rule; a_set->user_tag=relation;
struct Maat_group_inner*group=NULL; struct Maat_group_inner*group=NULL;
assert(compile_rule->group_cnt<=MAX_ITEMS_PER_BOOL_EXPR); assert(relation->group_cnt<=MAX_ITEMS_PER_BOOL_EXPR);
for(i=0,j=0;i<compile_rule->group_boundary&&j<MAX_ITEMS_PER_BOOL_EXPR;i++) for(i=0,j=0;i<relation->group_boundary&&j<MAX_ITEMS_PER_BOOL_EXPR;i++)
{ {
group=(struct Maat_group_inner*)dynamic_array_read(compile_rule->groups, i); group=(struct Maat_group_inner*)dynamic_array_read(relation->groups, i);
if(group==NULL) if(group==NULL)
{ {
continue; continue;
} }
a_set->items[j].item_id=group->group_id; a_set->items[j].item_id=group->group_id;
a_set->items[j].not_flag=compile_rule->not_flag[j]; a_set->items[j].not_flag=relation->not_flag[j];
if(a_set->items[j].not_flag) if(a_set->items[j].not_flag)
{ {
*has_not=1; *has_not=1;
} }
j++; j++;
} }
assert(j==compile_rule->group_cnt); assert(j==relation->group_cnt);
a_set->item_num=j; a_set->item_num=j;
} }
struct compile_walker struct compile_walker
@@ -935,23 +935,23 @@ struct compile_walker
void walk_compile_hash(const uchar * key, uint size, void * data, void * user) void walk_compile_hash(const uchar * key, uint size, void * data, void * user)
{ {
struct bool_expr* one_set=NULL; struct bool_expr* one_set=NULL;
struct Maat_compile_inner* compile_rule=(struct Maat_compile_inner*)data; struct Maat_compile_group_relation* relation=(struct Maat_compile_group_relation*)data;
struct compile_walker* walker=(struct compile_walker*)user; struct compile_walker* walker=(struct compile_walker*)user;
unsigned char has_not_flag=0; unsigned char has_not_flag=0;
MESA_lqueue_head update_q=walker->update_q; MESA_lqueue_head update_q=walker->update_q;
if(compile_rule->db_c_rule==NULL) if(relation->compile==NULL)
{ {
return; return;
} }
//make sure compile rule's each group has loadded. //make sure compile rule's each group has loadded.
if((compile_rule->group_cnt==compile_rule->db_c_rule->declare_grp_num if((relation->group_cnt==relation->compile->declared_grp_num
||compile_rule->db_c_rule->declare_grp_num==0)//for compatible old version || relation->compile->declared_grp_num==0)//for compatible old version
&&compile_rule->group_cnt>0 && relation->group_cnt>0
&&compile_rule->group_cnt!=compile_rule->not_group_cnt) && relation->group_cnt!=relation->not_group_cnt)
{ {
one_set=ALLOC(struct bool_expr, 1); one_set=ALLOC(struct bool_expr, 1);
//reading compile rule is safe in update thread, mutex lock called when modified //reading compile rule is safe in update thread, mutex lock called when modified
make_group_set(compile_rule, one_set, &has_not_flag); make_group_set(relation, one_set, &has_not_flag);
if(has_not_flag) if(has_not_flag)
{ {
walker->compile_has_not_flag++; walker->compile_has_not_flag++;
@@ -1023,61 +1023,86 @@ void EMPTY_FREE(void*p)
{ {
return; return;
} }
struct Maat_compile_inner * create_compile_rule(int compile_id, struct Maat_scanner_t *scanner)
struct Maat_compile_rule* create_compile_rule(struct Maat_rule_head* p_head, const char* service_define, int declared_grp_num, const struct Maat_table_desc* table)
{
int i=0;
struct Maat_compile_rule*p=ALLOC(struct Maat_compile_rule, 1);
p->head=*p_head;
p->declared_grp_num=declared_grp_num;
p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM);
//protect by feather->background_update_mutex
p->ref_table=table;
p->head.serv_def_len=strlen(service_define)+1;
p->service_defined=ALLOC(char, p->head.serv_def_len);
memcpy(p->service_defined, service_define, p->head.serv_def_len);
for(i=0; i<table->compile.ex_data_num; i++)
{
p->ads[i]=rule_ex_data_new(&p->head, p->service_defined, table->compile.ex_desc+i);
}
p->is_valid=1;
return p;
}
void destroy_compile_rule(struct Maat_compile_rule* compile_rule)
{
int i=0;
const struct compile_table_desc* compile_desc= &(compile_rule->ref_table->compile);
for(i=0; i<compile_desc->ex_data_num; i++)
{
rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ads+i, compile_desc->ex_desc+i);
compile_rule->ads[i]=NULL;
}
free(compile_rule->ads);
compile_rule->is_valid=0;
compile_rule->declared_grp_num=-1;
free(compile_rule->service_defined);
compile_rule->service_defined=NULL;
free(compile_rule);
return;
}
struct Maat_compile_group_relation * create_compile_group_relation(int compile_id, struct Maat_scanner_t *scanner)
{ {
int ret=0; int ret=0;
struct Maat_compile_inner* p=ALLOC(struct Maat_compile_inner, 1); struct Maat_compile_group_relation* p=ALLOC(struct Maat_compile_group_relation, 1);
p->compile_id=compile_id; p->compile_id=compile_id;
p->group_cnt=0; p->group_cnt=0;
p->group_boundary=1; p->group_boundary=1;
p->groups=dynamic_array_create(1, 1); p->groups=dynamic_array_create(1, 1);
p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM);
pthread_rwlock_init(&(p->rwlock), NULL); pthread_rwlock_init(&(p->rwlock), NULL);
ret=HASH_add_by_id(scanner->compile_hash, compile_id, p); ret=HASH_add_by_id(scanner->compile_hash, compile_id, p);
assert(ret>0); assert(ret>0);
return p; return p;
} }
void _destroy_compile_rule(struct Maat_compile_inner * compile_rule)
void _destroy_compile_group_relation(struct Maat_compile_group_relation * cg_relation)
{ {
const struct Maat_table_desc* table=compile_rule->ref_table; pthread_rwlock_wrlock(&(cg_relation->rwlock));
const struct compile_table_desc* compile_desc=&(table->compile); cg_relation->compile_id=-1;
struct db_compile_rule* db_compile_rule=compile_rule->db_c_rule; dynamic_array_destroy(cg_relation->groups, NULL);
int i=0; pthread_rwlock_unlock(&(cg_relation->rwlock));
compile_rule->compile_id=-1;
dynamic_array_destroy(compile_rule->groups,NULL);
pthread_rwlock_destroy(&(cg_relation->rwlock));
pthread_rwlock_wrlock(&(compile_rule->rwlock));
if(db_compile_rule!=NULL) free(cg_relation);
{
for(i=0; i<compile_desc->ex_data_num; i++)
{
rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, compile_desc->ex_desc+i);
compile_rule->ads[i]=NULL;
}
free(db_compile_rule->service_defined);
free(db_compile_rule);
compile_rule->db_c_rule=NULL;
}
free(compile_rule->ads);
pthread_rwlock_unlock(&(compile_rule->rwlock));
pthread_rwlock_destroy(&(compile_rule->rwlock));
free(compile_rule);
} }
void destroy_compile_rule(struct Maat_compile_inner * p) void destroy_compile_group_relation(struct Maat_compile_group_relation * p, struct Maat_scanner_t *scanner)
{ {
int i=0; int i=0;
UNUSED struct Maat_compile_inner* p_group=NULL; UNUSED struct Maat_group_inner* p_group=NULL;
assert(p->group_cnt==0); assert(p->group_cnt==0);
for(i=0;i<p->group_boundary;i++) for(i=0;i<p->group_boundary;i++)
{ {
p_group=(struct Maat_compile_inner*)dynamic_array_read(p->groups,i); p_group=(struct Maat_group_inner*)dynamic_array_read(p->groups, i);
assert(p_group==NULL); assert(p_group==NULL);
} }
_destroy_compile_rule(p); HASH_delete_by_id(scanner->compile_hash, p->compile_id);
_destroy_compile_group_relation(p);
} }
scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset) scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset)
{ {
@@ -1253,7 +1278,6 @@ struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table
{ {
table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size,
table_desc->plugin.ex_desc.key2index_func); table_desc->plugin.ex_desc.key2index_func);
pthread_rwlock_init(&(table_rt->plugin.rwlock), NULL);
} }
break; break;
default: default:
@@ -1306,7 +1330,6 @@ void table_runtime_free(struct Maat_table_runtime* p)
{ {
MESA_htable_destroy(p->plugin.key2ex_hash, NULL); MESA_htable_destroy(p->plugin.key2ex_hash, NULL);
} }
break;
default: default:
break; break;
} }
@@ -1348,11 +1371,11 @@ struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
scanner=ALLOC(struct Maat_scanner_t, 1); scanner=ALLOC(struct Maat_scanner_t, 1);
//Function Maat_cmd_append will access compile_hash in user thread. //Function Maat_cmd_append will access compile_hash in user thread.
hargs.thread_safe=1; hargs.thread_safe=8;
scanner->compile_hash=MESA_htable_create(&hargs, sizeof(hargs)); scanner->compile_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(scanner->compile_hash,0); MESA_htable_print_crtl(scanner->compile_hash,0);
hargs.thread_safe=1; hargs.thread_safe=8;
scanner->group_hash=MESA_htable_create(&hargs, sizeof(hargs)); scanner->group_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(scanner->group_hash,0); MESA_htable_print_crtl(scanner->group_hash,0);
@@ -1417,7 +1440,7 @@ void destroy_maat_scanner(struct Maat_scanner_t*scanner)
return; return;
} }
rulescan_destroy(scanner->region); rulescan_destroy(scanner->region);
MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_rule); MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_group_relation);
MESA_htable_destroy(scanner->group_hash, (void (*)(void*))_destroy_group_rule); MESA_htable_destroy(scanner->group_hash, (void (*)(void*))_destroy_group_rule);
MESA_htable_destroy(scanner->region_hash, NULL); MESA_htable_destroy(scanner->region_hash, NULL);
MESA_htable_destroy(scanner->vertex_id2group, NULL); MESA_htable_destroy(scanner->vertex_id2group, NULL);
@@ -1783,23 +1806,24 @@ unsigned int del_region_from_group(struct Maat_group_inner* group,int region_id,
return j; return j;
} }
int add_group_to_compile(struct Maat_compile_inner*a_compile_rule,struct Maat_group_inner* a_rule_group, int not_flag) int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int not_flag)
{ {
int i=0,ret=-1; int i=0,ret=-1;
int write_pos=-1; int write_pos=-1;
struct Maat_group_inner* p=NULL; struct Maat_group_inner* p=NULL;
pthread_rwlock_wrlock(&(a_compile_rule->rwlock));
if(a_compile_rule->db_c_rule!=NULL pthread_rwlock_wrlock(&(relation->rwlock));
&&a_compile_rule->group_cnt>=a_compile_rule->db_c_rule->declare_grp_num if(relation->compile!=NULL
&&a_compile_rule->db_c_rule->declare_grp_num!=0) && relation->group_cnt>=relation->compile->declared_grp_num
&& relation->compile->declared_grp_num!=0)
{ {
ret=-1; ret=-1;
goto error_out; goto error_out;
} }
for(i=0;i<a_compile_rule->group_boundary;i++) for(i=0;i<relation->group_boundary;i++)
{ {
p=(struct Maat_group_inner*)dynamic_array_read(a_compile_rule->groups,i); p=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i);
if(p==NULL) if(p==NULL)
{ {
write_pos=i; write_pos=i;
@@ -1813,31 +1837,31 @@ int add_group_to_compile(struct Maat_compile_inner*a_compile_rule,struct Maat_gr
} }
} }
} }
if(write_pos<0&&a_compile_rule->group_boundary==MAX_EXPR_ITEM_NUM) if(write_pos<0&&relation->group_boundary==MAX_EXPR_ITEM_NUM)
{ {
ret=-1; ret=-1;
goto error_out; goto error_out;
} }
if(write_pos<0) if(write_pos<0)
{ {
write_pos=a_compile_rule->group_boundary; write_pos=relation->group_boundary;
a_compile_rule->group_boundary++; relation->group_boundary++;
} }
dynamic_array_write(a_compile_rule->groups, write_pos, a_rule_group); dynamic_array_write(relation->groups, write_pos, a_rule_group);
if(not_flag) if(not_flag)
{ {
a_compile_rule->not_flag[write_pos]=1; relation->not_flag[write_pos]=1;
a_compile_rule->not_group_cnt++; relation->not_group_cnt++;
} }
else else
{ {
a_compile_rule->not_flag[write_pos]=0; relation->not_flag[write_pos]=0;
} }
a_compile_rule->group_cnt++; relation->group_cnt++;
//member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish. //member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish.
if(a_rule_group->parent_ref_cnt==1&&a_compile_rule->group_cnt==1) if(a_rule_group->parent_ref_cnt==1&&relation->group_cnt==1)
{ {
a_rule_group->compile_shortcut=a_compile_rule; a_rule_group->compile_shortcut=relation;
} }
else else
{ {
@@ -1845,11 +1869,11 @@ int add_group_to_compile(struct Maat_compile_inner*a_compile_rule,struct Maat_gr
} }
a_rule_group->parent_ref_cnt++; a_rule_group->parent_ref_cnt++;
//update group's shortcut when compile has more than one group. //update group's shortcut when compile has more than one group.
if(a_compile_rule->group_cnt!=1) if(relation->group_cnt!=1)
{ {
for(i=0;i<a_compile_rule->group_boundary;i++) for(i=0;i<relation->group_boundary;i++)
{ {
p=(struct Maat_group_inner*)dynamic_array_read(a_compile_rule->groups,i); p=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i);
if(p!=NULL) if(p!=NULL)
{ {
p->compile_shortcut=NULL; p->compile_shortcut=NULL;
@@ -1858,18 +1882,18 @@ int add_group_to_compile(struct Maat_compile_inner*a_compile_rule,struct Maat_gr
} }
ret=1; ret=1;
error_out: error_out:
pthread_rwlock_unlock(&(a_compile_rule->rwlock)); pthread_rwlock_unlock(&(relation->rwlock));
return ret; return ret;
} }
struct Maat_group_inner* del_group_from_compile(struct Maat_compile_inner*a_compile_rule,int group_id) struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relation*relation, int group_id)
{ {
int i=0; int i=0;
struct Maat_group_inner* group_rule=NULL; struct Maat_group_inner* group_rule=NULL;
pthread_rwlock_wrlock(&(a_compile_rule->rwlock)); pthread_rwlock_wrlock(&(relation->rwlock));
for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++) for(i=0;i<MAAT_MAX_EXPR_ITEM_NUM;i++)
{ {
group_rule=(struct Maat_group_inner*)dynamic_array_read(a_compile_rule->groups,i); group_rule=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i);
if(group_rule==NULL) if(group_rule==NULL)
{ {
continue; continue;
@@ -1877,19 +1901,22 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_inner*a_comp
if(group_rule->group_id==group_id) if(group_rule->group_id==group_id)
{ {
group_rule->parent_ref_cnt--; group_rule->parent_ref_cnt--;
dynamic_array_write(a_compile_rule->groups,i,NULL); dynamic_array_write(relation->groups,i,NULL);
if(a_compile_rule->not_flag[i]==1) if(relation->not_flag[i]==1)
{ {
a_compile_rule->not_group_cnt--; relation->not_group_cnt--;
a_compile_rule->not_flag[i]=0; relation->not_flag[i]=0;
} }
a_compile_rule->group_cnt--; relation->group_cnt--;
pthread_rwlock_unlock(&(a_compile_rule->rwlock)); break;
return group_rule; }
else
{
group_rule=NULL;
} }
} }
pthread_rwlock_unlock(&(a_compile_rule->rwlock)); pthread_rwlock_unlock(&(relation->rwlock));
return NULL; return group_rule;
} }
int MAAT_MAGIC=0xaaaa; int MAAT_MAGIC=0xaaaa;
int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_name,int is_valid,void*logger) int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_name,int is_valid,void*logger)
@@ -2358,7 +2385,7 @@ int del_region_rule(struct Maat_table_desc* table,int region_id,int group_id,int
int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner_t *scanner, void* logger) int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner_t *scanner, void* logger)
{ {
struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
struct Maat_compile_inner*compile_rule=NULL; struct Maat_compile_group_relation*compile_rule=NULL;
int ret=0; int ret=0;
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
@@ -2379,10 +2406,10 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
else else
{ {
group_rule->has_compile_neighbors=1; group_rule->has_compile_neighbors=1;
compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
compile_rule=create_compile_rule(db_group_rule->parent_id, scanner); compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner);
} }
ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->not_flag); ret=add_group_to_compile(compile_rule, group_rule, db_group_rule->not_flag);
if(ret<0) if(ret<0)
@@ -2399,7 +2426,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
} }
int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner_t *scanner, void* logger) int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner_t *scanner, void* logger)
{ {
struct Maat_compile_inner*compile_rule=NULL; struct Maat_compile_group_relation* relation=NULL;
struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
igraph_es_t es; igraph_es_t es;
int ret=0; int ret=0;
@@ -2427,8 +2454,13 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
db_group_rule->parent_id); db_group_rule->parent_id);
return 0; return 0;
} }
// The edges between the given pairs of vertices will be included in the edge selection.
igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id); //The vertex pairs must be given as the arguments of the function call, the third argument
//is the first vertex of the first edge, the fourth argument is the second vertex of the
//first edge, the fifth is the first vertex of the second edge and so on. The last element
//of the argument list must be -1 to denote the end of the argument list.
//https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small
igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id, -1);
ret=igraph_delete_edges(&scanner->group_graph, es); ret=igraph_delete_edges(&scanner->group_graph, es);
if(ret!=IGRAPH_SUCCESS) if(ret!=IGRAPH_SUCCESS)
{ {
@@ -2442,8 +2474,8 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
} }
else else
{ {
compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id);
if(compile_rule==NULL) if(relation==NULL)
{ {
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"update error, delete %s group %d error : compile id %d does not exisit.", "update error, delete %s group %d error : compile id %d does not exisit.",
@@ -2452,7 +2484,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
db_group_rule->parent_id); db_group_rule->parent_id);
return 0; return 0;
} }
group_rule=del_group_from_compile(compile_rule, db_group_rule->group_id); group_rule=del_group_from_compile(relation, db_group_rule->group_id);
if(group_rule==NULL) if(group_rule==NULL)
{ {
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
@@ -2462,66 +2494,57 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
,db_group_rule->parent_id); ,db_group_rule->parent_id);
return 0; return 0;
} }
if(compile_rule->group_cnt==0&&compile_rule->is_valid==0) if(relation->group_cnt==0)
{ {
HASH_delete_by_id(scanner->compile_hash, db_group_rule->parent_id); destroy_compile_group_relation(relation, scanner);
garbage_bagging(GARBAGE_COMPILE_RULE, compile_rule, scanner->tomb_ref);
} }
} }
destroy_group_rule(group_rule, DESTROY_GROUP_BY_PARENT, scanner); destroy_group_rule(group_rule, DESTROY_GROUP_BY_PARENT, scanner);
return 1; return 1;
} }
int add_compile_rule(struct Maat_table_desc* table, struct db_compile_rule* db_compile_rule, struct Maat_scanner_t *scanner, void* logger) int add_compile_rule(struct Maat_table_desc* table, struct Maat_compile_rule* db_compile_rule, struct Maat_scanner_t *scanner, void* logger)
{ {
struct Maat_compile_inner *compile_rule=NULL; struct Maat_compile_group_relation *cg_relation=NULL;
struct _head_Maat_rule *p_maat_rule_head=&(db_compile_rule->m_rule_head); struct Maat_rule_head *p_maat_rule_head=&(db_compile_rule->head);
int i=0;
compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
if(compile_rule==NULL) if(cg_relation==NULL)
{ {
compile_rule=create_compile_rule(p_maat_rule_head->config_id, scanner); cg_relation=create_compile_group_relation(p_maat_rule_head->config_id, scanner);
} }
else else
{ {
if(compile_rule->db_c_rule!=NULL)//duplicate config if(cg_relation->compile!=NULL)//duplicate config
{ {
return -1; return -1;
} }
} }
cg_relation->compile=db_compile_rule;
pthread_rwlock_wrlock(&(compile_rule->rwlock));
compile_rule->ref_table=table;
compile_rule->db_c_rule=db_compile_rule;
for(i=0; i<table->compile.ex_data_num; i++)
{
compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->compile.ex_desc+i);
}
compile_rule->is_valid=1;
pthread_rwlock_unlock(&(compile_rule->rwlock));
return 0; return 0;
} }
int del_compile_rule(struct Maat_table_desc* table,struct db_compile_rule* db_compile_rule,struct Maat_scanner_t *scanner,void* logger) int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_scanner_t *scanner, void* logger)
{ {
struct Maat_compile_inner *compile_rule=NULL; struct Maat_compile_group_relation *cg_relation=NULL;
compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, db_compile_rule->m_rule_head.config_id); cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, compile_id);
if(compile_rule==NULL) if(cg_relation==NULL || cg_relation->compile==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,delete %s compile rule error : congfig id %d does not exisit." "update error, delete %s compile rule error : config id %d does not exist."
,table->table_name[table->updating_name] ,table->table_name[table->updating_name]
,db_compile_rule->m_rule_head.config_id); ,compile_id);
return -1; return -1;
} }
pthread_rwlock_wrlock(&(compile_rule->rwlock));
compile_rule->is_valid=0;
pthread_rwlock_unlock(&(compile_rule->rwlock));
if(compile_rule->group_cnt==0) pthread_rwlock_wrlock(&(cg_relation->rwlock));
garbage_bagging(GARBAGE_COMPILE_RULE, cg_relation->compile, scanner->tomb_ref);
cg_relation->compile=NULL;
pthread_rwlock_unlock(&(cg_relation->rwlock));
if(cg_relation->group_cnt==0)
{ {
HASH_delete_by_id(scanner->compile_hash, compile_rule->compile_id); destroy_compile_group_relation(cg_relation, scanner);
garbage_bagging(GARBAGE_COMPILE_RULE,compile_rule, scanner->tomb_ref);
} }
return 1; return 1;
} }
@@ -3164,92 +3187,89 @@ error_out:
intval_rule=NULL; intval_rule=NULL;
} }
void update_compile_rule(struct Maat_table_desc* table,const char* table_line,struct Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger) void update_compile_rule(struct Maat_table_desc* table,const char* table_line ,struct Maat_scanner_t *scanner, const struct rule_tag* tags, int n_tags,void* logger)
{ {
struct compile_table_desc* compile_desc=&(table->compile); struct compile_table_desc* compile_desc=&(table->compile);
struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id]; struct Maat_table_runtime* table_rt=scanner->table_rt[table->table_id];
struct db_compile_rule *p_compile=ALLOC(struct db_compile_rule, 1); struct Maat_compile_rule *p_compile=NULL;
struct _head_Maat_rule* p_m_rule=&(p_compile->m_rule_head); struct Maat_rule_head m_rule_tmp;
char user_region[MAX_TABLE_LINE_SIZE]={0}; memset(&m_rule_tmp, 0, sizeof(m_rule_tmp));
char service_define[MAX_TABLE_LINE_SIZE]={0};
char tag_str[MAX_TABLE_LINE_SIZE]={0}; char tag_str[MAX_TABLE_LINE_SIZE]={0};
int ret=0; int ret=0;
p_compile->declare_grp_num=0; int is_valid=0, declared_grp_num=0;
ret=sscanf(table_line,"%d\t%d\t%hhd\t%hhd\t%hhd\t%s\t%s\t%d\t%d",&(p_m_rule->config_id) ret=sscanf(table_line,"%d\t%d\t%hhd\t%hhd\t%hhd\t%s\t%s\t%d\t%d",&(m_rule_tmp.config_id),
,&(p_m_rule->service_id) &(m_rule_tmp.service_id),
,&(p_m_rule->action) &(m_rule_tmp.action),
,&(p_m_rule->do_blacklist) &(m_rule_tmp.do_blacklist),
,&(p_m_rule->do_log) &(m_rule_tmp.do_log),
,tag_str tag_str,
,user_region service_define,
,&(p_compile->is_valid) &is_valid,
,&(p_compile->declare_grp_num)); &declared_grp_num);
if((ret!=8&&ret!=9)||p_compile->declare_grp_num>MAAT_MAX_EXPR_ITEM_NUM) if((ret!=8&&ret!=9)||declared_grp_num>MAAT_MAX_EXPR_ITEM_NUM)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module ,
"update error,invalid format of compile table %s:%s" "update error, invalid format of compile table %s:%s"
,table->table_name[table->updating_name],table_line); ,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++; goto error_out;
goto no_save;
} }
if(n_tags>0&&strlen(tag_str)>2) if(n_tags>0&&strlen(tag_str)>2)
{ {
ret=compare_accept_tag(tag_str, tags, n_tags); ret=compare_accept_tag(tag_str, tags, n_tags);
if(ret<0) if(ret<0)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module ,
"update error,invalid tag format of compile table %s:%s" "update error, invalid tag format of compile table %s:%s"
,table->table_name[table->updating_name],table_line); ,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++; goto error_out;
goto no_save;
} }
if(ret==0) if(ret==0)
{ {
table->unmatch_tag_cnt++; table->unmatch_tag_cnt++;
goto no_save; return;
} }
} }
switch(compile_desc->user_region_encoding) switch(compile_desc->user_region_encoding)
{ {
case USER_REGION_ENCODE_ESCAPE: case USER_REGION_ENCODE_ESCAPE:
str_unescape(user_region); str_unescape(service_define);
break; break;
default: default:
break; break;
} }
p_m_rule->serv_def_len=strlen(user_region)+1;
p_compile->service_defined= ALLOC(char, p_m_rule->serv_def_len);
memcpy(p_compile->service_defined,user_region,p_m_rule->serv_def_len);
if(p_compile->is_valid==FALSE) if(is_valid==FALSE)
{ {
ret=del_compile_rule(table,p_compile,scanner, logger); ret=del_compile_rule(table, m_rule_tmp.config_id, scanner, logger);
if(ret>0) if(ret>0)
{ {
table_rt->origin_rule_num--; table_rt->origin_rule_num--;
} }
goto no_save; goto error_out;
} }
else else
{ {
ret=add_compile_rule(table, p_compile, scanner,logger); p_compile=create_compile_rule(&m_rule_tmp, service_define, declared_grp_num, table);
ret=add_compile_rule(table, p_compile, scanner, logger);
if(ret<0) if(ret<0)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
"duplicate config of compile table %s config_id=%d" "duplicate config of compile table %s config_id=%d",
,table->table_name[table->updating_name],p_m_rule->config_id); table->table_name[table->updating_name], m_rule_tmp.config_id);
table->udpate_err_cnt++; table->udpate_err_cnt++;
goto no_save; destroy_compile_rule(p_compile);
p_compile=NULL;
goto error_out;
} }
table_rt->origin_rule_num++; table_rt->origin_rule_num++;
} }
return; return;
no_save: error_out:
free(p_compile->service_defined); table->udpate_err_cnt++;
p_compile->service_defined=NULL;
free(p_compile);
p_compile=NULL;
return; return;
} }
@@ -3399,7 +3419,7 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
switch(bag->type) switch(bag->type)
{ {
case GARBAGE_COMPILE_RULE: case GARBAGE_COMPILE_RULE:
destroy_compile_rule(bag->compile_rule); destroy_compile_rule(bag->compile_rule);
break; break;
case GARBAGE_GROUP_RULE: case GARBAGE_GROUP_RULE:
_destroy_group_rule(bag->group_rule); _destroy_group_rule(bag->group_rule);
@@ -3512,8 +3532,7 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma
if(plugin_desc->have_exdata) if(plugin_desc->have_exdata)
{ {
ret=Maat_helper_read_column(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len); ret=Maat_helper_read_column(table_line, plugin_desc->valid_flag_column, &is_valid_offset, &valid_len);
//thread safe is protected by background_update_mutex
pthread_rwlock_wrlock(&(table_rt->plugin.rwlock));
if(atoi(table_line+is_valid_offset)==1) if(atoi(table_line+is_valid_offset)==1)
{ {
plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger); plugin_EX_data_new(table, table_line, table_rt->plugin.key2ex_hash, logger);
@@ -3522,7 +3541,6 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma
{ {
plugin_EX_data_free(table, table_line, table_rt->plugin.key2ex_hash, logger); plugin_EX_data_free(table, table_line, table_rt->plugin.key2ex_hash, logger);
} }
pthread_rwlock_unlock(&(table_rt->plugin.rwlock));
} }
if(plugin_desc->cb_plug_cnt>0) if(plugin_desc->cb_plug_cnt>0)
{ {

37
src/inc_internal/Maat_rule_internal.h
View File

@@ -85,7 +85,7 @@ struct db_digest_rule
short confidence_degree; short confidence_degree;
int is_valid; int is_valid;
}; };
struct _head_Maat_rule struct Maat_rule_head
{ {
int config_id; int config_id;
int service_id; int service_id;
@@ -95,13 +95,14 @@ struct _head_Maat_rule
char resevered; char resevered;
int serv_def_len; int serv_def_len;
}; };
struct db_compile_rule struct Maat_compile_rule
{ {
struct _head_Maat_rule m_rule_head;// fix len of Maat_rule_t struct Maat_rule_head head;// fix len of Maat_rule_t
char* service_defined; char* service_defined;
long long effective_range;
int is_valid; int is_valid;
int declare_grp_num; int declared_grp_num;
const struct Maat_table_desc* ref_table;
MAAT_RULE_EX_DATA* ads;
}; };
#define PARENT_TYPE_COMPILE 0 #define PARENT_TYPE_COMPILE 0
#define PARENT_TYPE_GROUP 1 #define PARENT_TYPE_GROUP 1
@@ -149,19 +150,17 @@ struct Maat_group_inner
void* compile_shortcut; void* compile_shortcut;
pthread_mutex_t mutex; pthread_mutex_t mutex;
}; };
struct Maat_compile_inner struct Maat_compile_group_relation
{ {
struct db_compile_rule *db_c_rule; struct Maat_compile_rule *compile;
dynamic_array_t *groups; //element is struct Maat_group_inner* dynamic_array_t *groups; //element is struct Maat_group_inner*
char not_flag[MAX_ITEMS_PER_BOOL_EXPR]; char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
char is_valid; int compile_id;//equal to compile->m_rule.config_id
int compile_id;//equal to db_c_rule->m_rule.config_id
const struct Maat_table_desc* ref_table;
int group_boundary; int group_boundary;
int group_cnt; int group_cnt;
int not_group_cnt; int not_group_cnt;
MAAT_RULE_EX_DATA* ads;
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
}; };
struct _compile_result_t struct _compile_result_t
{ {
@@ -225,13 +224,13 @@ struct similar_runtime
GIE_handle_t* gie_handle; GIE_handle_t* gie_handle;
MESA_lqueue_head update_q; MESA_lqueue_head update_q;
}; };
struct plugin_runtime struct plugin_runtime
{ {
dynamic_array_t *cache_lines; dynamic_array_t *cache_lines;
long long cache_line_num; long long cache_line_num;
long long acc_line_num; long long acc_line_num;
long long cache_size; long long cache_size;
pthread_rwlock_t rwlock;
MESA_htable_handle key2ex_hash; MESA_htable_handle key2ex_hash;
}; };
struct expr_runtime struct expr_runtime
@@ -259,7 +258,7 @@ struct Maat_table_runtime
struct plugin_runtime plugin; struct plugin_runtime plugin;
struct expr_runtime expr; struct expr_runtime expr;
struct ip_runtime ip; struct ip_runtime ip;
struct group_runtime group; struct group_runtime group;
void * other; void * other;
}; };
mcore_long_t scan_cnt; mcore_long_t scan_cnt;
@@ -286,6 +285,8 @@ struct Maat_scanner_t
MESA_htable_handle region_hash; MESA_htable_handle region_hash;
MESA_htable_handle group_hash; MESA_htable_handle group_hash;
MESA_htable_handle compile_hash; MESA_htable_handle compile_hash;
MESA_htable_handle compile_group_relation_hash;
MESA_htable_handle district_map; MESA_htable_handle district_map;
MESA_htable_handle tmp_district_map; MESA_htable_handle tmp_district_map;
MESA_htable_handle vertex_id2group; MESA_htable_handle vertex_id2group;
@@ -427,7 +428,7 @@ struct _maat_garbage_t
{ {
struct Maat_scanner_t* scanner; struct Maat_scanner_t* scanner;
struct Maat_group_inner* group_rule; struct Maat_group_inner* group_rule;
struct Maat_compile_inner* compile_rule; struct Maat_compile_rule* compile_rule;
struct bool_matcher* bool_matcher; struct bool_matcher* bool_matcher;
void * raw; void * raw;
MESA_htable_handle str2int_map; MESA_htable_handle str2int_map;
@@ -457,7 +458,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger);
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(struct Maat_compile_inner* compile_rule, struct bool_expr* a_set, unsigned char *has_not); void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger); int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
void maat_start_cb(long long new_version,int update_type,void*u_para); void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para);
@@ -480,9 +481,9 @@ int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_lis
void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger); void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger);
void rewrite_table_line_with_foreign(struct serial_rule_t*p); void rewrite_table_line_with_foreign(struct serial_rule_t*p);
void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule* rule_head, const char* srv_def, int srv_def_len); void fill_maat_rule(struct Maat_rule_t *rule, const struct Maat_rule_head* rule_head, const char* srv_def, int srv_def_len);
MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc); MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
void rule_ex_data_free(const struct _head_Maat_rule * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc); void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index); MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index);
int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);
int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger); int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line, MESA_htable_handle key2ex_hash, void *logger);