gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

在编译配置级别支持合取范式，编译配置最多8个子句（Clause）组成，子句内的分组是“或”运算，子句之间是“与”运算。

Browse Source
This commit is contained in:
zhengchao
2020-05-30 20:56:49 +08:00
parent 48881faafe
commit 1179e9c76a
8 changed files with 259 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/entry/Maat_api.cpp
View File

@@ -104,7 +104,7 @@ struct compile_sort_para
int compile_id; int compile_id;
void* user; void* user;
}; };
static void compile_sort_para_set(struct compile_sort_para* para, const struct Maat_compile_group_relation* compile_relation, void* user) static void compile_sort_para_set(struct compile_sort_para* para, const struct Maat_compile_inner* compile_relation, void* user)
{ {
para->compile_id=compile_relation->compile_id; para->compile_id=compile_relation->compile_id;
para->evaluation_order=compile_relation->compile->evaluation_order; para->evaluation_order=compile_relation->compile->evaluation_order;
@@ -144,8 +144,8 @@ static int compile_sort_para_compare(const struct compile_sort_para* a, const st
} }
static int compare_compile_inner(const void *a, const void *b) static int compare_compile_inner(const void *a, const void *b)
{ {
const struct Maat_compile_group_relation *ra=*(const struct Maat_compile_group_relation **)a; const struct Maat_compile_inner *ra=*(const struct Maat_compile_inner **)a;
const struct Maat_compile_group_relation *rb=*(const struct Maat_compile_group_relation **)b; const struct Maat_compile_inner *rb=*(const struct Maat_compile_inner **)b;
struct compile_sort_para sa, sb; struct compile_sort_para sa, sb;
compile_sort_para_set(&sa, ra, NULL); compile_sort_para_set(&sa, ra, NULL);
@@ -166,12 +166,12 @@ size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_ru
memcpy(copy_rule_array, rule_array, sizeof(struct Maat_rule_t)*n_rule); memcpy(copy_rule_array, rule_array, sizeof(struct Maat_rule_t)*n_rule);
struct Maat_compile_group_relation *p=NULL; struct Maat_compile_inner *p=NULL;
size_t i=0, j=0; size_t i=0, j=0;
for(i=0; i<n_rule; i++) for(i=0; i<n_rule; i++)
{ {
p=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule_array[i].config_id); p=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule_array[i].config_id);
if(p && 0==pthread_rwlock_tryrdlock(&(p->rwlock)))//rule maybe already deleted. if(p && 0==pthread_rwlock_tryrdlock(&(p->rwlock)))//rule maybe already deleted.
{ {
compile_sort_para_set(sort_para+i, p, copy_rule_array+i); compile_sort_para_set(sort_para+i, p, copy_rule_array+i);
@@ -302,7 +302,7 @@ void scan_hit_status_update_by_group(struct scan_hit_status* hit_status, struct
} }
return; return;
} }
size_t scan_hit_status_update_by_compile(struct scan_hit_status* hit_status, struct Maat_compile_group_relation* compile_rule, int Nth_scan) size_t scan_hit_status_update_by_compile(struct scan_hit_status* hit_status, struct Maat_compile_inner* compile_rule, int Nth_scan)
{ {
size_t i=0, j=0; size_t i=0, j=0;
struct Maat_hit_path_inner* p=NULL, *q=NULL; struct Maat_hit_path_inner* p=NULL, *q=NULL;
@@ -410,8 +410,8 @@ int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, const
size_t r_in_c_cnt=0; size_t r_in_c_cnt=0;
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
struct Maat_group_inner* group_rule=NULL; struct Maat_group_inner* group_rule=NULL;
struct Maat_compile_group_relation* relation_array[MAX_SCANNER_HIT_NUM]; struct Maat_compile_inner* relation_array[MAX_SCANNER_HIT_NUM];
struct Maat_compile_group_relation* relation=NULL; struct Maat_compile_inner* relation=NULL;
int virtual_table_id=0; int virtual_table_id=0;
const unsigned long long* hit_group_ids=NULL; const unsigned long long* hit_group_ids=NULL;
size_t hit_group_id_cnt=0; size_t hit_group_id_cnt=0;
@@ -452,7 +452,7 @@ int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, const
} }
if(scan_ret>1) if(scan_ret>1)
{ {
qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation*), qsort(relation_array, scan_ret, sizeof(struct Maat_compile_inner*),
compare_compile_inner); compare_compile_inner);
} }
for(i=0; i<(unsigned int)scan_ret&&result_cnt<size; i++) for(i=0; i<(unsigned int)scan_ret&&result_cnt<size; i++)
@@ -1245,7 +1245,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user) void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
{ {
struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user; struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
struct Maat_compile_group_relation *relation=(struct Maat_compile_group_relation *)data; struct Maat_compile_inner *relation=(struct Maat_compile_inner *)data;
struct Maat_compile_rule* compile_rule=relation->compile; struct Maat_compile_rule* compile_rule=relation->compile;
MAAT_RULE_EX_DATA ad=NULL; MAAT_RULE_EX_DATA ad=NULL;
@@ -1301,13 +1301,13 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx) MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
{ {
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct Maat_compile_group_relation *relation=NULL; struct Maat_compile_inner *relation=NULL;
struct Maat_compile_rule* compile=NULL; struct Maat_compile_rule* compile=NULL;
const struct compile_table_schema* compile_desc=NULL; const struct compile_table_schema* compile_desc=NULL;
const struct compile_ex_data_idx* ex_desc=NULL; const struct compile_ex_data_idx* ex_desc=NULL;
MAAT_RULE_EX_DATA ad=NULL; MAAT_RULE_EX_DATA ad=NULL;
relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); relation=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(relation==NULL||relation->compile==NULL) if(relation==NULL||relation->compile==NULL)
{ {
return NULL; return NULL;
@@ -2478,11 +2478,11 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
{ {
int ret=0; int ret=0;
struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
struct Maat_compile_group_relation *compile_inner=NULL; struct Maat_compile_inner *compile_inner=NULL;
switch(type) switch(type)
{ {
case MAAT_RULE_SERV_DEFINE: case MAAT_RULE_SERV_DEFINE:
compile_inner=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); compile_inner=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
if(compile_inner==NULL) if(compile_inner==NULL)
{ {
ret=0; ret=0;

8
src/entry/Maat_command.cpp
View File

@@ -873,7 +873,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
struct Maat_group_t* group_cmd=NULL; struct Maat_group_t* group_cmd=NULL;
struct Maat_region_t* region_cmd=NULL; struct Maat_region_t* region_cmd=NULL;
struct Maat_compile_group_relation *relation=NULL; struct Maat_compile_inner *relation=NULL;
struct Maat_group_inner* group_inner=NULL; struct Maat_group_inner* group_inner=NULL;
struct Maat_region_inner* region_inner=NULL; struct Maat_region_inner* region_inner=NULL;
void* logger=feather->logger; void* logger=feather->logger;
@@ -885,7 +885,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
,"MAAT not ready."); ,"MAAT not ready.");
return -1; return -1;
} }
relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id); relation=(struct Maat_compile_inner *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
//Operation on relation is thread safe, no immediate memory free when delete a compile rule or a scanner. //Operation on relation is thread safe, no immediate memory free when delete a compile rule or a scanner.
//In another words, if the relation is accessable from compile means, its was valid in at least 10 seconds (garbage bury). //In another words, if the relation is accessable from compile means, its was valid in at least 10 seconds (garbage bury).
if(relation==NULL) if(relation==NULL)
@@ -1389,10 +1389,10 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
struct Maat_region_t* p_region=NULL; struct Maat_region_t* p_region=NULL;
enum MAAT_TABLE_TYPE table_type; enum MAAT_TABLE_TYPE table_type;
struct Maat_compile_group_relation *compile_rule=NULL; struct Maat_compile_inner *compile_rule=NULL;
if(feather->scanner!=NULL) if(feather->scanner!=NULL)
{ {
compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id); compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
if(compile_rule!=NULL) if(compile_rule!=NULL)
{ {
MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module

89
src/entry/Maat_rule.cpp
View File

@@ -624,7 +624,7 @@ void destroy_group_rule(struct Maat_group_inner* group_rule, int by_whom, struct
} }
} }
void make_group_set(struct Maat_compile_group_relation* relation, struct bool_expr* a_set, unsigned char *has_not) void make_group_set(struct Maat_compile_inner* relation, struct bool_expr* a_set, unsigned char *has_not)
{ {
int i=0,j=0; int i=0,j=0;
a_set->user_tag=relation; a_set->user_tag=relation;
@@ -657,23 +657,23 @@ struct compile_walker
void walk_compile_hash(const uchar * key, uint size, void * data, void * user) void walk_compile_hash(const uchar * key, uint size, void * data, void * user)
{ {
struct bool_expr* one_set=NULL; struct bool_expr* one_set=NULL;
struct Maat_compile_group_relation* relation=(struct Maat_compile_group_relation*)data; struct Maat_compile_inner* compile_inner=(struct Maat_compile_inner*)data;
struct compile_walker* walker=(struct compile_walker*)user; struct compile_walker* walker=(struct compile_walker*)user;
unsigned char has_not_flag=0; unsigned char has_not_flag=0;
MESA_lqueue_head update_q=walker->update_q; MESA_lqueue_head update_q=walker->update_q;
if(relation->compile==NULL) if(compile_inner->compile==NULL)
{ {
return; return;
} }
//make sure compile rule's each group has loadded. //make sure compile rule's each group has loadded.
if((relation->group_cnt==relation->compile->declared_grp_num if((compile_inner->group_cnt==compile_inner->compile->declared_grp_num
|| relation->compile->declared_grp_num==0)//for compatible old version || compile_inner->compile->declared_grp_num==0)//for compatible old version
&& relation->group_cnt>0 && compile_inner->group_cnt>0
&& relation->group_cnt!=relation->not_group_cnt) && compile_inner->group_cnt!=compile_inner->not_group_cnt)
{ {
one_set=ALLOC(struct bool_expr, 1); one_set=ALLOC(struct bool_expr, 1);
//reading compile rule is safe in update thread, mutex lock called when modified //reading compile rule is safe in update thread, mutex lock called when modified
make_group_set(relation, one_set, &has_not_flag); make_group_set(compile_inner, one_set, &has_not_flag);
if(has_not_flag) if(has_not_flag)
{ {
walker->compile_has_not_flag++; walker->compile_has_not_flag++;
@@ -788,10 +788,10 @@ void destroy_compile_rule(struct Maat_compile_rule* compile_rule)
free(compile_rule); free(compile_rule);
return; return;
} }
struct Maat_compile_group_relation * create_compile_group_relation(int compile_id, struct Maat_scanner *scanner) struct Maat_compile_inner * create_compile_group_relation(int compile_id, struct Maat_scanner *scanner)
{ {
int ret=0; int ret=0;
struct Maat_compile_group_relation* p=ALLOC(struct Maat_compile_group_relation, 1); struct Maat_compile_inner* p=ALLOC(struct Maat_compile_inner, 1);
p->magic_num=COMPILE_RELATION_MAGIC; p->magic_num=COMPILE_RELATION_MAGIC;
p->compile_id=compile_id; p->compile_id=compile_id;
p->group_cnt=0; p->group_cnt=0;
@@ -803,7 +803,7 @@ struct Maat_compile_group_relation * create_compile_group_relation(int compile_i
return p; return p;
} }
void _destroy_compile_group_relation(struct Maat_compile_group_relation * cg_relation) void _destroy_compile_group_relation(struct Maat_compile_inner * cg_relation)
{ {
assert(cg_relation->magic_num==COMPILE_RELATION_MAGIC); assert(cg_relation->magic_num==COMPILE_RELATION_MAGIC);
pthread_rwlock_wrlock(&(cg_relation->rwlock)); pthread_rwlock_wrlock(&(cg_relation->rwlock));
@@ -815,7 +815,7 @@ void _destroy_compile_group_relation(struct Maat_compile_group_relation * cg_rel
free(cg_relation); free(cg_relation);
} }
void destroy_compile_group_relation(struct Maat_compile_group_relation * p, struct Maat_scanner *scanner) void destroy_compile_group_relation(struct Maat_compile_inner * p, struct Maat_scanner *scanner)
{ {
int i=0; int i=0;
UNUSED struct Maat_group_inner* p_group=NULL; UNUSED struct Maat_group_inner* p_group=NULL;
@@ -1278,12 +1278,6 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
free(to_update_expr); free(to_update_expr);
} }
struct region_group_relation
{
int region_id;
int group_id;
int array_idx;
};
int region_group_relation_add(MESA_htable_handle region_hash, int region_id, int group_id, int array_idx) int region_group_relation_add(MESA_htable_handle region_hash, int region_id, int group_id, int array_idx)
{ {
struct region_group_relation* relation=ALLOC(struct region_group_relation, 1); struct region_group_relation* relation=ALLOC(struct region_group_relation, 1);
@@ -1423,67 +1417,68 @@ unsigned int del_region_from_group(struct Maat_group_inner* group,int region_id,
return i; return i;
} }
int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag) int add_group_to_compile(struct Maat_compile_inner* compile_inner, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag)
{ {
int i=0,ret=-1; int i=0,ret=-1;
int write_pos=-1; int write_pos=-1;
struct Maat_group_inner* p=NULL; struct Maat_group_inner* p=NULL;
pthread_rwlock_wrlock(&(relation->rwlock)); pthread_rwlock_wrlock(&(compile_inner->rwlock));
if(relation->compile!=NULL if(compile_inner->compile!=NULL
&& relation->group_cnt>=relation->compile->declared_grp_num && compile_inner->group_cnt>=compile_inner->compile->declared_grp_num
&& relation->compile->declared_grp_num!=0) && compile_inner->compile->declared_grp_num!=0)
{ {
ret=-1; ret=-1;
goto error_out; goto error_out;
} }
for(i=0;i<relation->group_boundary;i++) for(i=0;i<compile_inner->group_boundary;i++)
{ {
p=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i); p=(struct Maat_group_inner*)dynamic_array_read(compile_inner->groups,i);
if(p==NULL) if(p==NULL)
{ {
write_pos=i; write_pos=i;
} }
else else
{ {
if(p->group_id==a_rule_group->group_id && relation->virtual_table_id[i]==virual_table_id)//duplicate group if(p->group_id==a_rule_group->group_id && compile_inner->virtual_table_id[i]==virual_table_id)//duplicate group
{ {
ret=-1; ret=-1;
goto error_out; goto error_out;
} }
} }
} }
if(write_pos<0&&relation->group_boundary==MAX_EXPR_ITEM_NUM) if(write_pos<0&&compile_inner->group_boundary==MAX_EXPR_ITEM_NUM)
{ {
ret=-1; ret=-1;
goto error_out; goto error_out;
} }
if(write_pos<0) if(write_pos<0)
{ {
write_pos=relation->group_boundary; write_pos=compile_inner->group_boundary;
relation->group_boundary++; compile_inner->group_boundary++;
} }
dynamic_array_write(relation->groups, write_pos, a_rule_group); dynamic_array_write(compile_inner->groups, write_pos, a_rule_group);
if(not_flag) if(not_flag)
{ {
relation->not_flag[write_pos]=1; compile_inner->not_flag[write_pos]=1;
relation->not_group_cnt++; compile_inner->not_group_cnt++;
} }
else else
{ {
relation->not_flag[write_pos]=0; compile_inner->not_flag[write_pos]=0;
} }
relation->virtual_table_id[write_pos]=virual_table_id; compile_inner->virtual_table_id[write_pos]=virual_table_id;
relation->group_cnt++; compile_inner->group_cnt++;
a_rule_group->ref_by_parent_cnt++; a_rule_group->ref_by_parent_cnt++;
a_rule_group->ref_by_compile_cnt++;
ret=1; ret=1;
error_out: error_out:
pthread_rwlock_unlock(&(relation->rwlock)); pthread_rwlock_unlock(&(compile_inner->rwlock));
return ret; return ret;
} }
struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relation*relation, int group_id, int virual_table_id) struct Maat_group_inner* del_group_from_compile(struct Maat_compile_inner*relation, int group_id, int virual_table_id)
{ {
int i=0; int i=0;
struct Maat_group_inner* group_rule=NULL; struct Maat_group_inner* group_rule=NULL;
@@ -1513,6 +1508,7 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati
} }
} }
pthread_rwlock_unlock(&(relation->rwlock)); pthread_rwlock_unlock(&(relation->rwlock));
group_rule->ref_by_compile_cnt--;
return group_rule; return group_rule;
} }
@@ -1981,7 +1977,7 @@ int del_region_rule(struct Maat_table_schema* table,int region_id,int group_id,i
int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger) int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger)
{ {
struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
struct Maat_compile_group_relation*compile_rule=NULL; struct Maat_compile_inner*compile_rule=NULL;
int ret=0; int ret=0;
igraph_integer_t edge_id; igraph_integer_t edge_id;
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
@@ -2014,8 +2010,7 @@ int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
} }
else else
{ {
group_rule->ref_by_compile_cnt++; compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id);
compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id);
if(compile_rule==NULL) if(compile_rule==NULL)
{ {
compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner); compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner);
@@ -2036,7 +2031,7 @@ int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
} }
int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger) int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger)
{ {
struct Maat_compile_group_relation* relation=NULL; struct Maat_compile_inner* relation=NULL;
struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
igraph_es_t es; igraph_es_t es;
int ret=0; int ret=0;
@@ -2092,7 +2087,7 @@ int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
} }
else else
{ {
relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); relation=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id);
if(relation==NULL) if(relation==NULL)
{ {
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
@@ -2116,7 +2111,7 @@ int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
{ {
destroy_compile_group_relation(relation, scanner); destroy_compile_group_relation(relation, scanner);
} }
group_rule->ref_by_compile_cnt--;
} }
destroy_group_rule(group_rule, DESTROY_GROUP_BY_PARENT, scanner); destroy_group_rule(group_rule, DESTROY_GROUP_BY_PARENT, scanner);
scanner->to_update_group_cnt++; scanner->to_update_group_cnt++;
@@ -2124,10 +2119,10 @@ int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
} }
int add_compile_rule(struct Maat_table_schema* table, struct Maat_compile_rule* db_compile_rule, struct Maat_scanner *scanner, void* logger) int add_compile_rule(struct Maat_table_schema* table, struct Maat_compile_rule* db_compile_rule, struct Maat_scanner *scanner, void* logger)
{ {
struct Maat_compile_group_relation *cg_relation=NULL; struct Maat_compile_inner *cg_relation=NULL;
struct Maat_rule_head *p_maat_rule_head=&(db_compile_rule->head); struct Maat_rule_head *p_maat_rule_head=&(db_compile_rule->head);
cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); cg_relation=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
if(cg_relation==NULL) if(cg_relation==NULL)
{ {
cg_relation=create_compile_group_relation(p_maat_rule_head->config_id, scanner); cg_relation=create_compile_group_relation(p_maat_rule_head->config_id, scanner);
@@ -2146,8 +2141,8 @@ int add_compile_rule(struct Maat_table_schema* table, struct Maat_compile_rule*
} }
int del_compile_rule(struct Maat_table_schema* table, int compile_id, struct Maat_scanner *scanner, void* logger) int del_compile_rule(struct Maat_table_schema* table, int compile_id, struct Maat_scanner *scanner, void* logger)
{ {
struct Maat_compile_group_relation *cg_relation=NULL; struct Maat_compile_inner *cg_relation=NULL;
cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, compile_id); cg_relation=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, compile_id);
if(cg_relation==NULL || cg_relation->compile==NULL) if(cg_relation==NULL || cg_relation->compile==NULL)
{ {
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,

73
src/entry/json2iris.cpp
View File

@@ -183,6 +183,7 @@ int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_k
map_register(iris_cfg->str2int_map, "left",2); map_register(iris_cfg->str2int_map, "left",2);
map_register(iris_cfg->str2int_map, "prefix",2); map_register(iris_cfg->str2int_map, "prefix",2);
map_register(iris_cfg->str2int_map, "complete",3); map_register(iris_cfg->str2int_map, "complete",3);
map_register(iris_cfg->str2int_map, "exact",3);
map_register(iris_cfg->str2int_map, "uncase plain",0); map_register(iris_cfg->str2int_map, "uncase plain",0);
map_register(iris_cfg->str2int_map, "hexbin",1); map_register(iris_cfg->str2int_map, "hexbin",1);
@@ -934,6 +935,28 @@ int write_index_file(struct iris_description_t *p_iris,void* logger)
p_iris->idx_fp=NULL; p_iris->idx_fp=NULL;
return 0; return 0;
} }
static struct group_info_t* group_info_read(MESA_htable_handle table, const char* group_name)
{
return (struct group_info_t*)MESA_htable_search(table, (const unsigned char*)group_name, strlen(group_name));
}
static struct group_info_t* group_info_add_unsafe(struct iris_description_t* p_iris, MESA_htable_handle table, const char* group_name)
{
static struct group_info_t untitled_group;
struct group_info_t *group_info=NULL;
if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name)))
{
group_info=&untitled_group;
group_info->group_id=get_group_seq(p_iris);
}
else
{
group_info=ALLOC(struct group_info_t, 1);
group_info->group_id=get_group_seq(p_iris);
strncpy(group_info->group_name, group_name, sizeof(group_info->group_name));
MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info);
}
return group_info;
}
int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger) int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger)
{ {
const char* _str_parent_type[2]={"compile", "group"}; const char* _str_parent_type[2]={"compile", "group"};
@@ -943,7 +966,6 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
cJSON *sub_groups=NULL, *region_rule=NULL; cJSON *sub_groups=NULL, *region_rule=NULL;
const char* group_name=NULL, *virtual_table=NULL; const char* group_name=NULL, *virtual_table=NULL;
struct group_info_t *group_info=NULL; struct group_info_t *group_info=NULL;
struct group_info_t untitled_group;
item=cJSON_GetObjectItem(group_json, "group_name"); item=cJSON_GetObjectItem(group_json, "group_name");
if(item==NULL||item->type!=cJSON_String) if(item==NULL||item->type!=cJSON_String)
@@ -977,22 +999,10 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
{ {
group_not_flag=0; group_not_flag=0;
} }
group_info=group_info_read(p_iris->group_name_map, group_name);
group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name));
if(group_info==NULL)//exist group name, regions and sub groups will be ommit. if(group_info==NULL)//exist group name, regions and sub groups will be ommit.
{ {
if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name))) group_info=group_info_add_unsafe(p_iris, p_iris->group_name_map, group_name);
{
group_info=&untitled_group;
group_info->group_id=get_group_seq(p_iris);
}
else
{
group_info=ALLOC(struct group_info_t, 1);
group_info->group_id=get_group_seq(p_iris);
strncpy(group_info->group_name, group_name, sizeof(group_info->group_name));
MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info);
}
region_json=cJSON_GetObjectItem(group_json,"regions"); region_json=cJSON_GetObjectItem(group_json,"regions");
if(region_json!=NULL) if(region_json!=NULL)
{ {
@@ -1043,7 +1053,10 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
int compile_id=-1, compile_cnt=0, group_cnt=0; int compile_id=-1, compile_cnt=0, group_cnt=0;
int ret=0; int ret=0;
cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL; cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL;
cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL; cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL, *item=NULL;
static struct group_info_t* parent_group=NULL;
const char* parent_group_name=NULL;
plug_tables=cJSON_GetObjectItem(json,"plugin_table"); plug_tables=cJSON_GetObjectItem(json,"plugin_table");
if(NULL!=plug_tables) if(NULL!=plug_tables)
{ {
@@ -1052,6 +1065,34 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
write_plugin_line(each_plug_table, i, p_iris, logger); write_plugin_line(each_plug_table, i, p_iris, logger);
} }
} }
g_rules=cJSON_GetObjectItem(json, "groups");//sub-group to group
if(g_rules!=NULL)
{
cJSON_ArrayForEach(group_rule, g_rules)
{
item=cJSON_GetObjectItem(group_rule, "parent_group");
if(item==NULL || item->type!=cJSON_String)
{
parent_group_name=untitled_group_name;
}
else
{
parent_group_name=item->string;
}
parent_group=group_info_read(p_iris->group_name_map, parent_group_name);
if(parent_group==NULL)
{
parent_group=group_info_add_unsafe(p_iris, p_iris->group_name_map, item->string);
}
ret=write_group_rule(group_rule, parent_group->group_id, PARENT_TYPE_GROUP, 0, p_iris, logger);
if(ret<0)
{
return -1;
}
}
}
c_rules=cJSON_GetObjectItem(json,"rules"); c_rules=cJSON_GetObjectItem(json,"rules");
if(c_rules==NULL) if(c_rules==NULL)
{ {

65
src/inc_internal/Maat_rule_internal.h
View File

@@ -108,6 +108,7 @@ struct db_group_rule_t
int not_flag; int not_flag;
int parent_type; //PARENT_TYPE_**, 0:compile, 1: group. int parent_type; //PARENT_TYPE_**, 0:compile, 1: group.
int virtual_table_id; int virtual_table_id;
int clause_id;
}; };
struct op_expr_t struct op_expr_t
{ {
@@ -119,6 +120,13 @@ struct op_expr_t
int rule_type; int rule_type;
}; };
struct region_group_relation
{
int region_id;
int group_id;
int array_idx;
};
struct Maat_region_inner struct Maat_region_inner
{ {
int region_id; int region_id;
@@ -129,22 +137,6 @@ struct Maat_region_inner
int expr_id_ub; int expr_id_ub;
enum MAAT_TABLE_TYPE table_type; enum MAAT_TABLE_TYPE table_type;
}; };
#define COMPILE_RELATION_MAGIC 0x1a2b3c4d
struct Maat_compile_group_relation
{
long long magic_num;
struct Maat_compile_rule *compile;
dynamic_array_t *groups; //element is struct Maat_group_inner*
int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR];
char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
int compile_id;//equal to compile->m_rule.config_id
int group_boundary;
int group_cnt;
int not_group_cnt;
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
};
struct Maat_group_inner struct Maat_group_inner
{ {
int group_id; int group_id;
@@ -162,6 +154,35 @@ struct Maat_group_inner
pthread_mutex_t mutex; pthread_mutex_t mutex;
}; };
#define COMPILE_RELATION_MAGIC 0x1a2b3c4d
struct Maat_group_reference
{
struct Maat_group_inner* ref_group;
char not_flag;
int virtual_table_id;
int clause_id;
TAILQ_ENTRY(Maat_group_reference) entries;
};
TAILQ_HEAD(ref_group_q, Maat_group_reference);
struct Maat_compile_inner
{
long long magic_num;
struct Maat_compile_rule *compile;
struct ref_group_q ref_group_qhead;
dynamic_array_t *groups; //element is struct Maat_group_inner*
int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR];
char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
int compile_id;//equal to compile->m_rule.config_id
int group_boundary;
int group_cnt;
int not_group_cnt;
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
};
struct _compile_result_t struct _compile_result_t
{ {
int compile_id; int compile_id;
@@ -253,10 +274,14 @@ struct Maat_scanner
struct Maat_table_runtime_manager* table_rt_mgr; struct Maat_table_runtime_manager* table_rt_mgr;
size_t max_table_num; size_t max_table_num;
MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation* //Access in both UPDATE thread and SCAN thread
MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions; MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions;
MESA_htable_handle compile_hash;//key: compile_id, value: struct Maat_compile_inner *
MESA_htable_handle clause_hash; //key: global_clause_id, value:
//Access in UPDATE thread ONLY.
MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation*
MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_inner* MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_inner*
MESA_htable_handle compile_hash;//key: compile_id, value: struct Maat_compile_group_relation *
MESA_htable_handle district_map; MESA_htable_handle district_map;
@@ -402,7 +427,7 @@ struct _maat_garbage_t
struct Maat_scanner* scanner; struct Maat_scanner* scanner;
struct Maat_group_inner* group_rule; struct Maat_group_inner* group_rule;
struct Maat_compile_rule* compile_rule; struct Maat_compile_rule* compile_rule;
struct Maat_compile_group_relation * compile_group_relation; struct Maat_compile_inner * compile_group_relation;
struct bool_matcher* bool_matcher; struct bool_matcher* bool_matcher;
struct ip_matcher* a_ip_matcher; struct ip_matcher* a_ip_matcher;
void * raw; void * raw;
@@ -432,7 +457,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger);
void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
void garbage_bury(MESA_lqueue_head garbage_q,void *logger); void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not); void make_group_set(struct Maat_compile_inner* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
void maat_start_cb(long long new_version,int update_type,void*u_para); void maat_start_cb(long long new_version,int update_type,void*u_para);
int maat_update_cb(const char* table_name,const char* line,void *u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para);
void maat_finish_cb(void* u_para); void maat_finish_cb(void* u_para);

57
test/maat_json.json
View File

@@ -1,6 +1,42 @@
{ {
"compile_table": "COMPILE", "compile_table": "COMPILE",
"group_table": "GROUP", "group_table": "GROUP",
"groups": [
{
"group_name": "ASN1234",
"parent_group": "SRC_IP_ASN_1234",
"virtual_table":"SOURCE_IP_ASN",
"regions": [
{
"table_name": "AS_NUMBER",
"table_type": "string",
"table_content": {
"keywords": "AS1234",
"expr_type": "none",
"match_method": "exact",
"format": "uncase plain"
}
}
]
},
{
"group_name": "ASN2345",
"parent_group": "DEST_IP_ASN_2345",
"virtual_table":"DESTINATION_IP_ASN",
"regions": [
{
"table_name": "AS_NUMBER",
"table_type": "string",
"table_content": {
"keywords": "AS2345",
"expr_type": "none",
"match_method": "exact",
"format": "uncase plain"
}
}
]
}
],
"rules": [ "rules": [
{ {
"compile_id": 123, "compile_id": 123,
@@ -1718,7 +1754,26 @@
"not_flag":0 "not_flag":0
} }
] ]
} },
{
"compile_id": 178,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "non-top-object-as-virtual",
"is_valid": "yes",
"groups": [
{
"group_name":"SRC_IP_ASN_1234",
"not_flag":0
},
{
"group_name":"DEST_IP_ASN_2345",
"not_flag":0
}
]
}
], ],
"plugin_table": [ "plugin_table": [
{ {

11
test/table_info.conf
View File

@@ -28,15 +28,15 @@
7 FILE_DIGEST digest -- 7 FILE_DIGEST digest --
8 HTTP_SIGNATURE expr_plus GBK GBK yes 0 8 HTTP_SIGNATURE expr_plus GBK GBK yes 0
9 SIM_URL similar -- 9 SIM_URL similar --
10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff 10 IMAGE_FP expr UTF8 UTF8 yes 128
11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} -- 11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} --
12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":[6,8],"tag":3} -- 12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":[6,8],"tag":3} --
13 COMPILE_ALIAS compile escape -- 13 COMPILE_ALIAS compile escape --
14 TEST_PLUGIN_EXDATA_TABLE plugin {"key":2,"valid":4,"tag":5,"estimate_size":1024} -- 14 TEST_PLUGIN_EXDATA_TABLE plugin {"key":2,"valid":4,"tag":5,"estimate_size":1024} --
15 IR_INTERCEPT_IP plugin {"valid":14,"tag":18} 15 IR_INTERCEPT_IP plugin {"valid":14,"tag":18}
16 APP_PAYLOAD expr_plus UTF8 UTF8 yes 0 quickoff 16 APP_PAYLOAD expr_plus UTF8 UTF8 yes 0
17 TROJAN_PAYLOAD expr UTF8 UTF8 yes 0 quickoff 17 TROJAN_PAYLOAD expr UTF8 UTF8 yes 0
18 MAIL_ADDR expr UTF8 UTF8 yes 0 quickoff 18 MAIL_ADDR expr UTF8 UTF8 yes 0
19 IP_PLUS_CONFIG ip_plus -- 19 IP_PLUS_CONFIG ip_plus --
20 HTTP_RESPONSE_KEYWORDS virtual KEYWORDS_TABLE -- 20 HTTP_RESPONSE_KEYWORDS virtual KEYWORDS_TABLE --
21 HTTP_REQUEST_HEADER virtual HTTP_SIGNATURE -- 21 HTTP_REQUEST_HEADER virtual HTTP_SIGNATURE --
@@ -49,3 +49,6 @@
26 COMPOSITION_IP_SESSION virtual IP_PLUS_CONFIG -- 26 COMPOSITION_IP_SESSION virtual IP_PLUS_CONFIG --
27 COMPOSITION_IP composition {"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"} 27 COMPOSITION_IP composition {"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"}
28 TEST_IP_PLUGIN_WITH_EXDATA ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":6} -- 28 TEST_IP_PLUGIN_WITH_EXDATA ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":6} --
29 AS_NUMBER expr UTF8 UTF8 yes 0
30 SOURCE_IP_ASN virtual AS_NUMBER --
31 DESTINATION_IP_ASN virtual AS_NUMBER --

36
test/test_maatframe.cpp
View File

@@ -1732,7 +1732,7 @@ TEST(ScanResult, LongerServiceDefine)
} }
TEST(VirtualTable, VirtualWithPhysical) TEST(VirtualTable, VirtualWithPhysical)
{ {
#define TestVirtualTable1 #define VT_VirtualWithPhysical
int ret=0, table_id=0; int ret=0, table_id=0;
const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me."; const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058"; const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058";
@@ -1772,7 +1772,7 @@ TEST(VirtualTable, VirtualWithPhysical)
} }
TEST(VirtualTable, VirtualWithVirtual) TEST(VirtualTable, VirtualWithVirtual)
{ {
#define TestVirtualTable2 #define VT_VirtualWithVirtual
int ret=0, table_id=0; int ret=0, table_id=0;
const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;";
@@ -1812,7 +1812,7 @@ TEST(VirtualTable, VirtualWithVirtual)
} }
TEST(VirtualTable, OneGroupInTwoVirtual) TEST(VirtualTable, OneGroupInTwoVirtual)
{ {
#define TestVirtualTable3 #define VT_OneGroupInTwoVirtual
int ret=0, table_id=0; int ret=0, table_id=0;
const char* http_resp_hdr_cookie="sessionid=888888;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; const char* http_resp_hdr_cookie="sessionid=888888;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;";
@@ -1849,6 +1849,36 @@ TEST(VirtualTable, OneGroupInTwoVirtual)
return; return;
} }
TEST(VirtualTable, VirtualAsNonTopObject)
{
#define VT_AsNonTopObject
int ret=0, table_id=0;
const char* src_asn="AS1234", *dst_asn="AS2345";
struct Maat_rule_t result[4];
memset(result, 0, sizeof(result));
scan_status_t mid=NULL;
table_id=Maat_table_register(g_feather, "SOURCE_IP_ASN");
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, src_asn, strlen(src_asn),
result, NULL, 4, &mid, 0);
EXPECT_EQ(ret, -2);
table_id=Maat_table_register(g_feather, "DESTINATION_IP_ASN");
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, dst_asn, strlen(dst_asn),
result, NULL, 4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 178);
Maat_clean_status(&mid);
return;
}
class MaatFileTest : public testing::Test class MaatFileTest : public testing::Test