diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index b25867a..5c623fb 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -104,7 +104,7 @@ struct compile_sort_para int compile_id; void* user; }; -static void compile_sort_para_set(struct compile_sort_para* para, const struct Maat_compile_group_relation* compile_relation, void* user) +static void compile_sort_para_set(struct compile_sort_para* para, const struct Maat_compile_inner* compile_relation, void* user) { para->compile_id=compile_relation->compile_id; para->evaluation_order=compile_relation->compile->evaluation_order; @@ -144,8 +144,8 @@ static int compile_sort_para_compare(const struct compile_sort_para* a, const st } static int compare_compile_inner(const void *a, const void *b) { - const struct Maat_compile_group_relation *ra=*(const struct Maat_compile_group_relation **)a; - const struct Maat_compile_group_relation *rb=*(const struct Maat_compile_group_relation **)b; + const struct Maat_compile_inner *ra=*(const struct Maat_compile_inner **)a; + const struct Maat_compile_inner *rb=*(const struct Maat_compile_inner **)b; struct compile_sort_para sa, sb; compile_sort_para_set(&sa, ra, NULL); @@ -166,12 +166,12 @@ size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_ru memcpy(copy_rule_array, rule_array, sizeof(struct Maat_rule_t)*n_rule); - struct Maat_compile_group_relation *p=NULL; + struct Maat_compile_inner *p=NULL; size_t i=0, j=0; for(i=0; iscanner->compile_hash, rule_array[i].config_id); + p=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule_array[i].config_id); if(p && 0==pthread_rwlock_tryrdlock(&(p->rwlock)))//rule maybe already deleted. { compile_sort_para_set(sort_para+i, p, copy_rule_array+i); @@ -302,7 +302,7 @@ void scan_hit_status_update_by_group(struct scan_hit_status* hit_status, struct } return; } -size_t scan_hit_status_update_by_compile(struct scan_hit_status* hit_status, struct Maat_compile_group_relation* compile_rule, int Nth_scan) +size_t scan_hit_status_update_by_compile(struct scan_hit_status* hit_status, struct Maat_compile_inner* compile_rule, int Nth_scan) { size_t i=0, j=0; struct Maat_hit_path_inner* p=NULL, *q=NULL; @@ -410,8 +410,8 @@ int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, const size_t r_in_c_cnt=0; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; struct Maat_group_inner* group_rule=NULL; - struct Maat_compile_group_relation* relation_array[MAX_SCANNER_HIT_NUM]; - struct Maat_compile_group_relation* relation=NULL; + struct Maat_compile_inner* relation_array[MAX_SCANNER_HIT_NUM]; + struct Maat_compile_inner* relation=NULL; int virtual_table_id=0; const unsigned long long* hit_group_ids=NULL; size_t hit_group_id_cnt=0; @@ -452,7 +452,7 @@ int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, const } if(scan_ret>1) { - qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation*), + qsort(relation_array, scan_ret, sizeof(struct Maat_compile_inner*), compare_compile_inner); } for(i=0; i<(unsigned int)scan_ret&&result_cntcompile; MAAT_RULE_EX_DATA ad=NULL; @@ -1301,13 +1301,13 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx) { struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - struct Maat_compile_group_relation *relation=NULL; + struct Maat_compile_inner *relation=NULL; struct Maat_compile_rule* compile=NULL; const struct compile_table_schema* compile_desc=NULL; const struct compile_ex_data_idx* ex_desc=NULL; MAAT_RULE_EX_DATA ad=NULL; - relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); + relation=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); if(relation==NULL||relation->compile==NULL) { return NULL; @@ -2478,11 +2478,11 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum { int ret=0; struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather; - struct Maat_compile_group_relation *compile_inner=NULL; + struct Maat_compile_inner *compile_inner=NULL; switch(type) { case MAAT_RULE_SERV_DEFINE: - compile_inner=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); + compile_inner=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id); if(compile_inner==NULL) { ret=0; diff --git a/src/entry/Maat_command.cpp b/src/entry/Maat_command.cpp index b8062d6..1801e90 100644 --- a/src/entry/Maat_command.cpp +++ b/src/entry/Maat_command.cpp @@ -873,7 +873,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ struct Maat_group_t* group_cmd=NULL; struct Maat_region_t* region_cmd=NULL; - struct Maat_compile_group_relation *relation=NULL; + struct Maat_compile_inner *relation=NULL; struct Maat_group_inner* group_inner=NULL; struct Maat_region_inner* region_inner=NULL; void* logger=feather->logger; @@ -885,7 +885,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _ ,"MAAT not ready."); return -1; } - relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id); + relation=(struct Maat_compile_inner *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id); //Operation on relation is thread safe, no immediate memory free when delete a compile rule or a scanner. //In another words, if the relation is accessable from compile means, its was valid in at least 10 seconds (garbage bury). if(relation==NULL) @@ -1389,10 +1389,10 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd) struct Maat_region_t* p_region=NULL; enum MAAT_TABLE_TYPE table_type; - struct Maat_compile_group_relation *compile_rule=NULL; + struct Maat_compile_inner *compile_rule=NULL; if(feather->scanner!=NULL) { - compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id); + compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id); if(compile_rule!=NULL) { MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 43dcd99..868bde4 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -624,7 +624,7 @@ void destroy_group_rule(struct Maat_group_inner* group_rule, int by_whom, struct } } -void make_group_set(struct Maat_compile_group_relation* relation, struct bool_expr* a_set, unsigned char *has_not) +void make_group_set(struct Maat_compile_inner* relation, struct bool_expr* a_set, unsigned char *has_not) { int i=0,j=0; a_set->user_tag=relation; @@ -657,23 +657,23 @@ struct compile_walker void walk_compile_hash(const uchar * key, uint size, void * data, void * user) { struct bool_expr* one_set=NULL; - struct Maat_compile_group_relation* relation=(struct Maat_compile_group_relation*)data; + struct Maat_compile_inner* compile_inner=(struct Maat_compile_inner*)data; struct compile_walker* walker=(struct compile_walker*)user; unsigned char has_not_flag=0; MESA_lqueue_head update_q=walker->update_q; - if(relation->compile==NULL) + if(compile_inner->compile==NULL) { return; } //make sure compile rule's each group has loadded. - if((relation->group_cnt==relation->compile->declared_grp_num - || relation->compile->declared_grp_num==0)//for compatible old version - && relation->group_cnt>0 - && relation->group_cnt!=relation->not_group_cnt) + if((compile_inner->group_cnt==compile_inner->compile->declared_grp_num + || compile_inner->compile->declared_grp_num==0)//for compatible old version + && compile_inner->group_cnt>0 + && compile_inner->group_cnt!=compile_inner->not_group_cnt) { one_set=ALLOC(struct bool_expr, 1); //reading compile rule is safe in update thread, mutex lock called when modified - make_group_set(relation, one_set, &has_not_flag); + make_group_set(compile_inner, one_set, &has_not_flag); if(has_not_flag) { walker->compile_has_not_flag++; @@ -788,10 +788,10 @@ void destroy_compile_rule(struct Maat_compile_rule* compile_rule) free(compile_rule); return; } -struct Maat_compile_group_relation * create_compile_group_relation(int compile_id, struct Maat_scanner *scanner) +struct Maat_compile_inner * create_compile_group_relation(int compile_id, struct Maat_scanner *scanner) { int ret=0; - struct Maat_compile_group_relation* p=ALLOC(struct Maat_compile_group_relation, 1); + struct Maat_compile_inner* p=ALLOC(struct Maat_compile_inner, 1); p->magic_num=COMPILE_RELATION_MAGIC; p->compile_id=compile_id; p->group_cnt=0; @@ -803,7 +803,7 @@ struct Maat_compile_group_relation * create_compile_group_relation(int compile_i return p; } -void _destroy_compile_group_relation(struct Maat_compile_group_relation * cg_relation) +void _destroy_compile_group_relation(struct Maat_compile_inner * cg_relation) { assert(cg_relation->magic_num==COMPILE_RELATION_MAGIC); pthread_rwlock_wrlock(&(cg_relation->rwlock)); @@ -815,7 +815,7 @@ void _destroy_compile_group_relation(struct Maat_compile_group_relation * cg_rel free(cg_relation); } -void destroy_compile_group_relation(struct Maat_compile_group_relation * p, struct Maat_scanner *scanner) +void destroy_compile_group_relation(struct Maat_compile_inner * p, struct Maat_scanner *scanner) { int i=0; UNUSED struct Maat_group_inner* p_group=NULL; @@ -1278,12 +1278,6 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, free(to_update_expr); } -struct region_group_relation -{ - int region_id; - int group_id; - int array_idx; -}; int region_group_relation_add(MESA_htable_handle region_hash, int region_id, int group_id, int array_idx) { struct region_group_relation* relation=ALLOC(struct region_group_relation, 1); @@ -1423,67 +1417,68 @@ unsigned int del_region_from_group(struct Maat_group_inner* group,int region_id, return i; } -int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag) +int add_group_to_compile(struct Maat_compile_inner* compile_inner, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag) { int i=0,ret=-1; int write_pos=-1; struct Maat_group_inner* p=NULL; - pthread_rwlock_wrlock(&(relation->rwlock)); - if(relation->compile!=NULL - && relation->group_cnt>=relation->compile->declared_grp_num - && relation->compile->declared_grp_num!=0) + pthread_rwlock_wrlock(&(compile_inner->rwlock)); + if(compile_inner->compile!=NULL + && compile_inner->group_cnt>=compile_inner->compile->declared_grp_num + && compile_inner->compile->declared_grp_num!=0) { ret=-1; goto error_out; } - for(i=0;igroup_boundary;i++) + for(i=0;igroup_boundary;i++) { - p=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i); + p=(struct Maat_group_inner*)dynamic_array_read(compile_inner->groups,i); if(p==NULL) { write_pos=i; } else { - if(p->group_id==a_rule_group->group_id && relation->virtual_table_id[i]==virual_table_id)//duplicate group + if(p->group_id==a_rule_group->group_id && compile_inner->virtual_table_id[i]==virual_table_id)//duplicate group { ret=-1; goto error_out; } } } - if(write_pos<0&&relation->group_boundary==MAX_EXPR_ITEM_NUM) + if(write_pos<0&&compile_inner->group_boundary==MAX_EXPR_ITEM_NUM) { ret=-1; goto error_out; } if(write_pos<0) { - write_pos=relation->group_boundary; - relation->group_boundary++; + write_pos=compile_inner->group_boundary; + compile_inner->group_boundary++; } - dynamic_array_write(relation->groups, write_pos, a_rule_group); + dynamic_array_write(compile_inner->groups, write_pos, a_rule_group); if(not_flag) { - relation->not_flag[write_pos]=1; - relation->not_group_cnt++; + compile_inner->not_flag[write_pos]=1; + compile_inner->not_group_cnt++; } else { - relation->not_flag[write_pos]=0; + compile_inner->not_flag[write_pos]=0; } - relation->virtual_table_id[write_pos]=virual_table_id; - relation->group_cnt++; + compile_inner->virtual_table_id[write_pos]=virual_table_id; + compile_inner->group_cnt++; a_rule_group->ref_by_parent_cnt++; + a_rule_group->ref_by_compile_cnt++; ret=1; error_out: - pthread_rwlock_unlock(&(relation->rwlock)); + pthread_rwlock_unlock(&(compile_inner->rwlock)); return ret; } -struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relation*relation, int group_id, int virual_table_id) +struct Maat_group_inner* del_group_from_compile(struct Maat_compile_inner*relation, int group_id, int virual_table_id) { int i=0; struct Maat_group_inner* group_rule=NULL; @@ -1513,6 +1508,7 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati } } pthread_rwlock_unlock(&(relation->rwlock)); + group_rule->ref_by_compile_cnt--; return group_rule; } @@ -1981,7 +1977,7 @@ int del_region_rule(struct Maat_table_schema* table,int region_id,int group_id,i int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger) { struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; - struct Maat_compile_group_relation*compile_rule=NULL; + struct Maat_compile_inner*compile_rule=NULL; int ret=0; igraph_integer_t edge_id; group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); @@ -2014,8 +2010,7 @@ int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g } else { - group_rule->ref_by_compile_cnt++; - compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); + compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); if(compile_rule==NULL) { compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner); @@ -2036,7 +2031,7 @@ int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g } int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger) { - struct Maat_compile_group_relation* relation=NULL; + struct Maat_compile_inner* relation=NULL; struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; igraph_es_t es; int ret=0; @@ -2092,7 +2087,7 @@ int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g } else { - relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); + relation=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id); if(relation==NULL) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, @@ -2116,7 +2111,7 @@ int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g { destroy_compile_group_relation(relation, scanner); } - group_rule->ref_by_compile_cnt--; + } destroy_group_rule(group_rule, DESTROY_GROUP_BY_PARENT, scanner); scanner->to_update_group_cnt++; @@ -2124,10 +2119,10 @@ int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g } int add_compile_rule(struct Maat_table_schema* table, struct Maat_compile_rule* db_compile_rule, struct Maat_scanner *scanner, void* logger) { - struct Maat_compile_group_relation *cg_relation=NULL; + struct Maat_compile_inner *cg_relation=NULL; struct Maat_rule_head *p_maat_rule_head=&(db_compile_rule->head); - cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); + cg_relation=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id); if(cg_relation==NULL) { cg_relation=create_compile_group_relation(p_maat_rule_head->config_id, scanner); @@ -2146,8 +2141,8 @@ int add_compile_rule(struct Maat_table_schema* table, struct Maat_compile_rule* } int del_compile_rule(struct Maat_table_schema* table, int compile_id, struct Maat_scanner *scanner, void* logger) { - struct Maat_compile_group_relation *cg_relation=NULL; - cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, compile_id); + struct Maat_compile_inner *cg_relation=NULL; + cg_relation=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, compile_id); if(cg_relation==NULL || cg_relation->compile==NULL) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 5305426..56d7a01 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -183,6 +183,7 @@ int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_k map_register(iris_cfg->str2int_map, "left",2); map_register(iris_cfg->str2int_map, "prefix",2); map_register(iris_cfg->str2int_map, "complete",3); + map_register(iris_cfg->str2int_map, "exact",3); map_register(iris_cfg->str2int_map, "uncase plain",0); map_register(iris_cfg->str2int_map, "hexbin",1); @@ -934,6 +935,28 @@ int write_index_file(struct iris_description_t *p_iris,void* logger) p_iris->idx_fp=NULL; return 0; } +static struct group_info_t* group_info_read(MESA_htable_handle table, const char* group_name) +{ + return (struct group_info_t*)MESA_htable_search(table, (const unsigned char*)group_name, strlen(group_name)); +} +static struct group_info_t* group_info_add_unsafe(struct iris_description_t* p_iris, MESA_htable_handle table, const char* group_name) +{ + static struct group_info_t untitled_group; + struct group_info_t *group_info=NULL; + if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name))) + { + group_info=&untitled_group; + group_info->group_id=get_group_seq(p_iris); + } + else + { + group_info=ALLOC(struct group_info_t, 1); + group_info->group_id=get_group_seq(p_iris); + strncpy(group_info->group_name, group_name, sizeof(group_info->group_name)); + MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info); + } + return group_info; +} int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger) { const char* _str_parent_type[2]={"compile", "group"}; @@ -943,7 +966,6 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac cJSON *sub_groups=NULL, *region_rule=NULL; const char* group_name=NULL, *virtual_table=NULL; struct group_info_t *group_info=NULL; - struct group_info_t untitled_group; item=cJSON_GetObjectItem(group_json, "group_name"); if(item==NULL||item->type!=cJSON_String) @@ -977,22 +999,10 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac { group_not_flag=0; } - - group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name)); + group_info=group_info_read(p_iris->group_name_map, group_name); if(group_info==NULL)//exist group name, regions and sub groups will be ommit. { - if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name))) - { - group_info=&untitled_group; - group_info->group_id=get_group_seq(p_iris); - } - else - { - group_info=ALLOC(struct group_info_t, 1); - group_info->group_id=get_group_seq(p_iris); - strncpy(group_info->group_name, group_name, sizeof(group_info->group_name)); - MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info); - } + group_info=group_info_add_unsafe(p_iris, p_iris->group_name_map, group_name); region_json=cJSON_GetObjectItem(group_json,"regions"); if(region_json!=NULL) { @@ -1043,7 +1053,10 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) int compile_id=-1, compile_cnt=0, group_cnt=0; int ret=0; cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL; - cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL; + cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL, *item=NULL; + static struct group_info_t* parent_group=NULL; + const char* parent_group_name=NULL; + plug_tables=cJSON_GetObjectItem(json,"plugin_table"); if(NULL!=plug_tables) { @@ -1052,6 +1065,34 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) write_plugin_line(each_plug_table, i, p_iris, logger); } } + g_rules=cJSON_GetObjectItem(json, "groups");//sub-group to group + if(g_rules!=NULL) + { + cJSON_ArrayForEach(group_rule, g_rules) + { + item=cJSON_GetObjectItem(group_rule, "parent_group"); + if(item==NULL || item->type!=cJSON_String) + { + parent_group_name=untitled_group_name; + + } + else + { + parent_group_name=item->string; + } + parent_group=group_info_read(p_iris->group_name_map, parent_group_name); + if(parent_group==NULL) + { + parent_group=group_info_add_unsafe(p_iris, p_iris->group_name_map, item->string); + } + ret=write_group_rule(group_rule, parent_group->group_id, PARENT_TYPE_GROUP, 0, p_iris, logger); + if(ret<0) + { + return -1; + } + } + + } c_rules=cJSON_GetObjectItem(json,"rules"); if(c_rules==NULL) { diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 8638f0b..2db3142 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -108,6 +108,7 @@ struct db_group_rule_t int not_flag; int parent_type; //PARENT_TYPE_**, 0:compile, 1: group. int virtual_table_id; + int clause_id; }; struct op_expr_t { @@ -119,6 +120,13 @@ struct op_expr_t int rule_type; }; +struct region_group_relation +{ + int region_id; + int group_id; + int array_idx; +}; + struct Maat_region_inner { int region_id; @@ -129,22 +137,6 @@ struct Maat_region_inner int expr_id_ub; enum MAAT_TABLE_TYPE table_type; }; -#define COMPILE_RELATION_MAGIC 0x1a2b3c4d -struct Maat_compile_group_relation -{ - long long magic_num; - struct Maat_compile_rule *compile; - dynamic_array_t *groups; //element is struct Maat_group_inner* - int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR]; - char not_flag[MAX_ITEMS_PER_BOOL_EXPR]; - int compile_id;//equal to compile->m_rule.config_id - int group_boundary; - int group_cnt; - int not_group_cnt; - pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read - -}; - struct Maat_group_inner { int group_id; @@ -162,6 +154,35 @@ struct Maat_group_inner pthread_mutex_t mutex; }; +#define COMPILE_RELATION_MAGIC 0x1a2b3c4d +struct Maat_group_reference +{ + struct Maat_group_inner* ref_group; + char not_flag; + int virtual_table_id; + int clause_id; + TAILQ_ENTRY(Maat_group_reference) entries; +}; +TAILQ_HEAD(ref_group_q, Maat_group_reference); + +struct Maat_compile_inner +{ + long long magic_num; + struct Maat_compile_rule *compile; + struct ref_group_q ref_group_qhead; + + dynamic_array_t *groups; //element is struct Maat_group_inner* + int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR]; + char not_flag[MAX_ITEMS_PER_BOOL_EXPR]; + int compile_id;//equal to compile->m_rule.config_id + int group_boundary; + int group_cnt; + int not_group_cnt; + pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read + +}; + + struct _compile_result_t { int compile_id; @@ -253,10 +274,14 @@ struct Maat_scanner struct Maat_table_runtime_manager* table_rt_mgr; size_t max_table_num; - MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation* + //Access in both UPDATE thread and SCAN thread MESA_htable_handle exprid_hash; //key: expr_id, value: int array_idx of Maat_group_inner->regions; + MESA_htable_handle compile_hash;//key: compile_id, value: struct Maat_compile_inner * + MESA_htable_handle clause_hash; //key: global_clause_id, value: + + //Access in UPDATE thread ONLY. + MESA_htable_handle region_hash; //key: region_id, value: struct region_group_relation* MESA_htable_handle group_hash; //key: group_id, value: struct Maat_group_inner* - MESA_htable_handle compile_hash;//key: compile_id, value: struct Maat_compile_group_relation * MESA_htable_handle district_map; @@ -402,7 +427,7 @@ struct _maat_garbage_t struct Maat_scanner* scanner; struct Maat_group_inner* group_rule; struct Maat_compile_rule* compile_rule; - struct Maat_compile_group_relation * compile_group_relation; + struct Maat_compile_inner * compile_group_relation; struct bool_matcher* bool_matcher; struct ip_matcher* a_ip_matcher; void * raw; @@ -432,7 +457,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger); void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q); void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q); void garbage_bury(MESA_lqueue_head garbage_q,void *logger); -void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not); +void make_group_set(struct Maat_compile_inner* compile_rule, struct bool_expr* a_set, unsigned char *has_not); void maat_start_cb(long long new_version,int update_type,void*u_para); int maat_update_cb(const char* table_name,const char* line,void *u_para); void maat_finish_cb(void* u_para); diff --git a/test/maat_json.json b/test/maat_json.json index 5692650..02e8e9c 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -1,6 +1,42 @@ { "compile_table": "COMPILE", "group_table": "GROUP", + "groups": [ + { + "group_name": "ASN1234", + "parent_group": "SRC_IP_ASN_1234", + "virtual_table":"SOURCE_IP_ASN", + "regions": [ + { + "table_name": "AS_NUMBER", + "table_type": "string", + "table_content": { + "keywords": "AS1234", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "ASN2345", + "parent_group": "DEST_IP_ASN_2345", + "virtual_table":"DESTINATION_IP_ASN", + "regions": [ + { + "table_name": "AS_NUMBER", + "table_type": "string", + "table_content": { + "keywords": "AS2345", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + } + ], "rules": [ { "compile_id": 123, @@ -1718,7 +1754,26 @@ "not_flag":0 } ] - } + }, + { + "compile_id": 178, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "non-top-object-as-virtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"SRC_IP_ASN_1234", + "not_flag":0 + }, + { + "group_name":"DEST_IP_ASN_2345", + "not_flag":0 + } + ] + } ], "plugin_table": [ { diff --git a/test/table_info.conf b/test/table_info.conf index f577ffb..f7f24c8 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -28,15 +28,15 @@ 7 FILE_DIGEST digest -- 8 HTTP_SIGNATURE expr_plus GBK GBK yes 0 9 SIM_URL similar -- -10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff +10 IMAGE_FP expr UTF8 UTF8 yes 128 11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} -- 12 TEST_FOREIGN_KEY plugin {"valid":4,"foreign":[6,8],"tag":3} -- 13 COMPILE_ALIAS compile escape -- 14 TEST_PLUGIN_EXDATA_TABLE plugin {"key":2,"valid":4,"tag":5,"estimate_size":1024} -- 15 IR_INTERCEPT_IP plugin {"valid":14,"tag":18} -16 APP_PAYLOAD expr_plus UTF8 UTF8 yes 0 quickoff -17 TROJAN_PAYLOAD expr UTF8 UTF8 yes 0 quickoff -18 MAIL_ADDR expr UTF8 UTF8 yes 0 quickoff +16 APP_PAYLOAD expr_plus UTF8 UTF8 yes 0 +17 TROJAN_PAYLOAD expr UTF8 UTF8 yes 0 +18 MAIL_ADDR expr UTF8 UTF8 yes 0 19 IP_PLUS_CONFIG ip_plus -- 20 HTTP_RESPONSE_KEYWORDS virtual KEYWORDS_TABLE -- 21 HTTP_REQUEST_HEADER virtual HTTP_SIGNATURE -- @@ -48,4 +48,7 @@ 25 COMPOSITION_IP_DESTINATION virtual IP_PLUS_CONFIG -- 26 COMPOSITION_IP_SESSION virtual IP_PLUS_CONFIG -- 27 COMPOSITION_IP composition {"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"} -28 TEST_IP_PLUGIN_WITH_EXDATA ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":6} -- \ No newline at end of file +28 TEST_IP_PLUGIN_WITH_EXDATA ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":6} -- +29 AS_NUMBER expr UTF8 UTF8 yes 0 +30 SOURCE_IP_ASN virtual AS_NUMBER -- +31 DESTINATION_IP_ASN virtual AS_NUMBER -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index d17c940..76c4dbb 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -1732,7 +1732,7 @@ TEST(ScanResult, LongerServiceDefine) } TEST(VirtualTable, VirtualWithPhysical) { -#define TestVirtualTable1 +#define VT_VirtualWithPhysical int ret=0, table_id=0; const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me."; const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058"; @@ -1772,7 +1772,7 @@ TEST(VirtualTable, VirtualWithPhysical) } TEST(VirtualTable, VirtualWithVirtual) { -#define TestVirtualTable2 +#define VT_VirtualWithVirtual int ret=0, table_id=0; const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; @@ -1812,7 +1812,7 @@ TEST(VirtualTable, VirtualWithVirtual) } TEST(VirtualTable, OneGroupInTwoVirtual) { -#define TestVirtualTable3 +#define VT_OneGroupInTwoVirtual int ret=0, table_id=0; const char* http_resp_hdr_cookie="sessionid=888888;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; @@ -1849,6 +1849,36 @@ TEST(VirtualTable, OneGroupInTwoVirtual) return; } +TEST(VirtualTable, VirtualAsNonTopObject) +{ +#define VT_AsNonTopObject + int ret=0, table_id=0; + + const char* src_asn="AS1234", *dst_asn="AS2345"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "SOURCE_IP_ASN"); + ASSERT_GT(table_id, 0); + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, src_asn, strlen(src_asn), + result, NULL, 4, &mid, 0); + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(g_feather, "DESTINATION_IP_ASN"); + ASSERT_GT(table_id, 0); + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, dst_asn, strlen(dst_asn), + result, NULL, 4, &mid, 0); + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 178); + + Maat_clean_status(&mid); + + return; +} class MaatFileTest : public testing::Test