diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index 79b86e9..6908002 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -159,8 +159,9 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int unsigned char has_not_flag=0; struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler; struct Maat_group_inner* group_rule=NULL; - struct Maat_compile_inner* array_mi_rule[MAX_SCANNER_HIT_NUM]; - struct Maat_compile_inner* _mi_rule=NULL; + struct Maat_compile_inner* compile_arrary[MAX_SCANNER_HIT_NUM]; + struct Maat_compile_inner* compile_rule=NULL; + int region_pos[MAX_SCANNER_HIT_NUM]; _mid->cur_hit_cnt=0; for(i=0;icompile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cntcompile_shortcut); + compile_arrary[shortcut_avilable_cnt]=(struct Maat_compile_inner*)(group_rule->compile_shortcut); shortcut_avilable_cnt++; } - _mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id; - _mid->cur_hit_cnt++; for(j=0; jendpoint_cnt; j++) { + _mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->endpoints[j]; + _mid->cur_hit_cnt++; ret=insert_set_id(&(_mid->hitted_group_id), &(_mid->hit_group_size), _mid->hit_group_cnt, @@ -205,40 +206,40 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int { scan_ret=bool_matcher_match(bm, thread_num, _mid->hitted_group_id, _mid->hit_group_cnt, - (void **)array_mi_rule, MAX_SCANNER_HIT_NUM); + (void **)compile_arrary, MAX_SCANNER_HIT_NUM); } for(i=0;irwlock))) + if(0==pthread_rwlock_tryrdlock(&(compile_rule->rwlock))) { - if(_mi_rule->is_valid==1) + if(compile_rule->is_valid==1) { - if(_mi_rule->not_group_cnt>0 && !is_last_region) + if(compile_rule->not_group_cnt>0 && !is_last_region) { _mid->not_grp_compile_hitted_flag=1; } else { - make_group_set(_mi_rule, &(rs_result[result_cnt].group_set), &has_not_flag); + make_group_set(compile_rule, &(rs_result[result_cnt].group_set), &has_not_flag); r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt, region_pos, MAX_SCANNER_HIT_NUM); if(r_in_c_cnt>0 || //compile config hitted becasue of new reigon _mid->cur_hit_cnt==0) //or ever hit a compile that refer a NOT-logic group { - fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head), - _mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len); - rs_result[result_cnt].compile_id=_mi_rule->compile_id; + fill_maat_rule(&(result[result_cnt]), &(compile_rule->db_c_rule->m_rule_head), + compile_rule->db_c_rule->service_defined ,compile_rule->db_c_rule->m_rule_head.serv_def_len); + rs_result[result_cnt].compile_id=compile_rule->compile_id; result_cnt++; } } } - pthread_rwlock_unlock(&(_mi_rule->rwlock)); + pthread_rwlock_unlock(&(compile_rule->rwlock)); } } if(result_cnt>0) diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 37bfed1..909a41e 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -836,7 +836,7 @@ error_jump: map_destroy(string2int_map); return table_cnt; } -struct Maat_group_inner* create_group_rule(int group_id) +struct Maat_group_inner* create_group_rule(int group_id, struct Maat_scanner_t *scanner) { struct Maat_group_inner* group=ALLOC(struct Maat_group_inner, 1); group->group_id=group_id; @@ -847,6 +847,10 @@ struct Maat_group_inner* create_group_rule(int group_id) group->compile_shortcut=NULL; group->table_id=0; group->group_name=NULL; + group->vertex_id=scanner->grp_vertex_id_generator++; + assert(igraph_vcount(&scanner->group_graph)==group->vertex_id); + igraph_add_vertices(&scanner->group_graph, 1, NULL); + HASH_add_by_id(scanner->vertex_id2group, group->vertex_id, group); pthread_mutex_init(&(group->mutex), NULL); return group; } @@ -1326,6 +1330,9 @@ struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t scanner->group_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(scanner->group_hash,0); + scanner->vertex_id2group=MESA_htable_create(&hargs, sizeof(hargs)); + MESA_htable_print_crtl(scanner->vertex_id2group,0); + hargs.thread_safe=0; scanner->region_hash=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(scanner->region_hash,0); @@ -1387,6 +1394,8 @@ void destroy_maat_scanner(struct Maat_scanner_t*scanner) MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_rule); MESA_htable_destroy(scanner->group_hash, (void (*)(void*))_destroy_group_rule); MESA_htable_destroy(scanner->region_hash, NULL); + MESA_htable_destroy(scanner->vertex_id2group, NULL); + map_destroy(scanner->district_map); scanner->district_map=NULL; assert(scanner->tmp_district_map==NULL); @@ -1933,7 +1942,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_rule->group_id); + group_rule=create_group_rule(db_rule->group_id, scanner); HASH_add_by_id(scanner->group_hash, db_rule->group_id, group_rule); } switch(db_rule->expr_type) @@ -2178,7 +2187,7 @@ int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,st group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_ip_rule->group_id); + group_rule=create_group_rule(db_ip_rule->group_id, scanner); HASH_add_by_id(scanner->group_hash, db_ip_rule->group_id, group_rule); } @@ -2210,7 +2219,7 @@ int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule* intval_ group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(intval_rule->group_id); + group_rule=create_group_rule(intval_rule->group_id, scanner); HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule); } expr_id=scanner->exprid_generator++; @@ -2241,7 +2250,7 @@ int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule* db_dige group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_digest_rule->group_id); + group_rule=create_group_rule(db_digest_rule->group_id, scanner); HASH_add_by_id(scanner->group_hash, db_digest_rule->group_id, group_rule); } expr_id=scanner->exprid_generator++; @@ -2334,7 +2343,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); if(group_rule==NULL) { - group_rule=create_group_rule(db_group_rule->group_id); + group_rule=create_group_rule(db_group_rule->group_id, scanner); group_rule->table_id=table->table_id; ret=HASH_add_by_id(scanner->group_hash, group_rule->group_id, group_rule); assert(ret>=0); @@ -2345,13 +2354,13 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->parent_id); if(parent_group==NULL) { - parent_group=create_group_rule(db_group_rule->parent_id); + parent_group=create_group_rule(db_group_rule->parent_id, scanner); parent_group->table_id=table->table_id; ret=HASH_add_by_id(scanner->group_hash, parent_group->group_id, parent_group); assert(ret>=0); } group_rule->ref_cnt++; - igraph_add_edge(&(scanner->group_graph), db_group_rule->group_id, db_group_rule->parent_id); + igraph_add_edge(&scanner->group_graph, group_rule->vertex_id, parent_group->vertex_id); } else { @@ -2379,13 +2388,36 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro void del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner_t *scanner, void* logger) { struct Maat_compile_inner*compile_rule=NULL; - struct Maat_group_inner* group_rule=NULL; + struct Maat_group_inner* group_rule=NULL, *parent_group=NULL; igraph_es_t es; int ret=0; + + if(db_group_rule->parent_type==PARENT_TYPE_GROUP) { + group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); + parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id); + if(group_rule==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module , + "update error, delete %s group %d from parent group %d error, target group not exisit.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + return; + } + if(parent_group==NULL) + { + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module , + "update error, delete %s group %d from parent group %d error, parent group not exisit.", + table->table_name[table->updating_name], + db_group_rule->group_id, + db_group_rule->parent_id); + return; + } group_rule->ref_cnt--; - igraph_es_pairs_small(&es, IGRAPH_DIRECTED, db_group_rule->group_id, db_group_rule->parent_id); + + igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id); ret=igraph_delete_edges(&scanner->group_graph, es); if(ret!=IGRAPH_SUCCESS) { @@ -2500,7 +2532,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru if(ret!=3&&ret!=4&&ret!=5) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "update error,invalid format of group table %s:%s", + "update error, invalid format of group table %s:%s", table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; return; @@ -2509,7 +2541,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru { db_group_rule.not_flag=0; } - if(!db_group_rule.parent_type==PARENT_TYPE_GROUP && db_group_rule.not_flag) + if(db_group_rule.parent_type==PARENT_TYPE_GROUP && db_group_rule.not_flag) { MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , "update error, invalid format of group table %s:%s not operation is forbidden for non-compile parent.", @@ -3320,21 +3352,39 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma table_rt->plugin.cache_line_num++; } } +void vector_print(igraph_vector_t *v) { + long int i; + for (i=0; igroup_graph), group_rule->group_id, IGRAPH_OUT, - /*NO search unreachable*/ 0, &vids, NULL, NULL, NULL, NULL, NULL, NULL); - long int i; + igraph_vector_init(&vids, 0); + + igraph_dfs(&(scanner->group_graph), group_rule->vertex_id, IGRAPH_OUT, + 0, &vids, NULL, NULL, NULL, NULL, NULL, NULL); + + + long int i=0; int* temp_group_ids=ALLOC(int, igraph_vector_size(&vids)); size_t path_endpoint_cnt=0; for(i=0; igroup_hash, (int) VECTOR(vids)[i]); - if(parent_group->has_compile_neighbors)//including itself? + tmp_vid=(int) VECTOR(vids)[i]; + if(tmp_vid<0) + { + break; + } + parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->vertex_id2group, tmp_vid); + if(parent_group->has_compile_neighbors)//including itself { temp_group_ids[path_endpoint_cnt]=parent_group->group_id; path_endpoint_cnt++; @@ -3367,6 +3417,8 @@ void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q GIE_create_para_t para; para.gram_value=7; para.position_accuracy=10; + find_group_paths(scanner); + tmp1=create_bool_matcher(scanner->compile_hash, scan_thread_num, logger); diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 4f3186a..8632f50 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -849,7 +849,7 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac { group_not_flag=item->valueint; } - if(parent_type!=PARENT_TYPE_GROUP) + if(parent_type==PARENT_TYPE_GROUP) { group_not_flag=0; } diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index ee50471..2527f95 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -142,6 +142,7 @@ struct Maat_group_inner int ref_cnt; char* group_name; int has_compile_neighbors; + int vertex_id; int endpoint_cnt; int* endpoints; dynamic_array_t *regions; @@ -287,8 +288,11 @@ struct Maat_scanner_t MESA_htable_handle compile_hash; MESA_htable_handle district_map; MESA_htable_handle tmp_district_map; + MESA_htable_handle vertex_id2group; igraph_t group_graph; + int grp_vertex_id_generator; + unsigned int district_num; unsigned int cfg_num; unsigned int exprid_generator; diff --git a/test/maat_json.json b/test/maat_json.json index d837d85..eb1da97 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -12,7 +12,7 @@ "is_valid": "yes", "groups": [ { - "group_name": "IP_group", + "group_name": "123_IP_group", "regions": [ { "table_name": "IP_CONFIG", @@ -77,7 +77,7 @@ "is_valid": "yes", "groups": [ { - "group_name": "IP_group" + "group_name": "123_IP_group" }, { "group_name": "Untitled", @@ -144,7 +144,7 @@ ] }, { - "group_name": "Untitled", + "group_name": "126_interval_group", "regions": [ { "table_name": "CONTENT_SIZE", @@ -595,7 +595,7 @@ "match_method": "sub", "format": "uncase plain" } - } + } ] }, { @@ -682,7 +682,7 @@ }, { "not_flag":1, - "group_name": "IP_group" + "group_name": "123_IP_group" } ] }, @@ -879,9 +879,25 @@ "format": "uncase plain", "match_method": "prefix" } + }, + { + "table_type": "expr", + "table_name": "MAIL_ADDR", + "table_content": { + "keywords": "ceshi6@mailhost.cn", + "expr_type": "none", + "format": "uncase plain", + "match_method": "prefix" + } } ], "group_name": "152_mail_addr" + }, + { + "group_name": "interval_group_refered", + "sub_groups": [ + {"group_name": "126_interval_group"} + ] } ] }, @@ -917,7 +933,7 @@ { "group_name": "IP_group_refered", "sub_groups": [ - {"group_name": "IP_group"} + {"group_name": "123_IP_group"} ] } ] diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index cda125f..b3c5002 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -308,31 +308,37 @@ TEST(StringScan, BugReport20190325) TEST(StringScan, PrefixAndSuffix) { int ret=0; - int table_id=0; + int mail_addr_table=0, cont_sz_table=0; struct Maat_rule_t result[4]; int found_pos[4]; - const char* table_name="MAIL_ADDR"; scan_status_t mid=NULL; const char* hit_twice="ceshi3@mailhost.cn"; const char* hit_suffix="11111111111ceshi3@mailhost.cn"; const char* hit_prefix="ceshi3@mailhost.cn11111111111"; - table_id=Maat_table_register(g_feather,table_name); - ASSERT_GT(table_id, 0); + - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_twice, strlen(hit_twice), + cont_sz_table=Maat_table_register(g_feather, "CONTENT_SIZE"); + ret=Maat_scan_intval(g_feather, cont_sz_table, 2015, + result, 4, &mid, 0); + mail_addr_table=Maat_table_register(g_feather, "MAIL_ADDR"); + ASSERT_GT(mail_addr_table, 0); + ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_twice, strlen(hit_twice), result,found_pos, 4, &mid, 0); EXPECT_EQ(ret, 2); EXPECT_EQ(result[0].config_id, 152); EXPECT_EQ(result[1].config_id, 151); Maat_clean_status(&mid); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_suffix, strlen(hit_suffix), + ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_suffix, strlen(hit_suffix), result,found_pos, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 151); Maat_clean_status(&mid); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_prefix, strlen(hit_prefix), + ret=Maat_scan_intval(g_feather, cont_sz_table, 2015, + result, 4, &mid, 0); + + ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_prefix, strlen(hit_prefix), result,found_pos, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 152); @@ -489,15 +495,15 @@ TEST(NOTLogic, OneRegion) ASSERT_GT(table_id, 0); Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit), - result,found_pos, 4, &mid, 0); + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, string_should_hit, strlen(string_should_hit), + result, found_pos, 4, &mid, 0); EXPECT_GE(ret, 1); Maat_clean_status(&mid); Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit), - result,found_pos, 4, &mid, 0); + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit), + result, found_pos, 4, &mid, 0); EXPECT_EQ(ret, -2); Maat_clean_status(&mid); @@ -1124,7 +1130,7 @@ TEST(Policy, SubGroup) { #define TestSubGroup int ret=0, table_id=0; - const char* scan_string="ceshi3@mailhost.cn"; + const char* scan_string="ceshi6@mailhost.cn"; struct Maat_rule_t result[4]; memset(result, 0, sizeof(result)); @@ -1141,14 +1147,14 @@ TEST(Policy, SubGroup) table_id=Maat_table_register(g_feather,"MAIL_ADDR"); ASSERT_GT(table_id, 0); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_string, strlen(scan_string), + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_string, strlen(scan_string), result,NULL, 4, &mid, 0); EXPECT_EQ(ret, -2); table_id=Maat_table_register(g_feather, "IP_CONFIG"); ASSERT_GT(table_id, 0); - ret=Maat_scan_proto_addr(g_feather,table_id,&ipv4_addr,6,result,4, &mid,0); + ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid,0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 153);