gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

子分组功能单元测试通过。

Browse Source
This commit is contained in:
zhengchao
2019-05-12 14:20:57 +08:00
parent 02b97ca014
commit 63e6d24c19
6 changed files with 133 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/entry/Maat_api.cpp
View File

@@ -159,8 +159,9 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
unsigned char has_not_flag=0;
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
struct Maat_group_inner* group_rule=NULL;
struct Maat_compile_inner* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct Maat_compile_inner* _mi_rule=NULL;
struct Maat_compile_inner* compile_arrary[MAX_SCANNER_HIT_NUM];
struct Maat_compile_inner* compile_rule=NULL;
int region_pos[MAX_SCANNER_HIT_NUM];
_mid->cur_hit_cnt=0;
for(i=0;i<region_hit_num;i++)
@@ -172,13 +173,13 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
}
if(group_rule->compile_shortcut!=NULL&&group_rule->ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
{
array_mi_rule[shortcut_avilable_cnt]=(struct Maat_compile_inner*)(group_rule->compile_shortcut);
compile_arrary[shortcut_avilable_cnt]=(struct Maat_compile_inner*)(group_rule->compile_shortcut);
shortcut_avilable_cnt++;
}
_mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id;
_mid->cur_hit_cnt++;
for(j=0; j<group_rule->endpoint_cnt; j++)
{
_mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->endpoints[j];
_mid->cur_hit_cnt++;
ret=insert_set_id(&(_mid->hitted_group_id),
&(_mid->hit_group_size),
_mid->hit_group_cnt,
@@ -205,40 +206,40 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
{
scan_ret=bool_matcher_match(bm, thread_num,
_mid->hitted_group_id, _mid->hit_group_cnt,
(void **)array_mi_rule, MAX_SCANNER_HIT_NUM);
(void **)compile_arrary, MAX_SCANNER_HIT_NUM);
}
for(i=0;i<scan_ret&&result_cnt<size;i++)
{
_mi_rule=array_mi_rule[i];
if(_mi_rule==NULL)
compile_rule=compile_arrary[i];
if(compile_rule==NULL)
{
continue;
}
if(0==pthread_rwlock_tryrdlock(&(_mi_rule->rwlock)))
if(0==pthread_rwlock_tryrdlock(&(compile_rule->rwlock)))
{
if(_mi_rule->is_valid==1)
if(compile_rule->is_valid==1)
{
if(_mi_rule->not_group_cnt>0 && !is_last_region)
if(compile_rule->not_group_cnt>0 && !is_last_region)
{
_mid->not_grp_compile_hitted_flag=1;
}
else
{
make_group_set(_mi_rule, &(rs_result[result_cnt].group_set), &has_not_flag);
make_group_set(compile_rule, &(rs_result[result_cnt].group_set), &has_not_flag);
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0 || //compile config hitted becasue of new reigon
_mid->cur_hit_cnt==0) //or ever hit a compile that refer a NOT-logic group
{
fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head),
_mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len);
rs_result[result_cnt].compile_id=_mi_rule->compile_id;
fill_maat_rule(&(result[result_cnt]), &(compile_rule->db_c_rule->m_rule_head),
compile_rule->db_c_rule->service_defined ,compile_rule->db_c_rule->m_rule_head.serv_def_len);
rs_result[result_cnt].compile_id=compile_rule->compile_id;
result_cnt++;
}
}
}
pthread_rwlock_unlock(&(_mi_rule->rwlock));
pthread_rwlock_unlock(&(compile_rule->rwlock));
}
}
if(result_cnt>0)

86
src/entry/Maat_rule.cpp
View File

@@ -836,7 +836,7 @@ error_jump:
map_destroy(string2int_map);
return table_cnt;
}
struct Maat_group_inner* create_group_rule(int group_id)
struct Maat_group_inner* create_group_rule(int group_id, struct Maat_scanner_t *scanner)
{
struct Maat_group_inner* group=ALLOC(struct Maat_group_inner, 1);
group->group_id=group_id;
@@ -847,6 +847,10 @@ struct Maat_group_inner* create_group_rule(int group_id)
group->compile_shortcut=NULL;
group->table_id=0;
group->group_name=NULL;
group->vertex_id=scanner->grp_vertex_id_generator++;
assert(igraph_vcount(&scanner->group_graph)==group->vertex_id);
igraph_add_vertices(&scanner->group_graph, 1, NULL);
HASH_add_by_id(scanner->vertex_id2group, group->vertex_id, group);
pthread_mutex_init(&(group->mutex), NULL);
return group;
}
@@ -1326,6 +1330,9 @@ struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
scanner->group_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(scanner->group_hash,0);
scanner->vertex_id2group=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(scanner->vertex_id2group,0);
hargs.thread_safe=0;
scanner->region_hash=MESA_htable_create(&hargs, sizeof(hargs));
MESA_htable_print_crtl(scanner->region_hash,0);
@@ -1387,6 +1394,8 @@ void destroy_maat_scanner(struct Maat_scanner_t*scanner)
MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_rule);
MESA_htable_destroy(scanner->group_hash, (void (*)(void*))_destroy_group_rule);
MESA_htable_destroy(scanner->region_hash, NULL);
MESA_htable_destroy(scanner->vertex_id2group, NULL);
map_destroy(scanner->district_map);
scanner->district_map=NULL;
assert(scanner->tmp_district_map==NULL);
@@ -1933,7 +1942,7 @@ int add_expr_rule(struct Maat_table_desc* table,struct db_str_rule_t* db_rule,st
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(db_rule->group_id);
group_rule=create_group_rule(db_rule->group_id, scanner);
HASH_add_by_id(scanner->group_hash, db_rule->group_id, group_rule);
}
switch(db_rule->expr_type)
@@ -2178,7 +2187,7 @@ int add_ip_rule(struct Maat_table_desc* table,struct db_ip_rule_t* db_ip_rule,st
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_ip_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(db_ip_rule->group_id);
group_rule=create_group_rule(db_ip_rule->group_id, scanner);
HASH_add_by_id(scanner->group_hash, db_ip_rule->group_id, group_rule);
}
@@ -2210,7 +2219,7 @@ int add_intval_rule(struct Maat_table_desc* table,struct db_intval_rule* intval_
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, intval_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(intval_rule->group_id);
group_rule=create_group_rule(intval_rule->group_id, scanner);
HASH_add_by_id(scanner->group_hash, intval_rule->group_id, group_rule);
}
expr_id=scanner->exprid_generator++;
@@ -2241,7 +2250,7 @@ int add_digest_rule(struct Maat_table_desc* table,struct db_digest_rule* db_dige
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_digest_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(db_digest_rule->group_id);
group_rule=create_group_rule(db_digest_rule->group_id, scanner);
HASH_add_by_id(scanner->group_hash, db_digest_rule->group_id, group_rule);
}
expr_id=scanner->exprid_generator++;
@@ -2334,7 +2343,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
if(group_rule==NULL)
{
group_rule=create_group_rule(db_group_rule->group_id);
group_rule=create_group_rule(db_group_rule->group_id, scanner);
group_rule->table_id=table->table_id;
ret=HASH_add_by_id(scanner->group_hash, group_rule->group_id, group_rule);
assert(ret>=0);
@@ -2345,13 +2354,13 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->parent_id);
if(parent_group==NULL)
{
parent_group=create_group_rule(db_group_rule->parent_id);
parent_group=create_group_rule(db_group_rule->parent_id, scanner);
parent_group->table_id=table->table_id;
ret=HASH_add_by_id(scanner->group_hash, parent_group->group_id, parent_group);
assert(ret>=0);
}
group_rule->ref_cnt++;
igraph_add_edge(&(scanner->group_graph), db_group_rule->group_id, db_group_rule->parent_id);
igraph_add_edge(&scanner->group_graph, group_rule->vertex_id, parent_group->vertex_id);
}
else
{
@@ -2379,13 +2388,36 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
void del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner_t *scanner, void* logger)
{
struct Maat_compile_inner*compile_rule=NULL;
struct Maat_group_inner* group_rule=NULL;
struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
igraph_es_t es;
int ret=0;
if(db_group_rule->parent_type==PARENT_TYPE_GROUP)
{
group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
if(group_rule==NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module ,
"update error, delete %s group %d from parent group %d error, target group not exisit.",
table->table_name[table->updating_name],
db_group_rule->group_id,
db_group_rule->parent_id);
return;
}
if(parent_group==NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module ,
"update error, delete %s group %d from parent group %d error, parent group not exisit.",
table->table_name[table->updating_name],
db_group_rule->group_id,
db_group_rule->parent_id);
return;
}
group_rule->ref_cnt--;
igraph_es_pairs_small(&es, IGRAPH_DIRECTED, db_group_rule->group_id, db_group_rule->parent_id);
igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id);
ret=igraph_delete_edges(&scanner->group_graph, es);
if(ret!=IGRAPH_SUCCESS)
{
@@ -2500,7 +2532,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru
if(ret!=3&&ret!=4&&ret!=5)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"update error,invalid format of group table %s:%s",
"update error, invalid format of group table %s:%s",
table->table_name[table->updating_name], table_line);
table->udpate_err_cnt++;
return;
@@ -2509,7 +2541,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru
{
db_group_rule.not_flag=0;
}
if(!db_group_rule.parent_type==PARENT_TYPE_GROUP && db_group_rule.not_flag)
if(db_group_rule.parent_type==PARENT_TYPE_GROUP && db_group_rule.not_flag)
{
MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
"update error, invalid format of group table %s:%s not operation is forbidden for non-compile parent.",
@@ -3320,21 +3352,39 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma
table_rt->plugin.cache_line_num++;
}
}
void vector_print(igraph_vector_t *v) {
long int i;
for (i=0; i<igraph_vector_size(v); i++) {
printf(" %li", (long int) VECTOR(*v)[i]);
}
printf("\n");
}
void walk_group_hash(const uchar * key, uint size, void * data, void * user)
{
struct Maat_group_inner* group_rule=(struct Maat_group_inner*)data;
struct Maat_group_inner* parent_group=NULL;
struct Maat_scanner_t* scanner=(struct Maat_scanner_t*)user;
int tmp_vid=0;
igraph_vector_t vids;
igraph_dfs(&(scanner->group_graph), group_rule->group_id, IGRAPH_OUT,
/*NO search unreachable*/ 0, &vids, NULL, NULL, NULL, NULL, NULL, NULL);
long int i;
igraph_vector_init(&vids, 0);
igraph_dfs(&(scanner->group_graph), group_rule->vertex_id, IGRAPH_OUT,
0, &vids, NULL, NULL, NULL, NULL, NULL, NULL);
long int i=0;
int* temp_group_ids=ALLOC(int, igraph_vector_size(&vids));
size_t path_endpoint_cnt=0;
for(i=0; i<igraph_vector_size(&vids); i++)
{
parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, (int) VECTOR(vids)[i]);
if(parent_group->has_compile_neighbors)//including itself?
tmp_vid=(int) VECTOR(vids)[i];
if(tmp_vid<0)
{
break;
}
parent_group=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->vertex_id2group, tmp_vid);
if(parent_group->has_compile_neighbors)//including itself
{
temp_group_ids[path_endpoint_cnt]=parent_group->group_id;
path_endpoint_cnt++;
@@ -3367,6 +3417,8 @@ void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q
GIE_create_para_t para;
para.gram_value=7;
para.position_accuracy=10;
find_group_paths(scanner);
tmp1=create_bool_matcher(scanner->compile_hash,
scan_thread_num,
logger);

2
src/entry/json2iris.cpp
View File

@@ -849,7 +849,7 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
{
group_not_flag=item->valueint;
}
if(parent_type!=PARENT_TYPE_GROUP)
if(parent_type==PARENT_TYPE_GROUP)
{
group_not_flag=0;
}

4
src/inc_internal/Maat_rule_internal.h
View File

@@ -142,6 +142,7 @@ struct Maat_group_inner
int ref_cnt;
char* group_name;
int has_compile_neighbors;
int vertex_id;
int endpoint_cnt;
int* endpoints;
dynamic_array_t *regions;
@@ -287,8 +288,11 @@ struct Maat_scanner_t
MESA_htable_handle compile_hash;
MESA_htable_handle district_map;
MESA_htable_handle tmp_district_map;
MESA_htable_handle vertex_id2group;
igraph_t group_graph;
int grp_vertex_id_generator;
unsigned int district_num;
unsigned int cfg_num;
unsigned int exprid_generator;

26
test/maat_json.json
View File

@@ -12,7 +12,7 @@
"is_valid": "yes",
"groups": [
{
"group_name": "IP_group",
"group_name": "123_IP_group",
"regions": [
{
"table_name": "IP_CONFIG",
@@ -77,7 +77,7 @@
"is_valid": "yes",
"groups": [
{
"group_name": "IP_group"
"group_name": "123_IP_group"
},
{
"group_name": "Untitled",
@@ -144,7 +144,7 @@
]
},
{
"group_name": "Untitled",
"group_name": "126_interval_group",
"regions": [
{
"table_name": "CONTENT_SIZE",
@@ -682,7 +682,7 @@
},
{
"not_flag":1,
"group_name": "IP_group"
"group_name": "123_IP_group"
}
]
},
@@ -879,9 +879,25 @@
"format": "uncase plain",
"match_method": "prefix"
}
},
{
"table_type": "expr",
"table_name": "MAIL_ADDR",
"table_content": {
"keywords": "ceshi6@mailhost.cn",
"expr_type": "none",
"format": "uncase plain",
"match_method": "prefix"
}
}
],
"group_name": "152_mail_addr"
},
{
"group_name": "interval_group_refered",
"sub_groups": [
{"group_name": "126_interval_group"}
]
}
]
},
@@ -917,7 +933,7 @@
{
"group_name": "IP_group_refered",
"sub_groups": [
{"group_name": "IP_group"}
{"group_name": "123_IP_group"}
]
}
]

34
test/test_maatframe.cpp
View File

@@ -308,31 +308,37 @@ TEST(StringScan, BugReport20190325)
TEST(StringScan, PrefixAndSuffix)
{
int ret=0;
int table_id=0;
int mail_addr_table=0, cont_sz_table=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* table_name="MAIL_ADDR";
scan_status_t mid=NULL;
const char* hit_twice="ceshi3@mailhost.cn";
const char* hit_suffix="11111111111ceshi3@mailhost.cn";
const char* hit_prefix="ceshi3@mailhost.cn11111111111";
table_id=Maat_table_register(g_feather,table_name);
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_twice, strlen(hit_twice),
cont_sz_table=Maat_table_register(g_feather, "CONTENT_SIZE");
ret=Maat_scan_intval(g_feather, cont_sz_table, 2015,
result, 4, &mid, 0);
mail_addr_table=Maat_table_register(g_feather, "MAIL_ADDR");
ASSERT_GT(mail_addr_table, 0);
ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_twice, strlen(hit_twice),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, 2);
EXPECT_EQ(result[0].config_id, 152);
EXPECT_EQ(result[1].config_id, 151);
Maat_clean_status(&mid);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_suffix, strlen(hit_suffix),
ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_suffix, strlen(hit_suffix),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 151);
Maat_clean_status(&mid);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, hit_prefix, strlen(hit_prefix),
ret=Maat_scan_intval(g_feather, cont_sz_table, 2015,
result, 4, &mid, 0);
ret=Maat_full_scan_string(g_feather, mail_addr_table,CHARSET_GBK, hit_prefix, strlen(hit_prefix),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 152);
@@ -489,15 +495,15 @@ TEST(NOTLogic, OneRegion)
ASSERT_GT(table_id, 0);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result,found_pos, 4, &mid, 0);
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result, found_pos, 4, &mid, 0);
EXPECT_GE(ret, 1);
Maat_clean_status(&mid);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit),
result,found_pos, 4, &mid, 0);
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit),
result, found_pos, 4, &mid, 0);
EXPECT_EQ(ret, -2);
Maat_clean_status(&mid);
@@ -1124,7 +1130,7 @@ TEST(Policy, SubGroup)
{
#define TestSubGroup
int ret=0, table_id=0;
const char* scan_string="ceshi3@mailhost.cn";
const char* scan_string="ceshi6@mailhost.cn";
struct Maat_rule_t result[4];
memset(result, 0, sizeof(result));
@@ -1141,14 +1147,14 @@ TEST(Policy, SubGroup)
table_id=Maat_table_register(g_feather,"MAIL_ADDR");
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_string, strlen(scan_string),
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_string, strlen(scan_string),
result,NULL, 4, &mid, 0);
EXPECT_EQ(ret, -2);
table_id=Maat_table_register(g_feather, "IP_CONFIG");
ASSERT_GT(table_id, 0);
ret=Maat_scan_proto_addr(g_feather,table_id,&ipv4_addr,6,result,4, &mid,0);
ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid,0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 153);