gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[BUGFIX]fix hit paths miss path if sub group(has super group) is referenced by compile

Browse Source
This commit is contained in:
liuwentan
2023-05-17 14:34:34 +08:00
parent 6830966084
commit 6626cbd57c
7 changed files with 227 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/inc_internal/maat_compile.h
View File

@@ -66,7 +66,7 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
struct maat_compile_state *compile_state,
struct maat_hit_path *hit_path_array,
size_t array_size, size_t hit_path_cnt);
size_t array_size, size_t n_internal_hit_path);
void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema,
@@ -100,6 +100,7 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
size_t n_hit_item, struct maat_state *state);
size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *compile_state,
struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt,
struct maat_hit_path *hit_path_array,
size_t array_size);

14
src/maat_api.c
View File

@@ -40,6 +40,7 @@
#include "maat_bool_plugin.h"
#include "maat_virtual.h"
#include "maat_stat.h"
#include "uthash/utarray.h"
#define MODULE_MAAT_API module_name_str("maat.api")
@@ -1804,16 +1805,17 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat
compile_table_id = maat_instance->default_compile_table_id;
}
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
size_t internal_hit_path_cnt = maat_compile_state_get_internal_hit_paths(state->compile_state,
(struct group2group_runtime *)g2g_runtime,
paths, n_path);
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
if (NULL == compile_rt) {
return internal_hit_path_cnt;
return -1;
}
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
size_t internal_hit_path_cnt = maat_compile_state_get_internal_hit_paths(state->compile_state,
(struct compile_runtime *)compile_rt,
(struct group2group_runtime *)g2g_runtime,
paths, n_path);
return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt,
state->compile_state, paths, n_path,
internal_hit_path_cnt);

126
src/maat_compile.c
View File

@@ -93,6 +93,12 @@ struct compile_rule {
char table_name[NAME_MAX];
};
struct group_reference {
long long group_id;
size_t ref_by_compile_cnt;
UT_hash_handle hh;
};
/* compile_runtime and group2compile_runtime share compile_hash_map */
struct compile_runtime {
struct bool_matcher *bm;
@@ -101,6 +107,8 @@ struct compile_runtime {
time_t version;
struct maat_clause *clause_by_literals_hash;
struct literal_clause *literal2clause_hash;
struct group_reference *group_ref_hash;
pthread_mutex_t mutex;
long long rule_num;
long long update_err_cnt;
struct bool_expr_match *expr_match_buff;
@@ -573,8 +581,11 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
compile_rt->version = time(NULL);
compile_rt->cfg_hash_tbl = rcu_hash_new(rcu_maat_compile_free, NULL);
compile_rt->clause_by_literals_hash = NULL;
compile_rt->literal2clause_hash = NULL;
compile_rt->group_ref_hash = NULL;
compile_rt->logger = logger;
compile_rt->ref_garbage_bin = garbage_bin;
pthread_mutex_init(&(compile_rt->mutex), NULL);
return compile_rt;
}
@@ -591,7 +602,7 @@ static void maat_clause_hash_free(struct maat_clause *clause_hash)
}
}
void literal2clause_hash_free(struct literal_clause *hash)
static void literal2clause_hash_free(struct literal_clause *hash)
{
struct literal_clause *l2c_val = NULL, *tmp_l2c_val = NULL;
@@ -607,6 +618,17 @@ void literal2clause_hash_free(struct literal_clause *hash)
assert(hash == NULL);
}
static void group_reference_hash_free(struct group_reference *group_ref_hash)
{
struct group_reference *group_ref = NULL, *tmp_group_ref = NULL;
HASH_ITER(hh, group_ref_hash, group_ref, tmp_group_ref) {
HASH_DEL(group_ref_hash, group_ref);
FREE(group_ref);
}
assert(group_ref_hash == NULL);
}
void garbage_literal2clause_hash_free(void *l2c_hash, void *arg)
{
literal2clause_hash_free((struct literal_clause *)l2c_hash);
@@ -640,6 +662,14 @@ void compile_runtime_free(void *compile_runtime)
compile_rt->clause_by_literals_hash = NULL;
}
pthread_mutex_lock(&(compile_rt->mutex));
if (compile_rt->group_ref_hash != NULL) {
group_reference_hash_free(compile_rt->group_ref_hash);
compile_rt->group_ref_hash = NULL;
}
pthread_mutex_unlock(&(compile_rt->mutex));
pthread_mutex_destroy(&(compile_rt->mutex));
if (compile_rt->expr_match_buff != NULL) {
FREE(compile_rt->expr_match_buff);
}
@@ -1457,7 +1487,7 @@ static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id, lon
return 1;
}
static int maat_compile_has_literal(struct maat_compile* compile,
static int maat_compile_has_literal(struct maat_compile *compile,
struct maat_literal_id *literal_id)
{
int i = 0;
@@ -1503,7 +1533,6 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
size_t new_hit_path_cnt = 0;
struct maat_compile *compile = NULL;
struct maat_literal_id literal_id = {0, 0};
struct maat_hit_path tmp_path;
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM;
assert(compile_state->thread_id >= 0);
@@ -1540,7 +1569,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
hit_path_array[j].compile_id = compile->compile_id;
} else {
// means same literal_id hit more than one compile_id
tmp_path = hit_path_array[j];
struct maat_hit_path tmp_path = hit_path_array[j];
tmp_path.compile_id = compile->compile_id;
if(maat_compile_is_hit_path_existed(hit_path_array, n_internal_hit_path + new_hit_path_cnt, &tmp_path)) {
hit_path_array[n_internal_hit_path + new_hit_path_cnt] = tmp_path;
@@ -1846,9 +1875,24 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
return -1;
}
struct group_reference *group_ref = NULL;
if (0 == is_valid) {
//delete
ret = maat_remove_group_from_compile(compile_rt->cfg_hash_tbl, g2c_item, compile_rt->logger);
pthread_mutex_lock(&(compile_rt->mutex));
HASH_FIND(hh, compile_rt->group_ref_hash, (char *)&(g2c_item->group_id),
sizeof(long long), group_ref);
if (group_ref != NULL) {
if (0 == group_ref->ref_by_compile_cnt) {
HASH_DEL(compile_rt->group_ref_hash, group_ref);
FREE(group_ref);
} else {
group_ref->ref_by_compile_cnt--;
}
}
pthread_mutex_unlock(&(compile_rt->mutex));
ret = maat_remove_group_from_compile(compile_rt->cfg_hash_tbl, g2c_item,
compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
g2c_rt->not_flag_group--;
@@ -1859,6 +1903,18 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
}
} else {
//add
pthread_mutex_lock(&(compile_rt->mutex));
HASH_FIND(hh, compile_rt->group_ref_hash, (char *)&(g2c_item->group_id),
sizeof(long long), group_ref);
if (NULL == group_ref) {
group_ref = ALLOC(struct group_reference, 1);
group_ref->group_id = g2c_item->group_id;
HASH_ADD_KEYPTR(hh, compile_rt->group_ref_hash, (char *)&(group_ref->group_id),
sizeof(long long), group_ref);
}
group_ref->ref_by_compile_cnt++;
pthread_mutex_unlock(&(compile_rt->mutex));
ret = maat_add_group_to_compile(compile_rt->cfg_hash_tbl, g2c_item, compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
@@ -1874,6 +1930,20 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
return ret;
}
int group_referenced_by_compile(struct compile_runtime *compile_rt, long long group_id)
{
struct group_reference *group_ref = NULL;
pthread_mutex_lock(&(compile_rt->mutex));
HASH_FIND(hh, compile_rt->group_ref_hash, (char *)&group_id, sizeof(long long), group_ref);
if (group_ref != NULL) {
pthread_mutex_unlock(&(compile_rt->mutex));
return 1;
}
pthread_mutex_unlock(&(compile_rt->mutex));
return 0;
}
long long group2compile_runtime_not_group_count(void *g2c_runtime)
{
if (NULL == g2c_runtime) {
@@ -2090,7 +2160,9 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
}
}
UT_icd ut_compile_group_id_icd = {sizeof(long long), NULL, NULL, NULL};
size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *compile_state,
struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt,
struct maat_hit_path *hit_path_array,
size_t array_size)
@@ -2104,33 +2176,40 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp
NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths
*/
long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
memset(super_group_ids, -1, sizeof(super_group_ids));
UT_array *valid_super_group_ids;
utarray_new(valid_super_group_ids, &ut_compile_group_id_icd);
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, &(internal_path->group_id), 1,
super_group_ids, MAX_SCANNER_HIT_GROUP_NUM);
if (0 == super_group_cnt) {
/*
item->group_id has no top group, this group can only be referenced by compile
------------------------------------------------------------------------------
for example:
compile1 -> group1 -> group2 -> item1
group3 -> item2
group1 and group3 has no top group
group1 is referenced by compile1, group3 is not referenced by any compile
NOTE: Add the hit path as long as the item is hit
*/
super_group_cnt = 1; // add one hit path which top_group_ids[0] = -1
/* if super group is not referenced by compile, drop it */
for (size_t idx = 0; idx < super_group_cnt; idx++) {
if (0 == group_referenced_by_compile(compile_rt, super_group_ids[idx])) {
continue;
}
utarray_push_back(valid_super_group_ids, &super_group_ids[idx]);
}
/*
if internal_path->group_id can be referenced directly by compile,
so add it to hit_path which super_group_ids is -1
------------------------------------------------------------------------------
NOTE: Add the hit path as long as the item is hit
*/
if (0 == utarray_len(valid_super_group_ids)) {
long long super_group_id = -1;
utarray_push_back(valid_super_group_ids, &super_group_id);
}
long long *p = NULL;
struct maat_hit_path tmp_path;
for (int j = 0; j < super_group_cnt && hit_path_cnt < array_size; j++) {
for (p = utarray_front(valid_super_group_ids); p != NULL && hit_path_cnt < array_size;
p = utarray_next(valid_super_group_ids, p)) {
memset(&tmp_path, 0, sizeof(tmp_path));
tmp_path.Nth_scan = internal_path->Nth_scan;
tmp_path.item_id = internal_path->item_id;
tmp_path.sub_group_id = internal_path->group_id;
tmp_path.top_group_id = super_group_ids[j];
tmp_path.top_group_id = *p;
tmp_path.vtable_id = internal_path->vtable_id;
tmp_path.compile_id = -1;
@@ -2144,6 +2223,7 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp
hit_path_array[hit_path_cnt] = tmp_path;
hit_path_cnt++;
}
utarray_free(valid_super_group_ids);
}
return hit_path_cnt;

112
test/group_exclude/group_exclude_L3.conf
View File

@@ -1,90 +1,100 @@
{
"add_items": [
{
"group_id": 1,
"super_group_id": 6,
"group_id": 8,
"super_group_id": 5,
"is_exclude": 0
},
{
"group_id": 2,
"super_group_id": 9,
"is_exclude": 0
},
{
"group_id": 4,
"super_group_id": 7,
"is_exclude": 0
},
{
"group_id": 4,
"super_group_id": 8,
"is_exclude": 0
},
{
"group_id": 5,
"super_group_id": 8,
"group_id": 9,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 6,
"super_group_id": 9,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 9,
"is_exclude": 0
},
{
"group_id": 8,
"super_group_id": 10,
"is_exclude": 0
},
{
"group_id": 8,
"super_group_id": 13,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 11,
"super_group_id": 12,
"super_group_id": 6,
"is_exclude": 0
},
{
"group_id": 2,
"super_group_id": 6,
"group_id": 11,
"super_group_id": 7,
"is_exclude": 0
},
{
"group_id": 12,
"super_group_id": 7,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 2,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 4,
"is_exclude": 0
},
{
"group_id": 13,
"super_group_id": 3,
"is_exclude": 0
},
{
"group_id": 9,
"super_group_id": 5,
"is_exclude": 1
},
{
"group_id": 3,
"group_id": 5,
"super_group_id": 1,
"is_exclude": 1
},
{
"group_id": 10,
"super_group_id":6,
"is_exclude": 1
},
{
"group_id": 10,
"super_group_id": 7,
"is_exclude": 1
},
{
"group_id": 3,
"super_group_id": 8,
"is_exclude": 1
},
{
"group_id": 7,
"super_group_id": 10,
"super_group_id": 6,
"is_exclude": 1
},
{
"group_id": 7,
"super_group_id": 12,
"group_id": 6,
"super_group_id": 2,
"is_exclude": 1
},
{
"group_id": 11,
"super_group_id": 13,
"group_id": 6,
"super_group_id": 3,
"is_exclude": 1
},
{
"group_id": 13,
"super_group_id": 4,
"is_exclude": 1
}
],
"del_items": [
{
"group_id": 11,
"super_group_id": 13,
"group_id": 7,
"super_group_id": 6,
"is_exclude": 1
},
{
"group_id": 13,
"super_group_id": 4,
"is_exclude": 1
}
]

BIN
test/group_exclude/group_exclude_L3.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 202 KiB

After

Width:  |  Height:  |  Size: 200 KiB

38
test/group_exclude/group_exclude_gtest.cpp
View File

@@ -169,7 +169,6 @@ TEST_F(MaatGroupExclude, level_3_basic) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
//struct log_handle *logger = MaatGroupExclude::logger;
int ret = parse_config_file("group_exclude_L3.conf", &rules);
EXPECT_EQ(ret, 0);
@@ -184,13 +183,42 @@ TEST_F(MaatGroupExclude, level_3_basic) {
group2group_runtime_commit(g2g_runtime, table_name, 1);
long long hit_group_ids[2] = {11, 4};
long long hit_group_ids[2] = {11, 13};
long long super_group_ids[MAX_ITEM_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
for (size_t i = 0; i < super_group_cnt; i++) {
printf("super_group_ids[%zu]:%lld\n", i, super_group_ids[i]);
}
EXPECT_EQ(super_group_cnt, 2);
EXPECT_EQ(super_group_ids[0], 2);
EXPECT_EQ(super_group_ids[1], 7);
//delete group_id = 7, super_group_id = 6, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[0].group_id,
rules.del_items[0].super_group_id, rules.del_items[0].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 2);
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 3);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 6);
EXPECT_EQ(super_group_ids[2], 7);
//delete group_id = 13, super_group_id = 4, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[1].group_id,
rules.del_items[1].super_group_id, rules.del_items[1].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 3);
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 4);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 6);
EXPECT_EQ(super_group_ids[3], 7);
}
int main(int argc, char ** argv)

12
test/maat_framework_gtest.cpp
View File

@@ -5415,6 +5415,18 @@ TEST_F(MaatCmdTest, HitPath) {
"time=2020-02-11", "Cookie", 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/
EXPECT_EQ(ret, 1);
/*
item1 -> group1 -> group11
\
\ -> compile1
/
item2 -> group2 -> group21 _/
*/
long long group11_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
ret = group2group_table_set_line(maat_instance, g2g_table_name, MAAT_OP_ADD, group1_id,
group11_id, 0, 0);
EXPECT_EQ(ret, 1);
//item3 -> group3, group3 is not referenced by any compile.
long long item3_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1);
long long group3_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);