gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix compile conjunction bug

Browse Source
This commit is contained in:
liuwentan
2023-03-06 16:45:34 +08:00
parent 9ff724af02
commit 33c9c10467
4 changed files with 131 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/inc_internal/maat_compile.h
View File

@@ -56,11 +56,10 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name);
int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile_ids, int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile_ids,
int ids_index, size_t compile_ids_size, struct maat_state *state); int ids_index, size_t compile_ids_size, struct maat_state *state);
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, size_t compile_runtime_get_new_hit_paths(struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt, struct maat_compile_state *compile_state,
struct maat_compile_state *compile_state, struct maat_hit_path *hit_path_array,
struct maat_hit_path *hit_paths, size_t array_size, size_t hit_path_cnt);
size_t hit_path_index, size_t n_hit_path);
void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt, void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema, struct compile_schema *compile_schema,
@@ -86,7 +85,12 @@ void maat_compile_state_free(struct maat_compile_state *compile_state);
int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
long long *hit_item_ids, size_t hit_item_cnt, long long *hit_item_ids, size_t hit_item_cnt,
size_t *n_hit_group_id, struct maat_state *state); size_t *n_hit_group_id, struct maat_state *state);
size_t maat_compile_state_get_hit_paths(struct maat_compile_state *compile_state,
struct group2group_runtime *g2g_rt,
struct maat_hit_path *hit_path_array,
size_t array_size);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);
#ifdef __cplusplus #ifdef __cplusplus

22
src/maat_api.c
View File

@@ -1789,20 +1789,22 @@ size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state,
compile_table_cnt = state->n_compile_table; compile_table_cnt = state->n_compile_table;
} }
size_t sum_hit_path = 0; void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
assert(g2g_runtime != NULL);
size_t compile_state_hit_path_cnt = maat_compile_state_get_hit_paths(state->compile_state,
(struct group2group_runtime *)g2g_runtime,
paths, n_path);
size_t new_hit_path_cnt = 0;
for (size_t i = 0; i < compile_table_cnt; i++) { for (size_t i = 0; i < compile_table_cnt; i++) {
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]); void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]);
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); assert(NULL != compile_rt);
assert(NULL != compile_rt && NULL != g2g_runtime);
size_t n_hit_path = compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, new_hit_path_cnt += compile_runtime_get_new_hit_paths((struct compile_runtime *)compile_rt,
(struct group2group_runtime *)g2g_runtime, state->compile_state, paths, n_path,
state->compile_state, paths, sum_hit_path, compile_state_hit_path_cnt + new_hit_path_cnt);
n_path);
sum_hit_path += n_hit_path;
} }
return sum_hit_path; return (compile_state_hit_path_cnt + new_hit_path_cnt);
} }
size_t maat_get_hit_objects(struct maat_compile_state *compile_state, size_t maat_get_hit_objects(struct maat_compile_state *compile_state,

139
src/maat_compile.c
View File

@@ -137,7 +137,6 @@ struct maat_internal_hit_path {
struct maat_compile_state { struct maat_compile_state {
int thread_id; int thread_id;
int Nth_scan; int Nth_scan;
int Nth_get_hit_path;
time_t hier_ver; time_t hier_ver;
size_t this_scan_hit_item_cnt; size_t this_scan_hit_item_cnt;
int not_clause_hitted_flag; int not_clause_hitted_flag;
@@ -1267,61 +1266,18 @@ static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_path
return 0; return 0;
} }
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, size_t compile_runtime_get_new_hit_paths(struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt,
struct maat_compile_state *compile_state, struct maat_compile_state *compile_state,
struct maat_hit_path *hit_paths, struct maat_hit_path *hit_path_array,
size_t hit_path_index, size_t hit_path_size) size_t array_size, size_t hit_path_cnt)
{ {
size_t i = 0, j = 0; /* assign hit_path_array[].compile_id */
struct maat_internal_hit_path *internal_path = NULL;
size_t hit_path_cnt = hit_path_index;
size_t new_hit_path_cnt = 0; size_t new_hit_path_cnt = 0;
size_t tmp_path_cnt = utarray_len(compile_state->internal_hit_paths);
if (0 == compile_state->Nth_get_hit_path) {
for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) {
internal_path = (struct maat_internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i);
/*
NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths
*/
long long top_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
memset(top_group_ids, 0, sizeof(top_group_ids));
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &(internal_path->group_id),
1, top_group_ids);
if (top_group_cnt <= 0) {
/*
item->group_id has no top group, this group can only be referenced by compile
------------------------------------------------------------------------------
for example:
compile1 -> group1 -> group2 -> item1
group3 -> item2
group1 and group3 has no top group
group1 is referenced by compile1, group3 is not referenced by any compile
NOTE: Add the hit path as long as the item is hit
*/
top_group_cnt = 1; // add one hit path which top_group_ids[0] = -1
}
for (int j = 0; j < top_group_cnt && hit_path_cnt < hit_path_size; j++, hit_path_cnt++) {
hit_paths[hit_path_cnt].Nth_scan = internal_path->Nth_scan;
hit_paths[hit_path_cnt].item_id = internal_path->item_id;
hit_paths[hit_path_cnt].sub_group_id = internal_path->group_id;
hit_paths[hit_path_cnt].top_group_id = top_group_ids[j]; // top_group_id may be -1
hit_paths[hit_path_cnt].vtable_id = internal_path->vtable_id;
hit_paths[hit_path_cnt].compile_id = -1;
}
}
}
/* assign hit_paths[].compile_id */
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
struct maat_literal_id literal_id = {0, 0}; struct maat_literal_id literal_id = {0, 0};
struct maat_hit_path tmp_path; struct maat_hit_path tmp_path;
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM; struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM;
int bool_match_ret = bool_matcher_match(compile_rt->bm, int bool_match_ret = bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0), (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses), expr_match, utarray_len(compile_state->all_hit_clauses), expr_match,
@@ -1334,37 +1290,30 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
continue; continue;
} }
for (j = 0; j < hit_path_cnt && (hit_path_cnt + new_hit_path_cnt) < hit_path_size; j++) { for (size_t j = 0; j < hit_path_cnt && (hit_path_cnt + new_hit_path_cnt) < array_size; j++) {
if (hit_paths[j].top_group_id < 0) { if (hit_path_array[j].top_group_id < 0) {
continue; continue;
} }
literal_id.group_id = hit_paths[j].top_group_id; literal_id.group_id = hit_path_array[j].top_group_id;
literal_id.vtable_id = hit_paths[j].vtable_id; literal_id.vtable_id = hit_path_array[j].vtable_id;
if (maat_compile_has_literal(compile, &literal_id)) { if (maat_compile_has_literal(compile, &literal_id)) {
if (hit_paths[j].compile_id < 0) { if (hit_path_array[j].compile_id < 0) {
hit_paths[j].compile_id = compile->compile_id; hit_path_array[j].compile_id = compile->compile_id;
} else { } else {
// means same literal_id hit more than one compile_id // means same literal_id hit more than one compile_id
tmp_path = hit_paths[j]; tmp_path = hit_path_array[j];
tmp_path.compile_id = compile->compile_id; tmp_path.compile_id = compile->compile_id;
if(maat_compile_is_hit_path_existed(hit_paths, hit_path_cnt + new_hit_path_cnt, &tmp_path)) { if(maat_compile_is_hit_path_existed(hit_path_array, hit_path_cnt + new_hit_path_cnt, &tmp_path)) {
hit_paths[hit_path_cnt + new_hit_path_cnt] = tmp_path; hit_path_array[hit_path_cnt + new_hit_path_cnt] = tmp_path;
new_hit_path_cnt++; new_hit_path_cnt++;
} }
} }
} }
} }
} }
if (0 == compile_state->Nth_get_hit_path) {
hit_path_cnt += new_hit_path_cnt;
} else {
hit_path_cnt = new_hit_path_cnt;
}
compile_state->Nth_get_hit_path++;
return hit_path_cnt; return new_hit_path_cnt;
} }
void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state,
@@ -1847,3 +1796,61 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
return 0; return 0;
} }
size_t maat_compile_state_get_hit_paths(struct maat_compile_state *compile_state,
struct group2group_runtime *g2g_rt,
struct maat_hit_path *hit_path_array,
size_t array_size)
{
size_t hit_path_cnt = 0;
struct maat_internal_hit_path *internal_path = NULL;
for (int i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) {
internal_path = (struct maat_internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i);
/*
NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths
*/
long long top_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
memset(top_group_ids, 0, sizeof(top_group_ids));
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &(internal_path->group_id),
1, top_group_ids);
if (top_group_cnt <= 0) {
/*
item->group_id has no top group, this group can only be referenced by compile
------------------------------------------------------------------------------
for example:
compile1 -> group1 -> group2 -> item1
group3 -> item2
group1 and group3 has no top group
group1 is referenced by compile1, group3 is not referenced by any compile
NOTE: Add the hit path as long as the item is hit
*/
top_group_cnt = 1; // add one hit path which top_group_ids[0] = -1
}
struct maat_hit_path tmp_path;
for (int j = 0; j < top_group_cnt && hit_path_cnt < array_size; j++, hit_path_cnt++) {
memset(&tmp_path, 0, sizeof(tmp_path));
tmp_path.Nth_scan = internal_path->Nth_scan;
tmp_path.item_id = internal_path->item_id;
tmp_path.sub_group_id = internal_path->group_id;
tmp_path.top_group_id = top_group_ids[j];
tmp_path.vtable_id = internal_path->vtable_id;
tmp_path.compile_id = -1;
/* check if internal_path is duplicated from hit_path_array[] element */
if (hit_path_cnt > 0) {
if (maat_compile_is_hit_path_existed(hit_path_array, hit_path_cnt, &tmp_path)) {
continue;
}
}
hit_path_array[hit_path_cnt] = tmp_path;
}
}
return hit_path_cnt;
}

37
test/maat_framework_gtest.cpp
View File

@@ -1427,7 +1427,7 @@ TEST_F(CompileTable, CompileRuleUpdate) {
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
} }
TEST_F(CompileTable, Conjunction) { TEST_F(CompileTable, Conjunction1) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
@@ -1453,6 +1453,41 @@ TEST_F(CompileTable, Conjunction) {
maat_state_free(&state); maat_state_free(&state);
} }
TEST_F(CompileTable, Conjunction2) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL_LITERAL";
const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"};
int table_id = maat_get_table_id(g_maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 197);
struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0};
int n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE);
EXPECT_EQ(n_read, 2);
maat_state_set_scan_compile_tables(g_maat_instance, &state, compile_tables, 2);
ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 141);
memset(hit_path, 0, sizeof(hit_path));
n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE);
EXPECT_EQ(n_read, 4);
maat_state_free(&state);
}
class Policy : public testing::Test class Policy : public testing::Test
{ {
protected: protected: