gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[PATCH]add get direct/indirect hit groups API

Browse Source
This commit is contained in:
liuwentan
2023-09-11 12:00:33 +08:00
parent c237d7dbaf
commit 2e1a14eca3
5 changed files with 144 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
include/maat.h
View File

@@ -281,8 +281,8 @@ int maat_state_disable_compile_NOT(struct maat_state *state);
int maat_state_set_scan_compile_table(struct maat_state *state, int compile_table_id);
int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths,
size_t n_path);
int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *path_array,
size_t array_size);
/**
* @brief get the total number of scans after maat_state_new
@@ -290,15 +290,24 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat
size_t maat_state_get_scan_count(struct maat_state *state);
/**
* @brief get hit groups(full or incremental)
* @brief get direct hit groups(full or incremental)
*
* @param type:
* MAAT_LIST_TYPE_FULL => get all hit groups after maat_state_new
* MAAT_LIST_TYPE_INC => get hit groups for this scan
* NOTE: hit groups may be duplicated, only return hit sub group id
* NOTE: hit groups may be duplicated
*/
int maat_state_get_hit_groups(struct maat_state *state, enum maat_list_type type,
struct maat_hit_group *groups, size_t n_group);
int maat_state_get_direct_hit_groups(struct maat_state *state, enum maat_list_type type,
struct maat_hit_group *group_array, size_t array_size);
/**
* @brief get indirect hit groups
*
* NOTE: hit groups may be duplicated
*/
int maat_state_get_indirect_hit_groups(struct maat_state *state,
struct maat_hit_group *group_array,
size_t array_size);
/* return hit object compile_id */
int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group);

13
src/inc_internal/maat_compile.h
View File

@@ -68,7 +68,7 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thread_id,
struct maat_compile_state *compile_state,
struct maat_hit_path *hit_path_array,
size_t array_size, size_t n_internal_hit_path);
size_t array_size, size_t n_hit_path);
void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema,
@@ -113,11 +113,14 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp
struct maat_hit_path *hit_path_array,
size_t array_size);
size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state,
struct group2group_runtime *g2g_rt,
size_t maat_compile_state_get_direct_hit_groups(struct maat_compile_state *compile_state,
enum maat_list_type type,
struct maat_hit_group *hit_group_array,
size_t group_array_size);
struct maat_hit_group *group_array,
size_t array_size);
size_t maat_compile_state_get_indirect_hit_groups(struct maat_compile_state *compile_state,
struct maat_hit_group *group_array,
size_t array_size);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);

40
src/maat_api.c
View File

@@ -1906,10 +1906,10 @@ int maat_state_set_scan_compile_table(struct maat_state *state, int compile_tabl
return 0;
}
int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths,
size_t n_path)
int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *path_array,
size_t array_size)
{
if (NULL == state || NULL == paths || 0 == n_path) {
if (NULL == state || NULL == path_array || 0 == array_size) {
return -1;
}
@@ -1937,13 +1937,13 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat
int g2g_table_id = table_manager_get_group2group_table_id(maat_inst->tbl_mgr);
void *g2g_runtime = table_manager_get_runtime(maat_inst->tbl_mgr, g2g_table_id);
size_t internal_hit_path_cnt = maat_compile_state_get_internal_hit_paths(state->compile_state,
size_t hit_path_cnt = maat_compile_state_get_internal_hit_paths(state->compile_state,
(struct compile_runtime *)compile_rt,
(struct group2group_runtime *)g2g_runtime,
paths, n_path);
path_array, array_size);
return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, state->thread_id,
state->compile_state, paths, n_path, internal_hit_path_cnt);
state->compile_state, path_array, array_size, hit_path_cnt);
}
size_t maat_state_get_scan_count(struct maat_state *state)
@@ -1955,10 +1955,10 @@ size_t maat_state_get_scan_count(struct maat_state *state)
return state->scan_cnt;
}
int maat_state_get_hit_groups(struct maat_state *state, enum maat_list_type type,
struct maat_hit_group *groups, size_t n_group)
int maat_state_get_direct_hit_groups(struct maat_state *state, enum maat_list_type type,
struct maat_hit_group *group_array, size_t array_size)
{
if (NULL == state || NULL == groups || 0 == n_group) {
if (NULL == state || NULL == group_array || 0 == array_size) {
return -1;
}
@@ -1966,12 +1966,24 @@ int maat_state_get_hit_groups(struct maat_state *state, enum maat_list_type type
return 0;
}
int g2g_table_id = table_manager_get_group2group_table_id(state->maat_inst->tbl_mgr);
void *g2g_runtime = table_manager_get_runtime(state->maat_inst->tbl_mgr, g2g_table_id);
return maat_compile_state_get_direct_hit_groups(state->compile_state, type,
group_array, array_size);
}
return maat_compile_state_get_hit_groups(state->compile_state,
(struct group2group_runtime *)g2g_runtime,
type, groups, n_group);
int maat_state_get_indirect_hit_groups(struct maat_state *state,
struct maat_hit_group *group_array,
size_t array_size)
{
if (NULL == state || NULL == group_array || 0 == array_size) {
return -1;
}
if (NULL == state->compile_state) {
return 0;
}
return maat_compile_state_get_indirect_hit_groups(state->compile_state,
group_array, array_size);
}
int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group)

84
src/maat_compile.c
View File

@@ -147,10 +147,12 @@ struct maat_compile_state {
UT_array *internal_inc_hit_paths;
UT_array *all_hit_clauses;
UT_array *this_scan_hit_clauses;
UT_array *indirect_hit_groups;
};
UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL};
UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL};
UT_icd ut_indirect_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL};
UT_icd ut_hit_path_icd = {sizeof(struct maat_internal_hit_path), NULL, NULL, NULL};
static struct maat_compile *maat_compile_new(long long compile_id)
@@ -1347,6 +1349,7 @@ struct maat_compile_state *maat_compile_state_new(void)
utarray_new(compile_state->internal_inc_hit_paths, &ut_hit_path_icd);
utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->indirect_hit_groups, &ut_indirect_hit_group_icd);
return compile_state;
}
@@ -1366,6 +1369,7 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state)
utarray_clear(compile_state->internal_inc_hit_paths);
utarray_clear(compile_state->all_hit_clauses);
utarray_clear(compile_state->this_scan_hit_clauses);
utarray_clear(compile_state->indirect_hit_groups);
}
void maat_compile_state_free(struct maat_compile_state *compile_state,
@@ -1377,13 +1381,15 @@ void maat_compile_state_free(struct maat_compile_state *compile_state,
long long free_bytes = 0;
if (compile_state->internal_hit_paths != NULL) {
free_bytes += utarray_len(compile_state->internal_hit_paths) * sizeof(struct maat_internal_hit_path);
free_bytes += utarray_len(compile_state->internal_hit_paths) *
sizeof(struct maat_internal_hit_path);
utarray_free(compile_state->internal_hit_paths);
compile_state->internal_hit_paths = NULL;
}
if (compile_state->internal_inc_hit_paths != NULL) {
free_bytes += utarray_len(compile_state->internal_inc_hit_paths) * sizeof(struct maat_internal_hit_path);
free_bytes += utarray_len(compile_state->internal_inc_hit_paths) *
sizeof(struct maat_internal_hit_path);
utarray_free(compile_state->internal_inc_hit_paths);
compile_state->internal_inc_hit_paths = NULL;
}
@@ -1400,10 +1406,17 @@ void maat_compile_state_free(struct maat_compile_state *compile_state,
compile_state->this_scan_hit_clauses = NULL;
}
if (compile_state->indirect_hit_groups != NULL) {
free_bytes += utarray_len(compile_state->indirect_hit_groups) * sizeof(struct maat_hit_group);
utarray_free(compile_state->indirect_hit_groups);
compile_state->indirect_hit_groups = NULL;
}
FREE(compile_state);
free_bytes += sizeof(struct maat_compile_state);
alignment_int64_array_add(maat_inst->stat->maat_state_free_bytes, thread_id, free_bytes);
alignment_int64_array_add(maat_inst->stat->maat_state_free_bytes,
thread_id, free_bytes);
}
static void maat_compile_hit_path_add(UT_array *hit_paths, long long item_id,
@@ -1457,7 +1470,7 @@ static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_path
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thread_id,
struct maat_compile_state *compile_state,
struct maat_hit_path *hit_path_array,
size_t array_size, size_t n_internal_hit_path)
size_t array_size, size_t n_hit_path)
{
/* assign hit_path_array[].compile_id */
size_t new_hit_path_cnt = 0;
@@ -1483,7 +1496,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
continue;
}
for (size_t j = 0; j < n_internal_hit_path && (n_internal_hit_path + new_hit_path_cnt) < array_size; j++) {
for (size_t j = 0; j < n_hit_path && (n_hit_path + new_hit_path_cnt) < array_size; j++) {
if (hit_path_array[j].top_group_id < 0) {
literal_id.group_id = hit_path_array[j].sub_group_id;
} else {
@@ -1502,8 +1515,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
// means same literal_id hit more than one compile_id
struct maat_hit_path tmp_path = hit_path_array[j];
tmp_path.compile_id = compile->compile_id;
if(!maat_compile_is_hit_path_existed(hit_path_array, n_internal_hit_path + new_hit_path_cnt, &tmp_path)) {
hit_path_array[n_internal_hit_path + new_hit_path_cnt] = tmp_path;
if(!maat_compile_is_hit_path_existed(hit_path_array, n_hit_path + new_hit_path_cnt, &tmp_path)) {
hit_path_array[n_hit_path + new_hit_path_cnt] = tmp_path;
new_hit_path_cnt++;
}
}
@@ -1511,7 +1524,24 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
}
}
return (n_internal_hit_path + new_hit_path_cnt);
return (n_hit_path + new_hit_path_cnt);
}
static void maat_compile_state_update_indirect_group(struct maat_compile_state *compile_state,
long long *group_ids, size_t n_group_id,
int vtable_id)
{
if (NULL == compile_state) {
return;
}
struct maat_hit_group hit_group;
for (size_t i = 0; i < n_group_id; i++) {
hit_group.item_id = 0;
hit_group.group_id = group_ids[i];
hit_group.vtable_id = vtable_id;
utarray_push_back(compile_state->indirect_hit_groups, &hit_group);
}
}
static void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state,
@@ -2059,6 +2089,8 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
hit_items[i].group_id, vtable_id, state->scan_cnt);
}
maat_compile_state_update_indirect_group(state->compile_state, super_group_ids, super_group_cnt, vtable_id);
/* update hit clause */
int compile_table_id = table_manager_get_default_compile_table_id(maat_inst->tbl_mgr);
if (state->compile_table_id > 0) {
@@ -2082,11 +2114,31 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
}
}
size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state,
struct group2group_runtime *g2g_rt,
size_t maat_compile_state_get_indirect_hit_groups(struct maat_compile_state *compile_state,
struct maat_hit_group *group_array, size_t array_size)
{
if (NULL == compile_state) {
return 0;
}
size_t i = 0;
struct maat_hit_group *hit_group = NULL;
for (i = 0; i < utarray_len(compile_state->indirect_hit_groups) && i < array_size; i++) {
hit_group = (struct maat_hit_group *)utarray_eltptr(compile_state->indirect_hit_groups, i);
group_array[i].item_id = hit_group->item_id;
group_array[i].group_id = hit_group->group_id;
group_array[i].vtable_id = hit_group->vtable_id;
}
utarray_clear(compile_state->indirect_hit_groups);
return i;
}
size_t maat_compile_state_get_direct_hit_groups(struct maat_compile_state *compile_state,
enum maat_list_type type,
struct maat_hit_group *hit_group_array,
size_t group_array_size)
struct maat_hit_group *group_array,
size_t array_size)
{
if (NULL == compile_state) {
return 0;
@@ -2102,11 +2154,11 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
size_t i = 0;
struct maat_internal_hit_path *path = NULL;
for (i = 0; i < utarray_len(wanted_hit_path) && i < group_array_size; i++) {
for (i = 0; i < utarray_len(wanted_hit_path) && i < array_size; i++) {
path = (struct maat_internal_hit_path *)utarray_eltptr(wanted_hit_path, i);
hit_group_array[i].item_id = path->item_id;
hit_group_array[i].group_id = path->group_id;
hit_group_array[i].vtable_id = path->vtable_id;
group_array[i].item_id = path->item_id;
group_array[i].group_id = path->group_id;
group_array[i].vtable_id = path->vtable_id;
}
if (type == MAAT_LIST_TYPE_INC) {

32
test/maat_framework_gtest.cpp
View File

@@ -6443,19 +6443,26 @@ TEST_F(MaatCmdTest, HitGroup) {
struct maat_hit_group hit_groups[128];
memset(hit_groups, 0, sizeof(hit_groups));
int n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
int n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
EXPECT_EQ(n_hit_group, 1);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
EXPECT_EQ(hit_groups[0].group_id, group1_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id);
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
EXPECT_EQ(n_hit_group, 1);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
EXPECT_EQ(hit_groups[0].group_id, group1_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id);
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_indirect_hit_groups(state, hit_groups, 128);
EXPECT_EQ(n_hit_group, 1);
EXPECT_EQ(hit_groups[0].item_id, 0);
EXPECT_EQ(hit_groups[0].group_id, group11_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id);
int http_res_table_id = maat_get_table_id(maat_inst, "HTTP_RESPONSE_HEADER");
ASSERT_GT(http_res_table_id, 0);
@@ -6470,7 +6477,7 @@ TEST_F(MaatCmdTest, HitGroup) {
scan_count = maat_state_get_scan_count(state);
EXPECT_EQ(scan_count, 2);
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
EXPECT_EQ(n_hit_group, 2);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
@@ -6482,13 +6489,20 @@ TEST_F(MaatCmdTest, HitGroup) {
EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id);
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
EXPECT_EQ(n_hit_group, 1);
EXPECT_EQ(hit_groups[0].item_id, item2_id);
EXPECT_EQ(hit_groups[0].group_id, group2_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_res_table_id);
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_indirect_hit_groups(state, hit_groups, 128);
EXPECT_EQ(n_hit_group, 1);
EXPECT_EQ(hit_groups[0].item_id, 0);
EXPECT_EQ(hit_groups[0].group_id, group21_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_res_table_id);
const char* keywords1="In graph theory, hit group item forth";
const char *keywords2="To test one group hit group item fifth";
@@ -6502,7 +6516,7 @@ TEST_F(MaatCmdTest, HitGroup) {
scan_count = maat_state_get_scan_count(state);
EXPECT_EQ(scan_count, 3);
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
EXPECT_EQ(n_hit_group, 3);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
@@ -6529,7 +6543,7 @@ TEST_F(MaatCmdTest, HitGroup) {
scan_count = maat_state_get_scan_count(state);
EXPECT_EQ(scan_count, 4);
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
EXPECT_EQ(n_hit_group, 4);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
@@ -6549,7 +6563,7 @@ TEST_F(MaatCmdTest, HitGroup) {
EXPECT_EQ(hit_groups[3].vtable_id, 0); //physical table(keywords_table) vtable_id is 0
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
EXPECT_EQ(n_hit_group, 2);
EXPECT_EQ(hit_groups[0].item_id, item4_id);
@@ -6568,7 +6582,7 @@ TEST_F(MaatCmdTest, HitGroup) {
EXPECT_EQ(scan_count, 5);
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
EXPECT_EQ(n_hit_group, 5);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
@@ -6592,7 +6606,7 @@ TEST_F(MaatCmdTest, HitGroup) {
EXPECT_EQ(hit_groups[4].vtable_id, 0);
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
EXPECT_EQ(n_hit_group, 1);
EXPECT_EQ(hit_groups[0].item_id, item5_id);