From ac8a0a512cc6831fae0b93456dd02c693ddc18ac Mon Sep 17 00:00:00 2001 From: liuwentan Date: Mon, 5 Jun 2023 15:14:45 +0800 Subject: [PATCH] [FEATURE]support get hit groups --- include/maat.h | 10 ++-- src/inc_internal/maat_compile.h | 19 +++++--- src/maat_api.c | 17 ++++--- src/maat_compile.c | 63 +++++++++++++++++++++++++ test/maat_framework_gtest.cpp | 81 ++++++++++++++++++++++++++------- 5 files changed, 157 insertions(+), 33 deletions(-) diff --git a/include/maat.h b/include/maat.h index 9fd2fad..3a994a5 100644 --- a/include/maat.h +++ b/include/maat.h @@ -36,9 +36,9 @@ struct maat_hit_path { long long compile_id; }; -struct maat_hit_object { - int vtable_id; +struct maat_hit_group { long long group_id; + int vtable_id; }; enum maat_scan_status { @@ -235,11 +235,11 @@ int maat_state_set_scan_compile_table(struct maat_state *state, int compile_tabl int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths, size_t n_path); -int maat_state_get_hit_objects(struct maat_state *state, struct maat_hit_object *objs, - size_t n_obj); +int maat_state_get_hit_groups(struct maat_state *state, struct maat_hit_group *groups, + size_t n_group); /* return hit object compile_id */ -int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj); +int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group); #ifdef __cplusplus } diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index 07d352b..67937d4 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -51,10 +51,12 @@ void compile_runtime_free(void *compile_runtime); void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt); -int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *table_name, - const char *line, int valid_column); +int compile_runtime_update(void *compile_runtime, void *compile_schema, + const char *table_name, const char *line, + int valid_column); -int compile_runtime_commit(void *compile_runtime, const char *table_name, long long maat_rt_version); +int compile_runtime_commit(void *compile_runtime, const char *table_name, + long long maat_rt_version); long long compile_runtime_rule_count(void *compile_runtime); @@ -78,11 +80,13 @@ void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt, void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); -void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime); +void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, + void *g2g_runtime); void group2compile_runtime_free(void *g2c_runtime); -int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *table_name, - const char *line, int valid_column); +int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, + const char *table_name, const char *line, + int valid_column); long long group2compile_runtime_not_group_count(void *g2c_runtime); @@ -104,6 +108,9 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp struct group2group_runtime *g2g_rt, struct maat_hit_path *hit_path_array, size_t array_size); +size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state, + struct maat_hit_group *hit_group_array, + size_t array_size); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); diff --git a/src/maat_api.c b/src/maat_api.c index 6939c4a..fa384ba 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -1846,16 +1846,21 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat paths, n_path); return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, - state->compile_state, paths, n_path, - internal_hit_path_cnt); + state->compile_state, paths, n_path, + internal_hit_path_cnt); } -int maat_state_get_hit_objects(struct maat_state *state, struct maat_hit_object *objs, size_t n_obj) +int maat_state_get_hit_groups(struct maat_state *state, struct maat_hit_group *groups, + size_t n_group) { - return 0; + if (NULL == state || NULL == groups || 0 == n_group) { + return -1; + } + + return maat_compile_state_get_hit_groups(state->compile_state, groups, n_group); } -int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj) +int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group) { return 0; -} +} \ No newline at end of file diff --git a/src/maat_compile.c b/src/maat_compile.c index 8ea93b4..e42aef3 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -14,6 +14,7 @@ #include "maat_utils.h" #include "log/log.h" +#include "maat.h" #include "uthash/utarray.h" #include "uthash/uthash.h" #include "bool_matcher.h" @@ -169,10 +170,12 @@ struct maat_compile_state { UT_array *internal_hit_paths; UT_array *all_hit_clauses; UT_array *this_scan_hit_clauses; + UT_array *all_hit_groups; }; UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL}; UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL}; +UT_icd ut_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL}; UT_icd ut_hit_path_icd = {sizeof(struct maat_internal_hit_path), NULL, NULL, NULL}; struct maat_compile *maat_compile_new(long long compile_id) @@ -1029,6 +1032,19 @@ static inline int compare_clause_id(const void *a, const void *b) } } +static inline int compare_hit_group(const void *pa, const void *pb) +{ + struct maat_hit_group *la=(struct maat_hit_group *)pa; + struct maat_hit_group *lb=(struct maat_hit_group *)pb; + + long long ret = la->group_id - lb->group_id; + if (ret == 0) { + ret = la->vtable_id - lb->vtable_id; + } + + return ret; +} + struct literal_clause *maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt) { if (NULL == compile_rt) { @@ -1447,6 +1463,7 @@ struct maat_compile_state *maat_compile_state_new(int thread_id) utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd); utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd); + utarray_new(compile_state->all_hit_groups, &ut_hit_group_icd); return compile_state; } @@ -1466,6 +1483,7 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state) utarray_clear(compile_state->internal_hit_paths); utarray_clear(compile_state->all_hit_clauses); utarray_clear(compile_state->this_scan_hit_clauses); + utarray_clear(compile_state->all_hit_groups); } void maat_compile_state_free(struct maat_compile_state *compile_state) @@ -1489,6 +1507,11 @@ void maat_compile_state_free(struct maat_compile_state *compile_state) compile_state->this_scan_hit_clauses = NULL; } + if (compile_state->all_hit_groups != NULL) { + utarray_free(compile_state->all_hit_groups); + compile_state->all_hit_groups = NULL; + } + FREE(compile_state); } @@ -1663,6 +1686,22 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta } } +void maat_compile_state_update_hit_group(struct maat_compile_state *compile_state, + long long group_id, int vtable_id) +{ + if (NULL == compile_state) { + return; + } + + struct maat_hit_group hit_group = {group_id, vtable_id}; + if (utarray_find(compile_state->all_hit_groups, &hit_group, compare_hit_group)) { + return; + } + + utarray_push_back(compile_state->all_hit_groups, &hit_group); + utarray_sort(compile_state->all_hit_groups, compare_hit_group); +} + int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state) { return compile_state->not_clause_hitted_flag; @@ -2189,14 +2228,38 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items, for (int j = 0; j < super_group_cnt; j++) { maat_compile_state_update_hit_clause(state->compile_state, compile_rt, super_group_ids[j], vtable_id); + maat_compile_state_update_hit_group(state->compile_state, super_group_ids[j], + vtable_id); } for (int j = 0; j < hit_cnt; j++) { maat_compile_state_update_hit_clause(state->compile_state, compile_rt, hit_group_ids[j], vtable_id); + maat_compile_state_update_hit_group(state->compile_state, hit_group_ids[j], + vtable_id); } } +size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state, + struct maat_hit_group *hit_group_array, + size_t array_size) +{ + if (NULL == compile_state) { + return 0; + } + + size_t i = 0; + struct maat_hit_group *hit_group = NULL; + + for (i = 0; i < utarray_len(compile_state->all_hit_groups) && i < array_size; i++) { + hit_group = (struct maat_hit_group *)utarray_eltptr(compile_state->all_hit_groups, i); + hit_group_array[i].group_id = hit_group->group_id; + hit_group_array[i].vtable_id = hit_group->vtable_id; + } + + return i; +} + UT_icd ut_compile_group_id_icd = {sizeof(long long), NULL, NULL, NULL}; size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *compile_state, struct compile_runtime *compile_rt, diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 81bcd42..d0691e2 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -5366,10 +5366,10 @@ TEST_F(MaatCmdTest, HitPath) { const char* http_resp_hdr_cookie = "laptop=thinkpad X1 extrem;time=2020-02-11T15:34:00;" "main[XWJOKE]=hoho; Hm_lvt_bbac0322e6ee13093f98d5c4b5a10912=1578874808;"; - int table_id = maat_get_table_id(maat_instance, "HTTP_REQUEST_HEADER"); - ASSERT_GT(table_id, 0); + int http_req_table_id = maat_get_table_id(maat_instance, "HTTP_REQUEST_HEADER"); + ASSERT_GT(http_req_table_id, 0); - ret = maat_state_set_scan_district(state, table_id, "URL", strlen("URL")); + ret = maat_state_set_scan_district(state, http_req_table_id, "URL", strlen("URL")); EXPECT_EQ(ret, 0); int Nth_scan = 0; @@ -5377,10 +5377,19 @@ TEST_F(MaatCmdTest, HitPath) { Nth_scan++; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; - ret = maat_scan_string(maat_instance, table_id, http_url, strlen(http_url), + ret = maat_scan_string(maat_instance, http_req_table_id, http_url, strlen(http_url), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + struct maat_hit_group hit_groups[128]; + memset(hit_groups, 0, sizeof(hit_groups)); + int n_hit_group = maat_state_get_hit_groups(state, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 2); + EXPECT_EQ(hit_groups[0].group_id, group1_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[1].group_id, group11_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_req_table_id); + struct maat_hit_path hit_path[128]; memset(hit_path, 0, sizeof(hit_path)); int n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); @@ -5392,22 +5401,33 @@ TEST_F(MaatCmdTest, HitPath) { EXPECT_EQ(hit_path[path_idx].sub_group_id, group1_id); //EXPECT_EQ(hit_path[path_idx].top_group_id, group1_id); EXPECT_EQ(hit_path[path_idx].top_group_id, -1); - EXPECT_EQ(hit_path[path_idx].vtable_id, table_id); + EXPECT_EQ(hit_path[path_idx].vtable_id, http_req_table_id); EXPECT_EQ(hit_path[path_idx].compile_id, -1); - table_id = maat_get_table_id(maat_instance, "HTTP_RESPONSE_HEADER"); - ASSERT_GT(table_id, 0); + int http_res_table_id = maat_get_table_id(maat_instance, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(http_res_table_id, 0); - ret = maat_state_set_scan_district(state, table_id, "Cookie", strlen("Cookie")); + ret = maat_state_set_scan_district(state, http_res_table_id, "Cookie", strlen("Cookie")); EXPECT_EQ(ret, 0); Nth_scan++; - ret = maat_scan_string(maat_instance, table_id, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + ret = maat_scan_string(maat_instance, http_res_table_id, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], compile1_id); + n_hit_group = maat_state_get_hit_groups(state, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 4); + EXPECT_EQ(hit_groups[0].group_id, group1_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[1].group_id, group21_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[2].group_id, group2_id); + EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[3].group_id, group11_id); + EXPECT_EQ(hit_groups[3].vtable_id, http_req_table_id); + n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); EXPECT_EQ(n_read, 2); EXPECT_EQ(hit_path[path_idx].Nth_scan, Nth_scan-1); @@ -5422,7 +5442,7 @@ TEST_F(MaatCmdTest, HitPath) { EXPECT_EQ(hit_path[path_idx].item_id, item2_id); EXPECT_EQ(hit_path[path_idx].sub_group_id, group2_id); EXPECT_EQ(hit_path[path_idx].top_group_id, group21_id); - EXPECT_EQ(hit_path[path_idx].vtable_id, table_id); + EXPECT_EQ(hit_path[path_idx].vtable_id, http_res_table_id); EXPECT_EQ(hit_path[path_idx].compile_id, compile1_id); const char* keywords1="In graph theory, a path in a graph is a finite or infinite \ @@ -5432,14 +5452,28 @@ are all distinct (and since the vertices are distinct, so are the edges). "; sequence of edges which joins a sequence of distinct vertices, but with the added restriction\ that the edges be all directed in the same direction."; - table_id = maat_get_table_id(maat_instance, keywords_table_name); - ASSERT_GT(table_id, 0); + int keywords_table_id = maat_get_table_id(maat_instance, keywords_table_name); + ASSERT_GT(keywords_table_id, 0); - struct maat_stream *stream = maat_stream_new(maat_instance, table_id, state); + struct maat_stream *stream = maat_stream_new(maat_instance, keywords_table_id, state); Nth_scan++; ret = maat_stream_scan(stream, keywords1, strlen(keywords1), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + n_hit_group = maat_state_get_hit_groups(state, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 5); + EXPECT_EQ(hit_groups[0].group_id, group1_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[1].group_id, group21_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[2].group_id, group2_id); + EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[3].group_id, group11_id); + EXPECT_EQ(hit_groups[3].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[4].group_id, group4_id); + EXPECT_EQ(hit_groups[4].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 + n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); EXPECT_EQ(n_read, 3); @@ -5452,17 +5486,32 @@ that the edges be all directed in the same direction."; EXPECT_EQ(hit_path[path_idx].vtable_id, 0); EXPECT_EQ(hit_path[path_idx].compile_id, -1); - table_id = maat_get_table_id(maat_instance, ip_table_name); - ASSERT_GT(table_id, 0); + int ip_table_id = maat_get_table_id(maat_instance, ip_table_name); + ASSERT_GT(ip_table_id, 0); Nth_scan++; uint32_t ip_addr; inet_pton(AF_INET, "220.181.38.148", &ip_addr); uint16_t port = htons(17272); - ret = maat_scan_ipv4(maat_instance, table_id, ip_addr, port, 6, results, ARRAY_SIZE, + ret = maat_scan_ipv4(maat_instance, ip_table_id, ip_addr, port, 6, results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + n_hit_group = maat_state_get_hit_groups(state, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 6); + EXPECT_EQ(hit_groups[0].group_id, group1_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[1].group_id, group21_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[2].group_id, group2_id); + EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[3].group_id, group11_id); + EXPECT_EQ(hit_groups[3].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[4].group_id, group3_id); + EXPECT_EQ(hit_groups[4].vtable_id, 0); //physical table(ip_table) vtable_id is 0 + EXPECT_EQ(hit_groups[5].group_id, group4_id); + EXPECT_EQ(hit_groups[5].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 + n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); EXPECT_EQ(n_read, 4);