diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index e6e588e..36e0de5 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -117,7 +117,7 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat struct group2group_runtime *g2g_rt, enum maat_list_type type, struct maat_hit_group *hit_group_array, - size_t array_size); + size_t array_size, struct log_handle *logger); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); diff --git a/src/maat_api.c b/src/maat_api.c index 1048638..6cbc117 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -1964,7 +1964,7 @@ int maat_state_get_hit_groups(struct maat_state *state, enum maat_list_type type return maat_compile_state_get_hit_groups(state->compile_state, (struct group2group_runtime *)g2g_runtime, - type, groups, n_group); + type, groups, n_group, state->maat_inst->logger); } int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group) diff --git a/src/maat_compile.c b/src/maat_compile.c index 9ec95ae..6005676 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -28,6 +28,7 @@ #define MODULE_COMPILE module_name_str("maat.compile") #define DEFAULT_GC_TIMEOUT_S 10 +#define MAX_SUPER_GROUP_CNT 128 struct compile_schema { int compile_id_column; @@ -132,6 +133,9 @@ struct maat_compile { struct maat_internal_hit_path { long long item_id; long long group_id; + long long all_group_ids[MAX_SUPER_GROUP_CNT]; // group_id + super_group_ids + int n_all_group_ids; + int excced_max_group_flag; //if all hit groups count exceed MAX_SUPER_GROUP_CNT int Nth_scan; int Nth_hit_item; int vtable_id; @@ -989,9 +993,9 @@ static inline int compare_hit_group(const void *pa, const void *pb) struct maat_hit_group *lb=(struct maat_hit_group *)pb; long long ret = la->item_id - lb->item_id; - if (ret == 0) { + if (0 == ret) { ret = la->group_id - lb->group_id; - if (ret == 0) { + if (0 == ret) { ret = la->vtable_id - lb->vtable_id; } } @@ -1427,7 +1431,8 @@ void maat_compile_state_free(struct maat_compile_state *compile_state, } static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id, - long long group_id, int vtable_id, + long long group_id, long long *super_group_ids, + size_t n_super_group_ids, int vtable_id, int Nth_scan, int Nth_item_result) { struct maat_internal_hit_path new_path; @@ -1437,7 +1442,19 @@ static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id, new_path.Nth_scan = Nth_scan; new_path.group_id = group_id; new_path.vtable_id = vtable_id; - + new_path.n_all_group_ids = 0; + new_path.excced_max_group_flag = 0; + + for (size_t i = 0; i < n_super_group_ids && i < MAX_SUPER_GROUP_CNT; i++) { + new_path.all_group_ids[new_path.n_all_group_ids++] = super_group_ids[i]; + } + + if (new_path.n_all_group_ids < MAX_SUPER_GROUP_CNT) { + new_path.all_group_ids[new_path.n_all_group_ids++] = group_id; + } else { + new_path.excced_max_group_flag = 1; + } + utarray_push_back(hit_paths, &new_path); return 1; @@ -1541,6 +1558,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr static void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, long long item_id, long long group_id, + long long *super_group_ids, size_t n_super_group_ids, int vtable_id, int Nth_scan, int Nth_item_result) { if (compile_state->Nth_scan != Nth_scan) { @@ -1555,10 +1573,10 @@ static void maat_compile_state_update_hit_path(struct maat_compile_state *compil } maat_compile_hit_path_add(compile_state->internal_inc_hit_paths, item_id, group_id, - vtable_id, Nth_scan, Nth_item_result); + super_group_ids, n_super_group_ids, vtable_id, Nth_scan, Nth_item_result); - maat_compile_hit_path_add(compile_state->internal_hit_paths, item_id, group_id, - vtable_id, Nth_scan, Nth_item_result); + maat_compile_hit_path_add(compile_state->internal_hit_paths, item_id, group_id, + super_group_ids, n_super_group_ids, vtable_id, Nth_scan, Nth_item_result); compile_state->this_scan_hit_item_flag = 1; } @@ -2073,10 +2091,25 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items, state->thread_id, 1); } + int g2g_table_id = table_manager_get_group2group_table_id(maat_inst->tbl_mgr); + void *g2g_rt = table_manager_get_runtime(maat_inst->tbl_mgr, g2g_table_id); + + for (i = 0; i < hit_cnt; i++) { + hit_group_ids[i] = hit_items[i].group_id; + } + + long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; + size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids, + hit_cnt, super_group_ids, + MAX_SCANNER_HIT_GROUP_NUM); + if (super_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) { + super_group_cnt = MAX_SCANNER_HIT_GROUP_NUM; + } + for (i = 0; i < hit_cnt; i++) { maat_compile_state_update_hit_path(state->compile_state, hit_items[i].item_id, - hit_items[i].group_id, vtable_id, state->scan_cnt, i); - hit_group_ids[i] = hit_items[i].group_id; + hit_items[i].group_id, super_group_ids, super_group_cnt, + vtable_id, state->scan_cnt, i); } /* update hit clause */ @@ -2091,17 +2124,6 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items, return; } - int g2g_table_id = table_manager_get_group2group_table_id(maat_inst->tbl_mgr); - void *g2g_rt = table_manager_get_runtime(maat_inst->tbl_mgr, g2g_table_id); - - long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; - size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids, - hit_cnt, super_group_ids, - MAX_SCANNER_HIT_GROUP_NUM); - if (super_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) { - super_group_cnt = MAX_SCANNER_HIT_GROUP_NUM; - } - for (int j = 0; j < super_group_cnt; j++) { maat_compile_state_update_hit_clause(state->compile_state, compile_rt, super_group_ids[j], vtable_id); @@ -2113,11 +2135,12 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items, } } +#define MAX_HIT_PATH_CNT 10 size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state, struct group2group_runtime *g2g_rt, enum maat_list_type type, struct maat_hit_group *hit_group_array, - size_t array_size) + size_t array_size, struct log_handle *logger) { if (NULL == compile_state) { return 0; @@ -2138,19 +2161,22 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat } for (i = 0; i < utarray_len(tmp_hit_path); i++) { - internal_path = (struct maat_internal_hit_path *)utarray_eltptr(tmp_hit_path, i); - - long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; - size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, &(internal_path->group_id), 1, - super_group_ids, MAX_SCANNER_HIT_GROUP_NUM); - if (super_group_cnt + 1 <= MAX_SCANNER_HIT_GROUP_NUM) { - super_group_ids[super_group_cnt++] = internal_path->group_id; + if (i > MAX_HIT_PATH_CNT) { + break; } - for (size_t idx = 0; idx < super_group_cnt; idx++) { + internal_path = (struct maat_internal_hit_path *)utarray_eltptr(tmp_hit_path, i); + if (internal_path->excced_max_group_flag == 1) { + log_error(logger, MODULE_COMPILE, + "[%s:%d]group_id:%lld has too much super group ids, exceed maxium:%d", + __FUNCTION__, __LINE__, internal_path->group_id, MAX_SUPER_GROUP_CNT); + internal_path->excced_max_group_flag = 0; + } + + for (size_t idx = 0; idx < internal_path->n_all_group_ids; idx++) { struct maat_hit_group hit_group; hit_group.item_id = internal_path->item_id; - hit_group.group_id = super_group_ids[idx]; + hit_group.group_id = internal_path->all_group_ids[idx]; hit_group.vtable_id = internal_path->vtable_id; if (utarray_find(all_hit_groups, &hit_group, compare_hit_group)) { continue; diff --git a/src/maat_kv.c b/src/maat_kv.c index 2ce1a73..4850434 100644 --- a/src/maat_kv.c +++ b/src/maat_kv.c @@ -129,7 +129,6 @@ int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase)); HASH_FIND(hh, store->hash, key_lowercase, key_len, kv); - int i = 0; if (kv) { *value = kv->val; return 1; @@ -181,4 +180,4 @@ struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map) } return target; -} \ No newline at end of file +} diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index fc65c42..d11c4ff 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -6323,9 +6323,9 @@ TEST_F(MaatCmdTest, GroupInMassCompiles) { } TEST_F(MaatCmdTest, HitGroup) { - const char *g2g_table_name = "GROUP2GROUP"; + const char *compile_table_name = "COMPILE"; const char *g2c_table_name = "GROUP2COMPILE"; - const char *compile_table_name = "COMPILE"; + const char *g2g_table_name = "GROUP2GROUP"; const char *http_sig_table_name = "HTTP_SIGNATURE"; const char *ip_table_name = "IP_CONFIG"; const char *keywords_table_name = "KEYWORDS_TABLE"; @@ -6480,6 +6480,7 @@ TEST_F(MaatCmdTest, HitGroup) { n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); EXPECT_EQ(n_hit_group, 4); + EXPECT_EQ(hit_groups[0].item_id, item1_id); EXPECT_EQ(hit_groups[0].group_id, group1_id); EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); @@ -6499,6 +6500,7 @@ TEST_F(MaatCmdTest, HitGroup) { memset(hit_groups, 0, sizeof(hit_groups)); n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); EXPECT_EQ(n_hit_group, 2); + EXPECT_EQ(hit_groups[0].item_id, item2_id); EXPECT_EQ(hit_groups[0].group_id, group21_id); EXPECT_EQ(hit_groups[0].vtable_id, http_res_table_id); @@ -6507,7 +6509,6 @@ TEST_F(MaatCmdTest, HitGroup) { EXPECT_EQ(hit_groups[1].group_id, group2_id); EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); - const char* keywords1="In graph theory, hit group item forth"; const char *keywords2="To test one group hit group item fifth"; @@ -6523,6 +6524,7 @@ TEST_F(MaatCmdTest, HitGroup) { n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); EXPECT_EQ(n_hit_group, 5); + EXPECT_EQ(hit_groups[0].item_id, item1_id); EXPECT_EQ(hit_groups[0].group_id, group1_id); EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); @@ -6557,6 +6559,7 @@ TEST_F(MaatCmdTest, HitGroup) { n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); EXPECT_EQ(n_hit_group, 6); + EXPECT_EQ(hit_groups[0].item_id, item1_id); EXPECT_EQ(hit_groups[0].group_id, group1_id); EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); @@ -6584,6 +6587,7 @@ TEST_F(MaatCmdTest, HitGroup) { memset(hit_groups, 0, sizeof(hit_groups)); n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); EXPECT_EQ(n_hit_group, 2); + EXPECT_EQ(hit_groups[0].item_id, item3_id); EXPECT_EQ(hit_groups[0].group_id, group3_id); EXPECT_EQ(hit_groups[0].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 @@ -6602,6 +6606,7 @@ TEST_F(MaatCmdTest, HitGroup) { memset(hit_groups, 0, sizeof(hit_groups)); n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); EXPECT_EQ(n_hit_group, 8); + EXPECT_EQ(hit_groups[0].item_id, item1_id); EXPECT_EQ(hit_groups[0].group_id, group1_id); EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); @@ -6638,6 +6643,7 @@ TEST_F(MaatCmdTest, HitGroup) { memset(hit_groups, 0, sizeof(hit_groups)); n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); EXPECT_EQ(n_hit_group, 2); + EXPECT_EQ(hit_groups[0].item_id, item5_id); EXPECT_EQ(hit_groups[0].group_id, group1_id); EXPECT_EQ(hit_groups[0].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 @@ -6872,7 +6878,7 @@ that the edges be all directed in the same direction."; EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); scan_count = maat_state_get_scan_count(state); EXPECT_EQ(scan_count, 4); - + n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); EXPECT_EQ(n_read, 6); @@ -8079,4 +8085,4 @@ int main(int argc, char ** argv) ret=RUN_ALL_TESTS(); return ret; -} \ No newline at end of file +}