gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FEATURE]support get hit groups and items

Browse Source
This commit is contained in:
刘文坛
2023-08-10 12:21:04 +00:00
parent 42f4480271
commit d030e5a452
5 changed files with 70 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/inc_internal/maat_compile.h
View File

@@ -117,7 +117,7 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
struct group2group_runtime *g2g_rt,
enum maat_list_type type,
struct maat_hit_group *hit_group_array,
size_t array_size);
size_t array_size, struct log_handle *logger);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);

2
src/maat_api.c
View File

@@ -1964,7 +1964,7 @@ int maat_state_get_hit_groups(struct maat_state *state, enum maat_list_type type
return maat_compile_state_get_hit_groups(state->compile_state,
(struct group2group_runtime *)g2g_runtime,
type, groups, n_group);
type, groups, n_group, state->maat_inst->logger);
}
int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group)

82
src/maat_compile.c
View File

@@ -28,6 +28,7 @@
#define MODULE_COMPILE module_name_str("maat.compile")
#define DEFAULT_GC_TIMEOUT_S 10
#define MAX_SUPER_GROUP_CNT 128
struct compile_schema {
int compile_id_column;
@@ -132,6 +133,9 @@ struct maat_compile {
struct maat_internal_hit_path {
long long item_id;
long long group_id;
long long all_group_ids[MAX_SUPER_GROUP_CNT]; // group_id + super_group_ids
int n_all_group_ids;
int excced_max_group_flag; //if all hit groups count exceed MAX_SUPER_GROUP_CNT
int Nth_scan;
int Nth_hit_item;
int vtable_id;
@@ -989,9 +993,9 @@ static inline int compare_hit_group(const void *pa, const void *pb)
struct maat_hit_group *lb=(struct maat_hit_group *)pb;
long long ret = la->item_id - lb->item_id;
if (ret == 0) {
if (0 == ret) {
ret = la->group_id - lb->group_id;
if (ret == 0) {
if (0 == ret) {
ret = la->vtable_id - lb->vtable_id;
}
}
@@ -1427,7 +1431,8 @@ void maat_compile_state_free(struct maat_compile_state *compile_state,
}
static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id,
long long group_id, int vtable_id,
long long group_id, long long *super_group_ids,
size_t n_super_group_ids, int vtable_id,
int Nth_scan, int Nth_item_result)
{
struct maat_internal_hit_path new_path;
@@ -1437,6 +1442,18 @@ static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id,
new_path.Nth_scan = Nth_scan;
new_path.group_id = group_id;
new_path.vtable_id = vtable_id;
new_path.n_all_group_ids = 0;
new_path.excced_max_group_flag = 0;
for (size_t i = 0; i < n_super_group_ids && i < MAX_SUPER_GROUP_CNT; i++) {
new_path.all_group_ids[new_path.n_all_group_ids++] = super_group_ids[i];
}
if (new_path.n_all_group_ids < MAX_SUPER_GROUP_CNT) {
new_path.all_group_ids[new_path.n_all_group_ids++] = group_id;
} else {
new_path.excced_max_group_flag = 1;
}
utarray_push_back(hit_paths, &new_path);
@@ -1541,6 +1558,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
static void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state,
long long item_id, long long group_id,
long long *super_group_ids, size_t n_super_group_ids,
int vtable_id, int Nth_scan, int Nth_item_result)
{
if (compile_state->Nth_scan != Nth_scan) {
@@ -1555,10 +1573,10 @@ static void maat_compile_state_update_hit_path(struct maat_compile_state *compil
}
maat_compile_hit_path_add(compile_state->internal_inc_hit_paths, item_id, group_id,
vtable_id, Nth_scan, Nth_item_result);
super_group_ids, n_super_group_ids, vtable_id, Nth_scan, Nth_item_result);
maat_compile_hit_path_add(compile_state->internal_hit_paths, item_id, group_id,
vtable_id, Nth_scan, Nth_item_result);
super_group_ids, n_super_group_ids, vtable_id, Nth_scan, Nth_item_result);
compile_state->this_scan_hit_item_flag = 1;
}
@@ -2073,10 +2091,25 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
state->thread_id, 1);
}
int g2g_table_id = table_manager_get_group2group_table_id(maat_inst->tbl_mgr);
void *g2g_rt = table_manager_get_runtime(maat_inst->tbl_mgr, g2g_table_id);
for (i = 0; i < hit_cnt; i++) {
hit_group_ids[i] = hit_items[i].group_id;
}
long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids,
hit_cnt, super_group_ids,
MAX_SCANNER_HIT_GROUP_NUM);
if (super_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
super_group_cnt = MAX_SCANNER_HIT_GROUP_NUM;
}
for (i = 0; i < hit_cnt; i++) {
maat_compile_state_update_hit_path(state->compile_state, hit_items[i].item_id,
hit_items[i].group_id, vtable_id, state->scan_cnt, i);
hit_group_ids[i] = hit_items[i].group_id;
hit_items[i].group_id, super_group_ids, super_group_cnt,
vtable_id, state->scan_cnt, i);
}
/* update hit clause */
@@ -2091,17 +2124,6 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
return;
}
int g2g_table_id = table_manager_get_group2group_table_id(maat_inst->tbl_mgr);
void *g2g_rt = table_manager_get_runtime(maat_inst->tbl_mgr, g2g_table_id);
long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids,
hit_cnt, super_group_ids,
MAX_SCANNER_HIT_GROUP_NUM);
if (super_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
super_group_cnt = MAX_SCANNER_HIT_GROUP_NUM;
}
for (int j = 0; j < super_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
super_group_ids[j], vtable_id);
@@ -2113,11 +2135,12 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
}
}
#define MAX_HIT_PATH_CNT 10
size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state,
struct group2group_runtime *g2g_rt,
enum maat_list_type type,
struct maat_hit_group *hit_group_array,
size_t array_size)
size_t array_size, struct log_handle *logger)
{
if (NULL == compile_state) {
return 0;
@@ -2138,19 +2161,22 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
}
for (i = 0; i < utarray_len(tmp_hit_path); i++) {
internal_path = (struct maat_internal_hit_path *)utarray_eltptr(tmp_hit_path, i);
long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, &(internal_path->group_id), 1,
super_group_ids, MAX_SCANNER_HIT_GROUP_NUM);
if (super_group_cnt + 1 <= MAX_SCANNER_HIT_GROUP_NUM) {
super_group_ids[super_group_cnt++] = internal_path->group_id;
if (i > MAX_HIT_PATH_CNT) {
break;
}
for (size_t idx = 0; idx < super_group_cnt; idx++) {
internal_path = (struct maat_internal_hit_path *)utarray_eltptr(tmp_hit_path, i);
if (internal_path->excced_max_group_flag == 1) {
log_error(logger, MODULE_COMPILE,
"[%s:%d]group_id:%lld has too much super group ids, exceed maxium:%d",
__FUNCTION__, __LINE__, internal_path->group_id, MAX_SUPER_GROUP_CNT);
internal_path->excced_max_group_flag = 0;
}
for (size_t idx = 0; idx < internal_path->n_all_group_ids; idx++) {
struct maat_hit_group hit_group;
hit_group.item_id = internal_path->item_id;
hit_group.group_id = super_group_ids[idx];
hit_group.group_id = internal_path->all_group_ids[idx];
hit_group.vtable_id = internal_path->vtable_id;
if (utarray_find(all_hit_groups, &hit_group, compare_hit_group)) {
continue;

1
src/maat_kv.c
View File

@@ -129,7 +129,6 @@ int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key
strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase));
HASH_FIND(hh, store->hash, key_lowercase, key_len, kv);
int i = 0;
if (kv) {
*value = kv->val;
return 1;

12
test/maat_framework_gtest.cpp
View File

@@ -6323,9 +6323,9 @@ TEST_F(MaatCmdTest, GroupInMassCompiles) {
}
TEST_F(MaatCmdTest, HitGroup) {
const char *g2g_table_name = "GROUP2GROUP";
const char *g2c_table_name = "GROUP2COMPILE";
const char *compile_table_name = "COMPILE";
const char *g2c_table_name = "GROUP2COMPILE";
const char *g2g_table_name = "GROUP2GROUP";
const char *http_sig_table_name = "HTTP_SIGNATURE";
const char *ip_table_name = "IP_CONFIG";
const char *keywords_table_name = "KEYWORDS_TABLE";
@@ -6480,6 +6480,7 @@ TEST_F(MaatCmdTest, HitGroup) {
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
EXPECT_EQ(n_hit_group, 4);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
EXPECT_EQ(hit_groups[0].group_id, group1_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id);
@@ -6499,6 +6500,7 @@ TEST_F(MaatCmdTest, HitGroup) {
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
EXPECT_EQ(n_hit_group, 2);
EXPECT_EQ(hit_groups[0].item_id, item2_id);
EXPECT_EQ(hit_groups[0].group_id, group21_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_res_table_id);
@@ -6507,7 +6509,6 @@ TEST_F(MaatCmdTest, HitGroup) {
EXPECT_EQ(hit_groups[1].group_id, group2_id);
EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id);
const char* keywords1="In graph theory, hit group item forth";
const char *keywords2="To test one group hit group item fifth";
@@ -6523,6 +6524,7 @@ TEST_F(MaatCmdTest, HitGroup) {
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
EXPECT_EQ(n_hit_group, 5);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
EXPECT_EQ(hit_groups[0].group_id, group1_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id);
@@ -6557,6 +6559,7 @@ TEST_F(MaatCmdTest, HitGroup) {
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
EXPECT_EQ(n_hit_group, 6);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
EXPECT_EQ(hit_groups[0].group_id, group1_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id);
@@ -6584,6 +6587,7 @@ TEST_F(MaatCmdTest, HitGroup) {
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
EXPECT_EQ(n_hit_group, 2);
EXPECT_EQ(hit_groups[0].item_id, item3_id);
EXPECT_EQ(hit_groups[0].group_id, group3_id);
EXPECT_EQ(hit_groups[0].vtable_id, 0); //physical table(keywords_table) vtable_id is 0
@@ -6602,6 +6606,7 @@ TEST_F(MaatCmdTest, HitGroup) {
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128);
EXPECT_EQ(n_hit_group, 8);
EXPECT_EQ(hit_groups[0].item_id, item1_id);
EXPECT_EQ(hit_groups[0].group_id, group1_id);
EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id);
@@ -6638,6 +6643,7 @@ TEST_F(MaatCmdTest, HitGroup) {
memset(hit_groups, 0, sizeof(hit_groups));
n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128);
EXPECT_EQ(n_hit_group, 2);
EXPECT_EQ(hit_groups[0].item_id, item5_id);
EXPECT_EQ(hit_groups[0].group_id, group1_id);
EXPECT_EQ(hit_groups[0].vtable_id, 0); //physical table(keywords_table) vtable_id is 0