gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[BUGFIX]maat_scan_group support virtual table statistics

Browse Source
This commit is contained in:
liuwentan
2024-04-09 09:57:11 +08:00
parent 7e25f48fdd
commit 1b97f76bf5
8 changed files with 118 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/inc_internal/maat_table.h
View File

@@ -77,7 +77,8 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_
int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name,
long long *table_ids_array, size_t n_table_ids_array);
const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id);
const char *table_manager_get_table_name(struct table_manager *tbl_mgr,
int table_id);
const char *table_manager_get_table_schema_tag(struct table_manager *tbl_mgr, int table_id);

4
src/inc_internal/maat_virtual.h
View File

@@ -49,8 +49,8 @@ void virtual_runtime_hit_times_inc(struct virtual_runtime *virt_rt,
long long virtual_runtime_hit_times(void *virtual_runtime);
void virtual_runtime_hit_item_num_add(void *virtual_runtime, int thread_id,
long long val);
void virtual_runtime_hit_item_num_add(struct virtual_runtime *virt_rt,
int thread_id, long long val);
long long virtual_runtime_hit_item_num(void *virtual_runtime);

40
src/maat_api.c
View File

@@ -1074,7 +1074,8 @@ flag_scan(struct table_manager *tbl_mgr, int thread_id, long long flag,
if (virt_rt != NULL) {
//Note: group_hit_cnt is equivalent to item_hit_cnt
virtual_runtime_hit_item_num_add(virt_rt, thread_id, group_hit_cnt);
virtual_runtime_hit_item_num_add((struct virtual_runtime *)virt_rt,
thread_id, group_hit_cnt);
virtual_runtime_hit_times_inc((struct virtual_runtime *)virt_rt, thread_id);
}
@@ -1125,7 +1126,8 @@ interval_scan(struct table_manager *tbl_mgr, int thread_id, long long integer,
if (virt_rt != NULL) {
//Note: group_hit_cnt is equivalent to item_hit_cnt
virtual_runtime_hit_item_num_add(virt_rt, thread_id, group_hit_cnt);
virtual_runtime_hit_item_num_add((struct virtual_runtime *)virt_rt,
thread_id, group_hit_cnt);
virtual_runtime_hit_times_inc((struct virtual_runtime *)virt_rt, thread_id);
}
@@ -1170,7 +1172,8 @@ ipv4_scan(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_addr,
if (virt_rt != NULL) {
//Note: group_hit_cnt is equivalent to item_hit_cnt
virtual_runtime_hit_item_num_add(virt_rt, thread_id, group_hit_cnt);
virtual_runtime_hit_item_num_add((struct virtual_runtime *)virt_rt,
thread_id, group_hit_cnt);
virtual_runtime_hit_times_inc((struct virtual_runtime *)virt_rt, thread_id);
}
@@ -1215,7 +1218,8 @@ ipv6_scan(struct table_manager *tbl_mgr, int thread_id, uint8_t *ip_addr,
if (virt_rt != NULL) {
//Note: group_hit_cnt is equivalent to item_hit_cnt
virtual_runtime_hit_item_num_add(virt_rt, thread_id, group_hit_cnt);
virtual_runtime_hit_item_num_add((struct virtual_runtime *)virt_rt,
thread_id, group_hit_cnt);
virtual_runtime_hit_times_inc((struct virtual_runtime *)virt_rt, thread_id);
}
@@ -1270,7 +1274,8 @@ string_scan(struct table_manager *tbl_mgr, int thread_id,
if (virt_rt != NULL) {
//Note: group_hit_cnt is equivalent to item_hit_cnt
virtual_runtime_hit_item_num_add(virt_rt, thread_id, group_hit_cnt);
virtual_runtime_hit_item_num_add((struct virtual_runtime *)virt_rt,
thread_id, group_hit_cnt);
virtual_runtime_hit_times_inc((struct virtual_runtime *)virt_rt, thread_id);
}
@@ -1329,7 +1334,6 @@ int maat_scan_flag(struct maat *maat_inst, int table_id,
table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
phy_table_id = vtable_get_physical_table_id(maat_inst->tbl_mgr, table_id);
vtable_id = table_id;
}
if (phy_table_id < 0) {
@@ -1414,7 +1418,6 @@ int maat_scan_integer(struct maat *maat_inst, int table_id,
table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
phy_table_id = vtable_get_physical_table_id(maat_inst->tbl_mgr, table_id);
vtable_id = table_id;
}
if (phy_table_id < 0) {
@@ -1500,7 +1503,6 @@ int maat_scan_ipv4_port(struct maat *maat_inst, int table_id, uint32_t ip_addr,
table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
phy_table_id = vtable_get_physical_table_id(maat_inst->tbl_mgr, table_id);
vtable_id = table_id;
}
if (phy_table_id < 0) {
@@ -1585,7 +1587,6 @@ int maat_scan_ipv6_port(struct maat *maat_inst, int table_id, uint8_t *ip_addr,
table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
phy_table_id = vtable_get_physical_table_id(maat_inst->tbl_mgr, table_id);
vtable_id = table_id;
}
if (phy_table_id < 0) {
@@ -1689,7 +1690,6 @@ int maat_scan_string(struct maat *maat_inst, int table_id,
table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
phy_table_id = vtable_get_physical_table_id(maat_inst->tbl_mgr, table_id);
vtable_id = table_id;
}
if (phy_table_id < 0) {
@@ -1815,6 +1815,22 @@ int maat_scan_group(struct maat *maat_inst, int table_id,
return MAAT_SCAN_OK;
}
int vtable_id = table_id;
enum table_type table_type = TABLE_TYPE_INVALID;
table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
void *virt_rt = table_manager_get_runtime(maat_inst->tbl_mgr, vtable_id);
if (virt_rt != NULL) {
virtual_runtime_scan_times_inc((struct virtual_runtime *)virt_rt,
state->thread_id);
virtual_runtime_hit_times_inc((struct virtual_runtime *)virt_rt,
state->thread_id);
virtual_runtime_hit_item_num_add((struct virtual_runtime *)virt_rt,
state->thread_id, n_group);
}
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
@@ -1890,7 +1906,6 @@ struct maat_stream *maat_stream_new(struct maat *maat_inst, int table_id,
if (table_type == TABLE_TYPE_VIRTUAL) {
stream->phy_table_id = vtable_get_physical_table_id(maat_inst->tbl_mgr,
table_id);
stream->vtable_id = table_id;
}
if (stream->phy_table_id < 0) {
@@ -1972,7 +1987,8 @@ static int expr_stream_scan(struct maat_stream *stream, const char *data,
if (virt_rt != NULL) {
//Note: group_hit_cnt is equivalent to item_hit_cnt
virtual_runtime_hit_item_num_add(virt_rt, stream->thread_id, group_hit_cnt);
virtual_runtime_hit_item_num_add((struct virtual_runtime *)virt_rt,
stream->thread_id, group_hit_cnt);
virtual_runtime_hit_times_inc((struct virtual_runtime *)virt_rt, stream->thread_id);
}

45
src/maat_group.c
View File

@@ -777,6 +777,7 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
int ret = 0;
size_t i = 0;
int err_flag = 0;
long long *sub_group_id = NULL;
if (0 == is_valid) {
//delete
@@ -784,10 +785,8 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
sub_group_id = (long long *)utarray_eltptr(g2g_item->incl_sub_group_ids, i);
ret = group_topology_del_group_from_group(g2g_rt->updating_group_topo,
g2g_item->group_id, *sub_group_id, 0);
if (0 == ret) {
g2g_rt->rule_num--;
} else {
g2g_rt->update_err_cnt++;
if (ret != 0) {
err_flag = 1;
}
}
@@ -795,23 +794,27 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
sub_group_id = (long long *)utarray_eltptr(g2g_item->excl_sub_group_ids, i);
ret = group_topology_del_group_from_group(g2g_rt->updating_group_topo,
g2g_item->group_id, *sub_group_id, 1);
if (0 == ret) {
g2g_rt->rule_num--;
g2g_rt->excl_rule_num--;
} else {
g2g_rt->update_err_cnt++;
if (ret != 0) {
err_flag = 1;
}
}
if (1 == err_flag) {
g2g_rt->update_err_cnt++;
} else {
if (utarray_len(g2g_item->excl_sub_group_ids) > 0) {
g2g_rt->excl_rule_num--;
}
g2g_rt->rule_num--;
}
} else {
//add
for (i = 0; i < utarray_len(g2g_item->incl_sub_group_ids); i++) {
sub_group_id = (long long *)utarray_eltptr(g2g_item->incl_sub_group_ids, i);
ret = group_topology_add_group_to_group(g2g_rt->updating_group_topo,
g2g_item->group_id, *sub_group_id, 0);
if (0 == ret) {
g2g_rt->rule_num++;
} else {
g2g_rt->update_err_cnt++;
if (ret != 0) {
err_flag = 1;
}
}
@@ -819,13 +822,19 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
sub_group_id = (long long *)utarray_eltptr(g2g_item->excl_sub_group_ids, i);
ret = group_topology_add_group_to_group(g2g_rt->updating_group_topo,
g2g_item->group_id, *sub_group_id, 1);
if (0 == ret) {
g2g_rt->rule_num++;
g2g_rt->excl_rule_num++;
} else {
g2g_rt->update_err_cnt++;
if (ret != 0) {
err_flag = 1;
}
}
if (1 == err_flag) {
g2g_rt->update_err_cnt++;
} else {
if (utarray_len(g2g_item->excl_sub_group_ids) > 0) {
g2g_rt->excl_rule_num++;
}
g2g_rt->rule_num++;
}
}
group2group_item_free(g2g_item);

15
src/maat_stat.c
View File

@@ -241,8 +241,7 @@ static void fs_table_row_output(FILE *fp, struct maat_stat *stat, int perf_on)
long long plugin_cache_num = 0, plugin_rule_num = 0;
long long total_rule_num = 0, total_scan_bytes = 0, total_update_err = 0;
long long total_scan_times = 0, total_hit_times = 0, total_scan_cpu_time = 0;
long long total_regex_num = 0, total_ipv6_num = 0;
long long total_hit_item_num = 0, total_hit_pattern_num = 0;
long long total_regv6_num = 0, total_hit_item_num = 0, total_hit_pattern_num = 0;
long long g2c_not_clause_num = 0, g2g_excl_rule_num = 0;
struct fieldstat_tag cell_tag = {
.key = "TBL",
@@ -319,18 +318,18 @@ static void fs_table_row_output(FILE *fp, struct maat_stat *stat, int perf_on)
fieldstat_easy_counter_set(stat->fs_handle, 0,
stat->fs_column_id[COLUMN_HIT_ITEM_NUM],
&cell_tag, 1, hit_item_num);
if (table_type != TABLE_TYPE_VIRTUAL) {
total_regex_num += regex_rule_num;
total_ipv6_num += ipv6_rule_num;
total_regv6_num += regex_rule_num;
total_regv6_num += ipv6_rule_num;
total_rule_num += rule_num;
}
total_scan_times += scan_times;
total_hit_times += hit_times;
total_scan_bytes += scan_bytes;
total_hit_item_num += hit_item_num;
total_update_err +=
table_manager_runtime_update_err_count(stat->ref_tbl_mgr, i);
}
if (table_type == TABLE_TYPE_EXPR ||
table_type == TABLE_TYPE_EXPR_PLUS) {
@@ -375,7 +374,7 @@ static void fs_table_row_output(FILE *fp, struct maat_stat *stat, int perf_on)
&cell_tag, 1, total_rule_num);
fieldstat_easy_counter_set(stat->fs_handle, 0,
stat->fs_column_id[COLUMN_REGEX_NUM],
&cell_tag, 1, total_regex_num);
&cell_tag, 1, total_regv6_num);
fieldstat_easy_counter_set(stat->fs_handle, 0,
stat->fs_column_id[COLUMN_SCAN_TIMES],
&cell_tag, 1, total_scan_times);
@@ -434,7 +433,7 @@ static void fs_table_row_output(FILE *fp, struct maat_stat *stat, int perf_on)
void maat_stat_output(struct maat_stat *stat, struct table_manager *tbl_mgr,
long long maat_version, int perf_on)
{
if (NULL == stat) {
if (NULL == stat || NULL == stat->fs_handle) {
return;
}

7
src/maat_virtual.c
View File

@@ -222,14 +222,13 @@ long long virtual_runtime_hit_times(void *virtual_runtime)
return sum;
}
void virtual_runtime_hit_item_num_add(void *virtual_runtime, int thread_id,
long long val)
void virtual_runtime_hit_item_num_add(struct virtual_runtime *virt_rt,
int thread_id, long long val)
{
if (NULL == virtual_runtime) {
if (NULL == virt_rt || thread_id < 0) {
return;
}
struct virtual_runtime *virt_rt = (struct virtual_runtime *)virtual_runtime;
alignment_int64_array_add(virt_rt->hit_item_num, thread_id, val);
}

29
test/maat_framework_gtest.cpp
View File

@@ -3180,7 +3180,7 @@ protected:
struct maat *MaatGroupScan::_shared_maat_inst;
struct log_handle *MaatGroupScan::logger;
TEST_F(MaatGroupScan, basic) {
TEST_F(MaatGroupScan, PhysicalTable) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
@@ -3202,6 +3202,32 @@ TEST_F(MaatGroupScan, basic) {
maat_state_free(state);
state = NULL;
sleep(2);
}
TEST_F(MaatGroupScan, VirtualTable) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
const char *table_name = "HTTP_RESPONSE_KEYWORDS";
struct maat *maat_inst = MaatGroupScan::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GE(table_id, 0);
struct maat_hit_group hit_group;
hit_group.group_id = 259;
hit_group.vtable_id = table_id;
int ret = maat_scan_group(maat_inst, table_id, &hit_group, 1, results,
ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 233);
maat_state_free(state);
state = NULL;
sleep(2);
}
TEST_F(MaatGroupScan, SetScanCompileTable) {
@@ -3232,6 +3258,7 @@ TEST_F(MaatGroupScan, SetScanCompileTable) {
maat_state_free(state);
state = NULL;
sleep(2);
}
class NOTLogic : public testing::Test

16
test/maat_json.json
View File

@@ -4084,6 +4084,22 @@
]
}
]
},
{
"compile_id": 233,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "maat_scan_group",
"is_valid": "yes",
"groups": [
{
"virtual_table": "HTTP_RESPONSE_KEYWORDS",
"group_name": "233_url_group",
"group_id": 259
}
]
}
],
"plugin_table": [