gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

table_info.conf support table_name & db_tables to implement all physical tables conjunction

Browse Source
This commit is contained in:
liuwentan
2023-03-22 20:40:36 +08:00
parent 6fef5354fa
commit a67d24381e
10 changed files with 148 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
include/maat.h
View File

@@ -214,8 +214,8 @@ int maat_state_set_scan_district(struct maat *instance, struct maat_state **stat
int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state);
int maat_state_set_scan_compile_tables(struct maat *maat_instance, struct maat_state **state,
const char *compile_table[], size_t n_table);
int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state,
int compile_table_id);
int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state,
struct maat_hit_path *paths, size_t n_path);

2
src/inc_internal/maat_compile.h
View File

@@ -57,7 +57,7 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
int compile_runtime_commit(void *compile_runtime, const char *table_name);
int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile_ids,
int ids_index, size_t compile_ids_size, struct maat_state *state);
size_t compile_ids_size, struct maat_state *state);
size_t compile_runtime_get_new_hit_paths(struct compile_runtime *compile_rt,
struct maat_compile_state *compile_state,

3
src/inc_internal/maat_rule.h
View File

@@ -235,8 +235,7 @@ struct maat {
struct maat_state {
struct maat *maat_instance;
int16_t thread_id;
size_t n_compile_table;
char compile_tables[MAX_COMPILE_TABLE_NUM][NAME_MAX]; //caller can select compile table to scan
int compile_table_id;
unsigned char is_set_district;
unsigned char is_last_scan;
long long district_id; //-1: Any District; -2: Unkonwn District;

80
src/maat_api.c
View File

@@ -936,7 +936,7 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
outer_state->maat_instance = maat_instance;
outer_state->district_id = DISTRICT_ANY;
outer_state->thread_id = (signed short)thread_id;
outer_state->n_compile_table = 0;
outer_state->compile_table_id = 0;
return outer_state;
}
@@ -978,11 +978,11 @@ static inline int scan_status_should_compile_NOT(struct maat_state *state)
return 0;
}
size_t hit_group_to_compile(void *compile_runtime, long long *compile_ids, int ids_index,
size_t compile_ids_size, struct maat_state *mid)
size_t hit_group_to_compile(void *compile_runtime, long long *compile_ids,
size_t compile_ids_size, struct maat_state *state)
{
size_t n_hit_compile = compile_runtime_match((struct compile_runtime *)compile_runtime,
compile_ids, ids_index, compile_ids_size, mid);
compile_ids, compile_ids_size, state);
return n_hit_compile;
}
@@ -1188,34 +1188,25 @@ int expr_stream_scan(struct maat_stream *stream, const char *data, size_t data_l
}
size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n_result,
struct maat_state *mid)
struct maat_state *state)
{
int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0};
size_t compile_table_cnt = 0;
size_t sum_hit_compile_cnt = 0;
int compile_table_id = -1;
if (0 == mid->n_compile_table) {
compile_table_id[0] = maat_instance->default_compile_table_id;
compile_table_cnt = 1;
if (state->compile_table_id != 0) {
compile_table_id = state->compile_table_id;
} else {
for (size_t i = 0; i < mid->n_compile_table; i++) {
compile_table_id[i] = maat_get_table_id(maat_instance, mid->compile_tables[i]);
}
compile_table_cnt = mid->n_compile_table;
compile_table_id = maat_instance->default_compile_table_id;
}
for (size_t i = 0; i < compile_table_cnt; i++) {
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]);
size_t n_hit_compile = hit_group_to_compile(compile_rt, results, sum_hit_compile_cnt, n_result, mid);
sum_hit_compile_cnt += n_hit_compile;
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, state);
assert(state->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == state->is_last_scan) {
state->is_last_scan = LAST_SCAN_FINISHED;
}
assert(mid->is_last_scan < LAST_SCAN_FINISHED);
if (LAST_SCAN_SET == mid->is_last_scan) {
mid->is_last_scan = LAST_SCAN_FINISHED;
}
return sum_hit_compile_cnt;
return n_hit_compile;
}
int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
@@ -1798,19 +1789,15 @@ int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **sta
return 0;
}
int maat_state_set_scan_compile_tables(struct maat *maat_instance, struct maat_state **state,
const char *compile_tables[], size_t n_table)
int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state,
int compile_table_id)
{
if (NULL == maat_instance->maat_rt) {
if (NULL == maat_instance->maat_rt || compile_table_id < 0) {
return -1;
}
struct maat_state *mid = grab_state(state, maat_instance, -1);
mid->n_compile_table = n_table;
for (size_t i = 0; i < mid->n_compile_table; i++) {
strncpy(mid->compile_tables[i], compile_tables[i], strlen(compile_tables[i]));
}
mid->compile_table_id = compile_table_id;
return 0;
}
@@ -1818,17 +1805,12 @@ int maat_state_set_scan_compile_tables(struct maat *maat_instance, struct maat_s
size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state,
struct maat_hit_path *paths, size_t n_path)
{
int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0};
size_t compile_table_cnt = 0;
int compile_table_id = -1;
if (0 == state->n_compile_table) {
compile_table_ids[0] = maat_instance->default_compile_table_id;
compile_table_cnt = 1;
if (state->compile_table_id != 0) {
compile_table_id = state->compile_table_id;
} else {
for (size_t i = 0; i < state->n_compile_table; i++) {
compile_table_ids[i] = maat_get_table_id(maat_instance, state->compile_tables[i]);
}
compile_table_cnt = state->n_compile_table;
compile_table_id = maat_instance->default_compile_table_id;
}
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
@@ -1837,14 +1819,14 @@ size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state,
(struct group2group_runtime *)g2g_runtime,
paths, n_path);
size_t new_hit_path_cnt = 0;
for (size_t i = 0; i < compile_table_cnt; i++) {
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]);
assert(NULL != compile_rt);
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
assert(NULL != compile_rt);
new_hit_path_cnt += compile_runtime_get_new_hit_paths((struct compile_runtime *)compile_rt,
state->compile_state, paths, n_path,
compile_state_hit_path_cnt + new_hit_path_cnt);
}
new_hit_path_cnt = compile_runtime_get_new_hit_paths((struct compile_runtime *)compile_rt,
state->compile_state, paths, n_path,
compile_state_hit_path_cnt);
return (compile_state_hit_path_cnt + new_hit_path_cnt);
}

44
src/maat_compile.c
View File

@@ -1709,7 +1709,7 @@ static int compare_compile_rule(const void *a, const void *b)
}
int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile_ids,
int ids_index, size_t compile_ids_size, struct maat_state *state)
size_t compile_ids_size, struct maat_state *state)
{
struct maat_compile_state *compile_state = state->compile_state;
int is_last_scan = state->is_last_scan;
@@ -1724,7 +1724,7 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
}
for (size_t i = 0; i < bool_match_ret; i++) {
compile_ids[ids_index + i] = compile_rules[i]->compile_id;
compile_ids[i] = compile_rules[i]->compile_id;
}
return MIN(bool_match_ret, compile_ids_size);
@@ -1768,30 +1768,28 @@ int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
*n_hit_group_id = hit_group_cnt;
/* update hit clause */
int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0};
size_t compile_table_cnt = 0;
if (0 == state->n_compile_table) {
compile_table_ids[0] = state->maat_instance->default_compile_table_id;
compile_table_cnt = 1;
int compile_table_id = -1;
if (state->compile_table_id != 0) {
compile_table_id = state->compile_table_id;
} else {
for (size_t i = 0; i < state->n_compile_table; i++) {
compile_table_ids[i] = maat_get_table_id(state->maat_instance, state->compile_tables[i]);
}
compile_table_cnt = state->n_compile_table;
compile_table_id = state->maat_instance->default_compile_table_id;
}
for (size_t idx = 0; idx < compile_table_cnt; idx++) {
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
compile_table_ids[idx]);
for (size_t i = 0; i < hit_group_cnt; i++) {
long long top_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
memset(top_group_ids, 0, sizeof(top_group_ids));
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &hit_group_ids[i],
1, top_group_ids);
for (int j = 0; j < top_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
top_group_ids[j], vtable_id);
}
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
compile_table_id);
assert(compile_rt != NULL);
for (size_t i = 0; i < hit_group_cnt; i++) {
long long top_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
memset(top_group_ids, 0, sizeof(top_group_ids));
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &hit_group_ids[i],
1, top_group_ids);
if (top_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
top_group_cnt = MAX_SCANNER_HIT_GROUP_NUM;
}
for (int j = 0; j < top_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
top_group_ids[j], vtable_id);
}
}

49
src/maat_table.c
View File

@@ -287,29 +287,29 @@ static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id
}
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "table_name");
if (NULL == item || (item->type != cJSON_String && item->type != cJSON_Array)) {
item = cJSON_GetObjectItem(json, "db_tables");
if (item != NULL && item->type != cJSON_Array) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) has no table name",
"[%s:%d] table(table_id:%d) has db_tables, but format is invalid, should be array",
__FUNCTION__, __LINE__, table_id);
return -1;
}
if (item->type == cJSON_Array) {
if (item != NULL) {
int n_table_name = cJSON_GetArraySize(item);
cJSON *tmp_item = NULL;
for (int i = 0; i < n_table_name; i++) {
tmp_item = cJSON_GetArrayItem(item, i);
cJSON *tmp_item = cJSON_GetArrayItem(item, i);
if (NULL == tmp_item || tmp_item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) table_name format invalid",
"[%s:%d] table(table_id:%d) db_tables element format invalid, should be string",
__FUNCTION__, __LINE__, table_id);
return -1;
}
if (strlen(tmp_item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) name %s length too long",
"[%s:%d] table(table_id:%d) db_tables element string %s length too long",
__FUNCTION__, __LINE__, table_id, tmp_item->valuestring);
return -1;
}
@@ -318,20 +318,27 @@ static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id
log_info(logger, MODULE_TABLE, "tablename[%s] -> table_id:[%d]",
tmp_item->valuestring, table_id);
}
} else {
//cJSON_String
if (strlen(item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) name %s length too long",
__FUNCTION__, __LINE__, table_id, item->valuestring);
return -1;
}
maat_kv_register(tablename2id_map, item->valuestring, table_id);
log_info(logger, MODULE_TABLE, "table_name[%s] -> table_id:[%d]",
item->valuestring, table_id);
}
item = cJSON_GetObjectItem(json, "table_name");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) has no table_name",
__FUNCTION__, __LINE__, table_id);
return -1;
}
if (strlen(item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) table_name %s length too long",
__FUNCTION__, __LINE__, table_id, item->valuestring);
return -1;
}
maat_kv_register(tablename2id_map, item->valuestring, table_id);
log_info(logger, MODULE_TABLE, "table_name[%s] -> table_id:[%d]",
item->valuestring, table_id);
return 0;
}

2
test/adapter_hs_gtest.cpp
View File

@@ -652,7 +652,7 @@ int main(int argc, char **argv)
{
int ret = 0;
::testing::InitGoogleTest(&argc, argv);
g_logger = log_handle_create("./tmp.log", 0);
g_logger = log_handle_create("./adapter_hs_gtest.log", 0);
ret = RUN_ALL_TESTS();

39
test/maat_framework_gtest.cpp
View File

@@ -539,6 +539,24 @@ protected:
struct maat *MaatStringScan::_shared_maat_instance;
struct log_handle *MaatStringScan::logger;
TEST_F(MaatStringScan, ScanDataOnlyOneByte) {
const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char scan_data = 0x20;
int ret = maat_scan_string(maat_instance, table_id, 0, &scan_data, sizeof(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state);
}
TEST_F(MaatStringScan, Full) {
const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
@@ -2370,13 +2388,11 @@ TEST_F(CompileTable, Conjunction1) {
struct maat_state *state = NULL;
const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL";
const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"};
struct maat *maat_instance = CompileTable::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
maat_state_set_scan_compile_tables(maat_instance, &state, compile_tables, 2);
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
@@ -2397,7 +2413,6 @@ TEST_F(CompileTable, Conjunction2) {
struct maat_state *state = NULL;
const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL";
const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"};
struct maat *maat_instance = CompileTable::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
@@ -2406,19 +2421,17 @@ TEST_F(CompileTable, Conjunction2) {
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 197);
EXPECT_EQ(results[1], 141);
struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0};
int n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE);
EXPECT_EQ(n_read, 2);
maat_state_set_scan_compile_tables(maat_instance, &state, compile_tables, 2);
ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 141);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
memset(hit_path, 0, sizeof(hit_path));
n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE);
@@ -2550,9 +2563,9 @@ TEST_F(Policy, CompileEXData) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *url = "firewall should hit";
const char *table_name = "HTTP_URL";
const char *compile_table_name = "COMPILE_ALIAS";
const char *compile_table_name = "COMPILE_FIREWALL";
const char *expect_name = "I have a name";
struct maat *maat_instance = Policy::_shared_maat_instance;
@@ -2568,14 +2581,14 @@ TEST_F(Policy, CompileEXData) {
ASSERT_TRUE(ret == 0);
EXPECT_EQ(ex_data_counter, 1);
ret = maat_state_set_scan_compile_tables(maat_instance, &state, &compile_table_name, 1);
ret = maat_state_set_scan_compile_table(maat_instance, &state, compile_table_id);
EXPECT_EQ(ret, 0);
ret = maat_scan_string(maat_instance, table_id, 0, url, strlen(url),
results, ARRAY_SIZE, &n_hit_result, &state);
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 141);
EXPECT_EQ(results[0], 198);
void *ex_data = maat_plugin_table_get_ex_data(maat_instance, compile_table_id,
(char *)&results[0]);

32
test/maat_json.json
View File

@@ -1,6 +1,6 @@
{
"compile_table": "COMPILE",
"group2compile_table": "GROUP2COMPILE",
"compile_table": "COMPILE_DEFAULT",
"group2compile_table": "GROUP2COMPILE_DEFAULT",
"group2group_table": "GROUP2GROUP",
"groups": [
{
@@ -2447,6 +2447,34 @@
]
}
]
},
{
"compile_id": 198,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "Something:I\\bhave\\ba\\bname,7799",
"compile_table_name": "COMPILE_FIREWALL",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"g2c_table_name": "GROUP2COMPILE_FIREWALL",
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "firewall",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
],
"plugin_table": [

12
test/table_info.conf
View File

@@ -2,6 +2,7 @@
{
"table_id":0,
"table_name":"COMPILE",
"db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"],
"table_type":"compile",
"valid_column":8,
"custom": {
@@ -14,6 +15,7 @@
{
"table_id":1,
"table_name":"GROUP2COMPILE",
"db_tables":["GROUP2COMPILE_DEFAULT", "GROUP2COMPILE_ALIAS"],
"table_type":"group2compile",
"associated_compile_table_id":0,
"valid_column":3,
@@ -27,9 +29,9 @@
},
{
"table_id":2,
"table_name":"COMPILE_ALIAS",
"table_name":"COMPILE_FIREWALL",
"table_type":"compile",
"valid_column":4,
"valid_column":8,
"custom": {
"compile_id":1,
"tags":6,
@@ -39,7 +41,7 @@
},
{
"table_id":3,
"table_name":"GROUP2COMPILE_ALIAS",
"table_name":"GROUP2COMPILE_FIREWALL",
"table_type":"group2compile",
"associated_compile_table_id":2,
"valid_column":3,
@@ -63,7 +65,8 @@
},
{
"table_id":5,
"table_name":["HTTP_URL", "HTTP_HOST"],
"table_name":"HTTP_REGION",
"db_tables":["HTTP_URL", "HTTP_HOST"],
"table_type":"expr",
"valid_column":7,
"custom": {
@@ -81,7 +84,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,