gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FEATURE]support physical NOT clause => TSG-17286

Browse Source
This commit is contained in:
刘文坛
2023-10-24 08:21:41 +00:00
parent bf05309f2a
commit d24d6c1a5f
13 changed files with 123 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/inc_internal/maat_compile.h
View File

@@ -104,7 +104,7 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state);
void maat_compile_state_free(struct maat_compile_state *compile_state,
struct maat *maat_instance, int thread_id);
int maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
int maat_compile_state_update(int phy_table_id, int vtable_id, struct maat_item *hit_items,
size_t n_hit_item, struct maat_state *state);
size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *compile_state,

5
src/inc_internal/maat_expr.h
View File

@@ -50,12 +50,13 @@ long long expr_runtime_get_version(void *expr_runtime);
* @retval the num of hit group_id
*/
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int vtable_id, struct maat_state *state);
size_t data_len, int phy_table_id, int vtable_id, struct maat_state *state);
struct expr_matcher_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct expr_matcher_stream *s_handle,
const char *data, size_t data_len, int vtable_id, struct maat_state *state);
const char *data, size_t data_len, int phy_table_id, int vtable_id,
struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,
struct expr_matcher_stream *stream);

2
src/inc_internal/maat_flag.h
View File

@@ -48,7 +48,7 @@ long long flag_runtime_rule_count(void *flag_runtime);
* @retval the num of hit group_id
*/
int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, long long flag,
int vtable_id, struct maat_state *state);
int phy_table_id, int vtable_id, struct maat_state *state);
int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *district,
size_t district_len, long long *district_id);

3
src/inc_internal/maat_interval.h
View File

@@ -47,7 +47,8 @@ long long interval_runtime_rule_count(void *interval_runtime);
* @retval the num of hit group_id
*/
int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
long long integer, int vtable_id, struct maat_state *state);
long long integer, int phy_table_id, int vtable_id,
struct maat_state *state);
int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, const char *district,
size_t district_len, long long *district_id);

4
src/inc_internal/maat_ip.h
View File

@@ -42,8 +42,8 @@ long long ip_runtime_ipv6_rule_count(void *ip_runtime);
/* ip runtime scan API */
int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, uint16_t port, int proto, int vtable_id,
struct maat_state *state);
uint8_t *ip_addr, uint16_t port, int proto, int phy_table_id,
int vtable_id, struct maat_state *state);
void ip_runtime_hit_inc(struct ip_runtime *ip_rt, int thread_id);

17
src/maat_api.c
View File

@@ -1043,7 +1043,7 @@ static int flag_scan(struct table_manager *tbl_mgr, int thread_id, long long fla
}
int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id,
flag, vtable_id, state);
flag, phy_table_id, vtable_id, state);
if (group_hit_cnt <= 0) {
return group_hit_cnt;
}
@@ -1073,7 +1073,8 @@ static int interval_scan(struct table_manager *tbl_mgr, int thread_id, long long
}
int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt,
thread_id, integer, vtable_id, state);
thread_id, integer, phy_table_id,
vtable_id, state);
if (group_hit_cnt <= 0) {
return group_hit_cnt;
}
@@ -1099,8 +1100,8 @@ static int ipv4_scan(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_a
}
int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4,
(uint8_t *)&ip_addr, port, proto, vtable_id,
state);
(uint8_t *)&ip_addr, port, proto, phy_table_id,
vtable_id, state);
if (group_hit_cnt <= 0) {
return group_hit_cnt;
}
@@ -1126,7 +1127,8 @@ static int ipv6_scan(struct table_manager *tbl_mgr, int thread_id,
}
int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6,
ip_addr, port, proto, vtable_id, state);
ip_addr, port, proto, phy_table_id, vtable_id,
state);
if (group_hit_cnt <= 0) {
return group_hit_cnt;
}
@@ -1157,7 +1159,7 @@ static int string_scan(struct table_manager *tbl_mgr, int thread_id,
int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt,
thread_id, data, data_len,
vtable_id, state);
phy_table_id, vtable_id, state);
if (group_hit_cnt <= 0) {
return group_hit_cnt;
}
@@ -1193,7 +1195,8 @@ static int expr_stream_scan(struct maat_stream *stream, const char *data,
int group_hit_cnt = expr_runtime_stream_scan((struct expr_runtime *)expr_rt,
stream->handle, data, data_len,
stream->vtable_id, state);
stream->phy_table_id, stream->vtable_id,
state);
if (group_hit_cnt <= 0) {
return group_hit_cnt;
}

48
src/maat_compile.c
View File

@@ -91,8 +91,8 @@ struct literal_clause {
UT_hash_handle hh;
};
struct vtable_not_clause {
int vtable_id;
struct table_not_clause {
int table_id;
int not_clause_num;
UT_hash_handle hh;
};
@@ -118,7 +118,7 @@ struct group2compile_runtime {
long long rule_num;
long long update_err_cnt;
struct compile_runtime *ref_compile_rt;
struct vtable_not_clause *not_clause_hash;
struct table_not_clause *not_clause_hash;
};
struct maat_clause {
@@ -728,7 +728,7 @@ void group2compile_runtime_free(void *g2c_runtime)
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
if (g2c_rt->not_clause_hash != NULL) {
struct vtable_not_clause *not_clause = NULL, *tmp_not_clause = NULL;
struct table_not_clause *not_clause = NULL, *tmp_not_clause = NULL;
HASH_ITER(hh, g2c_rt->not_clause_hash, not_clause, tmp_not_clause) {
HASH_DEL(g2c_rt->not_clause_hash, not_clause);
FREE(not_clause);
@@ -1949,20 +1949,12 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
return 0;
}
int validate_vtable_not_clause(struct group2compile_runtime *g2c_rt,
struct table_manager *tbl_mgr, int vtable_id,
static int validate_table_not_clause(struct group2compile_runtime *g2c_rt,
struct table_manager *tbl_mgr, int table_id,
int is_valid, struct log_handle *logger)
{
enum table_type table_type = table_manager_get_table_type(tbl_mgr, vtable_id);
if (table_type != TABLE_TYPE_VIRTUAL) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]table(table_id:%d) is not virtual table, can't own NOT clause.",
__FUNCTION__, __LINE__, vtable_id);
return -1;
}
struct vtable_not_clause *not_clause = NULL;
HASH_FIND_INT(g2c_rt->not_clause_hash, &vtable_id, not_clause);
struct table_not_clause *not_clause = NULL;
HASH_FIND_INT(g2c_rt->not_clause_hash, &table_id, not_clause);
if (0 == is_valid) {
//delete
@@ -1974,15 +1966,15 @@ int validate_vtable_not_clause(struct group2compile_runtime *g2c_rt,
} else {
//add
if (NULL == not_clause) {
not_clause = ALLOC(struct vtable_not_clause, 1);
not_clause->vtable_id = vtable_id;
not_clause = ALLOC(struct table_not_clause, 1);
not_clause->table_id = table_id;
not_clause->not_clause_num++;
HASH_ADD_INT(g2c_rt->not_clause_hash, vtable_id, not_clause);
HASH_ADD_INT(g2c_rt->not_clause_hash, table_id, not_clause);
} else {
if (not_clause->not_clause_num >= MAX_NOT_CLAUSE_NUM) {
const char *table_name = table_manager_get_table_name(tbl_mgr, vtable_id);
const char *table_name = table_manager_get_table_name(tbl_mgr, table_id);
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]virtual table:<%s> NOT clause num exceed maximum:%d",
"[%s:%d]table:<%s> NOT clause num exceed maximum:%d",
__FUNCTION__, __LINE__, table_name, MAX_NOT_CLAUSE_NUM);
return -1;
}
@@ -2024,7 +2016,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
}
if (1 == g2c_item->not_flag) {
ret = validate_vtable_not_clause(g2c_rt, schema->ref_tbl_mgr,
ret = validate_table_not_clause(g2c_rt, schema->ref_tbl_mgr,
g2c_item->vtable_id, is_valid,
compile_rt->logger);
if (ret < 0) {
@@ -2251,7 +2243,7 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
return MIN(bool_match_ret, compile_ids_size);
}
int maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
int maat_compile_state_update(int phy_table_id, int vtable_id, struct maat_item *hit_items,
size_t n_hit_item, struct maat_state *state)
{
size_t i = 0, j = 0;
@@ -2266,12 +2258,8 @@ int maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
}
struct maat_compile_state *compile_state = state->compile_state;
if (hit_cnt > 0) {
if (compile_state->Nth_scan != state->scan_cnt) {
compile_state->Nth_scan = state->scan_cnt;
utarray_clear(compile_state->this_scan_hit_clauses);
}
}
compile_state->Nth_scan = state->scan_cnt;
for (i = 0; i < hit_cnt; i++) {
hit_group_ids[i] = hit_items[i].group_id;
@@ -2319,6 +2307,10 @@ int maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
hit_group_ids[i], vtable_id);
}
if (vtable_id == 0) {
vtable_id = phy_table_id;
}
long long hit_NOT_group_ids[VTABLE_MAX_NOT_GROUP_NUM];
size_t hit_not_cnt = maat_compile_state_update_hit_not_clauses(state->compile_state, compile_rt,
hit_group_ids, hit_cnt, vtable_id,

11
src/maat_expr.c
View File

@@ -955,8 +955,8 @@ long long expr_runtime_get_version(void *expr_runtime)
}
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len, int vtable_id,
struct maat_state *state)
const char *data, size_t data_len, int phy_table_id,
int vtable_id, struct maat_state *state)
{
if (0 == expr_rt->rule_num) {
//empty expr table
@@ -1002,7 +1002,7 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
}
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
struct expr_matcher_stream *
@@ -1024,7 +1024,8 @@ expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
struct expr_matcher_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state)
int phy_table_id, int vtable_id,
struct maat_state *state)
{
if (0 == expr_rt->rule_num) {
//empty expr table
@@ -1064,7 +1065,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
}
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,

5
src/maat_flag.c
View File

@@ -544,7 +544,8 @@ long long flag_runtime_rule_count(void *flag_runtime)
}
int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
long long flag, int vtable_id, struct maat_state *state)
long long flag, int phy_table_id, int vtable_id,
struct maat_state *state)
{
if (0 == flag_rt->rule_num) {
//empty flag table
@@ -588,7 +589,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
}
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
void flag_runtime_hit_inc(struct flag_runtime *flag_rt, int thread_id)

5
src/maat_interval.c
View File

@@ -545,7 +545,8 @@ long long interval_runtime_rule_count(void *interval_runtime)
}
int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
long long integer, int vtable_id, struct maat_state *state)
long long integer, int phy_table_id, int vtable_id,
struct maat_state *state)
{
if (0 == interval_rt->rule_num) {
//empty interval table
@@ -589,7 +590,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
}
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
void interval_runtime_hit_inc(struct interval_runtime *interval_rt, int thread_id)

6
src/maat_ip.c
View File

@@ -700,8 +700,8 @@ static int validate_port_proto(struct ip_item *item, uint16_t port, int proto)
}
int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, uint16_t port, int proto, int vtable_id,
struct maat_state *state)
uint8_t *ip_addr, uint16_t port, int proto, int phy_table_id,
int vtable_id, struct maat_state *state)
{
if (0 == ip_rt->rule_num) {
//empty ip table
@@ -761,7 +761,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
}
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
void ip_runtime_hit_inc(struct ip_runtime *ip_rt, int thread_id)

39
test/maat_framework_gtest.cpp
View File

@@ -1064,7 +1064,7 @@ TEST_F(MaatHsStringScan, HexBinCaseSensitive) {
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
maat_state_reset(state);
ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2),
@@ -1793,7 +1793,7 @@ TEST_F(MaatRsStringScan, HexBinCaseSensitive) {
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
maat_state_reset(state);
ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2),
@@ -3221,45 +3221,42 @@ TEST_F(NOTLogic, NotUrlAndNotIp) {
state = NULL;
}
#if 0
TEST_F(NOTLogic, MultiNotClause) {
const char *string1 = "This string ONLY contains not_logic_compile_223_1.";
const char *string2 = "This string ONLY contains not_logic_compile_223_2.";
const char *string3 = "This string ONLY contains not_logic_compile_223_3.";
const char *string4 = "This string ONLY contains nothing";
TEST_F(NOTLogic, NotPhysicalTable) {
const char *string1 = "This string ONLY contains not_logic_compile_224_1.";
const char *string2 = "This string ONLY contains not_logic_compile_224_2.";
const char *string3 = "This string ONLY contains nothing.";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
const char *table_name = "HTTP_NOT_LOGIC";
const char *phy_table_name = "KEYWORDS_TABLE";
const char *vtable_name = "HTTP_RESPONSE_KEYWORDS";
struct maat *maat_inst = NOTLogic::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
int phy_table_id = maat_get_table_id(maat_inst, phy_table_name);
ASSERT_GT(phy_table_id, 0);
int ret = maat_scan_string(maat_inst, table_id, string1,
int vtable_id = maat_get_table_id(maat_inst, vtable_name);
ASSERT_GT(vtable_id, 0);
int ret = maat_scan_string(maat_inst, phy_table_id, string1,
strlen(string1), results, ARRAY_SIZE,
&n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
ret = maat_scan_string(maat_inst, table_id, string2, strlen(string2),
ret = maat_scan_string(maat_inst, vtable_id, string2, strlen(string2),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
ret = maat_scan_string(maat_inst, table_id, string3, strlen(string3),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
ret = maat_scan_string(maat_inst, table_id, string4, strlen(string4),
ret = maat_scan_string(maat_inst, phy_table_id, string3, strlen(string3),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 223);
EXPECT_EQ(results[0], 224);
maat_state_free(state);
state = NULL;
}
#endif
TEST_F(NOTLogic, EightNotClause) {
const char *string_nothing = "This string contain nothing";
@@ -3604,7 +3601,7 @@ TEST_F(ExcludeLogic, ScanIrrelavantAtLast) {
ret = maat_scan_string(maat_inst, not_hit_table_id, string_irrelevant,
strlen(string_irrelevant), results, ARRAY_SIZE,
&n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
maat_state_free(state);
state = NULL;

47
test/maat_json.json
View File

@@ -3747,6 +3747,53 @@
]
}
]
},
{
"compile_id": 224,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"user_region": "NOTLogic.NotPhysicalTable",
"is_valid": "yes",
"groups": [
{
"group_name": "NOTLogicGroup_224_1",
"virtual_table": "KEYWORDS_TABLE",
"not_flag": 1,
"clause_index": 0,
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "expr",
"table_content": {
"keywords": "not_logic_compile_224_1",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"group_name": "NOTLogicGroup_224_2",
"virtual_table": "HTTP_RESPONSE_KEYWORDS",
"not_flag": 0,
"clause_index": 1,
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "expr",
"table_content": {
"keywords": "not_logic_compile_224_2",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
],
"plugin_table": [