diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index de3b1e4..b5d895e 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -104,7 +104,7 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state); void maat_compile_state_free(struct maat_compile_state *compile_state, struct maat *maat_instance, int thread_id); -int maat_compile_state_update(int vtable_id, struct maat_item *hit_items, +int maat_compile_state_update(int phy_table_id, int vtable_id, struct maat_item *hit_items, size_t n_hit_item, struct maat_state *state); size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *compile_state, diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index d510d92..7b2a66e 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -50,12 +50,13 @@ long long expr_runtime_get_version(void *expr_runtime); * @retval the num of hit group_id */ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, - size_t data_len, int vtable_id, struct maat_state *state); + size_t data_len, int phy_table_id, int vtable_id, struct maat_state *state); struct expr_matcher_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct expr_matcher_stream *s_handle, - const char *data, size_t data_len, int vtable_id, struct maat_state *state); + const char *data, size_t data_len, int phy_table_id, int vtable_id, + struct maat_state *state); void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id, struct expr_matcher_stream *stream); diff --git a/src/inc_internal/maat_flag.h b/src/inc_internal/maat_flag.h index 99c088d..2811b24 100644 --- a/src/inc_internal/maat_flag.h +++ b/src/inc_internal/maat_flag.h @@ -48,7 +48,7 @@ long long flag_runtime_rule_count(void *flag_runtime); * @retval the num of hit group_id */ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, long long flag, - int vtable_id, struct maat_state *state); + int phy_table_id, int vtable_id, struct maat_state *state); int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *district, size_t district_len, long long *district_id); diff --git a/src/inc_internal/maat_interval.h b/src/inc_internal/maat_interval.h index 260b600..1f13d15 100644 --- a/src/inc_internal/maat_interval.h +++ b/src/inc_internal/maat_interval.h @@ -47,7 +47,8 @@ long long interval_runtime_rule_count(void *interval_runtime); * @retval the num of hit group_id */ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, - long long integer, int vtable_id, struct maat_state *state); + long long integer, int phy_table_id, int vtable_id, + struct maat_state *state); int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, const char *district, size_t district_len, long long *district_id); diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h index 725e393..05da8db 100644 --- a/src/inc_internal/maat_ip.h +++ b/src/inc_internal/maat_ip.h @@ -42,8 +42,8 @@ long long ip_runtime_ipv6_rule_count(void *ip_runtime); /* ip runtime scan API */ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, - uint8_t *ip_addr, uint16_t port, int proto, int vtable_id, - struct maat_state *state); + uint8_t *ip_addr, uint16_t port, int proto, int phy_table_id, + int vtable_id, struct maat_state *state); void ip_runtime_hit_inc(struct ip_runtime *ip_rt, int thread_id); diff --git a/src/maat_api.c b/src/maat_api.c index 933a7bb..703914e 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -1043,7 +1043,7 @@ static int flag_scan(struct table_manager *tbl_mgr, int thread_id, long long fla } int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id, - flag, vtable_id, state); + flag, phy_table_id, vtable_id, state); if (group_hit_cnt <= 0) { return group_hit_cnt; } @@ -1073,7 +1073,8 @@ static int interval_scan(struct table_manager *tbl_mgr, int thread_id, long long } int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt, - thread_id, integer, vtable_id, state); + thread_id, integer, phy_table_id, + vtable_id, state); if (group_hit_cnt <= 0) { return group_hit_cnt; } @@ -1099,8 +1100,8 @@ static int ipv4_scan(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_a } int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4, - (uint8_t *)&ip_addr, port, proto, vtable_id, - state); + (uint8_t *)&ip_addr, port, proto, phy_table_id, + vtable_id, state); if (group_hit_cnt <= 0) { return group_hit_cnt; } @@ -1126,7 +1127,8 @@ static int ipv6_scan(struct table_manager *tbl_mgr, int thread_id, } int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, - ip_addr, port, proto, vtable_id, state); + ip_addr, port, proto, phy_table_id, vtable_id, + state); if (group_hit_cnt <= 0) { return group_hit_cnt; } @@ -1157,7 +1159,7 @@ static int string_scan(struct table_manager *tbl_mgr, int thread_id, int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt, thread_id, data, data_len, - vtable_id, state); + phy_table_id, vtable_id, state); if (group_hit_cnt <= 0) { return group_hit_cnt; } @@ -1193,7 +1195,8 @@ static int expr_stream_scan(struct maat_stream *stream, const char *data, int group_hit_cnt = expr_runtime_stream_scan((struct expr_runtime *)expr_rt, stream->handle, data, data_len, - stream->vtable_id, state); + stream->phy_table_id, stream->vtable_id, + state); if (group_hit_cnt <= 0) { return group_hit_cnt; } diff --git a/src/maat_compile.c b/src/maat_compile.c index d573c7a..fd1dbb4 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -91,8 +91,8 @@ struct literal_clause { UT_hash_handle hh; }; -struct vtable_not_clause { - int vtable_id; +struct table_not_clause { + int table_id; int not_clause_num; UT_hash_handle hh; }; @@ -118,7 +118,7 @@ struct group2compile_runtime { long long rule_num; long long update_err_cnt; struct compile_runtime *ref_compile_rt; - struct vtable_not_clause *not_clause_hash; + struct table_not_clause *not_clause_hash; }; struct maat_clause { @@ -728,7 +728,7 @@ void group2compile_runtime_free(void *g2c_runtime) struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; if (g2c_rt->not_clause_hash != NULL) { - struct vtable_not_clause *not_clause = NULL, *tmp_not_clause = NULL; + struct table_not_clause *not_clause = NULL, *tmp_not_clause = NULL; HASH_ITER(hh, g2c_rt->not_clause_hash, not_clause, tmp_not_clause) { HASH_DEL(g2c_rt->not_clause_hash, not_clause); FREE(not_clause); @@ -1949,20 +1949,12 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, return 0; } -int validate_vtable_not_clause(struct group2compile_runtime *g2c_rt, - struct table_manager *tbl_mgr, int vtable_id, - int is_valid, struct log_handle *logger) +static int validate_table_not_clause(struct group2compile_runtime *g2c_rt, + struct table_manager *tbl_mgr, int table_id, + int is_valid, struct log_handle *logger) { - enum table_type table_type = table_manager_get_table_type(tbl_mgr, vtable_id); - if (table_type != TABLE_TYPE_VIRTUAL) { - log_fatal(logger, MODULE_COMPILE, - "[%s:%d]table(table_id:%d) is not virtual table, can't own NOT clause.", - __FUNCTION__, __LINE__, vtable_id); - return -1; - } - - struct vtable_not_clause *not_clause = NULL; - HASH_FIND_INT(g2c_rt->not_clause_hash, &vtable_id, not_clause); + struct table_not_clause *not_clause = NULL; + HASH_FIND_INT(g2c_rt->not_clause_hash, &table_id, not_clause); if (0 == is_valid) { //delete @@ -1974,15 +1966,15 @@ int validate_vtable_not_clause(struct group2compile_runtime *g2c_rt, } else { //add if (NULL == not_clause) { - not_clause = ALLOC(struct vtable_not_clause, 1); - not_clause->vtable_id = vtable_id; + not_clause = ALLOC(struct table_not_clause, 1); + not_clause->table_id = table_id; not_clause->not_clause_num++; - HASH_ADD_INT(g2c_rt->not_clause_hash, vtable_id, not_clause); + HASH_ADD_INT(g2c_rt->not_clause_hash, table_id, not_clause); } else { if (not_clause->not_clause_num >= MAX_NOT_CLAUSE_NUM) { - const char *table_name = table_manager_get_table_name(tbl_mgr, vtable_id); + const char *table_name = table_manager_get_table_name(tbl_mgr, table_id); log_fatal(logger, MODULE_COMPILE, - "[%s:%d]virtual table:<%s> NOT clause num exceed maximum:%d", + "[%s:%d]table:<%s> NOT clause num exceed maximum:%d", __FUNCTION__, __LINE__, table_name, MAX_NOT_CLAUSE_NUM); return -1; } @@ -2024,9 +2016,9 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, } if (1 == g2c_item->not_flag) { - ret = validate_vtable_not_clause(g2c_rt, schema->ref_tbl_mgr, - g2c_item->vtable_id, is_valid, - compile_rt->logger); + ret = validate_table_not_clause(g2c_rt, schema->ref_tbl_mgr, + g2c_item->vtable_id, is_valid, + compile_rt->logger); if (ret < 0) { log_fatal(compile_rt->logger, MODULE_COMPILE, "[%s:%d]validate NOT clause failed, abandon config:%s", @@ -2251,7 +2243,7 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile return MIN(bool_match_ret, compile_ids_size); } -int maat_compile_state_update(int vtable_id, struct maat_item *hit_items, +int maat_compile_state_update(int phy_table_id, int vtable_id, struct maat_item *hit_items, size_t n_hit_item, struct maat_state *state) { size_t i = 0, j = 0; @@ -2266,12 +2258,8 @@ int maat_compile_state_update(int vtable_id, struct maat_item *hit_items, } struct maat_compile_state *compile_state = state->compile_state; - if (hit_cnt > 0) { - if (compile_state->Nth_scan != state->scan_cnt) { - compile_state->Nth_scan = state->scan_cnt; - utarray_clear(compile_state->this_scan_hit_clauses); - } - } + utarray_clear(compile_state->this_scan_hit_clauses); + compile_state->Nth_scan = state->scan_cnt; for (i = 0; i < hit_cnt; i++) { hit_group_ids[i] = hit_items[i].group_id; @@ -2319,6 +2307,10 @@ int maat_compile_state_update(int vtable_id, struct maat_item *hit_items, hit_group_ids[i], vtable_id); } + if (vtable_id == 0) { + vtable_id = phy_table_id; + } + long long hit_NOT_group_ids[VTABLE_MAX_NOT_GROUP_NUM]; size_t hit_not_cnt = maat_compile_state_update_hit_not_clauses(state->compile_state, compile_rt, hit_group_ids, hit_cnt, vtable_id, diff --git a/src/maat_expr.c b/src/maat_expr.c index 8663631..b17df06 100644 --- a/src/maat_expr.c +++ b/src/maat_expr.c @@ -955,8 +955,8 @@ long long expr_runtime_get_version(void *expr_runtime) } int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, - const char *data, size_t data_len, int vtable_id, - struct maat_state *state) + const char *data, size_t data_len, int phy_table_id, + int vtable_id, struct maat_state *state) { if (0 == expr_rt->rule_num) { //empty expr table @@ -1002,7 +1002,7 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, } next: - return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state); + return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state); } struct expr_matcher_stream * @@ -1024,7 +1024,8 @@ expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct expr_matcher_stream *s_handle, const char *data, size_t data_len, - int vtable_id, struct maat_state *state) + int phy_table_id, int vtable_id, + struct maat_state *state) { if (0 == expr_rt->rule_num) { //empty expr table @@ -1064,7 +1065,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, } next: - return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state); + return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state); } void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id, diff --git a/src/maat_flag.c b/src/maat_flag.c index e002a92..d862088 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -544,7 +544,8 @@ long long flag_runtime_rule_count(void *flag_runtime) } int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, - long long flag, int vtable_id, struct maat_state *state) + long long flag, int phy_table_id, int vtable_id, + struct maat_state *state) { if (0 == flag_rt->rule_num) { //empty flag table @@ -588,7 +589,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, } next: - return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state); + return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state); } void flag_runtime_hit_inc(struct flag_runtime *flag_rt, int thread_id) diff --git a/src/maat_interval.c b/src/maat_interval.c index 92a739e..89f3a67 100644 --- a/src/maat_interval.c +++ b/src/maat_interval.c @@ -545,7 +545,8 @@ long long interval_runtime_rule_count(void *interval_runtime) } int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, - long long integer, int vtable_id, struct maat_state *state) + long long integer, int phy_table_id, int vtable_id, + struct maat_state *state) { if (0 == interval_rt->rule_num) { //empty interval table @@ -589,7 +590,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, } next: - return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state); + return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state); } void interval_runtime_hit_inc(struct interval_runtime *interval_rt, int thread_id) diff --git a/src/maat_ip.c b/src/maat_ip.c index cf74949..cc5c4a5 100644 --- a/src/maat_ip.c +++ b/src/maat_ip.c @@ -700,8 +700,8 @@ static int validate_port_proto(struct ip_item *item, uint16_t port, int proto) } int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, - uint8_t *ip_addr, uint16_t port, int proto, int vtable_id, - struct maat_state *state) + uint8_t *ip_addr, uint16_t port, int proto, int phy_table_id, + int vtable_id, struct maat_state *state) { if (0 == ip_rt->rule_num) { //empty ip table @@ -761,7 +761,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, } next: - return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state); + return maat_compile_state_update(phy_table_id, vtable_id, hit_maat_items, real_hit_item_cnt, state); } void ip_runtime_hit_inc(struct ip_runtime *ip_rt, int thread_id) diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index c54cfab..14eb286 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -1064,7 +1064,7 @@ TEST_F(MaatHsStringScan, HexBinCaseSensitive) { struct maat_state *state = maat_state_new(maat_inst, thread_id); int ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1), results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); maat_state_reset(state); ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2), @@ -1793,7 +1793,7 @@ TEST_F(MaatRsStringScan, HexBinCaseSensitive) { struct maat_state *state = maat_state_new(maat_inst, thread_id); int ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1), results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); maat_state_reset(state); ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2), @@ -3221,45 +3221,42 @@ TEST_F(NOTLogic, NotUrlAndNotIp) { state = NULL; } -#if 0 -TEST_F(NOTLogic, MultiNotClause) { - const char *string1 = "This string ONLY contains not_logic_compile_223_1."; - const char *string2 = "This string ONLY contains not_logic_compile_223_2."; - const char *string3 = "This string ONLY contains not_logic_compile_223_3."; - const char *string4 = "This string ONLY contains nothing"; +TEST_F(NOTLogic, NotPhysicalTable) { + const char *string1 = "This string ONLY contains not_logic_compile_224_1."; + const char *string2 = "This string ONLY contains not_logic_compile_224_2."; + const char *string3 = "This string ONLY contains nothing."; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - const char *table_name = "HTTP_NOT_LOGIC"; + const char *phy_table_name = "KEYWORDS_TABLE"; + const char *vtable_name = "HTTP_RESPONSE_KEYWORDS"; struct maat *maat_inst = NOTLogic::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); + int phy_table_id = maat_get_table_id(maat_inst, phy_table_name); + ASSERT_GT(phy_table_id, 0); - int ret = maat_scan_string(maat_inst, table_id, string1, + int vtable_id = maat_get_table_id(maat_inst, vtable_name); + ASSERT_GT(vtable_id, 0); + + int ret = maat_scan_string(maat_inst, phy_table_id, string1, strlen(string1), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_string(maat_inst, table_id, string2, strlen(string2), + ret = maat_scan_string(maat_inst, vtable_id, string2, strlen(string2), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_string(maat_inst, table_id, string3, strlen(string3), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - ret = maat_scan_string(maat_inst, table_id, string4, strlen(string4), + ret = maat_scan_string(maat_inst, phy_table_id, string3, strlen(string3), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 223); + EXPECT_EQ(results[0], 224); maat_state_free(state); state = NULL; } -#endif TEST_F(NOTLogic, EightNotClause) { const char *string_nothing = "This string contain nothing"; @@ -3604,7 +3601,7 @@ TEST_F(ExcludeLogic, ScanIrrelavantAtLast) { ret = maat_scan_string(maat_inst, not_hit_table_id, string_irrelevant, strlen(string_irrelevant), results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); maat_state_free(state); state = NULL; diff --git a/test/maat_json.json b/test/maat_json.json index 75e035e..89f8d6e 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -3747,6 +3747,53 @@ ] } ] + }, + { + "compile_id": 224, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "user_region": "NOTLogic.NotPhysicalTable", + "is_valid": "yes", + "groups": [ + { + "group_name": "NOTLogicGroup_224_1", + "virtual_table": "KEYWORDS_TABLE", + "not_flag": 1, + "clause_index": 0, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "not_logic_compile_224_1", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "NOTLogicGroup_224_2", + "virtual_table": "HTTP_RESPONSE_KEYWORDS", + "not_flag": 0, + "clause_index": 1, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "not_logic_compile_224_2", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] } ], "plugin_table": [