diff --git a/scanner/adapter_hs/adapter_hs.cpp b/scanner/adapter_hs/adapter_hs.cpp index 9afddc5..6c48df9 100644 --- a/scanner/adapter_hs/adapter_hs.cpp +++ b/scanner/adapter_hs/adapter_hs.cpp @@ -522,8 +522,6 @@ int matched_event_cb(unsigned int id, unsigned long long from, } int ret = 0; - long long start_offset = -1; - long long end_offset = -1; struct pattern_attribute pat_attr = matched_pat->ref_hs_attr[id]; switch (pat_attr.match_mode) { case HS_MATCH_MODE_EXACTLY: @@ -532,20 +530,28 @@ int matched_event_cb(unsigned int id, unsigned long long from, } break; case HS_MATCH_MODE_SUB: + if (pat_attr.offset.start == -1 && + pat_attr.offset.end == -1) { + ret = 1; + break; + } + if (pat_attr.offset.start == -1) { - start_offset = 0; - } else { - start_offset = pat_attr.offset.start; + if ((long long)(to - 1) <= pat_attr.offset.end) { + ret = 1; + break; + } } if (pat_attr.offset.end == -1) { - end_offset = matched_pat->scan_data_len; - } else { - end_offset = pat_attr.offset.end; + if ((long long)from >= pat_attr.offset.start) { + ret = 1; + break; + } } - if (start_offset <= (long long)from && - end_offset >= (long long)(to - 1)) { + if ((long long)from >= pat_attr.offset.start && + (long long)(to - 1) <= pat_attr.offset.end) { ret = 1; } break; diff --git a/src/maat_compile.c b/src/maat_compile.c index 14b9f52..04a7e79 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -72,7 +72,7 @@ struct compile_runtime { struct maat_compile *compile_hash; // struct maat_runtime *ref_maat_rt; uint32_t rule_num; - + struct maat_clause *clause_by_literals_hash; pthread_rwlock_t rwlock; /* TODO: replaced with mutex? */ struct bool_expr_match *expr_match_buff; @@ -499,6 +499,7 @@ void *compile_runtime_new(void *compile_schema, int max_thread_num, struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1); compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM); + compile_rt->clause_by_literals_hash = NULL; compile_rt->logger = logger; compile_rt->ref_garbage_bin = garbage_bin; pthread_rwlock_init(&compile_rt->rwlock, NULL); @@ -541,6 +542,18 @@ void maat_compile_hash_free(struct maat_compile **compile_hash) assert(*compile_hash == NULL); } +static void maat_clause_hash_free(struct maat_clause **clause_hash) +{ + struct maat_clause *clause = NULL, *tmp_clause = NULL; + + HASH_ITER (hh, *clause_hash, clause, tmp_clause) { + HASH_DEL(*clause_hash, clause); + FREE(clause->literal_ids); + clause->n_literal_id = 0; + FREE(clause); + } +} + void compile_runtime_free(void *compile_runtime) { if (NULL == compile_runtime) { @@ -553,12 +566,18 @@ void compile_runtime_free(void *compile_runtime) if (compile_rt->bm != NULL) { bool_matcher_free(compile_rt->bm); + compile_rt->bm = NULL; } if (compile_rt->compile_hash != NULL) { maat_compile_hash_free(&(compile_rt->compile_hash)); + compile_rt->compile_hash = NULL; } + if (compile_rt->clause_by_literals_hash != NULL) { + maat_clause_hash_free(&compile_rt->clause_by_literals_hash); + compile_rt->clause_by_literals_hash = NULL; + } if (compile_rt->expr_match_buff != NULL) { FREE(compile_rt->expr_match_buff); } @@ -723,6 +742,7 @@ struct maat_compile *maat_compile_new(long long compile_id) for(int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { utarray_new(compile->clause_states[i].ut_literal_ids, &ut_literal_id_icd); compile->clause_states[i].in_use=0; + compile->clause_states[i].clause_id = -1; } return compile; @@ -751,8 +771,7 @@ int maat_compile_hash_add(struct maat_compile **compile_hash, long long compile_ HASH_ADD(hh, *compile_hash, compile_id, sizeof(long long), compile); //TODO:mytest need to delete #if 0 - if (compile_id == 141) - { + size_t compile_cnt = HASH_COUNT(*compile_hash); struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL; HASH_ITER(hh, *compile_hash, compile1, tmp_compile1) @@ -760,7 +779,7 @@ int maat_compile_hash_add(struct maat_compile **compile_hash, long long compile_ printf(" compile_id:%lld, compile_cnt:%zu\n", compile1->compile_id, compile_cnt); } - } + #endif return ret; } @@ -879,40 +898,28 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile, } static const struct maat_clause * -maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, - struct maat_runtime *ref_maat_rt, +maat_clause_hash_fetch_clause(struct compile_runtime *compile_rt, struct maat_literal_id *literal_ids, size_t n_literal_id) { struct maat_clause *clause = NULL; - HASH_FIND(hh, *clause_hash, literal_ids, n_literal_id * sizeof(struct maat_literal_id), clause); + HASH_FIND(hh, compile_rt->clause_by_literals_hash, literal_ids, + n_literal_id * sizeof(struct maat_literal_id), clause); if (!clause) { clause = ALLOC(struct maat_clause, 1); - clause->clause_id = maat_runtime_get_sequence(ref_maat_rt, "clause_id"); + clause->clause_id = maat_runtime_get_sequence(compile_rt->ref_maat_rt, "clause_id"); clause->n_literal_id = n_literal_id; clause->literal_ids = ALLOC(struct maat_literal_id, n_literal_id); memcpy(clause->literal_ids, literal_ids, n_literal_id * sizeof(struct maat_literal_id)); - HASH_ADD_KEYPTR(hh, *clause_hash, clause->literal_ids, + HASH_ADD_KEYPTR(hh, compile_rt->clause_by_literals_hash, clause->literal_ids, n_literal_id * sizeof(struct maat_literal_id), clause); } return clause; } -static void maat_clause_hash_free(struct maat_clause **clause_hash) -{ - struct maat_clause *clause = NULL, *tmp_clause = NULL; - - HASH_ITER (hh, *clause_hash, clause, tmp_clause) { - HASH_DEL(*clause_hash, clause); - FREE(clause->literal_ids); - clause->n_literal_id = 0; - FREE(clause); - } -} - struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compile_rt) { if (NULL == compile_rt) { @@ -924,7 +931,7 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi struct bool_matcher *bm = NULL; struct maat_clause_state *clause_state = NULL; const struct maat_clause *clause = NULL; - struct maat_clause *clause_hash = NULL; // + //struct maat_clause *clause_hash = NULL; // pthread_rwlock_rdlock(&compile_rt->rwlock); //STEP 1, update clause_id of each compile and literal @@ -943,8 +950,7 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi has_clause_num++; literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->ut_literal_ids, 0); n_literal_id = utarray_len(clause_state->ut_literal_ids); - clause = maat_clause_hash_fetch_clause(&clause_hash, compile_rt->ref_maat_rt, - literal_ids, n_literal_id); + clause = maat_clause_hash_fetch_clause(compile_rt, literal_ids, n_literal_id); clause_state->clause_id = clause->clause_id; } assert(has_clause_num == compile->actual_clause_num); @@ -1023,13 +1029,7 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi } error: - if (clause_hash != NULL) { - maat_clause_hash_free(&clause_hash); - clause_hash = NULL; - } - FREE(bool_expr_array); - return bm; } @@ -1392,7 +1392,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta if (!clause_state->in_use) { continue; } - + size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses); tmp = (struct maat_literal_id *)utarray_find(clause_state->ut_literal_ids, &literal_id, compare_literal_id); @@ -1416,6 +1416,12 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta utarray_push_back(compile_state->all_hit_clauses, clause_id); } utarray_sort(compile_state->all_hit_clauses, compare_clause_id); + // printf(" all_hit_clause:"); + // for (i = 0; i < utarray_len(compile_state->all_hit_clauses); i++) { + // long long *tmp_clause_id = (long long *)utarray_eltptr(compile_state->all_hit_clauses, i); + // printf(" %lld ", *tmp_clause_id); + // } + // printf("\n"); } } } diff --git a/test/adapter_hs_gtest.cpp b/test/adapter_hs_gtest.cpp index 1c7b4e5..77881d4 100644 --- a/test/adapter_hs_gtest.cpp +++ b/test/adapter_hs_gtest.cpp @@ -241,7 +241,6 @@ TEST(adapter_hs_scan, literal_sub_has_normal_offset) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_expr_array, 12); struct adapter_hs *hs_instance = adapter_hs_new(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); @@ -653,6 +652,32 @@ TEST(adapter_hs_scan, same_pattern_different_offset) hs_instance = NULL; } +TEST(adapter_hs_scan, long_scan_data) +{ + struct hs_expr expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(1, expr_array, n_expr_array, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(expr_array, n_expr_array); + + const char* scan_data = "A directed path in a directed graph is a finite or infinite\ +sequence of edges which joins a sequence of distinct vertices, but with the added restriction\ +that the edges be all directed in the same direction."; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data, strlen(scan_data), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 113); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + int main(int argc, char **argv) { int ret = 0; diff --git a/test/literal_expr.conf b/test/literal_expr.conf index e25b42a..cf13fa2 100644 --- a/test/literal_expr.conf +++ b/test/literal_expr.conf @@ -161,6 +161,18 @@ "offset": "16~20" } ] + }, + { + "expr_id": 113, + "pattern_num": 1, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "a finite or infinite" + } + ] } ] } diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 1c8800b..73029d0 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -1286,7 +1286,7 @@ TEST_F(MaatIPScan, IPv4_IPPort) { uint32_t sip; int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); - uint16_t port = htons(65529); + uint16_t port = htons(65530); int proto = 6; long long results[ARRAY_SIZE] = {0}; @@ -1313,7 +1313,7 @@ TEST_F(MaatIPScan, IPv4_Port) { uint32_t sip; int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); - uint16_t port = htons(65528); + uint16_t port = htons(20303); int proto = 6; long long results[ARRAY_SIZE] = {0}; @@ -1324,12 +1324,12 @@ TEST_F(MaatIPScan, IPv4_Port) { EXPECT_EQ(ret, MAAT_SCAN_OK); maat_state_reset(state); - port = htons(65529); + port = htons(64999); ret = maat_scan_ipv4(maat_instance, table_id, sip, port, proto, results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 154); + EXPECT_EQ(results[0], 169); maat_state_free(state); state = NULL; @@ -2929,7 +2929,6 @@ protected: maat_cmd_flushDB(_shared_maat_instance); maat_free(_shared_maat_instance); - maat_options_set_deferred_load_on(opts); _shared_maat_instance = maat_new(opts, table_info_path); maat_options_free(opts); } @@ -4270,7 +4269,7 @@ TEST_F(MaatCmdTest, GroupInMassCompiles) { maat_state_free(state); state = NULL; } -#if 0 + TEST_F(MaatCmdTest, HitPath) { const char *g2g_table_name = "GROUP2GROUP"; const char *g2c_table_name = "GROUP2COMPILE"; @@ -4343,7 +4342,7 @@ TEST_F(MaatCmdTest, HitPath) { NULL, 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/ EXPECT_EQ(ret, 1); - sleep(WAIT_FOR_EFFECTIVE_S); + sleep(WAIT_FOR_EFFECTIVE_S * 2); const char* http_url = "en.wikipedia.org/wiki/Path_(graph_theory)"; const char* http_resp_hdr_cookie = "laptop=thinkpad X1 extrem;time=2020-02-11T15:34:00;main[XWJOKE]=hoho; Hm_lvt_bbac0322e6ee13093f98d5c4b5a10912=1578874808;"; @@ -4419,7 +4418,7 @@ that the edges be all directed in the same direction."; struct maat_stream *stream = maat_stream_new(maat_instance, table_id, state); Nth_scan++; ret = maat_stream_scan(stream, keywords1, strlen(keywords1), results, ARRAY_SIZE, - &n_hit_result, state); + &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); EXPECT_EQ(n_read, 3); @@ -4456,7 +4455,7 @@ that the edges be all directed in the same direction."; EXPECT_EQ(hit_path[path_idx].vtable_id, 0); EXPECT_EQ(hit_path[path_idx].compile_id, -1); - Nth_scan++; + Nth_scan++; ret = maat_stream_scan(stream, keywords2, strlen(keywords2), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); @@ -4502,19 +4501,19 @@ TEST_F(MaatCmdTest, SameScanStatusWhenClauseUpdate_TSG6419) { long long item11_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); ret = ip_table_set_line(maat_instance, ip_table_name, MAAT_OP_ADD, item11_id, group11_id, - IPv4, "192.168.2.1", "192.168.2.4", 0, 0, 0); + IPv4, "192.168.2.1", "192.168.2.4", 0, 65535, 0); EXPECT_EQ(ret, 1); long long group21_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); ret = group2compile_table_set_line(maat_instance, g2c_table_name, MAAT_OP_ADD, group21_id, - compile1_id, 0, "null", 1, 0); + compile1_id, 0, "null", 2, 0); EXPECT_EQ(ret, 1); long long item21_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); ret = intval_table_set_line(maat_instance, app_id_table_name, MAAT_OP_ADD, item21_id, group21_id, 31, 31, NULL, 0); EXPECT_EQ(ret, 1); - sleep(WAIT_FOR_EFFECTIVE_S); + sleep(WAIT_FOR_EFFECTIVE_S * 2); long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; @@ -4555,7 +4554,7 @@ TEST_F(MaatCmdTest, SameScanStatusWhenClauseUpdate_TSG6419) { group22_id, 32, 32, NULL, 0); EXPECT_EQ(ret, 1); - sleep(WAIT_FOR_EFFECTIVE_S); + sleep(WAIT_FOR_EFFECTIVE_S * 2); table_id = maat_get_table_id(maat_instance, app_id_table_name); ret = maat_scan_integer(maat_instance, table_id, 31, results, ARRAY_SIZE, @@ -4595,7 +4594,7 @@ TEST_F(MaatCmdTest, GroupEdit) { long long item11_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); ret = ip_table_set_line(maat_instance, ip_table_name, MAAT_OP_ADD, item11_id, group11_id, - IPv4, "192.168.3.1", "192.168.3.4", 0, 0, 0); + IPv4, "192.168.3.1", "192.168.3.4", 0, 65535, 0); EXPECT_EQ(ret, 1); long long group21_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); @@ -4747,11 +4746,11 @@ TEST_F(MaatCmdTest, CompileDelete_TSG6548) { while (now - update_time < 3) { ret = maat_scan_ipv4(maat_instance, table_id, ip_addr, port, 6, results, ARRAY_SIZE, &n_hit_result, state); - if (ret > 0) { + if (ret == MAAT_SCAN_HIT) { hit_cnt++; EXPECT_EQ(results[0], compile1_id); - } else { - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + } + if (ret == MAAT_SCAN_HALF_HIT) { miss_cnt++; } now = time(NULL); @@ -4761,6 +4760,7 @@ TEST_F(MaatCmdTest, CompileDelete_TSG6548) { maat_state_free(state); } +#if 0 TEST_F(MaatCmdTest, UpdateDeadLockDetection) { const char* g2c_table_name = "GROUP2COMPILE"; const char* compile_table_name = "COMPILE"; @@ -4878,7 +4878,7 @@ TEST_F(MaatCmdTest, StreamScanSegfaultWhenVersionRollBack_TSG6324) { EXPECT_EQ(results[0], compile1_id); //DON'T DO THIS!!! - //Roll back version, trigger full udpate. + //Roll back version, trigger full update. //This operation generates FATAL logs in test_maat_redis.log.yyyy-mm-dd. //For example: Add group 22 vt_id 0 to clause 2 of compile 979 failed, group is already exisited maat_cmd_incrby(maat_instance, "MAAT_VERSION", -100); diff --git a/test/maat_json.json b/test/maat_json.json index 4b50bfe..20a9f96 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -1064,7 +1064,7 @@ "ip1": "10.0.7.100", "ip2": "10.0.7.106", "port_format": "range", - "port1": "65529", + "port1": "65530", "port2": "65535", "protocol": 6 } @@ -1506,6 +1506,37 @@ } ] }, + { + "compile_id": 169, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv4_Any", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "addr_format": "CIDR", + "ip1": "0.0.0.0", + "ip2": "0", + "port_format": "range", + "port1": "64000", + "port2": "64999", + "protocol": 6 + } + } + ], + "not_flag" : 0 + } + ] + }, { "compile_id": 170, "service": 0,