diff --git a/CMakeLists.txt b/CMakeLists.txt index ab59515..48d5185 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,11 +12,30 @@ LINK_DIRECTORIES(/opt/MESA/lib /usr/lib64) include_directories(${PROJECT_SOURCE_DIR}/include) include_directories(/opt/MESA/include/) -set(CMAKE_C_FLAGS "-std=gnu99 -fgnu89-inline -fPIC -Wall") +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_FLAGS "-fPIC -Wall") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall") set(MAAT_DEPEND_DYN_LIB pthread m crypto z) include_directories(include) +#for ASAN +set(ASAN_OPTION "OFF" CACHE STRING " set asan type chosen by the user, using OFF as default") +set_property(CACHE ASAN_OPTION PROPERTY STRINGS OFF ADDRESS THREAD) +message(STATUS "ASAN_OPTION='${ASAN_OPTION}'") + +if(ASAN_OPTION MATCHES "ADDRESS") + set(CMAKE_C_FLAGS "${CMAKADDRESS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=address -fno-omit-frame-pointer") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=address -fno-omit-frame-pointer") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan") +elseif(ASAN_OPTION MATCHES "THREAD") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=thread -fno-omit-frame-pointer") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=thread -fno-omit-frame-pointer") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan") +endif() +# end of for ASAN + enable_testing() #add_subdirectory(source_dir [binary_dir] [EXCLUDE_FROM_ALL]) diff --git a/scanner/adapter_hs/adapter_hs.cpp b/scanner/adapter_hs/adapter_hs.cpp index 6c48df9..49b93a1 100644 --- a/scanner/adapter_hs/adapter_hs.cpp +++ b/scanner/adapter_hs/adapter_hs.cpp @@ -52,10 +52,8 @@ struct adapter_hs_runtime { hs_database_t *literal_db; hs_database_t *regex_db; - hs_scratch_t **literal_scratchs; - hs_scratch_t **regex_scratchs; - size_t literal_scratch_size; - size_t regex_scratch_size; + hs_scratch_t **literal_scratches; + hs_scratch_t **regex_scratches; struct bool_matcher *bm; }; @@ -95,14 +93,15 @@ struct adapter_hs_stream { hs_stream_t *regex_stream; struct adapter_hs_runtime *hs_rt; struct matched_pattern *matched_pat; + struct log_handle *logger; }; -int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratchs, size_t n_worker_thread, +int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches, size_t n_worker_thread, struct log_handle *logger) { size_t scratch_size = 0; - if (hs_alloc_scratch(db, &scratchs[0]) != HS_SUCCESS) { + if (hs_alloc_scratch(db, &scratches[0]) != HS_SUCCESS) { log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Unable to allocate scratch space. Exiting.", __FUNCTION__, __LINE__); @@ -110,18 +109,21 @@ int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratchs, size_t n_worke } for (size_t i = 1; i < n_worker_thread; i++) { - hs_error_t err = hs_clone_scratch(scratchs[0], &scratchs[i]); + hs_error_t err = hs_clone_scratch(scratches[0], &scratches[i]); if (err != HS_SUCCESS) { log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Unable to clone scratch", __FUNCTION__, __LINE__); return -1; } - err = hs_scratch_size(scratchs[i], &scratch_size); + err = hs_scratch_size(scratches[i], &scratch_size); if (err != HS_SUCCESS) { log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Unable to query scratch size", __FUNCTION__, __LINE__); return -1; + } else { + log_info(logger, MODULE_ADAPTER_HS, + "[%s:%d] scratch[%d] size:%zu", __FUNCTION__, __LINE__, i, scratch_size); } } @@ -134,17 +136,17 @@ static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_work int ret = 0; if (pattern_type == HS_PATTERN_TYPE_STR) { - hs_rt->literal_scratchs = ALLOC(hs_scratch_t *, n_worker_thread); - ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->literal_scratchs, n_worker_thread, logger); + hs_rt->literal_scratches = ALLOC(hs_scratch_t *, n_worker_thread); + ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->literal_scratches, n_worker_thread, logger); if (ret < 0) { - FREE(hs_rt->literal_scratchs); + FREE(hs_rt->literal_scratches); return -1; } } else { - hs_rt->regex_scratchs = ALLOC(hs_scratch_t *, n_worker_thread); - ret = _hs_alloc_scratch(hs_rt->regex_db, hs_rt->regex_scratchs, n_worker_thread, logger); + hs_rt->regex_scratches = ALLOC(hs_scratch_t *, n_worker_thread); + ret = _hs_alloc_scratch(hs_rt->regex_db, hs_rt->regex_scratches, n_worker_thread, logger); if (ret < 0) { - FREE(hs_rt->regex_scratchs); + FREE(hs_rt->regex_scratches); return -1; } } @@ -452,32 +454,42 @@ void adapter_hs_free(struct adapter_hs *hs_instance) if (hs_instance->hs_rt != NULL) { if (hs_instance->hs_rt->literal_db != NULL) { hs_free_database(hs_instance->hs_rt->literal_db); + hs_instance->hs_rt->literal_db = NULL; } if (hs_instance->hs_rt->regex_db != NULL) { hs_free_database(hs_instance->hs_rt->regex_db); + hs_instance->hs_rt->regex_db = NULL; } - if (hs_instance->hs_rt->literal_scratchs != NULL) { + if (hs_instance->hs_rt->literal_scratches != NULL) { for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { - if (hs_instance->hs_rt->literal_scratchs[i] != NULL) { - hs_free_scratch(hs_instance->hs_rt->literal_scratchs[i]); + if (hs_instance->hs_rt->literal_scratches[i] != NULL) { + hs_free_scratch(hs_instance->hs_rt->literal_scratches[i]); + hs_instance->hs_rt->literal_scratches[i] = NULL; } } - } - FREE(hs_instance->hs_rt->literal_scratchs); - if (hs_instance->hs_rt->regex_scratchs != NULL) { + FREE(hs_instance->hs_rt->literal_scratches); + hs_instance->hs_rt->literal_scratches = NULL; + } + + + if (hs_instance->hs_rt->regex_scratches != NULL) { for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { - if (hs_instance->hs_rt->regex_scratchs[i] != NULL) { - hs_free_scratch(hs_instance->hs_rt->regex_scratchs[i]); + if (hs_instance->hs_rt->regex_scratches[i] != NULL) { + hs_free_scratch(hs_instance->hs_rt->regex_scratches[i]); + hs_instance->hs_rt->regex_scratches[i] = NULL; } } - } - FREE(hs_instance->hs_rt->regex_scratchs); + FREE(hs_instance->hs_rt->regex_scratches); + hs_instance->hs_rt->regex_scratches = NULL; + } + if (hs_instance->hs_rt->bm != NULL) { bool_matcher_free(hs_instance->hs_rt->bm); + hs_instance->hs_rt->bm = NULL; } FREE(hs_instance->hs_rt); @@ -587,6 +599,7 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, struct adapter_hs_stream *hs_stream = ALLOC(struct adapter_hs_stream, 1); hs_error_t err; + hs_stream->logger = hs_instance->logger; hs_stream->thread_id = thread_id; hs_stream->n_expr = hs_instance->n_expr; hs_stream->hs_rt = hs_instance->hs_rt; @@ -619,7 +632,7 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, return hs_stream; error: - //TODO: hs_stream->hs_rt->scratchs[thread_id] may be free twice + //TODO: hs_stream->hs_rt->scratches[thread_id] may be free twice if (hs_stream->literal_stream != NULL) { hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL); hs_stream->literal_stream = NULL; @@ -686,21 +699,34 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data int thread_id = hs_stream->thread_id; hs_stream->matched_pat->scan_data_len = data_len; + int err_scratch_flag = 0; if (hs_stream->literal_stream != NULL) { - err = hs_scan_stream(hs_stream->literal_stream, data, data_len, - 0, hs_stream->hs_rt->literal_scratchs[thread_id], - matched_event_cb, hs_stream->matched_pat); - if (err != HS_SUCCESS) { - err_count++; + if (hs_stream->hs_rt->literal_scratches != NULL) { + err = hs_scan_stream(hs_stream->literal_stream, data, data_len, + 0, hs_stream->hs_rt->literal_scratches[thread_id], + matched_event_cb, hs_stream->matched_pat); + if (err != HS_SUCCESS) { + err_count++; + } + } else { + log_error(hs_stream->logger, MODULE_ADAPTER_HS, "literal_scratches is null, thread_id:%d", + thread_id); + err_scratch_flag++; } } - + if (hs_stream->regex_stream != NULL) { - err = hs_scan_stream(hs_stream->regex_stream, data, data_len, - 0, hs_stream->hs_rt->regex_scratchs[thread_id], - matched_event_cb, hs_stream->matched_pat); - if (err != HS_SUCCESS) { - err_count++; + if (hs_stream->hs_rt->regex_scratches != NULL) { + err = hs_scan_stream(hs_stream->regex_stream, data, data_len, + 0, hs_stream->hs_rt->regex_scratches[thread_id], + matched_event_cb, hs_stream->matched_pat); + if (err != HS_SUCCESS) { + err_count++; + } + } else { + log_error(hs_stream->logger, MODULE_ADAPTER_HS, "regex_scratches is null, thread_id:%d", + thread_id); + err_scratch_flag++; } } @@ -708,6 +734,10 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data return -1; } + if (err_scratch_flag != 0) { + return -1; + } + size_t n_pattern_id = utarray_len(hs_stream->matched_pat->pattern_ids); if (0 == n_pattern_id) { *n_hit_result = 0; diff --git a/src/maat_api.c b/src/maat_api.c index 58f6021..e81ae85 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -387,7 +387,6 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) if (ret < 0) { log_error(maat_instance->logger, MODULE_MAAT_API, "[%s:%d] maat read full config failed", __FUNCTION__, __LINE__); - goto failed; } } diff --git a/src/maat_config_monitor.c b/src/maat_config_monitor.c index f23670f..e2ca112 100644 --- a/src/maat_config_monitor.c +++ b/src/maat_config_monitor.c @@ -435,8 +435,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, } if (!maat_instance->is_running) { - strncpy(maat_instance->json_ctx.json_file, json_filename, - sizeof(maat_instance->json_ctx.json_file)); + size_t len = MIN(strlen(json_filename), sizeof(maat_instance->json_ctx.json_file)); + strncpy(maat_instance->json_ctx.json_file, json_filename, len); } ret = stat(json_filename, &fstat_buf); diff --git a/src/maat_rule.c b/src/maat_rule.c index 47a7be3..39db5e5 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -478,10 +478,8 @@ void *rule_monitor_loop(void *arg) ret = maat_read_full_config(maat_instance); if (ret < 0) { log_error(maat_instance->logger, MODULE_MAAT_RULE, - "[%s:%d] maat read full config failed, exit rule_monitor_loop thread.", + "[%s:%d] maat read full config failed", __FUNCTION__, __LINE__); - pthread_mutex_unlock(&(maat_instance->background_update_mutex)); - return NULL; } } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index e48dd71..cd26341 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -292,6 +292,130 @@ int ip_table_set_line(struct maat *maat_instance, const char *table_name, enum m return maat_cmd_set_line(maat_instance, &line_rule); } + +int test_add_expr_command(struct maat *maat_instance, const char *expr_table, + long long compile_id, int timeout, const char *keywords) +{ + char huge_serv_def[1024 * 2] = {0}; + + memset(huge_serv_def, 's', sizeof(huge_serv_def) - 1); + huge_serv_def[sizeof(huge_serv_def) - 1] = '\0'; + + int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id, + huge_serv_def, 1, timeout); + EXPECT_EQ(ret, 1); + + long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); + ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id, + compile_id, 0, "null", 1, timeout); + EXPECT_EQ(ret, 1); + + long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); + ret = expr_table_set_line(maat_instance, expr_table, MAAT_OP_ADD, item_id, group_id, keywords, + "null", 1, 0, 0, 0); + EXPECT_EQ(ret, 1); + + return ret; +} + +int del_command(struct maat *maat_instance, int compile_id) +{ + return compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_DEL, compile_id, NULL, 1, 0); +} +#if 0 +class MaatStreamScan : public testing::Test +{ +protected: + static void SetUpTestCase() { + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); + + _shared_maat_instance = maat_new(opts, table_info_path); + assert(_shared_maat_instance != NULL); + + maat_cmd_flushDB(_shared_maat_instance); + maat_free(_shared_maat_instance); + + maat_options_set_foreign_cont_dir(opts, "./foreign_files/"); + maat_options_set_rule_effect_interval_ms(opts, 20 * 1000); //20s for garbage collection + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + } + + static void TearDownTestCase() { + maat_free(_shared_maat_instance); + } + + static struct maat *_shared_maat_instance; +}; + +struct maat *MaatStreamScan::_shared_maat_instance; + +TEST_F(MaatStreamScan, dynamic_config) { + const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; + const char *scan_data2 = "hello world"; + const char *table_name = "HTTP_URL"; + const char *keywords1 = "action=search\\&query=(.*)"; + const char *keywords2 = "hello"; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + int thread_id = 0; + struct maat *maat_instance = MaatStreamScan::_shared_maat_instance; + struct maat_state *state = maat_state_new(maat_instance, thread_id); + + long long compile1_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + int ret = test_add_expr_command(maat_instance, table_name, compile1_id, 0, keywords1); + EXPECT_EQ(ret, 1); + sleep(WAIT_FOR_EFFECTIVE_S * 2); + + int table_id = maat_get_table_id(maat_instance, table_name); + ASSERT_GT(table_id, 0); + + struct maat_stream *sp = maat_stream_new(maat_instance, table_id, state); + ASSERT_TRUE(sp != NULL); + + ret = maat_stream_scan(sp, "www.cyberessays.com", strlen("www.cyberessays.com"), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_stream_scan(sp, scan_data1, strlen(scan_data1), results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(results[0], compile1_id); + maat_state_reset(state); + + long long compile2_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + ret = test_add_expr_command(maat_instance, table_name, compile2_id, 0, keywords2); + EXPECT_EQ(ret, 1); + + for (int i = 0; i < 100; i++) { + ret = maat_stream_scan(sp, "www.cyberessays.com", strlen("www.cyberessays.com"), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_stream_scan(sp, scan_data2, strlen(scan_data2), results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(results[0], compile2_id); + maat_state_reset(state); + usleep(500 * 1000); + + compile2_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + ret = test_add_expr_command(maat_instance, table_name, compile2_id, 0, keywords2); + EXPECT_EQ(ret, 1); + + } + maat_stream_free(sp); + maat_state_free(state); + sp = NULL; + state = NULL; +} +#endif #if 1 class MaatFlagScan : public testing::Test { @@ -3318,36 +3442,6 @@ protected: struct maat *MaatCmdTest::_shared_maat_instance; -int test_add_expr_command(struct maat *maat_instance, const char *expr_table, - long long compile_id, int timeout, const char *keywords) -{ - char huge_serv_def[1024 * 2] = {0}; - - memset(huge_serv_def, 's', sizeof(huge_serv_def) - 1); - huge_serv_def[sizeof(huge_serv_def) - 1] = '\0'; - - int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id, - huge_serv_def, 1, timeout); - EXPECT_EQ(ret, 1); - - long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id, - compile_id, 0, "null", 1, timeout); - EXPECT_EQ(ret, 1); - - long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); - ret = expr_table_set_line(maat_instance, expr_table, MAAT_OP_ADD, item_id, group_id, keywords, - "null", 1, 0, 0, 0); - EXPECT_EQ(ret, 1); - - return ret; -} - -int del_command(struct maat *maat_instance, int compile_id) -{ - return compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_DEL, compile_id, NULL, 1, 0); -} - TEST_F(MaatCmdTest, SetIP) { long long version_before = 0; long long results[ARRAY_SIZE] = {0}; @@ -5269,7 +5363,7 @@ TEST_F(MaatCmdTest, StreamScanSegfaultWhenVersionRollBack_TSG6324) { maat_state_free(state); state = NULL; } -#endif + class MaatRollbackTest : public testing::Test { protected: @@ -5446,7 +5540,7 @@ TEST_F(MaatRollbackTest, FullConfigRollback) { maat_state_free(state); state = NULL; } - +#endif int main(int argc, char ** argv) { int ret=0; diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index d133a3f..7c31055 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -5,7 +5,7 @@ include(ExternalProject) set(VENDOR_ROOT ${CMAKE_BINARY_DIR}/vendor) set(VENDOR_BUILD ${CMAKE_BINARY_DIR}/vendor/vbuild) -set(CMAKE_C_FLAGS "-std=gnu99 -fPIC -Wall") +set(CMAKE_C_FLAGS "-std=c99 -fPIC -Wall") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall") # GoogleTest