gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

read_full_config error can't abort maat_new

Browse Source
This commit is contained in:
liuwentan
2023-04-12 15:31:17 +08:00
parent 53cd03995d
commit e8fb0143e9
7 changed files with 216 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
CMakeLists.txt
View File

@@ -12,11 +12,30 @@ LINK_DIRECTORIES(/opt/MESA/lib /usr/lib64)
include_directories(${PROJECT_SOURCE_DIR}/include)
include_directories(/opt/MESA/include/)
set(CMAKE_C_FLAGS "-std=gnu99 -fgnu89-inline -fPIC -Wall")
set(CMAKE_C_STANDARD 11)
set(CMAKE_C_FLAGS "-fPIC -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall")
set(MAAT_DEPEND_DYN_LIB pthread m crypto z)
include_directories(include)
#for ASAN
set(ASAN_OPTION "OFF" CACHE STRING " set asan type chosen by the user, using OFF as default")
set_property(CACHE ASAN_OPTION PROPERTY STRINGS OFF ADDRESS THREAD)
message(STATUS "ASAN_OPTION='${ASAN_OPTION}'")
if(ASAN_OPTION MATCHES "ADDRESS")
set(CMAKE_C_FLAGS "${CMAKADDRESS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=address -fno-omit-frame-pointer")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=address -fno-omit-frame-pointer")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
elseif(ASAN_OPTION MATCHES "THREAD")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=thread -fno-omit-frame-pointer")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -DCMAKE_BUILD_TYPE=Debug -fsanitize=thread -fno-omit-frame-pointer")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lasan")
endif()
# end of for ASAN
enable_testing()
#add_subdirectory(source_dir [binary_dir] [EXCLUDE_FROM_ALL])

82
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -52,10 +52,8 @@ struct adapter_hs_runtime {
hs_database_t *literal_db;
hs_database_t *regex_db;
hs_scratch_t **literal_scratchs;
hs_scratch_t **regex_scratchs;
size_t literal_scratch_size;
size_t regex_scratch_size;
hs_scratch_t **literal_scratches;
hs_scratch_t **regex_scratches;
struct bool_matcher *bm;
};
@@ -95,14 +93,15 @@ struct adapter_hs_stream {
hs_stream_t *regex_stream;
struct adapter_hs_runtime *hs_rt;
struct matched_pattern *matched_pat;
struct log_handle *logger;
};
int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratchs, size_t n_worker_thread,
int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches, size_t n_worker_thread,
struct log_handle *logger)
{
size_t scratch_size = 0;
if (hs_alloc_scratch(db, &scratchs[0]) != HS_SUCCESS) {
if (hs_alloc_scratch(db, &scratches[0]) != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to allocate scratch space. Exiting.",
__FUNCTION__, __LINE__);
@@ -110,18 +109,21 @@ int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratchs, size_t n_worke
}
for (size_t i = 1; i < n_worker_thread; i++) {
hs_error_t err = hs_clone_scratch(scratchs[0], &scratchs[i]);
hs_error_t err = hs_clone_scratch(scratches[0], &scratches[i]);
if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to clone scratch", __FUNCTION__, __LINE__);
return -1;
}
err = hs_scratch_size(scratchs[i], &scratch_size);
err = hs_scratch_size(scratches[i], &scratch_size);
if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to query scratch size", __FUNCTION__, __LINE__);
return -1;
} else {
log_info(logger, MODULE_ADAPTER_HS,
"[%s:%d] scratch[%d] size:%zu", __FUNCTION__, __LINE__, i, scratch_size);
}
}
@@ -134,17 +136,17 @@ static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_work
int ret = 0;
if (pattern_type == HS_PATTERN_TYPE_STR) {
hs_rt->literal_scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->literal_scratchs, n_worker_thread, logger);
hs_rt->literal_scratches = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->literal_scratches, n_worker_thread, logger);
if (ret < 0) {
FREE(hs_rt->literal_scratchs);
FREE(hs_rt->literal_scratches);
return -1;
}
} else {
hs_rt->regex_scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->regex_db, hs_rt->regex_scratchs, n_worker_thread, logger);
hs_rt->regex_scratches = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->regex_db, hs_rt->regex_scratches, n_worker_thread, logger);
if (ret < 0) {
FREE(hs_rt->regex_scratchs);
FREE(hs_rt->regex_scratches);
return -1;
}
}
@@ -452,32 +454,42 @@ void adapter_hs_free(struct adapter_hs *hs_instance)
if (hs_instance->hs_rt != NULL) {
if (hs_instance->hs_rt->literal_db != NULL) {
hs_free_database(hs_instance->hs_rt->literal_db);
hs_instance->hs_rt->literal_db = NULL;
}
if (hs_instance->hs_rt->regex_db != NULL) {
hs_free_database(hs_instance->hs_rt->regex_db);
hs_instance->hs_rt->regex_db = NULL;
}
if (hs_instance->hs_rt->literal_scratchs != NULL) {
if (hs_instance->hs_rt->literal_scratches != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->literal_scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->literal_scratchs[i]);
if (hs_instance->hs_rt->literal_scratches[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->literal_scratches[i]);
hs_instance->hs_rt->literal_scratches[i] = NULL;
}
}
}
FREE(hs_instance->hs_rt->literal_scratchs);
if (hs_instance->hs_rt->regex_scratchs != NULL) {
FREE(hs_instance->hs_rt->literal_scratches);
hs_instance->hs_rt->literal_scratches = NULL;
}
if (hs_instance->hs_rt->regex_scratches != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->regex_scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->regex_scratchs[i]);
if (hs_instance->hs_rt->regex_scratches[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->regex_scratches[i]);
hs_instance->hs_rt->regex_scratches[i] = NULL;
}
}
FREE(hs_instance->hs_rt->regex_scratches);
hs_instance->hs_rt->regex_scratches = NULL;
}
FREE(hs_instance->hs_rt->regex_scratchs);
if (hs_instance->hs_rt->bm != NULL) {
bool_matcher_free(hs_instance->hs_rt->bm);
hs_instance->hs_rt->bm = NULL;
}
FREE(hs_instance->hs_rt);
@@ -587,6 +599,7 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
struct adapter_hs_stream *hs_stream = ALLOC(struct adapter_hs_stream, 1);
hs_error_t err;
hs_stream->logger = hs_instance->logger;
hs_stream->thread_id = thread_id;
hs_stream->n_expr = hs_instance->n_expr;
hs_stream->hs_rt = hs_instance->hs_rt;
@@ -619,7 +632,7 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
return hs_stream;
error:
//TODO: hs_stream->hs_rt->scratchs[thread_id] may be free twice
//TODO: hs_stream->hs_rt->scratches[thread_id] may be free twice
if (hs_stream->literal_stream != NULL) {
hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL);
hs_stream->literal_stream = NULL;
@@ -686,28 +699,45 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
int thread_id = hs_stream->thread_id;
hs_stream->matched_pat->scan_data_len = data_len;
int err_scratch_flag = 0;
if (hs_stream->literal_stream != NULL) {
if (hs_stream->hs_rt->literal_scratches != NULL) {
err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
0, hs_stream->hs_rt->literal_scratchs[thread_id],
0, hs_stream->hs_rt->literal_scratches[thread_id],
matched_event_cb, hs_stream->matched_pat);
if (err != HS_SUCCESS) {
err_count++;
}
} else {
log_error(hs_stream->logger, MODULE_ADAPTER_HS, "literal_scratches is null, thread_id:%d",
thread_id);
err_scratch_flag++;
}
}
if (hs_stream->regex_stream != NULL) {
if (hs_stream->hs_rt->regex_scratches != NULL) {
err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
0, hs_stream->hs_rt->regex_scratchs[thread_id],
0, hs_stream->hs_rt->regex_scratches[thread_id],
matched_event_cb, hs_stream->matched_pat);
if (err != HS_SUCCESS) {
err_count++;
}
} else {
log_error(hs_stream->logger, MODULE_ADAPTER_HS, "regex_scratches is null, thread_id:%d",
thread_id);
err_scratch_flag++;
}
}
if (err_count == 2) {
return -1;
}
if (err_scratch_flag != 0) {
return -1;
}
size_t n_pattern_id = utarray_len(hs_stream->matched_pat->pattern_ids);
if (0 == n_pattern_id) {
*n_hit_result = 0;

1
src/maat_api.c
View File

@@ -387,7 +387,6 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] maat read full config failed", __FUNCTION__, __LINE__);
goto failed;
}
}

4
src/maat_config_monitor.c
View File

@@ -435,8 +435,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
}
if (!maat_instance->is_running) {
strncpy(maat_instance->json_ctx.json_file, json_filename,
sizeof(maat_instance->json_ctx.json_file));
size_t len = MIN(strlen(json_filename), sizeof(maat_instance->json_ctx.json_file));
strncpy(maat_instance->json_ctx.json_file, json_filename, len);
}
ret = stat(json_filename, &fstat_buf);

4
src/maat_rule.c
View File

@@ -478,10 +478,8 @@ void *rule_monitor_loop(void *arg)
ret = maat_read_full_config(maat_instance);
if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_RULE,
"[%s:%d] maat read full config failed, exit rule_monitor_loop thread.",
"[%s:%d] maat read full config failed",
__FUNCTION__, __LINE__);
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return NULL;
}
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));

158
test/maat_framework_gtest.cpp
View File

@@ -292,6 +292,130 @@ int ip_table_set_line(struct maat *maat_instance, const char *table_name, enum m
return maat_cmd_set_line(maat_instance, &line_rule);
}
int test_add_expr_command(struct maat *maat_instance, const char *expr_table,
long long compile_id, int timeout, const char *keywords)
{
char huge_serv_def[1024 * 2] = {0};
memset(huge_serv_def, 's', sizeof(huge_serv_def) - 1);
huge_serv_def[sizeof(huge_serv_def) - 1] = '\0';
int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id,
huge_serv_def, 1, timeout);
EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, timeout);
EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1);
ret = expr_table_set_line(maat_instance, expr_table, MAAT_OP_ADD, item_id, group_id, keywords,
"null", 1, 0, 0, 0);
EXPECT_EQ(ret, 1);
return ret;
}
int del_command(struct maat *maat_instance, int compile_id)
{
return compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_DEL, compile_id, NULL, 1, 0);
}
#if 0
class MaatStreamScan : public testing::Test
{
protected:
static void SetUpTestCase() {
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
_shared_maat_instance = maat_new(opts, table_info_path);
assert(_shared_maat_instance != NULL);
maat_cmd_flushDB(_shared_maat_instance);
maat_free(_shared_maat_instance);
maat_options_set_foreign_cont_dir(opts, "./foreign_files/");
maat_options_set_rule_effect_interval_ms(opts, 20 * 1000); //20s for garbage collection
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
}
static struct maat *_shared_maat_instance;
};
struct maat *MaatStreamScan::_shared_maat_instance;
TEST_F(MaatStreamScan, dynamic_config) {
const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char *scan_data2 = "hello world";
const char *table_name = "HTTP_URL";
const char *keywords1 = "action=search\\&query=(.*)";
const char *keywords2 = "hello";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_instance = MaatStreamScan::_shared_maat_instance;
struct maat_state *state = maat_state_new(maat_instance, thread_id);
long long compile1_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1);
int ret = test_add_expr_command(maat_instance, table_name, compile1_id, 0, keywords1);
EXPECT_EQ(ret, 1);
sleep(WAIT_FOR_EFFECTIVE_S * 2);
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_stream_new(maat_instance, table_id, state);
ASSERT_TRUE(sp != NULL);
ret = maat_stream_scan(sp, "www.cyberessays.com", strlen("www.cyberessays.com"),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_stream_scan(sp, scan_data1, strlen(scan_data1), results, ARRAY_SIZE,
&n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], compile1_id);
maat_state_reset(state);
long long compile2_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1);
ret = test_add_expr_command(maat_instance, table_name, compile2_id, 0, keywords2);
EXPECT_EQ(ret, 1);
for (int i = 0; i < 100; i++) {
ret = maat_stream_scan(sp, "www.cyberessays.com", strlen("www.cyberessays.com"),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_stream_scan(sp, scan_data2, strlen(scan_data2), results, ARRAY_SIZE,
&n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], compile2_id);
maat_state_reset(state);
usleep(500 * 1000);
compile2_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1);
ret = test_add_expr_command(maat_instance, table_name, compile2_id, 0, keywords2);
EXPECT_EQ(ret, 1);
}
maat_stream_free(sp);
maat_state_free(state);
sp = NULL;
state = NULL;
}
#endif
#if 1
class MaatFlagScan : public testing::Test
{
@@ -3318,36 +3442,6 @@ protected:
struct maat *MaatCmdTest::_shared_maat_instance;
int test_add_expr_command(struct maat *maat_instance, const char *expr_table,
long long compile_id, int timeout, const char *keywords)
{
char huge_serv_def[1024 * 2] = {0};
memset(huge_serv_def, 's', sizeof(huge_serv_def) - 1);
huge_serv_def[sizeof(huge_serv_def) - 1] = '\0';
int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id,
huge_serv_def, 1, timeout);
EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, timeout);
EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1);
ret = expr_table_set_line(maat_instance, expr_table, MAAT_OP_ADD, item_id, group_id, keywords,
"null", 1, 0, 0, 0);
EXPECT_EQ(ret, 1);
return ret;
}
int del_command(struct maat *maat_instance, int compile_id)
{
return compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_DEL, compile_id, NULL, 1, 0);
}
TEST_F(MaatCmdTest, SetIP) {
long long version_before = 0;
long long results[ARRAY_SIZE] = {0};
@@ -5269,7 +5363,7 @@ TEST_F(MaatCmdTest, StreamScanSegfaultWhenVersionRollBack_TSG6324) {
maat_state_free(state);
state = NULL;
}
#endif
class MaatRollbackTest : public testing::Test
{
protected:
@@ -5446,7 +5540,7 @@ TEST_F(MaatRollbackTest, FullConfigRollback) {
maat_state_free(state);
state = NULL;
}
#endif
int main(int argc, char ** argv)
{
int ret=0;

2
vendor/CMakeLists.txt vendored
View File

@@ -5,7 +5,7 @@ include(ExternalProject)
set(VENDOR_ROOT ${CMAKE_BINARY_DIR}/vendor)
set(VENDOR_BUILD ${CMAKE_BINARY_DIR}/vendor/vbuild)
set(CMAKE_C_FLAGS "-std=gnu99 -fPIC -Wall")
set(CMAKE_C_FLAGS "-std=c99 -fPIC -Wall")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall")
# GoogleTest