gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adapter_hs engine only construct stream db

Browse Source
This commit is contained in:
liuwentan
2023-03-17 17:28:52 +08:00
parent 68533f9d43
commit 37447eef7f
15 changed files with 79389 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -157,7 +157,6 @@ static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_work
static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
struct adpt_hs_compile_data *compile_data,
enum hs_pattern_type pattern_type,
enum hs_scan_mode scan_mode,
struct log_handle *logger)
{
hs_error_t err;
@@ -170,7 +169,7 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
if (pattern_type == HS_PATTERN_TYPE_STR) {
err = hs_compile_lit_multi((const char *const *)compile_data->patterns, compile_data->flags,
compile_data->ids, compile_data->pattern_lens, compile_data->n_patterns,
scan_mode, NULL, &hs_rt->literal_db, &compile_err);
HS_MODE_STREAM, NULL, &hs_rt->literal_db, &compile_err);
if (err != HS_SUCCESS) {
if (compile_err) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s",
@@ -181,8 +180,10 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
return -1;
}
} else {
err = hs_compile_multi((const char *const *)compile_data->patterns, compile_data->flags,
compile_data->ids, compile_data->n_patterns, scan_mode, NULL,
err = hs_compile_multi((const char *const *)compile_data->patterns,
compile_data->flags, compile_data->ids,
compile_data->n_patterns,
HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL,
&hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) {
if (compile_err) {
@@ -258,14 +259,12 @@ void hs_tag_free(struct hs_tag *tag)
FREE(tag);
}
struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
enum hs_pattern_type pattern_type,
struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
size_t n_worker_thread,
struct hs_expr *exprs, size_t n_expr,
struct log_handle *logger)
{
if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) ||
(pattern_type != HS_PATTERN_TYPE_STR && pattern_type != HS_PATTERN_TYPE_REG) ||
if ((pattern_type != HS_PATTERN_TYPE_STR && pattern_type != HS_PATTERN_TYPE_REG) ||
0 == n_worker_thread || NULL == exprs || 0 == n_expr) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] input parameters illegal!",
__FUNCTION__, __LINE__);
@@ -360,8 +359,18 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
//mytest
// for (size_t i = 0; i < n_expr; i++) {
// printf("hs_instance:%p exprs[%zu] expr_id:%llu, item_num:%zu\n", hs_instance, i, bool_exprs[i].expr_id, bool_exprs[i].item_num);
// if (bool_exprs[i].expr_id == 37)
// {
// printf("<before bool_matcher_new> exprs[%zu] expr_id:%llu, item_num:%zu\n",
// i, bool_exprs[i].expr_id, bool_exprs[i].item_num);
// printf("item_id: ");
// for (size_t j = 0; j < bool_exprs[i].item_num; j++)
// {
// printf("%llu ", bool_exprs[i].items[j].item_id);
// }
// }
// }
// printf("\n");
/* create bool matcher */
hs_instance->hs_rt->bm = bool_matcher_new(bool_exprs, n_expr, &mem_size);
@@ -380,7 +389,7 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
FREE(bool_exprs);
/* build hs database */
ret = adpt_hs_build_database(hs_instance->hs_rt, compile_data, pattern_type, scan_mode, logger);
ret = adpt_hs_build_database(hs_instance->hs_rt, compile_data, pattern_type, logger);
if (ret < 0) {
goto error;
}
@@ -466,7 +475,7 @@ int matched_event_cb(unsigned int id, unsigned long long from,
unsigned long long pattern_id = id;
if (utarray_find(matched_pat_container->pat_ids, &pattern_id, compare_pattern_id)) {
return -1;
return 0;
}
utarray_push_back(matched_pat_container->pat_ids, &pattern_id);
@@ -540,106 +549,6 @@ int hs_tag_validate(struct hs_tag *hs_tag, struct matched_pattern_container *mat
return 0;
}
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
const char *data, size_t data_len,
struct hs_scan_result *results,
size_t n_result, size_t *n_hit_result)
{
if (NULL == hs_instance || NULL == data || (0 == data_len) ||
NULL == results || 0 == n_result || NULL == n_hit_result) {
return -1;
}
struct adapter_hs_runtime *hs_rt = hs_instance->hs_rt;
hs_scratch_t *scratch = hs_rt->scratchs[thread_id];
hs_error_t err;
struct matched_pattern_container matched_pat_container;
matched_pat_container.pat_hash = NULL;
utarray_new(matched_pat_container.pat_ids, &ut_pattern_id_icd);
utarray_reserve(matched_pat_container.pat_ids, hs_instance->n_patterns);
int err_count = 0;
if (hs_rt->literal_db != NULL) {
err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch,
matched_event_cb, &matched_pat_container);
if (err != HS_SUCCESS) {
err_count++;
}
}
if (hs_rt->regex_db != NULL) {
err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch,
matched_event_cb, &matched_pat_container);
if (err != HS_SUCCESS) {
err_count++;
}
}
if (err_count > 0) {
utarray_free(matched_pat_container.pat_ids);
return -1;
}
size_t matched_pattern_ids_cnt = utarray_len(matched_pat_container.pat_ids);
if (0 == matched_pattern_ids_cnt) {
*n_hit_result = 0;
utarray_free(matched_pat_container.pat_ids);
assert(matched_pat_container.pat_hash == NULL);
return 0;
}
size_t i = 0;
unsigned long long items[matched_pattern_ids_cnt];
memset(items, 0, sizeof(unsigned long long) * matched_pattern_ids_cnt);
for (i = 0; i < matched_pattern_ids_cnt; i++) {
items[i] = *(unsigned long long *)utarray_eltptr(matched_pat_container.pat_ids, i);
}
int ret = 0;
int real_matched_index = 0;
struct hs_tag *hs_tag = NULL;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr);
int bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, matched_pattern_ids_cnt,
bool_matcher_results, hs_instance->n_expr);
if (bool_matcher_ret < 0) {
ret = -1;
goto next;
}
if (bool_matcher_ret > (int)n_result) {
bool_matcher_ret = n_result;
}
for (int index = 0; index < bool_matcher_ret; index++) {
hs_tag = (struct hs_tag *)bool_matcher_results[index].user_tag;
int tag_ret = hs_tag_validate(hs_tag, &matched_pat_container, data_len);
if (tag_ret < 0) {
//bool_matcher_results[index] is invalid hit, continue
continue;
}
results[real_matched_index].item_id = bool_matcher_results[index].expr_id;
results[real_matched_index].user_tag = hs_tag->user_tag;
real_matched_index++;
}
*n_hit_result = real_matched_index;
next:
FREE(bool_matcher_results);
struct matched_pattern *pattern = NULL, *tmp_pattern = NULL;
HASH_ITER(hh, matched_pat_container.pat_hash, pattern, tmp_pattern) {
HASH_DELETE(hh, matched_pat_container.pat_hash, pattern);
FREE(pattern);
}
utarray_free(matched_pat_container.pat_ids);
return ret;
}
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id)
{
if (NULL == hs_instance || thread_id < 0) {
@@ -675,6 +584,44 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
return hs_stream;
}
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
{
if (NULL == hs_stream) {
return;
}
int thread_id = hs_stream->thread_id;
if (hs_stream->hs_rt != NULL) {
if (hs_stream->literal_stream != NULL) {
hs_close_stream(hs_stream->literal_stream,
hs_stream->hs_rt->scratchs[thread_id],
NULL, NULL);
hs_stream->literal_stream = NULL;
}
if (hs_stream->regex_stream != NULL) {
hs_close_stream(hs_stream->regex_stream,
hs_stream->hs_rt->scratchs[thread_id],
NULL, NULL);
hs_stream->regex_stream = NULL;
}
}
if (hs_stream->matched_pat_container.pat_hash != NULL) {
struct matched_pattern *pattern = NULL, *tmp_pattern = NULL;
HASH_ITER(hh, hs_stream->matched_pat_container.pat_hash, pattern, tmp_pattern) {
HASH_DELETE(hh, hs_stream->matched_pat_container.pat_hash, pattern);
FREE(pattern);
}
}
utarray_free(hs_stream->matched_pat_container.pat_ids);
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */
hs_stream->hs_rt = NULL;
FREE(hs_stream);
}
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len,
struct hs_scan_result *results, size_t n_result, size_t *n_hit_result)
{
@@ -701,7 +648,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, &hs_stream->matched_pat_container);
if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
if (err != HS_SUCCESS) {
err_count++;
}
}
@@ -710,7 +657,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, &hs_stream->matched_pat_container);
if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
if (err != HS_SUCCESS) {
err_count++;
}
}
@@ -732,10 +679,9 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
}
int ret = 0;
int matched_index = 0;
struct bool_expr_match *bool_matcher_results = NULL;
bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
int real_matched_index = 0;
struct hs_tag *hs_tag = NULL;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, matched_pattern_ids_cnt,
bool_matcher_results, hs_stream->n_expr);
if (bool_matcher_ret < 0) {
@@ -747,11 +693,20 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
bool_matcher_ret = n_result;
}
for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
results[matched_index].item_id = bool_matcher_results[matched_index].expr_id;
results[matched_index].user_tag = bool_matcher_results[matched_index].user_tag;
for (int index = 0; index < bool_matcher_ret; index++) {
hs_tag = (struct hs_tag *)bool_matcher_results[index].user_tag;
int tag_ret = hs_tag_validate(hs_tag, &hs_stream->matched_pat_container, data_len);
if (tag_ret < 0) {
//bool_matcher_results[index] is invalid hit, continue
continue;
}
results[real_matched_index].item_id = bool_matcher_results[index].expr_id;
results[real_matched_index].user_tag = hs_tag->user_tag;
real_matched_index++;
}
*n_hit_result = bool_matcher_ret;
*n_hit_result = real_matched_index;
next:
FREE(bool_matcher_results);
@@ -765,38 +720,19 @@ next:
return ret;
}
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
const char *data, size_t data_len,
struct hs_scan_result *results,
size_t n_result, size_t *n_hit_result)
{
if (NULL == hs_stream) {
return;
if (NULL == hs_instance || NULL == data || (0 == data_len) ||
NULL == results || 0 == n_result || NULL == n_hit_result) {
return -1;
}
int thread_id = hs_stream->thread_id;
struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(hs_instance, thread_id);
int ret = adapter_hs_scan_stream(hs_stream, data, data_len, results, n_result, n_hit_result);
adapter_hs_stream_close(hs_stream);
if (hs_stream->hs_rt != NULL) {
if (hs_stream->literal_stream != NULL) {
hs_close_stream(hs_stream->literal_stream,
hs_stream->hs_rt->scratchs[thread_id],
NULL, NULL);
hs_stream->literal_stream = NULL;
}
if (hs_stream->regex_stream != NULL) {
hs_close_stream(hs_stream->regex_stream,
hs_stream->hs_rt->scratchs[thread_id],
NULL, NULL);
hs_stream->regex_stream = NULL;
}
}
struct matched_pattern *pattern = NULL, *tmp_pattern = NULL;
HASH_ITER(hh, hs_stream->matched_pat_container.pat_hash, pattern, tmp_pattern) {
HASH_DELETE(hh, hs_stream->matched_pat_container.pat_hash, pattern);
FREE(pattern);
}
utarray_free(hs_stream->matched_pat_container.pat_ids);
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */
hs_stream->hs_rt = NULL;
FREE(hs_stream);
return ret;
}

14
scanner/adapter_hs/adapter_hs.h
View File

@@ -25,13 +25,6 @@ extern "C"
struct adapter_hs;
/* scan mode */
enum hs_scan_mode {
HS_SCAN_MODE_BLOCK = 1,
HS_SCAN_MODE_STREAM,
HS_SCAN_MODE_MAX
};
/* match method */
enum hs_match_mode {
HS_MATCH_MODE_INVALID = -1,
@@ -89,15 +82,14 @@ struct hs_expr {
/**
* @brief initialize adapter_hs instance
*
* @param scan_mode: the following scan as block or stream mode
* @param pattern_type: pure literal or regex expression
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
* @param expr_array: logic AND expression's array
* @param n_expr_arrays: the number of logic AND expression's array
*
* @retval the pointer to adapter_hs instance
*/
struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
enum hs_pattern_type pattern_type,
struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
size_t n_worker_thread,
struct hs_expr *exprs, size_t n_expr,
struct log_handle *logger);
@@ -140,4 +132,4 @@ void adapter_hs_stream_close(struct adapter_hs_stream *stream);
}
#endif
#endif
#endif

8
src/maat_ex_data.c
View File

@@ -202,21 +202,15 @@ struct ex_container *ex_container_new(void *ex_data, void *custom_data)
void ex_container_free(void *schema, void *data)
{
/* schema is NULL if not call ex_data_runtime_set_ex_container_schema */
if (NULL == data) {
if (NULL == schema || NULL == data) {
return;
}
struct ex_container *ex_container = (struct ex_container *)data;
//TODO:
if (NULL == schema) {
}
struct ex_container_schema *container_schema = (struct ex_container_schema *)schema;
long argl = container_schema->ex_schema->argl;
void *argp = container_schema->ex_schema->argp;
if (ex_container->ex_data != NULL
&& container_schema->ex_schema->free_func != NULL) {
container_schema->ex_schema->free_func(container_schema->table_id,

31
src/maat_expr.c
View File

@@ -34,7 +34,6 @@ struct expr_schema {
int expr_type_column;
int match_method_column;
int is_hexbin_column;
enum hs_scan_mode scan_mode; /* adapter_hs scan mode */
enum hs_pattern_type pattern_type; /* literal or regex */
int table_id; //ugly
struct table_manager *ref_tbl_mgr;
@@ -69,7 +68,6 @@ struct expr_item {
};
struct expr_runtime {
enum hs_scan_mode scan_mode;
enum hs_pattern_type pattern_type;
struct adapter_hs *hs;
struct adapter_hs_stream *hs_stream;
@@ -277,21 +275,6 @@ void expr_item_free(struct expr_item *expr_item)
FREE(expr_item);
}
enum hs_scan_mode scan_mode_str_to_enum(const char *mode_str)
{
enum hs_scan_mode scan_mode = HS_SCAN_MODE_MAX;
if (strcmp(mode_str, "block") == 0) {
scan_mode = HS_SCAN_MODE_BLOCK;
} else if (strcmp(mode_str, "stream") == 0) {
scan_mode = HS_SCAN_MODE_STREAM;
} else {
assert(0);
}
return scan_mode;
}
enum hs_pattern_type pattern_type_str_to_enum(const char *type_str)
{
enum hs_pattern_type pattern_type = HS_PATTERN_TYPE_MAX;
@@ -327,12 +310,6 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
goto error;
}
custom_item = cJSON_GetObjectItem(item, "scan_mode");
if (custom_item != NULL && custom_item->type == cJSON_String) {
expr_schema->scan_mode = scan_mode_str_to_enum(custom_item->valuestring);
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "pattern_type");
if (custom_item != NULL && custom_item->type == cJSON_String) {
expr_schema->pattern_type = pattern_type_str_to_enum(custom_item->valuestring);
@@ -383,7 +360,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
expr_schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 9) {
if (read_cnt < 8) {
goto error;
}
@@ -436,7 +413,6 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num,
expr_rt->htable = rcu_hash_new(expr_ex_data_free);
expr_rt->item_htable = rcu_hash_new(expr_maat_item_free);
expr_rt->scan_mode = schema->scan_mode;
expr_rt->pattern_type = schema->pattern_type;
expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin;
@@ -809,9 +785,8 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
int ret = 0;
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->pattern_type,
expr_rt->n_worker_thread, rules, rule_cnt,
expr_rt->logger);
new_adapter_hs = adapter_hs_initialize(expr_rt->pattern_type, expr_rt->n_worker_thread,
rules, rule_cnt, expr_rt->logger);
if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild adapter_hs engine failed when update %zu expr rules",

5
test/CMakeLists.txt
View File

@@ -24,6 +24,9 @@ add_executable(maat_ex_data_gtest maat_ex_data_gtest.cpp)
target_link_libraries(maat_ex_data_gtest maat_frame_static gtest_static)
file(COPY table_info.conf DESTINATION ./)
file(COPY file_test_tableinfo.conf DESTINATION ./)
file(COPY literal_expr.conf DESTINATION ./)
file(COPY regex_expr.conf DESTINATION ./)
file(COPY maat_json.json DESTINATION ./)
file(COPY maat_json.json DESTINATION ./)
file(COPY ntcrule DESTINATION ./)
file(COPY testdata DESTINATION ./)

67
test/adapter_hs_gtest.cpp
View File

@@ -204,44 +204,23 @@ void expr_array_free(struct hs_expr expr_array[], size_t n_expr_array)
}
}
TEST(block_mode_initialize, invalid_input_parameter)
{
struct adapter_hs *hs_instance = NULL;
struct hs_expr exprs[64];
/* case1: invalid scan_mode parameter */
hs_instance = adapter_hs_initialize(HS_SCAN_MODE_MAX, HS_PATTERN_TYPE_REG,
1, exprs, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
/* case2: invalid expr parameter */
hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG,
1, NULL, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
/* case3: invalid expr num */
hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG,
1, exprs, 0, g_logger);
EXPECT_TRUE(hs_instance == NULL);
}
TEST(block_mode_scan, invalid_input_parameter)
TEST(adapter_hs_init, invalid_input_parameter)
{
struct hs_expr expr_array[64];
size_t n_expr_array = 0;
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG,
1, NULL, 0, g_logger);
EXPECT_TRUE(hs_instance == NULL);
hs_instance = adapter_hs_initialize(HS_SCAN_MODE_MAX, HS_PATTERN_TYPE_REG, 1, expr_array,
hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array,
n_expr_array, g_logger);
EXPECT_TRUE(hs_instance == NULL);
n_expr_array = 1;
expr_array[0].expr_id = 101;
expr_array[0].n_patterns = 10;
hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG, 1, expr_array,
hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array,
n_expr_array, g_logger);
EXPECT_TRUE(hs_instance == NULL);
@@ -249,12 +228,12 @@ TEST(block_mode_scan, invalid_input_parameter)
n_expr_array = 1;
expr_array[0].expr_id = 101;
expr_array[0].n_patterns = 1;
hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG, 1, expr_array,
hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array,
n_expr_array, g_logger);
EXPECT_TRUE(hs_instance == NULL);
}
TEST(block_mode_scan, literal_sub_has_normal_offset)
TEST(adapter_hs_scan, literal_sub_has_normal_offset)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
@@ -263,7 +242,7 @@ TEST(block_mode_scan, literal_sub_has_normal_offset)
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_expr_array, 11);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -295,7 +274,7 @@ TEST(block_mode_scan, literal_sub_has_normal_offset)
hs_instance = NULL;
}
TEST(block_mode_scan, literal_sub_has_left_unlimit_offset)
TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
@@ -303,7 +282,7 @@ TEST(block_mode_scan, literal_sub_has_left_unlimit_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -336,7 +315,7 @@ TEST(block_mode_scan, literal_sub_has_left_unlimit_offset)
hs_instance = NULL;
}
TEST(block_mode_scan, literal_sub_has_right_unlimit_offset)
TEST(adapter_hs_scan, literal_sub_has_right_unlimit_offset)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
@@ -344,7 +323,7 @@ TEST(block_mode_scan, literal_sub_has_right_unlimit_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -392,7 +371,7 @@ TEST(block_mode_scan, literal_sub_has_right_unlimit_offset)
hs_instance = NULL;
}
TEST(block_mode_scan, literal_sub_with_no_offset)
TEST(adapter_hs_scan, literal_sub_with_no_offset)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
@@ -400,7 +379,7 @@ TEST(block_mode_scan, literal_sub_with_no_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -440,7 +419,7 @@ TEST(block_mode_scan, literal_sub_with_no_offset)
hs_instance = NULL;
}
TEST(block_mode_scan, literal_exactly)
TEST(adapter_hs_scan, literal_exactly)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
@@ -448,7 +427,7 @@ TEST(block_mode_scan, literal_exactly)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -482,7 +461,7 @@ TEST(block_mode_scan, literal_exactly)
hs_instance = NULL;
}
TEST(block_mode_scan, literal_prefix)
TEST(adapter_hs_scan, literal_prefix)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
@@ -490,7 +469,7 @@ TEST(block_mode_scan, literal_prefix)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -533,7 +512,7 @@ TEST(block_mode_scan, literal_prefix)
hs_instance = NULL;
}
TEST(block_mode_scan, literal_suffix)
TEST(adapter_hs_scan, literal_suffix)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
@@ -541,7 +520,7 @@ TEST(block_mode_scan, literal_suffix)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -584,7 +563,7 @@ TEST(block_mode_scan, literal_suffix)
hs_instance = NULL;
}
TEST(block_mode_scan, literal_sub_with_hexbin)
TEST(adapter_hs_scan, literal_sub_with_hexbin)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
@@ -592,7 +571,7 @@ TEST(block_mode_scan, literal_sub_with_hexbin)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -617,7 +596,7 @@ TEST(block_mode_scan, literal_sub_with_hexbin)
hs_instance = NULL;
}
TEST(block_mode_scan, literal_with_chinese)
TEST(adapter_hs_scan, literal_with_chinese)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
@@ -625,7 +604,7 @@ TEST(block_mode_scan, literal_with_chinese)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1,
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);

7
test/file_test_tableinfo.conf
View File

@@ -81,7 +81,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -97,7 +96,6 @@
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -114,7 +112,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -130,7 +127,6 @@
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -147,7 +143,6 @@
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -164,7 +159,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -180,7 +174,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,

469
test/maat_framework_gtest.cpp
View File

@@ -539,28 +539,25 @@ protected:
struct maat *MaatStringScan::_shared_maat_instance;
struct log_handle *MaatStringScan::logger;
TEST_F(MaatStringScan, Expr8) {
const char *table_name = "KEYWORDS_TABLE";
TEST_F(MaatStringScan, Full) {
const char *table_name = "HTTP_URL_REGEX";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567";
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 182);
struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0};
int n_read = 0;
n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE);
EXPECT_NE(n_read, 0);
EXPECT_EQ(results[0], 125);
maat_state_free(&state);
}
#if 0
TEST_F(MaatStringScan, Regex) {
int ret = 0;
long long results[ARRAY_SIZE] = {0};
@@ -599,7 +596,7 @@ TEST_F(MaatStringScan, Regex) {
}
#endif
}
#endif
TEST_F(MaatStringScan, ExprPlus) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
@@ -632,72 +629,14 @@ TEST_F(MaatStringScan, ExprPlus) {
EXPECT_EQ(results[0], 190);
maat_state_free(&state);
}
TEST_F(MaatStringScan, ExprAndExprPlus) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *expr_table_name = "HTTP_URL_LITERAL";
const char *expr_plus_table_name = "HTTP_SIGNATURE";
const char *region_name = "I love China";
const char *scan_data = "today is Monday and yesterday is Tuesday";
int expr_table_id = maat_get_table_id(maat_instance, expr_table_name);
int expr_plus_table_id = maat_get_table_id(maat_instance, expr_plus_table_name);
int ret = maat_scan_string(maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_ERR);
ret = maat_state_set_scan_district(maat_instance, &state, region_name, strlen(region_name));
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
ret = maat_scan_string(maat_instance, expr_table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 195);
maat_state_free(&state);
}
TEST_F(MaatStringScan, StreamInput) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char *table_name = "HTTP_URL_REGEX";
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_scan_stream_open(maat_instance, table_id, 0);
ASSERT_TRUE(sp != NULL);
int ret = maat_scan_stream(&sp, "www.cyberessays.com", strlen("www.cyberessays.com"),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_scan_stream(&sp, scan_data, strlen(scan_data), results, ARRAY_SIZE,
&n_hit_result, &state);
maat_scan_stream_close(&sp);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 125);
maat_state_free(&state);
}
//TODO:
#if 0
TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
long long results[ARRAY_SIZE] = {0};
TEST_F(MaatStringScan, ExprPlusWithOffset)
{
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *region_name = "tcp.payload";
const char *region_name = "Payload";
unsigned char udp_payload_not_hit[] = { /* Stun packet */
0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42,
0x4f, 0xc2, 0xc2, 0x70, 0xb3, 0xa8, 0x4e, 0x22,
@@ -710,17 +649,36 @@ TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f,
0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64,
0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a,
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a, 0xab, 0x00 };
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a };
unsigned char udp_payload_hit[] = { /* Stun packet */ //rule:"1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d"
0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42, //1-1:03
0x4f, 0xc2, 0x2d, 0x70, 0xb3, 0xa8, 0x4e, 0x2d, //10-10:2d
0x34, 0x22, 0x87, 0x4c, 0x2d, 0x00, 0x00, 0x46, //15-16:2d34
0x2d, 0x34, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01, //20-20:2d
0x03, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a, //24-24:2d
0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9,
0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b,
0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2,
0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f,
0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64,
0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a,
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a };
int table_id = maat_get_table_id(maat_instance, "APP_PAYLOAD");
ASSERT_GT(table_id, 0);
int ret = maat_state_set_scan_district(maat_instance, &state, region_name, strlen(region_name));
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_instance, table_id, 0, (char *)udp_payload_not_hit, sizeof(udp_payload_not_hit),
EXPECT_EQ(ret, 0);
ret = maat_scan_string(maat_instance, table_id, 0, (char*)udp_payload_not_hit, sizeof(udp_payload_not_hit),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_scan_string(maat_instance, table_id, 0, (char*)udp_payload_hit, sizeof(udp_payload_hit),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 148);
maat_state_free(&state);
}
@@ -757,14 +715,43 @@ TEST_F(MaatStringScan, ExprPlusWithHex) {
EXPECT_EQ(results[0], 132);
maat_state_free(&state);
}
TEST_F(MaatStringScan, ExprPlusWithOffset)
{
long long results[ARRAY_SIZE] = {0};
#endif
TEST_F(MaatStringScan, ExprAndExprPlus) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *region_name = "Payload";
const char *expr_table_name = "HTTP_URL_LITERAL";
const char *expr_plus_table_name = "HTTP_SIGNATURE";
const char *region_name = "I love China";
const char *scan_data = "today is Monday and yesterday is Tuesday";
int expr_table_id = maat_get_table_id(maat_instance, expr_table_name);
int expr_plus_table_id = maat_get_table_id(maat_instance, expr_plus_table_name);
int ret = maat_scan_string(maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_ERR);
ret = maat_state_set_scan_district(maat_instance, &state, region_name, strlen(region_name));
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
ret = maat_scan_string(maat_instance, expr_table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 195);
maat_state_free(&state);
}
#if 0
TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *region_name = "tcp.payload";
unsigned char udp_payload_not_hit[] = { /* Stun packet */
0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42,
0x4f, 0xc2, 0xc2, 0x70, 0xb3, 0xa8, 0x4e, 0x22,
@@ -777,39 +764,294 @@ TEST_F(MaatStringScan, ExprPlusWithOffset)
0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f,
0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64,
0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a,
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a };
unsigned char udp_payload_hit[] = { /* Stun packet */ //rule:"1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d"
0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42, //1-1:03
0x4f, 0xc2, 0x2d, 0x70, 0xb3, 0xa8, 0x4e, 0x2d, //10-10:2d
0x34, 0x22, 0x87, 0x4c, 0x2d, 0x00, 0x00, 0x46, //15-16:2d34&20-20:2d
0x2d, 0x34, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01, //24-24:2d
0x03, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a,
0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9,
0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b,
0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2,
0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f,
0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64,
0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a,
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a };
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a, 0xab, 0x00 };
int table_id = maat_get_table_id(maat_instance, "APP_PAYLOAD");
ASSERT_GT(table_id, 0);
int ret = maat_state_set_scan_district(maat_instance, &state, region_name, strlen(region_name));
EXPECT_EQ(ret, 0);
ret = maat_scan_string(maat_instance, table_id, 0, (char*)udp_payload_not_hit, sizeof(udp_payload_not_hit),
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_instance, table_id, 0, (char *)udp_payload_not_hit, sizeof(udp_payload_not_hit),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_scan_string(maat_instance, table_id, 0, (char*)udp_payload_hit, sizeof(udp_payload_hit),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 148);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
maat_state_free(&state);
}
#endif
TEST_F(MaatStringScan, Expr8) {
const char *table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 182);
struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0};
int n_read = 0;
n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE);
EXPECT_NE(n_read, 0);
maat_state_free(&state);
}
TEST_F(MaatStringScan, HexBinCaseSensitive) {
const char *table_name = "KEYWORDS_TABLE";
const char *scan_data1 = "String TeST should not hit.";
const char *scan_data2 = "String TEST should hit";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
maat_state_free(&state);
ret = maat_scan_string(maat_instance, table_id, 0, scan_data2, strlen(scan_data2),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191);
maat_state_free(&state);
}
TEST_F(MaatStringScan, BugReport20190325) {
unsigned char scan_data[] = {/* Packet 1 */
0x01, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00,
0x00, 0xf4, 0x01, 0x00, 0x00, 0x32, 0x00, 0x00,
0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
0x2d, 0x3d, 0x3d, 0x20, 0x48, 0x3d, 0x48, 0x20,
0x3d, 0x3d, 0x2d, 0x3a, 0x00, 0x02, 0x00, 0x00,
0x00, 0x07, 0x0e, 0x00, 0x00, 0xe8, 0x03, 0x00,
0x00, 0x4c, 0x69, 0x6e, 0x75, 0x78, 0x20, 0x33,
0x2e, 0x31, 0x39, 0x2e, 0x30, 0x2d, 0x31, 0x35,
0x2d, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63,
0x00, 0x31, 0x3a, 0x47, 0x32, 0x2e, 0x34, 0x30,
0x00};
const char *table_name = "TROJAN_PAYLOAD";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, (char *)scan_data, sizeof(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 150);
maat_state_free(&state);
}
#if 0
TEST_F(MaatStringScan, PrefixAndSuffix) {
const char *hit_twice = "ceshi3@mailhost.cn";
const char *hit_suffix = "11111111111ceshi3@mailhost.cn";
const char *hit_prefix = "ceshi3@mailhost.cn11111111111";
const char *cont_sz_table_name = "CONTENT_SIZE";
const char *mail_addr_table_name = "MAIL_ADDR";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int cont_sz_table_id = maat_get_table_id(maat_instance, cont_sz_table_name);
ASSERT_GT(cont_sz_table_id, 0);
int mail_addr_table_id = maat_get_table_id(maat_instance, mail_addr_table_name);
ASSERT_GT(mail_addr_table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_integer(maat_instance, cont_sz_table_id, 0, 2015, results,
ARRAY_SIZE, &n_hit_result, &state);
ret = maat_scan_string(maat_instance, mail_addr_table_id, 0, hit_twice, strlen(hit_twice),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 151);
EXPECT_EQ(results[1], 152);
maat_state_free(&state);
ret = maat_scan_string(maat_instance, mail_addr_table_id, 0, hit_suffix, strlen(hit_suffix),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 151);
ret = maat_scan_integer(maat_instance, cont_sz_table_id, 0, 2015, results,
ARRAY_SIZE, &n_hit_result, &state);
ret = maat_scan_string(maat_instance, mail_addr_table_id, 0, hit_prefix, strlen(hit_prefix),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 152);
maat_state_free(&state);
}
TEST_F(MaatStringScan, MaatUnescape) {
const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char *table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 132);
maat_state_free(&state);
}
TEST_F(MaatStringScan, RegexWithNotContains) {
const char *should_NOT_hit_scan_data = "new.qq.com/rain/a/TWF2021042600418000";
const char *should_hit_scan_data = "fakesina.com/rain/a/TWF2021042600418000";
const char *table_name = "HTTP_URL_REGEX";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, should_NOT_hit_scan_data,
strlen(should_NOT_hit_scan_data), results, ARRAY_SIZE,
&n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
maat_state_free(&state);
ret = maat_scan_string(maat_instance, table_id, 0, should_hit_scan_data,
strlen(should_hit_scan_data), results, ARRAY_SIZE,
&n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 183);
maat_state_free(&state);
}
#endif
TEST_F(MaatStringScan, OffsetChunk64) {
const char *table_name = "IMAGE_FP";
const char *file_name = "./testdata/mesa_logo.jpg";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
FILE *fp = fopen(file_name, "r");
ASSERT_FALSE(fp==NULL);
char scan_data[64];
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_scan_stream_open(maat_instance, table_id, 0);
ASSERT_TRUE(sp != NULL);
int ret = 0;
int read_size = 0;
int pass_flag = 0;
while (0 == feof(fp)) {
read_size = fread(scan_data, 1, sizeof(scan_data), fp);
ret = maat_scan_stream(&sp, scan_data, read_size,
results, ARRAY_SIZE, &n_hit_result, &state);
if (ret > 0) {
pass_flag = 1;
break;
}
}
EXPECT_EQ(pass_flag, 1);
EXPECT_EQ(results[0], 136);
maat_scan_stream_close(&sp);
fclose(fp);
maat_state_free(&state);
}
TEST_F(MaatStringScan, OffsetChunk1460) {
const char *table_name = "IMAGE_FP";
const char *file_name = "./testdata/mesa_logo.jpg";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
FILE *fp = fopen(file_name, "r");
ASSERT_FALSE(fp==NULL);
char scan_data[1460];
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_scan_stream_open(maat_instance, table_id, 0);
ASSERT_TRUE(sp != NULL);
int ret = 0;
int read_size = 0;
int pass_flag = 0;
while (0 == feof(fp)) {
read_size = fread(scan_data, 1, sizeof(scan_data), fp);
ret = maat_scan_stream(&sp, scan_data, read_size,
results, ARRAY_SIZE, &n_hit_result, &state);
if (ret > 0) {
pass_flag = 1;
break;
}
}
EXPECT_EQ(pass_flag, 1);
EXPECT_EQ(results[0], 136);
maat_scan_stream_close(&sp);
fclose(fp);
maat_state_free(&state);
}
TEST_F(MaatStringScan, StreamInput) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char *table_name = "HTTP_URL_REGEX";
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_scan_stream_open(maat_instance, table_id, 0);
ASSERT_TRUE(sp != NULL);
int ret = maat_scan_stream(&sp, "www.cyberessays.com", strlen("www.cyberessays.com"),
results, ARRAY_SIZE, &n_hit_result, &state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_scan_stream(&sp, scan_data, strlen(scan_data), results, ARRAY_SIZE,
&n_hit_result, &state);
maat_scan_stream_close(&sp);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 125);
maat_state_free(&state);
}
TEST_F(MaatStringScan, dynamic_config) {
const char *table_name = "HTTP_URL_LITERAL";
char data[128] = "hello world, welcome to maat version4, it's funny.";
@@ -2460,20 +2702,19 @@ class MaatFileTest : public testing::Test
{
protected:
static void SetUpTestCase() {
struct maat_options *opts = maat_options_new();
char json_path[PATH_MAX] = {0};
const char *rule_folder = "./ntcrule/full/index";
const char *table_info = "./file_test_tableinfo.conf";
snprintf(json_path, sizeof(json_path), "./%s", json_filename);
maat_options_set_json_file(opts, json_path);
struct maat_options *opts = maat_options_new();
maat_options_set_iris(opts, rule_folder, rule_folder);
maat_options_set_logger_path(opts, "./maat_input_mode_gtest.log");
struct maat *maat_instance = maat_new(opts, table_info);
EXPECT_TRUE(maat_instance != NULL);
_shared_maat_instance = maat_new(opts, table_info);
EXPECT_TRUE(_shared_maat_instance != NULL);
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
}
static struct maat *_shared_maat_instance;

5
test/maat_json.json
View File

@@ -878,11 +878,12 @@
]
},
{
"compile_id": 148,
"is_valid": "yes",
"do_log": 0,
"effective_rage": 0,
"action": 0,
"compile_id": 148,
"service": 0,
"do_blacklist": 0,
"user_region": "StringScan.ExprPlusWithOffset",
@@ -895,7 +896,7 @@
"format": "hexbin",
"match_method": "sub",
"district": "Payload",
"keywords": "1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d",
"keywords": "1-1:03&9-10:2d&14-16:2d34",
"expr_type": "offset"
},
"table_type": "expr_plus"

13
test/table_info.conf
View File

@@ -73,7 +73,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -89,7 +88,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"stream",
"pattern_type":"regex",
"item_id":1,
"group_id":2,
@@ -105,7 +103,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -158,7 +155,6 @@
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -175,7 +171,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -236,8 +231,7 @@
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"regex",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"district":3,
@@ -253,7 +247,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -270,7 +263,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -349,7 +341,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -377,7 +368,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
@@ -435,7 +425,6 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,

77893
test/testdata/bool-matcher-test-exprs.txt vendored Normal file
View File

File diff suppressed because it is too large Load Diff

48
test/testdata/charsetWindows1251.txt vendored Normal file
View File

@@ -0,0 +1,48 @@
<EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>!','JS_CORE_WINDOW_AUTH':'<27><><EFBFBD><EFBFBD><EFBFBD>','JS_CORE_IMAGE_FULL':'<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>'});</script>
<script type="text/javascript">(window.BX||top.BX).message({'LANGUAGE_ID':'ru','FORMAT_DATE':'DD.MM.YYYY','FORMAT_DATETIME':'DD.MM.YYYY HH:MI:SS','COOKIE_PREFIX':'BITRIX_SM','USER_ID':'','SERVER_TIME':'1578340589','SERVER_TZ_OFFSET':'18000','USER_TZ_OFFSET':'0','USER_TZ_AUTO':'Y','bitrix_sessid':'fadf964e9f5bb819f212e5abf5ffb255','SITE_ID':'s1'});</script>
<script type="text/javascript" src="/bitrix/cache/js/s1/web20/kernel_main/kernel_main.js?1402043622360126"></script>
<script type="text/javascript" src="/bitrix/js/main/rsasecurity.js?136876011925044"></script>
<script type="text/javascript">BX.setCSSList(['/bitrix/js/main/core/css/core.css','/bitrix/js/main/core/css/core_popup.css','/bitrix/js/main/core/css/core_date.css','/bitrix/js/main/core/css/core.css','/bitrix/js/main/core/css/core_popup.css','/bitrix/js/main/core/css/core_date.css']); </script>
<script type="text/javascript">BX.setJSList(['/bitrix/js/main/core/core.js','/bitrix/js/main/core/core_ajax.js','/bitrix/js/main/session.js','/bitrix/js/main/json/json2.min.js','/bitrix/js/main/core/core_ls.js','/bitrix/js/main/core/core_window.js','/bitrix/js/main/utils.js','/bitrix/js/main/core/core_popup.js','/bitrix/js/main/core/core_date.js','/bitrix/js/main/dd.js']); </script>
<script type="text/javascript">
bxSession.Expand(1440, 'fadf964e9f5bb819f212e5abf5ffb255', false, '35a74b06af8f9ea55ffbda20075b0894');
</script>
<script>new Image().src='http://www.sgaice.ru/bitrix/spread.php?s=QklUUklYX1NNX0dVRVNUX0lEATY4MTg5NQExNjA5NDQ0NTg5AS8BAQECQklUUklYX1NNX0xBU1RfVklTSVQBMDcuMDEuMjAyMCAwMDo1NjoyOQExNjA5NDQ0NTg5AS8BAQEC&k=71d3b79b44f9716b27b47feab4a206cf';
</script>
<script type="text/javascript" src="/bitrix/cache/js/s1/web20/template_1e341eb2f86845c7519566374f51d35a/template_1e341eb2f86845c7519566374f51d35a_368c1a68876fd1c32b307a10695f3654.js?14010848191120"></script>
<script type="text/javascript" src="/bitrix/js/imgzoom/thumb.js"></script>
<meta name="google-site-verification" content="gL_64SaiDgQcX5z-pvPZmBJ-exN-wS6KZNoDMcPtYtM" />
<title><3E><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><>ѻ</title>
</head>
<body>
<div id="maintop">
<table align="left1" width="100%" border="0" cellpadding="0" cellspacing="0">
<tr><td valign="top">
<script type="text/javascript">
top.BX.defer(top.rsasec_form_bind)({'formid':'system_auth_form6zOUGO','key':{'M':'HazQxsgvQCIFPf30iHR40R22fp7P9YLPXFhQu6uus68RZxf2IpMo9v0KDpxkgg43WXaZaXrTRvjg1e2126IOo66vH5bphkMP/69MSPlEoaXYzWjTokd+Yzy30WR6HEOyB9tJwADGyjysqoE4+jUfHZQv2JMaVZS0U4SHWOUPwNU=','E':'AQAB','chunk':'128'},'rsa_rand':'5e1390ed8a8e19.17355178','params':['USER_PASSWORD']});
</script>
<div id="login-form-window">
<a href="" onclick="return CloseLoginForm()" style="float:right;"><3E><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD></a>
<form method="post" target="_top" action="/index.php?login=yes">
<input type="hidden" name="backurl" value="/index.php" />
<input type="hidden" name="AUTH_FORM" value="Y" />
<input type="hidden" name="TYPE" value="AUTH" />
<table width="95%">
<tr>
<td colspan="2">
<09><><EFBFBD><EFBFBD><EFBFBD>:<br />
<input type="text" name="USER_LOGIN" maxlength="50" value="

BIN
test/testdata/digest_test.data vendored Normal file
View File

Binary file not shown.

968
test/testdata/jd.com.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

BIN
test/testdata/mesa_logo.jpg vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 105 KiB