diff --git a/scanner/adapter_hs/adapter_hs.cpp b/scanner/adapter_hs/adapter_hs.cpp index c031bf0..2242efc 100644 --- a/scanner/adapter_hs/adapter_hs.cpp +++ b/scanner/adapter_hs/adapter_hs.cpp @@ -22,6 +22,8 @@ #include "maat_utils.h" #include "../bool_matcher/bool_matcher.h" +#define MAX_OFFSET_NUM 1024 + pid_t hs_gettid() { return syscall(SYS_gettid); @@ -50,8 +52,10 @@ struct adapter_hs_runtime { hs_database_t *literal_db; hs_database_t *regex_db; - hs_scratch_t **scratchs; - size_t scratch_size; + hs_scratch_t **literal_scratchs; + hs_scratch_t **regex_scratchs; + size_t literal_scratch_size; + size_t regex_scratch_size; struct bool_matcher *bm; }; @@ -63,21 +67,25 @@ struct adapter_hs { size_t n_patterns; struct adapter_hs_runtime *hs_rt; struct hs_tag *tag_map; + struct pattern_attribute *pat_attr_by_str; + struct pattern_attribute *pat_attr_by_id; struct log_handle *logger; }; +struct matched_offset { + unsigned long long start_offset; + unsigned long long end_offset; +}; + struct matched_pattern { unsigned long long pattern_id; - unsigned long matched_l_offset; - unsigned long matched_r_offset; + struct matched_offset *offsets; + size_t offset_cnt; + size_t offset_size; UT_hash_handle hh; }; struct matched_pattern_container { - UT_array *pat_ids; - unsigned long long pattern_id; - unsigned long long l_matched; - unsigned long long r_matched; struct matched_pattern *pat_hash; }; @@ -92,10 +100,11 @@ struct adapter_hs_stream { }; struct pattern_attribute { + unsigned long long bool_expr_id; unsigned long long pattern_id; enum hs_match_mode match_mode; - int l_offset; - int r_offset; + int start_offset; + int end_offset; }; struct hs_tag { @@ -108,40 +117,54 @@ struct hs_tag { UT_hash_handle hh; }; -static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread, - enum hs_pattern_type pattern_type, struct log_handle *logger) +int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratchs, size_t n_worker_thread, + struct log_handle *logger) { - hs_database_t *database = NULL; - hs_rt->scratchs = ALLOC(hs_scratch_t *, n_worker_thread); + size_t scratch_size = 0; - if (pattern_type == HS_PATTERN_TYPE_STR) { - database = hs_rt->literal_db; - } else { - database = hs_rt->regex_db; - } - - if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) { + if (hs_alloc_scratch(db, &scratchs[0]) != HS_SUCCESS) { log_error(logger, MODULE_ADAPTER_HS, - "[%s:%d] Unable to allocate scratch space. Exiting.", + "[%s:%d] Unable to allocate scratch space. Exiting.", __FUNCTION__, __LINE__); - hs_free_database(database); return -1; } for (size_t i = 1; i < n_worker_thread; i++) { - hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]); + hs_error_t err = hs_clone_scratch(scratchs[0], &scratchs[i]); if (err != HS_SUCCESS) { - log_error(logger, MODULE_ADAPTER_HS, - "[%s:%d] Unable to clone scratch prototype", __FUNCTION__, __LINE__); - hs_free_database(database); + log_error(logger, MODULE_ADAPTER_HS, + "[%s:%d] Unable to clone scratch", __FUNCTION__, __LINE__); return -1; } - err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size); + err = hs_scratch_size(scratchs[i], &scratch_size); if (err != HS_SUCCESS) { - log_error(logger, MODULE_ADAPTER_HS, + log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Unable to query scratch size", __FUNCTION__, __LINE__); - hs_free_database(database); + return -1; + } + } + + return 0; +} + +static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread, + enum hs_pattern_type pattern_type, struct log_handle *logger) +{ + int ret = 0; + + if (pattern_type == HS_PATTERN_TYPE_STR) { + hs_rt->literal_scratchs = ALLOC(hs_scratch_t *, n_worker_thread); + ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->literal_scratchs, n_worker_thread, logger); + if (ret < 0) { + FREE(hs_rt->literal_scratchs); + return -1; + } + } else { + hs_rt->regex_scratchs = ALLOC(hs_scratch_t *, n_worker_thread); + ret = _hs_alloc_scratch(hs_rt->regex_db, hs_rt->regex_scratchs, n_worker_thread, logger); + if (ret < 0) { + FREE(hs_rt->regex_scratchs); return -1; } } @@ -155,20 +178,20 @@ static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_work * @retval 0(success) -1(failed) */ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, - struct adpt_hs_compile_data *compile_data, - enum hs_pattern_type pattern_type, + struct adpt_hs_compile_data *literal_cd, + struct adpt_hs_compile_data *regex_cd, struct log_handle *logger) { hs_error_t err; hs_compile_error_t *compile_err = NULL; - if (NULL == hs_rt || NULL == compile_data) { + if (NULL == hs_rt || (NULL == literal_cd && NULL == regex_cd)) { return -1; } - if (pattern_type == HS_PATTERN_TYPE_STR) { - err = hs_compile_lit_multi((const char *const *)compile_data->patterns, compile_data->flags, - compile_data->ids, compile_data->pattern_lens, compile_data->n_patterns, + if (literal_cd != NULL) { + err = hs_compile_lit_multi((const char *const *)literal_cd->patterns, literal_cd->flags, + literal_cd->ids, literal_cd->pattern_lens, literal_cd->n_patterns, HS_MODE_STREAM, NULL, &hs_rt->literal_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { @@ -179,12 +202,12 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, hs_free_compile_error(compile_err); return -1; } - } else { - err = hs_compile_multi((const char *const *)compile_data->patterns, - compile_data->flags, compile_data->ids, - compile_data->n_patterns, - HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL, - &hs_rt->regex_db, &compile_err); + } + + if (regex_cd != NULL) { + err = hs_compile_multi((const char *const *)regex_cd->patterns, regex_cd->flags, regex_cd->ids, + regex_cd->n_patterns, HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL, + &hs_rt->regex_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s", @@ -203,20 +226,21 @@ struct adpt_hs_compile_data *adpt_hs_compile_data_new(size_t n_patterns) struct adpt_hs_compile_data *hs_cd = ALLOC(struct adpt_hs_compile_data, 1); hs_cd->patterns = ALLOC(char *, n_patterns); hs_cd->pattern_lens = ALLOC(size_t, n_patterns); + hs_cd->n_patterns = n_patterns; hs_cd->ids = ALLOC(unsigned int, n_patterns); hs_cd->flags = ALLOC(unsigned int, n_patterns); return hs_cd; } -void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patterns) +void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd) { if (NULL == hs_cd) { return; } if (hs_cd->patterns != NULL) { - for (size_t i = 0; i < n_patterns; i++) { + for (size_t i = 0; i < hs_cd->n_patterns; i++) { FREE(hs_cd->patterns[i]); } @@ -259,21 +283,92 @@ void hs_tag_free(struct hs_tag *tag) FREE(tag); } -struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type, - size_t n_worker_thread, +void populate_compile_data(struct adpt_hs_compile_data *compile_data, int index, int pattern_id, + char *pat, size_t pat_len, int case_sensitive) +{ + compile_data->ids[index] = pattern_id; + + /* set flags */ + compile_data->flags[index] |= HS_FLAG_SOM_LEFTMOST; + if (case_sensitive == HS_CASE_INSESITIVE) { + compile_data->flags[index] |= HS_FLAG_CASELESS; + } + + compile_data->pattern_lens[index] = pat_len; + compile_data->patterns[index] = ALLOC(char, pat_len + 1); + memcpy(compile_data->patterns[index], pat, pat_len); +} + +struct bool_expr *bool_exprs_new(struct hs_expr *exprs, size_t n_expr, struct hs_tag **tag_hash, + struct adpt_hs_compile_data *literal_cd, struct adpt_hs_compile_data *regex_cd, + size_t *n_pattern) +{ + uint32_t pattern_index = 0; + uint32_t literal_index = 0; + uint32_t regex_index = 0; + + struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr); + if (NULL == bool_exprs) { + return NULL; + } + + /* populate adpt_hs_compile_data and bool_expr */ + for (size_t i = 0; i < n_expr; i++) { + struct hs_tag *hs_tag = hs_tag_new(exprs[i].expr_id, exprs[i].n_patterns); + hs_tag->user_tag = exprs[i].user_tag; + + for (size_t j = 0; j < exprs[i].n_patterns; j++) { + hs_tag->pat_attr[j].pattern_id = pattern_index; + hs_tag->pat_attr[j].match_mode = exprs[i].patterns[j].match_mode; + if (exprs[i].patterns[j].match_mode == HS_MATCH_MODE_SUB) { + hs_tag->pat_attr[j].start_offset = exprs[i].patterns[j].start_offset; + hs_tag->pat_attr[j].end_offset = exprs[i].patterns[j].end_offset; + } + + /* literal pattern */ + if (exprs[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) { + populate_compile_data(literal_cd, literal_index, pattern_index, + exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len, + exprs[i].patterns[j].case_sensitive); + literal_index++; + } else { + /* regex pattern */ + populate_compile_data(regex_cd, regex_index, pattern_index, + exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len, + exprs[i].patterns[j].case_sensitive); + regex_index++; + } + + bool_exprs[i].items[j].item_id = pattern_index++; + bool_exprs[i].items[j].not_flag = 0; + // printf("item_id:%llu, pat:%s pat_len:%zu\n", + // bool_exprs[i].items[j].item_id, exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len); + } + //printf("expr_id:%lld item_num:%zu\n", exprs[i].expr_id, exprs[i].n_patterns); + bool_exprs[i].expr_id = exprs[i].expr_id; + bool_exprs[i].item_num = exprs[i].n_patterns; + bool_exprs[i].user_tag = hs_tag; + HASH_ADD_KEYPTR(hh, *tag_hash, hs_tag->key, hs_tag->key_len, hs_tag); + } + + *n_pattern = pattern_index; + + return bool_exprs; +} + +struct adapter_hs *adapter_hs_initialize(size_t n_worker_thread, struct hs_expr *exprs, size_t n_expr, struct log_handle *logger) { - if ((pattern_type != HS_PATTERN_TYPE_STR && pattern_type != HS_PATTERN_TYPE_REG) || - 0 == n_worker_thread || NULL == exprs || 0 == n_expr) { + if (0 == n_worker_thread || NULL == exprs || 0 == n_expr) { log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] input parameters illegal!", __FUNCTION__, __LINE__); return NULL; } /* get the sum of pattern */ - size_t pattern_num = 0; - + size_t literal_pattern_num = 0; + size_t regex_pattern_num = 0; for (size_t i = 0; i < n_expr; i++) { if (exprs[i].n_patterns > MAX_EXPR_PATTERN_NUM) { log_error(logger, MODULE_ADAPTER_HS, @@ -283,83 +378,53 @@ struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type, } for (size_t j = 0; j < exprs[i].n_patterns; j++) { + /* pat_len should not 0 */ if (0 == exprs[i].patterns[j].pat_len) { - log_error(logger, MODULE_ADAPTER_HS, + log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] expr pattern length should not 0", __FUNCTION__, __LINE__); return NULL; } - pattern_num++; + if (exprs[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) { + literal_pattern_num++; + } else { + regex_pattern_num++; + } } } - if (0 == pattern_num) { - log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] expr array has no valid pattern", + if (0 == literal_pattern_num && 0 == regex_pattern_num) { + log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] exprs has no valid pattern", __FUNCTION__, __LINE__); return NULL; } - struct adpt_hs_compile_data *compile_data = NULL; - compile_data = adpt_hs_compile_data_new(pattern_num); + struct adpt_hs_compile_data *literal_cd = NULL; + struct adpt_hs_compile_data *regex_cd = NULL; + if (literal_pattern_num > 0) { + literal_cd = adpt_hs_compile_data_new(literal_pattern_num); + } + + if (regex_pattern_num > 0) { + regex_cd = adpt_hs_compile_data_new(regex_pattern_num); + } - uint32_t pattern_index = 0; + size_t pattern_cnt = 0; struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1); hs_instance->tag_map = NULL; hs_instance->logger = logger; - - struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr); - /* populate adpt_hs_compile_data and bool_expr */ - for (size_t i = 0; i < n_expr; i++) { - struct hs_tag *hs_tag = hs_tag_new(exprs[i].expr_id, exprs[i].n_patterns); - hs_tag->user_tag = exprs[i].user_tag; - - for (size_t j = 0; j < exprs[i].n_patterns; j++) { - size_t pat_len = 0; - - hs_tag->pat_attr[j].pattern_id = pattern_index; - hs_tag->pat_attr[j].match_mode = exprs[i].patterns[j].match_mode; - if (exprs[i].patterns[j].match_mode == HS_MATCH_MODE_SUB) { - hs_tag->pat_attr[j].l_offset = exprs[i].patterns[j].l_offset; - hs_tag->pat_attr[j].r_offset = exprs[i].patterns[j].r_offset; - } - - compile_data->ids[pattern_index] = pattern_index; - if (pattern_type == HS_PATTERN_TYPE_STR) { - compile_data->flags[pattern_index] |= HS_FLAG_SOM_LEFTMOST; - } - - if (exprs[i].patterns[j].case_sensitive == HS_CASE_INSESITIVE) { - compile_data->flags[pattern_index] |= HS_FLAG_CASELESS; - } - - pat_len = exprs[i].patterns[j].pat_len; - compile_data->pattern_lens[pattern_index] = pat_len; - compile_data->patterns[pattern_index] = ALLOC(char, pat_len + 1); - memcpy(compile_data->patterns[pattern_index], exprs[i].patterns[j].pat, - exprs[i].patterns[j].pat_len); - - bool_exprs[i].items[j].item_id = pattern_index; - pattern_index++; - } - - bool_exprs[i].expr_id = exprs[i].expr_id; - bool_exprs[i].item_num = exprs[i].n_patterns; - bool_exprs[i].user_tag = hs_tag; - HASH_ADD_KEYPTR(hh, hs_instance->tag_map, hs_tag->key, hs_tag->key_len, hs_tag); - } - compile_data->n_patterns = pattern_index; - - int ret = -1; - size_t mem_size = 0; - hs_instance->n_worker_thread = n_worker_thread; - hs_instance->n_patterns = pattern_index; hs_instance->n_expr = n_expr; - hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1); + struct bool_expr *bool_exprs = bool_exprs_new(exprs, n_expr, &hs_instance->tag_map, + literal_cd, regex_cd, &pattern_cnt); + if (NULL == bool_exprs) { + return NULL; + } + hs_instance->n_patterns = pattern_cnt; + //mytest // for (size_t i = 0; i < n_expr; i++) { - // if (bool_exprs[i].expr_id == 37) // { // printf(" exprs[%zu] expr_id:%llu, item_num:%zu\n", // i, bool_exprs[i].expr_id, bool_exprs[i].item_num); @@ -369,10 +434,14 @@ struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type, // printf("%llu ", bool_exprs[i].items[j].item_id); // } // } + // printf("\n"); // } - // printf("\n"); + /* create bool matcher */ + size_t mem_size = 0; + int hs_ret = 0; + hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1); hs_instance->hs_rt->bm = bool_matcher_new(bool_exprs, n_expr, &mem_size); if (hs_instance->hs_rt->bm != NULL) { log_info(logger, MODULE_ADAPTER_HS, @@ -381,30 +450,47 @@ struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type, } else { log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Adapter_hs module: build bool matcher failed", __FUNCTION__, __LINE__); - adpt_hs_compile_data_free(compile_data, pattern_index); - FREE(bool_exprs); - adapter_hs_destroy(hs_instance); - return NULL; + + hs_ret = -1; } FREE(bool_exprs); /* build hs database */ - ret = adpt_hs_build_database(hs_instance->hs_rt, compile_data, pattern_type, logger); + int ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, logger); if (ret < 0) { + hs_ret = -1; + } + + if (literal_cd != NULL) { + adpt_hs_compile_data_free(literal_cd); + } + + if (regex_cd != NULL) { + adpt_hs_compile_data_free(regex_cd); + } + + if (hs_ret < 0) { goto error; } - ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, pattern_type, logger); - if (ret < 0) { - goto error; + /* literal and regex scratch can't reuse */ + if (literal_pattern_num > 0) { + ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_STR, logger); + if (ret < 0) { + goto error; + } + } + + if (regex_pattern_num > 0) { + ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_REG, logger); + if (ret < 0) { + goto error; + } } - adpt_hs_compile_data_free(compile_data, pattern_index); return hs_instance; error: - adpt_hs_compile_data_free(compile_data, pattern_index); adapter_hs_destroy(hs_instance); - return NULL; } @@ -423,14 +509,23 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance) hs_free_database(hs_instance->hs_rt->regex_db); } - if (hs_instance->hs_rt->scratchs != NULL) { + if (hs_instance->hs_rt->literal_scratchs != NULL) { for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { - if (hs_instance->hs_rt->scratchs[i] != NULL) { - hs_free_scratch(hs_instance->hs_rt->scratchs[i]); + if (hs_instance->hs_rt->literal_scratchs[i] != NULL) { + hs_free_scratch(hs_instance->hs_rt->literal_scratchs[i]); } } } - FREE(hs_instance->hs_rt->scratchs); + FREE(hs_instance->hs_rt->literal_scratchs); + + if (hs_instance->hs_rt->regex_scratchs != NULL) { + for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { + if (hs_instance->hs_rt->regex_scratchs[i] != NULL) { + hs_free_scratch(hs_instance->hs_rt->regex_scratchs[i]); + } + } + } + FREE(hs_instance->hs_rt->regex_scratchs); if (hs_instance->hs_rt->bm != NULL) { bool_matcher_free(hs_instance->hs_rt->bm); @@ -450,20 +545,19 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance) FREE(hs_instance); } -static inline int compare_pattern_id(const void *a, const void *b) +int find_same_pattern_offset(struct matched_pattern *matched_pat, unsigned long long from, + unsigned long long to) { - long long ret = *(unsigned long long *)a - *(unsigned long long *)b; + for (size_t i = 0; i < matched_pat->offset_cnt; i++) { + if (matched_pat->offsets[i].start_offset == from && + matched_pat->offsets[i].end_offset == to - 1) { + return 0; + } + } - if (0 == ret) { - return 0; - } else if (ret < 0) { - return -1; - } else { - return 1; - } + return -1; } -UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; /** * @param id: pattern id */ @@ -474,50 +568,75 @@ int matched_event_cb(unsigned int id, unsigned long long from, struct matched_pattern_container *matched_pat_container = (struct matched_pattern_container *)ctx; unsigned long long pattern_id = id; - if (utarray_find(matched_pat_container->pat_ids, &pattern_id, compare_pattern_id)) { + struct matched_pattern *matched_pat = NULL; + HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), matched_pat); + if (matched_pat != NULL) { + // same pattern_id, offset maybe different + int ret = find_same_pattern_offset(matched_pat, from, to); + if (ret < 0) { /* different offset */ + // TODO: use realloc + if (matched_pat->offset_cnt >= matched_pat->offset_size) { + matched_pat->offset_size *= 2; + matched_pat->offsets = (struct matched_offset *)realloc(matched_pat->offsets, + matched_pat->offset_size*sizeof(struct matched_offset)); + } + matched_pat->offsets[matched_pat->offset_cnt].start_offset = from; + matched_pat->offsets[matched_pat->offset_cnt].end_offset = to - 1; + matched_pat->offset_cnt++; + } return 0; + } else { + // different pattern_id + struct matched_pattern *matched_pat = ALLOC(struct matched_pattern, 1); + matched_pat->pattern_id = pattern_id; + matched_pat->offsets = ALLOC(struct matched_offset, MAX_OFFSET_NUM); + matched_pat->offset_size = MAX_OFFSET_NUM; + matched_pat->offsets[matched_pat->offset_cnt].start_offset = from; + matched_pat->offsets[matched_pat->offset_cnt].end_offset = to - 1; + matched_pat->offset_cnt++; + + HASH_ADD(hh, matched_pat_container->pat_hash, pattern_id, sizeof(unsigned long long), matched_pat); } - utarray_push_back(matched_pat_container->pat_ids, &pattern_id); - utarray_sort(matched_pat_container->pat_ids, compare_pattern_id); - - struct matched_pattern *matched_pat = ALLOC(struct matched_pattern, 1); - matched_pat->pattern_id = pattern_id; - matched_pat->matched_l_offset = from; - matched_pat->matched_r_offset = to; - - HASH_ADD(hh, matched_pat_container->pat_hash, pattern_id, sizeof(unsigned long long), matched_pat); return 0; } int is_real_matched_pattern(struct matched_pattern *matched_pat, enum hs_match_mode match_mode, - size_t data_len, int attr_l_offset, int attr_r_offset) + size_t data_len, int attr_start_offset, int attr_end_offset) { if (match_mode == HS_MATCH_MODE_EXACTLY) { - if (matched_pat->matched_l_offset == 0 && - matched_pat->matched_r_offset == data_len) { - return 0; + for (size_t i = 0; i < matched_pat->offset_cnt; i++) { + if (matched_pat->offsets[i].start_offset == 0 && + matched_pat->offsets[i].end_offset == data_len - 1) { + return 0; + } } } else if (match_mode == HS_MATCH_MODE_PREFIX) { - if (matched_pat->matched_l_offset == 0) { - return 0; + for (size_t i = 0; i < matched_pat->offset_cnt; i++) { + if (matched_pat->offsets[i].start_offset == 0) { + return 0; + } } } else if (match_mode == HS_MATCH_MODE_SUFFIX) { - if (matched_pat->matched_r_offset == data_len) { - return 0; + for (size_t i = 0; i < matched_pat->offset_cnt; i++) { + if (matched_pat->offsets[i].end_offset == data_len - 1) { + return 0; + } } } else if (match_mode == HS_MATCH_MODE_SUB) { - if (attr_l_offset == -1) { - attr_l_offset = 0; + if (attr_start_offset == -1) { + attr_start_offset = 0; } - if (attr_r_offset == -1) { - attr_r_offset = (int)data_len; + if (attr_end_offset == -1) { + attr_end_offset = (int)data_len - 1; } - if (matched_pat->matched_l_offset >= (unsigned long)attr_l_offset && - matched_pat->matched_r_offset <= (unsigned long)attr_r_offset) { - return 0; + for (size_t i = 0; i < matched_pat->offset_cnt; i++) { + if (matched_pat->offsets[i].start_offset >= (unsigned long long)attr_start_offset && + matched_pat->offsets[i].end_offset <= (unsigned long long)attr_end_offset) { + return 0; + } } } else { assert(0); @@ -531,15 +650,13 @@ int hs_tag_validate(struct hs_tag *hs_tag, struct matched_pattern_container *mat { /* check if real matched pattern, because pattern match_mode is different */ for (size_t i = 0; i < hs_tag->n_pat_attr; i++) { - struct matched_pattern *tmp_matched_pat = NULL; + struct matched_pattern *matched_pat = NULL; unsigned long long pattern_id = hs_tag->pat_attr[i].pattern_id; - HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), tmp_matched_pat); - if (tmp_matched_pat) { - int matched_ret = is_real_matched_pattern(tmp_matched_pat, - hs_tag->pat_attr[i].match_mode, - data_len, - hs_tag->pat_attr[i].l_offset, - hs_tag->pat_attr[i].r_offset); + HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), matched_pat); + if (matched_pat) { + int matched_ret = is_real_matched_pattern(matched_pat, hs_tag->pat_attr[i].match_mode, + data_len, hs_tag->pat_attr[i].start_offset, + hs_tag->pat_attr[i].end_offset); if (matched_ret < 0) { return -1; } @@ -562,14 +679,13 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, hs_stream->n_expr = hs_instance->n_expr; hs_stream->n_patterns = hs_instance->n_patterns; hs_stream->hs_rt = hs_instance->hs_rt; - utarray_new(hs_stream->matched_pat_container.pat_ids, &ut_pattern_id_icd); - utarray_reserve(hs_stream->matched_pat_container.pat_ids, hs_stream->n_patterns); + int err_count = 0; if (hs_instance->hs_rt->literal_db != NULL) { err = hs_open_stream(hs_instance->hs_rt->literal_db, 0, &hs_stream->literal_stream); if (err != HS_SUCCESS) { log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err); - return NULL; + err_count++; } } @@ -577,11 +693,29 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, err = hs_open_stream(hs_instance->hs_rt->regex_db, 0, &hs_stream->regex_stream); if (err != HS_SUCCESS) { log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err); - return NULL; + err_count++; } } + if (err_count > 0) { + goto error; + } + return hs_stream; +error: + //TODO: hs_stream->hs_rt->scratchs[thread_id] may be free twice + if (hs_stream->literal_stream != NULL) { + hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL); + hs_stream->literal_stream = NULL; + } + + if (hs_stream->regex_stream != NULL) { + hs_close_stream(hs_stream->regex_stream, NULL, NULL, NULL); + hs_stream->regex_stream = NULL; + } + + FREE(hs_stream); + return NULL; } void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream) @@ -590,20 +724,14 @@ void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream) return; } - int thread_id = hs_stream->thread_id; - if (hs_stream->hs_rt != NULL) { if (hs_stream->literal_stream != NULL) { - hs_close_stream(hs_stream->literal_stream, - hs_stream->hs_rt->scratchs[thread_id], - NULL, NULL); + hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL); hs_stream->literal_stream = NULL; } if (hs_stream->regex_stream != NULL) { - hs_close_stream(hs_stream->regex_stream, - hs_stream->hs_rt->scratchs[thread_id], - NULL, NULL); + hs_close_stream(hs_stream->regex_stream, NULL, NULL, NULL); hs_stream->regex_stream = NULL; } } @@ -615,13 +743,23 @@ void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream) FREE(pattern); } } - utarray_free(hs_stream->matched_pat_container.pat_ids); /* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */ hs_stream->hs_rt = NULL; FREE(hs_stream); } +static int cmp_ull_p(const void *p1, const void *p2) +{ + if(* (unsigned long long*) p1 > * (unsigned long long*) p2) { + return 1; + } else if(* (unsigned long long*) p1 < * (unsigned long long*) p2) { + return -1; + } else { + return 0; + } +} + int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len, struct hs_scan_result *results, size_t n_result, size_t *n_hit_result) { @@ -646,7 +784,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data int thread_id = hs_stream->thread_id; if (hs_stream->literal_stream != NULL) { err = hs_scan_stream(hs_stream->literal_stream, data, data_len, - 0, hs_stream->hs_rt->scratchs[thread_id], + 0, hs_stream->hs_rt->literal_scratchs[thread_id], matched_event_cb, &hs_stream->matched_pat_container); if (err != HS_SUCCESS) { err_count++; @@ -655,34 +793,42 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data if (hs_stream->regex_stream != NULL) { err = hs_scan_stream(hs_stream->regex_stream, data, data_len, - 0, hs_stream->hs_rt->scratchs[thread_id], + 0, hs_stream->hs_rt->regex_scratchs[thread_id], matched_event_cb, &hs_stream->matched_pat_container); if (err != HS_SUCCESS) { err_count++; } } - if (err_count > 0) { + if (err_count == 2) { return -1; } - size_t matched_pattern_ids_cnt = utarray_len(hs_stream->matched_pat_container.pat_ids); - if (0 == matched_pattern_ids_cnt) { + size_t n_item = HASH_COUNT(hs_stream->matched_pat_container.pat_hash); + if (0 == n_item) { *n_hit_result = 0; return 0; } - unsigned long long items[matched_pattern_ids_cnt]; - memset(items, 0, sizeof(unsigned long long) * matched_pattern_ids_cnt); - for (size_t i = 0; i < matched_pattern_ids_cnt; i++) { - items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->matched_pat_container.pat_ids, i); + if (n_item > MAX_SCANNER_HIT_ITEM_NUM) { + n_item = MAX_SCANNER_HIT_ITEM_NUM; } + unsigned long long item_ids[MAX_SCANNER_HIT_ITEM_NUM]; + memset(item_ids, 0, sizeof(unsigned long long) * MAX_SCANNER_HIT_ITEM_NUM); + + int i = 0; + struct matched_pattern *pat = NULL, *tmp_pat = NULL; + HASH_ITER(hh, hs_stream->matched_pat_container.pat_hash, pat, tmp_pat) { + item_ids[i++] = pat->pattern_id; + } + qsort(item_ids, n_item, sizeof(unsigned long long), cmp_ull_p); + int ret = 0; int real_matched_index = 0; struct hs_tag *hs_tag = NULL; struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); - int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, matched_pattern_ids_cnt, + int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, item_ids, n_item, bool_matcher_results, hs_stream->n_expr); if (bool_matcher_ret < 0) { ret = -1; @@ -715,7 +861,6 @@ next: HASH_DELETE(hh, hs_stream->matched_pat_container.pat_hash, pattern); FREE(pattern); } - utarray_clear(hs_stream->matched_pat_container.pat_ids); return ret; } diff --git a/scanner/adapter_hs/adapter_hs.h b/scanner/adapter_hs/adapter_hs.h index f0d293e..9ac51c6 100644 --- a/scanner/adapter_hs/adapter_hs.h +++ b/scanner/adapter_hs/adapter_hs.h @@ -54,16 +54,18 @@ struct hs_scan_result { struct hs_pattern { enum hs_case_sensitive case_sensitive; enum hs_match_mode match_mode; + enum hs_pattern_type pattern_type; + int is_hexbin; /* 1(yes) 0(no) */ /* - * just match in scan_data's range of [l_offset, r_offset], -1 means no limits + * just match in scan_data's range of [start_offset, end_offset], -1 means no limits * for example: - * [-1, r_offset] means the pattern must in scan_data's [0 ~ r_offset] - * [l_offset, -1] means the pattern must in scan_data's [l_offset ~ end] + * [-1, end_offset] means the pattern must in scan_data's [0 ~ start_offset] + * [start_offset, -1] means the pattern must in scan_data's [start_offset ~ data_end] */ - int l_offset; - int r_offset; + int start_offset; + int end_offset; /* start pointer of pattern */ char *pat; @@ -82,15 +84,13 @@ struct hs_expr { /** * @brief initialize adapter_hs instance * - * @param pattern_type: pure literal or regex expression * @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan() * @param expr_array: logic AND expression's array * @param n_expr_arrays: the number of logic AND expression's array * * @retval the pointer to adapter_hs instance */ -struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type, - size_t n_worker_thread, +struct adapter_hs *adapter_hs_initialize(size_t n_worker_thread, struct hs_expr *exprs, size_t n_expr, struct log_handle *logger); diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index 56342ee..26148fb 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -45,10 +45,11 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name); int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len, int vtable_id, struct maat_state *state); -int expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); -int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, +struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); +int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle, + const char *data, size_t data_len, int vtable_id, struct maat_state *state); -void expr_runtime_stream_close(struct expr_runtime *expr_rt); +void expr_runtime_stream_close(struct adapter_hs_stream *s_handle); void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id); long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread); diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 12f0614..3d56159 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -35,7 +35,6 @@ extern "C" #define MAX_TABLE_NUM 1024 #define MAX_COMPILE_TABLE_NUM 16 -#define MAX_PHYSICAL_TABLE_NUM 16 #define DISTRICT_ANY -1 #define DISTRICT_UNKNOWN -2 diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h index 3120652..c69d062 100644 --- a/src/inc_internal/maat_virtual.h +++ b/src/inc_internal/maat_virtual.h @@ -39,7 +39,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, void virtual_schema_free(void *virtual_schema); int virtual_table_get_id(void *virtual_schema); -size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]); +int virtual_table_get_physical_table_id(void *virtual_schema); #ifdef __cplusplus } diff --git a/src/maat_api.c b/src/maat_api.c index e152ad0..5464ece 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -48,11 +48,10 @@ enum district_set_flag { struct maat_stream { struct maat *ref_maat_instance; + struct adapter_hs_stream *s_handle; //each physical table open one stream int thread_id; - int table_id; int vtable_id; - int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; - size_t n_physical_table; + int physical_table_id; struct log_handle *logger; }; @@ -987,243 +986,205 @@ size_t hit_group_to_compile(void *compile_runtime, long long *compile_ids, int i return n_hit_compile; } -static int vtable_get_physical_table_ids(struct table_manager *tbl_mgr, int table_id, - int *physical_table_ids, size_t n_table_id_array, - int *vtable_id) +static int vtable_get_physical_table_id(struct table_manager *tbl_mgr, int table_id) { - size_t physical_table_cnt = 0; enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); - if (table_type == TABLE_TYPE_VIRTUAL) { - //find physical table id - *vtable_id = table_id; - void *virtual_schema = table_manager_get_schema(tbl_mgr, table_id); - assert(virtual_schema != NULL); - physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); - return physical_table_cnt; - } else { - *vtable_id = 0; - physical_table_ids[0] = table_id; - physical_table_cnt = 1; + if (table_type != TABLE_TYPE_VIRTUAL) { + return -1; } - return physical_table_cnt; + // find physical table id + void *virtual_schema = table_manager_get_schema(tbl_mgr, table_id); + assert(virtual_schema != NULL); + return virtual_table_get_physical_table_id(virtual_schema); } -int flag_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long flag, - int physical_table_ids[], int physical_table_cnt, int vtable_id, - struct maat_state *mid) +int flag_scan(struct table_manager *tbl_mgr, int thread_id, long long flag, + int physical_table_id, int vtable_id, struct maat_state *mid) { - int sum_hit_group_cnt = 0; - - for (int i = 0; i < physical_table_cnt; i++) { - enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); - if ((table_type == TABLE_TYPE_FLAG_PLUS) && - (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { - return -1; - } - - if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) { - continue; - } - - void *flag_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); - if (NULL == flag_rt) { - return -1; - } - - int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id, - flag, vtable_id, mid); - if (group_hit_cnt < 0) { - return -1; - } - - if (group_hit_cnt > 0) { - flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id); - } - sum_hit_group_cnt += group_hit_cnt; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); + if ((table_type == TABLE_TYPE_FLAG_PLUS) && + (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { + return -1; } - return sum_hit_group_cnt; + if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) { + return -1; + } + + void *flag_rt = table_manager_get_runtime(tbl_mgr, physical_table_id); + if (NULL == flag_rt) { + return -1; + } + + int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id, + flag, vtable_id, mid); + if (group_hit_cnt < 0) { + return -1; + } + + if (group_hit_cnt > 0) { + flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id); + } + + return group_hit_cnt; } -int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long integer, - int physical_table_ids[], int physical_table_cnt, int vtable_id, - struct maat_state *mid) +int interval_scan(struct table_manager *tbl_mgr, int thread_id, long long integer, + int physical_table_id, int vtable_id, struct maat_state *mid) { - int sum_hit_group_cnt = 0; - for (size_t i = 0; i < physical_table_cnt; i++) { - enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); - if ((table_type == TABLE_TYPE_INTERVAL_PLUS) && - (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { - //maat_instance->scan_err_cnt++; - return -1; - } - - if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { - continue; - } - - void *interval_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); - if (NULL == interval_rt) { - return -1; - } - - int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt, - thread_id, integer, vtable_id, mid); - if (group_hit_cnt < 0) { - return -1; - } - - if (group_hit_cnt > 0) { - interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id); - } - sum_hit_group_cnt += group_hit_cnt; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); + if ((table_type == TABLE_TYPE_INTERVAL_PLUS) && + (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { + // maat_instance->scan_err_cnt++; + return -1; } - return sum_hit_group_cnt; + if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { + return -1; + } + + void *interval_rt = table_manager_get_runtime(tbl_mgr, physical_table_id); + if (NULL == interval_rt) { + return -1; + } + + int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt, + thread_id, integer, vtable_id, mid); + if (group_hit_cnt < 0) { + return -1; + } + + if (group_hit_cnt > 0) { + interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id); + } + + return group_hit_cnt; } -int ipv4_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_addr, - int physical_table_ids[], int physical_table_cnt, int vtable_id, - struct maat_state *mid) +int ipv4_scan(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_addr, + int physical_table_id, int vtable_id, struct maat_state *mid) { - int sum_hit_group_cnt = 0; - for (size_t i = 0; i < physical_table_cnt; i++) { - enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); - if (table_type != TABLE_TYPE_IP_PLUS) { - continue; - } - - void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); - if (NULL == ip_rt) { - return -1; - } - - int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4, - (uint8_t *)&ip_addr, vtable_id, mid); - if (group_hit_cnt < 0) { - return -1; - } - - if (group_hit_cnt > 0) { - ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); - } - sum_hit_group_cnt += group_hit_cnt; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); + if (table_type != TABLE_TYPE_IP_PLUS) { + return -1; } - return sum_hit_group_cnt; + void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_id); + if (NULL == ip_rt) { + return -1; + } + + int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4, + (uint8_t *)&ip_addr, vtable_id, mid); + if (group_hit_cnt < 0) { + return -1; + } + + if (group_hit_cnt > 0) { + ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); + } + + return group_hit_cnt; } -int ipv6_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint8_t *ip_addr, - int physical_table_ids[], int physical_table_cnt, int vtable_id, - struct maat_state *mid) +int ipv6_scan(struct table_manager *tbl_mgr, int thread_id, uint8_t *ip_addr, + int physical_table_id, int vtable_id, struct maat_state *mid) { - int sum_hit_group_cnt = 0; - for (size_t i = 0; i < physical_table_cnt; i++) { - enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); - if (table_type != TABLE_TYPE_IP_PLUS) { - continue; - } - - void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); - if (NULL == ip_rt) { - return -1; - } - - int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, - ip_addr, vtable_id, mid); - if (group_hit_cnt < 0) { - return -1; - } - - if (group_hit_cnt > 0) { - ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); - } - sum_hit_group_cnt += group_hit_cnt; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); + if (table_type != TABLE_TYPE_IP_PLUS) { + return -1; } - return sum_hit_group_cnt; + void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_id); + if (NULL == ip_rt) { + return -1; + } + + int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, + ip_addr, vtable_id, mid); + if (group_hit_cnt < 0) { + return -1; + } + + if (group_hit_cnt > 0) { + ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); + } + + return group_hit_cnt; } -int string_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, const char *data, - size_t data_len, int physical_table_ids[], int physical_table_cnt, - int vtable_id, struct maat_state *mid) +int string_scan(struct table_manager *tbl_mgr, int thread_id, const char *data, size_t data_len, + int physical_table_id, int vtable_id, struct maat_state *mid) { - int sum_hit_group_cnt = 0; - - for (size_t i = 0; i < physical_table_cnt; i++) { - enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); - if ((table_type == TABLE_TYPE_EXPR_PLUS) && - (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { - //maat_instance->scan_err_cnt++; - return -1; - } - - if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { - continue; - } - - void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); - if (NULL == expr_rt) { - return -1; - } - - int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt, - thread_id, data, data_len, - vtable_id, mid); - if (group_hit_cnt < 0) { - return -1; - } - - if (group_hit_cnt > 0) { - expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); - } - sum_hit_group_cnt += group_hit_cnt; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); + if ((table_type == TABLE_TYPE_EXPR_PLUS) && + (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { + // maat_instance->scan_err_cnt++; + return -1; } - return sum_hit_group_cnt; + if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { + return -1; + } + + void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_id); + if (NULL == expr_rt) { + return -1; + } + + int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt, + thread_id, data, data_len, + vtable_id, mid); + if (group_hit_cnt < 0) { + return -1; + } + + if (group_hit_cnt > 0) { + expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); + } + + return group_hit_cnt; } -int stream_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, const char *data, - size_t data_len, int physical_table_ids[], int physical_table_cnt, - int vtable_id, struct maat_state *mid) +int expr_stream_scan(struct maat_stream *stream, const char *data, size_t data_len, struct maat_state *state) { - int sum_hit_group_cnt = 0; - - for (size_t i = 0; i < physical_table_cnt; i++) { - enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); - if ((table_type == TABLE_TYPE_EXPR_PLUS) && - (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { - //maat_instance->scan_err_cnt++; - return -1; - } - - if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { - continue; - } - - void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); - if (NULL == expr_rt) { - return -1; - } - - int group_hit_cnt = expr_runtime_stream_scan((struct expr_runtime *)expr_rt, - data, data_len, vtable_id, mid); - if (group_hit_cnt < 0) { - return -1; - } - - if (group_hit_cnt > 0) { - expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); - } - sum_hit_group_cnt += group_hit_cnt; + if (NULL == stream || NULL == data) { + return 0; } - return sum_hit_group_cnt; + struct table_manager *tbl_mgr = stream->ref_maat_instance->tbl_mgr; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, stream->physical_table_id); + if ((table_type == TABLE_TYPE_EXPR_PLUS) && + (NULL == state || DISTRICT_FLAG_UNSET == state->is_set_district)) { + // maat_instance->scan_err_cnt++; + return -1; + } + + if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { + return -1; + } + + void *expr_rt = table_manager_get_runtime(tbl_mgr, stream->physical_table_id); + if (NULL == expr_rt) { + return -1; + } + + int group_hit_cnt = expr_runtime_stream_scan((struct expr_runtime *)expr_rt, stream->s_handle, + data, data_len, stream->vtable_id, state); + if (group_hit_cnt < 0) { + return -1; + } + + if (group_hit_cnt > 0) { + expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, stream->thread_id); + } + + return group_hit_cnt; } size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n_result, @@ -1279,31 +1240,29 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, } int vtable_id = 0; - int physical_table_cnt = 0; - int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; - memset(physical_table_ids, -1, sizeof(physical_table_ids)); + int physical_table_id = -1; + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id); + vtable_id = table_id; + } else { + physical_table_id = table_id; + } - physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, - physical_table_ids, - MAX_PHYSICAL_TABLE_NUM, &vtable_id); - if (physical_table_cnt <= 0) { + if (physical_table_id < 0) { return MAAT_SCAN_ERR; } - enum table_type table_type = TABLE_TYPE_INVALID; - if (0 == vtable_id) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); - if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) { - return MAAT_SCAN_ERR; - } + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); + if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) { + return MAAT_SCAN_ERR; } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int hit_group_cnt = flag_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, flag, - physical_table_ids, - physical_table_cnt, vtable_id, mid); + int hit_group_cnt = flag_scan(maat_instance->tbl_mgr, thread_id, flag, + physical_table_id, vtable_id, mid); if (hit_group_cnt < 0) { return MAAT_SCAN_ERR; } @@ -1356,31 +1315,29 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, } int vtable_id = 0; - int physical_table_cnt = 0; - int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; - memset(physical_table_ids, -1, sizeof(physical_table_ids)); + int physical_table_id = -1; + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id); + vtable_id = table_id; + } else { + physical_table_id = table_id; + } - physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, - physical_table_ids, - MAX_PHYSICAL_TABLE_NUM, &vtable_id); - if (physical_table_cnt <= 0) { + if (physical_table_id < 0) { return MAAT_SCAN_ERR; } - enum table_type table_type = TABLE_TYPE_INVALID; - if (0 == vtable_id) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); - if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { - return MAAT_SCAN_ERR; - } + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); + if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { + return MAAT_SCAN_ERR; } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int hit_group_cnt = interval_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, integer, - physical_table_ids, - physical_table_cnt, vtable_id, mid); + int hit_group_cnt = interval_scan(maat_instance->tbl_mgr, thread_id, integer, + physical_table_id, vtable_id, mid); if (hit_group_cnt < 0) { return MAAT_SCAN_ERR; } @@ -1433,31 +1390,29 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, } int vtable_id = 0; - int physical_table_cnt = 0; - int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; - memset(physical_table_ids, -1, sizeof(physical_table_ids)); + int physical_table_id = -1; + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id); + vtable_id = table_id; + } else { + physical_table_id = table_id; + } - physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, - physical_table_ids, - MAX_PHYSICAL_TABLE_NUM, &vtable_id); - if (physical_table_cnt <= 0) { + if (physical_table_id < 0) { return MAAT_SCAN_ERR; } - enum table_type table_type = TABLE_TYPE_INVALID; - if (0 == vtable_id) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); - if (table_type != TABLE_TYPE_IP_PLUS) { - return MAAT_SCAN_ERR; - } + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); + if (table_type != TABLE_TYPE_IP_PLUS) { + return MAAT_SCAN_ERR; } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int hit_group_cnt = ipv4_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, ip_addr, - physical_table_ids, - physical_table_cnt, vtable_id, mid); + int hit_group_cnt = ipv4_scan(maat_instance->tbl_mgr, thread_id, ip_addr, + physical_table_id, vtable_id, mid); if (hit_group_cnt < 0) { return MAAT_SCAN_ERR; } @@ -1518,31 +1473,29 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, } int vtable_id = 0; - int physical_table_cnt = 0; - int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; - memset(physical_table_ids, -1, sizeof(physical_table_ids)); + int physical_table_id = -1; + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id); + vtable_id = table_id; + } else { + physical_table_id = table_id; + } - physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, - physical_table_ids, - MAX_PHYSICAL_TABLE_NUM, &vtable_id); - if (physical_table_cnt <= 0) { + if (physical_table_id < 0) { return MAAT_SCAN_ERR; } - enum table_type table_type = TABLE_TYPE_INVALID; - if (0 == vtable_id) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); - if (table_type != TABLE_TYPE_IP_PLUS) { - return MAAT_SCAN_ERR; - } + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); + if (table_type != TABLE_TYPE_IP_PLUS) { + return MAAT_SCAN_ERR; } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int hit_group_cnt = ipv6_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, ip_addr, - physical_table_ids, - physical_table_cnt, vtable_id, mid); + int hit_group_cnt = ipv6_scan(maat_instance->tbl_mgr, thread_id, ip_addr, + physical_table_id, vtable_id, mid); if (hit_group_cnt < 0) { return MAAT_SCAN_ERR; } @@ -1603,31 +1556,29 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, } int vtable_id = 0; - int physical_table_cnt = 0; - int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; - memset(physical_table_ids, -1, sizeof(physical_table_ids)); + int physical_table_id = -1; + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id); + vtable_id = table_id; + } else { + physical_table_id = table_id; + } - physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, - physical_table_ids, - MAX_PHYSICAL_TABLE_NUM, &vtable_id); - if (physical_table_cnt <= 0) { + if (physical_table_id < 0) { return MAAT_SCAN_ERR; } - enum table_type table_type = TABLE_TYPE_INVALID; - if (0 == vtable_id) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); - if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { - return MAAT_SCAN_ERR; - } + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); + if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { + return MAAT_SCAN_ERR; } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int hit_group_cnt = string_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, data, - data_len, physical_table_ids, - physical_table_cnt, vtable_id, mid); + int hit_group_cnt = string_scan(maat_instance->tbl_mgr, thread_id, data, data_len, + physical_table_id, vtable_id, mid); if (hit_group_cnt < 0) { return MAAT_SCAN_ERR; } @@ -1666,52 +1617,43 @@ struct maat_stream *maat_scan_stream_open(struct maat *maat_instance, int table_ } struct maat_stream *stream = ALLOC(struct maat_stream, 1); - memset(stream->physical_table_ids, -1, sizeof(stream->physical_table_ids)); - stream->ref_maat_instance = maat_instance; - stream->table_id = table_id; stream->thread_id = thread_id; stream->logger = maat_instance->logger; - stream->n_physical_table = vtable_get_physical_table_ids(stream->ref_maat_instance->tbl_mgr, - stream->table_id, stream->physical_table_ids, - MAX_PHYSICAL_TABLE_NUM, &stream->vtable_id); - if (stream->n_physical_table <= 0) { + + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + stream->physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id); + stream->vtable_id = table_id; + } else { + stream->physical_table_id = table_id; + stream->vtable_id = 0; + } + + if (stream->physical_table_id < 0) { return NULL; } - enum table_type table_type = TABLE_TYPE_INVALID; - if (0 == stream->vtable_id) { - table_type = table_manager_get_table_type(stream->ref_maat_instance->tbl_mgr, - stream->physical_table_ids[0]); - if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { - log_error(maat_instance->logger, MODULE_MAAT_API, - "[%s:%d] table(table_id:%d) thread_id:%d scan stream's physical table must be expr or expr_plus", - __FUNCTION__, __LINE__, table_id, thread_id); - return NULL; - } + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, stream->physical_table_id); + if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { + return NULL; } - for (size_t i = 0; i < stream->n_physical_table; i++) { - enum table_type table_type = table_manager_get_table_type(stream->ref_maat_instance->tbl_mgr, - stream->physical_table_ids[i]); - if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { - log_error(maat_instance->logger, MODULE_MAAT_API, - "%s:%d] table(table_id:%d) thread_id:%d scan stream's physical table must be expr or expr_plus", - __FUNCTION__, __LINE__, table_id, thread_id); - return NULL; - } + void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr, + stream->physical_table_id); + assert(expr_rt != NULL); - void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr, - stream->physical_table_ids[i]); - assert(expr_rt != NULL); - - int ret = expr_runtime_stream_open((struct expr_runtime *)expr_rt, thread_id); - if (ret < 0) { - return NULL; - } + struct adapter_hs_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt, thread_id); + if (NULL == handle) { + goto error; } + stream->s_handle = handle; return stream; + +error: + FREE(stream); + return NULL; } int maat_scan_stream(struct maat_stream **maat_stream, const char *data, int data_len, @@ -1728,20 +1670,23 @@ int maat_scan_stream(struct maat_stream **maat_stream, const char *data, int dat mid = grab_state(state, stream->ref_maat_instance, stream->thread_id); mid->scan_cnt++; + int valid_table_id = -1; + if (stream->vtable_id != 0) { + valid_table_id = stream->vtable_id; + } else { + valid_table_id = stream->physical_table_id; + } + if (NULL == stream->ref_maat_instance->maat_rt) { log_error(stream->logger, MODULE_MAAT_API, "[%s:%d] table(table_id:%d) thread_id:%d maat_scan_stream error because of maat_runtime is NULL", - __FUNCTION__, __LINE__, stream->table_id, stream->thread_id); + __FUNCTION__, __LINE__, valid_table_id, stream->thread_id); return MAAT_SCAN_OK; } alignment_int64_array_add(stream->ref_maat_instance->thread_call_cnt, stream->thread_id, 1); - int hit_group_cnt = stream_scan_hit_group_count(stream->ref_maat_instance->tbl_mgr, - stream->thread_id, data, data_len, - stream->physical_table_ids, - stream->n_physical_table, - stream->vtable_id, mid); + int hit_group_cnt = expr_stream_scan(stream, data, data_len, mid); if (hit_group_cnt < 0) { return MAAT_SCAN_ERR; } @@ -1774,13 +1719,7 @@ void maat_scan_stream_close(struct maat_stream **maat_stream) { struct maat_stream *stream = *maat_stream; - for (size_t i = 0; i < stream->n_physical_table; i++) { - void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr, - stream->physical_table_ids[i]); - assert(expr_rt != NULL); - expr_runtime_stream_close((struct expr_runtime *)expr_rt); - } - + expr_runtime_stream_close(stream->s_handle); FREE(stream); } diff --git a/src/maat_bool_plugin.c b/src/maat_bool_plugin.c index 44d76ae..bfae01f 100644 --- a/src/maat_bool_plugin.c +++ b/src/maat_bool_plugin.c @@ -144,7 +144,6 @@ int bool_plugin_table_set_ex_data_schema(void *bool_plugin_schema, return 0; } - static int cmp_ull_p(const void *p1, const void *p2) { if(* (unsigned long long*) p1 > * (unsigned long long*) p2) { diff --git a/src/maat_expr.c b/src/maat_expr.c index 9857d27..70eb92a 100644 --- a/src/maat_expr.c +++ b/src/maat_expr.c @@ -34,7 +34,6 @@ struct expr_schema { int expr_type_column; int match_method_column; int is_hexbin_column; - enum hs_pattern_type pattern_type; /* literal or regex */ int table_id; //ugly struct table_manager *ref_tbl_mgr; }; @@ -68,12 +67,9 @@ struct expr_item { }; struct expr_runtime { - enum hs_pattern_type pattern_type; struct adapter_hs *hs; - struct adapter_hs_stream *hs_stream; struct rcu_hash_table *htable; // store hs_expr rule for rebuild adapter_hs instance struct rcu_hash_table *item_htable; // store this expr table's all maat_item which will be used in expr_runtime_scan - struct group2group_runtime *ref_g2g_rt; uint32_t rule_num; int n_worker_thread; @@ -275,34 +271,26 @@ void expr_item_free(struct expr_item *expr_item) FREE(expr_item); } -enum hs_pattern_type pattern_type_str_to_enum(const char *type_str) -{ - enum hs_pattern_type pattern_type = HS_PATTERN_TYPE_MAX; - - if (strcmp(type_str, "literal") == 0) { - pattern_type = HS_PATTERN_TYPE_STR; - } else if (strcmp(type_str, "regex") == 0) { - pattern_type = HS_PATTERN_TYPE_REG; - } else { - assert(0); - } - - return pattern_type; -} - void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger) { - int read_cnt = 0; + char table_type[NAME_MAX] = {0}; struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1); cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); if (item != NULL && item->type == cJSON_Number) { expr_schema->table_id = item->valueint; - read_cnt++; + } else { + log_error(logger, MODULE_EXPR, + "[%s:%d] table %s has no table_id column", table_name); + goto error; } + /* table_type already validate in maat_table_new() */ + item = cJSON_GetObjectItem(json, "table_type"); + memcpy(table_type, item->valuestring, strlen(item->valuestring)); + item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { log_error(logger, MODULE_EXPR, @@ -310,59 +298,73 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, goto error; } - custom_item = cJSON_GetObjectItem(item, "pattern_type"); - if (custom_item != NULL && custom_item->type == cJSON_String) { - expr_schema->pattern_type = pattern_type_str_to_enum(custom_item->valuestring); - read_cnt++; - } - custom_item = cJSON_GetObjectItem(item, "item_id"); if (custom_item != NULL && custom_item->type == cJSON_Number) { expr_schema->item_id_column = custom_item->valueint; - read_cnt++; + } else { + log_error(logger, MODULE_EXPR, + "[%s:%d] table %s has no item_id column", table_name); + goto error; } custom_item = cJSON_GetObjectItem(item, "group_id"); if (custom_item != NULL && custom_item->type == cJSON_Number) { expr_schema->group_id_column = custom_item->valueint; - read_cnt++; + } else { + log_error(logger, MODULE_EXPR, + "[%s:%d] table %s has no group_id column", table_name); + goto error; } custom_item = cJSON_GetObjectItem(item, "keywords"); if (custom_item != NULL && custom_item->type == cJSON_Number) { expr_schema->keywords_column = custom_item->valueint; - read_cnt++; + } else { + log_error(logger, MODULE_EXPR, + "[%s:%d] table %s has no keywords column", table_name); + goto error; } /* expr_plus has district */ - custom_item = cJSON_GetObjectItem(item, "district"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - expr_schema->district_column = custom_item->valueint; - } + if (strcmp(table_type, "expr_plus") == 0) { + custom_item = cJSON_GetObjectItem(item, "district"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + expr_schema->district_column = custom_item->valueint; + } else { + log_error(logger, MODULE_EXPR, + "[%s:%d] expr_plus table %s has no district column", table_name); + goto error; + } + } - custom_item = cJSON_GetObjectItem(item, "expr_type"); + custom_item = cJSON_GetObjectItem(item, "expr_type"); if (custom_item != NULL && custom_item->type == cJSON_Number) { expr_schema->expr_type_column = custom_item->valueint; - read_cnt++; + } else { + log_error(logger, MODULE_EXPR, + "[%s:%d] table %s has no expr_type column", table_name); + goto error; } custom_item = cJSON_GetObjectItem(item, "match_method"); if (custom_item != NULL && custom_item->type == cJSON_Number) { expr_schema->match_method_column = custom_item->valueint; - read_cnt++; + } else { + log_error(logger, MODULE_EXPR, + "[%s:%d] table %s has no match_method column", table_name); + goto error; } custom_item = cJSON_GetObjectItem(item, "is_hexbin"); if (custom_item != NULL && custom_item->type == cJSON_Number) { expr_schema->is_hexbin_column = custom_item->valueint; - read_cnt++; + } else { + log_error(logger, MODULE_EXPR, + "[%s:%d] table %s has no is_hexbin column", table_name); + goto error; } expr_schema->ref_tbl_mgr = tbl_mgr; - - if (read_cnt < 8) { - goto error; - } return expr_schema; error: @@ -408,12 +410,10 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num, return NULL; } - struct expr_schema *schema = (struct expr_schema *)expr_schema; struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1); expr_rt->htable = rcu_hash_new(expr_ex_data_free); expr_rt->item_htable = rcu_hash_new(expr_maat_item_free); - expr_rt->pattern_type = schema->pattern_type; expr_rt->n_worker_thread = max_thread_num; expr_rt->ref_garbage_bin = garbage_bin; expr_rt->logger = logger; @@ -436,10 +436,10 @@ void expr_runtime_free(void *expr_runtime) expr_rt->hs = NULL; } - if (expr_rt->hs_stream != NULL) { - adapter_hs_stream_close(expr_rt->hs_stream); - expr_rt->hs_stream = NULL; - } + // if (expr_rt->hs_stream != NULL) { + // adapter_hs_stream_close(expr_rt->hs_stream); + // expr_rt->hs_stream = NULL; + // } if (expr_rt->htable != NULL) { rcu_hash_free(expr_rt->htable); @@ -563,9 +563,9 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d if (i >= MAAT_MAX_EXPR_ITEM_NUM) { log_error(logger, MODULE_EXPR, - "[%s:%d] expr item_id:%d too many patterns", + "[%s:%d]abandon config expr_item(item_id:%d) too many patterns", __FUNCTION__, __LINE__, expr_item->item_id); - return NULL; + goto error; } sub_key_array[i] = tmp; @@ -586,9 +586,9 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d if (i >= MAAT_MAX_EXPR_ITEM_NUM) { log_error(logger, MODULE_EXPR, - "[%s:%d] expr item_id:%d too many patterns", + "[%s:%d]abandon config expr_item(item_id:%d) too many patterns", __FUNCTION__, __LINE__, expr_item->item_id); - return NULL; + goto error; } sub_key_array[i] = tmp; @@ -596,17 +596,17 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d if (!(key_left_offset[i] >= 0 && key_right_offset[i] > 0 && key_left_offset[i] <= key_right_offset[i])) { log_error(logger, MODULE_EXPR, - "[%s:%d] expr item:%d has invalid offset.", + "[%s:%d]abandon config expr_item(item_id:%d) has invalid offset.", __FUNCTION__, __LINE__, expr_item->item_id); - return NULL; + goto error; } sub_key_array[i] = (char *)memchr(sub_key_array[i], ':', strlen(sub_key_array[i])); if (NULL == sub_key_array[i]) { log_error(logger, MODULE_EXPR, - "[%s:%d] expr item:%d has invalid offset keyword format.", + "[%s:%d]abandon config expr_item(item_id:%d) has invalid offset keyword format.", __FUNCTION__, __LINE__, expr_item->item_id); - return NULL; + goto error; } sub_key_array[i]++;//jump over ':' @@ -620,24 +620,26 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d sub_key_array[0] = str_unescape(sub_key_array[0]); break; default: - break; + log_error(logger, MODULE_EXPR, + "[%s:%d]abandon config expr_item(item_id:%lld) has invalid expr type=%d", + __FUNCTION__, __LINE__, expr_item->item_id, expr_item->expr_type); + goto error; } - size_t region_str_len = 0; - char *region_string = NULL; - size_t sub_key_len = 0; - for (i = 0; i < sub_expr_cnt; i++) { - + size_t region_str_len = 0; + char *region_string = NULL; + size_t sub_key_len = 0; + if (FALSE == expr_item->is_case_sensitive) { // insensitive expr_rule->patterns[i].case_sensitive = HS_CASE_INSESITIVE; } - - enum hs_pattern_type pattern_type = expr_type2pattern_type(expr_item->expr_type); - if (TRUE == expr_item->is_hexbin && pattern_type != HS_PATTERN_TYPE_REG) { - region_str_len = strlen(sub_key_array[i]) + 1; + expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type); + + if (TRUE == expr_item->is_hexbin && expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) { + region_str_len = strlen(sub_key_array[i]) * 8 + 1; region_string = ALLOC(char, region_str_len); region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]), region_string, region_str_len); } @@ -656,8 +658,8 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d expr_rule->patterns[i].match_mode = expr_item->match_mode; if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) { - expr_rule->patterns[i].l_offset = key_left_offset[i]; - expr_rule->patterns[i].r_offset = key_right_offset[i]; + expr_rule->patterns[i].start_offset = key_left_offset[i]; + expr_rule->patterns[i].end_offset = key_right_offset[i]; } } expr_rule->expr_id = expr_item->item_id; @@ -674,6 +676,9 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d // printf("expr_rule->patterns[%zu].r_offset:%d\n", i, expr_rule->patterns[i].r_offset); // } return expr_rule; +error: + FREE(expr_rule); + return NULL; } int expr_runtime_update(void *expr_runtime, void *expr_schema, @@ -763,18 +768,11 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name) struct hs_expr *rules = NULL; void **ex_data_array = NULL; size_t rule_cnt = rcu_hash_list(expr_rt->htable, &ex_data_array); - //printf("rcu_hash_commit rule_cnt:%zu\n", rule_cnt); + if (rule_cnt > 0) { rules = ALLOC(struct hs_expr, rule_cnt); for (size_t i = 0; i < rule_cnt; i++) { rules[i] = *(struct hs_expr *)ex_data_array[i]; - // if (rules[i].expr_id == 13) - // { - // for (size_t j = 0; j < rules[i].n_patterns; j++) - // { - // printf("rules[%zu].patterns[%zu]:%s\n", i, j, rules[i].patterns[j].pat); - // } - // } } } @@ -785,8 +783,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name) int ret = 0; struct adapter_hs *new_adapter_hs = NULL; struct adapter_hs *old_adapter_hs = NULL; - new_adapter_hs = adapter_hs_initialize(expr_rt->pattern_type, expr_rt->n_worker_thread, - rules, rule_cnt, expr_rt->logger); + new_adapter_hs = adapter_hs_initialize(expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger); if (NULL == new_adapter_hs) { log_error(expr_rt->logger, MODULE_EXPR, "[%s:%d] table[%s] rebuild adapter_hs engine failed when update %zu expr rules", @@ -867,24 +864,17 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d return group_hit_cnt; } -int expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) +struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) { if (NULL == expr_rt || thread_id < 0) { - return -1; + return NULL; } - struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(expr_rt->hs, thread_id); - if (NULL == hs_stream) { - return -1; - } - - expr_rt->hs_stream = hs_stream; - - return 0; + return adapter_hs_stream_open(expr_rt->hs, thread_id); } -int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, - int vtable_id, struct maat_state *state) +int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle, + const char *data, size_t data_len, int vtable_id, struct maat_state *state) { if (0 == expr_rt->rule_num) { //empty expr table @@ -893,9 +883,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz size_t n_hit_item = 0; struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; - int ret = adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, - hit_results, MAX_SCANNER_HIT_ITEM_NUM, - &n_hit_item); + int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item); if (ret < 0) { return -1; } @@ -928,12 +916,9 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz return group_hit_cnt; } -void expr_runtime_stream_close(struct expr_runtime *expr_rt) +void expr_runtime_stream_close(struct adapter_hs_stream *s_handle) { - if (expr_rt != NULL) { - adapter_hs_stream_close(expr_rt->hs_stream); - expr_rt->hs_stream = NULL; - } + adapter_hs_stream_close(s_handle); } void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id) diff --git a/src/maat_plugin.c b/src/maat_plugin.c index b07aff5..a551579 100644 --- a/src/maat_plugin.c +++ b/src/maat_plugin.c @@ -94,60 +94,57 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, } schema->table_id = item->valueint; + /* custom is optional */ item = cJSON_GetObjectItem(json, "custom"); - if (item == NULL || item->type != cJSON_Object) { - log_error(logger, MODULE_PLUGIN, - "[%s:%d] plugin table %s has no custom column", - __FUNCTION__, __LINE__, table_name); - goto error; - } + if (item != NULL && item->type == cJSON_Object) { + custom_item = cJSON_GetObjectItem(item, "key"); + if (NULL == custom_item || custom_item->type != cJSON_Number) { + log_error(logger, MODULE_PLUGIN, + "[%s:%d] plugin table: %s has no key column", + __FUNCTION__, __LINE__, table_name); + goto error; + } + schema->key_column = custom_item->valueint; - custom_item = cJSON_GetObjectItem(item, "key"); - if (NULL == custom_item || custom_item->type != cJSON_Number) { - log_error(logger, MODULE_PLUGIN, - "[%s:%d] plugin table: %s has no key column", - __FUNCTION__, __LINE__, table_name); - goto error; - } - schema->key_column = custom_item->valueint; - - custom_item = cJSON_GetObjectItem(item, "key_type"); - if (NULL == custom_item || custom_item->type != cJSON_String) { - log_error(logger, MODULE_PLUGIN, - "[%s:%d] plugin table: %s has no key_type column", - __FUNCTION__, __LINE__, table_name); - goto error; - } + custom_item = cJSON_GetObjectItem(item, "key_type"); + if (NULL == custom_item || custom_item->type != cJSON_String) { + log_error(logger, MODULE_PLUGIN, + "[%s:%d] plugin table: %s has no key_type column", + __FUNCTION__, __LINE__, table_name); + goto error; + } - if (strcmp(custom_item->valuestring, "pointer") == 0) { - schema->key_type = PLUGIN_KEY_TYPE_POINTER; - } else if (strcmp(custom_item->valuestring, "integer") == 0) { - schema->key_type = PLUGIN_KEY_TYPE_INTEGER; - } else { - log_error(logger, MODULE_PLUGIN, - "[%s:%d] plugin table: %s key_type:%s illegal", - __FUNCTION__, __LINE__, table_name, - custom_item->valuestring); - goto error; - } - - custom_item = cJSON_GetObjectItem(item, "tag"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - schema->rule_tag_column = custom_item->valueint; - } + if (strcmp(custom_item->valuestring, "pointer") == 0) { + schema->key_type = PLUGIN_KEY_TYPE_POINTER; + } else if (strcmp(custom_item->valuestring, "integer") == 0) { + schema->key_type = PLUGIN_KEY_TYPE_INTEGER; + } else { + log_error(logger, MODULE_PLUGIN, + "[%s:%d] plugin table: %s key_type:%s illegal", + __FUNCTION__, __LINE__, table_name, + custom_item->valuestring); + goto error; + } - custom_item = cJSON_GetObjectItem(item, "foreign"); - if (custom_item != NULL) { - if (custom_item->type == cJSON_String) { - schema->n_foreign = read_integer_array(custom_item->valuestring, - schema->foreign_columns, - MAX_FOREIGN_CLMN_NUM); - } else if (custom_item->type == cJSON_Array) { - schema->n_foreign = cJSON_GetArraySize(custom_item); - for (int i = 0; i < schema->n_foreign; i++) { - cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i); - assert(foreign_item->type == cJSON_Number); - schema->foreign_columns[i] = foreign_item->valueint; + custom_item = cJSON_GetObjectItem(item, "tag"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->rule_tag_column = custom_item->valueint; + } + + custom_item = cJSON_GetObjectItem(item, "foreign"); + if (custom_item != NULL) { + if (custom_item->type == cJSON_String) { + schema->n_foreign = read_integer_array(custom_item->valuestring, + schema->foreign_columns, + MAX_FOREIGN_CLMN_NUM); + } else if (custom_item->type == cJSON_Array) { + schema->n_foreign = cJSON_GetArraySize(custom_item); + for (int i = 0; i < schema->n_foreign; i++) + { + cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i); + assert(foreign_item->type == cJSON_Number); + schema->foreign_columns[i] = foreign_item->valueint; + } } } } diff --git a/src/maat_virtual.c b/src/maat_virtual.c index ee11974..251398d 100644 --- a/src/maat_virtual.c +++ b/src/maat_virtual.c @@ -20,8 +20,7 @@ #define MODULE_VIRTUAL module_name_str("maat.virtual") struct virtual_schema { - char physical_tables[MAX_PHYSICAL_TABLE_NUM][NAME_MAX]; - size_t n_physical_table; + char physical_table[NAME_MAX]; int table_id; struct table_manager *ref_tbl_mgr; }; @@ -42,23 +41,15 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, schema->table_id = item->valueint; item = cJSON_GetObjectItem(json, "physical_table"); - if (NULL == item || item->type != cJSON_Array) { + if (NULL == item || item->type != cJSON_String) { log_error(logger, MODULE_VIRTUAL, "[%s:%d] virtual table %s has no physical_table column", __FUNCTION__, __LINE__, table_name); goto error; } - int cnt = cJSON_GetArraySize(item); - for (int i = 0; i < cnt; i++) { - cJSON *tmp_item = cJSON_GetArrayItem(item, i); - if (tmp_item != NULL && tmp_item->type == cJSON_String) { - memcpy(schema->physical_tables[i], tmp_item->valuestring, - strlen(tmp_item->valuestring)); - } - } - schema->n_physical_table = cnt; - + memcpy(schema->physical_table, item->valuestring, strlen(item->valuestring)); + return schema; error: FREE(schema); @@ -80,7 +71,7 @@ int virtual_table_get_id(void *virtual_schema) return schema->table_id; } -size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]) +int virtual_table_get_physical_table_id(void *virtual_schema) { if (NULL == virtual_schema) { return 0; @@ -88,10 +79,5 @@ size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_ta struct virtual_schema *schema = (struct virtual_schema *)virtual_schema; - for (size_t i = 0; i < schema->n_physical_table; i++) { - int table_id = table_manager_get_table_id(schema->ref_tbl_mgr, schema->physical_tables[i]); - physical_table_ids[i] = table_id; - } - - return schema->n_physical_table; + return table_manager_get_table_id(schema->ref_tbl_mgr, schema->physical_table); } \ No newline at end of file diff --git a/test/adapter_hs_gtest.cpp b/test/adapter_hs_gtest.cpp index 9ef6d36..0fe2624 100644 --- a/test/adapter_hs_gtest.cpp +++ b/test/adapter_hs_gtest.cpp @@ -174,11 +174,11 @@ int parse_config_file(const char *filename, struct hs_expr exprs[], size_t *n_ex printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n", key_left_offset, key_right_offset); } - exprs[i].patterns[j].l_offset = key_left_offset; - exprs[i].patterns[j].r_offset = key_right_offset; + exprs[i].patterns[j].start_offset = key_left_offset; + exprs[i].patterns[j].end_offset = key_right_offset; } else { - exprs[i].patterns[j].l_offset = -1; - exprs[i].patterns[j].r_offset = -1; + exprs[i].patterns[j].start_offset = -1; + exprs[i].patterns[j].end_offset = -1; } } } @@ -209,27 +209,23 @@ TEST(adapter_hs_init, invalid_input_parameter) struct hs_expr expr_array[64]; size_t n_expr_array = 0; - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, - 1, NULL, 0, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, NULL, 0, g_logger); EXPECT_TRUE(hs_instance == NULL); - hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array, - n_expr_array, g_logger); + hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance == NULL); n_expr_array = 1; expr_array[0].expr_id = 101; expr_array[0].n_patterns = 10; - hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array, - n_expr_array, g_logger); + hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance == NULL); memset(expr_array, 0, sizeof(expr_array)); n_expr_array = 1; expr_array[0].expr_id = 101; expr_array[0].n_patterns = 1; - hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array, - n_expr_array, g_logger); + hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance == NULL); } @@ -240,10 +236,9 @@ TEST(adapter_hs_scan, literal_sub_has_normal_offset) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_expr_array, 11); + EXPECT_EQ(n_expr_array, 12); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, - expr_array, n_expr_array, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); expr_array_free(expr_array, n_expr_array); @@ -268,6 +263,14 @@ TEST(adapter_hs_scan, literal_sub_has_normal_offset) n_result = 0; ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 101); + + char scan_data4[64] = "Aaahello aaa"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 0); adapter_hs_destroy(hs_instance); @@ -282,8 +285,7 @@ TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, - expr_array, n_expr_array, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); expr_array_free(expr_array, n_expr_array); @@ -309,6 +311,14 @@ TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset) n_result = 0; ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 102); + + char scan_data4[64] = "Aaahello bbb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 0); adapter_hs_destroy(hs_instance); @@ -323,8 +333,7 @@ TEST(adapter_hs_scan, literal_sub_has_right_unlimit_offset) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, - expr_array, n_expr_array, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); expr_array_free(expr_array, n_expr_array); @@ -379,8 +388,7 @@ TEST(adapter_hs_scan, literal_sub_with_no_offset) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, - expr_array, n_expr_array, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); expr_array_free(expr_array, n_expr_array); @@ -427,8 +435,7 @@ TEST(adapter_hs_scan, literal_exactly) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, - expr_array, n_expr_array, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); expr_array_free(expr_array, n_expr_array); @@ -469,8 +476,7 @@ TEST(adapter_hs_scan, literal_prefix) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, - expr_array, n_expr_array, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); expr_array_free(expr_array, n_expr_array); @@ -520,8 +526,7 @@ TEST(adapter_hs_scan, literal_suffix) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, - expr_array, n_expr_array, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); expr_array_free(expr_array, n_expr_array); @@ -571,8 +576,7 @@ TEST(adapter_hs_scan, literal_sub_with_hexbin) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, - expr_array, n_expr_array, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); expr_array_free(expr_array, n_expr_array); @@ -604,8 +608,7 @@ TEST(adapter_hs_scan, literal_with_chinese) int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, - expr_array, n_expr_array, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); EXPECT_TRUE(hs_instance != NULL); expr_array_free(expr_array, n_expr_array); @@ -621,6 +624,30 @@ TEST(adapter_hs_scan, literal_with_chinese) hs_instance = NULL; } +TEST(adapter_hs_scan, same_pattern_different_offset) +{ + struct hs_expr expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(expr_array, n_expr_array); + + char data[64] = "onetoday,anothertoday"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, data, strlen(data), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 112); + + adapter_hs_destroy(hs_instance); + hs_instance = NULL; +} + int main(int argc, char **argv) { int ret = 0; diff --git a/test/literal_expr.conf b/test/literal_expr.conf index dfbf055..e25b42a 100644 --- a/test/literal_expr.conf +++ b/test/literal_expr.conf @@ -141,6 +141,26 @@ "pattern": "يىلىدىكى" } ] + }, + { + "expr_id": 112, + "pattern_num": 2, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "today", + "offset": "3~7" + }, + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "today", + "offset": "16~20" + } + ] } ] } diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 4fe7965..224cb65 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -353,7 +353,7 @@ TEST_F(MaatFlagScan, basic) { TEST_F(MaatFlagScan, withExprRegion) { const char *flag_table_name = "FLAG_CONFIG"; - const char *expr_table_name = "HTTP_URL_LITERAL"; + const char *expr_table_name = "HTTP_URL"; struct maat *maat_instance = MaatFlagScan::_shared_maat_instance; int flag_table_id = maat_get_table_id(maat_instance, flag_table_name); @@ -540,7 +540,7 @@ struct maat *MaatStringScan::_shared_maat_instance; struct log_handle *MaatStringScan::logger; TEST_F(MaatStringScan, Full) { - const char *table_name = "HTTP_URL_REGEX"; + const char *table_name = "HTTP_URL"; struct maat *maat_instance = MaatStringScan::_shared_maat_instance; int table_id = maat_get_table_id(maat_instance, table_name); @@ -564,7 +564,7 @@ TEST_F(MaatStringScan, Regex) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *cookie = "Cookie: Txa123aheadBCAxd"; - const char *table_name = "HTTP_URL_REGEX"; + const char *table_name = "HTTP_URL"; struct maat *maat_instance = MaatStringScan::_shared_maat_instance; int table_id = maat_get_table_id(maat_instance, table_name); @@ -629,7 +629,7 @@ TEST_F(MaatStringScan, ExprPlus) { EXPECT_EQ(results[0], 190); maat_state_free(&state); } -#if 0 + TEST_F(MaatStringScan, ExprPlusWithOffset) { long long results[ARRAY_SIZE] = {0}; @@ -681,7 +681,7 @@ TEST_F(MaatStringScan, ExprPlusWithOffset) maat_state_free(&state); } - +#if 0 TEST_F(MaatStringScan, ExprPlusWithHex) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; @@ -721,7 +721,7 @@ TEST_F(MaatStringScan, ExprAndExprPlus) { size_t n_hit_result = 0; struct maat_state *state = NULL; struct maat *maat_instance = MaatStringScan::_shared_maat_instance; - const char *expr_table_name = "HTTP_URL_LITERAL"; + const char *expr_table_name = "HTTP_URL"; const char *expr_plus_table_name = "HTTP_SIGNATURE"; const char *region_name = "I love China"; const char *scan_data = "today is Monday and yesterday is Tuesday"; @@ -904,7 +904,8 @@ TEST_F(MaatStringScan, PrefixAndSuffix) { EXPECT_EQ(results[0], 152); maat_state_free(&state); } - +#endif +#if 0 TEST_F(MaatStringScan, MaatUnescape) { const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me."; const char *table_name = "KEYWORDS_TABLE"; @@ -923,11 +924,12 @@ TEST_F(MaatStringScan, MaatUnescape) { EXPECT_EQ(results[0], 132); maat_state_free(&state); } - +#endif +#if 0 TEST_F(MaatStringScan, RegexWithNotContains) { const char *should_NOT_hit_scan_data = "new.qq.com/rain/a/TWF2021042600418000"; const char *should_hit_scan_data = "fakesina.com/rain/a/TWF2021042600418000"; - const char *table_name = "HTTP_URL_REGEX"; + const char *table_name = "HTTP_URL"; struct maat *maat_instance = MaatStringScan::_shared_maat_instance; int table_id = maat_get_table_id(maat_instance, table_name); @@ -1031,7 +1033,7 @@ TEST_F(MaatStringScan, StreamInput) { struct maat_state *state = NULL; struct maat *maat_instance = MaatStringScan::_shared_maat_instance; const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; - const char *table_name = "HTTP_URL_REGEX"; + const char *table_name = "HTTP_URL"; int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); @@ -1053,7 +1055,7 @@ TEST_F(MaatStringScan, StreamInput) { } TEST_F(MaatStringScan, dynamic_config) { - const char *table_name = "HTTP_URL_LITERAL"; + const char *table_name = "HTTP_URL"; char data[128] = "hello world, welcome to maat version4, it's funny."; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; @@ -1414,7 +1416,7 @@ TEST_F(NOTLogic, OneRegion) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - const char *table_name = "HTTP_URL_LITERAL"; + const char *table_name = "HTTP_URL"; struct maat *maat_instance = NOTLogic::_shared_maat_instance; int table_id = maat_get_table_id(maat_instance, table_name); @@ -1442,7 +1444,7 @@ TEST_F(NOTLogic, ScanNotAtLast) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - const char *hit_table_name = "HTTP_URL_LITERAL"; + const char *hit_table_name = "HTTP_URL"; const char *not_hit_table_name = "KEYWORDS_TABLE"; struct maat *maat_instance = NOTLogic::_shared_maat_instance; @@ -1469,7 +1471,7 @@ TEST_F(NOTLogic, ScanIrrelavantAtLast) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - const char *hit_table_name = "HTTP_URL_LITERAL"; + const char *hit_table_name = "HTTP_URL"; const char *not_hit_table_name = "KEYWORDS_TABLE"; struct maat *maat_instance = NOTLogic::_shared_maat_instance; @@ -1498,7 +1500,7 @@ TEST_F(NOTLogic, ScanHitAtLastEmptyExpr) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - const char *not_hit_table_name = "HTTP_URL_LITERAL"; + const char *not_hit_table_name = "HTTP_URL"; const char *hit_table_name = "IP_PLUS_CONFIG"; const char *empty_table_name = "EMPTY_KEYWORD"; struct maat *maat_instance = NOTLogic::_shared_maat_instance; @@ -1537,7 +1539,7 @@ TEST_F(NOTLogic, ScanHitAtLastEmptyInteger) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - const char *not_hit_table_name = "HTTP_URL_LITERAL"; + const char *not_hit_table_name = "HTTP_URL"; const char *hit_table_name = "IP_PLUS_CONFIG"; const char *empty_table_name = "EMPTY_INTERGER"; struct maat *maat_instance = NOTLogic::_shared_maat_instance; @@ -1575,7 +1577,7 @@ TEST_F(NOTLogic, ScanNotIP) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - const char *hit_table_name = "HTTP_URL_LITERAL"; + const char *hit_table_name = "HTTP_URL"; const char *not_hit_table_name = "IP_CONFIG"; struct maat *maat_instance = NOTLogic::_shared_maat_instance; @@ -2367,7 +2369,7 @@ TEST_F(CompileTable, Conjunction1) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; - const char *table_name = "HTTP_URL_LITERAL"; + const char *table_name = "HTTP_URL"; const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"}; struct maat *maat_instance = CompileTable::_shared_maat_instance; @@ -2394,7 +2396,7 @@ TEST_F(CompileTable, Conjunction2) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; - const char *table_name = "HTTP_URL_LITERAL"; + const char *table_name = "HTTP_URL"; const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"}; struct maat *maat_instance = CompileTable::_shared_maat_instance; @@ -2525,7 +2527,7 @@ TEST_F(Policy, CompileRuleTags) { struct maat_state *state = NULL; const char *should_hit = "string bbb should hit"; const char *should_not_hit = "string aaa should not hit"; - const char *table_name = "HTTP_URL_LITERAL"; + const char *table_name = "HTTP_URL"; struct maat *maat_instance = Policy::_shared_maat_instance; int table_id = maat_get_table_id(maat_instance, table_name); @@ -2549,7 +2551,7 @@ TEST_F(Policy, CompileEXData) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; - const char *table_name = "HTTP_URL_LITERAL"; + const char *table_name = "HTTP_URL"; const char *compile_table_name = "COMPILE_ALIAS"; const char *expect_name = "I have a name"; struct maat *maat_instance = Policy::_shared_maat_instance; @@ -2679,8 +2681,8 @@ TEST_F(TableInfo, Conjunction) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *scan_data = "soq is using table conjunction function.http://www.3300av.com/novel/27122.txt"; - const char *table_name = "HTTP_URL_LITERAL"; - const char *conj_table_name = "HTTP_HOST_LITERAL"; + const char *table_name = "HTTP_URL"; + const char *conj_table_name = "HTTP_HOST"; struct maat *maat_instance = TableInfo::_shared_maat_instance; int table_id = maat_get_table_id(maat_instance, table_name); @@ -2869,7 +2871,7 @@ TEST_F(MaatCmdTest, SetIP) { TEST_F(MaatCmdTest, SetExpr) { const char *scan_data = "Hiredis is a minimalistic C client library for the Redis database.\r\n"; - const char *table_name = "HTTP_URL_LITERAL"; + const char *table_name = "HTTP_URL"; const char *keywords1 = "Hiredis"; const char *keywords2 = "C Client"; diff --git a/test/maat_json.json b/test/maat_json.json index 4d48fb9..a4a4ee0 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -243,7 +243,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "abckkk&123", @@ -295,7 +295,7 @@ { "regions": [ { - "table_name": "HTTP_URL_REGEX", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "action=search\\&query=(.*)", @@ -321,7 +321,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "should_not_hit_any_rule", @@ -387,7 +387,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "C#中国", @@ -491,7 +491,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_HOST_LITERAL", + "table_name": "HTTP_HOST", "table_type": "expr", "table_content": { "keywords": "www.3300av.com", @@ -517,7 +517,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "novel&27122.txt", @@ -596,7 +596,7 @@ { "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "should&hit&aaa", @@ -623,7 +623,7 @@ { "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "should&hit&bbb", @@ -677,7 +677,7 @@ "g2c_table_name": "GROUP2COMPILE_ALIAS", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "i.ytimg.com", @@ -702,7 +702,7 @@ { "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": ",IgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE", @@ -728,7 +728,7 @@ "not_flag":0, "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "must-contained-string-of-rule-143", @@ -743,7 +743,7 @@ "not_flag":1, "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "must-not-contained-string-of-rule-143", @@ -769,7 +769,7 @@ "not_flag":0, "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "must-contained-string-of-rule-144", @@ -810,7 +810,7 @@ "not_flag":0, "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "must-contained-string-of-rule-145", @@ -839,7 +839,7 @@ { "regions": [ { - "table_name": "HTTP_URL_REGEX", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "Cookie:\\s&head", @@ -864,7 +864,7 @@ { "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "googlevideo.com/videoplayback&mn=sn-35153iuxa-5a56%2Csn-n8v7znz7", @@ -877,35 +877,34 @@ } ] }, - { - "compile_id": 148, - "is_valid": "yes", - "do_log": 0, - "effective_rage": 0, - "action": 0, - - "service": 0, - "do_blacklist": 0, - "user_region": "StringScan.ExprPlusWithOffset", - "groups": [ - { - "regions": [ - { - "table_name": "APP_PAYLOAD", - "table_content": { - "format": "hexbin", - "match_method": "sub", - "district": "Payload", - "keywords": "1-1:03&9-10:2d&14-16:2d34", - "expr_type": "offset" - }, - "table_type": "expr_plus" - } - ], - "group_name": "Untitled" - } - ] - }, + { + "compile_id": 148, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "user_region": "StringScan.ExprPlusWithOffset", + "effective_rage": 0, + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "APP_PAYLOAD", + "table_type": "expr_plus", + "table_content": { + "format": "hexbin", + "match_method": "sub", + "district": "Payload", + "keywords": "1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d", + "expr_type": "offset" + } + } + ] + } + ] + }, { "compile_id": 149, "service": 1, @@ -1304,7 +1303,7 @@ "not_flag":0, "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "https://blog.csdn.net/littlefang/article/details/8213058", @@ -1458,7 +1457,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "cavemancircus.com/", @@ -1500,7 +1499,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "2019/12/27/pretty-girls-6", @@ -1527,7 +1526,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "2019/12/27", @@ -1554,7 +1553,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "2019/12/27", @@ -2051,7 +2050,7 @@ "not_flag": 1, "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "must-not-contained-string-of-rule-186", @@ -2103,7 +2102,7 @@ "not_flag": 1, "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "must-not-contained-string-of-rule-187", @@ -2155,7 +2154,7 @@ "not_flag": 1, "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "must-not-contained-string-of-rule-188", @@ -2323,7 +2322,7 @@ { "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "hello", @@ -2386,7 +2385,7 @@ { "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "Monday", @@ -2436,7 +2435,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_URL_LITERAL", + "table_name": "HTTP_URL", "table_type": "expr", "table_content": { "keywords": "hqdefault.jpg", diff --git a/test/table_info.conf b/test/table_info.conf index c375362..ccf602d 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -29,16 +29,10 @@ "table_id":2, "table_name":"COMPILE_ALIAS", "table_type":"compile", - "user_region_encoded":"escape", "valid_column":4, "custom": { "compile_id":1, - "service_id":2, - "action":3, - "do_blacklist":4, - "do_log":5, "tags":6, - "user_region":7, "clause_num":9, "evaluation_order":10 } @@ -69,11 +63,10 @@ }, { "table_id":5, - "table_name":["HTTP_URL_LITERAL", "HTTP_HOST_LITERAL"], + "table_name":["HTTP_URL", "HTTP_HOST"], "table_type":"expr", "valid_column":7, "custom": { - "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -84,21 +77,6 @@ }, { "table_id":6, - "table_name":"HTTP_URL_REGEX", - "table_type":"expr", - "valid_column":7, - "custom": { - "pattern_type":"regex", - "item_id":1, - "group_id":2, - "keywords":3, - "expr_type":4, - "match_method":5, - "is_hexbin":6 - } - }, - { - "table_id":7, "table_name":"KEYWORDS_TABLE", "table_type":"expr", "valid_column":7, @@ -113,7 +91,7 @@ } }, { - "table_id":8, + "table_id":7, "table_name":"IP_CONFIG", "table_type":"ip_plus", "valid_column":18, @@ -127,7 +105,7 @@ } }, { - "table_id":9, + "table_id":8, "table_name":"CONTENT_SIZE", "table_type":"intval", "valid_column":5, @@ -139,23 +117,17 @@ } }, { - "table_id":10, + "table_id":9, "table_name":"QD_ENTRY_INFO", "table_type":"plugin", - "valid_column":4, - "custom": { - "key_type":"pointer", - "key":1, - "tag":3 - } + "valid_column":4 }, { - "table_id":11, + "table_id":10, "table_name":"HTTP_SIGNATURE", "table_type":"expr_plus", "valid_column":8, "custom": { - "pattern_type":"literal", "item_id":1, "group_id":2, "district":3, @@ -166,12 +138,11 @@ } }, { - "table_id":12, + "table_id":11, "table_name":"IMAGE_FP", "table_type":"expr", "valid_column":7, "custom": { - "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -181,7 +152,7 @@ } }, { - "table_id":13, + "table_id":12, "table_name":"TEST_EFFECTIVE_RANGE_TABLE", "table_type":"plugin", "valid_column":4, @@ -192,7 +163,7 @@ } }, { - "table_id":14, + "table_id":13, "table_name":"TEST_FOREIGN_KEY", "table_type":"plugin", "valid_column":4, @@ -204,7 +175,7 @@ } }, { - "table_id":15, + "table_id":14, "table_name":"TEST_PLUGIN_EXDATA_TABLE", "table_type":"plugin", "valid_column":4, @@ -215,7 +186,7 @@ } }, { - "table_id":16, + "table_id":15, "table_name":"IR_INTERCEPT_IP", "table_type":"plugin", "valid_column":14, @@ -226,12 +197,11 @@ } }, { - "table_id":17, + "table_id":16, "table_name":"APP_PAYLOAD", "table_type":"expr_plus", "valid_column":8, "custom": { - "pattern_type":"literal", "item_id":1, "group_id":2, "district":3, @@ -242,12 +212,11 @@ } }, { - "table_id":18, + "table_id":17, "table_name":"TROJAN_PAYLOAD", "table_type":"expr", "valid_column":7, "custom": { - "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -258,12 +227,11 @@ } }, { - "table_id":19, + "table_id":18, "table_name":"MAIL_ADDR", "table_type":"expr", "valid_column":7, "custom": { - "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -273,7 +241,7 @@ } }, { - "table_id":20, + "table_id":19, "table_name":"IP_PLUS_CONFIG", "table_type":"ip_plus", "valid_column":18, @@ -287,43 +255,43 @@ } }, { - "table_id":21, + "table_id":20, "table_name":"HTTP_RESPONSE_KEYWORDS", "table_type":"virtual", - "physical_table": ["KEYWORDS_TABLE"] + "physical_table": "KEYWORDS_TABLE" + }, + { + "table_id":21, + "table_name":"HTTP_REQUEST_HEADER", + "table_type":"virtual", + "physical_table": "HTTP_SIGNATURE" }, { "table_id":22, - "table_name":"HTTP_REQUEST_HEADER", + "table_name":"HTTP_RESPONSE_HEADER", "table_type":"virtual", - "physical_table": ["HTTP_SIGNATURE"] + "physical_table": "HTTP_SIGNATURE" }, { "table_id":23, - "table_name":"HTTP_RESPONSE_HEADER", + "table_name":"VIRTUAL_IP_PLUS_TABLE", "table_type":"virtual", - "physical_table": ["HTTP_SIGNATURE"] + "physical_table": "IP_PLUS_CONFIG" }, { "table_id":24, - "table_name":"VIRTUAL_IP_PLUS_TABLE", + "table_name":"VIRTUAL_IP_PLUS_SOURCE", "table_type":"virtual", - "physical_table": ["IP_PLUS_CONFIG"] + "physical_table": "IP_PLUS_CONFIG" }, { "table_id":25, - "table_name":"VIRTUAL_IP_PLUS_SOURCE", + "table_name":"VIRTUAL_IP_PLUS_DESTINATION", "table_type":"virtual", - "physical_table": ["IP_PLUS_CONFIG"] + "physical_table": "IP_PLUS_CONFIG" }, { "table_id":26, - "table_name":"VIRTUAL_IP_PLUS_DESTINATION", - "table_type":"virtual", - "physical_table": ["IP_PLUS_CONFIG"] - }, - { - "table_id":27, "table_name":"TEST_IP_PLUGIN_WITH_EXDATA", "table_type":"ip_plugin", "valid_column":6, @@ -336,12 +304,11 @@ } }, { - "table_id":28, + "table_id":27, "table_name":"AS_NUMBER", "table_type":"expr", "valid_column":7, "custom": { - "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -351,24 +318,23 @@ } }, { - "table_id":29, + "table_id":28, "table_name":"SOURCE_IP_ASN", "table_type":"virtual", - "physical_table":["AS_NUMBER"] + "physical_table":"AS_NUMBER" + }, + { + "table_id":29, + "table_name":"DESTINATION_IP_ASN", + "table_type":"virtual", + "physical_table":"AS_NUMBER" }, { "table_id":30, - "table_name":"DESTINATION_IP_ASN", - "table_type":"virtual", - "physical_table":["AS_NUMBER"] - }, - { - "table_id":31, "table_name":"GeoLocation", "table_type":"expr", "valid_column":7, "custom": { - "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -378,13 +344,13 @@ } }, { - "table_id":32, + "table_id":31, "table_name":"SOURCE_IP_GEO", "table_type":"virtual", - "physical_table":["GeoLocation"] + "physical_table":"GeoLocation" }, { - "table_id":33, + "table_id":32, "table_name":"INTERGER_PLUS", "table_type":"intval_plus", "valid_column":6, @@ -397,7 +363,7 @@ } }, { - "table_id":34, + "table_id":33, "table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA", "table_type":"fqdn_plugin", "valid_column":5, @@ -408,7 +374,7 @@ } }, { - "table_id":35, + "table_id":34, "table_name":"APP_ID", "table_type":"intval", "valid_column":5, @@ -420,12 +386,11 @@ } }, { - "table_id":36, + "table_id":35, "table_name":"EMPTY_KEYWORD", "table_type":"expr", "valid_column":7, "custom": { - "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -435,7 +400,7 @@ } }, { - "table_id":37, + "table_id":36, "table_name":"EMPTY_INTERGER", "table_type":"intval", "valid_column":5, @@ -447,7 +412,7 @@ } }, { - "table_id":38, + "table_id":37, "table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA", "table_type":"bool_plugin", "valid_column":4, @@ -457,7 +422,7 @@ } }, { - "table_id":39, + "table_id":38, "table_name":"FLAG_CONFIG", "table_type":"flag", "valid_column":5, @@ -469,7 +434,7 @@ } }, { - "table_id":40, + "table_id":39, "table_name":"FLAG_PLUS_CONFIG", "table_type":"flag_plus", "valid_column":6, @@ -482,7 +447,7 @@ } }, { - "table_id":41, + "table_id":40, "table_name":"PORT_CONFIG", "table_type":"port", "valid_column":7, @@ -496,19 +461,19 @@ } }, { - "table_id":42, + "table_id":41, "table_name":"VIRTUAL_PORT_SOURCE", "table_type":"virtual", - "physical_table": ["PORT_CONFIG"] + "physical_table": "PORT_CONFIG" + }, + { + "table_id":42, + "table_name":"VIRTUAL_PORT_DESTINATION", + "table_type":"virtual", + "physical_table": "PORT_CONFIG" }, { "table_id":43, - "table_name":"VIRTUAL_PORT_DESTINATION", - "table_type":"virtual", - "physical_table": ["PORT_CONFIG"] - }, - { - "table_id":44, "table_name":"COMPOSITION_CONFIG", "table_type":"composition", "custom": { @@ -519,7 +484,7 @@ } }, { - "table_id":45, + "table_id":44, "table_name":"TEST_PLUGIN_KEY_TYPE_TABLE", "table_type":"plugin", "valid_column":4,