gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support same pattern different offset(x-x:pat1 & y-y:pat1)

Browse Source
This commit is contained in:
liuwentan
2023-03-22 11:10:00 +08:00
parent 37447eef7f
commit 23ef2c3797
15 changed files with 970 additions and 906 deletions
Download Patch File Download Diff File Expand all files Collapse all files

523
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -22,6 +22,8 @@
#include "maat_utils.h"
#include "../bool_matcher/bool_matcher.h"
#define MAX_OFFSET_NUM 1024
pid_t hs_gettid()
{
return syscall(SYS_gettid);
@@ -50,8 +52,10 @@ struct adapter_hs_runtime {
hs_database_t *literal_db;
hs_database_t *regex_db;
hs_scratch_t **scratchs;
size_t scratch_size;
hs_scratch_t **literal_scratchs;
hs_scratch_t **regex_scratchs;
size_t literal_scratch_size;
size_t regex_scratch_size;
struct bool_matcher *bm;
};
@@ -63,21 +67,25 @@ struct adapter_hs {
size_t n_patterns;
struct adapter_hs_runtime *hs_rt;
struct hs_tag *tag_map;
struct pattern_attribute *pat_attr_by_str;
struct pattern_attribute *pat_attr_by_id;
struct log_handle *logger;
};
struct matched_offset {
unsigned long long start_offset;
unsigned long long end_offset;
};
struct matched_pattern {
unsigned long long pattern_id;
unsigned long matched_l_offset;
unsigned long matched_r_offset;
struct matched_offset *offsets;
size_t offset_cnt;
size_t offset_size;
UT_hash_handle hh;
};
struct matched_pattern_container {
UT_array *pat_ids;
unsigned long long pattern_id;
unsigned long long l_matched;
unsigned long long r_matched;
struct matched_pattern *pat_hash;
};
@@ -92,10 +100,11 @@ struct adapter_hs_stream {
};
struct pattern_attribute {
unsigned long long bool_expr_id;
unsigned long long pattern_id;
enum hs_match_mode match_mode;
int l_offset;
int r_offset;
int start_offset;
int end_offset;
};
struct hs_tag {
@@ -108,40 +117,54 @@ struct hs_tag {
UT_hash_handle hh;
};
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread,
enum hs_pattern_type pattern_type, struct log_handle *logger)
int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratchs, size_t n_worker_thread,
struct log_handle *logger)
{
hs_database_t *database = NULL;
hs_rt->scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
size_t scratch_size = 0;
if (pattern_type == HS_PATTERN_TYPE_STR) {
database = hs_rt->literal_db;
} else {
database = hs_rt->regex_db;
}
if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) {
if (hs_alloc_scratch(db, &scratchs[0]) != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to allocate scratch space. Exiting.",
"[%s:%d] Unable to allocate scratch space. Exiting.",
__FUNCTION__, __LINE__);
hs_free_database(database);
return -1;
}
for (size_t i = 1; i < n_worker_thread; i++) {
hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]);
hs_error_t err = hs_clone_scratch(scratchs[0], &scratchs[i]);
if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to clone scratch prototype", __FUNCTION__, __LINE__);
hs_free_database(database);
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to clone scratch", __FUNCTION__, __LINE__);
return -1;
}
err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size);
err = hs_scratch_size(scratchs[i], &scratch_size);
if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS,
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to query scratch size", __FUNCTION__, __LINE__);
hs_free_database(database);
return -1;
}
}
return 0;
}
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread,
enum hs_pattern_type pattern_type, struct log_handle *logger)
{
int ret = 0;
if (pattern_type == HS_PATTERN_TYPE_STR) {
hs_rt->literal_scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->literal_scratchs, n_worker_thread, logger);
if (ret < 0) {
FREE(hs_rt->literal_scratchs);
return -1;
}
} else {
hs_rt->regex_scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->regex_db, hs_rt->regex_scratchs, n_worker_thread, logger);
if (ret < 0) {
FREE(hs_rt->regex_scratchs);
return -1;
}
}
@@ -155,20 +178,20 @@ static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_work
* @retval 0(success) -1(failed)
*/
static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
struct adpt_hs_compile_data *compile_data,
enum hs_pattern_type pattern_type,
struct adpt_hs_compile_data *literal_cd,
struct adpt_hs_compile_data *regex_cd,
struct log_handle *logger)
{
hs_error_t err;
hs_compile_error_t *compile_err = NULL;
if (NULL == hs_rt || NULL == compile_data) {
if (NULL == hs_rt || (NULL == literal_cd && NULL == regex_cd)) {
return -1;
}
if (pattern_type == HS_PATTERN_TYPE_STR) {
err = hs_compile_lit_multi((const char *const *)compile_data->patterns, compile_data->flags,
compile_data->ids, compile_data->pattern_lens, compile_data->n_patterns,
if (literal_cd != NULL) {
err = hs_compile_lit_multi((const char *const *)literal_cd->patterns, literal_cd->flags,
literal_cd->ids, literal_cd->pattern_lens, literal_cd->n_patterns,
HS_MODE_STREAM, NULL, &hs_rt->literal_db, &compile_err);
if (err != HS_SUCCESS) {
if (compile_err) {
@@ -179,12 +202,12 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
hs_free_compile_error(compile_err);
return -1;
}
} else {
err = hs_compile_multi((const char *const *)compile_data->patterns,
compile_data->flags, compile_data->ids,
compile_data->n_patterns,
HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL,
&hs_rt->regex_db, &compile_err);
}
if (regex_cd != NULL) {
err = hs_compile_multi((const char *const *)regex_cd->patterns, regex_cd->flags, regex_cd->ids,
regex_cd->n_patterns, HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL,
&hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) {
if (compile_err) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s",
@@ -203,20 +226,21 @@ struct adpt_hs_compile_data *adpt_hs_compile_data_new(size_t n_patterns)
struct adpt_hs_compile_data *hs_cd = ALLOC(struct adpt_hs_compile_data, 1);
hs_cd->patterns = ALLOC(char *, n_patterns);
hs_cd->pattern_lens = ALLOC(size_t, n_patterns);
hs_cd->n_patterns = n_patterns;
hs_cd->ids = ALLOC(unsigned int, n_patterns);
hs_cd->flags = ALLOC(unsigned int, n_patterns);
return hs_cd;
}
void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patterns)
void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd)
{
if (NULL == hs_cd) {
return;
}
if (hs_cd->patterns != NULL) {
for (size_t i = 0; i < n_patterns; i++) {
for (size_t i = 0; i < hs_cd->n_patterns; i++) {
FREE(hs_cd->patterns[i]);
}
@@ -259,21 +283,92 @@ void hs_tag_free(struct hs_tag *tag)
FREE(tag);
}
struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
size_t n_worker_thread,
void populate_compile_data(struct adpt_hs_compile_data *compile_data, int index, int pattern_id,
char *pat, size_t pat_len, int case_sensitive)
{
compile_data->ids[index] = pattern_id;
/* set flags */
compile_data->flags[index] |= HS_FLAG_SOM_LEFTMOST;
if (case_sensitive == HS_CASE_INSESITIVE) {
compile_data->flags[index] |= HS_FLAG_CASELESS;
}
compile_data->pattern_lens[index] = pat_len;
compile_data->patterns[index] = ALLOC(char, pat_len + 1);
memcpy(compile_data->patterns[index], pat, pat_len);
}
struct bool_expr *bool_exprs_new(struct hs_expr *exprs, size_t n_expr, struct hs_tag **tag_hash,
struct adpt_hs_compile_data *literal_cd, struct adpt_hs_compile_data *regex_cd,
size_t *n_pattern)
{
uint32_t pattern_index = 0;
uint32_t literal_index = 0;
uint32_t regex_index = 0;
struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr);
if (NULL == bool_exprs) {
return NULL;
}
/* populate adpt_hs_compile_data and bool_expr */
for (size_t i = 0; i < n_expr; i++) {
struct hs_tag *hs_tag = hs_tag_new(exprs[i].expr_id, exprs[i].n_patterns);
hs_tag->user_tag = exprs[i].user_tag;
for (size_t j = 0; j < exprs[i].n_patterns; j++) {
hs_tag->pat_attr[j].pattern_id = pattern_index;
hs_tag->pat_attr[j].match_mode = exprs[i].patterns[j].match_mode;
if (exprs[i].patterns[j].match_mode == HS_MATCH_MODE_SUB) {
hs_tag->pat_attr[j].start_offset = exprs[i].patterns[j].start_offset;
hs_tag->pat_attr[j].end_offset = exprs[i].patterns[j].end_offset;
}
/* literal pattern */
if (exprs[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) {
populate_compile_data(literal_cd, literal_index, pattern_index,
exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len,
exprs[i].patterns[j].case_sensitive);
literal_index++;
} else {
/* regex pattern */
populate_compile_data(regex_cd, regex_index, pattern_index,
exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len,
exprs[i].patterns[j].case_sensitive);
regex_index++;
}
bool_exprs[i].items[j].item_id = pattern_index++;
bool_exprs[i].items[j].not_flag = 0;
// printf("item_id:%llu, pat:%s pat_len:%zu\n",
// bool_exprs[i].items[j].item_id, exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len);
}
//printf("expr_id:%lld item_num:%zu\n", exprs[i].expr_id, exprs[i].n_patterns);
bool_exprs[i].expr_id = exprs[i].expr_id;
bool_exprs[i].item_num = exprs[i].n_patterns;
bool_exprs[i].user_tag = hs_tag;
HASH_ADD_KEYPTR(hh, *tag_hash, hs_tag->key, hs_tag->key_len, hs_tag);
}
*n_pattern = pattern_index;
return bool_exprs;
}
struct adapter_hs *adapter_hs_initialize(size_t n_worker_thread,
struct hs_expr *exprs, size_t n_expr,
struct log_handle *logger)
{
if ((pattern_type != HS_PATTERN_TYPE_STR && pattern_type != HS_PATTERN_TYPE_REG) ||
0 == n_worker_thread || NULL == exprs || 0 == n_expr) {
if (0 == n_worker_thread || NULL == exprs || 0 == n_expr) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] input parameters illegal!",
__FUNCTION__, __LINE__);
return NULL;
}
/* get the sum of pattern */
size_t pattern_num = 0;
size_t literal_pattern_num = 0;
size_t regex_pattern_num = 0;
for (size_t i = 0; i < n_expr; i++) {
if (exprs[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
log_error(logger, MODULE_ADAPTER_HS,
@@ -283,83 +378,53 @@ struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
}
for (size_t j = 0; j < exprs[i].n_patterns; j++) {
/* pat_len should not 0 */
if (0 == exprs[i].patterns[j].pat_len) {
log_error(logger, MODULE_ADAPTER_HS,
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] expr pattern length should not 0", __FUNCTION__, __LINE__);
return NULL;
}
pattern_num++;
if (exprs[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) {
literal_pattern_num++;
} else {
regex_pattern_num++;
}
}
}
if (0 == pattern_num) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] expr array has no valid pattern",
if (0 == literal_pattern_num && 0 == regex_pattern_num) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] exprs has no valid pattern",
__FUNCTION__, __LINE__);
return NULL;
}
struct adpt_hs_compile_data *compile_data = NULL;
compile_data = adpt_hs_compile_data_new(pattern_num);
struct adpt_hs_compile_data *literal_cd = NULL;
struct adpt_hs_compile_data *regex_cd = NULL;
if (literal_pattern_num > 0) {
literal_cd = adpt_hs_compile_data_new(literal_pattern_num);
}
if (regex_pattern_num > 0) {
regex_cd = adpt_hs_compile_data_new(regex_pattern_num);
}
uint32_t pattern_index = 0;
size_t pattern_cnt = 0;
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1);
hs_instance->tag_map = NULL;
hs_instance->logger = logger;
struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr);
/* populate adpt_hs_compile_data and bool_expr */
for (size_t i = 0; i < n_expr; i++) {
struct hs_tag *hs_tag = hs_tag_new(exprs[i].expr_id, exprs[i].n_patterns);
hs_tag->user_tag = exprs[i].user_tag;
for (size_t j = 0; j < exprs[i].n_patterns; j++) {
size_t pat_len = 0;
hs_tag->pat_attr[j].pattern_id = pattern_index;
hs_tag->pat_attr[j].match_mode = exprs[i].patterns[j].match_mode;
if (exprs[i].patterns[j].match_mode == HS_MATCH_MODE_SUB) {
hs_tag->pat_attr[j].l_offset = exprs[i].patterns[j].l_offset;
hs_tag->pat_attr[j].r_offset = exprs[i].patterns[j].r_offset;
}
compile_data->ids[pattern_index] = pattern_index;
if (pattern_type == HS_PATTERN_TYPE_STR) {
compile_data->flags[pattern_index] |= HS_FLAG_SOM_LEFTMOST;
}
if (exprs[i].patterns[j].case_sensitive == HS_CASE_INSESITIVE) {
compile_data->flags[pattern_index] |= HS_FLAG_CASELESS;
}
pat_len = exprs[i].patterns[j].pat_len;
compile_data->pattern_lens[pattern_index] = pat_len;
compile_data->patterns[pattern_index] = ALLOC(char, pat_len + 1);
memcpy(compile_data->patterns[pattern_index], exprs[i].patterns[j].pat,
exprs[i].patterns[j].pat_len);
bool_exprs[i].items[j].item_id = pattern_index;
pattern_index++;
}
bool_exprs[i].expr_id = exprs[i].expr_id;
bool_exprs[i].item_num = exprs[i].n_patterns;
bool_exprs[i].user_tag = hs_tag;
HASH_ADD_KEYPTR(hh, hs_instance->tag_map, hs_tag->key, hs_tag->key_len, hs_tag);
}
compile_data->n_patterns = pattern_index;
int ret = -1;
size_t mem_size = 0;
hs_instance->n_worker_thread = n_worker_thread;
hs_instance->n_patterns = pattern_index;
hs_instance->n_expr = n_expr;
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
struct bool_expr *bool_exprs = bool_exprs_new(exprs, n_expr, &hs_instance->tag_map,
literal_cd, regex_cd, &pattern_cnt);
if (NULL == bool_exprs) {
return NULL;
}
hs_instance->n_patterns = pattern_cnt;
//mytest
// for (size_t i = 0; i < n_expr; i++) {
// if (bool_exprs[i].expr_id == 37)
// {
// printf("<before bool_matcher_new> exprs[%zu] expr_id:%llu, item_num:%zu\n",
// i, bool_exprs[i].expr_id, bool_exprs[i].item_num);
@@ -369,10 +434,14 @@ struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
// printf("%llu ", bool_exprs[i].items[j].item_id);
// }
// }
// printf("\n");
// }
// printf("\n");
/* create bool matcher */
size_t mem_size = 0;
int hs_ret = 0;
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
hs_instance->hs_rt->bm = bool_matcher_new(bool_exprs, n_expr, &mem_size);
if (hs_instance->hs_rt->bm != NULL) {
log_info(logger, MODULE_ADAPTER_HS,
@@ -381,30 +450,47 @@ struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
} else {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Adapter_hs module: build bool matcher failed",
__FUNCTION__, __LINE__);
adpt_hs_compile_data_free(compile_data, pattern_index);
FREE(bool_exprs);
adapter_hs_destroy(hs_instance);
return NULL;
hs_ret = -1;
}
FREE(bool_exprs);
/* build hs database */
ret = adpt_hs_build_database(hs_instance->hs_rt, compile_data, pattern_type, logger);
int ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, logger);
if (ret < 0) {
hs_ret = -1;
}
if (literal_cd != NULL) {
adpt_hs_compile_data_free(literal_cd);
}
if (regex_cd != NULL) {
adpt_hs_compile_data_free(regex_cd);
}
if (hs_ret < 0) {
goto error;
}
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, pattern_type, logger);
if (ret < 0) {
goto error;
/* literal and regex scratch can't reuse */
if (literal_pattern_num > 0) {
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_STR, logger);
if (ret < 0) {
goto error;
}
}
if (regex_pattern_num > 0) {
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_REG, logger);
if (ret < 0) {
goto error;
}
}
adpt_hs_compile_data_free(compile_data, pattern_index);
return hs_instance;
error:
adpt_hs_compile_data_free(compile_data, pattern_index);
adapter_hs_destroy(hs_instance);
return NULL;
}
@@ -423,14 +509,23 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance)
hs_free_database(hs_instance->hs_rt->regex_db);
}
if (hs_instance->hs_rt->scratchs != NULL) {
if (hs_instance->hs_rt->literal_scratchs != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->scratchs[i]);
if (hs_instance->hs_rt->literal_scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->literal_scratchs[i]);
}
}
}
FREE(hs_instance->hs_rt->scratchs);
FREE(hs_instance->hs_rt->literal_scratchs);
if (hs_instance->hs_rt->regex_scratchs != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->regex_scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->regex_scratchs[i]);
}
}
}
FREE(hs_instance->hs_rt->regex_scratchs);
if (hs_instance->hs_rt->bm != NULL) {
bool_matcher_free(hs_instance->hs_rt->bm);
@@ -450,20 +545,19 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance)
FREE(hs_instance);
}
static inline int compare_pattern_id(const void *a, const void *b)
int find_same_pattern_offset(struct matched_pattern *matched_pat, unsigned long long from,
unsigned long long to)
{
long long ret = *(unsigned long long *)a - *(unsigned long long *)b;
for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
if (matched_pat->offsets[i].start_offset == from &&
matched_pat->offsets[i].end_offset == to - 1) {
return 0;
}
}
if (0 == ret) {
return 0;
} else if (ret < 0) {
return -1;
} else {
return 1;
}
return -1;
}
UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
/**
* @param id: pattern id
*/
@@ -474,50 +568,75 @@ int matched_event_cb(unsigned int id, unsigned long long from,
struct matched_pattern_container *matched_pat_container = (struct matched_pattern_container *)ctx;
unsigned long long pattern_id = id;
if (utarray_find(matched_pat_container->pat_ids, &pattern_id, compare_pattern_id)) {
struct matched_pattern *matched_pat = NULL;
HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), matched_pat);
if (matched_pat != NULL) {
// same pattern_id, offset maybe different
int ret = find_same_pattern_offset(matched_pat, from, to);
if (ret < 0) { /* different offset */
// TODO: use realloc
if (matched_pat->offset_cnt >= matched_pat->offset_size) {
matched_pat->offset_size *= 2;
matched_pat->offsets = (struct matched_offset *)realloc(matched_pat->offsets,
matched_pat->offset_size*sizeof(struct matched_offset));
}
matched_pat->offsets[matched_pat->offset_cnt].start_offset = from;
matched_pat->offsets[matched_pat->offset_cnt].end_offset = to - 1;
matched_pat->offset_cnt++;
}
return 0;
} else {
// different pattern_id
struct matched_pattern *matched_pat = ALLOC(struct matched_pattern, 1);
matched_pat->pattern_id = pattern_id;
matched_pat->offsets = ALLOC(struct matched_offset, MAX_OFFSET_NUM);
matched_pat->offset_size = MAX_OFFSET_NUM;
matched_pat->offsets[matched_pat->offset_cnt].start_offset = from;
matched_pat->offsets[matched_pat->offset_cnt].end_offset = to - 1;
matched_pat->offset_cnt++;
HASH_ADD(hh, matched_pat_container->pat_hash, pattern_id, sizeof(unsigned long long), matched_pat);
}
utarray_push_back(matched_pat_container->pat_ids, &pattern_id);
utarray_sort(matched_pat_container->pat_ids, compare_pattern_id);
struct matched_pattern *matched_pat = ALLOC(struct matched_pattern, 1);
matched_pat->pattern_id = pattern_id;
matched_pat->matched_l_offset = from;
matched_pat->matched_r_offset = to;
HASH_ADD(hh, matched_pat_container->pat_hash, pattern_id, sizeof(unsigned long long), matched_pat);
return 0;
}
int is_real_matched_pattern(struct matched_pattern *matched_pat, enum hs_match_mode match_mode,
size_t data_len, int attr_l_offset, int attr_r_offset)
size_t data_len, int attr_start_offset, int attr_end_offset)
{
if (match_mode == HS_MATCH_MODE_EXACTLY) {
if (matched_pat->matched_l_offset == 0 &&
matched_pat->matched_r_offset == data_len) {
return 0;
for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
if (matched_pat->offsets[i].start_offset == 0 &&
matched_pat->offsets[i].end_offset == data_len - 1) {
return 0;
}
}
} else if (match_mode == HS_MATCH_MODE_PREFIX) {
if (matched_pat->matched_l_offset == 0) {
return 0;
for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
if (matched_pat->offsets[i].start_offset == 0) {
return 0;
}
}
} else if (match_mode == HS_MATCH_MODE_SUFFIX) {
if (matched_pat->matched_r_offset == data_len) {
return 0;
for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
if (matched_pat->offsets[i].end_offset == data_len - 1) {
return 0;
}
}
} else if (match_mode == HS_MATCH_MODE_SUB) {
if (attr_l_offset == -1) {
attr_l_offset = 0;
if (attr_start_offset == -1) {
attr_start_offset = 0;
}
if (attr_r_offset == -1) {
attr_r_offset = (int)data_len;
if (attr_end_offset == -1) {
attr_end_offset = (int)data_len - 1;
}
if (matched_pat->matched_l_offset >= (unsigned long)attr_l_offset &&
matched_pat->matched_r_offset <= (unsigned long)attr_r_offset) {
return 0;
for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
if (matched_pat->offsets[i].start_offset >= (unsigned long long)attr_start_offset &&
matched_pat->offsets[i].end_offset <= (unsigned long long)attr_end_offset) {
return 0;
}
}
} else {
assert(0);
@@ -531,15 +650,13 @@ int hs_tag_validate(struct hs_tag *hs_tag, struct matched_pattern_container *mat
{
/* check if real matched pattern, because pattern match_mode is different */
for (size_t i = 0; i < hs_tag->n_pat_attr; i++) {
struct matched_pattern *tmp_matched_pat = NULL;
struct matched_pattern *matched_pat = NULL;
unsigned long long pattern_id = hs_tag->pat_attr[i].pattern_id;
HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), tmp_matched_pat);
if (tmp_matched_pat) {
int matched_ret = is_real_matched_pattern(tmp_matched_pat,
hs_tag->pat_attr[i].match_mode,
data_len,
hs_tag->pat_attr[i].l_offset,
hs_tag->pat_attr[i].r_offset);
HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), matched_pat);
if (matched_pat) {
int matched_ret = is_real_matched_pattern(matched_pat, hs_tag->pat_attr[i].match_mode,
data_len, hs_tag->pat_attr[i].start_offset,
hs_tag->pat_attr[i].end_offset);
if (matched_ret < 0) {
return -1;
}
@@ -562,14 +679,13 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
hs_stream->n_expr = hs_instance->n_expr;
hs_stream->n_patterns = hs_instance->n_patterns;
hs_stream->hs_rt = hs_instance->hs_rt;
utarray_new(hs_stream->matched_pat_container.pat_ids, &ut_pattern_id_icd);
utarray_reserve(hs_stream->matched_pat_container.pat_ids, hs_stream->n_patterns);
int err_count = 0;
if (hs_instance->hs_rt->literal_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->literal_db, 0, &hs_stream->literal_stream);
if (err != HS_SUCCESS) {
log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err);
return NULL;
err_count++;
}
}
@@ -577,11 +693,29 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
err = hs_open_stream(hs_instance->hs_rt->regex_db, 0, &hs_stream->regex_stream);
if (err != HS_SUCCESS) {
log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err);
return NULL;
err_count++;
}
}
if (err_count > 0) {
goto error;
}
return hs_stream;
error:
//TODO: hs_stream->hs_rt->scratchs[thread_id] may be free twice
if (hs_stream->literal_stream != NULL) {
hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL);
hs_stream->literal_stream = NULL;
}
if (hs_stream->regex_stream != NULL) {
hs_close_stream(hs_stream->regex_stream, NULL, NULL, NULL);
hs_stream->regex_stream = NULL;
}
FREE(hs_stream);
return NULL;
}
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
@@ -590,20 +724,14 @@ void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
return;
}
int thread_id = hs_stream->thread_id;
if (hs_stream->hs_rt != NULL) {
if (hs_stream->literal_stream != NULL) {
hs_close_stream(hs_stream->literal_stream,
hs_stream->hs_rt->scratchs[thread_id],
NULL, NULL);
hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL);
hs_stream->literal_stream = NULL;
}
if (hs_stream->regex_stream != NULL) {
hs_close_stream(hs_stream->regex_stream,
hs_stream->hs_rt->scratchs[thread_id],
NULL, NULL);
hs_close_stream(hs_stream->regex_stream, NULL, NULL, NULL);
hs_stream->regex_stream = NULL;
}
}
@@ -615,13 +743,23 @@ void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
FREE(pattern);
}
}
utarray_free(hs_stream->matched_pat_container.pat_ids);
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */
hs_stream->hs_rt = NULL;
FREE(hs_stream);
}
static int cmp_ull_p(const void *p1, const void *p2)
{
if(* (unsigned long long*) p1 > * (unsigned long long*) p2) {
return 1;
} else if(* (unsigned long long*) p1 < * (unsigned long long*) p2) {
return -1;
} else {
return 0;
}
}
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len,
struct hs_scan_result *results, size_t n_result, size_t *n_hit_result)
{
@@ -646,7 +784,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
int thread_id = hs_stream->thread_id;
if (hs_stream->literal_stream != NULL) {
err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
0, hs_stream->hs_rt->literal_scratchs[thread_id],
matched_event_cb, &hs_stream->matched_pat_container);
if (err != HS_SUCCESS) {
err_count++;
@@ -655,34 +793,42 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
if (hs_stream->regex_stream != NULL) {
err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
0, hs_stream->hs_rt->regex_scratchs[thread_id],
matched_event_cb, &hs_stream->matched_pat_container);
if (err != HS_SUCCESS) {
err_count++;
}
}
if (err_count > 0) {
if (err_count == 2) {
return -1;
}
size_t matched_pattern_ids_cnt = utarray_len(hs_stream->matched_pat_container.pat_ids);
if (0 == matched_pattern_ids_cnt) {
size_t n_item = HASH_COUNT(hs_stream->matched_pat_container.pat_hash);
if (0 == n_item) {
*n_hit_result = 0;
return 0;
}
unsigned long long items[matched_pattern_ids_cnt];
memset(items, 0, sizeof(unsigned long long) * matched_pattern_ids_cnt);
for (size_t i = 0; i < matched_pattern_ids_cnt; i++) {
items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->matched_pat_container.pat_ids, i);
if (n_item > MAX_SCANNER_HIT_ITEM_NUM) {
n_item = MAX_SCANNER_HIT_ITEM_NUM;
}
unsigned long long item_ids[MAX_SCANNER_HIT_ITEM_NUM];
memset(item_ids, 0, sizeof(unsigned long long) * MAX_SCANNER_HIT_ITEM_NUM);
int i = 0;
struct matched_pattern *pat = NULL, *tmp_pat = NULL;
HASH_ITER(hh, hs_stream->matched_pat_container.pat_hash, pat, tmp_pat) {
item_ids[i++] = pat->pattern_id;
}
qsort(item_ids, n_item, sizeof(unsigned long long), cmp_ull_p);
int ret = 0;
int real_matched_index = 0;
struct hs_tag *hs_tag = NULL;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, matched_pattern_ids_cnt,
int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, item_ids, n_item,
bool_matcher_results, hs_stream->n_expr);
if (bool_matcher_ret < 0) {
ret = -1;
@@ -715,7 +861,6 @@ next:
HASH_DELETE(hh, hs_stream->matched_pat_container.pat_hash, pattern);
FREE(pattern);
}
utarray_clear(hs_stream->matched_pat_container.pat_ids);
return ret;
}

16
scanner/adapter_hs/adapter_hs.h
View File

@@ -54,16 +54,18 @@ struct hs_scan_result {
struct hs_pattern {
enum hs_case_sensitive case_sensitive;
enum hs_match_mode match_mode;
enum hs_pattern_type pattern_type;
int is_hexbin; /* 1(yes) 0(no) */
/*
* just match in scan_data's range of [l_offset, r_offset], -1 means no limits
* just match in scan_data's range of [start_offset, end_offset], -1 means no limits
* for example:
* [-1, r_offset] means the pattern must in scan_data's [0 ~ r_offset]
* [l_offset, -1] means the pattern must in scan_data's [l_offset ~ end]
* [-1, end_offset] means the pattern must in scan_data's [0 ~ start_offset]
* [start_offset, -1] means the pattern must in scan_data's [start_offset ~ data_end]
*/
int l_offset;
int r_offset;
int start_offset;
int end_offset;
/* start pointer of pattern */
char *pat;
@@ -82,15 +84,13 @@ struct hs_expr {
/**
* @brief initialize adapter_hs instance
*
* @param pattern_type: pure literal or regex expression
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
* @param expr_array: logic AND expression's array
* @param n_expr_arrays: the number of logic AND expression's array
*
* @retval the pointer to adapter_hs instance
*/
struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
size_t n_worker_thread,
struct adapter_hs *adapter_hs_initialize(size_t n_worker_thread,
struct hs_expr *exprs, size_t n_expr,
struct log_handle *logger);

7
src/inc_internal/maat_expr.h
View File

@@ -45,10 +45,11 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name);
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int vtable_id, struct maat_state *state);
int expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len,
struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt);
void expr_runtime_stream_close(struct adapter_hs_stream *s_handle);
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id);
long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread);

1
src/inc_internal/maat_rule.h
View File

@@ -35,7 +35,6 @@ extern "C"
#define MAX_TABLE_NUM 1024
#define MAX_COMPILE_TABLE_NUM 16
#define MAX_PHYSICAL_TABLE_NUM 16
#define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2

2
src/inc_internal/maat_virtual.h
View File

@@ -39,7 +39,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void virtual_schema_free(void *virtual_schema);
int virtual_table_get_id(void *virtual_schema);
size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]);
int virtual_table_get_physical_table_id(void *virtual_schema);
#ifdef __cplusplus
}

597
src/maat_api.c
View File

@@ -48,11 +48,10 @@ enum district_set_flag {
struct maat_stream {
struct maat *ref_maat_instance;
struct adapter_hs_stream *s_handle; //each physical table open one stream
int thread_id;
int table_id;
int vtable_id;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
size_t n_physical_table;
int physical_table_id;
struct log_handle *logger;
};
@@ -987,243 +986,205 @@ size_t hit_group_to_compile(void *compile_runtime, long long *compile_ids, int i
return n_hit_compile;
}
static int vtable_get_physical_table_ids(struct table_manager *tbl_mgr, int table_id,
int *physical_table_ids, size_t n_table_id_array,
int *vtable_id)
static int vtable_get_physical_table_id(struct table_manager *tbl_mgr, int table_id)
{
size_t physical_table_cnt = 0;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
//find physical table id
*vtable_id = table_id;
void *virtual_schema = table_manager_get_schema(tbl_mgr, table_id);
assert(virtual_schema != NULL);
physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids);
return physical_table_cnt;
} else {
*vtable_id = 0;
physical_table_ids[0] = table_id;
physical_table_cnt = 1;
if (table_type != TABLE_TYPE_VIRTUAL) {
return -1;
}
return physical_table_cnt;
// find physical table id
void *virtual_schema = table_manager_get_schema(tbl_mgr, table_id);
assert(virtual_schema != NULL);
return virtual_table_get_physical_table_id(virtual_schema);
}
int flag_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long flag,
int physical_table_ids[], int physical_table_cnt, int vtable_id,
struct maat_state *mid)
int flag_scan(struct table_manager *tbl_mgr, int thread_id, long long flag,
int physical_table_id, int vtable_id, struct maat_state *mid)
{
int sum_hit_group_cnt = 0;
for (int i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if ((table_type == TABLE_TYPE_FLAG_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
return -1;
}
if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) {
continue;
}
void *flag_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == flag_rt) {
return -1;
}
int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id,
flag, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id);
}
sum_hit_group_cnt += group_hit_cnt;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if ((table_type == TABLE_TYPE_FLAG_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
return -1;
}
return sum_hit_group_cnt;
if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) {
return -1;
}
void *flag_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == flag_rt) {
return -1;
}
int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id,
flag, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id);
}
return group_hit_cnt;
}
int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long integer,
int physical_table_ids[], int physical_table_cnt, int vtable_id,
struct maat_state *mid)
int interval_scan(struct table_manager *tbl_mgr, int thread_id, long long integer,
int physical_table_id, int vtable_id, struct maat_state *mid)
{
int sum_hit_group_cnt = 0;
for (size_t i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if ((table_type == TABLE_TYPE_INTERVAL_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
//maat_instance->scan_err_cnt++;
return -1;
}
if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
continue;
}
void *interval_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == interval_rt) {
return -1;
}
int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt,
thread_id, integer, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id);
}
sum_hit_group_cnt += group_hit_cnt;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if ((table_type == TABLE_TYPE_INTERVAL_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
// maat_instance->scan_err_cnt++;
return -1;
}
return sum_hit_group_cnt;
if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
return -1;
}
void *interval_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == interval_rt) {
return -1;
}
int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt,
thread_id, integer, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id);
}
return group_hit_cnt;
}
int ipv4_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_addr,
int physical_table_ids[], int physical_table_cnt, int vtable_id,
struct maat_state *mid)
int ipv4_scan(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_addr,
int physical_table_id, int vtable_id, struct maat_state *mid)
{
int sum_hit_group_cnt = 0;
for (size_t i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if (table_type != TABLE_TYPE_IP_PLUS) {
continue;
}
void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == ip_rt) {
return -1;
}
int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4,
(uint8_t *)&ip_addr, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
}
sum_hit_group_cnt += group_hit_cnt;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_IP_PLUS) {
return -1;
}
return sum_hit_group_cnt;
void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == ip_rt) {
return -1;
}
int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4,
(uint8_t *)&ip_addr, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
}
return group_hit_cnt;
}
int ipv6_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint8_t *ip_addr,
int physical_table_ids[], int physical_table_cnt, int vtable_id,
struct maat_state *mid)
int ipv6_scan(struct table_manager *tbl_mgr, int thread_id, uint8_t *ip_addr,
int physical_table_id, int vtable_id, struct maat_state *mid)
{
int sum_hit_group_cnt = 0;
for (size_t i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if (table_type != TABLE_TYPE_IP_PLUS) {
continue;
}
void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == ip_rt) {
return -1;
}
int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6,
ip_addr, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
}
sum_hit_group_cnt += group_hit_cnt;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_IP_PLUS) {
return -1;
}
return sum_hit_group_cnt;
void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == ip_rt) {
return -1;
}
int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6,
ip_addr, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
}
return group_hit_cnt;
}
int string_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, const char *data,
size_t data_len, int physical_table_ids[], int physical_table_cnt,
int vtable_id, struct maat_state *mid)
int string_scan(struct table_manager *tbl_mgr, int thread_id, const char *data, size_t data_len,
int physical_table_id, int vtable_id, struct maat_state *mid)
{
int sum_hit_group_cnt = 0;
for (size_t i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
//maat_instance->scan_err_cnt++;
return -1;
}
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
continue;
}
void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == expr_rt) {
return -1;
}
int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt,
thread_id, data, data_len,
vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
}
sum_hit_group_cnt += group_hit_cnt;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
// maat_instance->scan_err_cnt++;
return -1;
}
return sum_hit_group_cnt;
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
return -1;
}
void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == expr_rt) {
return -1;
}
int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt,
thread_id, data, data_len,
vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
}
return group_hit_cnt;
}
int stream_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, const char *data,
size_t data_len, int physical_table_ids[], int physical_table_cnt,
int vtable_id, struct maat_state *mid)
int expr_stream_scan(struct maat_stream *stream, const char *data, size_t data_len, struct maat_state *state)
{
int sum_hit_group_cnt = 0;
for (size_t i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
//maat_instance->scan_err_cnt++;
return -1;
}
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
continue;
}
void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]);
if (NULL == expr_rt) {
return -1;
}
int group_hit_cnt = expr_runtime_stream_scan((struct expr_runtime *)expr_rt,
data, data_len, vtable_id, mid);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
}
sum_hit_group_cnt += group_hit_cnt;
if (NULL == stream || NULL == data) {
return 0;
}
return sum_hit_group_cnt;
struct table_manager *tbl_mgr = stream->ref_maat_instance->tbl_mgr;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, stream->physical_table_id);
if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == state || DISTRICT_FLAG_UNSET == state->is_set_district)) {
// maat_instance->scan_err_cnt++;
return -1;
}
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
return -1;
}
void *expr_rt = table_manager_get_runtime(tbl_mgr, stream->physical_table_id);
if (NULL == expr_rt) {
return -1;
}
int group_hit_cnt = expr_runtime_stream_scan((struct expr_runtime *)expr_rt, stream->s_handle,
data, data_len, stream->vtable_id, state);
if (group_hit_cnt < 0) {
return -1;
}
if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, stream->thread_id);
}
return group_hit_cnt;
}
size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n_result,
@@ -1279,31 +1240,29 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
}
int vtable_id = 0;
int physical_table_cnt = 0;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
memset(physical_table_ids, -1, sizeof(physical_table_ids));
int physical_table_id = -1;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id,
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
if (physical_table_id < 0) {
return MAAT_SCAN_ERR;
}
enum table_type table_type = TABLE_TYPE_INVALID;
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) {
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) {
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = flag_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, flag,
physical_table_ids,
physical_table_cnt, vtable_id, mid);
int hit_group_cnt = flag_scan(maat_instance->tbl_mgr, thread_id, flag,
physical_table_id, vtable_id, mid);
if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR;
}
@@ -1356,31 +1315,29 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id,
}
int vtable_id = 0;
int physical_table_cnt = 0;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
memset(physical_table_ids, -1, sizeof(physical_table_ids));
int physical_table_id = -1;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id,
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
if (physical_table_id < 0) {
return MAAT_SCAN_ERR;
}
enum table_type table_type = TABLE_TYPE_INVALID;
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = interval_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, integer,
physical_table_ids,
physical_table_cnt, vtable_id, mid);
int hit_group_cnt = interval_scan(maat_instance->tbl_mgr, thread_id, integer,
physical_table_id, vtable_id, mid);
if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR;
}
@@ -1433,31 +1390,29 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
}
int vtable_id = 0;
int physical_table_cnt = 0;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
memset(physical_table_ids, -1, sizeof(physical_table_ids));
int physical_table_id = -1;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id,
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
if (physical_table_id < 0) {
return MAAT_SCAN_ERR;
}
enum table_type table_type = TABLE_TYPE_INVALID;
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_IP_PLUS) {
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_IP_PLUS) {
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = ipv4_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, ip_addr,
physical_table_ids,
physical_table_cnt, vtable_id, mid);
int hit_group_cnt = ipv4_scan(maat_instance->tbl_mgr, thread_id, ip_addr,
physical_table_id, vtable_id, mid);
if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR;
}
@@ -1518,31 +1473,29 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id,
}
int vtable_id = 0;
int physical_table_cnt = 0;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
memset(physical_table_ids, -1, sizeof(physical_table_ids));
int physical_table_id = -1;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id,
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
if (physical_table_id < 0) {
return MAAT_SCAN_ERR;
}
enum table_type table_type = TABLE_TYPE_INVALID;
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_IP_PLUS) {
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_IP_PLUS) {
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = ipv6_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, ip_addr,
physical_table_ids,
physical_table_cnt, vtable_id, mid);
int hit_group_cnt = ipv6_scan(maat_instance->tbl_mgr, thread_id, ip_addr,
physical_table_id, vtable_id, mid);
if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR;
}
@@ -1603,31 +1556,29 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
}
int vtable_id = 0;
int physical_table_cnt = 0;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
memset(physical_table_ids, -1, sizeof(physical_table_ids));
int physical_table_id = -1;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id,
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
if (physical_table_id < 0) {
return MAAT_SCAN_ERR;
}
enum table_type table_type = TABLE_TYPE_INVALID;
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
return MAAT_SCAN_ERR;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = string_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, data,
data_len, physical_table_ids,
physical_table_cnt, vtable_id, mid);
int hit_group_cnt = string_scan(maat_instance->tbl_mgr, thread_id, data, data_len,
physical_table_id, vtable_id, mid);
if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR;
}
@@ -1666,52 +1617,43 @@ struct maat_stream *maat_scan_stream_open(struct maat *maat_instance, int table_
}
struct maat_stream *stream = ALLOC(struct maat_stream, 1);
memset(stream->physical_table_ids, -1, sizeof(stream->physical_table_ids));
stream->ref_maat_instance = maat_instance;
stream->table_id = table_id;
stream->thread_id = thread_id;
stream->logger = maat_instance->logger;
stream->n_physical_table = vtable_get_physical_table_ids(stream->ref_maat_instance->tbl_mgr,
stream->table_id, stream->physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &stream->vtable_id);
if (stream->n_physical_table <= 0) {
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) {
stream->physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
stream->vtable_id = table_id;
} else {
stream->physical_table_id = table_id;
stream->vtable_id = 0;
}
if (stream->physical_table_id < 0) {
return NULL;
}
enum table_type table_type = TABLE_TYPE_INVALID;
if (0 == stream->vtable_id) {
table_type = table_manager_get_table_type(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[0]);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] table(table_id:%d) thread_id:%d scan stream's physical table must be expr or expr_plus",
__FUNCTION__, __LINE__, table_id, thread_id);
return NULL;
}
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, stream->physical_table_id);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
return NULL;
}
for (size_t i = 0; i < stream->n_physical_table; i++) {
enum table_type table_type = table_manager_get_table_type(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[i]);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"%s:%d] table(table_id:%d) thread_id:%d scan stream's physical table must be expr or expr_plus",
__FUNCTION__, __LINE__, table_id, thread_id);
return NULL;
}
void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_id);
assert(expr_rt != NULL);
void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[i]);
assert(expr_rt != NULL);
int ret = expr_runtime_stream_open((struct expr_runtime *)expr_rt, thread_id);
if (ret < 0) {
return NULL;
}
struct adapter_hs_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt, thread_id);
if (NULL == handle) {
goto error;
}
stream->s_handle = handle;
return stream;
error:
FREE(stream);
return NULL;
}
int maat_scan_stream(struct maat_stream **maat_stream, const char *data, int data_len,
@@ -1728,20 +1670,23 @@ int maat_scan_stream(struct maat_stream **maat_stream, const char *data, int dat
mid = grab_state(state, stream->ref_maat_instance, stream->thread_id);
mid->scan_cnt++;
int valid_table_id = -1;
if (stream->vtable_id != 0) {
valid_table_id = stream->vtable_id;
} else {
valid_table_id = stream->physical_table_id;
}
if (NULL == stream->ref_maat_instance->maat_rt) {
log_error(stream->logger, MODULE_MAAT_API,
"[%s:%d] table(table_id:%d) thread_id:%d maat_scan_stream error because of maat_runtime is NULL",
__FUNCTION__, __LINE__, stream->table_id, stream->thread_id);
__FUNCTION__, __LINE__, valid_table_id, stream->thread_id);
return MAAT_SCAN_OK;
}
alignment_int64_array_add(stream->ref_maat_instance->thread_call_cnt, stream->thread_id, 1);
int hit_group_cnt = stream_scan_hit_group_count(stream->ref_maat_instance->tbl_mgr,
stream->thread_id, data, data_len,
stream->physical_table_ids,
stream->n_physical_table,
stream->vtable_id, mid);
int hit_group_cnt = expr_stream_scan(stream, data, data_len, mid);
if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR;
}
@@ -1774,13 +1719,7 @@ void maat_scan_stream_close(struct maat_stream **maat_stream)
{
struct maat_stream *stream = *maat_stream;
for (size_t i = 0; i < stream->n_physical_table; i++) {
void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[i]);
assert(expr_rt != NULL);
expr_runtime_stream_close((struct expr_runtime *)expr_rt);
}
expr_runtime_stream_close(stream->s_handle);
FREE(stream);
}

1
src/maat_bool_plugin.c
View File

@@ -144,7 +144,6 @@ int bool_plugin_table_set_ex_data_schema(void *bool_plugin_schema,
return 0;
}
static int cmp_ull_p(const void *p1, const void *p2)
{
if(* (unsigned long long*) p1 > * (unsigned long long*) p2) {

181
src/maat_expr.c
View File

@@ -34,7 +34,6 @@ struct expr_schema {
int expr_type_column;
int match_method_column;
int is_hexbin_column;
enum hs_pattern_type pattern_type; /* literal or regex */
int table_id; //ugly
struct table_manager *ref_tbl_mgr;
};
@@ -68,12 +67,9 @@ struct expr_item {
};
struct expr_runtime {
enum hs_pattern_type pattern_type;
struct adapter_hs *hs;
struct adapter_hs_stream *hs_stream;
struct rcu_hash_table *htable; // store hs_expr rule for rebuild adapter_hs instance
struct rcu_hash_table *item_htable; // store this expr table's all maat_item which will be used in expr_runtime_scan
struct group2group_runtime *ref_g2g_rt;
uint32_t rule_num;
int n_worker_thread;
@@ -275,34 +271,26 @@ void expr_item_free(struct expr_item *expr_item)
FREE(expr_item);
}
enum hs_pattern_type pattern_type_str_to_enum(const char *type_str)
{
enum hs_pattern_type pattern_type = HS_PATTERN_TYPE_MAX;
if (strcmp(type_str, "literal") == 0) {
pattern_type = HS_PATTERN_TYPE_STR;
} else if (strcmp(type_str, "regex") == 0) {
pattern_type = HS_PATTERN_TYPE_REG;
} else {
assert(0);
}
return pattern_type;
}
void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
int read_cnt = 0;
char table_type[NAME_MAX] = {0};
struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
expr_schema->table_id = item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no table_id column", table_name);
goto error;
}
/* table_type already validate in maat_table_new() */
item = cJSON_GetObjectItem(json, "table_type");
memcpy(table_type, item->valuestring, strlen(item->valuestring));
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_EXPR,
@@ -310,59 +298,73 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
goto error;
}
custom_item = cJSON_GetObjectItem(item, "pattern_type");
if (custom_item != NULL && custom_item->type == cJSON_String) {
expr_schema->pattern_type = pattern_type_str_to_enum(custom_item->valuestring);
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->item_id_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no item_id column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->group_id_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no group_id column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "keywords");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->keywords_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no keywords column", table_name);
goto error;
}
/* expr_plus has district */
custom_item = cJSON_GetObjectItem(item, "district");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->district_column = custom_item->valueint;
}
if (strcmp(table_type, "expr_plus") == 0) {
custom_item = cJSON_GetObjectItem(item, "district");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->district_column = custom_item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] expr_plus table %s has no district column", table_name);
goto error;
}
}
custom_item = cJSON_GetObjectItem(item, "expr_type");
custom_item = cJSON_GetObjectItem(item, "expr_type");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->expr_type_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no expr_type column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "match_method");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->match_method_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no match_method column", table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "is_hexbin");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->is_hexbin_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no is_hexbin column", table_name);
goto error;
}
expr_schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 8) {
goto error;
}
return expr_schema;
error:
@@ -408,12 +410,10 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num,
return NULL;
}
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->htable = rcu_hash_new(expr_ex_data_free);
expr_rt->item_htable = rcu_hash_new(expr_maat_item_free);
expr_rt->pattern_type = schema->pattern_type;
expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger;
@@ -436,10 +436,10 @@ void expr_runtime_free(void *expr_runtime)
expr_rt->hs = NULL;
}
if (expr_rt->hs_stream != NULL) {
adapter_hs_stream_close(expr_rt->hs_stream);
expr_rt->hs_stream = NULL;
}
// if (expr_rt->hs_stream != NULL) {
// adapter_hs_stream_close(expr_rt->hs_stream);
// expr_rt->hs_stream = NULL;
// }
if (expr_rt->htable != NULL) {
rcu_hash_free(expr_rt->htable);
@@ -563,9 +563,9 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_EXPR,
"[%s:%d] expr item_id:%d too many patterns",
"[%s:%d]abandon config expr_item(item_id:%d) too many patterns",
__FUNCTION__, __LINE__, expr_item->item_id);
return NULL;
goto error;
}
sub_key_array[i] = tmp;
@@ -586,9 +586,9 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_EXPR,
"[%s:%d] expr item_id:%d too many patterns",
"[%s:%d]abandon config expr_item(item_id:%d) too many patterns",
__FUNCTION__, __LINE__, expr_item->item_id);
return NULL;
goto error;
}
sub_key_array[i] = tmp;
@@ -596,17 +596,17 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
if (!(key_left_offset[i] >= 0 && key_right_offset[i] > 0
&& key_left_offset[i] <= key_right_offset[i])) {
log_error(logger, MODULE_EXPR,
"[%s:%d] expr item:%d has invalid offset.",
"[%s:%d]abandon config expr_item(item_id:%d) has invalid offset.",
__FUNCTION__, __LINE__, expr_item->item_id);
return NULL;
goto error;
}
sub_key_array[i] = (char *)memchr(sub_key_array[i], ':', strlen(sub_key_array[i]));
if (NULL == sub_key_array[i]) {
log_error(logger, MODULE_EXPR,
"[%s:%d] expr item:%d has invalid offset keyword format.",
"[%s:%d]abandon config expr_item(item_id:%d) has invalid offset keyword format.",
__FUNCTION__, __LINE__, expr_item->item_id);
return NULL;
goto error;
}
sub_key_array[i]++;//jump over ':'
@@ -620,24 +620,26 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
sub_key_array[0] = str_unescape(sub_key_array[0]);
break;
default:
break;
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%lld) has invalid expr type=%d",
__FUNCTION__, __LINE__, expr_item->item_id, expr_item->expr_type);
goto error;
}
size_t region_str_len = 0;
char *region_string = NULL;
size_t sub_key_len = 0;
for (i = 0; i < sub_expr_cnt; i++) {
size_t region_str_len = 0;
char *region_string = NULL;
size_t sub_key_len = 0;
if (FALSE == expr_item->is_case_sensitive) {
// insensitive
expr_rule->patterns[i].case_sensitive = HS_CASE_INSESITIVE;
}
enum hs_pattern_type pattern_type = expr_type2pattern_type(expr_item->expr_type);
if (TRUE == expr_item->is_hexbin && pattern_type != HS_PATTERN_TYPE_REG) {
region_str_len = strlen(sub_key_array[i]) + 1;
expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type);
if (TRUE == expr_item->is_hexbin && expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) {
region_str_len = strlen(sub_key_array[i]) * 8 + 1;
region_string = ALLOC(char, region_str_len);
region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]), region_string, region_str_len);
}
@@ -656,8 +658,8 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
expr_rule->patterns[i].match_mode = expr_item->match_mode;
if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) {
expr_rule->patterns[i].l_offset = key_left_offset[i];
expr_rule->patterns[i].r_offset = key_right_offset[i];
expr_rule->patterns[i].start_offset = key_left_offset[i];
expr_rule->patterns[i].end_offset = key_right_offset[i];
}
}
expr_rule->expr_id = expr_item->item_id;
@@ -674,6 +676,9 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
// printf("expr_rule->patterns[%zu].r_offset:%d\n", i, expr_rule->patterns[i].r_offset);
// }
return expr_rule;
error:
FREE(expr_rule);
return NULL;
}
int expr_runtime_update(void *expr_runtime, void *expr_schema,
@@ -763,18 +768,11 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
struct hs_expr *rules = NULL;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list(expr_rt->htable, &ex_data_array);
//printf("rcu_hash_commit rule_cnt:%zu\n", rule_cnt);
if (rule_cnt > 0) {
rules = ALLOC(struct hs_expr, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct hs_expr *)ex_data_array[i];
// if (rules[i].expr_id == 13)
// {
// for (size_t j = 0; j < rules[i].n_patterns; j++)
// {
// printf("rules[%zu].patterns[%zu]:%s\n", i, j, rules[i].patterns[j].pat);
// }
// }
}
}
@@ -785,8 +783,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
int ret = 0;
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
new_adapter_hs = adapter_hs_initialize(expr_rt->pattern_type, expr_rt->n_worker_thread,
rules, rule_cnt, expr_rt->logger);
new_adapter_hs = adapter_hs_initialize(expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger);
if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild adapter_hs engine failed when update %zu expr rules",
@@ -867,24 +864,17 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
return group_hit_cnt;
}
int expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
{
if (NULL == expr_rt || thread_id < 0) {
return -1;
return NULL;
}
struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(expr_rt->hs, thread_id);
if (NULL == hs_stream) {
return -1;
}
expr_rt->hs_stream = hs_stream;
return 0;
return adapter_hs_stream_open(expr_rt->hs, thread_id);
}
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len,
int vtable_id, struct maat_state *state)
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle,
const char *data, size_t data_len, int vtable_id, struct maat_state *state)
{
if (0 == expr_rt->rule_num) {
//empty expr table
@@ -893,9 +883,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz
size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int ret = adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item);
int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
if (ret < 0) {
return -1;
}
@@ -928,12 +916,9 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz
return group_hit_cnt;
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt)
void expr_runtime_stream_close(struct adapter_hs_stream *s_handle)
{
if (expr_rt != NULL) {
adapter_hs_stream_close(expr_rt->hs_stream);
expr_rt->hs_stream = NULL;
}
adapter_hs_stream_close(s_handle);
}
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id)

97
src/maat_plugin.c
View File

@@ -94,60 +94,57 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
}
schema->table_id = item->valueint;
/* custom is optional */
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table %s has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
if (item != NULL && item->type == cJSON_Object) {
custom_item = cJSON_GetObjectItem(item, "key");
if (NULL == custom_item || custom_item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s has no key column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
schema->key_column = custom_item->valueint;
custom_item = cJSON_GetObjectItem(item, "key");
if (NULL == custom_item || custom_item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s has no key column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
schema->key_column = custom_item->valueint;
custom_item = cJSON_GetObjectItem(item, "key_type");
if (NULL == custom_item || custom_item->type != cJSON_String) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s has no key_type column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "key_type");
if (NULL == custom_item || custom_item->type != cJSON_String) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s has no key_type column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
if (strcmp(custom_item->valuestring, "pointer") == 0) {
schema->key_type = PLUGIN_KEY_TYPE_POINTER;
} else if (strcmp(custom_item->valuestring, "integer") == 0) {
schema->key_type = PLUGIN_KEY_TYPE_INTEGER;
} else {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s key_type:%s illegal",
__FUNCTION__, __LINE__, table_name,
custom_item->valuestring);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "tag");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->rule_tag_column = custom_item->valueint;
}
if (strcmp(custom_item->valuestring, "pointer") == 0) {
schema->key_type = PLUGIN_KEY_TYPE_POINTER;
} else if (strcmp(custom_item->valuestring, "integer") == 0) {
schema->key_type = PLUGIN_KEY_TYPE_INTEGER;
} else {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s key_type:%s illegal",
__FUNCTION__, __LINE__, table_name,
custom_item->valuestring);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "foreign");
if (custom_item != NULL) {
if (custom_item->type == cJSON_String) {
schema->n_foreign = read_integer_array(custom_item->valuestring,
schema->foreign_columns,
MAX_FOREIGN_CLMN_NUM);
} else if (custom_item->type == cJSON_Array) {
schema->n_foreign = cJSON_GetArraySize(custom_item);
for (int i = 0; i < schema->n_foreign; i++) {
cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i);
assert(foreign_item->type == cJSON_Number);
schema->foreign_columns[i] = foreign_item->valueint;
custom_item = cJSON_GetObjectItem(item, "tag");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->rule_tag_column = custom_item->valueint;
}
custom_item = cJSON_GetObjectItem(item, "foreign");
if (custom_item != NULL) {
if (custom_item->type == cJSON_String) {
schema->n_foreign = read_integer_array(custom_item->valuestring,
schema->foreign_columns,
MAX_FOREIGN_CLMN_NUM);
} else if (custom_item->type == cJSON_Array) {
schema->n_foreign = cJSON_GetArraySize(custom_item);
for (int i = 0; i < schema->n_foreign; i++)
{
cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i);
assert(foreign_item->type == cJSON_Number);
schema->foreign_columns[i] = foreign_item->valueint;
}
}
}
}

26
src/maat_virtual.c
View File

@@ -20,8 +20,7 @@
#define MODULE_VIRTUAL module_name_str("maat.virtual")
struct virtual_schema {
char physical_tables[MAX_PHYSICAL_TABLE_NUM][NAME_MAX];
size_t n_physical_table;
char physical_table[NAME_MAX];
int table_id;
struct table_manager *ref_tbl_mgr;
};
@@ -42,23 +41,15 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_Array) {
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_VIRTUAL,
"[%s:%d] virtual table %s has no physical_table column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
int cnt = cJSON_GetArraySize(item);
for (int i = 0; i < cnt; i++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, i);
if (tmp_item != NULL && tmp_item->type == cJSON_String) {
memcpy(schema->physical_tables[i], tmp_item->valuestring,
strlen(tmp_item->valuestring));
}
}
schema->n_physical_table = cnt;
memcpy(schema->physical_table, item->valuestring, strlen(item->valuestring));
return schema;
error:
FREE(schema);
@@ -80,7 +71,7 @@ int virtual_table_get_id(void *virtual_schema)
return schema->table_id;
}
size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[])
int virtual_table_get_physical_table_id(void *virtual_schema)
{
if (NULL == virtual_schema) {
return 0;
@@ -88,10 +79,5 @@ size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_ta
struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
for (size_t i = 0; i < schema->n_physical_table; i++) {
int table_id = table_manager_get_table_id(schema->ref_tbl_mgr, schema->physical_tables[i]);
physical_table_ids[i] = table_id;
}
return schema->n_physical_table;
return table_manager_get_table_id(schema->ref_tbl_mgr, schema->physical_table);
}

89
test/adapter_hs_gtest.cpp
View File

@@ -174,11 +174,11 @@ int parse_config_file(const char *filename, struct hs_expr exprs[], size_t *n_ex
printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n",
key_left_offset, key_right_offset);
}
exprs[i].patterns[j].l_offset = key_left_offset;
exprs[i].patterns[j].r_offset = key_right_offset;
exprs[i].patterns[j].start_offset = key_left_offset;
exprs[i].patterns[j].end_offset = key_right_offset;
} else {
exprs[i].patterns[j].l_offset = -1;
exprs[i].patterns[j].r_offset = -1;
exprs[i].patterns[j].start_offset = -1;
exprs[i].patterns[j].end_offset = -1;
}
}
}
@@ -209,27 +209,23 @@ TEST(adapter_hs_init, invalid_input_parameter)
struct hs_expr expr_array[64];
size_t n_expr_array = 0;
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG,
1, NULL, 0, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, NULL, 0, g_logger);
EXPECT_TRUE(hs_instance == NULL);
hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array,
n_expr_array, g_logger);
hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance == NULL);
n_expr_array = 1;
expr_array[0].expr_id = 101;
expr_array[0].n_patterns = 10;
hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array,
n_expr_array, g_logger);
hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance == NULL);
memset(expr_array, 0, sizeof(expr_array));
n_expr_array = 1;
expr_array[0].expr_id = 101;
expr_array[0].n_patterns = 1;
hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array,
n_expr_array, g_logger);
hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance == NULL);
}
@@ -240,10 +236,9 @@ TEST(adapter_hs_scan, literal_sub_has_normal_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_expr_array, 11);
EXPECT_EQ(n_expr_array, 12);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -268,6 +263,14 @@ TEST(adapter_hs_scan, literal_sub_has_normal_offset)
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].item_id, 101);
char scan_data4[64] = "Aaahello aaa";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_destroy(hs_instance);
@@ -282,8 +285,7 @@ TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -309,6 +311,14 @@ TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset)
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].item_id, 102);
char scan_data4[64] = "Aaahello bbb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_destroy(hs_instance);
@@ -323,8 +333,7 @@ TEST(adapter_hs_scan, literal_sub_has_right_unlimit_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -379,8 +388,7 @@ TEST(adapter_hs_scan, literal_sub_with_no_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -427,8 +435,7 @@ TEST(adapter_hs_scan, literal_exactly)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -469,8 +476,7 @@ TEST(adapter_hs_scan, literal_prefix)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -520,8 +526,7 @@ TEST(adapter_hs_scan, literal_suffix)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -571,8 +576,7 @@ TEST(adapter_hs_scan, literal_sub_with_hexbin)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -604,8 +608,7 @@ TEST(adapter_hs_scan, literal_with_chinese)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1,
expr_array, n_expr_array, g_logger);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
@@ -621,6 +624,30 @@ TEST(adapter_hs_scan, literal_with_chinese)
hs_instance = NULL;
}
TEST(adapter_hs_scan, same_pattern_different_offset)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
char data[64] = "onetoday,anothertoday";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, data, strlen(data), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].item_id, 112);
adapter_hs_destroy(hs_instance);
hs_instance = NULL;
}
int main(int argc, char **argv)
{
int ret = 0;

20
test/literal_expr.conf
View File

@@ -141,6 +141,26 @@
"pattern": "يىلىدىكى"
}
]
},
{
"expr_id": 112,
"pattern_num": 2,
"patterns": [
{
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
"pattern": "today",
"offset": "3~7"
},
{
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
"pattern": "today",
"offset": "16~20"
}
]
}
]
}

50
test/maat_framework_gtest.cpp
View File

@@ -353,7 +353,7 @@ TEST_F(MaatFlagScan, basic) {
TEST_F(MaatFlagScan, withExprRegion) {
const char *flag_table_name = "FLAG_CONFIG";
const char *expr_table_name = "HTTP_URL_LITERAL";
const char *expr_table_name = "HTTP_URL";
struct maat *maat_instance = MaatFlagScan::_shared_maat_instance;
int flag_table_id = maat_get_table_id(maat_instance, flag_table_name);
@@ -540,7 +540,7 @@ struct maat *MaatStringScan::_shared_maat_instance;
struct log_handle *MaatStringScan::logger;
TEST_F(MaatStringScan, Full) {
const char *table_name = "HTTP_URL_REGEX";
const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
@@ -564,7 +564,7 @@ TEST_F(MaatStringScan, Regex) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *cookie = "Cookie: Txa123aheadBCAxd";
const char *table_name = "HTTP_URL_REGEX";
const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
@@ -629,7 +629,7 @@ TEST_F(MaatStringScan, ExprPlus) {
EXPECT_EQ(results[0], 190);
maat_state_free(&state);
}
#if 0
TEST_F(MaatStringScan, ExprPlusWithOffset)
{
long long results[ARRAY_SIZE] = {0};
@@ -681,7 +681,7 @@ TEST_F(MaatStringScan, ExprPlusWithOffset)
maat_state_free(&state);
}
#if 0
TEST_F(MaatStringScan, ExprPlusWithHex) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
@@ -721,7 +721,7 @@ TEST_F(MaatStringScan, ExprAndExprPlus) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *expr_table_name = "HTTP_URL_LITERAL";
const char *expr_table_name = "HTTP_URL";
const char *expr_plus_table_name = "HTTP_SIGNATURE";
const char *region_name = "I love China";
const char *scan_data = "today is Monday and yesterday is Tuesday";
@@ -904,7 +904,8 @@ TEST_F(MaatStringScan, PrefixAndSuffix) {
EXPECT_EQ(results[0], 152);
maat_state_free(&state);
}
#endif
#if 0
TEST_F(MaatStringScan, MaatUnescape) {
const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char *table_name = "KEYWORDS_TABLE";
@@ -923,11 +924,12 @@ TEST_F(MaatStringScan, MaatUnescape) {
EXPECT_EQ(results[0], 132);
maat_state_free(&state);
}
#endif
#if 0
TEST_F(MaatStringScan, RegexWithNotContains) {
const char *should_NOT_hit_scan_data = "new.qq.com/rain/a/TWF2021042600418000";
const char *should_hit_scan_data = "fakesina.com/rain/a/TWF2021042600418000";
const char *table_name = "HTTP_URL_REGEX";
const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
@@ -1031,7 +1033,7 @@ TEST_F(MaatStringScan, StreamInput) {
struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char *table_name = "HTTP_URL_REGEX";
const char *table_name = "HTTP_URL";
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
@@ -1053,7 +1055,7 @@ TEST_F(MaatStringScan, StreamInput) {
}
TEST_F(MaatStringScan, dynamic_config) {
const char *table_name = "HTTP_URL_LITERAL";
const char *table_name = "HTTP_URL";
char data[128] = "hello world, welcome to maat version4, it's funny.";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
@@ -1414,7 +1416,7 @@ TEST_F(NOTLogic, OneRegion) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *table_name = "HTTP_URL_LITERAL";
const char *table_name = "HTTP_URL";
struct maat *maat_instance = NOTLogic::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
@@ -1442,7 +1444,7 @@ TEST_F(NOTLogic, ScanNotAtLast) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *hit_table_name = "HTTP_URL_LITERAL";
const char *hit_table_name = "HTTP_URL";
const char *not_hit_table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -1469,7 +1471,7 @@ TEST_F(NOTLogic, ScanIrrelavantAtLast) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *hit_table_name = "HTTP_URL_LITERAL";
const char *hit_table_name = "HTTP_URL";
const char *not_hit_table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -1498,7 +1500,7 @@ TEST_F(NOTLogic, ScanHitAtLastEmptyExpr) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *not_hit_table_name = "HTTP_URL_LITERAL";
const char *not_hit_table_name = "HTTP_URL";
const char *hit_table_name = "IP_PLUS_CONFIG";
const char *empty_table_name = "EMPTY_KEYWORD";
struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -1537,7 +1539,7 @@ TEST_F(NOTLogic, ScanHitAtLastEmptyInteger) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *not_hit_table_name = "HTTP_URL_LITERAL";
const char *not_hit_table_name = "HTTP_URL";
const char *hit_table_name = "IP_PLUS_CONFIG";
const char *empty_table_name = "EMPTY_INTERGER";
struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -1575,7 +1577,7 @@ TEST_F(NOTLogic, ScanNotIP) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *hit_table_name = "HTTP_URL_LITERAL";
const char *hit_table_name = "HTTP_URL";
const char *not_hit_table_name = "IP_CONFIG";
struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -2367,7 +2369,7 @@ TEST_F(CompileTable, Conjunction1) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL_LITERAL";
const char *table_name = "HTTP_URL";
const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"};
struct maat *maat_instance = CompileTable::_shared_maat_instance;
@@ -2394,7 +2396,7 @@ TEST_F(CompileTable, Conjunction2) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL_LITERAL";
const char *table_name = "HTTP_URL";
const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"};
struct maat *maat_instance = CompileTable::_shared_maat_instance;
@@ -2525,7 +2527,7 @@ TEST_F(Policy, CompileRuleTags) {
struct maat_state *state = NULL;
const char *should_hit = "string bbb should hit";
const char *should_not_hit = "string aaa should not hit";
const char *table_name = "HTTP_URL_LITERAL";
const char *table_name = "HTTP_URL";
struct maat *maat_instance = Policy::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
@@ -2549,7 +2551,7 @@ TEST_F(Policy, CompileEXData) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL_LITERAL";
const char *table_name = "HTTP_URL";
const char *compile_table_name = "COMPILE_ALIAS";
const char *expect_name = "I have a name";
struct maat *maat_instance = Policy::_shared_maat_instance;
@@ -2679,8 +2681,8 @@ TEST_F(TableInfo, Conjunction) {
size_t n_hit_result = 0;
struct maat_state *state = NULL;
const char *scan_data = "soq is using table conjunction function.http://www.3300av.com/novel/27122.txt";
const char *table_name = "HTTP_URL_LITERAL";
const char *conj_table_name = "HTTP_HOST_LITERAL";
const char *table_name = "HTTP_URL";
const char *conj_table_name = "HTTP_HOST";
struct maat *maat_instance = TableInfo::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
@@ -2869,7 +2871,7 @@ TEST_F(MaatCmdTest, SetIP) {
TEST_F(MaatCmdTest, SetExpr) {
const char *scan_data = "Hiredis is a minimalistic C client library for the Redis database.\r\n";
const char *table_name = "HTTP_URL_LITERAL";
const char *table_name = "HTTP_URL";
const char *keywords1 = "Hiredis";
const char *keywords2 = "C Client";

111
test/maat_json.json
View File

@@ -243,7 +243,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "abckkk&123",
@@ -295,7 +295,7 @@
{
"regions": [
{
"table_name": "HTTP_URL_REGEX",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "action=search\\&query=(.*)",
@@ -321,7 +321,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "should_not_hit_any_rule",
@@ -387,7 +387,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "C#中国",
@@ -491,7 +491,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_HOST_LITERAL",
"table_name": "HTTP_HOST",
"table_type": "expr",
"table_content": {
"keywords": "www.3300av.com",
@@ -517,7 +517,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "novel&27122.txt",
@@ -596,7 +596,7 @@
{
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "should&hit&aaa",
@@ -623,7 +623,7 @@
{
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "should&hit&bbb",
@@ -677,7 +677,7 @@
"g2c_table_name": "GROUP2COMPILE_ALIAS",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "i.ytimg.com",
@@ -702,7 +702,7 @@
{
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": ",IgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE",
@@ -728,7 +728,7 @@
"not_flag":0,
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-contained-string-of-rule-143",
@@ -743,7 +743,7 @@
"not_flag":1,
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-not-contained-string-of-rule-143",
@@ -769,7 +769,7 @@
"not_flag":0,
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-contained-string-of-rule-144",
@@ -810,7 +810,7 @@
"not_flag":0,
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-contained-string-of-rule-145",
@@ -839,7 +839,7 @@
{
"regions": [
{
"table_name": "HTTP_URL_REGEX",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "Cookie:\\s&head",
@@ -864,7 +864,7 @@
{
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "googlevideo.com/videoplayback&mn=sn-35153iuxa-5a56%2Csn-n8v7znz7",
@@ -877,35 +877,34 @@
}
]
},
{
"compile_id": 148,
"is_valid": "yes",
"do_log": 0,
"effective_rage": 0,
"action": 0,
"service": 0,
"do_blacklist": 0,
"user_region": "StringScan.ExprPlusWithOffset",
"groups": [
{
"regions": [
{
"table_name": "APP_PAYLOAD",
"table_content": {
"format": "hexbin",
"match_method": "sub",
"district": "Payload",
"keywords": "1-1:03&9-10:2d&14-16:2d34",
"expr_type": "offset"
},
"table_type": "expr_plus"
}
],
"group_name": "Untitled"
}
]
},
{
"compile_id": 148,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"user_region": "StringScan.ExprPlusWithOffset",
"effective_rage": 0,
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "APP_PAYLOAD",
"table_type": "expr_plus",
"table_content": {
"format": "hexbin",
"match_method": "sub",
"district": "Payload",
"keywords": "1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d",
"expr_type": "offset"
}
}
]
}
]
},
{
"compile_id": 149,
"service": 1,
@@ -1304,7 +1303,7 @@
"not_flag":0,
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "https://blog.csdn.net/littlefang/article/details/8213058",
@@ -1458,7 +1457,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "cavemancircus.com/",
@@ -1500,7 +1499,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "2019/12/27/pretty-girls-6",
@@ -1527,7 +1526,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "2019/12/27",
@@ -1554,7 +1553,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "2019/12/27",
@@ -2051,7 +2050,7 @@
"not_flag": 1,
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-not-contained-string-of-rule-186",
@@ -2103,7 +2102,7 @@
"not_flag": 1,
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-not-contained-string-of-rule-187",
@@ -2155,7 +2154,7 @@
"not_flag": 1,
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-not-contained-string-of-rule-188",
@@ -2323,7 +2322,7 @@
{
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "hello",
@@ -2386,7 +2385,7 @@
{
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "Monday",
@@ -2436,7 +2435,7 @@
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_URL_LITERAL",
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "hqdefault.jpg",

155
test/table_info.conf
View File

@@ -29,16 +29,10 @@
"table_id":2,
"table_name":"COMPILE_ALIAS",
"table_type":"compile",
"user_region_encoded":"escape",
"valid_column":4,
"custom": {
"compile_id":1,
"service_id":2,
"action":3,
"do_blacklist":4,
"do_log":5,
"tags":6,
"user_region":7,
"clause_num":9,
"evaluation_order":10
}
@@ -69,11 +63,10 @@
},
{
"table_id":5,
"table_name":["HTTP_URL_LITERAL", "HTTP_HOST_LITERAL"],
"table_name":["HTTP_URL", "HTTP_HOST"],
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
@@ -84,21 +77,6 @@
},
{
"table_id":6,
"table_name":"HTTP_URL_REGEX",
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"regex",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":7,
"table_name":"KEYWORDS_TABLE",
"table_type":"expr",
"valid_column":7,
@@ -113,7 +91,7 @@
}
},
{
"table_id":8,
"table_id":7,
"table_name":"IP_CONFIG",
"table_type":"ip_plus",
"valid_column":18,
@@ -127,7 +105,7 @@
}
},
{
"table_id":9,
"table_id":8,
"table_name":"CONTENT_SIZE",
"table_type":"intval",
"valid_column":5,
@@ -139,23 +117,17 @@
}
},
{
"table_id":10,
"table_id":9,
"table_name":"QD_ENTRY_INFO",
"table_type":"plugin",
"valid_column":4,
"custom": {
"key_type":"pointer",
"key":1,
"tag":3
}
"valid_column":4
},
{
"table_id":11,
"table_id":10,
"table_name":"HTTP_SIGNATURE",
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"district":3,
@@ -166,12 +138,11 @@
}
},
{
"table_id":12,
"table_id":11,
"table_name":"IMAGE_FP",
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
@@ -181,7 +152,7 @@
}
},
{
"table_id":13,
"table_id":12,
"table_name":"TEST_EFFECTIVE_RANGE_TABLE",
"table_type":"plugin",
"valid_column":4,
@@ -192,7 +163,7 @@
}
},
{
"table_id":14,
"table_id":13,
"table_name":"TEST_FOREIGN_KEY",
"table_type":"plugin",
"valid_column":4,
@@ -204,7 +175,7 @@
}
},
{
"table_id":15,
"table_id":14,
"table_name":"TEST_PLUGIN_EXDATA_TABLE",
"table_type":"plugin",
"valid_column":4,
@@ -215,7 +186,7 @@
}
},
{
"table_id":16,
"table_id":15,
"table_name":"IR_INTERCEPT_IP",
"table_type":"plugin",
"valid_column":14,
@@ -226,12 +197,11 @@
}
},
{
"table_id":17,
"table_id":16,
"table_name":"APP_PAYLOAD",
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"district":3,
@@ -242,12 +212,11 @@
}
},
{
"table_id":18,
"table_id":17,
"table_name":"TROJAN_PAYLOAD",
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
@@ -258,12 +227,11 @@
}
},
{
"table_id":19,
"table_id":18,
"table_name":"MAIL_ADDR",
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
@@ -273,7 +241,7 @@
}
},
{
"table_id":20,
"table_id":19,
"table_name":"IP_PLUS_CONFIG",
"table_type":"ip_plus",
"valid_column":18,
@@ -287,43 +255,43 @@
}
},
{
"table_id":21,
"table_id":20,
"table_name":"HTTP_RESPONSE_KEYWORDS",
"table_type":"virtual",
"physical_table": ["KEYWORDS_TABLE"]
"physical_table": "KEYWORDS_TABLE"
},
{
"table_id":21,
"table_name":"HTTP_REQUEST_HEADER",
"table_type":"virtual",
"physical_table": "HTTP_SIGNATURE"
},
{
"table_id":22,
"table_name":"HTTP_REQUEST_HEADER",
"table_name":"HTTP_RESPONSE_HEADER",
"table_type":"virtual",
"physical_table": ["HTTP_SIGNATURE"]
"physical_table": "HTTP_SIGNATURE"
},
{
"table_id":23,
"table_name":"HTTP_RESPONSE_HEADER",
"table_name":"VIRTUAL_IP_PLUS_TABLE",
"table_type":"virtual",
"physical_table": ["HTTP_SIGNATURE"]
"physical_table": "IP_PLUS_CONFIG"
},
{
"table_id":24,
"table_name":"VIRTUAL_IP_PLUS_TABLE",
"table_name":"VIRTUAL_IP_PLUS_SOURCE",
"table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"]
"physical_table": "IP_PLUS_CONFIG"
},
{
"table_id":25,
"table_name":"VIRTUAL_IP_PLUS_SOURCE",
"table_name":"VIRTUAL_IP_PLUS_DESTINATION",
"table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"]
"physical_table": "IP_PLUS_CONFIG"
},
{
"table_id":26,
"table_name":"VIRTUAL_IP_PLUS_DESTINATION",
"table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"]
},
{
"table_id":27,
"table_name":"TEST_IP_PLUGIN_WITH_EXDATA",
"table_type":"ip_plugin",
"valid_column":6,
@@ -336,12 +304,11 @@
}
},
{
"table_id":28,
"table_id":27,
"table_name":"AS_NUMBER",
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
@@ -351,24 +318,23 @@
}
},
{
"table_id":29,
"table_id":28,
"table_name":"SOURCE_IP_ASN",
"table_type":"virtual",
"physical_table":["AS_NUMBER"]
"physical_table":"AS_NUMBER"
},
{
"table_id":29,
"table_name":"DESTINATION_IP_ASN",
"table_type":"virtual",
"physical_table":"AS_NUMBER"
},
{
"table_id":30,
"table_name":"DESTINATION_IP_ASN",
"table_type":"virtual",
"physical_table":["AS_NUMBER"]
},
{
"table_id":31,
"table_name":"GeoLocation",
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
@@ -378,13 +344,13 @@
}
},
{
"table_id":32,
"table_id":31,
"table_name":"SOURCE_IP_GEO",
"table_type":"virtual",
"physical_table":["GeoLocation"]
"physical_table":"GeoLocation"
},
{
"table_id":33,
"table_id":32,
"table_name":"INTERGER_PLUS",
"table_type":"intval_plus",
"valid_column":6,
@@ -397,7 +363,7 @@
}
},
{
"table_id":34,
"table_id":33,
"table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA",
"table_type":"fqdn_plugin",
"valid_column":5,
@@ -408,7 +374,7 @@
}
},
{
"table_id":35,
"table_id":34,
"table_name":"APP_ID",
"table_type":"intval",
"valid_column":5,
@@ -420,12 +386,11 @@
}
},
{
"table_id":36,
"table_id":35,
"table_name":"EMPTY_KEYWORD",
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
@@ -435,7 +400,7 @@
}
},
{
"table_id":37,
"table_id":36,
"table_name":"EMPTY_INTERGER",
"table_type":"intval",
"valid_column":5,
@@ -447,7 +412,7 @@
}
},
{
"table_id":38,
"table_id":37,
"table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA",
"table_type":"bool_plugin",
"valid_column":4,
@@ -457,7 +422,7 @@
}
},
{
"table_id":39,
"table_id":38,
"table_name":"FLAG_CONFIG",
"table_type":"flag",
"valid_column":5,
@@ -469,7 +434,7 @@
}
},
{
"table_id":40,
"table_id":39,
"table_name":"FLAG_PLUS_CONFIG",
"table_type":"flag_plus",
"valid_column":6,
@@ -482,7 +447,7 @@
}
},
{
"table_id":41,
"table_id":40,
"table_name":"PORT_CONFIG",
"table_type":"port",
"valid_column":7,
@@ -496,19 +461,19 @@
}
},
{
"table_id":42,
"table_id":41,
"table_name":"VIRTUAL_PORT_SOURCE",
"table_type":"virtual",
"physical_table": ["PORT_CONFIG"]
"physical_table": "PORT_CONFIG"
},
{
"table_id":42,
"table_name":"VIRTUAL_PORT_DESTINATION",
"table_type":"virtual",
"physical_table": "PORT_CONFIG"
},
{
"table_id":43,
"table_name":"VIRTUAL_PORT_DESTINATION",
"table_type":"virtual",
"physical_table": ["PORT_CONFIG"]
},
{
"table_id":44,
"table_name":"COMPOSITION_CONFIG",
"table_type":"composition",
"custom": {
@@ -519,7 +484,7 @@
}
},
{
"table_id":45,
"table_id":44,
"table_name":"TEST_PLUGIN_KEY_TYPE_TABLE",
"table_type":"plugin",
"valid_column":4,