gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support same pattern different offset(x-x:pat1 & y-y:pat1)

Browse Source
This commit is contained in:
liuwentan
2023-03-22 11:10:00 +08:00
parent 37447eef7f
commit 23ef2c3797
15 changed files with 970 additions and 906 deletions
Download Patch File Download Diff File Expand all files Collapse all files

495
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -22,6 +22,8 @@
#include "maat_utils.h" #include "maat_utils.h"
#include "../bool_matcher/bool_matcher.h" #include "../bool_matcher/bool_matcher.h"
#define MAX_OFFSET_NUM 1024
pid_t hs_gettid() pid_t hs_gettid()
{ {
return syscall(SYS_gettid); return syscall(SYS_gettid);
@@ -50,8 +52,10 @@ struct adapter_hs_runtime {
hs_database_t *literal_db; hs_database_t *literal_db;
hs_database_t *regex_db; hs_database_t *regex_db;
hs_scratch_t **scratchs; hs_scratch_t **literal_scratchs;
size_t scratch_size; hs_scratch_t **regex_scratchs;
size_t literal_scratch_size;
size_t regex_scratch_size;
struct bool_matcher *bm; struct bool_matcher *bm;
}; };
@@ -63,21 +67,25 @@ struct adapter_hs {
size_t n_patterns; size_t n_patterns;
struct adapter_hs_runtime *hs_rt; struct adapter_hs_runtime *hs_rt;
struct hs_tag *tag_map; struct hs_tag *tag_map;
struct pattern_attribute *pat_attr_by_str;
struct pattern_attribute *pat_attr_by_id;
struct log_handle *logger; struct log_handle *logger;
}; };
struct matched_offset {
unsigned long long start_offset;
unsigned long long end_offset;
};
struct matched_pattern { struct matched_pattern {
unsigned long long pattern_id; unsigned long long pattern_id;
unsigned long matched_l_offset; struct matched_offset *offsets;
unsigned long matched_r_offset; size_t offset_cnt;
size_t offset_size;
UT_hash_handle hh; UT_hash_handle hh;
}; };
struct matched_pattern_container { struct matched_pattern_container {
UT_array *pat_ids;
unsigned long long pattern_id;
unsigned long long l_matched;
unsigned long long r_matched;
struct matched_pattern *pat_hash; struct matched_pattern *pat_hash;
}; };
@@ -92,10 +100,11 @@ struct adapter_hs_stream {
}; };
struct pattern_attribute { struct pattern_attribute {
unsigned long long bool_expr_id;
unsigned long long pattern_id; unsigned long long pattern_id;
enum hs_match_mode match_mode; enum hs_match_mode match_mode;
int l_offset; int start_offset;
int r_offset; int end_offset;
}; };
struct hs_tag { struct hs_tag {
@@ -108,40 +117,54 @@ struct hs_tag {
UT_hash_handle hh; UT_hash_handle hh;
}; };
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread, int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratchs, size_t n_worker_thread,
enum hs_pattern_type pattern_type, struct log_handle *logger) struct log_handle *logger)
{ {
hs_database_t *database = NULL; size_t scratch_size = 0;
hs_rt->scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
if (pattern_type == HS_PATTERN_TYPE_STR) { if (hs_alloc_scratch(db, &scratchs[0]) != HS_SUCCESS) {
database = hs_rt->literal_db;
} else {
database = hs_rt->regex_db;
}
if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to allocate scratch space. Exiting.", "[%s:%d] Unable to allocate scratch space. Exiting.",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
hs_free_database(database);
return -1; return -1;
} }
for (size_t i = 1; i < n_worker_thread; i++) { for (size_t i = 1; i < n_worker_thread; i++) {
hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]); hs_error_t err = hs_clone_scratch(scratchs[0], &scratchs[i]);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to clone scratch prototype", __FUNCTION__, __LINE__); "[%s:%d] Unable to clone scratch", __FUNCTION__, __LINE__);
hs_free_database(database);
return -1; return -1;
} }
err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size); err = hs_scratch_size(scratchs[i], &scratch_size);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] Unable to query scratch size", __FUNCTION__, __LINE__); "[%s:%d] Unable to query scratch size", __FUNCTION__, __LINE__);
hs_free_database(database); return -1;
}
}
return 0;
}
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread,
enum hs_pattern_type pattern_type, struct log_handle *logger)
{
int ret = 0;
if (pattern_type == HS_PATTERN_TYPE_STR) {
hs_rt->literal_scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->literal_scratchs, n_worker_thread, logger);
if (ret < 0) {
FREE(hs_rt->literal_scratchs);
return -1;
}
} else {
hs_rt->regex_scratchs = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->regex_db, hs_rt->regex_scratchs, n_worker_thread, logger);
if (ret < 0) {
FREE(hs_rt->regex_scratchs);
return -1; return -1;
} }
} }
@@ -155,20 +178,20 @@ static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_work
* @retval 0(success) -1(failed) * @retval 0(success) -1(failed)
*/ */
static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
struct adpt_hs_compile_data *compile_data, struct adpt_hs_compile_data *literal_cd,
enum hs_pattern_type pattern_type, struct adpt_hs_compile_data *regex_cd,
struct log_handle *logger) struct log_handle *logger)
{ {
hs_error_t err; hs_error_t err;
hs_compile_error_t *compile_err = NULL; hs_compile_error_t *compile_err = NULL;
if (NULL == hs_rt || NULL == compile_data) { if (NULL == hs_rt || (NULL == literal_cd && NULL == regex_cd)) {
return -1; return -1;
} }
if (pattern_type == HS_PATTERN_TYPE_STR) { if (literal_cd != NULL) {
err = hs_compile_lit_multi((const char *const *)compile_data->patterns, compile_data->flags, err = hs_compile_lit_multi((const char *const *)literal_cd->patterns, literal_cd->flags,
compile_data->ids, compile_data->pattern_lens, compile_data->n_patterns, literal_cd->ids, literal_cd->pattern_lens, literal_cd->n_patterns,
HS_MODE_STREAM, NULL, &hs_rt->literal_db, &compile_err); HS_MODE_STREAM, NULL, &hs_rt->literal_db, &compile_err);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
if (compile_err) { if (compile_err) {
@@ -179,11 +202,11 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
hs_free_compile_error(compile_err); hs_free_compile_error(compile_err);
return -1; return -1;
} }
} else { }
err = hs_compile_multi((const char *const *)compile_data->patterns,
compile_data->flags, compile_data->ids, if (regex_cd != NULL) {
compile_data->n_patterns, err = hs_compile_multi((const char *const *)regex_cd->patterns, regex_cd->flags, regex_cd->ids,
HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL, regex_cd->n_patterns, HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL,
&hs_rt->regex_db, &compile_err); &hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
if (compile_err) { if (compile_err) {
@@ -203,20 +226,21 @@ struct adpt_hs_compile_data *adpt_hs_compile_data_new(size_t n_patterns)
struct adpt_hs_compile_data *hs_cd = ALLOC(struct adpt_hs_compile_data, 1); struct adpt_hs_compile_data *hs_cd = ALLOC(struct adpt_hs_compile_data, 1);
hs_cd->patterns = ALLOC(char *, n_patterns); hs_cd->patterns = ALLOC(char *, n_patterns);
hs_cd->pattern_lens = ALLOC(size_t, n_patterns); hs_cd->pattern_lens = ALLOC(size_t, n_patterns);
hs_cd->n_patterns = n_patterns;
hs_cd->ids = ALLOC(unsigned int, n_patterns); hs_cd->ids = ALLOC(unsigned int, n_patterns);
hs_cd->flags = ALLOC(unsigned int, n_patterns); hs_cd->flags = ALLOC(unsigned int, n_patterns);
return hs_cd; return hs_cd;
} }
void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patterns) void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd)
{ {
if (NULL == hs_cd) { if (NULL == hs_cd) {
return; return;
} }
if (hs_cd->patterns != NULL) { if (hs_cd->patterns != NULL) {
for (size_t i = 0; i < n_patterns; i++) { for (size_t i = 0; i < hs_cd->n_patterns; i++) {
FREE(hs_cd->patterns[i]); FREE(hs_cd->patterns[i]);
} }
@@ -259,21 +283,92 @@ void hs_tag_free(struct hs_tag *tag)
FREE(tag); FREE(tag);
} }
struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type, void populate_compile_data(struct adpt_hs_compile_data *compile_data, int index, int pattern_id,
size_t n_worker_thread, char *pat, size_t pat_len, int case_sensitive)
{
compile_data->ids[index] = pattern_id;
/* set flags */
compile_data->flags[index] |= HS_FLAG_SOM_LEFTMOST;
if (case_sensitive == HS_CASE_INSESITIVE) {
compile_data->flags[index] |= HS_FLAG_CASELESS;
}
compile_data->pattern_lens[index] = pat_len;
compile_data->patterns[index] = ALLOC(char, pat_len + 1);
memcpy(compile_data->patterns[index], pat, pat_len);
}
struct bool_expr *bool_exprs_new(struct hs_expr *exprs, size_t n_expr, struct hs_tag **tag_hash,
struct adpt_hs_compile_data *literal_cd, struct adpt_hs_compile_data *regex_cd,
size_t *n_pattern)
{
uint32_t pattern_index = 0;
uint32_t literal_index = 0;
uint32_t regex_index = 0;
struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr);
if (NULL == bool_exprs) {
return NULL;
}
/* populate adpt_hs_compile_data and bool_expr */
for (size_t i = 0; i < n_expr; i++) {
struct hs_tag *hs_tag = hs_tag_new(exprs[i].expr_id, exprs[i].n_patterns);
hs_tag->user_tag = exprs[i].user_tag;
for (size_t j = 0; j < exprs[i].n_patterns; j++) {
hs_tag->pat_attr[j].pattern_id = pattern_index;
hs_tag->pat_attr[j].match_mode = exprs[i].patterns[j].match_mode;
if (exprs[i].patterns[j].match_mode == HS_MATCH_MODE_SUB) {
hs_tag->pat_attr[j].start_offset = exprs[i].patterns[j].start_offset;
hs_tag->pat_attr[j].end_offset = exprs[i].patterns[j].end_offset;
}
/* literal pattern */
if (exprs[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) {
populate_compile_data(literal_cd, literal_index, pattern_index,
exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len,
exprs[i].patterns[j].case_sensitive);
literal_index++;
} else {
/* regex pattern */
populate_compile_data(regex_cd, regex_index, pattern_index,
exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len,
exprs[i].patterns[j].case_sensitive);
regex_index++;
}
bool_exprs[i].items[j].item_id = pattern_index++;
bool_exprs[i].items[j].not_flag = 0;
// printf("item_id:%llu, pat:%s pat_len:%zu\n",
// bool_exprs[i].items[j].item_id, exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len);
}
//printf("expr_id:%lld item_num:%zu\n", exprs[i].expr_id, exprs[i].n_patterns);
bool_exprs[i].expr_id = exprs[i].expr_id;
bool_exprs[i].item_num = exprs[i].n_patterns;
bool_exprs[i].user_tag = hs_tag;
HASH_ADD_KEYPTR(hh, *tag_hash, hs_tag->key, hs_tag->key_len, hs_tag);
}
*n_pattern = pattern_index;
return bool_exprs;
}
struct adapter_hs *adapter_hs_initialize(size_t n_worker_thread,
struct hs_expr *exprs, size_t n_expr, struct hs_expr *exprs, size_t n_expr,
struct log_handle *logger) struct log_handle *logger)
{ {
if ((pattern_type != HS_PATTERN_TYPE_STR && pattern_type != HS_PATTERN_TYPE_REG) || if (0 == n_worker_thread || NULL == exprs || 0 == n_expr) {
0 == n_worker_thread || NULL == exprs || 0 == n_expr) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] input parameters illegal!", log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] input parameters illegal!",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
return NULL; return NULL;
} }
/* get the sum of pattern */ /* get the sum of pattern */
size_t pattern_num = 0; size_t literal_pattern_num = 0;
size_t regex_pattern_num = 0;
for (size_t i = 0; i < n_expr; i++) { for (size_t i = 0; i < n_expr; i++) {
if (exprs[i].n_patterns > MAX_EXPR_PATTERN_NUM) { if (exprs[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
@@ -283,83 +378,53 @@ struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
} }
for (size_t j = 0; j < exprs[i].n_patterns; j++) { for (size_t j = 0; j < exprs[i].n_patterns; j++) {
/* pat_len should not 0 */
if (0 == exprs[i].patterns[j].pat_len) { if (0 == exprs[i].patterns[j].pat_len) {
log_error(logger, MODULE_ADAPTER_HS, log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] expr pattern length should not 0", __FUNCTION__, __LINE__); "[%s:%d] expr pattern length should not 0", __FUNCTION__, __LINE__);
return NULL; return NULL;
} }
pattern_num++; if (exprs[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) {
literal_pattern_num++;
} else {
regex_pattern_num++;
}
} }
} }
if (0 == pattern_num) { if (0 == literal_pattern_num && 0 == regex_pattern_num) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] expr array has no valid pattern", log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] exprs has no valid pattern",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
return NULL; return NULL;
} }
struct adpt_hs_compile_data *compile_data = NULL; struct adpt_hs_compile_data *literal_cd = NULL;
compile_data = adpt_hs_compile_data_new(pattern_num); struct adpt_hs_compile_data *regex_cd = NULL;
if (literal_pattern_num > 0) {
literal_cd = adpt_hs_compile_data_new(literal_pattern_num);
}
uint32_t pattern_index = 0; if (regex_pattern_num > 0) {
regex_cd = adpt_hs_compile_data_new(regex_pattern_num);
}
size_t pattern_cnt = 0;
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1); struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1);
hs_instance->tag_map = NULL; hs_instance->tag_map = NULL;
hs_instance->logger = logger; hs_instance->logger = logger;
struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr);
/* populate adpt_hs_compile_data and bool_expr */
for (size_t i = 0; i < n_expr; i++) {
struct hs_tag *hs_tag = hs_tag_new(exprs[i].expr_id, exprs[i].n_patterns);
hs_tag->user_tag = exprs[i].user_tag;
for (size_t j = 0; j < exprs[i].n_patterns; j++) {
size_t pat_len = 0;
hs_tag->pat_attr[j].pattern_id = pattern_index;
hs_tag->pat_attr[j].match_mode = exprs[i].patterns[j].match_mode;
if (exprs[i].patterns[j].match_mode == HS_MATCH_MODE_SUB) {
hs_tag->pat_attr[j].l_offset = exprs[i].patterns[j].l_offset;
hs_tag->pat_attr[j].r_offset = exprs[i].patterns[j].r_offset;
}
compile_data->ids[pattern_index] = pattern_index;
if (pattern_type == HS_PATTERN_TYPE_STR) {
compile_data->flags[pattern_index] |= HS_FLAG_SOM_LEFTMOST;
}
if (exprs[i].patterns[j].case_sensitive == HS_CASE_INSESITIVE) {
compile_data->flags[pattern_index] |= HS_FLAG_CASELESS;
}
pat_len = exprs[i].patterns[j].pat_len;
compile_data->pattern_lens[pattern_index] = pat_len;
compile_data->patterns[pattern_index] = ALLOC(char, pat_len + 1);
memcpy(compile_data->patterns[pattern_index], exprs[i].patterns[j].pat,
exprs[i].patterns[j].pat_len);
bool_exprs[i].items[j].item_id = pattern_index;
pattern_index++;
}
bool_exprs[i].expr_id = exprs[i].expr_id;
bool_exprs[i].item_num = exprs[i].n_patterns;
bool_exprs[i].user_tag = hs_tag;
HASH_ADD_KEYPTR(hh, hs_instance->tag_map, hs_tag->key, hs_tag->key_len, hs_tag);
}
compile_data->n_patterns = pattern_index;
int ret = -1;
size_t mem_size = 0;
hs_instance->n_worker_thread = n_worker_thread; hs_instance->n_worker_thread = n_worker_thread;
hs_instance->n_patterns = pattern_index;
hs_instance->n_expr = n_expr; hs_instance->n_expr = n_expr;
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
struct bool_expr *bool_exprs = bool_exprs_new(exprs, n_expr, &hs_instance->tag_map,
literal_cd, regex_cd, &pattern_cnt);
if (NULL == bool_exprs) {
return NULL;
}
hs_instance->n_patterns = pattern_cnt;
//mytest //mytest
// for (size_t i = 0; i < n_expr; i++) { // for (size_t i = 0; i < n_expr; i++) {
// if (bool_exprs[i].expr_id == 37)
// { // {
// printf("<before bool_matcher_new> exprs[%zu] expr_id:%llu, item_num:%zu\n", // printf("<before bool_matcher_new> exprs[%zu] expr_id:%llu, item_num:%zu\n",
// i, bool_exprs[i].expr_id, bool_exprs[i].item_num); // i, bool_exprs[i].expr_id, bool_exprs[i].item_num);
@@ -369,10 +434,14 @@ struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
// printf("%llu ", bool_exprs[i].items[j].item_id); // printf("%llu ", bool_exprs[i].items[j].item_id);
// } // }
// } // }
// }
// printf("\n"); // printf("\n");
// }
/* create bool matcher */ /* create bool matcher */
size_t mem_size = 0;
int hs_ret = 0;
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
hs_instance->hs_rt->bm = bool_matcher_new(bool_exprs, n_expr, &mem_size); hs_instance->hs_rt->bm = bool_matcher_new(bool_exprs, n_expr, &mem_size);
if (hs_instance->hs_rt->bm != NULL) { if (hs_instance->hs_rt->bm != NULL) {
log_info(logger, MODULE_ADAPTER_HS, log_info(logger, MODULE_ADAPTER_HS,
@@ -381,30 +450,47 @@ struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type,
} else { } else {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Adapter_hs module: build bool matcher failed", log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Adapter_hs module: build bool matcher failed",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
adpt_hs_compile_data_free(compile_data, pattern_index);
FREE(bool_exprs); hs_ret = -1;
adapter_hs_destroy(hs_instance);
return NULL;
} }
FREE(bool_exprs); FREE(bool_exprs);
/* build hs database */ /* build hs database */
ret = adpt_hs_build_database(hs_instance->hs_rt, compile_data, pattern_type, logger); int ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, logger);
if (ret < 0) { if (ret < 0) {
hs_ret = -1;
}
if (literal_cd != NULL) {
adpt_hs_compile_data_free(literal_cd);
}
if (regex_cd != NULL) {
adpt_hs_compile_data_free(regex_cd);
}
if (hs_ret < 0) {
goto error; goto error;
} }
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, pattern_type, logger); /* literal and regex scratch can't reuse */
if (literal_pattern_num > 0) {
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_STR, logger);
if (ret < 0) { if (ret < 0) {
goto error; goto error;
} }
}
if (regex_pattern_num > 0) {
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_REG, logger);
if (ret < 0) {
goto error;
}
}
adpt_hs_compile_data_free(compile_data, pattern_index);
return hs_instance; return hs_instance;
error: error:
adpt_hs_compile_data_free(compile_data, pattern_index);
adapter_hs_destroy(hs_instance); adapter_hs_destroy(hs_instance);
return NULL; return NULL;
} }
@@ -423,14 +509,23 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance)
hs_free_database(hs_instance->hs_rt->regex_db); hs_free_database(hs_instance->hs_rt->regex_db);
} }
if (hs_instance->hs_rt->scratchs != NULL) { if (hs_instance->hs_rt->literal_scratchs != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->scratchs[i] != NULL) { if (hs_instance->hs_rt->literal_scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->scratchs[i]); hs_free_scratch(hs_instance->hs_rt->literal_scratchs[i]);
} }
} }
} }
FREE(hs_instance->hs_rt->scratchs); FREE(hs_instance->hs_rt->literal_scratchs);
if (hs_instance->hs_rt->regex_scratchs != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->regex_scratchs[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->regex_scratchs[i]);
}
}
}
FREE(hs_instance->hs_rt->regex_scratchs);
if (hs_instance->hs_rt->bm != NULL) { if (hs_instance->hs_rt->bm != NULL) {
bool_matcher_free(hs_instance->hs_rt->bm); bool_matcher_free(hs_instance->hs_rt->bm);
@@ -450,20 +545,19 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance)
FREE(hs_instance); FREE(hs_instance);
} }
static inline int compare_pattern_id(const void *a, const void *b) int find_same_pattern_offset(struct matched_pattern *matched_pat, unsigned long long from,
unsigned long long to)
{ {
long long ret = *(unsigned long long *)a - *(unsigned long long *)b; for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
if (matched_pat->offsets[i].start_offset == from &&
if (0 == ret) { matched_pat->offsets[i].end_offset == to - 1) {
return 0; return 0;
} else if (ret < 0) {
return -1;
} else {
return 1;
} }
}
return -1;
} }
UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
/** /**
* @param id: pattern id * @param id: pattern id
*/ */
@@ -474,51 +568,76 @@ int matched_event_cb(unsigned int id, unsigned long long from,
struct matched_pattern_container *matched_pat_container = (struct matched_pattern_container *)ctx; struct matched_pattern_container *matched_pat_container = (struct matched_pattern_container *)ctx;
unsigned long long pattern_id = id; unsigned long long pattern_id = id;
if (utarray_find(matched_pat_container->pat_ids, &pattern_id, compare_pattern_id)) { struct matched_pattern *matched_pat = NULL;
return 0; HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), matched_pat);
if (matched_pat != NULL) {
// same pattern_id, offset maybe different
int ret = find_same_pattern_offset(matched_pat, from, to);
if (ret < 0) { /* different offset */
// TODO: use realloc
if (matched_pat->offset_cnt >= matched_pat->offset_size) {
matched_pat->offset_size *= 2;
matched_pat->offsets = (struct matched_offset *)realloc(matched_pat->offsets,
matched_pat->offset_size*sizeof(struct matched_offset));
} }
matched_pat->offsets[matched_pat->offset_cnt].start_offset = from;
utarray_push_back(matched_pat_container->pat_ids, &pattern_id); matched_pat->offsets[matched_pat->offset_cnt].end_offset = to - 1;
utarray_sort(matched_pat_container->pat_ids, compare_pattern_id); matched_pat->offset_cnt++;
}
return 0;
} else {
// different pattern_id
struct matched_pattern *matched_pat = ALLOC(struct matched_pattern, 1); struct matched_pattern *matched_pat = ALLOC(struct matched_pattern, 1);
matched_pat->pattern_id = pattern_id; matched_pat->pattern_id = pattern_id;
matched_pat->matched_l_offset = from; matched_pat->offsets = ALLOC(struct matched_offset, MAX_OFFSET_NUM);
matched_pat->matched_r_offset = to; matched_pat->offset_size = MAX_OFFSET_NUM;
matched_pat->offsets[matched_pat->offset_cnt].start_offset = from;
matched_pat->offsets[matched_pat->offset_cnt].end_offset = to - 1;
matched_pat->offset_cnt++;
HASH_ADD(hh, matched_pat_container->pat_hash, pattern_id, sizeof(unsigned long long), matched_pat); HASH_ADD(hh, matched_pat_container->pat_hash, pattern_id, sizeof(unsigned long long), matched_pat);
}
return 0; return 0;
} }
int is_real_matched_pattern(struct matched_pattern *matched_pat, enum hs_match_mode match_mode, int is_real_matched_pattern(struct matched_pattern *matched_pat, enum hs_match_mode match_mode,
size_t data_len, int attr_l_offset, int attr_r_offset) size_t data_len, int attr_start_offset, int attr_end_offset)
{ {
if (match_mode == HS_MATCH_MODE_EXACTLY) { if (match_mode == HS_MATCH_MODE_EXACTLY) {
if (matched_pat->matched_l_offset == 0 && for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
matched_pat->matched_r_offset == data_len) { if (matched_pat->offsets[i].start_offset == 0 &&
matched_pat->offsets[i].end_offset == data_len - 1) {
return 0; return 0;
} }
}
} else if (match_mode == HS_MATCH_MODE_PREFIX) { } else if (match_mode == HS_MATCH_MODE_PREFIX) {
if (matched_pat->matched_l_offset == 0) { for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
if (matched_pat->offsets[i].start_offset == 0) {
return 0; return 0;
} }
}
} else if (match_mode == HS_MATCH_MODE_SUFFIX) { } else if (match_mode == HS_MATCH_MODE_SUFFIX) {
if (matched_pat->matched_r_offset == data_len) { for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
if (matched_pat->offsets[i].end_offset == data_len - 1) {
return 0; return 0;
} }
}
} else if (match_mode == HS_MATCH_MODE_SUB) { } else if (match_mode == HS_MATCH_MODE_SUB) {
if (attr_l_offset == -1) { if (attr_start_offset == -1) {
attr_l_offset = 0; attr_start_offset = 0;
} }
if (attr_r_offset == -1) { if (attr_end_offset == -1) {
attr_r_offset = (int)data_len; attr_end_offset = (int)data_len - 1;
} }
if (matched_pat->matched_l_offset >= (unsigned long)attr_l_offset && for (size_t i = 0; i < matched_pat->offset_cnt; i++) {
matched_pat->matched_r_offset <= (unsigned long)attr_r_offset) { if (matched_pat->offsets[i].start_offset >= (unsigned long long)attr_start_offset &&
matched_pat->offsets[i].end_offset <= (unsigned long long)attr_end_offset) {
return 0; return 0;
} }
}
} else { } else {
assert(0); assert(0);
} }
@@ -531,15 +650,13 @@ int hs_tag_validate(struct hs_tag *hs_tag, struct matched_pattern_container *mat
{ {
/* check if real matched pattern, because pattern match_mode is different */ /* check if real matched pattern, because pattern match_mode is different */
for (size_t i = 0; i < hs_tag->n_pat_attr; i++) { for (size_t i = 0; i < hs_tag->n_pat_attr; i++) {
struct matched_pattern *tmp_matched_pat = NULL; struct matched_pattern *matched_pat = NULL;
unsigned long long pattern_id = hs_tag->pat_attr[i].pattern_id; unsigned long long pattern_id = hs_tag->pat_attr[i].pattern_id;
HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), tmp_matched_pat); HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), matched_pat);
if (tmp_matched_pat) { if (matched_pat) {
int matched_ret = is_real_matched_pattern(tmp_matched_pat, int matched_ret = is_real_matched_pattern(matched_pat, hs_tag->pat_attr[i].match_mode,
hs_tag->pat_attr[i].match_mode, data_len, hs_tag->pat_attr[i].start_offset,
data_len, hs_tag->pat_attr[i].end_offset);
hs_tag->pat_attr[i].l_offset,
hs_tag->pat_attr[i].r_offset);
if (matched_ret < 0) { if (matched_ret < 0) {
return -1; return -1;
} }
@@ -562,14 +679,13 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
hs_stream->n_expr = hs_instance->n_expr; hs_stream->n_expr = hs_instance->n_expr;
hs_stream->n_patterns = hs_instance->n_patterns; hs_stream->n_patterns = hs_instance->n_patterns;
hs_stream->hs_rt = hs_instance->hs_rt; hs_stream->hs_rt = hs_instance->hs_rt;
utarray_new(hs_stream->matched_pat_container.pat_ids, &ut_pattern_id_icd);
utarray_reserve(hs_stream->matched_pat_container.pat_ids, hs_stream->n_patterns);
int err_count = 0;
if (hs_instance->hs_rt->literal_db != NULL) { if (hs_instance->hs_rt->literal_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->literal_db, 0, &hs_stream->literal_stream); err = hs_open_stream(hs_instance->hs_rt->literal_db, 0, &hs_stream->literal_stream);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err); log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err);
return NULL; err_count++;
} }
} }
@@ -577,11 +693,29 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
err = hs_open_stream(hs_instance->hs_rt->regex_db, 0, &hs_stream->regex_stream); err = hs_open_stream(hs_instance->hs_rt->regex_db, 0, &hs_stream->regex_stream);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err); log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err);
return NULL; err_count++;
} }
} }
if (err_count > 0) {
goto error;
}
return hs_stream; return hs_stream;
error:
//TODO: hs_stream->hs_rt->scratchs[thread_id] may be free twice
if (hs_stream->literal_stream != NULL) {
hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL);
hs_stream->literal_stream = NULL;
}
if (hs_stream->regex_stream != NULL) {
hs_close_stream(hs_stream->regex_stream, NULL, NULL, NULL);
hs_stream->regex_stream = NULL;
}
FREE(hs_stream);
return NULL;
} }
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream) void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
@@ -590,20 +724,14 @@ void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
return; return;
} }
int thread_id = hs_stream->thread_id;
if (hs_stream->hs_rt != NULL) { if (hs_stream->hs_rt != NULL) {
if (hs_stream->literal_stream != NULL) { if (hs_stream->literal_stream != NULL) {
hs_close_stream(hs_stream->literal_stream, hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL);
hs_stream->hs_rt->scratchs[thread_id],
NULL, NULL);
hs_stream->literal_stream = NULL; hs_stream->literal_stream = NULL;
} }
if (hs_stream->regex_stream != NULL) { if (hs_stream->regex_stream != NULL) {
hs_close_stream(hs_stream->regex_stream, hs_close_stream(hs_stream->regex_stream, NULL, NULL, NULL);
hs_stream->hs_rt->scratchs[thread_id],
NULL, NULL);
hs_stream->regex_stream = NULL; hs_stream->regex_stream = NULL;
} }
} }
@@ -615,13 +743,23 @@ void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
FREE(pattern); FREE(pattern);
} }
} }
utarray_free(hs_stream->matched_pat_container.pat_ids);
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */ /* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */
hs_stream->hs_rt = NULL; hs_stream->hs_rt = NULL;
FREE(hs_stream); FREE(hs_stream);
} }
static int cmp_ull_p(const void *p1, const void *p2)
{
if(* (unsigned long long*) p1 > * (unsigned long long*) p2) {
return 1;
} else if(* (unsigned long long*) p1 < * (unsigned long long*) p2) {
return -1;
} else {
return 0;
}
}
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len, int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len,
struct hs_scan_result *results, size_t n_result, size_t *n_hit_result) struct hs_scan_result *results, size_t n_result, size_t *n_hit_result)
{ {
@@ -646,7 +784,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
int thread_id = hs_stream->thread_id; int thread_id = hs_stream->thread_id;
if (hs_stream->literal_stream != NULL) { if (hs_stream->literal_stream != NULL) {
err = hs_scan_stream(hs_stream->literal_stream, data, data_len, err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id], 0, hs_stream->hs_rt->literal_scratchs[thread_id],
matched_event_cb, &hs_stream->matched_pat_container); matched_event_cb, &hs_stream->matched_pat_container);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
err_count++; err_count++;
@@ -655,34 +793,42 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
if (hs_stream->regex_stream != NULL) { if (hs_stream->regex_stream != NULL) {
err = hs_scan_stream(hs_stream->regex_stream, data, data_len, err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id], 0, hs_stream->hs_rt->regex_scratchs[thread_id],
matched_event_cb, &hs_stream->matched_pat_container); matched_event_cb, &hs_stream->matched_pat_container);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
err_count++; err_count++;
} }
} }
if (err_count > 0) { if (err_count == 2) {
return -1; return -1;
} }
size_t matched_pattern_ids_cnt = utarray_len(hs_stream->matched_pat_container.pat_ids); size_t n_item = HASH_COUNT(hs_stream->matched_pat_container.pat_hash);
if (0 == matched_pattern_ids_cnt) { if (0 == n_item) {
*n_hit_result = 0; *n_hit_result = 0;
return 0; return 0;
} }
unsigned long long items[matched_pattern_ids_cnt]; if (n_item > MAX_SCANNER_HIT_ITEM_NUM) {
memset(items, 0, sizeof(unsigned long long) * matched_pattern_ids_cnt); n_item = MAX_SCANNER_HIT_ITEM_NUM;
for (size_t i = 0; i < matched_pattern_ids_cnt; i++) {
items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->matched_pat_container.pat_ids, i);
} }
unsigned long long item_ids[MAX_SCANNER_HIT_ITEM_NUM];
memset(item_ids, 0, sizeof(unsigned long long) * MAX_SCANNER_HIT_ITEM_NUM);
int i = 0;
struct matched_pattern *pat = NULL, *tmp_pat = NULL;
HASH_ITER(hh, hs_stream->matched_pat_container.pat_hash, pat, tmp_pat) {
item_ids[i++] = pat->pattern_id;
}
qsort(item_ids, n_item, sizeof(unsigned long long), cmp_ull_p);
int ret = 0; int ret = 0;
int real_matched_index = 0; int real_matched_index = 0;
struct hs_tag *hs_tag = NULL; struct hs_tag *hs_tag = NULL;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, matched_pattern_ids_cnt, int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, item_ids, n_item,
bool_matcher_results, hs_stream->n_expr); bool_matcher_results, hs_stream->n_expr);
if (bool_matcher_ret < 0) { if (bool_matcher_ret < 0) {
ret = -1; ret = -1;
@@ -715,7 +861,6 @@ next:
HASH_DELETE(hh, hs_stream->matched_pat_container.pat_hash, pattern); HASH_DELETE(hh, hs_stream->matched_pat_container.pat_hash, pattern);
FREE(pattern); FREE(pattern);
} }
utarray_clear(hs_stream->matched_pat_container.pat_ids);
return ret; return ret;
} }

16
scanner/adapter_hs/adapter_hs.h
View File

@@ -54,16 +54,18 @@ struct hs_scan_result {
struct hs_pattern { struct hs_pattern {
enum hs_case_sensitive case_sensitive; enum hs_case_sensitive case_sensitive;
enum hs_match_mode match_mode; enum hs_match_mode match_mode;
enum hs_pattern_type pattern_type;
int is_hexbin; /* 1(yes) 0(no) */ int is_hexbin; /* 1(yes) 0(no) */
/* /*
* just match in scan_data's range of [l_offset, r_offset], -1 means no limits * just match in scan_data's range of [start_offset, end_offset], -1 means no limits
* for example: * for example:
* [-1, r_offset] means the pattern must in scan_data's [0 ~ r_offset] * [-1, end_offset] means the pattern must in scan_data's [0 ~ start_offset]
* [l_offset, -1] means the pattern must in scan_data's [l_offset ~ end] * [start_offset, -1] means the pattern must in scan_data's [start_offset ~ data_end]
*/ */
int l_offset; int start_offset;
int r_offset; int end_offset;
/* start pointer of pattern */ /* start pointer of pattern */
char *pat; char *pat;
@@ -82,15 +84,13 @@ struct hs_expr {
/** /**
* @brief initialize adapter_hs instance * @brief initialize adapter_hs instance
* *
* @param pattern_type: pure literal or regex expression
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan() * @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
* @param expr_array: logic AND expression's array * @param expr_array: logic AND expression's array
* @param n_expr_arrays: the number of logic AND expression's array * @param n_expr_arrays: the number of logic AND expression's array
* *
* @retval the pointer to adapter_hs instance * @retval the pointer to adapter_hs instance
*/ */
struct adapter_hs *adapter_hs_initialize(enum hs_pattern_type pattern_type, struct adapter_hs *adapter_hs_initialize(size_t n_worker_thread,
size_t n_worker_thread,
struct hs_expr *exprs, size_t n_expr, struct hs_expr *exprs, size_t n_expr,
struct log_handle *logger); struct log_handle *logger);

7
src/inc_internal/maat_expr.h
View File

@@ -45,10 +45,11 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name);
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int vtable_id, struct maat_state *state); size_t data_len, int vtable_id, struct maat_state *state);
int expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state); int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt); void expr_runtime_stream_close(struct adapter_hs_stream *s_handle);
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id); void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id);
long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread); long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread);

1
src/inc_internal/maat_rule.h
View File

@@ -35,7 +35,6 @@ extern "C"
#define MAX_TABLE_NUM 1024 #define MAX_TABLE_NUM 1024
#define MAX_COMPILE_TABLE_NUM 16 #define MAX_COMPILE_TABLE_NUM 16
#define MAX_PHYSICAL_TABLE_NUM 16
#define DISTRICT_ANY -1 #define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2 #define DISTRICT_UNKNOWN -2

2
src/inc_internal/maat_virtual.h
View File

@@ -39,7 +39,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void virtual_schema_free(void *virtual_schema); void virtual_schema_free(void *virtual_schema);
int virtual_table_get_id(void *virtual_schema); int virtual_table_get_id(void *virtual_schema);
size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]); int virtual_table_get_physical_table_id(void *virtual_schema);
#ifdef __cplusplus #ifdef __cplusplus
} }

353
src/maat_api.c
View File

@@ -48,11 +48,10 @@ enum district_set_flag {
struct maat_stream { struct maat_stream {
struct maat *ref_maat_instance; struct maat *ref_maat_instance;
struct adapter_hs_stream *s_handle; //each physical table open one stream
int thread_id; int thread_id;
int table_id;
int vtable_id; int vtable_id;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; int physical_table_id;
size_t n_physical_table;
struct log_handle *logger; struct log_handle *logger;
}; };
@@ -987,47 +986,34 @@ size_t hit_group_to_compile(void *compile_runtime, long long *compile_ids, int i
return n_hit_compile; return n_hit_compile;
} }
static int vtable_get_physical_table_ids(struct table_manager *tbl_mgr, int table_id, static int vtable_get_physical_table_id(struct table_manager *tbl_mgr, int table_id)
int *physical_table_ids, size_t n_table_id_array,
int *vtable_id)
{ {
size_t physical_table_cnt = 0;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
if (table_type == TABLE_TYPE_VIRTUAL) { if (table_type != TABLE_TYPE_VIRTUAL) {
//find physical table id return -1;
*vtable_id = table_id;
void *virtual_schema = table_manager_get_schema(tbl_mgr, table_id);
assert(virtual_schema != NULL);
physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids);
return physical_table_cnt;
} else {
*vtable_id = 0;
physical_table_ids[0] = table_id;
physical_table_cnt = 1;
} }
return physical_table_cnt; // find physical table id
void *virtual_schema = table_manager_get_schema(tbl_mgr, table_id);
assert(virtual_schema != NULL);
return virtual_table_get_physical_table_id(virtual_schema);
} }
int flag_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long flag, int flag_scan(struct table_manager *tbl_mgr, int thread_id, long long flag,
int physical_table_ids[], int physical_table_cnt, int vtable_id, int physical_table_id, int vtable_id, struct maat_state *mid)
struct maat_state *mid)
{ {
int sum_hit_group_cnt = 0; enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
for (int i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if ((table_type == TABLE_TYPE_FLAG_PLUS) && if ((table_type == TABLE_TYPE_FLAG_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
return -1; return -1;
} }
if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) { if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) {
continue; return -1;
} }
void *flag_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); void *flag_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == flag_rt) { if (NULL == flag_rt) {
return -1; return -1;
} }
@@ -1041,31 +1027,26 @@ int flag_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long
if (group_hit_cnt > 0) { if (group_hit_cnt > 0) {
flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id); flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id);
} }
sum_hit_group_cnt += group_hit_cnt;
}
return sum_hit_group_cnt; return group_hit_cnt;
} }
int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long integer, int interval_scan(struct table_manager *tbl_mgr, int thread_id, long long integer,
int physical_table_ids[], int physical_table_cnt, int vtable_id, int physical_table_id, int vtable_id, struct maat_state *mid)
struct maat_state *mid)
{ {
int sum_hit_group_cnt = 0;
for (size_t i = 0; i < physical_table_cnt; i++) { enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if ((table_type == TABLE_TYPE_INTERVAL_PLUS) && if ((table_type == TABLE_TYPE_INTERVAL_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
//maat_instance->scan_err_cnt++; // maat_instance->scan_err_cnt++;
return -1; return -1;
} }
if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
continue; return -1;
} }
void *interval_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); void *interval_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == interval_rt) { if (NULL == interval_rt) {
return -1; return -1;
} }
@@ -1079,25 +1060,20 @@ int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id,
if (group_hit_cnt > 0) { if (group_hit_cnt > 0) {
interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id); interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id);
} }
sum_hit_group_cnt += group_hit_cnt;
}
return sum_hit_group_cnt; return group_hit_cnt;
} }
int ipv4_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_addr, int ipv4_scan(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_addr,
int physical_table_ids[], int physical_table_cnt, int vtable_id, int physical_table_id, int vtable_id, struct maat_state *mid)
struct maat_state *mid)
{ {
int sum_hit_group_cnt = 0;
for (size_t i = 0; i < physical_table_cnt; i++) { enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if (table_type != TABLE_TYPE_IP_PLUS) { if (table_type != TABLE_TYPE_IP_PLUS) {
continue; return -1;
} }
void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == ip_rt) { if (NULL == ip_rt) {
return -1; return -1;
} }
@@ -1111,25 +1087,20 @@ int ipv4_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint
if (group_hit_cnt > 0) { if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
} }
sum_hit_group_cnt += group_hit_cnt;
}
return sum_hit_group_cnt; return group_hit_cnt;
} }
int ipv6_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint8_t *ip_addr, int ipv6_scan(struct table_manager *tbl_mgr, int thread_id, uint8_t *ip_addr,
int physical_table_ids[], int physical_table_cnt, int vtable_id, int physical_table_id, int vtable_id, struct maat_state *mid)
struct maat_state *mid)
{ {
int sum_hit_group_cnt = 0;
for (size_t i = 0; i < physical_table_cnt; i++) { enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if (table_type != TABLE_TYPE_IP_PLUS) { if (table_type != TABLE_TYPE_IP_PLUS) {
continue; return -1;
} }
void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == ip_rt) { if (NULL == ip_rt) {
return -1; return -1;
} }
@@ -1143,31 +1114,25 @@ int ipv6_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint
if (group_hit_cnt > 0) { if (group_hit_cnt > 0) {
ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
} }
sum_hit_group_cnt += group_hit_cnt;
}
return sum_hit_group_cnt; return group_hit_cnt;
} }
int string_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, const char *data, int string_scan(struct table_manager *tbl_mgr, int thread_id, const char *data, size_t data_len,
size_t data_len, int physical_table_ids[], int physical_table_cnt, int physical_table_id, int vtable_id, struct maat_state *mid)
int vtable_id, struct maat_state *mid)
{ {
int sum_hit_group_cnt = 0; enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
for (size_t i = 0; i < physical_table_cnt; i++) {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]);
if ((table_type == TABLE_TYPE_EXPR_PLUS) && if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
//maat_instance->scan_err_cnt++; // maat_instance->scan_err_cnt++;
return -1; return -1;
} }
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
continue; return -1;
} }
void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_id);
if (NULL == expr_rt) { if (NULL == expr_rt) {
return -1; return -1;
} }
@@ -1182,48 +1147,44 @@ int string_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, co
if (group_hit_cnt > 0) { if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
} }
sum_hit_group_cnt += group_hit_cnt;
}
return sum_hit_group_cnt; return group_hit_cnt;
} }
int stream_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, const char *data, int expr_stream_scan(struct maat_stream *stream, const char *data, size_t data_len, struct maat_state *state)
size_t data_len, int physical_table_ids[], int physical_table_cnt,
int vtable_id, struct maat_state *mid)
{ {
int sum_hit_group_cnt = 0; if (NULL == stream || NULL == data) {
return 0;
}
for (size_t i = 0; i < physical_table_cnt; i++) { struct table_manager *tbl_mgr = stream->ref_maat_instance->tbl_mgr;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); enum table_type table_type = table_manager_get_table_type(tbl_mgr, stream->physical_table_id);
if ((table_type == TABLE_TYPE_EXPR_PLUS) && if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { (NULL == state || DISTRICT_FLAG_UNSET == state->is_set_district)) {
//maat_instance->scan_err_cnt++; // maat_instance->scan_err_cnt++;
return -1; return -1;
} }
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
continue; return -1;
} }
void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); void *expr_rt = table_manager_get_runtime(tbl_mgr, stream->physical_table_id);
if (NULL == expr_rt) { if (NULL == expr_rt) {
return -1; return -1;
} }
int group_hit_cnt = expr_runtime_stream_scan((struct expr_runtime *)expr_rt, int group_hit_cnt = expr_runtime_stream_scan((struct expr_runtime *)expr_rt, stream->s_handle,
data, data_len, vtable_id, mid); data, data_len, stream->vtable_id, state);
if (group_hit_cnt < 0) { if (group_hit_cnt < 0) {
return -1; return -1;
} }
if (group_hit_cnt > 0) { if (group_hit_cnt > 0) {
expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, stream->thread_id);
}
sum_hit_group_cnt += group_hit_cnt;
} }
return sum_hit_group_cnt; return group_hit_cnt;
} }
size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n_result, size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n_result,
@@ -1279,31 +1240,29 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
} }
int vtable_id = 0; int vtable_id = 0;
int physical_table_cnt = 0; int physical_table_id = -1;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
memset(physical_table_ids, -1, sizeof(physical_table_ids)); if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, if (physical_table_id < 0) {
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
enum table_type table_type = TABLE_TYPE_INVALID; table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) { if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = flag_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, flag, int hit_group_cnt = flag_scan(maat_instance->tbl_mgr, thread_id, flag,
physical_table_ids, physical_table_id, vtable_id, mid);
physical_table_cnt, vtable_id, mid);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
@@ -1356,31 +1315,29 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id,
} }
int vtable_id = 0; int vtable_id = 0;
int physical_table_cnt = 0; int physical_table_id = -1;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
memset(physical_table_ids, -1, sizeof(physical_table_ids)); if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, if (physical_table_id < 0) {
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
enum table_type table_type = TABLE_TYPE_INVALID; table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = interval_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, integer, int hit_group_cnt = interval_scan(maat_instance->tbl_mgr, thread_id, integer,
physical_table_ids, physical_table_id, vtable_id, mid);
physical_table_cnt, vtable_id, mid);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
@@ -1433,31 +1390,29 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
} }
int vtable_id = 0; int vtable_id = 0;
int physical_table_cnt = 0; int physical_table_id = -1;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
memset(physical_table_ids, -1, sizeof(physical_table_ids)); if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, if (physical_table_id < 0) {
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
enum table_type table_type = TABLE_TYPE_INVALID; table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_IP_PLUS) { if (table_type != TABLE_TYPE_IP_PLUS) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = ipv4_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, ip_addr, int hit_group_cnt = ipv4_scan(maat_instance->tbl_mgr, thread_id, ip_addr,
physical_table_ids, physical_table_id, vtable_id, mid);
physical_table_cnt, vtable_id, mid);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
@@ -1518,31 +1473,29 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id,
} }
int vtable_id = 0; int vtable_id = 0;
int physical_table_cnt = 0; int physical_table_id = -1;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
memset(physical_table_ids, -1, sizeof(physical_table_ids)); if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, if (physical_table_id < 0) {
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
enum table_type table_type = TABLE_TYPE_INVALID; table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_IP_PLUS) { if (table_type != TABLE_TYPE_IP_PLUS) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = ipv6_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, ip_addr, int hit_group_cnt = ipv6_scan(maat_instance->tbl_mgr, thread_id, ip_addr,
physical_table_ids, physical_table_id, vtable_id, mid);
physical_table_cnt, vtable_id, mid);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
@@ -1603,31 +1556,29 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
} }
int vtable_id = 0; int vtable_id = 0;
int physical_table_cnt = 0; int physical_table_id = -1;
int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
memset(physical_table_ids, -1, sizeof(physical_table_ids)); if (table_type == TABLE_TYPE_VIRTUAL) {
physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
vtable_id = table_id;
} else {
physical_table_id = table_id;
}
physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, if (physical_table_id < 0) {
physical_table_ids,
MAX_PHYSICAL_TABLE_NUM, &vtable_id);
if (physical_table_cnt <= 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
enum table_type table_type = TABLE_TYPE_INVALID; table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if (0 == vtable_id) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int hit_group_cnt = string_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, data, int hit_group_cnt = string_scan(maat_instance->tbl_mgr, thread_id, data, data_len,
data_len, physical_table_ids, physical_table_id, vtable_id, mid);
physical_table_cnt, vtable_id, mid);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
@@ -1666,52 +1617,43 @@ struct maat_stream *maat_scan_stream_open(struct maat *maat_instance, int table_
} }
struct maat_stream *stream = ALLOC(struct maat_stream, 1); struct maat_stream *stream = ALLOC(struct maat_stream, 1);
memset(stream->physical_table_ids, -1, sizeof(stream->physical_table_ids));
stream->ref_maat_instance = maat_instance; stream->ref_maat_instance = maat_instance;
stream->table_id = table_id;
stream->thread_id = thread_id; stream->thread_id = thread_id;
stream->logger = maat_instance->logger; stream->logger = maat_instance->logger;
stream->n_physical_table = vtable_get_physical_table_ids(stream->ref_maat_instance->tbl_mgr,
stream->table_id, stream->physical_table_ids, enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
MAX_PHYSICAL_TABLE_NUM, &stream->vtable_id); if (table_type == TABLE_TYPE_VIRTUAL) {
if (stream->n_physical_table <= 0) { stream->physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, table_id);
stream->vtable_id = table_id;
} else {
stream->physical_table_id = table_id;
stream->vtable_id = 0;
}
if (stream->physical_table_id < 0) {
return NULL; return NULL;
} }
enum table_type table_type = TABLE_TYPE_INVALID; table_type = table_manager_get_table_type(maat_instance->tbl_mgr, stream->physical_table_id);
if (0 == stream->vtable_id) {
table_type = table_manager_get_table_type(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[0]);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] table(table_id:%d) thread_id:%d scan stream's physical table must be expr or expr_plus",
__FUNCTION__, __LINE__, table_id, thread_id);
return NULL;
}
}
for (size_t i = 0; i < stream->n_physical_table; i++) {
enum table_type table_type = table_manager_get_table_type(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[i]);
if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"%s:%d] table(table_id:%d) thread_id:%d scan stream's physical table must be expr or expr_plus",
__FUNCTION__, __LINE__, table_id, thread_id);
return NULL; return NULL;
} }
void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr, void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[i]); stream->physical_table_id);
assert(expr_rt != NULL); assert(expr_rt != NULL);
int ret = expr_runtime_stream_open((struct expr_runtime *)expr_rt, thread_id); struct adapter_hs_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt, thread_id);
if (ret < 0) { if (NULL == handle) {
return NULL; goto error;
}
} }
stream->s_handle = handle;
return stream; return stream;
error:
FREE(stream);
return NULL;
} }
int maat_scan_stream(struct maat_stream **maat_stream, const char *data, int data_len, int maat_scan_stream(struct maat_stream **maat_stream, const char *data, int data_len,
@@ -1728,20 +1670,23 @@ int maat_scan_stream(struct maat_stream **maat_stream, const char *data, int dat
mid = grab_state(state, stream->ref_maat_instance, stream->thread_id); mid = grab_state(state, stream->ref_maat_instance, stream->thread_id);
mid->scan_cnt++; mid->scan_cnt++;
int valid_table_id = -1;
if (stream->vtable_id != 0) {
valid_table_id = stream->vtable_id;
} else {
valid_table_id = stream->physical_table_id;
}
if (NULL == stream->ref_maat_instance->maat_rt) { if (NULL == stream->ref_maat_instance->maat_rt) {
log_error(stream->logger, MODULE_MAAT_API, log_error(stream->logger, MODULE_MAAT_API,
"[%s:%d] table(table_id:%d) thread_id:%d maat_scan_stream error because of maat_runtime is NULL", "[%s:%d] table(table_id:%d) thread_id:%d maat_scan_stream error because of maat_runtime is NULL",
__FUNCTION__, __LINE__, stream->table_id, stream->thread_id); __FUNCTION__, __LINE__, valid_table_id, stream->thread_id);
return MAAT_SCAN_OK; return MAAT_SCAN_OK;
} }
alignment_int64_array_add(stream->ref_maat_instance->thread_call_cnt, stream->thread_id, 1); alignment_int64_array_add(stream->ref_maat_instance->thread_call_cnt, stream->thread_id, 1);
int hit_group_cnt = stream_scan_hit_group_count(stream->ref_maat_instance->tbl_mgr, int hit_group_cnt = expr_stream_scan(stream, data, data_len, mid);
stream->thread_id, data, data_len,
stream->physical_table_ids,
stream->n_physical_table,
stream->vtable_id, mid);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
@@ -1774,13 +1719,7 @@ void maat_scan_stream_close(struct maat_stream **maat_stream)
{ {
struct maat_stream *stream = *maat_stream; struct maat_stream *stream = *maat_stream;
for (size_t i = 0; i < stream->n_physical_table; i++) { expr_runtime_stream_close(stream->s_handle);
void *expr_rt = table_manager_get_runtime(stream->ref_maat_instance->tbl_mgr,
stream->physical_table_ids[i]);
assert(expr_rt != NULL);
expr_runtime_stream_close((struct expr_runtime *)expr_rt);
}
FREE(stream); FREE(stream);
} }

1
src/maat_bool_plugin.c
View File

@@ -144,7 +144,6 @@ int bool_plugin_table_set_ex_data_schema(void *bool_plugin_schema,
return 0; return 0;
} }
static int cmp_ull_p(const void *p1, const void *p2) static int cmp_ull_p(const void *p1, const void *p2)
{ {
if(* (unsigned long long*) p1 > * (unsigned long long*) p2) { if(* (unsigned long long*) p1 > * (unsigned long long*) p2) {

163
src/maat_expr.c
View File

@@ -34,7 +34,6 @@ struct expr_schema {
int expr_type_column; int expr_type_column;
int match_method_column; int match_method_column;
int is_hexbin_column; int is_hexbin_column;
enum hs_pattern_type pattern_type; /* literal or regex */
int table_id; //ugly int table_id; //ugly
struct table_manager *ref_tbl_mgr; struct table_manager *ref_tbl_mgr;
}; };
@@ -68,12 +67,9 @@ struct expr_item {
}; };
struct expr_runtime { struct expr_runtime {
enum hs_pattern_type pattern_type;
struct adapter_hs *hs; struct adapter_hs *hs;
struct adapter_hs_stream *hs_stream;
struct rcu_hash_table *htable; // store hs_expr rule for rebuild adapter_hs instance struct rcu_hash_table *htable; // store hs_expr rule for rebuild adapter_hs instance
struct rcu_hash_table *item_htable; // store this expr table's all maat_item which will be used in expr_runtime_scan struct rcu_hash_table *item_htable; // store this expr table's all maat_item which will be used in expr_runtime_scan
struct group2group_runtime *ref_g2g_rt;
uint32_t rule_num; uint32_t rule_num;
int n_worker_thread; int n_worker_thread;
@@ -275,34 +271,26 @@ void expr_item_free(struct expr_item *expr_item)
FREE(expr_item); FREE(expr_item);
} }
enum hs_pattern_type pattern_type_str_to_enum(const char *type_str)
{
enum hs_pattern_type pattern_type = HS_PATTERN_TYPE_MAX;
if (strcmp(type_str, "literal") == 0) {
pattern_type = HS_PATTERN_TYPE_STR;
} else if (strcmp(type_str, "regex") == 0) {
pattern_type = HS_PATTERN_TYPE_REG;
} else {
assert(0);
}
return pattern_type;
}
void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger) const char *table_name, struct log_handle *logger)
{ {
int read_cnt = 0; char table_type[NAME_MAX] = {0};
struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1); struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1);
cJSON *custom_item = NULL; cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) { if (item != NULL && item->type == cJSON_Number) {
expr_schema->table_id = item->valueint; expr_schema->table_id = item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no table_id column", table_name);
goto error;
} }
/* table_type already validate in maat_table_new() */
item = cJSON_GetObjectItem(json, "table_type");
memcpy(table_type, item->valuestring, strlen(item->valuestring));
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
@@ -310,60 +298,74 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
goto error; goto error;
} }
custom_item = cJSON_GetObjectItem(item, "pattern_type");
if (custom_item != NULL && custom_item->type == cJSON_String) {
expr_schema->pattern_type = pattern_type_str_to_enum(custom_item->valuestring);
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "item_id"); custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->item_id_column = custom_item->valueint; expr_schema->item_id_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no item_id column", table_name);
goto error;
} }
custom_item = cJSON_GetObjectItem(item, "group_id"); custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->group_id_column = custom_item->valueint; expr_schema->group_id_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no group_id column", table_name);
goto error;
} }
custom_item = cJSON_GetObjectItem(item, "keywords"); custom_item = cJSON_GetObjectItem(item, "keywords");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->keywords_column = custom_item->valueint; expr_schema->keywords_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no keywords column", table_name);
goto error;
} }
/* expr_plus has district */ /* expr_plus has district */
if (strcmp(table_type, "expr_plus") == 0) {
custom_item = cJSON_GetObjectItem(item, "district"); custom_item = cJSON_GetObjectItem(item, "district");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->district_column = custom_item->valueint; expr_schema->district_column = custom_item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] expr_plus table %s has no district column", table_name);
goto error;
}
} }
custom_item = cJSON_GetObjectItem(item, "expr_type"); custom_item = cJSON_GetObjectItem(item, "expr_type");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->expr_type_column = custom_item->valueint; expr_schema->expr_type_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no expr_type column", table_name);
goto error;
} }
custom_item = cJSON_GetObjectItem(item, "match_method"); custom_item = cJSON_GetObjectItem(item, "match_method");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->match_method_column = custom_item->valueint; expr_schema->match_method_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no match_method column", table_name);
goto error;
} }
custom_item = cJSON_GetObjectItem(item, "is_hexbin"); custom_item = cJSON_GetObjectItem(item, "is_hexbin");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
expr_schema->is_hexbin_column = custom_item->valueint; expr_schema->is_hexbin_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no is_hexbin column", table_name);
goto error;
} }
expr_schema->ref_tbl_mgr = tbl_mgr; expr_schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 8) {
goto error;
}
return expr_schema; return expr_schema;
error: error:
FREE(expr_schema); FREE(expr_schema);
@@ -408,12 +410,10 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num,
return NULL; return NULL;
} }
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1); struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->htable = rcu_hash_new(expr_ex_data_free); expr_rt->htable = rcu_hash_new(expr_ex_data_free);
expr_rt->item_htable = rcu_hash_new(expr_maat_item_free); expr_rt->item_htable = rcu_hash_new(expr_maat_item_free);
expr_rt->pattern_type = schema->pattern_type;
expr_rt->n_worker_thread = max_thread_num; expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin; expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger; expr_rt->logger = logger;
@@ -436,10 +436,10 @@ void expr_runtime_free(void *expr_runtime)
expr_rt->hs = NULL; expr_rt->hs = NULL;
} }
if (expr_rt->hs_stream != NULL) { // if (expr_rt->hs_stream != NULL) {
adapter_hs_stream_close(expr_rt->hs_stream); // adapter_hs_stream_close(expr_rt->hs_stream);
expr_rt->hs_stream = NULL; // expr_rt->hs_stream = NULL;
} // }
if (expr_rt->htable != NULL) { if (expr_rt->htable != NULL) {
rcu_hash_free(expr_rt->htable); rcu_hash_free(expr_rt->htable);
@@ -563,9 +563,9 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
if (i >= MAAT_MAX_EXPR_ITEM_NUM) { if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"[%s:%d] expr item_id:%d too many patterns", "[%s:%d]abandon config expr_item(item_id:%d) too many patterns",
__FUNCTION__, __LINE__, expr_item->item_id); __FUNCTION__, __LINE__, expr_item->item_id);
return NULL; goto error;
} }
sub_key_array[i] = tmp; sub_key_array[i] = tmp;
@@ -586,9 +586,9 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
if (i >= MAAT_MAX_EXPR_ITEM_NUM) { if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"[%s:%d] expr item_id:%d too many patterns", "[%s:%d]abandon config expr_item(item_id:%d) too many patterns",
__FUNCTION__, __LINE__, expr_item->item_id); __FUNCTION__, __LINE__, expr_item->item_id);
return NULL; goto error;
} }
sub_key_array[i] = tmp; sub_key_array[i] = tmp;
@@ -596,17 +596,17 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
if (!(key_left_offset[i] >= 0 && key_right_offset[i] > 0 if (!(key_left_offset[i] >= 0 && key_right_offset[i] > 0
&& key_left_offset[i] <= key_right_offset[i])) { && key_left_offset[i] <= key_right_offset[i])) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"[%s:%d] expr item:%d has invalid offset.", "[%s:%d]abandon config expr_item(item_id:%d) has invalid offset.",
__FUNCTION__, __LINE__, expr_item->item_id); __FUNCTION__, __LINE__, expr_item->item_id);
return NULL; goto error;
} }
sub_key_array[i] = (char *)memchr(sub_key_array[i], ':', strlen(sub_key_array[i])); sub_key_array[i] = (char *)memchr(sub_key_array[i], ':', strlen(sub_key_array[i]));
if (NULL == sub_key_array[i]) { if (NULL == sub_key_array[i]) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"[%s:%d] expr item:%d has invalid offset keyword format.", "[%s:%d]abandon config expr_item(item_id:%d) has invalid offset keyword format.",
__FUNCTION__, __LINE__, expr_item->item_id); __FUNCTION__, __LINE__, expr_item->item_id);
return NULL; goto error;
} }
sub_key_array[i]++;//jump over ':' sub_key_array[i]++;//jump over ':'
@@ -620,24 +620,26 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
sub_key_array[0] = str_unescape(sub_key_array[0]); sub_key_array[0] = str_unescape(sub_key_array[0]);
break; break;
default: default:
break; log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%lld) has invalid expr type=%d",
__FUNCTION__, __LINE__, expr_item->item_id, expr_item->expr_type);
goto error;
} }
for (i = 0; i < sub_expr_cnt; i++) {
size_t region_str_len = 0; size_t region_str_len = 0;
char *region_string = NULL; char *region_string = NULL;
size_t sub_key_len = 0; size_t sub_key_len = 0;
for (i = 0; i < sub_expr_cnt; i++) {
if (FALSE == expr_item->is_case_sensitive) { if (FALSE == expr_item->is_case_sensitive) {
// insensitive // insensitive
expr_rule->patterns[i].case_sensitive = HS_CASE_INSESITIVE; expr_rule->patterns[i].case_sensitive = HS_CASE_INSESITIVE;
} }
enum hs_pattern_type pattern_type = expr_type2pattern_type(expr_item->expr_type); expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type);
if (TRUE == expr_item->is_hexbin && pattern_type != HS_PATTERN_TYPE_REG) { if (TRUE == expr_item->is_hexbin && expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) {
region_str_len = strlen(sub_key_array[i]) + 1; region_str_len = strlen(sub_key_array[i]) * 8 + 1;
region_string = ALLOC(char, region_str_len); region_string = ALLOC(char, region_str_len);
region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]), region_string, region_str_len); region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]), region_string, region_str_len);
} }
@@ -656,8 +658,8 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
expr_rule->patterns[i].match_mode = expr_item->match_mode; expr_rule->patterns[i].match_mode = expr_item->match_mode;
if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) { if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) {
expr_rule->patterns[i].l_offset = key_left_offset[i]; expr_rule->patterns[i].start_offset = key_left_offset[i];
expr_rule->patterns[i].r_offset = key_right_offset[i]; expr_rule->patterns[i].end_offset = key_right_offset[i];
} }
} }
expr_rule->expr_id = expr_item->item_id; expr_rule->expr_id = expr_item->item_id;
@@ -674,6 +676,9 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
// printf("expr_rule->patterns[%zu].r_offset:%d\n", i, expr_rule->patterns[i].r_offset); // printf("expr_rule->patterns[%zu].r_offset:%d\n", i, expr_rule->patterns[i].r_offset);
// } // }
return expr_rule; return expr_rule;
error:
FREE(expr_rule);
return NULL;
} }
int expr_runtime_update(void *expr_runtime, void *expr_schema, int expr_runtime_update(void *expr_runtime, void *expr_schema,
@@ -763,18 +768,11 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
struct hs_expr *rules = NULL; struct hs_expr *rules = NULL;
void **ex_data_array = NULL; void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list(expr_rt->htable, &ex_data_array); size_t rule_cnt = rcu_hash_list(expr_rt->htable, &ex_data_array);
//printf("rcu_hash_commit rule_cnt:%zu\n", rule_cnt);
if (rule_cnt > 0) { if (rule_cnt > 0) {
rules = ALLOC(struct hs_expr, rule_cnt); rules = ALLOC(struct hs_expr, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) { for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct hs_expr *)ex_data_array[i]; rules[i] = *(struct hs_expr *)ex_data_array[i];
// if (rules[i].expr_id == 13)
// {
// for (size_t j = 0; j < rules[i].n_patterns; j++)
// {
// printf("rules[%zu].patterns[%zu]:%s\n", i, j, rules[i].patterns[j].pat);
// }
// }
} }
} }
@@ -785,8 +783,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
int ret = 0; int ret = 0;
struct adapter_hs *new_adapter_hs = NULL; struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL; struct adapter_hs *old_adapter_hs = NULL;
new_adapter_hs = adapter_hs_initialize(expr_rt->pattern_type, expr_rt->n_worker_thread, new_adapter_hs = adapter_hs_initialize(expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger);
rules, rule_cnt, expr_rt->logger);
if (NULL == new_adapter_hs) { if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild adapter_hs engine failed when update %zu expr rules", "[%s:%d] table[%s] rebuild adapter_hs engine failed when update %zu expr rules",
@@ -867,24 +864,17 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
return group_hit_cnt; return group_hit_cnt;
} }
int expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
{ {
if (NULL == expr_rt || thread_id < 0) { if (NULL == expr_rt || thread_id < 0) {
return -1; return NULL;
} }
struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(expr_rt->hs, thread_id); return adapter_hs_stream_open(expr_rt->hs, thread_id);
if (NULL == hs_stream) {
return -1;
}
expr_rt->hs_stream = hs_stream;
return 0;
} }
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle,
int vtable_id, struct maat_state *state) const char *data, size_t data_len, int vtable_id, struct maat_state *state)
{ {
if (0 == expr_rt->rule_num) { if (0 == expr_rt->rule_num) {
//empty expr table //empty expr table
@@ -893,9 +883,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz
size_t n_hit_item = 0; size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
int ret = adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -928,12 +916,9 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz
return group_hit_cnt; return group_hit_cnt;
} }
void expr_runtime_stream_close(struct expr_runtime *expr_rt) void expr_runtime_stream_close(struct adapter_hs_stream *s_handle)
{ {
if (expr_rt != NULL) { adapter_hs_stream_close(s_handle);
adapter_hs_stream_close(expr_rt->hs_stream);
expr_rt->hs_stream = NULL;
}
} }
void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id) void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id)

13
src/maat_plugin.c
View File

@@ -94,14 +94,9 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
} }
schema->table_id = item->valueint; schema->table_id = item->valueint;
/* custom is optional */
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item != NULL && item->type == cJSON_Object) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table %s has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "key"); custom_item = cJSON_GetObjectItem(item, "key");
if (NULL == custom_item || custom_item->type != cJSON_Number) { if (NULL == custom_item || custom_item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN, log_error(logger, MODULE_PLUGIN,
@@ -144,13 +139,15 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
MAX_FOREIGN_CLMN_NUM); MAX_FOREIGN_CLMN_NUM);
} else if (custom_item->type == cJSON_Array) { } else if (custom_item->type == cJSON_Array) {
schema->n_foreign = cJSON_GetArraySize(custom_item); schema->n_foreign = cJSON_GetArraySize(custom_item);
for (int i = 0; i < schema->n_foreign; i++) { for (int i = 0; i < schema->n_foreign; i++)
{
cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i); cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i);
assert(foreign_item->type == cJSON_Number); assert(foreign_item->type == cJSON_Number);
schema->foreign_columns[i] = foreign_item->valueint; schema->foreign_columns[i] = foreign_item->valueint;
} }
} }
} }
}
schema->ref_tbl_mgr = tbl_mgr; schema->ref_tbl_mgr = tbl_mgr;
return schema; return schema;

24
src/maat_virtual.c
View File

@@ -20,8 +20,7 @@
#define MODULE_VIRTUAL module_name_str("maat.virtual") #define MODULE_VIRTUAL module_name_str("maat.virtual")
struct virtual_schema { struct virtual_schema {
char physical_tables[MAX_PHYSICAL_TABLE_NUM][NAME_MAX]; char physical_table[NAME_MAX];
size_t n_physical_table;
int table_id; int table_id;
struct table_manager *ref_tbl_mgr; struct table_manager *ref_tbl_mgr;
}; };
@@ -42,22 +41,14 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint; schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "physical_table"); item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_Array) { if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_VIRTUAL, log_error(logger, MODULE_VIRTUAL,
"[%s:%d] virtual table %s has no physical_table column", "[%s:%d] virtual table %s has no physical_table column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
int cnt = cJSON_GetArraySize(item); memcpy(schema->physical_table, item->valuestring, strlen(item->valuestring));
for (int i = 0; i < cnt; i++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, i);
if (tmp_item != NULL && tmp_item->type == cJSON_String) {
memcpy(schema->physical_tables[i], tmp_item->valuestring,
strlen(tmp_item->valuestring));
}
}
schema->n_physical_table = cnt;
return schema; return schema;
error: error:
@@ -80,7 +71,7 @@ int virtual_table_get_id(void *virtual_schema)
return schema->table_id; return schema->table_id;
} }
size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]) int virtual_table_get_physical_table_id(void *virtual_schema)
{ {
if (NULL == virtual_schema) { if (NULL == virtual_schema) {
return 0; return 0;
@@ -88,10 +79,5 @@ size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_ta
struct virtual_schema *schema = (struct virtual_schema *)virtual_schema; struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
for (size_t i = 0; i < schema->n_physical_table; i++) { return table_manager_get_table_id(schema->ref_tbl_mgr, schema->physical_table);
int table_id = table_manager_get_table_id(schema->ref_tbl_mgr, schema->physical_tables[i]);
physical_table_ids[i] = table_id;
}
return schema->n_physical_table;
} }

89
test/adapter_hs_gtest.cpp
View File

@@ -174,11 +174,11 @@ int parse_config_file(const char *filename, struct hs_expr exprs[], size_t *n_ex
printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n", printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n",
key_left_offset, key_right_offset); key_left_offset, key_right_offset);
} }
exprs[i].patterns[j].l_offset = key_left_offset; exprs[i].patterns[j].start_offset = key_left_offset;
exprs[i].patterns[j].r_offset = key_right_offset; exprs[i].patterns[j].end_offset = key_right_offset;
} else { } else {
exprs[i].patterns[j].l_offset = -1; exprs[i].patterns[j].start_offset = -1;
exprs[i].patterns[j].r_offset = -1; exprs[i].patterns[j].end_offset = -1;
} }
} }
} }
@@ -209,27 +209,23 @@ TEST(adapter_hs_init, invalid_input_parameter)
struct hs_expr expr_array[64]; struct hs_expr expr_array[64];
size_t n_expr_array = 0; size_t n_expr_array = 0;
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, struct adapter_hs *hs_instance = adapter_hs_initialize(1, NULL, 0, g_logger);
1, NULL, 0, g_logger);
EXPECT_TRUE(hs_instance == NULL); EXPECT_TRUE(hs_instance == NULL);
hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array, hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
n_expr_array, g_logger);
EXPECT_TRUE(hs_instance == NULL); EXPECT_TRUE(hs_instance == NULL);
n_expr_array = 1; n_expr_array = 1;
expr_array[0].expr_id = 101; expr_array[0].expr_id = 101;
expr_array[0].n_patterns = 10; expr_array[0].n_patterns = 10;
hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array, hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
n_expr_array, g_logger);
EXPECT_TRUE(hs_instance == NULL); EXPECT_TRUE(hs_instance == NULL);
memset(expr_array, 0, sizeof(expr_array)); memset(expr_array, 0, sizeof(expr_array));
n_expr_array = 1; n_expr_array = 1;
expr_array[0].expr_id = 101; expr_array[0].expr_id = 101;
expr_array[0].n_patterns = 1; expr_array[0].n_patterns = 1;
hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_REG, 1, expr_array, hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
n_expr_array, g_logger);
EXPECT_TRUE(hs_instance == NULL); EXPECT_TRUE(hs_instance == NULL);
} }
@@ -240,10 +236,9 @@ TEST(adapter_hs_scan, literal_sub_has_normal_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_expr_array, 11); EXPECT_EQ(n_expr_array, 12);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array); expr_array_free(expr_array, n_expr_array);
@@ -268,6 +263,14 @@ TEST(adapter_hs_scan, literal_sub_has_normal_offset)
n_result = 0; n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].item_id, 101);
char scan_data4[64] = "Aaahello aaa";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0); EXPECT_EQ(n_result, 0);
adapter_hs_destroy(hs_instance); adapter_hs_destroy(hs_instance);
@@ -282,8 +285,7 @@ TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array); expr_array_free(expr_array, n_expr_array);
@@ -309,6 +311,14 @@ TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset)
n_result = 0; n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].item_id, 102);
char scan_data4[64] = "Aaahello bbb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0); EXPECT_EQ(n_result, 0);
adapter_hs_destroy(hs_instance); adapter_hs_destroy(hs_instance);
@@ -323,8 +333,7 @@ TEST(adapter_hs_scan, literal_sub_has_right_unlimit_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array); expr_array_free(expr_array, n_expr_array);
@@ -379,8 +388,7 @@ TEST(adapter_hs_scan, literal_sub_with_no_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array); expr_array_free(expr_array, n_expr_array);
@@ -427,8 +435,7 @@ TEST(adapter_hs_scan, literal_exactly)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array); expr_array_free(expr_array, n_expr_array);
@@ -469,8 +476,7 @@ TEST(adapter_hs_scan, literal_prefix)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array); expr_array_free(expr_array, n_expr_array);
@@ -520,8 +526,7 @@ TEST(adapter_hs_scan, literal_suffix)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array); expr_array_free(expr_array, n_expr_array);
@@ -571,8 +576,7 @@ TEST(adapter_hs_scan, literal_sub_with_hexbin)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array); expr_array_free(expr_array, n_expr_array);
@@ -604,8 +608,7 @@ TEST(adapter_hs_scan, literal_with_chinese)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(HS_PATTERN_TYPE_STR, 1, struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL); EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array); expr_array_free(expr_array, n_expr_array);
@@ -621,6 +624,30 @@ TEST(adapter_hs_scan, literal_with_chinese)
hs_instance = NULL; hs_instance = NULL;
} }
TEST(adapter_hs_scan, same_pattern_different_offset)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_initialize(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
char data[64] = "onetoday,anothertoday";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, data, strlen(data), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].item_id, 112);
adapter_hs_destroy(hs_instance);
hs_instance = NULL;
}
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
int ret = 0; int ret = 0;

20
test/literal_expr.conf
View File

@@ -141,6 +141,26 @@
"pattern": "يىلىدىكى" "pattern": "يىلىدىكى"
} }
] ]
},
{
"expr_id": 112,
"pattern_num": 2,
"patterns": [
{
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
"pattern": "today",
"offset": "3~7"
},
{
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
"pattern": "today",
"offset": "16~20"
}
]
} }
] ]
} }

50
test/maat_framework_gtest.cpp
View File

@@ -353,7 +353,7 @@ TEST_F(MaatFlagScan, basic) {
TEST_F(MaatFlagScan, withExprRegion) { TEST_F(MaatFlagScan, withExprRegion) {
const char *flag_table_name = "FLAG_CONFIG"; const char *flag_table_name = "FLAG_CONFIG";
const char *expr_table_name = "HTTP_URL_LITERAL"; const char *expr_table_name = "HTTP_URL";
struct maat *maat_instance = MaatFlagScan::_shared_maat_instance; struct maat *maat_instance = MaatFlagScan::_shared_maat_instance;
int flag_table_id = maat_get_table_id(maat_instance, flag_table_name); int flag_table_id = maat_get_table_id(maat_instance, flag_table_name);
@@ -540,7 +540,7 @@ struct maat *MaatStringScan::_shared_maat_instance;
struct log_handle *MaatStringScan::logger; struct log_handle *MaatStringScan::logger;
TEST_F(MaatStringScan, Full) { TEST_F(MaatStringScan, Full) {
const char *table_name = "HTTP_URL_REGEX"; const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance; struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_instance, table_name);
@@ -564,7 +564,7 @@ TEST_F(MaatStringScan, Regex) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *cookie = "Cookie: Txa123aheadBCAxd"; const char *cookie = "Cookie: Txa123aheadBCAxd";
const char *table_name = "HTTP_URL_REGEX"; const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance; struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_instance, table_name);
@@ -629,7 +629,7 @@ TEST_F(MaatStringScan, ExprPlus) {
EXPECT_EQ(results[0], 190); EXPECT_EQ(results[0], 190);
maat_state_free(&state); maat_state_free(&state);
} }
#if 0
TEST_F(MaatStringScan, ExprPlusWithOffset) TEST_F(MaatStringScan, ExprPlusWithOffset)
{ {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
@@ -681,7 +681,7 @@ TEST_F(MaatStringScan, ExprPlusWithOffset)
maat_state_free(&state); maat_state_free(&state);
} }
#if 0
TEST_F(MaatStringScan, ExprPlusWithHex) { TEST_F(MaatStringScan, ExprPlusWithHex) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
@@ -721,7 +721,7 @@ TEST_F(MaatStringScan, ExprAndExprPlus) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance; struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *expr_table_name = "HTTP_URL_LITERAL"; const char *expr_table_name = "HTTP_URL";
const char *expr_plus_table_name = "HTTP_SIGNATURE"; const char *expr_plus_table_name = "HTTP_SIGNATURE";
const char *region_name = "I love China"; const char *region_name = "I love China";
const char *scan_data = "today is Monday and yesterday is Tuesday"; const char *scan_data = "today is Monday and yesterday is Tuesday";
@@ -904,7 +904,8 @@ TEST_F(MaatStringScan, PrefixAndSuffix) {
EXPECT_EQ(results[0], 152); EXPECT_EQ(results[0], 152);
maat_state_free(&state); maat_state_free(&state);
} }
#endif
#if 0
TEST_F(MaatStringScan, MaatUnescape) { TEST_F(MaatStringScan, MaatUnescape) {
const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me."; const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
@@ -923,11 +924,12 @@ TEST_F(MaatStringScan, MaatUnescape) {
EXPECT_EQ(results[0], 132); EXPECT_EQ(results[0], 132);
maat_state_free(&state); maat_state_free(&state);
} }
#endif
#if 0
TEST_F(MaatStringScan, RegexWithNotContains) { TEST_F(MaatStringScan, RegexWithNotContains) {
const char *should_NOT_hit_scan_data = "new.qq.com/rain/a/TWF2021042600418000"; const char *should_NOT_hit_scan_data = "new.qq.com/rain/a/TWF2021042600418000";
const char *should_hit_scan_data = "fakesina.com/rain/a/TWF2021042600418000"; const char *should_hit_scan_data = "fakesina.com/rain/a/TWF2021042600418000";
const char *table_name = "HTTP_URL_REGEX"; const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatStringScan::_shared_maat_instance; struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_instance, table_name);
@@ -1031,7 +1033,7 @@ TEST_F(MaatStringScan, StreamInput) {
struct maat_state *state = NULL; struct maat_state *state = NULL;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance; struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char *table_name = "HTTP_URL_REGEX"; const char *table_name = "HTTP_URL";
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
@@ -1053,7 +1055,7 @@ TEST_F(MaatStringScan, StreamInput) {
} }
TEST_F(MaatStringScan, dynamic_config) { TEST_F(MaatStringScan, dynamic_config) {
const char *table_name = "HTTP_URL_LITERAL"; const char *table_name = "HTTP_URL";
char data[128] = "hello world, welcome to maat version4, it's funny."; char data[128] = "hello world, welcome to maat version4, it's funny.";
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
@@ -1414,7 +1416,7 @@ TEST_F(NOTLogic, OneRegion) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *table_name = "HTTP_URL_LITERAL"; const char *table_name = "HTTP_URL";
struct maat *maat_instance = NOTLogic::_shared_maat_instance; struct maat *maat_instance = NOTLogic::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_instance, table_name);
@@ -1442,7 +1444,7 @@ TEST_F(NOTLogic, ScanNotAtLast) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *hit_table_name = "HTTP_URL_LITERAL"; const char *hit_table_name = "HTTP_URL";
const char *not_hit_table_name = "KEYWORDS_TABLE"; const char *not_hit_table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = NOTLogic::_shared_maat_instance; struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -1469,7 +1471,7 @@ TEST_F(NOTLogic, ScanIrrelavantAtLast) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *hit_table_name = "HTTP_URL_LITERAL"; const char *hit_table_name = "HTTP_URL";
const char *not_hit_table_name = "KEYWORDS_TABLE"; const char *not_hit_table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = NOTLogic::_shared_maat_instance; struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -1498,7 +1500,7 @@ TEST_F(NOTLogic, ScanHitAtLastEmptyExpr) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *not_hit_table_name = "HTTP_URL_LITERAL"; const char *not_hit_table_name = "HTTP_URL";
const char *hit_table_name = "IP_PLUS_CONFIG"; const char *hit_table_name = "IP_PLUS_CONFIG";
const char *empty_table_name = "EMPTY_KEYWORD"; const char *empty_table_name = "EMPTY_KEYWORD";
struct maat *maat_instance = NOTLogic::_shared_maat_instance; struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -1537,7 +1539,7 @@ TEST_F(NOTLogic, ScanHitAtLastEmptyInteger) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *not_hit_table_name = "HTTP_URL_LITERAL"; const char *not_hit_table_name = "HTTP_URL";
const char *hit_table_name = "IP_PLUS_CONFIG"; const char *hit_table_name = "IP_PLUS_CONFIG";
const char *empty_table_name = "EMPTY_INTERGER"; const char *empty_table_name = "EMPTY_INTERGER";
struct maat *maat_instance = NOTLogic::_shared_maat_instance; struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -1575,7 +1577,7 @@ TEST_F(NOTLogic, ScanNotIP) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *hit_table_name = "HTTP_URL_LITERAL"; const char *hit_table_name = "HTTP_URL";
const char *not_hit_table_name = "IP_CONFIG"; const char *not_hit_table_name = "IP_CONFIG";
struct maat *maat_instance = NOTLogic::_shared_maat_instance; struct maat *maat_instance = NOTLogic::_shared_maat_instance;
@@ -2367,7 +2369,7 @@ TEST_F(CompileTable, Conjunction1) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL_LITERAL"; const char *table_name = "HTTP_URL";
const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"}; const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"};
struct maat *maat_instance = CompileTable::_shared_maat_instance; struct maat *maat_instance = CompileTable::_shared_maat_instance;
@@ -2394,7 +2396,7 @@ TEST_F(CompileTable, Conjunction2) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL_LITERAL"; const char *table_name = "HTTP_URL";
const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"}; const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"};
struct maat *maat_instance = CompileTable::_shared_maat_instance; struct maat *maat_instance = CompileTable::_shared_maat_instance;
@@ -2525,7 +2527,7 @@ TEST_F(Policy, CompileRuleTags) {
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *should_hit = "string bbb should hit"; const char *should_hit = "string bbb should hit";
const char *should_not_hit = "string aaa should not hit"; const char *should_not_hit = "string aaa should not hit";
const char *table_name = "HTTP_URL_LITERAL"; const char *table_name = "HTTP_URL";
struct maat *maat_instance = Policy::_shared_maat_instance; struct maat *maat_instance = Policy::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_instance, table_name);
@@ -2549,7 +2551,7 @@ TEST_F(Policy, CompileEXData) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
const char *table_name = "HTTP_URL_LITERAL"; const char *table_name = "HTTP_URL";
const char *compile_table_name = "COMPILE_ALIAS"; const char *compile_table_name = "COMPILE_ALIAS";
const char *expect_name = "I have a name"; const char *expect_name = "I have a name";
struct maat *maat_instance = Policy::_shared_maat_instance; struct maat *maat_instance = Policy::_shared_maat_instance;
@@ -2679,8 +2681,8 @@ TEST_F(TableInfo, Conjunction) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
const char *scan_data = "soq is using table conjunction function.http://www.3300av.com/novel/27122.txt"; const char *scan_data = "soq is using table conjunction function.http://www.3300av.com/novel/27122.txt";
const char *table_name = "HTTP_URL_LITERAL"; const char *table_name = "HTTP_URL";
const char *conj_table_name = "HTTP_HOST_LITERAL"; const char *conj_table_name = "HTTP_HOST";
struct maat *maat_instance = TableInfo::_shared_maat_instance; struct maat *maat_instance = TableInfo::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_instance, table_name);
@@ -2869,7 +2871,7 @@ TEST_F(MaatCmdTest, SetIP) {
TEST_F(MaatCmdTest, SetExpr) { TEST_F(MaatCmdTest, SetExpr) {
const char *scan_data = "Hiredis is a minimalistic C client library for the Redis database.\r\n"; const char *scan_data = "Hiredis is a minimalistic C client library for the Redis database.\r\n";
const char *table_name = "HTTP_URL_LITERAL"; const char *table_name = "HTTP_URL";
const char *keywords1 = "Hiredis"; const char *keywords1 = "Hiredis";
const char *keywords2 = "C Client"; const char *keywords2 = "C Client";

73
test/maat_json.json
View File

@@ -243,7 +243,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "abckkk&123", "keywords": "abckkk&123",
@@ -295,7 +295,7 @@
{ {
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_REGEX", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "action=search\\&query=(.*)", "keywords": "action=search\\&query=(.*)",
@@ -321,7 +321,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "should_not_hit_any_rule", "keywords": "should_not_hit_any_rule",
@@ -387,7 +387,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "C#中国", "keywords": "C#中国",
@@ -491,7 +491,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_HOST_LITERAL", "table_name": "HTTP_HOST",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "www.3300av.com", "keywords": "www.3300av.com",
@@ -517,7 +517,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "novel&27122.txt", "keywords": "novel&27122.txt",
@@ -596,7 +596,7 @@
{ {
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "should&hit&aaa", "keywords": "should&hit&aaa",
@@ -623,7 +623,7 @@
{ {
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "should&hit&bbb", "keywords": "should&hit&bbb",
@@ -677,7 +677,7 @@
"g2c_table_name": "GROUP2COMPILE_ALIAS", "g2c_table_name": "GROUP2COMPILE_ALIAS",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "i.ytimg.com", "keywords": "i.ytimg.com",
@@ -702,7 +702,7 @@
{ {
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": ",IgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE", "keywords": ",IgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE",
@@ -728,7 +728,7 @@
"not_flag":0, "not_flag":0,
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-contained-string-of-rule-143", "keywords": "must-contained-string-of-rule-143",
@@ -743,7 +743,7 @@
"not_flag":1, "not_flag":1,
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-not-contained-string-of-rule-143", "keywords": "must-not-contained-string-of-rule-143",
@@ -769,7 +769,7 @@
"not_flag":0, "not_flag":0,
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-contained-string-of-rule-144", "keywords": "must-contained-string-of-rule-144",
@@ -810,7 +810,7 @@
"not_flag":0, "not_flag":0,
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-contained-string-of-rule-145", "keywords": "must-contained-string-of-rule-145",
@@ -839,7 +839,7 @@
{ {
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_REGEX", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "Cookie:\\s&head", "keywords": "Cookie:\\s&head",
@@ -864,7 +864,7 @@
{ {
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "googlevideo.com/videoplayback&mn=sn-35153iuxa-5a56%2Csn-n8v7znz7", "keywords": "googlevideo.com/videoplayback&mn=sn-35153iuxa-5a56%2Csn-n8v7znz7",
@@ -879,30 +879,29 @@
}, },
{ {
"compile_id": 148, "compile_id": 148,
"is_valid": "yes",
"do_log": 0,
"effective_rage": 0,
"action": 0,
"service": 0, "service": 0,
"action": 0,
"do_blacklist": 0, "do_blacklist": 0,
"do_log": 0,
"user_region": "StringScan.ExprPlusWithOffset", "user_region": "StringScan.ExprPlusWithOffset",
"effective_rage": 0,
"is_valid": "yes",
"groups": [ "groups": [
{ {
"group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "APP_PAYLOAD", "table_name": "APP_PAYLOAD",
"table_type": "expr_plus",
"table_content": { "table_content": {
"format": "hexbin", "format": "hexbin",
"match_method": "sub", "match_method": "sub",
"district": "Payload", "district": "Payload",
"keywords": "1-1:03&9-10:2d&14-16:2d34", "keywords": "1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d",
"expr_type": "offset" "expr_type": "offset"
},
"table_type": "expr_plus"
} }
], }
"group_name": "Untitled" ]
} }
] ]
}, },
@@ -1304,7 +1303,7 @@
"not_flag":0, "not_flag":0,
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "https://blog.csdn.net/littlefang/article/details/8213058", "keywords": "https://blog.csdn.net/littlefang/article/details/8213058",
@@ -1458,7 +1457,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "cavemancircus.com/", "keywords": "cavemancircus.com/",
@@ -1500,7 +1499,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "2019/12/27/pretty-girls-6", "keywords": "2019/12/27/pretty-girls-6",
@@ -1527,7 +1526,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "2019/12/27", "keywords": "2019/12/27",
@@ -1554,7 +1553,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "2019/12/27", "keywords": "2019/12/27",
@@ -2051,7 +2050,7 @@
"not_flag": 1, "not_flag": 1,
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-not-contained-string-of-rule-186", "keywords": "must-not-contained-string-of-rule-186",
@@ -2103,7 +2102,7 @@
"not_flag": 1, "not_flag": 1,
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-not-contained-string-of-rule-187", "keywords": "must-not-contained-string-of-rule-187",
@@ -2155,7 +2154,7 @@
"not_flag": 1, "not_flag": 1,
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-not-contained-string-of-rule-188", "keywords": "must-not-contained-string-of-rule-188",
@@ -2323,7 +2322,7 @@
{ {
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "hello", "keywords": "hello",
@@ -2386,7 +2385,7 @@
{ {
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "Monday", "keywords": "Monday",
@@ -2436,7 +2435,7 @@
"group_name": "Untitled", "group_name": "Untitled",
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL_LITERAL", "table_name": "HTTP_URL",
"table_type": "expr", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "hqdefault.jpg", "keywords": "hqdefault.jpg",

155
test/table_info.conf
View File

@@ -29,16 +29,10 @@
"table_id":2, "table_id":2,
"table_name":"COMPILE_ALIAS", "table_name":"COMPILE_ALIAS",
"table_type":"compile", "table_type":"compile",
"user_region_encoded":"escape",
"valid_column":4, "valid_column":4,
"custom": { "custom": {
"compile_id":1, "compile_id":1,
"service_id":2,
"action":3,
"do_blacklist":4,
"do_log":5,
"tags":6, "tags":6,
"user_region":7,
"clause_num":9, "clause_num":9,
"evaluation_order":10 "evaluation_order":10
} }
@@ -69,11 +63,10 @@
}, },
{ {
"table_id":5, "table_id":5,
"table_name":["HTTP_URL_LITERAL", "HTTP_HOST_LITERAL"], "table_name":["HTTP_URL", "HTTP_HOST"],
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
"custom": { "custom": {
"pattern_type":"literal",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"keywords":3, "keywords":3,
@@ -84,21 +77,6 @@
}, },
{ {
"table_id":6, "table_id":6,
"table_name":"HTTP_URL_REGEX",
"table_type":"expr",
"valid_column":7,
"custom": {
"pattern_type":"regex",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":7,
"table_name":"KEYWORDS_TABLE", "table_name":"KEYWORDS_TABLE",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
@@ -113,7 +91,7 @@
} }
}, },
{ {
"table_id":8, "table_id":7,
"table_name":"IP_CONFIG", "table_name":"IP_CONFIG",
"table_type":"ip_plus", "table_type":"ip_plus",
"valid_column":18, "valid_column":18,
@@ -127,7 +105,7 @@
} }
}, },
{ {
"table_id":9, "table_id":8,
"table_name":"CONTENT_SIZE", "table_name":"CONTENT_SIZE",
"table_type":"intval", "table_type":"intval",
"valid_column":5, "valid_column":5,
@@ -139,23 +117,17 @@
} }
}, },
{ {
"table_id":10, "table_id":9,
"table_name":"QD_ENTRY_INFO", "table_name":"QD_ENTRY_INFO",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4
"custom": {
"key_type":"pointer",
"key":1,
"tag":3
}
}, },
{ {
"table_id":11, "table_id":10,
"table_name":"HTTP_SIGNATURE", "table_name":"HTTP_SIGNATURE",
"table_type":"expr_plus", "table_type":"expr_plus",
"valid_column":8, "valid_column":8,
"custom": { "custom": {
"pattern_type":"literal",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"district":3, "district":3,
@@ -166,12 +138,11 @@
} }
}, },
{ {
"table_id":12, "table_id":11,
"table_name":"IMAGE_FP", "table_name":"IMAGE_FP",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
"custom": { "custom": {
"pattern_type":"literal",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"keywords":3, "keywords":3,
@@ -181,7 +152,7 @@
} }
}, },
{ {
"table_id":13, "table_id":12,
"table_name":"TEST_EFFECTIVE_RANGE_TABLE", "table_name":"TEST_EFFECTIVE_RANGE_TABLE",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
@@ -192,7 +163,7 @@
} }
}, },
{ {
"table_id":14, "table_id":13,
"table_name":"TEST_FOREIGN_KEY", "table_name":"TEST_FOREIGN_KEY",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
@@ -204,7 +175,7 @@
} }
}, },
{ {
"table_id":15, "table_id":14,
"table_name":"TEST_PLUGIN_EXDATA_TABLE", "table_name":"TEST_PLUGIN_EXDATA_TABLE",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
@@ -215,7 +186,7 @@
} }
}, },
{ {
"table_id":16, "table_id":15,
"table_name":"IR_INTERCEPT_IP", "table_name":"IR_INTERCEPT_IP",
"table_type":"plugin", "table_type":"plugin",
"valid_column":14, "valid_column":14,
@@ -226,12 +197,11 @@
} }
}, },
{ {
"table_id":17, "table_id":16,
"table_name":"APP_PAYLOAD", "table_name":"APP_PAYLOAD",
"table_type":"expr_plus", "table_type":"expr_plus",
"valid_column":8, "valid_column":8,
"custom": { "custom": {
"pattern_type":"literal",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"district":3, "district":3,
@@ -242,12 +212,11 @@
} }
}, },
{ {
"table_id":18, "table_id":17,
"table_name":"TROJAN_PAYLOAD", "table_name":"TROJAN_PAYLOAD",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
"custom": { "custom": {
"pattern_type":"literal",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"keywords":3, "keywords":3,
@@ -258,12 +227,11 @@
} }
}, },
{ {
"table_id":19, "table_id":18,
"table_name":"MAIL_ADDR", "table_name":"MAIL_ADDR",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
"custom": { "custom": {
"pattern_type":"literal",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"keywords":3, "keywords":3,
@@ -273,7 +241,7 @@
} }
}, },
{ {
"table_id":20, "table_id":19,
"table_name":"IP_PLUS_CONFIG", "table_name":"IP_PLUS_CONFIG",
"table_type":"ip_plus", "table_type":"ip_plus",
"valid_column":18, "valid_column":18,
@@ -287,43 +255,43 @@
} }
}, },
{ {
"table_id":21, "table_id":20,
"table_name":"HTTP_RESPONSE_KEYWORDS", "table_name":"HTTP_RESPONSE_KEYWORDS",
"table_type":"virtual", "table_type":"virtual",
"physical_table": ["KEYWORDS_TABLE"] "physical_table": "KEYWORDS_TABLE"
},
{
"table_id":21,
"table_name":"HTTP_REQUEST_HEADER",
"table_type":"virtual",
"physical_table": "HTTP_SIGNATURE"
}, },
{ {
"table_id":22, "table_id":22,
"table_name":"HTTP_REQUEST_HEADER", "table_name":"HTTP_RESPONSE_HEADER",
"table_type":"virtual", "table_type":"virtual",
"physical_table": ["HTTP_SIGNATURE"] "physical_table": "HTTP_SIGNATURE"
}, },
{ {
"table_id":23, "table_id":23,
"table_name":"HTTP_RESPONSE_HEADER", "table_name":"VIRTUAL_IP_PLUS_TABLE",
"table_type":"virtual", "table_type":"virtual",
"physical_table": ["HTTP_SIGNATURE"] "physical_table": "IP_PLUS_CONFIG"
}, },
{ {
"table_id":24, "table_id":24,
"table_name":"VIRTUAL_IP_PLUS_TABLE", "table_name":"VIRTUAL_IP_PLUS_SOURCE",
"table_type":"virtual", "table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"] "physical_table": "IP_PLUS_CONFIG"
}, },
{ {
"table_id":25, "table_id":25,
"table_name":"VIRTUAL_IP_PLUS_SOURCE", "table_name":"VIRTUAL_IP_PLUS_DESTINATION",
"table_type":"virtual", "table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"] "physical_table": "IP_PLUS_CONFIG"
}, },
{ {
"table_id":26, "table_id":26,
"table_name":"VIRTUAL_IP_PLUS_DESTINATION",
"table_type":"virtual",
"physical_table": ["IP_PLUS_CONFIG"]
},
{
"table_id":27,
"table_name":"TEST_IP_PLUGIN_WITH_EXDATA", "table_name":"TEST_IP_PLUGIN_WITH_EXDATA",
"table_type":"ip_plugin", "table_type":"ip_plugin",
"valid_column":6, "valid_column":6,
@@ -336,12 +304,11 @@
} }
}, },
{ {
"table_id":28, "table_id":27,
"table_name":"AS_NUMBER", "table_name":"AS_NUMBER",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
"custom": { "custom": {
"pattern_type":"literal",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"keywords":3, "keywords":3,
@@ -351,24 +318,23 @@
} }
}, },
{ {
"table_id":29, "table_id":28,
"table_name":"SOURCE_IP_ASN", "table_name":"SOURCE_IP_ASN",
"table_type":"virtual", "table_type":"virtual",
"physical_table":["AS_NUMBER"] "physical_table":"AS_NUMBER"
},
{
"table_id":29,
"table_name":"DESTINATION_IP_ASN",
"table_type":"virtual",
"physical_table":"AS_NUMBER"
}, },
{ {
"table_id":30, "table_id":30,
"table_name":"DESTINATION_IP_ASN",
"table_type":"virtual",
"physical_table":["AS_NUMBER"]
},
{
"table_id":31,
"table_name":"GeoLocation", "table_name":"GeoLocation",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
"custom": { "custom": {
"pattern_type":"literal",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"keywords":3, "keywords":3,
@@ -378,13 +344,13 @@
} }
}, },
{ {
"table_id":32, "table_id":31,
"table_name":"SOURCE_IP_GEO", "table_name":"SOURCE_IP_GEO",
"table_type":"virtual", "table_type":"virtual",
"physical_table":["GeoLocation"] "physical_table":"GeoLocation"
}, },
{ {
"table_id":33, "table_id":32,
"table_name":"INTERGER_PLUS", "table_name":"INTERGER_PLUS",
"table_type":"intval_plus", "table_type":"intval_plus",
"valid_column":6, "valid_column":6,
@@ -397,7 +363,7 @@
} }
}, },
{ {
"table_id":34, "table_id":33,
"table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA", "table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA",
"table_type":"fqdn_plugin", "table_type":"fqdn_plugin",
"valid_column":5, "valid_column":5,
@@ -408,7 +374,7 @@
} }
}, },
{ {
"table_id":35, "table_id":34,
"table_name":"APP_ID", "table_name":"APP_ID",
"table_type":"intval", "table_type":"intval",
"valid_column":5, "valid_column":5,
@@ -420,12 +386,11 @@
} }
}, },
{ {
"table_id":36, "table_id":35,
"table_name":"EMPTY_KEYWORD", "table_name":"EMPTY_KEYWORD",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
"custom": { "custom": {
"pattern_type":"literal",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"keywords":3, "keywords":3,
@@ -435,7 +400,7 @@
} }
}, },
{ {
"table_id":37, "table_id":36,
"table_name":"EMPTY_INTERGER", "table_name":"EMPTY_INTERGER",
"table_type":"intval", "table_type":"intval",
"valid_column":5, "valid_column":5,
@@ -447,7 +412,7 @@
} }
}, },
{ {
"table_id":38, "table_id":37,
"table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA", "table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA",
"table_type":"bool_plugin", "table_type":"bool_plugin",
"valid_column":4, "valid_column":4,
@@ -457,7 +422,7 @@
} }
}, },
{ {
"table_id":39, "table_id":38,
"table_name":"FLAG_CONFIG", "table_name":"FLAG_CONFIG",
"table_type":"flag", "table_type":"flag",
"valid_column":5, "valid_column":5,
@@ -469,7 +434,7 @@
} }
}, },
{ {
"table_id":40, "table_id":39,
"table_name":"FLAG_PLUS_CONFIG", "table_name":"FLAG_PLUS_CONFIG",
"table_type":"flag_plus", "table_type":"flag_plus",
"valid_column":6, "valid_column":6,
@@ -482,7 +447,7 @@
} }
}, },
{ {
"table_id":41, "table_id":40,
"table_name":"PORT_CONFIG", "table_name":"PORT_CONFIG",
"table_type":"port", "table_type":"port",
"valid_column":7, "valid_column":7,
@@ -496,19 +461,19 @@
} }
}, },
{ {
"table_id":42, "table_id":41,
"table_name":"VIRTUAL_PORT_SOURCE", "table_name":"VIRTUAL_PORT_SOURCE",
"table_type":"virtual", "table_type":"virtual",
"physical_table": ["PORT_CONFIG"] "physical_table": "PORT_CONFIG"
},
{
"table_id":42,
"table_name":"VIRTUAL_PORT_DESTINATION",
"table_type":"virtual",
"physical_table": "PORT_CONFIG"
}, },
{ {
"table_id":43, "table_id":43,
"table_name":"VIRTUAL_PORT_DESTINATION",
"table_type":"virtual",
"physical_table": ["PORT_CONFIG"]
},
{
"table_id":44,
"table_name":"COMPOSITION_CONFIG", "table_name":"COMPOSITION_CONFIG",
"table_type":"composition", "table_type":"composition",
"custom": { "custom": {
@@ -519,7 +484,7 @@
} }
}, },
{ {
"table_id":45, "table_id":44,
"table_name":"TEST_PLUGIN_KEY_TYPE_TABLE", "table_name":"TEST_PLUGIN_KEY_TYPE_TABLE",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,