gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rollback to v4.0.31

Browse Source
This commit is contained in:
liuwentan
2023-08-09 19:22:09 +08:00
parent d29eef0423
commit fb0cb5405d
40 changed files with 1907 additions and 5448 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
include/maat.h
View File

@@ -37,7 +37,6 @@ struct maat_hit_path {
}; };
struct maat_hit_group { struct maat_hit_group {
long long item_id;
long long group_id; long long group_id;
int vtable_id; int vtable_id;
}; };
@@ -60,11 +59,6 @@ enum maat_list_type {
MAAT_LIST_TYPE_INC MAAT_LIST_TYPE_INC
}; };
enum maat_expr_engine {
MAAT_EXPR_ENGINE_HS = 0, //default engine(hyperscan)
MAAT_EXPR_ENGINE_RS //rulescan
};
struct ip_addr { struct ip_addr {
int ip_type; //4: IPv4, 6: IPv6 int ip_type; //4: IPv4, 6: IPv6
union { union {
@@ -150,8 +144,6 @@ int maat_options_set_redis(struct maat_options *opts, const char *redis_ip,
int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filename); int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filename);
int maat_options_set_expr_engine(struct maat_options *opts, enum maat_expr_engine engine);
/* maat_instance API */ /* maat_instance API */
struct maat *maat_new(struct maat_options *opts, const char *table_info_path); struct maat *maat_new(struct maat_options *opts, const char *table_info_path);
void maat_free(struct maat *instance); void maat_free(struct maat *instance);
@@ -279,9 +271,6 @@ int maat_state_set_last_scan(struct maat_state *state);
int maat_state_set_scan_compile_table(struct maat_state *state, int compile_table_id); int maat_state_set_scan_compile_table(struct maat_state *state, int compile_table_id);
int maat_state_get_compile_table_ids(struct maat_state *state, long long *compile_ids,
size_t n_compile_ids, int *compile_table_ids);
int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths, int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths,
size_t n_path); size_t n_path);
@@ -307,4 +296,4 @@ int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *grou
} }
#endif #endif
#endif #endif

5
scanner/CMakeLists.txt
View File

@@ -7,9 +7,8 @@ include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
add_subdirectory(ip_matcher/IntervalIndex) add_subdirectory(ip_matcher/IntervalIndex)
add_library(adapter-static bool_matcher/bool_matcher.cpp expr_matcher/expr_matcher.cpp add_library(adapter-static adapter_hs/adapter_hs.cpp bool_matcher/bool_matcher.cpp
expr_matcher/adapter_hs/adapter_hs.cpp expr_matcher/adapter_rs/adapter_rs.cpp
fqdn_engine/fqdn_engine.cpp ip_matcher/ip_matcher.cpp ip_matcher/ipv4_match.cpp fqdn_engine/fqdn_engine.cpp ip_matcher/ip_matcher.cpp ip_matcher/ipv4_match.cpp
ip_matcher/ipv6_match.cpp flag_matcher/flag_matcher.cpp interval_matcher/cgranges.c ip_matcher/ipv6_match.cpp flag_matcher/flag_matcher.cpp interval_matcher/cgranges.c
interval_matcher/interval_matcher.cpp) interval_matcher/interval_matcher.cpp)
target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static rulescan_static interval_index_static) target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static interval_index_static)

240
scanner/expr_matcher/adapter_hs/adapter_hs.cpp → scanner/adapter_hs/adapter_hs.cpp
View File

@@ -1,10 +1,10 @@
/* /*
********************************************************************************************** **********************************************************************************************
* File: adapter_hs.c * File: adapter_hs.cpp
* Description: * Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com> * Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31 * Date: 2022-10-31
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved. * Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
*********************************************************************************************** ***********************************************************************************************
*/ */
@@ -20,8 +20,9 @@
#include "uthash/utarray.h" #include "uthash/utarray.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "../../bool_matcher/bool_matcher.h" #include "../bool_matcher/bool_matcher.h"
#define MAX_OFFSET_NUM 1024
#define MAX_HIT_PATTERN_NUM 512 #define MAX_HIT_PATTERN_NUM 512
pid_t hs_gettid() pid_t hs_gettid()
@@ -40,7 +41,6 @@ static const char *hs_module_name_str(const char *name)
#define MODULE_ADAPTER_HS hs_module_name_str("maat.adapter_hs") #define MODULE_ADAPTER_HS hs_module_name_str("maat.adapter_hs")
struct adpt_hs_compile_data { struct adpt_hs_compile_data {
enum expr_pattern_type pat_type;
unsigned int *ids; unsigned int *ids;
unsigned int *flags; unsigned int *flags;
char **patterns; char **patterns;
@@ -56,6 +56,7 @@ struct adapter_hs_scratch {
struct adapter_hs_stream { struct adapter_hs_stream {
int thread_id; int thread_id;
size_t n_expr;
hs_stream_t *literal_stream; hs_stream_t *literal_stream;
hs_stream_t *regex_stream; hs_stream_t *regex_stream;
struct adapter_hs_runtime *ref_hs_rt; struct adapter_hs_runtime *ref_hs_rt;
@@ -90,7 +91,7 @@ struct pattern_offset {
struct pattern_attribute { struct pattern_attribute {
long long pattern_id; long long pattern_id;
enum expr_match_mode match_mode; enum hs_match_mode match_mode;
struct pattern_offset offset; struct pattern_offset offset;
}; };
@@ -136,12 +137,12 @@ static int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches,
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt,
size_t n_worker_thread, size_t n_worker_thread,
enum expr_pattern_type pattern_type, enum hs_pattern_type pattern_type,
struct log_handle *logger) struct log_handle *logger)
{ {
int ret = 0; int ret = 0;
if (pattern_type == EXPR_PATTERN_TYPE_STR) { if (pattern_type == HS_PATTERN_TYPE_STR) {
hs_rt->scratch->literal_scratches = ALLOC(hs_scratch_t *, n_worker_thread); hs_rt->scratch->literal_scratches = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->scratch->literal_scratches, ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->scratch->literal_scratches,
n_worker_thread, logger); n_worker_thread, logger);
@@ -199,7 +200,7 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
if (regex_cd != NULL) { if (regex_cd != NULL) {
err = hs_compile_multi((const char *const *)regex_cd->patterns, err = hs_compile_multi((const char *const *)regex_cd->patterns,
regex_cd->flags, regex_cd->ids, regex_cd->n_patterns, regex_cd->flags, regex_cd->ids, regex_cd->n_patterns,
HS_MODE_STREAM, HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL,
NULL, &hs_rt->regex_db, &compile_err); NULL, &hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
if (compile_err) { if (compile_err) {
@@ -214,11 +215,9 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
return 0; return 0;
} }
static struct adpt_hs_compile_data * static struct adpt_hs_compile_data *adpt_hs_compile_data_new(size_t n_patterns)
adpt_hs_compile_data_new(enum expr_pattern_type pat_type, size_t n_patterns)
{ {
struct adpt_hs_compile_data *hs_cd = ALLOC(struct adpt_hs_compile_data, 1); struct adpt_hs_compile_data *hs_cd = ALLOC(struct adpt_hs_compile_data, 1);
hs_cd->pat_type = pat_type;
hs_cd->patterns = ALLOC(char *, n_patterns); hs_cd->patterns = ALLOC(char *, n_patterns);
hs_cd->pattern_lens = ALLOC(size_t, n_patterns); hs_cd->pattern_lens = ALLOC(size_t, n_patterns);
hs_cd->n_patterns = n_patterns; hs_cd->n_patterns = n_patterns;
@@ -264,11 +263,8 @@ static void populate_compile_data(struct adpt_hs_compile_data *compile_data,
compile_data->ids[index] = pattern_id; compile_data->ids[index] = pattern_id;
/* set flags */ /* set flags */
if (compile_data->pat_type == EXPR_PATTERN_TYPE_STR) { compile_data->flags[index] |= HS_FLAG_SOM_LEFTMOST;
compile_data->flags[index] |= HS_FLAG_SOM_LEFTMOST; if (case_sensitive == HS_CASE_INSENSITIVE) {
}
if (case_sensitive == EXPR_CASE_INSENSITIVE) {
compile_data->flags[index] |= HS_FLAG_CASELESS; compile_data->flags[index] |= HS_FLAG_CASELESS;
} }
@@ -299,14 +295,14 @@ static struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule,
pattern_attr[pattern_index].pattern_id = pattern_index; pattern_attr[pattern_index].pattern_id = pattern_index;
pattern_attr[pattern_index].match_mode = rules[i].patterns[j].match_mode; pattern_attr[pattern_index].match_mode = rules[i].patterns[j].match_mode;
if (pattern_attr[pattern_index].match_mode == EXPR_MATCH_MODE_SUB || if (pattern_attr[pattern_index].match_mode == HS_MATCH_MODE_SUB ||
pattern_attr[pattern_index].match_mode == EXPR_MATCH_MODE_EXACTLY) { pattern_attr[pattern_index].match_mode == HS_MATCH_MODE_EXACTLY) {
pattern_attr[pattern_index].offset.start = rules[i].patterns[j].start_offset; pattern_attr[pattern_index].offset.start = rules[i].patterns[j].start_offset;
pattern_attr[pattern_index].offset.end = rules[i].patterns[j].end_offset; pattern_attr[pattern_index].offset.end = rules[i].patterns[j].end_offset;
} }
/* literal pattern */ /* literal pattern */
if (rules[i].patterns[j].type == EXPR_PATTERN_TYPE_STR) { if (rules[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) {
populate_compile_data(literal_cd, literal_index, pattern_index, populate_compile_data(literal_cd, literal_index, pattern_index,
rules[i].patterns[j].pat, rules[i].patterns[j].pat_len, rules[i].patterns[j].pat, rules[i].patterns[j].pat_len,
rules[i].patterns[j].case_sensitive); rules[i].patterns[j].case_sensitive);
@@ -325,7 +321,7 @@ static struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule,
bool_exprs[i].expr_id = rules[i].expr_id; bool_exprs[i].expr_id = rules[i].expr_id;
bool_exprs[i].item_num = rules[i].n_patterns; bool_exprs[i].item_num = rules[i].n_patterns;
bool_exprs[i].user_tag = rules[i].tag; bool_exprs[i].user_tag = rules[i].user_tag;
} }
*n_pattern = pattern_index; *n_pattern = pattern_index;
@@ -349,43 +345,81 @@ static int verify_regex_expression(const char *regex_str, struct log_handle *log
FREE(info); FREE(info);
hs_free_compile_error(error); hs_free_compile_error(error);
return 0; return -1;
} }
if (info != NULL) { if (info != NULL) {
FREE(info); FREE(info);
} }
return 1; return 0;
} }
int adapter_hs_verify_regex_expression(const char *regex_expr, struct log_handle *logger) int adapter_hs_verify_regex_expression(const char *regex_expr, struct log_handle *logger)
{ {
if (NULL == regex_expr) { if (NULL == regex_expr) {
return 0; return -1;
} }
return verify_regex_expression(regex_expr, logger); return verify_regex_expression(regex_expr, logger);
} }
void *adapter_hs_new(struct expr_rule *rules, size_t n_rule, struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern, size_t n_worker_thread, struct log_handle *logger)
size_t n_worker_thread, struct log_handle *logger)
{ {
if (0 == n_worker_thread || NULL == rules || 0 == n_rule) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] input parameters illegal!", __FUNCTION__, __LINE__);
return NULL;
}
/* get the sum of pattern */ /* get the sum of pattern */
size_t i = 0; size_t i = 0, j = 0;
size_t literal_pattern_num = 0;
size_t regex_pattern_num = 0;
for (i = 0; i < n_rule; i++) {
if (rules[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] the number of patterns in one expression "
"should less than %d", __FUNCTION__, __LINE__,
MAX_EXPR_PATTERN_NUM);
return NULL;
}
for (j = 0; j < rules[i].n_patterns; j++) {
/* pat_len should not 0 */
if (0 == rules[i].patterns[j].pat_len) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] expr pattern length should not 0",
__FUNCTION__, __LINE__);
return NULL;
}
if (rules[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) {
literal_pattern_num++;
} else {
regex_pattern_num++;
}
}
}
if (0 == literal_pattern_num && 0 == regex_pattern_num) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] exprs has no valid pattern", __FUNCTION__, __LINE__);
return NULL;
}
struct adpt_hs_compile_data *literal_cd = NULL; struct adpt_hs_compile_data *literal_cd = NULL;
struct adpt_hs_compile_data *regex_cd = NULL; struct adpt_hs_compile_data *regex_cd = NULL;
if (literal_pattern_num > 0) {
if (n_literal_pattern > 0) { literal_cd = adpt_hs_compile_data_new(literal_pattern_num);
literal_cd = adpt_hs_compile_data_new(EXPR_PATTERN_TYPE_STR, n_literal_pattern);
} }
if (n_regex_pattern > 0) { if (regex_pattern_num > 0) {
regex_cd = adpt_hs_compile_data_new(EXPR_PATTERN_TYPE_REG, n_regex_pattern); regex_cd = adpt_hs_compile_data_new(regex_pattern_num);
} }
size_t pattern_cnt = n_literal_pattern + n_regex_pattern; size_t pattern_cnt = literal_pattern_num + regex_pattern_num;
struct adapter_hs *hs_inst = ALLOC(struct adapter_hs, 1); struct adapter_hs *hs_inst = ALLOC(struct adapter_hs, 1);
hs_inst->hs_attr = ALLOC(struct pattern_attribute, pattern_cnt); hs_inst->hs_attr = ALLOC(struct pattern_attribute, pattern_cnt);
hs_inst->logger = logger; hs_inst->logger = logger;
@@ -444,21 +478,21 @@ void *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
n_worker_thread); n_worker_thread);
for (i = 0; i < n_worker_thread; i++) { for (i = 0; i < n_worker_thread; i++) {
hs_inst->hs_rt->scratch->bool_match_buffs[i] = ALLOC(struct bool_expr_match, hs_inst->hs_rt->scratch->bool_match_buffs[i] = ALLOC(struct bool_expr_match,
MAX_HIT_EXPR_NUM); hs_inst->n_expr);
} }
/* literal and regex scratch can't reuse */ /* literal and regex scratch can't reuse */
if (n_literal_pattern > 0) { if (literal_pattern_num > 0) {
ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread, ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread,
EXPR_PATTERN_TYPE_STR, logger); HS_PATTERN_TYPE_STR, logger);
if (ret < 0) { if (ret < 0) {
goto error; goto error;
} }
} }
if (n_regex_pattern > 0) { if (regex_pattern_num > 0) {
ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread, ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread,
EXPR_PATTERN_TYPE_REG, logger); HS_PATTERN_TYPE_REG, logger);
if (ret < 0) { if (ret < 0) {
goto error; goto error;
} }
@@ -466,7 +500,7 @@ void *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
hs_inst->hs_rt->streams = ALLOC(struct adapter_hs_stream *, n_worker_thread); hs_inst->hs_rt->streams = ALLOC(struct adapter_hs_stream *, n_worker_thread);
for (i = 0; i < n_worker_thread; i++) { for (i = 0; i < n_worker_thread; i++) {
hs_inst->hs_rt->streams[i] = (struct adapter_hs_stream *)adapter_hs_stream_open(hs_inst, i); hs_inst->hs_rt->streams[i] = adapter_hs_stream_open(hs_inst, i);
} }
return hs_inst; return hs_inst;
@@ -475,15 +509,13 @@ error:
return NULL; return NULL;
} }
void adapter_hs_free(void *hs_instance) void adapter_hs_free(struct adapter_hs *hs_inst)
{ {
if (NULL == hs_instance) { if (NULL == hs_inst) {
return; return;
} }
struct adapter_hs *hs_inst = (struct adapter_hs *)hs_instance;
size_t i = 0; size_t i = 0;
if (hs_inst->hs_rt != NULL) { if (hs_inst->hs_rt != NULL) {
if (hs_inst->hs_rt->literal_db != NULL) { if (hs_inst->hs_rt->literal_db != NULL) {
hs_free_database(hs_inst->hs_rt->literal_db); hs_free_database(hs_inst->hs_rt->literal_db);
@@ -593,12 +625,12 @@ static int matched_event_cb(unsigned int id, unsigned long long from,
int ret = 0; int ret = 0;
struct pattern_attribute pat_attr = matched_pat->ref_hs_attr[id]; struct pattern_attribute pat_attr = matched_pat->ref_hs_attr[id];
switch (pat_attr.match_mode) { switch (pat_attr.match_mode) {
case EXPR_MATCH_MODE_EXACTLY: case HS_MATCH_MODE_EXACTLY:
if (0 == from && matched_pat->scan_data_len == to) { if (0 == from && matched_pat->scan_data_len == to) {
ret = 1; ret = 1;
} }
break; break;
case EXPR_MATCH_MODE_SUB: case HS_MATCH_MODE_SUB:
if (pat_attr.offset.start == -1 && if (pat_attr.offset.start == -1 &&
pat_attr.offset.end == -1) { pat_attr.offset.end == -1) {
ret = 1; ret = 1;
@@ -624,12 +656,12 @@ static int matched_event_cb(unsigned int id, unsigned long long from,
ret = 1; ret = 1;
} }
break; break;
case EXPR_MATCH_MODE_PREFIX: case HS_MATCH_MODE_PREFIX:
if (0 == from) { if (0 == from) {
ret = 1; ret = 1;
} }
break; break;
case EXPR_MATCH_MODE_SUFFIX: case HS_MATCH_MODE_SUFFIX:
if (to == matched_pat->scan_data_len) { if (to == matched_pat->scan_data_len) {
ret = 1; ret = 1;
} }
@@ -646,42 +678,43 @@ static int matched_event_cb(unsigned int id, unsigned long long from,
return 0; return 0;
} }
UT_icd ut_hs_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
void *adapter_hs_stream_open(void *hs_instance, int thread_id) struct adapter_hs_stream *
adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id)
{ {
if (NULL == hs_instance || thread_id < 0) { if (NULL == hs_instance || thread_id < 0) {
return NULL; return NULL;
} }
struct adapter_hs *hs_inst = (struct adapter_hs *)hs_instance;
struct adapter_hs_stream *hs_stream = ALLOC(struct adapter_hs_stream, 1); struct adapter_hs_stream *hs_stream = ALLOC(struct adapter_hs_stream, 1);
hs_error_t err; hs_error_t err;
hs_stream->logger = hs_inst->logger; hs_stream->logger = hs_instance->logger;
hs_stream->thread_id = thread_id; hs_stream->thread_id = thread_id;
hs_stream->ref_hs_rt = hs_inst->hs_rt; hs_stream->n_expr = hs_instance->n_expr;
hs_stream->ref_hs_rt = hs_instance->hs_rt;
hs_stream->matched_pat = ALLOC(struct matched_pattern, 1); hs_stream->matched_pat = ALLOC(struct matched_pattern, 1);
hs_stream->matched_pat->ref_hs_attr = hs_inst->hs_attr; hs_stream->matched_pat->ref_hs_attr = hs_instance->hs_attr;
hs_stream->matched_pat->n_patterns = hs_inst->n_patterns; hs_stream->matched_pat->n_patterns = hs_instance->n_patterns;
utarray_new(hs_stream->matched_pat->pattern_ids, &ut_hs_pattern_id_icd); utarray_new(hs_stream->matched_pat->pattern_ids, &ut_pattern_id_icd);
utarray_reserve(hs_stream->matched_pat->pattern_ids, MAX_HIT_PATTERN_NUM); utarray_reserve(hs_stream->matched_pat->pattern_ids, MAX_HIT_PATTERN_NUM);
int err_count = 0; int err_count = 0;
if (hs_inst->hs_rt->literal_db != NULL) { if (hs_instance->hs_rt->literal_db != NULL) {
err = hs_open_stream(hs_inst->hs_rt->literal_db, 0, err = hs_open_stream(hs_instance->hs_rt->literal_db, 0,
&hs_stream->literal_stream); &hs_stream->literal_stream);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(hs_inst->logger, MODULE_ADAPTER_HS, log_error(hs_instance->logger, MODULE_ADAPTER_HS,
"hs_open_stream failed, hs err:%d", err); "hs_open_stream failed, hs err:%d", err);
err_count++; err_count++;
} }
} }
if (hs_inst->hs_rt->regex_db != NULL) { if (hs_instance->hs_rt->regex_db != NULL) {
err = hs_open_stream(hs_inst->hs_rt->regex_db, 0, err = hs_open_stream(hs_instance->hs_rt->regex_db, 0,
&hs_stream->regex_stream); &hs_stream->regex_stream);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
log_error(hs_inst->logger, MODULE_ADAPTER_HS, log_error(hs_instance->logger, MODULE_ADAPTER_HS,
"hs_open_stream failed, hs err:%d", err); "hs_open_stream failed, hs err:%d", err);
err_count++; err_count++;
} }
@@ -707,37 +740,36 @@ error:
return NULL; return NULL;
} }
void adapter_hs_stream_close(void *hs_stream) void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
{ {
if (NULL == hs_stream) { if (NULL == hs_stream) {
return; return;
} }
struct adapter_hs_stream *stream = (struct adapter_hs_stream *)hs_stream; if (hs_stream->ref_hs_rt != NULL) {
if (stream->ref_hs_rt != NULL) { if (hs_stream->literal_stream != NULL) {
if (stream->literal_stream != NULL) { hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL);
hs_close_stream(stream->literal_stream, NULL, NULL, NULL); hs_stream->literal_stream = NULL;
stream->literal_stream = NULL;
} }
if (stream->regex_stream != NULL) { if (hs_stream->regex_stream != NULL) {
hs_close_stream(stream->regex_stream, NULL, NULL, NULL); hs_close_stream(hs_stream->regex_stream, NULL, NULL, NULL);
stream->regex_stream = NULL; hs_stream->regex_stream = NULL;
} }
} }
/* stream->hs_rt point to hs_instance->hs_rt which will call free /* hs_stream->hs_rt point to hs_instance->hs_rt which will call free
same as hs_attr */ same as hs_attr */
stream->ref_hs_rt = NULL; hs_stream->ref_hs_rt = NULL;
stream->matched_pat->ref_hs_attr = NULL; hs_stream->matched_pat->ref_hs_attr = NULL;
if (stream->matched_pat->pattern_ids != NULL) { if (hs_stream->matched_pat->pattern_ids != NULL) {
utarray_free(stream->matched_pat->pattern_ids); utarray_free(hs_stream->matched_pat->pattern_ids);
stream->matched_pat->pattern_ids = NULL; hs_stream->matched_pat->pattern_ids = NULL;
} }
FREE(stream->matched_pat); FREE(hs_stream->matched_pat);
FREE(stream); FREE(hs_stream);
} }
static void adapter_hs_stream_reset(struct adapter_hs_stream *hs_stream) static void adapter_hs_stream_reset(struct adapter_hs_stream *hs_stream)
@@ -762,9 +794,9 @@ static void adapter_hs_stream_reset(struct adapter_hs_stream *hs_stream)
utarray_clear(hs_stream->matched_pat->pattern_ids); utarray_clear(hs_stream->matched_pat->pattern_ids);
} }
int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len, int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data,
struct expr_scan_result *results, size_t n_result, size_t data_len, struct hs_scan_result *results,
size_t *n_hit_result) size_t n_result, size_t *n_hit_result)
{ {
hs_error_t err; hs_error_t err;
@@ -784,37 +816,36 @@ int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len,
*/ */
int err_count = 0; int err_count = 0;
struct adapter_hs_stream *stream = (struct adapter_hs_stream *)hs_stream; int thread_id = hs_stream->thread_id;
int thread_id = stream->thread_id; struct adapter_hs_scratch *scratch = hs_stream->ref_hs_rt->scratch;
struct adapter_hs_scratch *scratch = stream->ref_hs_rt->scratch; hs_stream->matched_pat->scan_data_len = data_len;
stream->matched_pat->scan_data_len = data_len;
int err_scratch_flag = 0; int err_scratch_flag = 0;
if (stream->literal_stream != NULL) { if (hs_stream->literal_stream != NULL) {
if (scratch->literal_scratches != NULL) { if (scratch->literal_scratches != NULL) {
err = hs_scan_stream(stream->literal_stream, data, data_len, err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
0, scratch->literal_scratches[thread_id], 0, scratch->literal_scratches[thread_id],
matched_event_cb, stream->matched_pat); matched_event_cb, hs_stream->matched_pat);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
err_count++; err_count++;
} }
} else { } else {
log_error(stream->logger, MODULE_ADAPTER_HS, log_error(hs_stream->logger, MODULE_ADAPTER_HS,
"literal_scratches is null, thread_id:%d", thread_id); "literal_scratches is null, thread_id:%d", thread_id);
err_scratch_flag++; err_scratch_flag++;
} }
} }
if (stream->regex_stream != NULL) { if (hs_stream->regex_stream != NULL) {
if (scratch->regex_scratches != NULL) { if (scratch->regex_scratches != NULL) {
err = hs_scan_stream(stream->regex_stream, data, data_len, err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
0, scratch->regex_scratches[thread_id], 0, scratch->regex_scratches[thread_id],
matched_event_cb, stream->matched_pat); matched_event_cb, hs_stream->matched_pat);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
err_count++; err_count++;
} }
} else { } else {
log_error(stream->logger, MODULE_ADAPTER_HS, log_error(hs_stream->logger, MODULE_ADAPTER_HS,
"regex_scratches is null, thread_id:%d", thread_id); "regex_scratches is null, thread_id:%d", thread_id);
err_scratch_flag++; err_scratch_flag++;
} }
@@ -828,7 +859,7 @@ int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len,
return -1; return -1;
} }
size_t n_pattern_id = utarray_len(stream->matched_pat->pattern_ids); size_t n_pattern_id = utarray_len(hs_stream->matched_pat->pattern_ids);
if (0 == n_pattern_id) { if (0 == n_pattern_id) {
*n_hit_result = 0; *n_hit_result = 0;
return 0; return 0;
@@ -837,13 +868,13 @@ int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len,
unsigned long long pattern_ids[n_pattern_id]; unsigned long long pattern_ids[n_pattern_id];
for (size_t i = 0; i < n_pattern_id; i++) { for (size_t i = 0; i < n_pattern_id; i++) {
pattern_ids[i] = *(unsigned long long *)utarray_eltptr(stream->matched_pat->pattern_ids, i); pattern_ids[i] = *(unsigned long long *)utarray_eltptr(hs_stream->matched_pat->pattern_ids, i);
} }
int ret = 0; int ret = 0;
struct bool_expr_match *bool_matcher_results = scratch->bool_match_buffs[thread_id]; struct bool_expr_match *bool_matcher_results = scratch->bool_match_buffs[thread_id];
int bool_matcher_ret = bool_matcher_match(stream->ref_hs_rt->bm, pattern_ids, n_pattern_id, int bool_matcher_ret = bool_matcher_match(hs_stream->ref_hs_rt->bm, pattern_ids, n_pattern_id,
bool_matcher_results, MAX_HIT_EXPR_NUM); bool_matcher_results, hs_stream->n_expr);
if (bool_matcher_ret < 0) { if (bool_matcher_ret < 0) {
ret = -1; ret = -1;
goto next; goto next;
@@ -860,21 +891,22 @@ int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len,
*n_hit_result = bool_matcher_ret; *n_hit_result = bool_matcher_ret;
next: next:
utarray_clear(stream->matched_pat->pattern_ids); utarray_clear(hs_stream->matched_pat->pattern_ids);
return ret; return ret;
} }
int adapter_hs_scan(void *hs_instance, int thread_id, const char *data, size_t data_len, int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
struct expr_scan_result *results, size_t n_result, size_t *n_hit_result) const char *data, size_t data_len,
struct hs_scan_result *results,
size_t n_result, size_t *n_hit_result)
{ {
if (NULL == hs_instance || NULL == data || (0 == data_len) || if (NULL == hs_instance || NULL == data || (0 == data_len) ||
NULL == results || 0 == n_result || NULL == n_hit_result) { NULL == results || 0 == n_result || NULL == n_hit_result) {
return -1; return -1;
} }
struct adapter_hs *hs_inst = (struct adapter_hs *)hs_instance; struct adapter_hs_stream *hs_stream = hs_instance->hs_rt->streams[thread_id];
struct adapter_hs_stream *hs_stream = hs_inst->hs_rt->streams[thread_id];
assert(hs_stream != NULL); assert(hs_stream != NULL);
adapter_hs_stream_reset(hs_stream); adapter_hs_stream_reset(hs_stream);

134
scanner/adapter_hs/adapter_hs.h Normal file
View File

@@ -0,0 +1,134 @@
/*
**********************************************************************************************
* File: adapter_hs.h
* Description: wrapper for raw hyperscan
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _ADAPTER_HS_H_
#define _ADAPTER_HS_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include <stdint.h>
#include "log/log.h"
#define MAX_EXPR_PATTERN_NUM 8
struct adapter_hs;
/* match method */
enum hs_match_mode {
HS_MATCH_MODE_INVALID = -1,
HS_MATCH_MODE_EXACTLY = 1, /* scan data must match pattern exactly */
HS_MATCH_MODE_PREFIX, /* pattern must in the head of scan_data */
HS_MATCH_MODE_SUFFIX, /* pattern must in the end of scan_data */
HS_MATCH_MODE_SUB /* pattern must in the range[l_offset, r_offset] of scan_data */
};
enum hs_pattern_type {
HS_PATTERN_TYPE_STR = 0, /* pure literal string */
HS_PATTERN_TYPE_REG /* regex expression */
};
enum hs_case_sensitive {
HS_CASE_SENSITIVE = 0,
HS_CASE_INSENSITIVE
};
struct hs_scan_result {
long long rule_id;
void *user_tag;
};
struct hs_pattern {
enum hs_case_sensitive case_sensitive;
enum hs_match_mode match_mode;
enum hs_pattern_type pattern_type;
int is_hexbin; /* 1(yes) 0(no) */
/*
* just match in scan_data's range of [start_offset, end_offset], -1 means no limits
* for example:
* [-1, end_offset] means the pattern must in scan_data's [0 ~ start_offset]
* [start_offset, -1] means the pattern must in scan_data's [start_offset ~ data_end]
*/
int start_offset;
int end_offset;
/* start pointer of pattern */
char *pat;
/* pattern length */
size_t pat_len;
};
/* logic AND expression, such as (pattern1 & pattern2) */
struct expr_rule {
long long expr_id;
size_t n_patterns;
struct hs_pattern patterns[MAX_EXPR_PATTERN_NUM];
void *user_tag;
};
int adapter_hs_verify_regex_expression(const char *regex_expr,
struct log_handle *logger);
/**
* @brief new adapter_hs instance
*
* @param rules: logic AND expression's array
* @param n_rule: the number of logic AND expression's array
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
*
* @retval the pointer to adapter_hs instance
*/
struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
size_t n_worker_thread, struct log_handle *logger);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
*
* @param instance: adapter_hs instance obtained by adapter_hs_new()
* @param thread_id: the thread_id of caller
* @param data: data to be scanned
* @param data_len: the length of data to be scanned
* @param results: the array of expr_id
* @param n_results: number of elements in array of expr_id
*/
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
const char *data, size_t data_len,
struct hs_scan_result *results,
size_t n_result, size_t *n_hit_result);
/**
* @brief destroy adapter_hs instance
*
* @param instance: adapter_hs instance obtained by adapter_hs_new()
*/
void adapter_hs_free(struct adapter_hs *instance);
struct adapter_hs_stream;
/**
* @brief open adapter_hs stream after adapter_hs instance initialized for stream scan
*
*/
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id);
int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, size_t data_len,
struct hs_scan_result *results, size_t n_result, size_t *n_hit_result);
void adapter_hs_stream_close(struct adapter_hs_stream *stream);
#ifdef __cplusplus
}
#endif
#endif

75
scanner/expr_matcher/adapter_hs/adapter_hs.h
View File

@@ -1,75 +0,0 @@
/*
**********************************************************************************************
* File: adapter_hs.h
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#ifndef _ADAPTER_HS_H_
#define _ADAPTER_HS_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include <stdint.h>
#include "log/log.h"
#include "../expr_matcher.h"
int adapter_hs_verify_regex_expression(const char *regex_expr, struct log_handle *logger);
/**
* @brief new adapter_hs instance
*
* @param rules: logic AND expression's array
* @param n_rule: the number of logic AND expression's array
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
*
* @retval the pointer to adapter_hs instance
*/
void *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern,
size_t n_worker_thread, struct log_handle *logger);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
*
* @param instance: adapter_hs instance obtained by adapter_hs_new()
* @param thread_id: the thread_id of caller
* @param data: data to be scanned
* @param data_len: the length of data to be scanned
* @param results: the array of expr_id
* @param n_results: number of elements in array of expr_id
*/
int adapter_hs_scan(void *hs_instance, int thread_id, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result, size_t *n_hit_result);
/**
* @brief destroy adapter_hs instance
*
* @param instance: adapter_hs instance obtained by adapter_hs_new()
*/
void adapter_hs_free(void *instance);
/**
* @brief open adapter_hs stream after adapter_hs instance initialized for stream scan
*
*/
void *adapter_hs_stream_open(void *hs_instance, int thread_id);
int adapter_hs_scan_stream(void *stream, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result,
size_t *n_hit_result);
void adapter_hs_stream_close(void *stream);
#ifdef __cplusplus
}
#endif
#endif

708
scanner/expr_matcher/adapter_rs/adapter_rs.cpp
View File

@@ -1,708 +0,0 @@
/*
**********************************************************************************************
* File: adapter_rs.cpp
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#include <stdint.h>
#include <stdio.h>
#include <stddef.h>
#include <assert.h>
#include <unistd.h>
#include <sys/syscall.h>
#include "rulescan.h"
#include "adapter_rs.h"
#include "uthash/utarray.h"
#include "uthash/uthash.h"
#include "maat_utils.h"
#include "../../bool_matcher/bool_matcher.h"
#define MAX_HIT_PATTERN_NUM 512
pid_t rs_gettid()
{
return syscall(SYS_gettid);
}
static const char *rs_module_name_str(const char *name)
{
static __thread char module[64];
snprintf(module, sizeof(module), "%s(%d)", name, rs_gettid());
return module;
}
#define MODULE_ADAPTER_RS rs_module_name_str("maat.adapter_rs")
struct adpt_rs_compile_data {
struct scan_pattern *patterns;
size_t n_patterns;
};
struct adapter_rs_stream {
int thread_id;
size_t offset; /* current stream offset */
rs_stream_t *literal_stream;
rs_stream_t *regex_stream;
struct adapter_rs_runtime *ref_rs_rt;
struct log_handle *logger;
};
/* adapter_rs runtime */
struct adapter_rs_runtime {
rs_database_t *literal_db;
rs_database_t *regex_db;
struct bool_expr_match **bool_match_buffs; /* per thread */
struct adapter_rs_stream **streams; /* per thread */
struct matched_pattern **matched_pats; /* per thread */
struct bool_matcher *bm;
};
/* adapter_rs instance */
struct adapter_rs {
size_t n_worker_thread;
size_t n_expr;
size_t n_patterns;
struct adapter_rs_runtime *rs_rt;
struct pattern_attribute *rs_attr;
struct log_handle *logger;
};
struct pattern_offset {
long long start;
long long end;
};
struct pattern_attribute {
long long pattern_id;
enum expr_match_mode match_mode;
struct pattern_offset offset;
size_t pattern_len;
};
struct matched_pattern {
UT_array *pattern_ids;
size_t n_patterns;
struct pattern_attribute *ref_rs_attr;
};
int adapter_rs_verify_regex_expression(const char *regex_expr,
struct log_handle *logger)
{
int ret = rs_verify_regex(regex_expr);
if (ret == 0) {
log_error(logger, MODULE_ADAPTER_RS,
"[%s:%d] illegal regex expression: \"%s\"",
__FUNCTION__, __LINE__, regex_expr);
}
return ret;
}
/**
* @brief build rs database for literal string and regex expression respectively
*
* @retval 0(success) -1(failed)
*/
static int adpt_rs_build_database(struct adapter_rs_runtime *rs_rt,
size_t n_worker_thread,
struct adpt_rs_compile_data *literal_cd,
struct adpt_rs_compile_data *regex_cd,
struct log_handle *logger)
{
if (NULL == rs_rt) {
return -1;
}
int ret = 0;
if (literal_cd != NULL) {
ret = rs_compile_lit(literal_cd->patterns, literal_cd->n_patterns,
&rs_rt->literal_db);
if (ret < 0) {
log_error(logger, MODULE_ADAPTER_RS, "[%s:%d] compile error",
__FUNCTION__, __LINE__);
return -1;
}
}
if (regex_cd != NULL) {
size_t n_failed_pats = 0;
ret = rs_compile_regex(regex_cd->patterns, regex_cd->n_patterns,
n_worker_thread, &rs_rt->regex_db, &n_failed_pats);
if (ret < 0) {
log_error(logger, MODULE_ADAPTER_RS, "[%s:%d] compile error",
__FUNCTION__, __LINE__);
return -1;
}
}
return 0;
}
static struct adpt_rs_compile_data *adpt_rs_compile_data_new(size_t n_patterns)
{
struct adpt_rs_compile_data *rs_cd = ALLOC(struct adpt_rs_compile_data, 1);
rs_cd->patterns = ALLOC(struct scan_pattern, n_patterns);
rs_cd->n_patterns = n_patterns;
return rs_cd;
}
static void adpt_rs_compile_data_free(struct adpt_rs_compile_data *rs_cd)
{
if (NULL == rs_cd) {
return;
}
if (rs_cd->patterns != NULL) {
for (size_t i = 0; i < rs_cd->n_patterns; i++) {
if (rs_cd->patterns[i].pattern != NULL) {
FREE(rs_cd->patterns[i].pattern);
}
}
FREE(rs_cd->patterns);
}
FREE(rs_cd);
}
static void populate_compile_data(struct adpt_rs_compile_data *compile_data,
size_t index, long long pattern_id, char *pat,
size_t pat_len, int case_sensitive)
{
compile_data->patterns[index].id = pattern_id;
compile_data->patterns[index].case_sensitive = case_sensitive;
compile_data->patterns[index].pattern = ALLOC(char, pat_len + 1);
memcpy(compile_data->patterns[index].pattern, pat, pat_len);
compile_data->patterns[index].pattern_len = pat_len;
}
static struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule,
struct pattern_attribute *pattern_attr,
struct adpt_rs_compile_data *literal_cd,
struct adpt_rs_compile_data *regex_cd,
size_t *n_pattern)
{
long long pattern_idx = 0;
size_t literal_idx = 0;
size_t regex_idx = 0;
struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_rule);
/* populate adpt_rs_compile_data and bool_expr */
for (size_t i = 0; i < n_rule; i++) {
for (size_t j = 0; j < rules[i].n_patterns; j++) {
pattern_attr[pattern_idx].pattern_id = pattern_idx;
pattern_attr[pattern_idx].match_mode = rules[i].patterns[j].match_mode;
pattern_attr[pattern_idx].pattern_len = rules[i].patterns[j].pat_len;
if (pattern_attr[pattern_idx].match_mode == EXPR_MATCH_MODE_SUB ||
pattern_attr[pattern_idx].match_mode == EXPR_MATCH_MODE_EXACTLY) {
pattern_attr[pattern_idx].offset.start = rules[i].patterns[j].start_offset;
pattern_attr[pattern_idx].offset.end = rules[i].patterns[j].end_offset;
}
/* literal pattern */
if (rules[i].patterns[j].type == EXPR_PATTERN_TYPE_STR) {
populate_compile_data(literal_cd, literal_idx, pattern_idx,
rules[i].patterns[j].pat, rules[i].patterns[j].pat_len,
rules[i].patterns[j].case_sensitive);
literal_idx++;
} else {
/* regex pattern */
populate_compile_data(regex_cd, regex_idx, pattern_idx,
rules[i].patterns[j].pat, rules[i].patterns[j].pat_len,
rules[i].patterns[j].case_sensitive);
regex_idx++;
}
bool_exprs[i].items[j].item_id = pattern_idx++;
bool_exprs[i].items[j].not_flag = 0;
}
bool_exprs[i].expr_id = rules[i].expr_id;
bool_exprs[i].item_num = rules[i].n_patterns;
bool_exprs[i].user_tag = rules[i].tag;
}
*n_pattern = pattern_idx;
return bool_exprs;
}
UT_icd ut_rs_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
void *adapter_rs_new(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern,
size_t n_worker_thread, struct log_handle *logger)
{
/* get the sum of pattern */
size_t i = 0;
struct adpt_rs_compile_data *literal_cd = NULL;
struct adpt_rs_compile_data *regex_cd = NULL;
if (n_literal_pattern > 0) {
literal_cd = adpt_rs_compile_data_new(n_literal_pattern);
}
if (n_regex_pattern > 0) {
regex_cd = adpt_rs_compile_data_new(n_regex_pattern);
}
size_t pattern_cnt = n_literal_pattern + n_regex_pattern;
struct adapter_rs *rs_inst = ALLOC(struct adapter_rs, 1);
rs_inst->rs_attr = ALLOC(struct pattern_attribute, pattern_cnt);
rs_inst->logger = logger;
rs_inst->n_worker_thread = n_worker_thread;
rs_inst->n_expr = n_rule;
struct bool_expr *bool_exprs = bool_exprs_new(rules, n_rule, rs_inst->rs_attr,
literal_cd, regex_cd, &pattern_cnt);
if (NULL == bool_exprs) {
return NULL;
}
rs_inst->n_patterns = pattern_cnt;
/* create bool matcher */
size_t mem_size = 0;
int rs_ret = 0;
rs_inst->rs_rt = ALLOC(struct adapter_rs_runtime, 1);
//rs_rt->bm
rs_inst->rs_rt->bm = bool_matcher_new(bool_exprs, n_rule, &mem_size);
if (rs_inst->rs_rt->bm != NULL) {
log_info(logger, MODULE_ADAPTER_RS,
"Adapter_rs module: build bool matcher of %zu expressions"
" with %zu bytes memory", n_rule, mem_size);
} else {
log_error(logger, MODULE_ADAPTER_RS,
"[%s:%d] Adapter_rs module: build bool matcher failed",
__FUNCTION__, __LINE__);
rs_ret = -1;
}
FREE(bool_exprs);
/* build rs database rs_rt->literal_db & rs_rt->regex_db */
int ret = adpt_rs_build_database(rs_inst->rs_rt, n_worker_thread,
literal_cd, regex_cd, logger);
if (ret < 0) {
rs_ret = -1;
}
if (literal_cd != NULL) {
adpt_rs_compile_data_free(literal_cd);
}
if (regex_cd != NULL) {
adpt_rs_compile_data_free(regex_cd);
}
if (rs_ret < 0) {
goto error;
}
/* alloc scratch */
rs_inst->rs_rt->bool_match_buffs = ALLOC(struct bool_expr_match *, n_worker_thread);
for (i = 0; i < n_worker_thread; i++) {
rs_inst->rs_rt->bool_match_buffs[i] = ALLOC(struct bool_expr_match, MAX_HIT_EXPR_NUM);
}
rs_inst->rs_rt->streams = ALLOC(struct adapter_rs_stream *, n_worker_thread);
for (i = 0; i < n_worker_thread; i++) {
rs_inst->rs_rt->streams[i] = (struct adapter_rs_stream *)adapter_rs_stream_open(rs_inst, i);
}
rs_inst->rs_rt->matched_pats = ALLOC(struct matched_pattern *, n_worker_thread);
for (i = 0; i < n_worker_thread; i++) {
rs_inst->rs_rt->matched_pats[i] = ALLOC(struct matched_pattern, 1);
rs_inst->rs_rt->matched_pats[i]->ref_rs_attr = rs_inst->rs_attr;
rs_inst->rs_rt->matched_pats[i]->n_patterns = rs_inst->n_patterns;
utarray_new(rs_inst->rs_rt->matched_pats[i]->pattern_ids, &ut_rs_pattern_id_icd);
utarray_reserve(rs_inst->rs_rt->matched_pats[i]->pattern_ids, MAX_HIT_PATTERN_NUM);
}
return rs_inst;
error:
adapter_rs_free(rs_inst);
return NULL;
}
void adapter_rs_free(void *rs_instance)
{
if (NULL == rs_instance) {
return;
}
size_t i = 0;
struct adapter_rs *rs_inst = (struct adapter_rs *)rs_instance;
if (rs_inst->rs_rt != NULL) {
if (rs_inst->rs_rt->literal_db != NULL) {
rs_free_database(rs_inst->rs_rt->literal_db);
rs_inst->rs_rt->literal_db = NULL;
}
if (rs_inst->rs_rt->regex_db != NULL) {
rs_free_database(rs_inst->rs_rt->regex_db);
rs_inst->rs_rt->regex_db = NULL;
}
if (rs_inst->rs_rt->bool_match_buffs != NULL) {
for (i = 0; i < rs_inst->n_worker_thread; i++) {
if (rs_inst->rs_rt->bool_match_buffs[i] != NULL) {
FREE(rs_inst->rs_rt->bool_match_buffs[i]);
}
}
FREE(rs_inst->rs_rt->bool_match_buffs);
}
if (rs_inst->rs_rt->bm != NULL) {
bool_matcher_free(rs_inst->rs_rt->bm);
rs_inst->rs_rt->bm = NULL;
}
if (rs_inst->rs_rt->streams != NULL) {
for (i = 0; i < rs_inst->n_worker_thread; i++) {
if (rs_inst->rs_rt->streams[i] != NULL) {
adapter_rs_stream_close(rs_inst->rs_rt->streams[i]);
rs_inst->rs_rt->streams[i] = NULL;
}
}
FREE(rs_inst->rs_rt->streams);
}
if (rs_inst->rs_rt->matched_pats != NULL) {
for (i = 0; i < rs_inst->n_worker_thread; i++) {
if (rs_inst->rs_rt->matched_pats[i] != NULL) {
utarray_free(rs_inst->rs_rt->matched_pats[i]->pattern_ids);
FREE(rs_inst->rs_rt->matched_pats[i]);
}
}
FREE(rs_inst->rs_rt->matched_pats);
}
FREE(rs_inst->rs_rt);
}
if (rs_inst->rs_attr != NULL) {
FREE(rs_inst->rs_attr);
}
FREE(rs_inst);
}
static inline int compare_pattern_id(const void *a, const void *b)
{
long long ret = *(const unsigned long long *)a - *(const unsigned long long *)b;
if (ret == 0) {
return 0;
} else if(ret < 0) {
return -1;
} else {
return 1;
}
}
/**
* @param id: pattern id
*/
static int matched_event_cb(unsigned int id, int pos_offset, int from, int to,
size_t data_len, void *ctx)
{
// put id in set
unsigned long long pattern_id = id;
struct matched_pattern *matched_pat = (struct matched_pattern *)ctx;
if (pattern_id > matched_pat->n_patterns || id < 0) {
return 0;
}
if (utarray_len(matched_pat->pattern_ids) >= MAX_HIT_PATTERN_NUM) {
return 0;
}
// duplicate pattern_id
if (utarray_find(matched_pat->pattern_ids, &pattern_id, compare_pattern_id)) {
return 0;
}
int ret = 0;
struct pattern_attribute pat_attr = matched_pat->ref_rs_attr[id];
switch (pat_attr.match_mode) {
case EXPR_MATCH_MODE_EXACTLY:
if (0 == (from + pos_offset) && (int)data_len == (to + pos_offset)) {
ret = 1;
}
break;
case EXPR_MATCH_MODE_SUB:
if (pat_attr.offset.start == -1 &&
pat_attr.offset.end == -1) {
ret = 1;
break;
}
if (pat_attr.offset.start == -1) {
if ((long long)(to + pos_offset - 1) <= pat_attr.offset.end) {
ret = 1;
break;
}
}
if (pat_attr.offset.end == -1) {
if ((long long)(from + pos_offset) >= pat_attr.offset.start) {
ret = 1;
break;
}
}
if ((long long)(from + pos_offset) >= pat_attr.offset.start &&
(long long)(to + pos_offset - 1) <= pat_attr.offset.end) {
ret = 1;
}
break;
case EXPR_MATCH_MODE_PREFIX:
if (0 == (from + pos_offset)) {
ret = 1;
}
break;
case EXPR_MATCH_MODE_SUFFIX:
if ((to + pos_offset) == (int)data_len) {
ret = 1;
}
break;
default:
break;
}
if (1 == ret) {
utarray_push_back(matched_pat->pattern_ids, &pattern_id);
utarray_sort(matched_pat->pattern_ids, compare_pattern_id);
}
return 0;
}
void *adapter_rs_stream_open(void *rs_instance, int thread_id)
{
if (NULL == rs_instance || thread_id < 0) {
return NULL;
}
struct adapter_rs *rs_inst = (struct adapter_rs *)rs_instance;
struct adapter_rs_stream *rs_stream = ALLOC(struct adapter_rs_stream, 1);
rs_stream->logger = rs_inst->logger;
rs_stream->thread_id = thread_id;
rs_stream->ref_rs_rt = rs_inst->rs_rt;
int err_count = 0;
if (rs_inst->rs_rt->literal_db != NULL) {
rs_stream->literal_stream = rs_open_stream(rs_inst->rs_rt->literal_db, 0, 128);
if (NULL == rs_stream->literal_stream) {
log_error(rs_inst->logger, MODULE_ADAPTER_RS, "rs_open_stream failed");
err_count++;
}
}
if (rs_inst->rs_rt->regex_db != NULL) {
rs_stream->regex_stream = rs_open_stream(rs_inst->rs_rt->regex_db, 0, 128);
if (NULL == rs_stream->regex_stream) {
log_error(rs_inst->logger, MODULE_ADAPTER_RS, "rs_open_stream failed");
err_count++;
}
}
if (err_count > 0) {
goto error;
}
return rs_stream;
error:
if (rs_stream->literal_stream != NULL) {
rs_close_stream(rs_stream->literal_stream);
rs_stream->literal_stream = NULL;
}
if (rs_stream->regex_stream != NULL) {
rs_close_stream(rs_stream->regex_stream);
rs_stream->regex_stream = NULL;
}
FREE(rs_stream);
return NULL;
}
void adapter_rs_stream_close(void *rs_stream)
{
if (NULL == rs_stream) {
return;
}
struct adapter_rs_stream *stream = (struct adapter_rs_stream *)rs_stream;
if (stream->ref_rs_rt != NULL) {
if (stream->literal_stream != NULL) {
rs_close_stream(stream->literal_stream);
stream->literal_stream = NULL;
}
if (stream->regex_stream != NULL) {
rs_close_stream(stream->regex_stream);
stream->regex_stream = NULL;
}
}
/* rs_stream->rs_rt point to rs_instance->rs_rt which will call free
same as rs_attr */
stream->ref_rs_rt = NULL;
FREE(stream);
}
int adapter_rs_scan_stream(void *rs_stream, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result,
size_t *n_hit_result)
{
if (NULL == rs_stream || NULL == data || 0 == data_len ||
NULL == results || 0 == n_result || NULL == n_hit_result) {
return -1;
}
int ret = 0, err_count = 0;
struct adapter_rs_stream *stream = (struct adapter_rs_stream *)rs_stream;
int thread_id = stream->thread_id;
struct adapter_rs_runtime *rs_rt = stream->ref_rs_rt;
struct matched_pattern *matched_pat = rs_rt->matched_pats[thread_id];
if (stream->literal_stream != NULL) {
ret = rs_scan_stream(stream->literal_stream, data, data_len,
matched_event_cb, matched_pat);
if (ret < 0) {
err_count++;
}
}
if (stream->regex_stream != NULL) {
ret = rs_scan_stream(stream->regex_stream, data, data_len,
matched_event_cb, matched_pat);
if (ret < 0) {
err_count++;
}
}
if (err_count == 2) {
return -1;
}
size_t n_pattern_id = utarray_len(matched_pat->pattern_ids);
if (0 == n_pattern_id) {
*n_hit_result = 0;
return 0;
}
unsigned long long pattern_ids[n_pattern_id];
for (size_t i = 0; i < n_pattern_id; i++) {
pattern_ids[i] = *(unsigned long long *)utarray_eltptr(matched_pat->pattern_ids, i);
}
struct bool_expr_match *bool_matcher_results = rs_rt->bool_match_buffs[thread_id];
int bool_matcher_ret = bool_matcher_match(rs_rt->bm, pattern_ids, n_pattern_id,
bool_matcher_results, MAX_HIT_EXPR_NUM);
if (bool_matcher_ret < 0) {
ret = -1;
goto next;
}
if (bool_matcher_ret > (int)n_result) {
bool_matcher_ret = n_result;
}
for (int index = 0; index < bool_matcher_ret; index++) {
results[index].rule_id = bool_matcher_results[index].expr_id;
results[index].user_tag = bool_matcher_results[index].user_tag;
}
*n_hit_result = bool_matcher_ret;
next:
utarray_clear(matched_pat->pattern_ids);
return ret;
}
int adapter_rs_scan(void *rs_instance, int thread_id, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result, size_t *n_hit_result)
{
if (NULL == rs_instance || NULL == data || (0 == data_len) ||
NULL == results || 0 == n_result || NULL == n_hit_result) {
return -1;
}
int ret = 0, err_count = 0;
struct adapter_rs *rs_inst = (struct adapter_rs *)rs_instance;
struct adapter_rs_runtime *rs_rt = rs_inst->rs_rt;
struct matched_pattern *matched_pat = rs_rt->matched_pats[thread_id];
if (rs_rt->literal_db != NULL) {
ret = rs_scan(rs_rt->literal_db, thread_id, data, data_len,
0, matched_event_cb, matched_pat);
if (ret < 0) {
err_count++;
}
}
if (rs_rt->regex_db != NULL) {
ret = rs_scan(rs_rt->regex_db, thread_id, data, data_len,
0, matched_event_cb, matched_pat);
if (ret < 0) {
err_count++;
}
}
if (err_count == 2) {
return -1;
}
size_t n_pattern_id = utarray_len(matched_pat->pattern_ids);
if (0 == n_pattern_id) {
*n_hit_result = 0;
return 0;
}
unsigned long long pattern_ids[n_pattern_id];
for (size_t i = 0; i < n_pattern_id; i++) {
pattern_ids[i] = *(unsigned long long *)utarray_eltptr(matched_pat->pattern_ids, i);
}
struct bool_expr_match *bool_matcher_results = rs_rt->bool_match_buffs[thread_id];
int bool_matcher_ret = bool_matcher_match(rs_rt->bm, pattern_ids, n_pattern_id,
bool_matcher_results, MAX_HIT_EXPR_NUM);
if (bool_matcher_ret < 0) {
ret = -1;
goto next;
}
if (bool_matcher_ret > (int)n_result) {
bool_matcher_ret = n_result;
}
for (int index = 0; index < bool_matcher_ret; index++) {
results[index].rule_id = bool_matcher_results[index].expr_id;
results[index].user_tag = bool_matcher_results[index].user_tag;
}
*n_hit_result = bool_matcher_ret;
next:
utarray_clear(matched_pat->pattern_ids);
return ret;
}

78
scanner/expr_matcher/adapter_rs/adapter_rs.h
View File

@@ -1,78 +0,0 @@
/*
**********************************************************************************************
* File: adapter_rs.h
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2023-06-30
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#ifndef _ADAPTER_RS_H_
#define _ADAPTER_RS_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include "log/log.h"
#include "../expr_matcher.h"
int adapter_rs_verify_regex_expression(const char *regex_expr,
struct log_handle *logger);
/**
* @brief new adapter_rs instance
*
* @param rules: logic AND expression's array
* @param n_rule: the number of logic AND expression's array
* @param n_worker_threads: the number of scan threads which will call adapter_rs_scan()
*
* @retval the pointer to adapter_rs instance
*/
void *adapter_rs_new(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern,
size_t n_worker_thread, struct log_handle *logger);
void adapter_rs_free(void *rs_instance);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
*
* @param rs_instance: adapter_rs instance obtained by adapter_rs_new()
* @param thread_id: the thread_id of caller
* @param scan_data: data to be scanned
* @param data_len: the length of data to be scanned
* @param result_array: the array to store hit expr_id which allocated by caller
* @param n_result_array: number of elements in array of expr_id
*/
int adapter_rs_scan(void *rs_instance, int thread_id,
const char *scan_data, size_t data_len,
struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results);
/**
* @brief
*/
void *adapter_rs_stream_open(void *rs_instance, int thread_id);
/**
* @brief
*/
int adapter_rs_scan_stream(void *rs_stream, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results);
/**
* @brief
*/
void adapter_rs_stream_close(void *rs_stream);
#ifdef __cplusplus
}
#endif
#endif

235
scanner/expr_matcher/expr_matcher.cpp
View File

@@ -1,235 +0,0 @@
/*
**********************************************************************************************
* File: expr_matcher.cpp
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2023-06-30
* Copyright: (c) Since 2023 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#include <unistd.h>
#include <assert.h>
#include <sys/syscall.h>
#include "log/log.h"
#include "expr_matcher.h"
#include "maat_utils.h"
#include "adapter_hs/adapter_hs.h"
#include "adapter_rs/adapter_rs.h"
pid_t expr_matcher_gettid()
{
return syscall(SYS_gettid);
}
static const char *expr_matcher_module_name_str(const char *name)
{
static __thread char module[64];
snprintf(module, sizeof(module), "%s(%d)", name, expr_matcher_gettid());
return module;
}
#define MODULE_EXPR_MATCHER expr_matcher_module_name_str("maat.expr_matcher")
struct expr_matcher {
enum expr_engine_type engine_type;
void *engine;
struct log_handle *logger;
};
struct expr_matcher_stream {
enum expr_engine_type engine_type;
void *handle;
};
struct expr_engine_operations {
enum expr_engine_type type;
void *(*engine_new)(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern,
size_t n_worker_thread, struct log_handle *logger);
void (*engine_free)(void *engine);
int (*engine_scan)(void *engine, int thread_id, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_result);
void *(*engine_stream_open)(void *engine, int thread_id);
void (*engine_stream_close)(void *stream);
int (*engine_scan_stream)(void *stream, const char *scan_data, size_t data_len,
struct expr_scan_result *result_array, size_t n_result_array,
size_t *n_hit_result);
};
struct expr_engine_operations expr_engine_ops[EXPR_ENGINE_TYPE_MAX] = {
{
.type = EXPR_ENGINE_TYPE_HS,
.engine_new = adapter_hs_new,
.engine_free = adapter_hs_free,
.engine_scan = adapter_hs_scan,
.engine_stream_open = adapter_hs_stream_open,
.engine_stream_close = adapter_hs_stream_close,
.engine_scan_stream = adapter_hs_scan_stream
},
{
.type = EXPR_ENGINE_TYPE_RS,
.engine_new = adapter_rs_new,
.engine_free = adapter_rs_free,
.engine_scan = adapter_rs_scan,
.engine_stream_open = adapter_rs_stream_open,
.engine_stream_close = adapter_rs_stream_close,
.engine_scan_stream = adapter_rs_scan_stream
}
};
int expr_matcher_verify_regex_expression(const char *regex_expr,
struct log_handle *logger)
{
int ret = adapter_hs_verify_regex_expression(regex_expr, logger);
if (ret == 0) {
return 0;
}
return adapter_rs_verify_regex_expression(regex_expr, logger);
}
struct expr_matcher *
expr_matcher_new(struct expr_rule *rules, size_t n_rule, enum expr_engine_type engine_type,
size_t n_worker_thread, struct log_handle *logger)
{
if (NULL == rules || 0 == n_rule || 0 == n_worker_thread ||
(engine_type != EXPR_ENGINE_TYPE_HS && engine_type != EXPR_ENGINE_TYPE_RS)) {
log_error(logger, MODULE_EXPR_MATCHER, "[%s:%d]engine type:%d is illegal",
__FUNCTION__, __LINE__, engine_type);
return NULL;
}
size_t i = 0, j = 0;
size_t literal_pat_num = 0;
size_t regex_pat_num = 0;
for (i = 0; i < n_rule; i++) {
if (rules[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
log_error(logger, MODULE_EXPR_MATCHER,
"[%s:%d] the number of patterns in one expression should less than"
" %d", __FUNCTION__, __LINE__, MAX_EXPR_PATTERN_NUM);
return NULL;
}
for (j = 0; j < rules[i].n_patterns; j++) {
/* pat_len should not 0 */
if (0 == rules[i].patterns[j].pat_len) {
log_error(logger, MODULE_EXPR_MATCHER,
"[%s:%d] expr pattern length should not 0",
__FUNCTION__, __LINE__);
return NULL;
}
if (rules[i].patterns[j].type == EXPR_PATTERN_TYPE_STR) {
literal_pat_num++;
} else {
regex_pat_num++;
}
}
}
if (0 == literal_pat_num && 0 == regex_pat_num) {
log_error(logger, MODULE_EXPR_MATCHER,
"[%s:%d] exprs has no valid pattern", __FUNCTION__, __LINE__);
return NULL;
}
void *engine = expr_engine_ops[engine_type].engine_new(rules, n_rule, literal_pat_num,
regex_pat_num, n_worker_thread,
logger);
if (NULL == engine) {
log_error(logger, MODULE_EXPR_MATCHER,
"[%s:%d]expr_matcher engine_new failed.", __FUNCTION__, __LINE__);
return NULL;
}
struct expr_matcher *matcher = ALLOC(struct expr_matcher, 1);
matcher->engine_type = engine_type;
matcher->engine = engine;
matcher->logger = logger;
return matcher;
}
void expr_matcher_free(struct expr_matcher *matcher)
{
if (NULL == matcher) {
return;
}
if (matcher->engine != NULL) {
expr_engine_ops[matcher->engine_type].engine_free(matcher->engine);
matcher->engine = NULL;
}
FREE(matcher);
}
int expr_matcher_match(struct expr_matcher *matcher, int thread_id, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results)
{
if (NULL == matcher || thread_id < 0 || NULL == scan_data || 0 == data_len
|| NULL == result_array || 0 == n_result_array || NULL == n_hit_results) {
return -1;
}
return expr_engine_ops[matcher->engine_type].engine_scan(matcher->engine, thread_id,
scan_data, data_len, result_array,
n_result_array, n_hit_results);
}
struct expr_matcher_stream *
expr_matcher_stream_open(struct expr_matcher *matcher, int thread_id)
{
if (NULL == matcher || thread_id < 0) {
return NULL;
}
void *s_handle = expr_engine_ops[matcher->engine_type].engine_stream_open(matcher->engine,
thread_id);
if (NULL == s_handle) {
log_error(matcher->logger, MODULE_EXPR_MATCHER,
"[%s:%d] expr_matcher engine_stream_open failed.",
__FUNCTION__, __LINE__);
return NULL;
}
struct expr_matcher_stream *stream = ALLOC(struct expr_matcher_stream, 1);
stream->engine_type = matcher->engine_type;
stream->handle = s_handle;
return stream;
}
int expr_matcher_stream_match(struct expr_matcher_stream *stream, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results)
{
if (NULL == stream || NULL == scan_data || 0 == data_len || NULL == result_array
|| 0 == n_result_array || NULL == n_hit_results) {
return -1;
}
return expr_engine_ops[stream->engine_type].engine_scan_stream(stream->handle, scan_data,
data_len, result_array,
n_result_array, n_hit_results);
}
void expr_matcher_stream_close(struct expr_matcher_stream *stream)
{
if (NULL == stream) {
return;
}
if (stream->handle != NULL) {
expr_engine_ops[stream->engine_type].engine_stream_close(stream->handle);
stream->handle = NULL;
}
FREE(stream);
}

134
scanner/expr_matcher/expr_matcher.h
View File

@@ -1,134 +0,0 @@
/*
**********************************************************************************************
* File: expr_matcher.h
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2023-06-30
* Copyright: (c) Since 2023 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#ifndef _EXPR_MATCHER_H_
#define _EXPR_MATCHER_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include "log/log.h"
#define MAX_EXPR_PATTERN_NUM 8 /* 每条与表达式最多由MAX_EXPR_ITEM_NUM个规则组成 */
#define MAX_HIT_EXPR_NUM 1024
enum expr_engine_type {
EXPR_ENGINE_TYPE_HS = 0, /* default engine */
EXPR_ENGINE_TYPE_RS,
EXPR_ENGINE_TYPE_MAX
};
enum expr_pattern_type {
EXPR_PATTERN_TYPE_STR = 0, /* pure literal string */
EXPR_PATTERN_TYPE_REG = 1, /* regex expression */
};
enum expr_case_sensitive {
EXPR_CASE_INSENSITIVE = 0,
EXPR_CASE_SENSITIVE
};
enum expr_match_mode {
EXPR_MATCH_MODE_INVALID = -1,
EXPR_MATCH_MODE_EXACTLY = 1, /* scan data must match pattern exactly */
EXPR_MATCH_MODE_PREFIX, /* pattern must in the head of scan_data */
EXPR_MATCH_MODE_SUFFIX, /* pattern must in the tail of scan_data */
EXPR_MATCH_MODE_SUB /* pattern must in the range[l_offset, r_offset] of scan_data */
};
struct expr_pattern {
enum expr_pattern_type type;
enum expr_match_mode match_mode;
enum expr_case_sensitive case_sensitive;
/*
* just match in scan_data's range of [start_offset, end_offset], -1 means no limits
* for example:
* [-1, end_offset] means the pattern must in scan_data's [0 ~ start_offset]
* [start_offset, -1] means the pattern must in scan_data's [start_offset ~ data_end]
*/
int start_offset;
int end_offset;
char *pat;
size_t pat_len;
};
struct expr_scan_result {
long long rule_id;
void *user_tag;
};
/* logic AND expression, such as (rule1 & rule2) */
struct expr_rule {
long long expr_id; /* AND expression ID */
size_t n_patterns;
struct expr_pattern patterns[MAX_EXPR_PATTERN_NUM];
void *tag; /* user defined data, return with hit result */
};
int expr_matcher_verify_regex_expression(const char *regex_expr,
struct log_handle *logger);
/**
* @brief new expr matcher instance
*
* @param expr_array: logic AND expression's array
* @param n_expr_array: the number of logic AND expression's array
* @param n_worker_threads: the number of scan threads which will call adapter_rs_scan()
*
*/
struct expr_matcher *
expr_matcher_new(struct expr_rule *rules, size_t n_rule, enum expr_engine_type type,
size_t n_worker_thread, struct log_handle *logger);
void expr_matcher_free(struct expr_matcher *matcher);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
*
* @param matcher: expr_matcher instance obtained by expr_matcher_new()
* @param thread_id: the thread_id of caller
* @param scan_data: data to be scanned
* @param data_len: the length of data to be scanned
* @param result_array: the array to store hit expr_id which allocated by caller
* @param n_result_array: number of elements in array of expr_id
*/
int expr_matcher_match(struct expr_matcher *matcher, int thread_id, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results);
/**
* @brief
*/
struct expr_matcher_stream *
expr_matcher_stream_open(struct expr_matcher *matcher, int thread_id);
/**
* @brief
*/
int expr_matcher_stream_match(struct expr_matcher_stream *stream, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results);
/**
* @brief
*/
void expr_matcher_stream_close(struct expr_matcher_stream *stream);
#ifdef __cplusplus
}
#endif
#endif

4
src/CMakeLists.txt
View File

@@ -23,15 +23,13 @@ include_directories(/opt/MESA/include/MESA/)
include_directories(${PROJECT_SOURCE_DIR}/include/) include_directories(${PROJECT_SOURCE_DIR}/include/)
include_directories(${PROJECT_SOURCE_DIR}/deps/) include_directories(${PROJECT_SOURCE_DIR}/deps/)
include_directories(${PROJECT_SOURCE_DIR}/scanner) include_directories(${PROJECT_SOURCE_DIR}/scanner)
include_directories(${PROJECT_SOURCE_DIR}/scanner/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/fqdn_engine) include_directories(${PROJECT_SOURCE_DIR}/scanner/fqdn_engine)
include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/flag_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/flag_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/interval_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/interval_matcher)
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_rs)
# Static Library Output # Static Library Output
add_library(maat_frame_static STATIC ${MAAT_SRC} ${LIB_SOURCE_FILES}) add_library(maat_frame_static STATIC ${MAAT_SRC} ${LIB_SOURCE_FILES})

19
src/inc_internal/maat_compile.h
View File

@@ -37,6 +37,12 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void group2compile_schema_free(void *g2c_schema); void group2compile_schema_free(void *g2c_schema);
int group2compile_associated_compile_table_id(void *g2c_schema); int group2compile_associated_compile_table_id(void *g2c_schema);
int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp);
/* compile runtime API */ /* compile runtime API */
void *compile_runtime_new(void *compile_schema, size_t max_thread_num, void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin, struct maat_garbage_bin *garbage_bin,
@@ -64,6 +70,14 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
struct maat_hit_path *hit_path_array, struct maat_hit_path *hit_path_array,
size_t array_size, size_t n_internal_hit_path); size_t array_size, size_t n_internal_hit_path);
void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema,
long long compile_id);
void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema);
void compile_runtime_garbage_collect_routine(void *compile_runtime);
/* group2compile runtime API */ /* group2compile runtime API */
void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num, void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin, struct maat_garbage_bin *garbage_bin,
@@ -105,13 +119,10 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
struct maat_hit_group *hit_group_array, struct maat_hit_group *hit_group_array,
size_t array_size); size_t array_size);
int maat_compile_state_get_compile_table_id(struct maat_compile_state *compile_state,
long long compile_id);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif #endif

11
src/inc_internal/maat_expr.h
View File

@@ -52,13 +52,14 @@ long long expr_runtime_get_version(void *expr_runtime);
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int vtable_id, struct maat_state *state); size_t data_len, int vtable_id, struct maat_state *state);
struct expr_matcher_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct expr_matcher_stream *s_handle, int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle,
const char *data, size_t data_len, int vtable_id, struct maat_state *state); const char *data, size_t data_len,
int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id, void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,
struct expr_matcher_stream *stream); struct adapter_hs_stream *s_handle);
int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district, int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district,
size_t district_len, long long *district_id); size_t district_len, long long *district_id);
@@ -83,4 +84,4 @@ long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt);
} }
#endif #endif
#endif #endif

7
src/inc_internal/maat_kv.h
View File

@@ -26,15 +26,12 @@ void maat_kv_store_free(struct maat_kv_store *store);
int maat_kv_register(struct maat_kv_store *store, const char *key, long long value); int maat_kv_register(struct maat_kv_store *store, const char *key, long long value);
int maat_kv_read(struct maat_kv_store *store, const char *key, long long *value_array, size_t n_array); int maat_kv_read(struct maat_kv_store *store, const char *key, long long *value);
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, long long *value);
long long *value_array, size_t n_array);
int maat_kv_write(struct maat_kv_store *store, const char *key, long long value); int maat_kv_write(struct maat_kv_store *store, const char *key, long long value);
int maat_kv_append(struct maat_kv_store *store, const char *key, long long value);
struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store); struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store);
#ifdef __cplusplus #ifdef __cplusplus

3
src/inc_internal/maat_rule.h
View File

@@ -128,7 +128,6 @@ struct maat_options {
int rule_effect_interval_ms; int rule_effect_interval_ms;
int rule_update_checking_interval_ms; int rule_update_checking_interval_ms;
enum maat_expr_engine expr_engine;
enum data_source input_mode; enum data_source input_mode;
union { union {
struct source_iris_ctx iris_ctx; struct source_iris_ctx iris_ctx;
@@ -226,4 +225,4 @@ void garbage_maat_kv_store_free(void *kv_store, void *arg);
} }
#endif #endif
#endif #endif

24
src/inc_internal/maat_table.h
View File

@@ -19,7 +19,6 @@ extern "C"
#include <stddef.h> #include <stddef.h>
#include <cJSON/cJSON.h> #include <cJSON/cJSON.h>
#include "maat.h"
#include "log/log.h" #include "log/log.h"
#include "maat_garbage_collection.h" #include "maat_garbage_collection.h"
@@ -48,8 +47,7 @@ struct table_manager;
struct table_manager * struct table_manager *
table_manager_create(const char *table_info_path, const char *accept_tags, table_manager_create(const char *table_info_path, const char *accept_tags,
enum maat_expr_engine expr_engine, struct maat_garbage_bin *garbage_bin, struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
struct log_handle *logger);
int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_thread_num, int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin); struct maat_garbage_bin *garbage_bin);
@@ -59,23 +57,7 @@ void table_manager_destroy(struct table_manager *tbl_mgr);
size_t table_manager_table_size(struct table_manager *tbl_mgr); size_t table_manager_table_size(struct table_manager *tbl_mgr);
size_t table_manager_table_count(struct table_manager *tbl_mgr); size_t table_manager_table_count(struct table_manager *tbl_mgr);
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_name); int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name);
/**
* @brief get table_name's all conjunction parents' table_id
*
* for example: "table_id": 1,
* "table_name":"COMPILE_CONJ"
* "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"]
*
* "table_id": 2,
* "table_name": "COMPILE_PLUGIN",
* "db_tables": ["COMPILE_DEFAULT"]
*
* so COMPILE_DEFAULT has two conjunction parents whose table_id is 1 and 2.
*/
int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name,
long long *table_ids_array, size_t n_table_ids_array);
const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id); const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id);
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id); enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id);
@@ -85,8 +67,6 @@ int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr);
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id); int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr);
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr); size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr);
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags); int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags);

4
src/json2iris.c
View File

@@ -377,7 +377,7 @@ static int direct_write_rule(cJSON *json, struct maat_kv_store *str2int,
if (1 == cmd[i].str2int_flag) { if (1 == cmd[i].str2int_flag) {
long long int_value = 0; long long int_value = 0;
char *p = item->valuestring; char *p = item->valuestring;
ret = maat_kv_read(str2int, p, &int_value, 1); ret = maat_kv_read(str2int, p, &int_value);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS, log_error(logger, MODULE_JSON2IRIS,
"[%s:%d] %s's value %s is not valid format", "[%s:%d] %s's value %s is not valid format",
@@ -636,7 +636,7 @@ static int write_region_rule(cJSON *region_json, int compile_id, int group_id,
const char *table_type_str = item->valuestring; const char *table_type_str = item->valuestring;
long long table_type_int; long long table_type_int;
int ret = maat_kv_read(p_iris->str2int_map, table_type_str, &(table_type_int), 1); int ret = maat_kv_read(p_iris->str2int_map, table_type_str, &table_type_int);
if (ret != 1) { if (ret != 1) {
log_error(logger, MODULE_JSON2IRIS, log_error(logger, MODULE_JSON2IRIS,
"[%s:%d] compile rule %d table name %s's table_type %s invalid", "[%s:%d] compile rule %d table name %s's table_type %s invalid",

106
src/maat_api.c
View File

@@ -51,7 +51,7 @@ enum district_flag {
struct maat_stream { struct maat_stream {
struct maat *ref_maat_inst; struct maat *ref_maat_inst;
struct expr_matcher_stream *handle; //each physical table open one stream struct adapter_hs_stream *handle; //each physical table open one stream
long long last_full_version; long long last_full_version;
long long expr_rt_version; long long expr_rt_version;
struct log_handle *logger; struct log_handle *logger;
@@ -70,7 +70,6 @@ struct maat_options* maat_options_new(void)
options->rule_update_checking_interval_ms = 1 * 1000; options->rule_update_checking_interval_ms = 1 * 1000;
options->gc_timeout_ms = 10 * 1000; options->gc_timeout_ms = 10 * 1000;
options->input_mode = DATA_SOURCE_NONE; options->input_mode = DATA_SOURCE_NONE;
options->expr_engine = MAAT_EXPR_ENGINE_HS;
options->log_level = 0; options->log_level = 0;
return options; return options;
@@ -255,19 +254,6 @@ int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filen
return 0; return 0;
} }
int maat_options_set_expr_engine(struct maat_options *opts,
enum maat_expr_engine expr_engine)
{
if (NULL == opts ||
(expr_engine != MAAT_EXPR_ENGINE_HS && expr_engine != MAAT_EXPR_ENGINE_RS)) {
return -1;
}
opts->expr_engine = expr_engine;
return 0;
}
int maat_options_set_logger(struct maat_options *opts, const char *log_path, int maat_options_set_logger(struct maat_options *opts, const char *log_path,
enum log_level level) enum log_level level)
{ {
@@ -371,8 +357,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
pthread_mutex_init(&(maat_inst->background_update_mutex), NULL); pthread_mutex_init(&(maat_inst->background_update_mutex), NULL);
maat_inst->tbl_mgr = table_manager_create(table_info_path, maat_inst->opts.accept_tags, maat_inst->tbl_mgr = table_manager_create(table_info_path, maat_inst->opts.accept_tags,
maat_inst->opts.expr_engine, maat_inst->garbage_bin, maat_inst->garbage_bin, maat_inst->logger);
maat_inst->logger);
if (NULL == maat_inst->tbl_mgr) { if (NULL == maat_inst->tbl_mgr) {
goto failed; goto failed;
} }
@@ -425,7 +410,12 @@ int maat_helper_verify_regex_expression(const char *regex_expr)
return 0; return 0;
} }
return expr_matcher_verify_regex_expression(regex_expr, NULL); int ret = adapter_hs_verify_regex_expression(regex_expr, NULL);
if (ret < 0) {
return 0;
} else {
return 1;
}
} }
int maat_get_table_id(struct maat *maat_inst, const char *table_name) int maat_get_table_id(struct maat *maat_inst, const char *table_name)
@@ -522,6 +512,32 @@ int maat_table_callback_register(struct maat *maat_inst, int table_id,
return 0; return 0;
} }
static int compile_table_ex_schema_register(struct maat *maat_inst, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp)
{
void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id);
assert(schema != NULL);
int ret = compile_table_set_ex_data_schema((struct compile_schema *)schema, table_id,
new_func, free_func, dup_func,
argl, argp);
if (ret < 0) {
return -1;
}
if (maat_inst->maat_rt != NULL) {
void *runtime = table_manager_get_runtime(maat_inst->tbl_mgr, table_id);
assert(runtime != NULL);
compile_runtime_ex_data_iterate((struct compile_runtime *)runtime,
(struct compile_schema *)schema);
}
return 0;
}
static int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_id, static int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_id,
maat_ex_new_func_t *new_func, maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func, maat_ex_free_func_t *free_func,
@@ -755,14 +771,16 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_inst,
int ret = -1; int ret = -1;
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr,
table_id); table_id);
if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_IP_PLUGIN || if (TABLE_TYPE_COMPILE == table_type) {
table_type == TABLE_TYPE_BOOL_PLUGIN || table_type == TABLE_TYPE_FQDN_PLUGIN ) { ret = compile_table_ex_schema_register(maat_inst, table_id, new_func,
free_func, dup_func, argl, argp);
} else {
ret = generic_plugin_table_ex_schema_register(maat_inst, table_name, ret = generic_plugin_table_ex_schema_register(maat_inst, table_name,
table_id, new_func, free_func, table_id, new_func, free_func,
dup_func, argl, argp); dup_func, argl, argp);
} }
pthread_mutex_unlock(&(maat_inst->background_update_mutex));
pthread_mutex_unlock(&(maat_inst->background_update_mutex));
return ret; return ret;
} }
@@ -793,7 +811,9 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_inst, int table_id,
void *ret = NULL; void *ret = NULL;
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr,
table_id); table_id);
if (TABLE_TYPE_PLUGIN == table_type) { if (TABLE_TYPE_COMPILE == table_type) {
ret = compile_runtime_get_ex_data(runtime, schema, *(long long *)key);
} else if (TABLE_TYPE_PLUGIN == table_type) {
ret = plugin_runtime_get_ex_data(runtime, schema, key, key_len); ret = plugin_runtime_get_ex_data(runtime, schema, key, key_len);
} }
@@ -1148,7 +1168,9 @@ int maat_scan_flag(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = flag_scan(maat_inst->tbl_mgr, state->thread_id, flag, int hit_group_cnt = flag_scan(maat_inst->tbl_mgr, state->thread_id, flag,
phy_table_id, vtable_id, state); phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
@@ -1156,8 +1178,6 @@ int maat_scan_flag(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0; size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1237,7 +1257,9 @@ int maat_scan_integer(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = interval_scan(maat_inst->tbl_mgr, state->thread_id, integer, int hit_group_cnt = interval_scan(maat_inst->tbl_mgr, state->thread_id, integer,
phy_table_id, vtable_id, state); phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
@@ -1245,8 +1267,6 @@ int maat_scan_integer(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0; size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1326,7 +1346,9 @@ int maat_scan_ipv4(struct maat *maat_inst, int table_id, uint32_t ip_addr,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = ipv4_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr, int hit_group_cnt = ipv4_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr,
port, protocol, phy_table_id, vtable_id, state); port, protocol, phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
@@ -1334,8 +1356,6 @@ int maat_scan_ipv4(struct maat *maat_inst, int table_id, uint32_t ip_addr,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0; size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1416,7 +1436,9 @@ int maat_scan_ipv6(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = ipv6_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr, int hit_group_cnt = ipv6_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr,
port, protocol, phy_table_id, vtable_id, state); port, protocol, phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
@@ -1424,8 +1446,6 @@ int maat_scan_ipv6(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0; size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1505,7 +1525,9 @@ int maat_scan_string(struct maat *maat_inst, int table_id, const char *data,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = string_scan(maat_inst->tbl_mgr, state->thread_id, data, int hit_group_cnt = string_scan(maat_inst->tbl_mgr, state->thread_id, data,
data_len, phy_table_id, vtable_id, state); data_len, phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) { if (hit_group_cnt < 0) {
@@ -1513,8 +1535,6 @@ int maat_scan_string(struct maat *maat_inst, int table_id, const char *data,
return MAAT_SCAN_ERR; return MAAT_SCAN_ERR;
} }
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0; size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1589,8 +1609,8 @@ struct maat_stream *maat_stream_new(struct maat *maat_inst, int table_id,
stream->expr_rt_version = expr_runtime_get_version(expr_rt); stream->expr_rt_version = expr_runtime_get_version(expr_rt);
maat_runtime_ref_inc(maat_inst->maat_rt, state->thread_id); maat_runtime_ref_inc(maat_inst->maat_rt, state->thread_id);
struct expr_matcher_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt, struct adapter_hs_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt,
state->thread_id); state->thread_id);
if (NULL == handle) { if (NULL == handle) {
goto error; goto error;
} }
@@ -1869,22 +1889,6 @@ int maat_state_set_scan_compile_table(struct maat_state *state, int compile_tabl
return 0; return 0;
} }
int maat_state_get_compile_table_ids(struct maat_state *state, long long *compile_ids,
size_t n_compile_ids, int *compile_table_ids)
{
if (NULL == state || NULL == compile_ids || 0 == n_compile_ids ||
NULL == compile_table_ids) {
return -1;
}
for (size_t i = 0; i < n_compile_ids; i++) {
compile_table_ids[i] = maat_compile_state_get_compile_table_id(state->compile_state,
compile_ids[i]);
}
return n_compile_ids;
}
int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths, int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths,
size_t n_path) size_t n_path)
{ {
@@ -1956,4 +1960,4 @@ int maat_state_get_hit_groups(struct maat_state *state, enum maat_list_type type
int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group) int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group)
{ {
return 0; return 0;
} }

281
src/maat_compile.c
View File

@@ -27,12 +27,16 @@
#include "alignment.h" #include "alignment.h"
#define MODULE_COMPILE module_name_str("maat.compile") #define MODULE_COMPILE module_name_str("maat.compile")
#define DEFAULT_GC_TIMEOUT_S 10
struct compile_schema { struct compile_schema {
int compile_id_column; int compile_id_column;
int rule_tag_column; int rule_tag_column;
int declared_clause_num_column; int declared_clause_num_column;
int set_flag;
int gc_timeout_s;
int table_id; //ugly int table_id; //ugly
struct ex_data_schema ex_schema;
struct table_manager *ref_tbl_mgr; struct table_manager *ref_tbl_mgr;
struct log_handle *logger; struct log_handle *logger;
}; };
@@ -53,6 +57,8 @@ struct compile_item {
long long compile_id; long long compile_id;
char *table_line; char *table_line;
size_t table_line_len; size_t table_line_len;
struct compile_schema *ref_schema;
void **ex_data;
char table_name[MAX_NAME_STR_LEN]; char table_name[MAX_NAME_STR_LEN];
}; };
@@ -79,8 +85,7 @@ struct literal_clause {
/* compile_runtime and group2compile_runtime share compile_hash_map */ /* compile_runtime and group2compile_runtime share compile_hash_map */
struct compile_runtime { struct compile_runtime {
struct bool_matcher *bm; struct bool_matcher *bm;
struct rcu_hash_table *cfg_hash; // <compile_id, struct maat_compile> struct rcu_hash_table *cfg_hash; // <compile_id, struct maat_compile>
struct rcu_hash_table *tbl_cfg_hash; // <compile_id, table_id>
struct maat_runtime *ref_maat_rt; struct maat_runtime *ref_maat_rt;
time_t version; time_t version;
struct literal_clause *literal2clause_hash; struct literal_clause *literal2clause_hash;
@@ -132,15 +137,9 @@ struct maat_internal_hit_path {
int vtable_id; int vtable_id;
}; };
struct maat_compile_table {
long long compile_id;
int table_id;
};
struct maat_compile_state { struct maat_compile_state {
uint8_t this_scan_hit_item_flag; uint8_t this_scan_hit_item_flag;
uint8_t not_clause_hit_flag; uint8_t not_clause_hit_flag;
uint8_t inc_hit_path_flag;
int Nth_scan; int Nth_scan;
time_t compile_rt_version; time_t compile_rt_version;
@@ -148,14 +147,12 @@ struct maat_compile_state {
UT_array *internal_inc_hit_paths; UT_array *internal_inc_hit_paths;
UT_array *all_hit_clauses; UT_array *all_hit_clauses;
UT_array *this_scan_hit_clauses; UT_array *this_scan_hit_clauses;
UT_array *hit_compile_tables;
}; };
UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL}; UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL};
UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL}; UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL};
UT_icd ut_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL}; UT_icd ut_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL};
UT_icd ut_hit_path_icd = {sizeof(struct maat_internal_hit_path), NULL, NULL, NULL}; UT_icd ut_hit_path_icd = {sizeof(struct maat_internal_hit_path), NULL, NULL, NULL};
UT_icd ut_hit_compile_table_icd = {sizeof(struct maat_compile_table)};
static struct maat_compile *maat_compile_new(long long compile_id) static struct maat_compile *maat_compile_new(long long compile_id)
{ {
@@ -188,6 +185,23 @@ static int maat_compile_set(struct maat_compile *compile, const char *table_name
return 0; return 0;
} }
static void *rule_ex_data_new(const char *table_name, int table_id,
const char *table_line,
struct ex_data_schema *ex_schema)
{
void *ex_data = NULL;
ex_schema->new_func(table_name, table_id, NULL, table_line, &ex_data,
ex_schema->argl, ex_schema->argp);
return ex_data;
}
static void rule_ex_data_free(int table_id, void **ex_data,
const struct ex_data_schema *ex_schema)
{
ex_schema->free_func(table_id, ex_data, ex_schema->argl, ex_schema->argp);
}
static int compile_accept_tag_match(struct compile_schema *schema, const char *line, static int compile_accept_tag_match(struct compile_schema *schema, const char *line,
const char *table_name, struct log_handle *logger) const char *table_name, struct log_handle *logger)
{ {
@@ -262,11 +276,18 @@ compile_item_new(const char *table_line, struct compile_schema *schema,
} }
compile_item->declared_clause_num = atoi(table_line + column_offset); compile_item->declared_clause_num = atoi(table_line + column_offset);
compile_item->ref_schema = schema;
compile_item->ex_data = ALLOC(void *, 1);
memcpy(compile_item->table_name, table_name, sizeof(compile_item->table_name)); memcpy(compile_item->table_name, table_name, sizeof(compile_item->table_name));
compile_item->table_line_len = strlen(table_line) + 1; compile_item->table_line_len = strlen(table_line) + 1;
compile_item->table_line = ALLOC(char, compile_item->table_line_len); compile_item->table_line = ALLOC(char, compile_item->table_line_len);
memcpy(compile_item->table_line, table_line, compile_item->table_line_len); memcpy(compile_item->table_line, table_line, compile_item->table_line_len);
if (1 == schema->set_flag) {
*(compile_item->ex_data) = rule_ex_data_new(table_name, schema->table_id,
compile_item->table_line,
&(schema->ex_schema));
}
return compile_item; return compile_item;
error: error:
FREE(compile_item); FREE(compile_item);
@@ -275,6 +296,17 @@ error:
static void compile_item_free(struct compile_item *item) static void compile_item_free(struct compile_item *item)
{ {
struct compile_schema *schema = item->ref_schema;
if (1 == schema->set_flag) {
rule_ex_data_free(schema->table_id, item->ex_data, &(schema->ex_schema));
*item->ex_data = NULL;
}
if (item->ex_data != NULL) {
FREE(item->ex_data);
}
item->declared_clause_num = -1; item->declared_clause_num = -1;
if (item->table_line != NULL) { if (item->table_line != NULL) {
@@ -315,9 +347,70 @@ static void rcu_compile_cfg_free(void *user_ctx, void *data)
maat_compile_free(compile); maat_compile_free(compile);
} }
static void rcu_compile_table_cfg_free(void *user_ctx, void *data) int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp)
{ {
FREE(data); if (1 == compile_schema->set_flag) {
log_error(compile_schema->logger, MODULE_COMPILE,
"[%s:%d] compile table(table_id:%d)ex schema has been set already,"
" can't set again", __FUNCTION__, __LINE__, table_id);
return -1;
}
compile_schema->ex_schema.new_func = new_func;
compile_schema->ex_schema.free_func = free_func;
compile_schema->ex_schema.dup_func = dup_func;
compile_schema->ex_schema.argl = argl;
compile_schema->ex_schema.argp = argp;
compile_schema->set_flag = 1;
return 0;
}
static void *compile_runtime_get_user_data(struct compile_runtime *compile_rt,
long long compile_id)
{
struct maat_compile *compile = rcu_hash_find(compile_rt->cfg_hash,
(char *)&compile_id,
sizeof(long long));
void *ret = NULL;
if (compile != NULL) {
ret = compile->user_data;
}
return ret;
}
static void rule_ex_data_new_cb(void *user_data, void *param,
const char *table_name, int table_id)
{
struct ex_data_schema *ex_schema = (struct ex_data_schema *)param;
struct compile_item *compile = (struct compile_item *)user_data;
void *ad = rule_ex_data_new(table_name, table_id, compile->table_line, ex_schema);
*compile->ex_data = ad;
}
static void compile_runtime_user_data_iterate(struct compile_runtime *compile_rt,
void (*callback)(void *user_data, void *param,
const char *table_name, int table_id),
void *param, int table_id)
{
/* I'm in background_update_mutex, config update can't happen, so no need to lock cfg_hash */
void **data_array = NULL;
size_t data_cnt = rcu_hash_list(compile_rt->cfg_hash, &data_array);
for (size_t i = 0; i < data_cnt; i++) {
struct maat_compile *compile = (struct maat_compile *)data_array[i];
if (compile->user_data) {
callback(compile->user_data, param, compile->table_name, table_id);
}
}
FREE(data_array);
} }
void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
@@ -371,6 +464,12 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
goto error; goto error;
} }
//gc_timeout_s is optional
custom_item = cJSON_GetObjectItem(item, "gc_timeout_s");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->gc_timeout_s = custom_item->valueint;
}
schema->ref_tbl_mgr = tbl_mgr; schema->ref_tbl_mgr = tbl_mgr;
return schema; return schema;
error: error:
@@ -494,13 +593,14 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
return NULL; return NULL;
} }
struct compile_schema *schema = (struct compile_schema *)compile_schema;
struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1); struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1);
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, compile_rt->expr_match_buff = ALLOC(struct bool_expr_match,
max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM); max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM);
compile_rt->version = time(NULL); compile_rt->version = time(NULL);
compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL, 0); compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL,
compile_rt->tbl_cfg_hash = rcu_hash_new(rcu_compile_table_cfg_free, NULL, 0); schema->gc_timeout_s + DEFAULT_GC_TIMEOUT_S);
compile_rt->literal2clause_hash = NULL; compile_rt->literal2clause_hash = NULL;
compile_rt->logger = logger; compile_rt->logger = logger;
compile_rt->ref_garbage_bin = garbage_bin; compile_rt->ref_garbage_bin = garbage_bin;
@@ -547,11 +647,6 @@ void compile_runtime_free(void *compile_runtime)
compile_rt->cfg_hash = NULL; compile_rt->cfg_hash = NULL;
} }
if (compile_rt->tbl_cfg_hash != NULL) {
rcu_hash_free(compile_rt->tbl_cfg_hash);
compile_rt->tbl_cfg_hash = NULL;
}
if (compile_rt->literal2clause_hash != NULL) { if (compile_rt->literal2clause_hash != NULL) {
literal2clause_hash_free(compile_rt->literal2clause_hash); literal2clause_hash_free(compile_rt->literal2clause_hash);
compile_rt->literal2clause_hash = NULL; compile_rt->literal2clause_hash = NULL;
@@ -892,30 +987,14 @@ static inline int compare_hit_group(const void *pa, const void *pb)
struct maat_hit_group *la=(struct maat_hit_group *)pa; struct maat_hit_group *la=(struct maat_hit_group *)pa;
struct maat_hit_group *lb=(struct maat_hit_group *)pb; struct maat_hit_group *lb=(struct maat_hit_group *)pb;
long long ret = la->item_id - lb->item_id; long long ret = la->group_id - lb->group_id;
if (ret == 0) { if (ret == 0) {
ret = la->group_id - lb->group_id; ret = la->vtable_id - lb->vtable_id;
if (ret == 0) {
ret = la->vtable_id - lb->vtable_id;
}
} }
return ret; return ret;
} }
static inline int compare_compile_id(const void *a, const void *b)
{
long long ret = *(const long long *)a - *(const long long *)b;
if (0 == ret) {
return 0;
} else if (ret < 0) {
return -1;
} else {
return 1;
}
}
static struct literal_clause * static struct literal_clause *
maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt) maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt)
{ {
@@ -996,29 +1075,6 @@ static size_t compile_state_if_new_hit_compile(struct maat_compile_state *compil
return r_in_c_cnt; return r_in_c_cnt;
} }
static void compile_state_update_hit_compile_table_id(struct maat_compile_state *compile_state,
long long compile_id, int table_id)
{
if (!utarray_find(compile_state->hit_compile_tables, &compile_id, compare_compile_id)) {
struct maat_compile_table compile_table = {compile_id, table_id};
utarray_push_back(compile_state->hit_compile_tables, &compile_table);
utarray_sort(compile_state->hit_compile_tables, compare_compile_id);
}
}
static int compile_runtime_get_compile_table_id(struct compile_runtime *compile_rt,
long long compile_id)
{
if (NULL == compile_rt || compile_id < 0) {
return -1;
}
int *table_id = rcu_hash_find(compile_rt->tbl_cfg_hash, (char *)&compile_id,
sizeof(long long));
return *table_id;
}
static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt,
int is_last_scan, int thread_id, int is_last_scan, int thread_id,
struct maat_compile_state *compile_state, struct maat_compile_state *compile_state,
@@ -1071,11 +1127,6 @@ static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt
hit a compile that refer a NOT-logic group in previous scan */ hit a compile that refer a NOT-logic group in previous scan */
user_data_array[ud_result_cnt] = compile->user_data; user_data_array[ud_result_cnt] = compile->user_data;
ud_result_cnt++; ud_result_cnt++;
int table_id = compile_runtime_get_compile_table_id(compile_rt, compile->compile_id);
if (table_id >= 0) {
compile_state_update_hit_compile_table_id(compile_state, compile->compile_id, table_id);
}
} }
} }
} }
@@ -1091,11 +1142,18 @@ static struct compile_item *compile_item_clone(struct compile_item *item)
new_item->compile_id = item->compile_id; new_item->compile_id = item->compile_id;
new_item->declared_clause_num = item->declared_clause_num; new_item->declared_clause_num = item->declared_clause_num;
new_item->ref_schema = item->ref_schema;
new_item->ex_data = ALLOC(void *, 1);
memcpy(new_item->table_name, item->table_name, sizeof(new_item->table_name)); memcpy(new_item->table_name, item->table_name, sizeof(new_item->table_name));
new_item->table_line_len = item->table_line_len; new_item->table_line_len = item->table_line_len;
new_item->table_line = ALLOC(char, new_item->table_line_len); new_item->table_line = ALLOC(char, new_item->table_line_len);
memcpy(new_item->table_line, item->table_line, new_item->table_line_len); memcpy(new_item->table_line, item->table_line, new_item->table_line_len);
if (1 == item->ref_schema->set_flag) {
*(new_item->ex_data) = rule_ex_data_new(item->table_name, item->ref_schema->table_id,
item->table_line, &(item->ref_schema->ex_schema));
}
return new_item; return new_item;
} }
@@ -1231,7 +1289,7 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
if (NULL == compile) { if (NULL == compile) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group_id:%lld from compile_id:%lld failed, compile" "[%s:%d] Remove group_id:%lld from compile_id:%lld failed, compile"
" is not existed.", __FUNCTION__, __LINE__, g2c_item->group_id, " is not exisited.", __FUNCTION__, __LINE__, g2c_item->group_id,
compile_id); compile_id);
return -1; return -1;
} else { } else {
@@ -1287,7 +1345,7 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group_id:%lld from compile_id:%lld failed, " "[%s:%d] Remove group_id:%lld from compile_id:%lld failed, "
"compile is not existed.", __FUNCTION__, __LINE__, "compile is not exisited.", __FUNCTION__, __LINE__,
g2c_item->group_id, compile_id); g2c_item->group_id, compile_id);
return -1; return -1;
} }
@@ -1304,7 +1362,6 @@ struct maat_compile_state *maat_compile_state_new(void)
utarray_new(compile_state->internal_inc_hit_paths, &ut_hit_path_icd); utarray_new(compile_state->internal_inc_hit_paths, &ut_hit_path_icd);
utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->hit_compile_tables, &ut_hit_compile_table_icd);
return compile_state; return compile_state;
} }
@@ -1319,14 +1376,11 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state)
compile_state->compile_rt_version = 0; compile_state->compile_rt_version = 0;
compile_state->this_scan_hit_item_flag = 0; compile_state->this_scan_hit_item_flag = 0;
compile_state->not_clause_hit_flag = 0; compile_state->not_clause_hit_flag = 0;
compile_state->inc_hit_path_flag = 0;
utarray_clear(compile_state->internal_hit_paths); utarray_clear(compile_state->internal_hit_paths);
utarray_clear(compile_state->internal_inc_hit_paths); utarray_clear(compile_state->internal_inc_hit_paths);
utarray_clear(compile_state->all_hit_clauses); utarray_clear(compile_state->all_hit_clauses);
utarray_clear(compile_state->this_scan_hit_clauses); utarray_clear(compile_state->this_scan_hit_clauses);
utarray_clear(compile_state->hit_compile_tables);
} }
void maat_compile_state_free(struct maat_compile_state *compile_state, void maat_compile_state_free(struct maat_compile_state *compile_state,
@@ -1361,12 +1415,6 @@ void maat_compile_state_free(struct maat_compile_state *compile_state,
compile_state->this_scan_hit_clauses = NULL; compile_state->this_scan_hit_clauses = NULL;
} }
if (compile_state->hit_compile_tables != NULL) {
free_bytes += utarray_len(compile_state->hit_compile_tables) * sizeof(struct maat_compile_table);
utarray_free(compile_state->hit_compile_tables);
compile_state->hit_compile_tables = NULL;
}
FREE(compile_state); FREE(compile_state);
free_bytes += sizeof(struct maat_compile_state); free_bytes += sizeof(struct maat_compile_state);
@@ -1493,14 +1541,10 @@ static void maat_compile_state_update_hit_path(struct maat_compile_state *compil
if (compile_state->Nth_scan != Nth_scan) { if (compile_state->Nth_scan != Nth_scan) {
assert(compile_state->this_scan_hit_item_flag == 0); assert(compile_state->this_scan_hit_item_flag == 0);
compile_state->Nth_scan = Nth_scan; compile_state->Nth_scan = Nth_scan;
utarray_clear(compile_state->internal_inc_hit_paths);
utarray_clear(compile_state->this_scan_hit_clauses); utarray_clear(compile_state->this_scan_hit_clauses);
} }
if (1 == compile_state->inc_hit_path_flag) {
compile_state->inc_hit_path_flag = 0;
utarray_clear(compile_state->internal_inc_hit_paths);
}
maat_compile_hit_path_add(compile_state->internal_inc_hit_paths, item_id, group_id, maat_compile_hit_path_add(compile_state->internal_inc_hit_paths, item_id, group_id,
vtable_id, Nth_scan, Nth_item_result); vtable_id, Nth_scan, Nth_item_result);
@@ -1550,25 +1594,46 @@ static void maat_compile_state_update_hit_clause(struct maat_compile_state *comp
} }
} }
int maat_compile_state_get_compile_table_id(struct maat_compile_state *compile_state,
long long compile_id)
{
struct maat_compile_table *compile_table = NULL;
compile_table = utarray_find(compile_state->hit_compile_tables, &compile_id,
compare_compile_id);
if (NULL == compile_table) {
return -1;
}
return compile_table->table_id;
}
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state) int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state)
{ {
return compile_state->not_clause_hit_flag; return compile_state->not_clause_hit_flag;
} }
void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema)
{
if (NULL == compile_rt || NULL == compile_schema ||
(0 == compile_schema->set_flag)) {
return;
}
compile_runtime_user_data_iterate(compile_rt, rule_ex_data_new_cb,
&(compile_schema->ex_schema),
compile_schema->table_id);
}
void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt,
struct compile_schema *schema,
long long compile_id)
{
if (NULL == compile_rt || NULL == schema || compile_id < 0
|| (0 == schema->set_flag)) {
return NULL;
}
struct compile_item *item = NULL;
item = (struct compile_item *)compile_runtime_get_user_data(compile_rt,
compile_id);
if (NULL == item) {
return NULL;
}
void *ex_data = NULL;
schema->ex_schema.dup_func(schema->table_id, &ex_data, item->ex_data,
schema->ex_schema.argl, schema->ex_schema.argp);
return ex_data;
}
static int compile_runtime_add_compile(struct compile_runtime *compile_rt, static int compile_runtime_add_compile(struct compile_runtime *compile_rt,
struct compile_schema *schema, struct compile_schema *schema,
long long compile_id, const char *table_name, long long compile_id, const char *table_name,
@@ -1737,7 +1802,6 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
if (0 == is_valid) { if (0 == is_valid) {
// delete // delete
compile_runtime_del_compile(compile_rt, compile_id); compile_runtime_del_compile(compile_rt, compile_id);
rcu_hash_del(compile_rt->tbl_cfg_hash, (char *)&compile_id, sizeof(long long));
} else { } else {
// add // add
int ret = compile_runtime_add_compile(compile_rt, schema, compile_id, int ret = compile_runtime_add_compile(compile_rt, schema, compile_id,
@@ -1745,14 +1809,6 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
if (ret < 0) { if (ret < 0) {
compile_rt->update_err_cnt++; compile_rt->update_err_cnt++;
} }
int *table_id = ALLOC(int, 1);
*table_id = table_manager_get_table_id(schema->ref_tbl_mgr, table_name);
ret = rcu_hash_add(compile_rt->tbl_cfg_hash, (char *)&compile_id,
sizeof(long long), table_id);
if (ret < 0) {
FREE(table_id);
}
} }
return 0; return 0;
@@ -1905,7 +1961,6 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
compile_rt->bm = new_bool_matcher; compile_rt->bm = new_bool_matcher;
compile_rt->literal2clause_hash = new_literal2clause; compile_rt->literal2clause_hash = new_literal2clause;
rcu_hash_commit(compile_rt->cfg_hash); rcu_hash_commit(compile_rt->cfg_hash);
rcu_hash_commit(compile_rt->tbl_cfg_hash);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL, maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL,
garbage_bool_matcher_free); garbage_bool_matcher_free);
@@ -2070,7 +2125,6 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
tmp_hit_path = compile_state->internal_hit_paths; tmp_hit_path = compile_state->internal_hit_paths;
} else if (type == MAAT_LIST_TYPE_INC) { } else if (type == MAAT_LIST_TYPE_INC) {
tmp_hit_path = compile_state->internal_inc_hit_paths; tmp_hit_path = compile_state->internal_inc_hit_paths;
compile_state->inc_hit_path_flag = 1;
} }
for (i = 0; i < utarray_len(tmp_hit_path); i++) { for (i = 0; i < utarray_len(tmp_hit_path); i++) {
@@ -2085,7 +2139,6 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
for (size_t idx = 0; idx < super_group_cnt; idx++) { for (size_t idx = 0; idx < super_group_cnt; idx++) {
struct maat_hit_group hit_group; struct maat_hit_group hit_group;
hit_group.item_id = internal_path->item_id;
hit_group.group_id = super_group_ids[idx]; hit_group.group_id = super_group_ids[idx];
hit_group.vtable_id = internal_path->vtable_id; hit_group.vtable_id = internal_path->vtable_id;
if (utarray_find(all_hit_groups, &hit_group, compare_hit_group)) { if (utarray_find(all_hit_groups, &hit_group, compare_hit_group)) {
@@ -2167,4 +2220,16 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp
} }
return hit_path_cnt; return hit_path_cnt;
}
void compile_runtime_garbage_collect_routine(void *compile_runtime)
{
if (NULL == compile_runtime) {
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
if (compile_rt->cfg_hash != NULL) {
rcu_hash_garbage_collect_routine(compile_rt->cfg_hash);
}
} }

133
src/maat_expr.c
View File

@@ -60,7 +60,7 @@ struct expr_item {
long long group_id; long long group_id;
char keywords[MAX_KEYWORDS_STR]; char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type; enum expr_type expr_type;
enum expr_match_mode match_mode; enum hs_match_mode match_mode;
int is_hexbin; int is_hexbin;
int is_case_sensitive; int is_case_sensitive;
void *user_data; void *user_data;
@@ -68,7 +68,7 @@ struct expr_item {
}; };
struct expr_runtime { struct expr_runtime {
struct expr_matcher *matcher; struct adapter_hs *hs;
struct rcu_hash_table *item_hash; // <item_id, struct expr_item> struct rcu_hash_table *item_hash; // <item_id, struct expr_item>
long long version; //expr_rt version long long version; //expr_rt version
@@ -79,7 +79,6 @@ struct expr_runtime {
struct log_handle *logger; struct log_handle *logger;
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
enum maat_expr_engine expr_engine;
int district_num; int district_num;
struct maat_kv_store *district_map; struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map; struct maat_kv_store *tmp_district_map;
@@ -115,22 +114,22 @@ static enum expr_type int_to_expr_type(int expr_type)
return type; return type;
} }
static enum expr_match_mode int_to_match_mode(int match_method) static enum hs_match_mode int_to_match_mode(int match_method)
{ {
enum expr_match_mode mode = EXPR_MATCH_MODE_INVALID; enum hs_match_mode mode = HS_MATCH_MODE_INVALID;
switch (match_method) { switch (match_method) {
case 0: case 0:
mode = EXPR_MATCH_MODE_SUB; mode = HS_MATCH_MODE_SUB;
break; break;
case 1: case 1:
mode = EXPR_MATCH_MODE_SUFFIX; mode = HS_MATCH_MODE_SUFFIX;
break; break;
case 2: case 2:
mode = EXPR_MATCH_MODE_PREFIX; mode = HS_MATCH_MODE_PREFIX;
break; break;
case 3: case 3:
mode = EXPR_MATCH_MODE_EXACTLY; mode = HS_MATCH_MODE_EXACTLY;
break; break;
default: default:
break; break;
@@ -144,13 +143,13 @@ static int expr_runtime_get_district_id(struct expr_runtime *expr_rt,
{ {
long long district_id = DISTRICT_ANY; long long district_id = DISTRICT_ANY;
int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id, 1); int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id);
if (map_ret < 0) { if (map_ret < 0) {
if (NULL == expr_rt->tmp_district_map) { if (NULL == expr_rt->tmp_district_map) {
expr_rt->tmp_district_map = maat_kv_store_duplicate(expr_rt->district_map); expr_rt->tmp_district_map = maat_kv_store_duplicate(expr_rt->district_map);
} }
map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id, 1); map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id);
if (map_ret < 0) { if (map_ret < 0) {
district_id = expr_rt->district_num; district_id = expr_rt->district_num;
maat_kv_register(expr_rt->tmp_district_map, district, district_id); maat_kv_register(expr_rt->tmp_district_map, district, district_id);
@@ -168,7 +167,7 @@ int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *dis
return -1; return -1;
} }
return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id, 1); return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id);
} }
static struct expr_item * static struct expr_item *
@@ -235,8 +234,8 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name,
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} else if (expr_item->expr_type == EXPR_TYPE_REGEX) { } else if (expr_item->expr_type == EXPR_TYPE_REGEX) {
ret = expr_matcher_verify_regex_expression(expr_item->keywords, expr_rt->logger); ret = adapter_hs_verify_regex_expression(expr_item->keywords, expr_rt->logger);
if (0 == ret) { if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> regex expression(item_id:%lld):%s illegal," "[%s:%d] expr table:<%s> regex expression(item_id:%lld):%s illegal,"
" will be dropped", __FUNCTION__, __LINE__, table_name, " will be dropped", __FUNCTION__, __LINE__, table_name,
@@ -278,7 +277,7 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name,
match_method_type = atoi(line + column_offset); match_method_type = atoi(line + column_offset);
expr_item->match_mode = int_to_match_mode(match_method_type); expr_item->match_mode = int_to_match_mode(match_method_type);
if (expr_item->match_mode == EXPR_MATCH_MODE_INVALID) { if (expr_item->match_mode == HS_MATCH_MODE_INVALID) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> has invalid match_method in line:%s", "[%s:%d] expr table:<%s> has invalid match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
@@ -473,14 +472,12 @@ void *expr_runtime_new(void *expr_schema, size_t max_thread_num,
return NULL; return NULL;
} }
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1); struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->item_hash = rcu_hash_new(expr_item_free_cb, NULL, 0); expr_rt->item_hash = rcu_hash_new(expr_item_free_cb, NULL, 0);
expr_rt->n_worker_thread = max_thread_num; expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin; expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger; expr_rt->logger = logger;
expr_rt->expr_engine = table_manager_get_expr_engine(schema->ref_tbl_mgr);
expr_rt->district_map = maat_kv_store_new(); expr_rt->district_map = maat_kv_store_new();
expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
@@ -498,9 +495,9 @@ void expr_runtime_free(void *expr_runtime)
} }
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
if (expr_rt->matcher != NULL) { if (expr_rt->hs != NULL) {
expr_matcher_free(expr_rt->matcher); adapter_hs_free(expr_rt->hs);
expr_rt->matcher = NULL; expr_rt->hs = NULL;
} }
if (expr_rt->item_hash != NULL) { if (expr_rt->item_hash != NULL) {
@@ -561,18 +558,18 @@ static int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
return 0; return 0;
} }
static enum expr_pattern_type expr_type2pattern_type(enum expr_type expr_type) static enum hs_pattern_type expr_type2pattern_type(enum expr_type expr_type)
{ {
enum expr_pattern_type pattern_type = EXPR_PATTERN_TYPE_STR; enum hs_pattern_type pattern_type;
switch (expr_type) { switch (expr_type) {
case EXPR_TYPE_STRING: case EXPR_TYPE_STRING:
case EXPR_TYPE_AND: case EXPR_TYPE_AND:
case EXPR_TYPE_OFFSET: case EXPR_TYPE_OFFSET:
pattern_type = EXPR_PATTERN_TYPE_STR; pattern_type = HS_PATTERN_TYPE_STR;
break; break;
case EXPR_TYPE_REGEX: case EXPR_TYPE_REGEX:
pattern_type = EXPR_PATTERN_TYPE_REG; pattern_type = HS_PATTERN_TYPE_REG;
break; break;
default: default:
break; break;
@@ -689,12 +686,12 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
} }
sub_expr_cnt = i; sub_expr_cnt = i;
break; break;
case EXPR_TYPE_STRING: //AND/OFFSET/STRING type expression use \b to represent blank(' ') case EXPR_TYPE_STRING:
sub_expr_cnt = 1; sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords; sub_key_array[0] = expr_item->keywords;
sub_key_array[0] = str_unescape(sub_key_array[0]); sub_key_array[0] = str_unescape(sub_key_array[0]);
break; break;
case EXPR_TYPE_REGEX: //only regex type expression use \s to represent blank(' ') case EXPR_TYPE_REGEX:
sub_expr_cnt = 1; sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords; sub_key_array[0] = expr_item->keywords;
break; break;
@@ -713,15 +710,15 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
if (TRUE == expr_item->is_case_sensitive) { if (TRUE == expr_item->is_case_sensitive) {
// insensitive // insensitive
expr_rule->patterns[i].case_sensitive = EXPR_CASE_SENSITIVE; expr_rule->patterns[i].case_sensitive = HS_CASE_SENSITIVE;
} else { } else {
expr_rule->patterns[i].case_sensitive = EXPR_CASE_INSENSITIVE; expr_rule->patterns[i].case_sensitive = HS_CASE_INSENSITIVE;
} }
expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type);
if (TRUE == expr_item->is_hexbin && if (TRUE == expr_item->is_hexbin &&
expr_rule->patterns[i].type != EXPR_PATTERN_TYPE_REG) { expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) {
region_str_len = strlen(sub_key_array[i]) * 8 + 1; region_str_len = strlen(sub_key_array[i]) * 8 + 1;
region_string = ALLOC(char, region_str_len); region_string = ALLOC(char, region_str_len);
region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]), region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]),
@@ -741,13 +738,13 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
} }
expr_rule->patterns[i].match_mode = expr_item->match_mode; expr_rule->patterns[i].match_mode = expr_item->match_mode;
if (expr_rule->patterns[i].match_mode == EXPR_MATCH_MODE_SUB) { if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) {
expr_rule->patterns[i].start_offset = key_left_offset[i]; expr_rule->patterns[i].start_offset = key_left_offset[i];
expr_rule->patterns[i].end_offset = key_right_offset[i]; expr_rule->patterns[i].end_offset = key_right_offset[i];
} }
} }
expr_rule->expr_id = expr_item->item_id; expr_rule->expr_id = expr_item->item_id;
expr_rule->tag = expr_item->user_data; expr_rule->user_tag = expr_item->user_data;
expr_rule->n_patterns = sub_expr_cnt; expr_rule->n_patterns = sub_expr_cnt;
return 0; return 0;
@@ -813,10 +810,10 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
return 0; return 0;
} }
static void garbage_expr_matcher_free(void *expr_matcher, void *arg) static void garbage_adapter_hs_free(void *adapter_hs, void *arg)
{ {
struct expr_matcher *matcher = (struct expr_matcher *)expr_matcher; struct adapter_hs *hs = (struct adapter_hs *)adapter_hs;
expr_matcher_free(matcher); adapter_hs_free(hs);
} }
int expr_runtime_commit(void *expr_runtime, const char *table_name, int expr_runtime_commit(void *expr_runtime, const char *table_name,
@@ -867,42 +864,38 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
} }
} }
struct expr_matcher *new_matcher = NULL; struct adapter_hs *new_adapter_hs = NULL;
struct expr_matcher *old_matcher = NULL; struct adapter_hs *old_adapter_hs = NULL;
if (rule_cnt > 0) { if (rule_cnt > 0) {
enum expr_engine_type engine_type = EXPR_ENGINE_TYPE_HS; new_adapter_hs = adapter_hs_new(rules, real_rule_cnt, expr_rt->n_worker_thread,
if (expr_rt->expr_engine == MAAT_EXPR_ENGINE_RS) { expr_rt->logger);
engine_type = EXPR_ENGINE_TYPE_RS; if (NULL == new_adapter_hs) {
}
new_matcher = expr_matcher_new(rules, real_rule_cnt, engine_type,
expr_rt->n_worker_thread, expr_rt->logger);
if (NULL == new_matcher) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild expr_matcher failed when update" "[%s:%d] table[%s] rebuild adapter_hs engine failed when update"
" %zu expr rules", __FUNCTION__, __LINE__, table_name, real_rule_cnt); " %zu expr rules", __FUNCTION__, __LINE__, table_name, real_rule_cnt);
ret = -1; ret = -1;
} else {
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, maat_rt_version);
} }
} }
old_matcher = expr_rt->matcher; old_adapter_hs = expr_rt->hs;
expr_rt->matcher = new_matcher; expr_rt->hs = new_adapter_hs;
rcu_hash_commit(expr_rt->item_hash); rcu_hash_commit(expr_rt->item_hash);
if (old_matcher != NULL) { if (old_adapter_hs != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_matcher, NULL, garbage_expr_matcher_free); maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL,
garbage_adapter_hs_free);
} }
expr_rt->rule_num = real_rule_cnt; expr_rt->rule_num = real_rule_cnt;
expr_rt->regex_rule_num = real_regex_rule_cnt; expr_rt->regex_rule_num = real_regex_rule_cnt;
expr_rt->version = maat_rt_version; expr_rt->version = maat_rt_version;
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, expr_rt->version);
if (rules != NULL) { if (rules != NULL) {
for (i = 0; i < rule_cnt; i++) { for (i = 0; i < rule_cnt; i++) {
expr_rule_reset(&rules[i]); expr_rule_reset(&rules[i]);
@@ -956,14 +949,15 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
return 0; return 0;
} }
if (NULL == expr_rt->matcher) { if (NULL == expr_rt->hs) {
return 0; return 0;
} }
size_t n_hit_item = 0; size_t n_hit_item = 0;
struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM]; struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = expr_matcher_match(expr_rt->matcher, thread_id, data, data_len, int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item); hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1006,15 +1000,14 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
return real_hit_item_cnt; return real_hit_item_cnt;
} }
struct expr_matcher_stream * struct adapter_hs_stream *
expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
{ {
if (NULL == expr_rt || thread_id < 0) { if (NULL == expr_rt || thread_id < 0) {
return NULL; return NULL;
} }
struct expr_matcher_stream *stream = expr_matcher_stream_open(expr_rt->matcher, struct adapter_hs_stream *stream = adapter_hs_stream_open(expr_rt->hs, thread_id);
thread_id);
if (NULL == stream) { if (NULL == stream) {
return NULL; return NULL;
} }
@@ -1023,7 +1016,7 @@ expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
} }
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
struct expr_matcher_stream *s_handle, struct adapter_hs_stream *s_handle,
const char *data, size_t data_len, const char *data, size_t data_len,
int vtable_id, struct maat_state *state) int vtable_id, struct maat_state *state)
{ {
@@ -1033,10 +1026,10 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
} }
size_t n_hit_item = 0; size_t n_hit_item = 0;
struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM]; struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = expr_matcher_stream_match(s_handle, data, data_len, hit_results, int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results,
MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item); MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1074,13 +1067,13 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
} }
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id, void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,
struct expr_matcher_stream *stream) struct adapter_hs_stream *s_handle)
{ {
if (NULL == expr_rt || thread_id < 0 || NULL == stream) { if (NULL == expr_rt || thread_id < 0 || NULL == s_handle) {
return; return;
} }
expr_matcher_stream_close(stream); adapter_hs_stream_close(s_handle);
} }
void expr_runtime_hit_inc(struct expr_runtime *expr_rt, int thread_id) void expr_runtime_hit_inc(struct expr_runtime *expr_rt, int thread_id)
@@ -1173,4 +1166,4 @@ long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt)
alignment_int64_array_reset(expr_rt->scan_bytes, expr_rt->n_worker_thread); alignment_int64_array_reset(expr_rt->scan_bytes, expr_rt->n_worker_thread);
return sum; return sum;
} }

6
src/maat_flag.c
View File

@@ -269,13 +269,13 @@ static int flag_runtime_get_district_id(struct flag_runtime *flag_rt,
{ {
long long district_id = DISTRICT_ANY; long long district_id = DISTRICT_ANY;
int map_ret = maat_kv_read(flag_rt->district_map, district, &district_id, 1); int map_ret = maat_kv_read(flag_rt->district_map, district, &district_id);
if (map_ret < 0) { if (map_ret < 0) {
if (NULL == flag_rt->tmp_district_map) { if (NULL == flag_rt->tmp_district_map) {
flag_rt->tmp_district_map = maat_kv_store_duplicate(flag_rt->district_map); flag_rt->tmp_district_map = maat_kv_store_duplicate(flag_rt->district_map);
} }
map_ret = maat_kv_read(flag_rt->tmp_district_map, district, &district_id, 1); map_ret = maat_kv_read(flag_rt->tmp_district_map, district, &district_id);
if (map_ret < 0) { if (map_ret < 0) {
district_id = flag_rt->district_num; district_id = flag_rt->district_num;
maat_kv_register(flag_rt->tmp_district_map, district, district_id); maat_kv_register(flag_rt->tmp_district_map, district, district_id);
@@ -293,7 +293,7 @@ int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *dis
return -1; return -1;
} }
return maat_kv_read_unNull(flag_rt->district_map, district, district_len, district_id, 1); return maat_kv_read_unNull(flag_rt->district_map, district, district_len, district_id);
} }
static struct flag_item * static struct flag_item *

6
src/maat_interval.c
View File

@@ -242,13 +242,13 @@ static int interval_runtime_get_district_id(struct interval_runtime *interval_rt
{ {
long long district_id = DISTRICT_ANY; long long district_id = DISTRICT_ANY;
int map_ret = maat_kv_read(interval_rt->district_map, district, &district_id, 1); int map_ret = maat_kv_read(interval_rt->district_map, district, &district_id);
if (map_ret < 0) { if (map_ret < 0) {
if (NULL == interval_rt->tmp_district_map) { if (NULL == interval_rt->tmp_district_map) {
interval_rt->tmp_district_map = maat_kv_store_duplicate(interval_rt->district_map); interval_rt->tmp_district_map = maat_kv_store_duplicate(interval_rt->district_map);
} }
map_ret = maat_kv_read(interval_rt->tmp_district_map, district, &district_id, 1); map_ret = maat_kv_read(interval_rt->tmp_district_map, district, &district_id);
if (map_ret < 0) { if (map_ret < 0) {
district_id = interval_rt->district_num; district_id = interval_rt->district_num;
maat_kv_register(interval_rt->tmp_district_map, district, district_id); maat_kv_register(interval_rt->tmp_district_map, district, district_id);
@@ -268,7 +268,7 @@ int interval_runtime_set_scan_district(struct interval_runtime *interval_rt,
} }
return maat_kv_read_unNull(interval_rt->district_map, district, return maat_kv_read_unNull(interval_rt->district_map, district,
district_len, district_id, 1); district_len, district_id);
} }
static struct interval_item * static struct interval_item *

101
src/maat_kv.c
View File

@@ -17,16 +17,11 @@
#define MAAT_KV_MAX_KEY_LEN 512 #define MAAT_KV_MAX_KEY_LEN 512
#define MAX_ENTITY_ID_CNT 4 #define MAX_ENTITY_ID_CNT 4
struct kv_entity {
long long id[MAX_ENTITY_ID_CNT];
int id_cnt; //0~4
};
struct maat_kv_pair struct maat_kv_pair
{ {
char* key; //must be lower case. char* key; //must be lower case.
size_t key_len; size_t key_len;
struct kv_entity *val; long long val;
UT_hash_handle hh; UT_hash_handle hh;
}; };
@@ -43,32 +38,15 @@ static void strlowercase(const char* src, size_t src_len, char* dst, size_t dst_
} }
static struct maat_kv_pair * static struct maat_kv_pair *
maat_kv_pair_new(const char* key, size_t keylen, long long value) maat_kv_pair_new(const char* key, size_t key_len, long long value)
{ {
struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1); struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1);
kv->key = ALLOC(char, keylen);
kv->key_len = keylen;
kv->val = ALLOC(struct kv_entity, 1);
strlowercase(key, keylen, kv->key, kv->key_len);
kv->val->id[0] = value;
kv->val->id_cnt = 1;
return kv;
}
static struct maat_kv_pair *
maat_kv_pair_clone(const char *key, size_t key_len, struct kv_entity *entity)
{
struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1);
kv->key = ALLOC(char, key_len); kv->key = ALLOC(char, key_len);
kv->key_len = key_len; kv->key_len = key_len;
kv->val = ALLOC(struct kv_entity, 1); kv->val = value;
strlowercase(key, key_len, kv->key, kv->key_len); strlowercase(key, key_len, kv->key, kv->key_len);
*(kv->val) = *entity;
return kv; return kv;
} }
@@ -83,10 +61,6 @@ static void maat_kv_pair_free(struct maat_kv_pair *kv)
FREE(kv->key); FREE(kv->key);
} }
if (kv->val != NULL) {
FREE(kv->val);
}
FREE(kv); FREE(kv);
} }
@@ -114,23 +88,23 @@ void maat_kv_store_free(struct maat_kv_store *store)
} }
static int maat_kv_register_unNull(struct maat_kv_store *store, const char *key, static int maat_kv_register_unNull(struct maat_kv_store *store, const char *key,
size_t keylen, long long value) size_t key_len, long long value)
{ {
if (keylen > MAAT_KV_MAX_KEY_LEN) { if (key_len > MAAT_KV_MAX_KEY_LEN) {
return -1; return -1;
} }
struct maat_kv_pair *kv = NULL; struct maat_kv_pair *kv = NULL;
struct maat_kv_pair *tmp_kv = NULL; struct maat_kv_pair *tmp_kv = NULL;
kv = maat_kv_pair_new(key, keylen, value); kv = maat_kv_pair_new(key, key_len, value);
HASH_FIND(hh, store->hash, kv->key, keylen, tmp_kv); HASH_FIND(hh, store->hash, kv->key, key_len, tmp_kv);
if (tmp_kv) { if (tmp_kv) {
maat_kv_pair_free(kv); maat_kv_pair_free(kv);
return -1; return -1;
} }
HASH_ADD_KEYPTR(hh, store->hash, kv->key, keylen, kv); HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv);
return 1; return 1;
} }
@@ -142,34 +116,31 @@ int maat_kv_register(struct maat_kv_store* store, const char *key, long long val
return ret; return ret;
} }
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t keylen, int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_len,
long long *value_array, size_t n_array) long long *value)
{ {
struct maat_kv_pair *kv = NULL; struct maat_kv_pair *kv = NULL;
char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0}; char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0};
if (keylen > MAAT_KV_MAX_KEY_LEN) { if (key_len > MAAT_KV_MAX_KEY_LEN) {
return -1; return -1;
} }
strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase)); strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase));
HASH_FIND(hh, store->hash, key_lowercase, keylen, kv); HASH_FIND(hh, store->hash, key_lowercase, key_len, kv);
int i = 0; int i = 0;
if (kv) { if (kv) {
for (i = 0; i < kv->val->id_cnt && i < (int)n_array; i++) { *value = kv->val;
value_array[i] = kv->val->id[i]; return 1;
}
return i;
} else { } else {
return -1; return -1;
} }
} }
int maat_kv_read(struct maat_kv_store *store, const char * key, int maat_kv_read(struct maat_kv_store *store, const char * key, long long *value)
long long *value_array, size_t n_array)
{ {
return maat_kv_read_unNull(store, key, strlen(key), value_array, n_array); return maat_kv_read_unNull(store, key, strlen(key), value);
} }
int maat_kv_write_unNull(struct maat_kv_store* store, const char* key, int maat_kv_write_unNull(struct maat_kv_store* store, const char* key,
@@ -185,8 +156,7 @@ int maat_kv_write_unNull(struct maat_kv_store* store, const char* key,
strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase)); strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase));
HASH_FIND(hh, store->hash, key_lowercase, key_len, kv); HASH_FIND(hh, store->hash, key_lowercase, key_len, kv);
if (kv) { if (kv) {
kv->val->id[0] = value; kv->val = value;
kv->val->id_cnt = 1;
} else { } else {
kv = maat_kv_pair_new(key, key_len, value); kv = maat_kv_pair_new(key, key_len, value);
HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv); HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv);
@@ -200,46 +170,13 @@ int maat_kv_write(struct maat_kv_store *store, const char *key, long long value)
return maat_kv_write_unNull(store, key, strlen(key), value); return maat_kv_write_unNull(store, key, strlen(key), value);
} }
int maat_kv_append_unNull(struct maat_kv_store* store, const char* key,
size_t key_len, long long value)
{
struct maat_kv_pair *kv = NULL;
char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0};
if (key_len > MAAT_KV_MAX_KEY_LEN) {
return -1;
}
strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase));
HASH_FIND(hh, store->hash, key_lowercase, key_len, kv);
if (kv) {
size_t id_cnt = kv->val->id_cnt;
if (id_cnt >= MAX_ENTITY_ID_CNT) {
return -1;
}
kv->val->id[id_cnt] = value;
kv->val->id_cnt++;
} else {
kv = maat_kv_pair_new(key, key_len, value);
HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv);
}
return 1;
}
int maat_kv_append(struct maat_kv_store *store, const char *key, long long value)
{
return maat_kv_append_unNull(store, key, strlen(key), value);
}
struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map) struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map)
{ {
struct maat_kv_store *target = maat_kv_store_new(); struct maat_kv_store *target = maat_kv_store_new();
struct maat_kv_pair *kv = NULL, *tmp_kv = NULL, *copy_kv = NULL; struct maat_kv_pair *kv = NULL, *tmp_kv = NULL, *copy_kv = NULL;
HASH_ITER (hh, origin_map->hash, kv, tmp_kv) { HASH_ITER (hh, origin_map->hash, kv, tmp_kv) {
copy_kv = maat_kv_pair_clone(kv->key, kv->key_len, kv->val); copy_kv = maat_kv_pair_new(kv->key, kv->key_len, kv->val);
HASH_ADD_KEYPTR(hh, target->hash, copy_kv->key, copy_kv->key_len, copy_kv); HASH_ADD_KEYPTR(hh, target->hash, copy_kv->key, copy_kv->key_len, copy_kv);
} }

54
src/maat_rule.c
View File

@@ -127,7 +127,8 @@ static int maat_update_cb(const char *table_name, const char *line, void *u_para
return 0; return 0;
} }
struct maat *maat_inst =(struct maat *)u_param; struct maat *maat_inst =(struct maat *)u_param;
struct maat_runtime* maat_rt = NULL;
int table_id = table_manager_get_table_id(maat_inst->tbl_mgr, table_name); int table_id = table_manager_get_table_id(maat_inst->tbl_mgr, table_name);
if (table_id < 0) { if (table_id < 0) {
log_error(maat_inst->logger, MODULE_MAAT_RULE, log_error(maat_inst->logger, MODULE_MAAT_RULE,
@@ -136,36 +137,23 @@ static int maat_update_cb(const char *table_name, const char *line, void *u_para
return -1; return -1;
} }
int update_type = MAAT_UPDATE_TYPE_INC; void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id);
if (maat_inst->creating_maat_rt != NULL) { // Full update if (NULL == schema) {
log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] update warning, table name %s doesn't have table schema",
__FUNCTION__, __LINE__, table_name);
return -1;
}
int update_type = MAAT_UPDATE_TYPE_INC;
if (maat_inst->creating_maat_rt != NULL) { //Full update
maat_rt = maat_inst->creating_maat_rt;
update_type = MAAT_UPDATE_TYPE_FULL; update_type = MAAT_UPDATE_TYPE_FULL;
} } else {
maat_rt = maat_inst->maat_rt;
// find conjunction id for table_id }
long long conj_parent_table_ids[4];
int conj_parent_table_cnt = table_manager_get_conj_parent_table_ids(maat_inst->tbl_mgr, table_name, table_manager_update_runtime(maat_rt->ref_tbl_mgr, table_name, table_id, line, update_type);
conj_parent_table_ids, 4);
if (conj_parent_table_cnt > 0) {
for (int i = 0; i < conj_parent_table_cnt; i++) {
int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name,
(int)conj_parent_table_ids[i], line, update_type);
if (ret < 0) {
log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] table<%s> update runtime error for rule:%s",
__FUNCTION__, __LINE__, table_name, line);
continue;
}
}
} else {
int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name,
table_id, line, update_type);
if (ret < 0) {
log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] table<%s> update runtime error for rules:%s",
__FUNCTION__, __LINE__, table_name, line);
return -1;
}
}
return 0; return 0;
} }
@@ -213,6 +201,10 @@ static void maat_plugin_table_garbage_collect_routine(struct table_manager *tbl_
table_type = table_manager_get_table_type(tbl_mgr, i); table_type = table_manager_get_table_type(tbl_mgr, i);
switch (table_type) { switch (table_type) {
case TABLE_TYPE_COMPILE:
runtime = table_manager_get_runtime(tbl_mgr, i);
compile_runtime_garbage_collect_routine(runtime);
break;
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
runtime = table_manager_get_runtime(tbl_mgr, i); runtime = table_manager_get_runtime(tbl_mgr, i);
ex_data_rt = plugin_runtime_get_ex_data_rt(runtime); ex_data_rt = plugin_runtime_get_ex_data_rt(runtime);
@@ -353,7 +345,7 @@ long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *ke
} }
long long sequence = 1; long long sequence = 1;
int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence, 1); int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence);
if (map_ret < 0) { if (map_ret < 0) {
maat_kv_register(maat_rt->sequence_map, key, sequence); maat_kv_register(maat_rt->sequence_map, key, sequence);
} else { } else {

318
src/maat_table.c
View File

@@ -47,11 +47,9 @@ struct table_manager {
struct rule_tag *accept_tags; struct rule_tag *accept_tags;
size_t n_accept_tag; size_t n_accept_tag;
enum maat_expr_engine expr_engine;
int default_compile_table_id; int default_compile_table_id;
int g2g_table_id; int g2g_table_id;
struct maat_kv_store *tablename2id_map; struct maat_kv_store *tablename2id_map;
struct maat_kv_store *conj_tablename2id_map;
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger; struct log_handle *logger;
@@ -518,7 +516,7 @@ maat_table_new(cJSON *json, struct maat_kv_store *reserved_word_map,
} }
ret = maat_kv_read(reserved_word_map, item->valuestring, ret = maat_kv_read(reserved_word_map, item->valuestring,
(long long *)&(ptable->table_type), 1); (long long *)&(ptable->table_type));
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_TABLE, log_error(logger, MODULE_TABLE,
"[%s:%d] table:%s table_type %s is illegal", "[%s:%d] table:%s table_type %s is illegal",
@@ -580,159 +578,90 @@ static void maat_table_free(struct maat_table *maat_tbl)
FREE(maat_tbl); FREE(maat_tbl);
} }
int register_single_tablename2id(struct maat_kv_store *tablename2id_map, const char *table_name, static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map,
int table_id, struct log_handle *logger) struct log_handle *logger)
{ {
if (strlen(table_name) >= NAME_MAX) { cJSON *item = cJSON_GetObjectItem(json, "table_id");
log_error(logger, MODULE_TABLE,
"[%s:%d] table:<%s> name length exceed maxium:%d",
__FUNCTION__, __LINE__, table_name, NAME_MAX);
return -1;
}
long long tmp_table_id = -1;
int ret = maat_kv_read(tablename2id_map, table_name, &tmp_table_id, 1);
if (ret > 0 && tmp_table_id != table_id) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table:<%s>(table_id:%lld) has already been registered"
", can't register again",
__FUNCTION__, __LINE__, table_name, tmp_table_id);
return -1;
}
maat_kv_register(tablename2id_map, table_name, table_id);
return 0;
}
static int register_conjunction_tablename2id(cJSON *root, struct maat_kv_store *conj_tablename2id_map,
struct log_handle *logger)
{
int json_array_size = cJSON_GetArraySize(root);
for (int i = 0; i < json_array_size; i++) {
cJSON *json = cJSON_GetArrayItem(root, i);
cJSON *item = cJSON_GetObjectItem(json, "table_id");
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "db_tables");
if (NULL == item || item->type != cJSON_Array) {
continue;
}
int n_table_name = cJSON_GetArraySize(item);
for (int j = 0; j < n_table_name; j++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, j);
if (NULL == tmp_item || tmp_item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) db_tables element format invalid"
", should be string", __FUNCTION__, __LINE__, table_id);
return -1;
}
int ret = maat_kv_append(conj_tablename2id_map, tmp_item->valuestring, table_id);
if (ret < 0) {
return -1;
}
}
}
return 0;
}
static int register_tablename2id(cJSON *root, struct maat_kv_store *tablename2id_map,
const char *table_info_path, struct log_handle *logger)
{
int i = 0;
int json_array_size = cJSON_GetArraySize(root);
//pre register tablename2id
for (i = 0; i < json_array_size; i++) {
cJSON *json = cJSON_GetArrayItem(root, i);
if (NULL == json || json->type != cJSON_Object) {
log_error(logger, MODULE_TABLE,
"[%s:%d] %s has invalid json object",
__FUNCTION__, __LINE__, table_info_path);
return -1;
}
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
log_error(logger, MODULE_TABLE,
"[%s:%d] %s has invalid json object, happened in table_id column ",
__FUNCTION__, __LINE__, table_info_path);
return -1;
}
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "table_name");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] %s has invalid json object, happened in table_name column",
__FUNCTION__, __LINE__, table_info_path);
return -1;
}
int ret = register_single_tablename2id(tablename2id_map, item->valuestring,
table_id, logger);
if (ret < 0) {
return -1;
}
}
//register db_tables's table_name
for (i = 0; i < json_array_size; i++) {
cJSON *json = cJSON_GetArrayItem(root, i);
cJSON *item = cJSON_GetObjectItem(json, "table_id");
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "db_tables");
if (NULL == item || item->type != cJSON_Array) {
continue;
}
int n_table_name = cJSON_GetArraySize(item);
for (int j = 0; j < n_table_name; j++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, j);
if (NULL == tmp_item || tmp_item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) db_tables element format invalid"
", should be string", __FUNCTION__, __LINE__, table_id);
return -1;
}
long long tmp_table_id = -1;
int ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &tmp_table_id, 1);
if (ret > 0) {
continue;
}
ret = register_single_tablename2id(tablename2id_map, tmp_item->valuestring,
table_id, logger);
if (ret < 0) {
return -1;
}
}
}
return 0;
}
int maat_default_compile_table_id(cJSON *json, struct log_handle *logger)
{
cJSON *item = cJSON_GetObjectItem(json, "default_compile_table");
if (NULL == item || item->type != cJSON_Number) { if (NULL == item || item->type != cJSON_Number) {
return -1; return -1;
} }
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "table_id"); item = cJSON_GetObjectItem(json, "db_tables");
//item is cJSON_Number which has been checked in maat_table_new if (item != NULL && item->type != cJSON_Array) {
return item->valueint; log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) has db_tables, but format is invalid"
", should be array", __FUNCTION__, __LINE__, table_id);
return -1;
}
int ret = 0;
long long tmp_table_id = 0;
if (item != NULL) {
int n_table_name = cJSON_GetArraySize(item);
for (int i = 0; i < n_table_name; i++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, i);
if (NULL == tmp_item || tmp_item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) db_tables element format invalid"
", should be string", __FUNCTION__, __LINE__, table_id);
return -1;
}
if (strlen(tmp_item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) db_tables string %s length "
"exceed maxium:%d", __FUNCTION__, __LINE__, table_id,
tmp_item->valuestring, NAME_MAX);
return -1;
}
ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &tmp_table_id);
if (ret > 0 && tmp_table_id != table_id) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table:<%s>(table_id:%lld) has already been registered"
", can't register again", __FUNCTION__, __LINE__,
tmp_item->valuestring, tmp_table_id);
return -1;
}
maat_kv_register(tablename2id_map, tmp_item->valuestring, table_id);
}
}
item = cJSON_GetObjectItem(json, "table_name");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) has no table_name",
__FUNCTION__, __LINE__, table_id);
return -1;
}
if (strlen(item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) table_name %s length too long",
__FUNCTION__, __LINE__, table_id, item->valuestring);
return -1;
}
ret = maat_kv_read(tablename2id_map, item->valuestring, &tmp_table_id);
if (ret > 0 && tmp_table_id != table_id) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table:<%s>(table_id:%lld) has already been registered,"
" can't register again", __FUNCTION__, __LINE__, item->valuestring,
tmp_table_id);
return -1;
}
maat_kv_register(tablename2id_map, item->valuestring, table_id);
return 0;
} }
struct table_manager * struct table_manager *
table_manager_create(const char *table_info_path, const char *accept_tags, table_manager_create(const char *table_info_path, const char *accept_tags,
enum maat_expr_engine expr_engine, struct maat_garbage_bin *garbage_bin, struct maat_garbage_bin *garbage_bin, struct log_handle *logger)
struct log_handle *logger)
{ {
if (NULL == table_info_path) { if (NULL == table_info_path) {
return NULL; return NULL;
@@ -773,28 +702,23 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger); tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger);
tbl_mgr->logger = logger; tbl_mgr->logger = logger;
tbl_mgr->tablename2id_map = maat_kv_store_new(); tbl_mgr->tablename2id_map = maat_kv_store_new();
tbl_mgr->conj_tablename2id_map = maat_kv_store_new();
tbl_mgr->expr_engine = expr_engine;
tbl_mgr->ref_garbage_bin = garbage_bin; tbl_mgr->ref_garbage_bin = garbage_bin;
ret = register_tablename2id(root, tbl_mgr->tablename2id_map, table_info_path, logger); for (int i = 0; i < json_array_size; i++) {
if (ret < 0) { json = cJSON_GetArrayItem(root, i);
log_error(logger, MODULE_TABLE,
"[%s:%d] register_tablename2id failed.", __FUNCTION__, __LINE__);
FREE(json_buff);
cJSON_Delete(root);
table_manager_destroy(tbl_mgr);
return NULL;
}
ret = register_conjunction_tablename2id(root, tbl_mgr->conj_tablename2id_map, logger); if (json != NULL && json->type == cJSON_Object) {
if (ret < 0) { ret = register_tablename2id(json, tbl_mgr->tablename2id_map, logger);
log_error(logger, MODULE_TABLE, if (ret < 0) {
"[%s:%d] register_conjunction_tablename2id failed.", __FUNCTION__, __LINE__); log_error(logger, MODULE_TABLE,
FREE(json_buff); "[%s:%d] register_tablename2id failed",
cJSON_Delete(root); __FUNCTION__, __LINE__);
table_manager_destroy(tbl_mgr); FREE(json_buff);
return NULL; cJSON_Delete(root);
table_manager_destroy(tbl_mgr);
return NULL;
}
}
} }
int default_compile_table_id = -1; int default_compile_table_id = -1;
@@ -812,27 +736,22 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
goto next; goto next;
} }
long long parent_table_ids[4]; maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name,
int parent_table_cnt = table_manager_get_conj_parent_table_ids(tbl_mgr, maat_tbl->table_name, maat_tbl->table_type, tbl_mgr, logger);
parent_table_ids, 4); if (NULL == maat_tbl->schema) {
if (parent_table_cnt <= 0) { log_error(logger, MODULE_TABLE,
// if table has conjuncion parent, which can share schema and "[%s:%d] Maat table schema new failed, table_name:%s",
// runtime with parent. __FUNCTION__, __LINE__, maat_tbl->table_name);
maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type, ret = -1;
tbl_mgr, logger); goto next;
if (NULL == maat_tbl->schema) {
log_error(logger, MODULE_TABLE,
"[%s:%d] Maat table schema new failed, table_name:%s",
__FUNCTION__, __LINE__, maat_tbl->table_name);
ret = -1;
goto next;
}
} }
if (maat_tbl->table_type == TABLE_TYPE_COMPILE) { if (maat_tbl->table_type == TABLE_TYPE_COMPILE) {
if (default_compile_table_id < 0) { if (default_compile_table_id < 0) {
default_compile_table_id = maat_default_compile_table_id(json, logger); default_compile_table_id = maat_tbl->table_id;
} } else if (maat_tbl->table_id < default_compile_table_id) {
default_compile_table_id = maat_tbl->table_id;
}
} }
if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) { if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) {
@@ -997,11 +916,6 @@ void table_manager_destroy(struct table_manager *tbl_mgr)
tbl_mgr->tablename2id_map = NULL; tbl_mgr->tablename2id_map = NULL;
} }
if (tbl_mgr->conj_tablename2id_map != NULL) {
maat_kv_store_free(tbl_mgr->conj_tablename2id_map);
tbl_mgr->conj_tablename2id_map = NULL;
}
FREE(tbl_mgr); FREE(tbl_mgr);
} }
@@ -1019,14 +933,14 @@ size_t table_manager_table_count(struct table_manager *tbl_mgr)
return tbl_mgr->n_table; return tbl_mgr->n_table;
} }
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_name) int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
{ {
if (NULL == tbl_mgr || NULL == table_name) { if (NULL == tbl_mgr || NULL == name) {
return -1; return -1;
} }
long long table_id = -1; long long table_id = -1;
int ret = maat_kv_read(tbl_mgr->tablename2id_map, table_name, &table_id, 1); int ret = maat_kv_read(tbl_mgr->tablename2id_map, name, &table_id);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1034,17 +948,6 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_
return (int)table_id; return (int)table_id;
} }
int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name,
long long *table_ids_array, size_t n_table_ids_array)
{
if (NULL == tbl_mgr || NULL == table_name) {
return -1;
}
return maat_kv_read(tbl_mgr->conj_tablename2id_map, table_name,
table_ids_array, n_table_ids_array);
}
const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id) const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id)
{ {
if (NULL == tbl_mgr || table_id < 0) { if (NULL == tbl_mgr || table_id < 0) {
@@ -1115,15 +1018,6 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->valid_column; return tbl_mgr->tbl[table_id]->valid_column;
} }
enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr)
{
if (NULL == tbl_mgr) {
return MAAT_EXPR_ENGINE_HS;
}
return tbl_mgr->expr_engine;
}
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr) size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr)
{ {
if (NULL == tbl_mgr) { if (NULL == tbl_mgr) {

13
test/CMakeLists.txt
View File

@@ -3,9 +3,7 @@ include_directories(${PROJECT_SOURCE_DIR}/include)
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
include_directories(${PROJECT_SOURCE_DIR}/deps) include_directories(${PROJECT_SOURCE_DIR}/deps)
include_directories(${PROJECT_SOURCE_DIR}/scanner) include_directories(${PROJECT_SOURCE_DIR}/scanner)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_rs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher)
@@ -21,8 +19,8 @@ target_link_libraries(maat_framework_gtest maat_frame_static gtest_static)
add_executable(maat_framework_perf_gtest maat_framework_perf_gtest.cpp) add_executable(maat_framework_perf_gtest maat_framework_perf_gtest.cpp)
target_link_libraries(maat_framework_perf_gtest maat_frame_static gtest_static) target_link_libraries(maat_framework_perf_gtest maat_frame_static gtest_static)
add_executable(expr_matcher_gtest expr_matcher_gtest.cpp) add_executable(adapter_hs_gtest adapter_hs_gtest.cpp)
target_link_libraries(expr_matcher_gtest maat_frame_static gtest_static) target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static)
add_executable(ip_matcher_gtest ip_matcher_gtest.cpp) add_executable(ip_matcher_gtest ip_matcher_gtest.cpp)
target_link_libraries(ip_matcher_gtest maat_frame_static gtest_static) target_link_libraries(ip_matcher_gtest maat_frame_static gtest_static)
@@ -34,7 +32,6 @@ add_executable(maat_ex_data_gtest maat_ex_data_gtest.cpp)
target_link_libraries(maat_ex_data_gtest maat_frame_static gtest_static) target_link_libraries(maat_ex_data_gtest maat_frame_static gtest_static)
add_subdirectory(group_exclude) add_subdirectory(group_exclude)
add_subdirectory(benchmark)
file(COPY table_info.conf DESTINATION ./) file(COPY table_info.conf DESTINATION ./)
file(COPY tsg_table_info.conf DESTINATION ./) file(COPY tsg_table_info.conf DESTINATION ./)
@@ -42,13 +39,13 @@ file(COPY file_test_tableinfo.conf DESTINATION ./)
file(COPY literal_expr.conf DESTINATION ./) file(COPY literal_expr.conf DESTINATION ./)
file(COPY regex_expr.conf DESTINATION ./) file(COPY regex_expr.conf DESTINATION ./)
file(COPY maat_json.json DESTINATION ./) file(COPY maat_json.json DESTINATION ./)
file(COPY maat_json.json DESTINATION ../tools/)
file(COPY ntcrule DESTINATION ./) file(COPY ntcrule DESTINATION ./)
file(COPY tsgrule DESTINATION ./) file(COPY tsgrule DESTINATION ./)
file(COPY testdata DESTINATION ./) file(COPY testdata DESTINATION ./)
file(COPY test_streamfiles DESTINATION ./) file(COPY test_streamfiles DESTINATION ./)
file(COPY json_update DESTINATION ./) file(COPY json_update DESTINATION ./)
file(COPY group_exclude DESTINATION ./) file(COPY group_exclude DESTINATION ./)
file(COPY benchmark DESTINATION ./)
include(GoogleTest) include(GoogleTest)
gtest_discover_tests(maat_framework_gtest) gtest_discover_tests(maat_framework_gtest)

730
test/adapter_hs_gtest.cpp Normal file
View File

@@ -0,0 +1,730 @@
#include <gtest/gtest.h>
#include "log/log.h"
#include "adapter_hs.h"
#include "maat_utils.h"
#include "cJSON/cJSON.h"
struct log_handle *g_logger = NULL;
enum hs_match_mode match_method_to_match_mode(const char *method)
{
enum hs_match_mode mode = HS_MATCH_MODE_INVALID;
if (strcmp(method, "sub") == 0) {
mode = HS_MATCH_MODE_SUB;
} else if (strcmp(method, "exactly") == 0) {
mode = HS_MATCH_MODE_EXACTLY;
} else if (strcmp(method, "prefix") == 0) {
mode = HS_MATCH_MODE_PREFIX;
} else if (strcmp(method, "suffix") == 0) {
mode = HS_MATCH_MODE_SUFFIX;
} else {
assert(0);
}
return mode;
}
enum hs_case_sensitive case_sensitive_str_to_enum(const char *str)
{
enum hs_case_sensitive case_sensitive = HS_CASE_SENSITIVE;
if (strcmp(str, "yes") == 0) {
case_sensitive = HS_CASE_SENSITIVE;
} else if (strcmp(str, "no") == 0) {
case_sensitive = HS_CASE_INSENSITIVE;
} else {
assert(0);
}
return case_sensitive;
}
int is_hexbin_str_to_int(const char *str)
{
int ret = 0;
if (strcmp(str, "yes") == 0) {
ret = 1;
}
return ret;
}
static int convertHextoint(char srctmp)
{
if (isdigit(srctmp)) {
return srctmp - '0';
} else {
char temp = toupper(srctmp);
temp = temp - 'A' + 10;
return temp;
}
}
static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size)
{
size_t resultlen = 0;
int high,low;
for (int i = 0; i < hex_len && size > resultlen; i += 2, resultlen++) {
high = convertHextoint(hex[i]);
low = convertHextoint(hex[i+1]);
binary[resultlen] = high * 16 + low;
}
size = resultlen;
binary[resultlen] = '\0';
return resultlen;
}
enum hs_pattern_type pattern_type_str_to_enum(const char *str)
{
enum hs_pattern_type pattern_type;
if (strcmp(str, "regex") == 0) {
pattern_type = HS_PATTERN_TYPE_REG;
} else if (strcmp(str, "literal") == 0) {
pattern_type = HS_PATTERN_TYPE_STR;
} else {
assert(0);
}
return pattern_type;
}
int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_expr)
{
unsigned char *json_buff = NULL;
size_t json_buff_size = 0;
int ret = load_file_to_memory(filename, &json_buff, &json_buff_size);
if (ret < 0) {
printf("load file:%s to memory failed.\n", filename);
return -1;
}
size_t rule_cnt = 0;
cJSON *rules_obj = NULL;
cJSON *root = cJSON_Parse((const char *)json_buff);
if (NULL == root) {
printf("Error before: %-200.200s\n", cJSON_GetErrorPtr());
ret = -1;
goto next;
}
rules_obj = cJSON_GetObjectItem(root, "expr_rules");
if (NULL == rules_obj) {
printf("Error before: %-200.200s\n", cJSON_GetErrorPtr());
ret = -1;
goto next;
}
rule_cnt = cJSON_GetArraySize(rules_obj);
for (size_t i = 0; i < rule_cnt; i++) {
cJSON *expr_obj = cJSON_GetArrayItem(rules_obj, i);
cJSON *tmp_item = cJSON_GetObjectItem(expr_obj, "expr_id");
if (tmp_item != NULL && tmp_item->type == cJSON_Number) {
exprs[i].expr_id = tmp_item->valueint;
}
tmp_item = cJSON_GetObjectItem(expr_obj, "pattern_num");
if (tmp_item != NULL && tmp_item->type == cJSON_Number) {
exprs[i].n_patterns = tmp_item->valueint;
}
tmp_item = cJSON_GetObjectItem(expr_obj, "patterns");
if (NULL == tmp_item || tmp_item->type != cJSON_Array) {
printf("json has no patterns array.\n");
ret = -1;
goto next;
}
size_t pattern_cnt = cJSON_GetArraySize(tmp_item);
for (size_t j = 0; j < pattern_cnt; j++) {
cJSON *pat_item = cJSON_GetArrayItem(tmp_item, j);
cJSON *item = cJSON_GetObjectItem(pat_item, "pattern_type");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].pattern_type = pattern_type_str_to_enum(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "match_method");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].match_mode = match_method_to_match_mode(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "case_sensitive");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].case_sensitive = case_sensitive_str_to_enum(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "is_hexbin");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].is_hexbin = is_hexbin_str_to_int(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "pattern");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].pat = ALLOC(char, strlen(item->valuestring) + 1);
if (exprs[i].patterns[j].is_hexbin == 1) {
size_t pat_str_len = strlen(item->valuestring) + 1;
char *pat_str = ALLOC(char, pat_str_len);
pat_str_len = hex2bin(item->valuestring, strlen(item->valuestring),
pat_str, pat_str_len);
memcpy(exprs[i].patterns[j].pat, pat_str, pat_str_len);
free(pat_str);
exprs[i].patterns[j].pat_len = pat_str_len;
} else {
memcpy(exprs[i].patterns[j].pat, item->valuestring,
strlen(item->valuestring));
exprs[i].patterns[j].pat_len = strlen(item->valuestring);
}
}
if (exprs[i].patterns->match_mode == HS_MATCH_MODE_SUB) {
item = cJSON_GetObjectItem(pat_item, "offset");
if (item != NULL && item->type == cJSON_String) {
int key_left_offset = -1;
int key_right_offset = -1;
sscanf(item->valuestring, "%d~%d", &key_left_offset, &key_right_offset);
if (key_left_offset < -1 || key_right_offset < -1) {
printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n",
key_left_offset, key_right_offset);
}
exprs[i].patterns[j].start_offset = key_left_offset;
exprs[i].patterns[j].end_offset = key_right_offset;
} else {
exprs[i].patterns[j].start_offset = -1;
exprs[i].patterns[j].end_offset = -1;
}
}
if (exprs[i].patterns->match_mode == HS_MATCH_MODE_EXACTLY) {
exprs[i].patterns[j].start_offset = 0;
exprs[i].patterns[j].end_offset = exprs[i].patterns[j].pat_len - 1;
}
}
exprs[i].n_patterns = pattern_cnt;
}
*n_expr = rule_cnt;
next:
cJSON_Delete(root);
FREE(json_buff);
return ret;
}
void expr_array_free(struct expr_rule rules[], size_t n_rule)
{
for (size_t i = 0; i < n_rule; i++) {
for (size_t j = 0; j < rules[i].n_patterns; j++) {
if (rules[i].patterns[j].pat != NULL) {
free(rules[i].patterns[j].pat);
rules[i].patterns[j].pat = NULL;
}
}
}
}
TEST(adapter_hs_init, invalid_input_parameter)
{
struct expr_rule rules[64];
size_t n_rule = 0;
struct adapter_hs *hs_instance = adapter_hs_new(NULL, 0, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
n_rule = 1;
rules[0].expr_id = 101;
rules[0].n_patterns = 10;
hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
memset(rules, 0, sizeof(rules));
n_rule = 1;
rules[0].expr_id = 101;
rules[0].n_patterns = 1;
hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
}
TEST(adapter_hs_scan, literal_sub_has_normal_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello aaa";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data2[64] = "Ahello aaa";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 101);
char scan_data3[64] = "Aahello aaa";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 101);
char scan_data4[64] = "Aaahello aaa";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello bbb";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 102);
char scan_data2[64] = "Ahello bbb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 102);
char scan_data3[64] = "Aahello bbb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 102);
char scan_data4[64] = "Aaahello bbb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_sub_has_right_unlimit_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello ccc";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data2[64] = "1234hello ccc";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data3[64] = "12345hello ccc";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 103);
char scan_data4[64] = "12345hello cccAaBb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 103);
char scan_data5[64] = "123456hello cccAaBb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data5, strlen(scan_data5), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 103);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_sub_with_no_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello ddd";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 104);
char scan_data2[64] = "123hello ddd";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 104);
char scan_data3[64] = "123hello ddd456";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 104);
char scan_data4[64] = "helloddd";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_exactly)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello eee";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 105);
char scan_data2[64] = "Ahello eee";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data3[64] = "hello eeeB";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_prefix)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello fff";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 106);
char scan_data2[64] = "Ahello fff";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data3[64] = "Ahello fffBCD";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data4[64] = "hello fffBCD";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 106);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_suffix)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello ggg";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 107);
char scan_data2[64] = "ABChello ggg";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 107);
char scan_data3[64] = "ABChello gggDEF";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data4[64] = "hello gggDEF";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_sub_with_hexbin)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "Content-Type: /html";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 108);
char scan_data2[64] = " html";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_with_chinese)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char data0[64] = "#中国 你好";
struct hs_scan_result result0[64] = {0};
size_t n_result0 = 0;
ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result0, 1);
EXPECT_EQ(result0[0].rule_id, 110);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, same_pattern_different_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char data[64] = "onetoday,anothertoday";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, data, strlen(data), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 112);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, long_scan_data)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
const char* scan_data = "A directed path in a directed graph is a finite or infinite\
sequence of edges which joins a sequence of distinct vertices, but with the added restriction\
that the edges be all directed in the same direction.";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data, strlen(scan_data), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 113);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, regex_expression_check)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./regex_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < n_rule; i++) {
for (size_t j = 0; j < rules[i].n_patterns; j++) {
adapter_hs_verify_regex_expression(rules[i].patterns[j].pat, g_logger);
}
}
expr_array_free(rules, n_rule);
}
int main(int argc, char **argv)
{
int ret = 0;
::testing::InitGoogleTest(&argc, argv);
g_logger = log_handle_create("./adapter_hs_gtest.log", 0);
ret = RUN_ALL_TESTS();
log_handle_destroy(g_logger);
return ret;
}

1330
test/expr_matcher_gtest.cpp
View File

File diff suppressed because it is too large Load Diff

2
test/json_update/corrupted.json
View File

@@ -1,5 +1,5 @@
{ {
"compile_table": "COMPILE_DEFAULT", "compile_table": "COMPILE",
"group_table": "GROUP", "group_table": "GROUP",
"rules": [ "rules": [
{ {

4
test/json_update/new.json
View File

@@ -1,6 +1,6 @@
{ {
"compile_table": "COMPILE_DEFAULT", "compile_table": "COMPILE",
"group2compile_table": "GROUP2COMPILE_DEFAULT", "group2compile_table": "GROUP2COMPILE",
"group2group_table": "GROUP2GROUP", "group2group_table": "GROUP2GROUP",
"rules": [ "rules": [
{ {

4
test/json_update/old.json
View File

@@ -1,6 +1,6 @@
{ {
"compile_table": "COMPILE_DEFAULT", "compile_table": "COMPILE",
"group2compile_table": "GROUP2COMPILE_DEFAULT", "group2compile_table": "GROUP2COMPILE",
"group2group_table": "GROUP2GROUP", "group2group_table": "GROUP2GROUP",
"rules": [ "rules": [
{ {

30
test/literal_expr.conf
View File

@@ -5,7 +5,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -19,7 +18,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -33,7 +31,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -47,7 +44,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -60,7 +56,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "exactly", "match_method": "exactly",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -73,7 +68,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "prefix", "match_method": "prefix",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -86,7 +80,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "suffix", "match_method": "suffix",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -99,7 +92,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "yes", "is_hexbin": "yes",
@@ -113,14 +105,12 @@
"pattern_num": 2, "pattern_num": 2,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
"pattern": "multi" "pattern": "multi"
}, },
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -133,7 +123,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -146,7 +135,6 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -159,7 +147,6 @@
"pattern_num": 2, "pattern_num": 2,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -167,7 +154,6 @@
"offset": "3~7" "offset": "3~7"
}, },
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -181,26 +167,12 @@
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "literal",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
"pattern": "a finite or infinite" "pattern": "a finite or infinite"
} }
] ]
},
{
"expr_id": 114,
"pattern_num": 1,
"patterns": [
{
"pattern_type": "regex",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
"pattern": "query=(.*)"
}
]
} }
] ]
} }

2
test/maat_ex_data_gtest.cpp
View File

@@ -163,4 +163,4 @@ int main(int argc, char ** argv)
maat_free(g_maat_inst); maat_free(g_maat_inst);
g_maat_inst = NULL; g_maat_inst = NULL;
return ret; return ret;
} }

1450
test/maat_framework_gtest.cpp
View File

File diff suppressed because it is too large Load Diff

575
test/maat_framework_perf_gtest.cpp
View File

@@ -55,6 +55,7 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
char *buff = ALLOC(char, strlen(line) + 1); char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1); memcpy(buff, line, strlen(line) + 1);
while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') {
buff[strlen(buff) - 1] = '\0'; buff[strlen(buff) - 1] = '\0';
} }
@@ -256,79 +257,33 @@ static int ip_table_set_line(struct maat *maat_inst, const char *table_name, enu
return maat_cmd_set_line(maat_inst, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
static int integer_table_set_line(struct maat *maat_inst, const char *table_name, void test_add_expr_command(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id, const char *keywords)
long long group_id, int low_bound, int up_bound,
int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) {
return 0;
}
sprintf(table_line, "%lld\t%lld\t%d\t%d\t%d",
item_id, group_id, low_bound, up_bound, op);
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
static int flag_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, long long flag,
long long flag_mask, int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) {
return 0;
}
sprintf(table_line, "%lld\t%lld\t%lld\t%lld\t%d",
item_id, group_id, flag, flag_mask, op);
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
static void test_add_expr_command(struct maat *maat_inst, const char *table_name,
const char *keywords)
{ {
long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, 0); compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1);
ret = expr_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id, ret = expr_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id,
keywords, "null", 1, 0, 0, 0); keywords, NULL, 1, 0, 0, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
} }
static void test_add_ip_command(struct maat *maat_inst, const char *table_name, void test_add_ip_command(struct maat *maat_inst, const char *table_name,
const char *ip, uint16_t port) const char *ip, uint16_t port)
{ {
long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, 0); compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
@@ -338,42 +293,6 @@ static void test_add_ip_command(struct maat *maat_inst, const char *table_name,
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
} }
static void test_add_integer_command(struct maat *maat_inst, const char *table_name,
int low_bound, int up_bound)
{
long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0);
EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1);
ret = integer_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id,
low_bound, up_bound, 0);
EXPECT_EQ(ret, 1);
}
static void test_add_flag_command(struct maat *maat_inst, const char *table_name,
long long flag, long long flag_mask)
{
long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0);
EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1);
ret = flag_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id,
flag, flag_mask, 0);
EXPECT_EQ(ret, 1);
}
class MaatPerfStringScan : public testing::Test class MaatPerfStringScan : public testing::Test
{ {
protected: protected:
@@ -426,7 +345,7 @@ void *perf_string_scan_thread(void *arg)
struct maat *maat_inst = param->maat_inst; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
struct timespec start, end; struct timespec start, end;
const char *scan_data = "today and yesterday should hit"; const char *scan_data = "String TEST should hit";
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
int hit_times = 0; int hit_times = 0;
size_t n_hit_result = 0; size_t n_hit_result = 0;
@@ -474,67 +393,13 @@ void *perf_string_update_thread(void *arg)
return is_all_hit; return is_all_hit;
} }
void *perf_regex_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name;
struct timespec start, end;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567";
long long results[ARRAY_SIZE] = {0};
int hit_times = 0;
size_t n_hit_result = 0;
struct maat_state *state = maat_state_new(maat_inst, param->thread_id);
int table_id = maat_get_table_id(maat_inst, table_name);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) {
int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
if (ret == MAAT_SCAN_HIT) {
hit_times++;
}
maat_state_reset(state);
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d regex_scan time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void *perf_regex_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10;
char keyword_buf[128];
for (int i = 0; i < CMD_EXPR_NUM; i++) {
random_keyword_generate(keyword_buf, sizeof(keyword_buf));
test_add_expr_command(maat_inst, table_name, keyword_buf);
sleep(1);
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;
return is_all_hit;
}
void *perf_ip_scan_thread(void *arg) void *perf_ip_scan_thread(void *arg)
{ {
struct thread_param *param = (struct thread_param *)arg; struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
struct timespec start, end; struct timespec start, end;
char ip_str[32] = "10.0.0.1"; char ip_str[32] = "10.0.7.100";
uint32_t ip_addr; uint32_t ip_addr;
uint16_t port = htons(65530); uint16_t port = htons(65530);
@@ -597,113 +462,8 @@ void *perf_ip_update_thread(void *arg)
return is_all_hit; return is_all_hit;
} }
void *perf_integer_scan_thread(void *arg) TEST_F(MaatPerfStringScan, MultiThread) {
{ const char *table_name = "KEYWORDS_TABLE";
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name;
struct timespec start, end;
int hit_times = 0;
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = maat_state_new(maat_inst, param->thread_id);
int table_id = maat_get_table_id(maat_inst, table_name);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) {
int ret = maat_scan_integer(maat_inst, table_id, 3000, results,
ARRAY_SIZE, &n_hit_result, state);
if (ret == MAAT_SCAN_HIT) {
hit_times++;
}
maat_state_reset(state);
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d integer_scan time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void *perf_integer_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10;
for (int i = 0; i < CMD_EXPR_NUM; i++) {
test_add_integer_command(maat_inst, table_name, 3001+i, 3001+i);
sleep(1);
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;
return is_all_hit;
}
void *perf_flag_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name;
struct timespec start, end;
int hit_times = 0;
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
long long scan_data = 15;
struct maat_state *state = maat_state_new(maat_inst, param->thread_id);
int table_id = maat_get_table_id(maat_inst, table_name);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) {
int ret = maat_scan_flag(maat_inst, table_id, scan_data, results,
ARRAY_SIZE, &n_hit_result, state);
if (ret == MAAT_SCAN_HIT) {
hit_times++;
}
maat_state_reset(state);
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d flag_scan time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void *perf_flag_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10;
for (int i = 0; i < CMD_EXPR_NUM; i++) {
test_add_flag_command(maat_inst, table_name, i, 15);
sleep(1);
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;
return is_all_hit;
}
TEST_F(MaatPerfStringScan, LiteralMultiThread) {
const char *table_name = "EXPR_LITERAL_PERF_CONFIG";
struct maat *maat_inst = MaatPerfStringScan::_shared_maat_inst; struct maat *maat_inst = MaatPerfStringScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
@@ -743,58 +503,12 @@ TEST_F(MaatPerfStringScan, LiteralMultiThread) {
free(is_all_hit); free(is_all_hit);
} }
scan_per_second = scan_count * 1000 / time_elapse_ms; scan_per_second = scan_count * 1000 / time_elapse_ms;
//EXPECT_GT(scan_per_second, 800 * 1000);
log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"StringScan match rate on %d-threads speed %lld lookups/s/thread", "StringScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second); PERF_THREAD_NUM, scan_per_second);
} }
TEST_F(MaatPerfStringScan, RegexMultiThread) {
const char *table_name = "EXPR_REGEX_PERF_CONFIG";
struct maat *maat_inst = MaatPerfStringScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_regex_scan_thread, thread_params+i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_regex_update_thread, thread_params+i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"RegexScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfStreamScan : public testing::Test class MaatPerfStreamScan : public testing::Test
{ {
protected: protected:
@@ -847,31 +561,34 @@ void *perf_stream_scan_thread(void *arg)
struct maat *maat_inst = param->maat_inst; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
struct timespec start, end; struct timespec start, end;
const char *scan_data = "http://www.cyberessays.com/search_results.php?today and yesterday"; const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
int ret = 0, hit_times = 0; int ret = 0, hit_times = 0;
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state_array[ARRAY_SIZE];
struct maat_stream *sp[ARRAY_SIZE];
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
struct maat_state *state = maat_state_new(maat_inst, param->thread_id);
struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state);
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) { for (int i = 0; i < param->test_count; i++) {
ret = maat_stream_scan(sp, scan_data, strlen(scan_data), results, ARRAY_SIZE, for (int j = 0; j < ARRAY_SIZE; j++) {
&n_hit_result, state); state_array[j] = maat_state_new(maat_inst, param->thread_id);
if (ret == MAAT_SCAN_HIT) { sp[j] = maat_stream_new(maat_inst, table_id, state_array[j]);
hit_times++;
ret = maat_stream_scan(sp[j], scan_data, strlen(scan_data), results, ARRAY_SIZE,
&n_hit_result, state_array[j]);
if (ret == MAAT_SCAN_HIT) {
hit_times++;
}
maat_stream_free(sp[j]);
maat_state_free(state_array[j]);
} }
maat_state_reset(state);
} }
clock_gettime(CLOCK_MONOTONIC, &end); clock_gettime(CLOCK_MONOTONIC, &end);
maat_stream_free(sp);
maat_state_free(state);
param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000; param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
int *is_all_hit = ALLOC(int, 1); int *is_all_hit = ALLOC(int, 1);
*is_all_hit = ((hit_times == param->test_count) ? 1 : 0); *is_all_hit = ((hit_times == param->test_count*ARRAY_SIZE) ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d stream_scan time_elapse:%lldms hit_times:%d", "thread_id:%d stream_scan time_elapse:%lldms hit_times:%d",
@@ -879,40 +596,64 @@ void *perf_stream_scan_thread(void *arg)
return is_all_hit; return is_all_hit;
} }
void *perf_stream_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10;
char keyword_buf[128];
for (int i = 0; i < CMD_EXPR_NUM; i++) {
random_keyword_generate(keyword_buf, sizeof(keyword_buf));
test_add_expr_command(maat_inst, table_name, keyword_buf);
sleep(1);
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;
return is_all_hit;
}
TEST_F(MaatPerfStreamScan, MultiThread) { TEST_F(MaatPerfStreamScan, MultiThread) {
const char *table_name = "EXPR_LITERAL_PERF_CONFIG"; const char *table_name = "HTTP_URL";
struct maat *maat_inst = MaatPerfStreamScan::_shared_maat_inst; struct maat *maat_inst = MaatPerfStreamScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM]; pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM]; struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0; int i = 0;
int *is_all_hit = NULL; int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM; i++) { for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_inst = maat_inst; thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i; thread_params[i].thread_id = i;
thread_params[i].table_name = table_name; thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT; thread_params[i].test_count = PERF_SCAN_COUNT / 10;
thread_params[i].time_elapse_ms = 0; thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_stream_scan_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_stream_scan_thread, thread_params+i);
} } else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_stream_update_thread, thread_params+i);
}
} }
long long time_elapse_ms = 0; long long time_elapse_ms = 0;
long long scan_count = 0; long long scan_count = 0;
long long scan_per_second = 0; long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM; i++) { for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit); pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms; time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count; scan_count += thread_params[i].test_count;
//maybe expr_runtime rebuild in stream_scan, so should not expect is_all_hit always 1 //maybe expr_runtime rebuild in stream_scan, so should not expect is_all_hit always 1
EXPECT_EQ(*is_all_hit, 1); //EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit); free(is_all_hit);
} }
scan_per_second = scan_count * 1000 / time_elapse_ms; scan_per_second = scan_count * 1000 / time_elapse_ms;
@@ -970,7 +711,7 @@ struct log_handle *MaatPerfIPScan::logger;
TEST_F(MaatPerfIPScan, MultiThread) TEST_F(MaatPerfIPScan, MultiThread)
{ {
const char *table_name = "IP_PERF_CONFIG"; const char *table_name = "IP_PLUS_CONFIG";
struct maat *maat_inst = MaatPerfIPScan::_shared_maat_inst; struct maat *maat_inst = MaatPerfIPScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
@@ -1016,192 +757,6 @@ TEST_F(MaatPerfIPScan, MultiThread)
PERF_THREAD_NUM, scan_per_second); PERF_THREAD_NUM, scan_per_second);
} }
class MaatPerfIntegerScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_inst);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_inst;
};
struct maat *MaatPerfIntegerScan::_shared_maat_inst;
struct log_handle *MaatPerfIntegerScan::logger;
TEST_F(MaatPerfIntegerScan, MultiThread) {
const char *table_name = "INTEGER_PERF_CONFIG";
struct maat *maat_inst = MaatPerfIntegerScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_integer_scan_thread, thread_params+i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_integer_update_thread, thread_params+i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"IntegerScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfFlagScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_inst);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_inst;
};
struct maat *MaatPerfFlagScan::_shared_maat_inst;
struct log_handle *MaatPerfFlagScan::logger;
TEST_F(MaatPerfFlagScan, MultiThread) {
const char *table_name = "FLAG_PERF_CONFIG";
struct maat *maat_inst = MaatPerfFlagScan::_shared_maat_inst;
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_inst = maat_inst;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = PERF_SCAN_COUNT;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_flag_scan_thread, thread_params+i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_flag_update_thread, thread_params+i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST,
"FlagScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfFQDNPluginScan : public testing::Test class MaatPerfFQDNPluginScan : public testing::Test
{ {
protected: protected:
@@ -1777,4 +1332,4 @@ int main(int argc, char ** argv)
ret=RUN_ALL_TESTS(); ret=RUN_ALL_TESTS();
return ret; return ret;
} }

2
test/maat_input_mode_gtest.cpp
View File

@@ -247,4 +247,4 @@ int main(int argc, char ** argv)
log_handle_destroy(g_logger); log_handle_destroy(g_logger);
return ret; return ret;
} }

243
test/maat_json.json
View File

@@ -1687,6 +1687,28 @@
} }
] ]
}, },
{
"compile_id": 175,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "ipv4_composition.match",
"is_valid": "yes",
"groups": [
{
"group_name": "ipv4_composition.source",
"virtual_table": "COMPOSITION_IP_SOURCE",
"not_flag": 0
},
{
"group_name": "ipv4_composition.destination",
"virtual_table": "COMPOSITION_IP_DESTINATION",
"not_flag": 0
}
]
},
{ {
"compile_id": 176, "compile_id": 176,
"service": 0, "service": 0,
@@ -1719,6 +1741,24 @@
} }
] ]
}, },
{
"compile_id": 177,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "ipv4_composition.session.match",
"is_valid": "yes",
"groups": [
{
"group_name": "ipv4_composition.session",
"virtual_table": "COMPOSITION_IP_SESSION",
"not_flag": 0,
"clause_index": 1
}
]
},
{ {
"compile_id": 178, "compile_id": 178,
"service": 1, "service": 1,
@@ -1779,6 +1819,47 @@
} }
] ]
}, },
{
"compile_id": 180,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "Hierarchy_VirtualWithTwoPhysical",
"is_valid": "yes",
"groups": [
{
"group_name": "FQDN_OBJ1",
"virtual_table": "VIRTUAL_SSL_SNI",
"not_flag": 0,
"clause_index": 0
},
{
"group_name": "FQDN_CAT1",
"virtual_table": "VIRTUAL_SSL_SNI",
"not_flag": 0,
"clause_index": 0
}
]
},
{
"compile_id": 181,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "ipv4_composition.match",
"is_valid": "yes",
"groups": [
{
"group_name": "IPv4-composition-source-only",
"virtual_table": "COMPOSITION_IP_SOURCE",
"not_flag": 0
}
]
},
{ {
"compile_id": 182, "compile_id": 182,
"service": 1, "service": 1,
@@ -1861,6 +1942,28 @@
} }
] ]
}, },
{
"compile_id": 185,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "ipv4_composition.NOT_match",
"is_valid": "yes",
"groups": [
{
"group_name": "IPv4-composition-NOT-client-ip",
"virtual_table": "COMPOSITION_IP_SOURCE",
"not_flag": 0
},
{
"group_name": "IPv4-composition-NOT-server-ip",
"virtual_table": "COMPOSITION_IP_DESTINATION",
"not_flag": 1
}
]
},
{ {
"compile_id": 186, "compile_id": 186,
"service": 1, "service": 1,
@@ -2258,7 +2361,7 @@
"do_blacklist": 1, "do_blacklist": 1,
"do_log": 1, "do_log": 1,
"user_region": "Something:I\\bhave\\ba\\bname,7799", "user_region": "Something:I\\bhave\\ba\\bname,7799",
"compile_table_name": "COMPILE_FIREWALL_DEFAULT", "compile_table_name": "COMPILE_FIREWALL",
"is_valid": "yes", "is_valid": "yes",
"groups": [ "groups": [
{ {
@@ -2870,135 +2973,7 @@
"not_flag": 0 "not_flag": 0
} }
] ]
}, }
{
"compile_id": 211,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "ip_perf_test",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PERF_CONFIG",
"table_content": {
"addr_type": "ipv4",
"addr_format": "range",
"ip1": "10.0.0.1",
"ip2": "10.0.0.6",
"port_format": "range",
"port1": "65530",
"port2": "65535",
"protocol": 6
}
}
],
"not_flag": 0
}
]
},
{
"compile_id": 212,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "integer_perf_test",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "INTEGER_PERF_CONFIG",
"table_type": "interval",
"table_content": {
"low_boundary": 3000,
"up_boundary": 3000
}
}
]
}
]
},
{
"compile_id": 213,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "expr_perf_test",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_name": "EXPR_LITERAL_PERF_CONFIG",
"table_type": "expr",
"table_content": {
"keywords": "today&yesterday",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 214,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"user_region": "flag_perf_test",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "flag",
"table_name": "FLAG_PERF_CONFIG",
"table_content": {
"flag": 15,
"flag_mask": 15
}
}
]
}
]
},
{
"compile_id": 215,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "expr_perf_test",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_name": "EXPR_REGEX_PERF_CONFIG",
"table_type": "expr",
"table_content": {
"keywords": "action=search\\&query=(.*)",
"expr_type": "regex",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
], ],
"plugin_table": [ "plugin_table": [
{ {
@@ -3009,6 +2984,14 @@
"3\t192.168.1.1\t103\t1" "3\t192.168.1.1\t103\t1"
] ]
}, },
{
"table_name": "TEST_PLUGIN_TABLE",
"table_content": [
"1\t3388\t99\t1",
"2\t3355\t66\t1",
"3\tcccc\t11\t1"
]
},
{ {
"table_name": "TEST_PLUGIN_EXDATA_TABLE", "table_name": "TEST_PLUGIN_EXDATA_TABLE",
"table_content": [ "table_content": [

243
test/table_info.conf
View File

@@ -1,10 +1,12 @@
[ [
{ {
"table_id":0, "table_id":0,
"table_name":"COMPILE_DEFAULT", "table_name":"COMPILE",
"db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"],
"table_type":"compile", "table_type":"compile",
"valid_column":8, "valid_column":8,
"custom": { "custom": {
"gc_timeout_s": 3,
"compile_id":1, "compile_id":1,
"tags":6, "tags":6,
"clause_num":9 "clause_num":9
@@ -12,20 +14,22 @@
}, },
{ {
"table_id":1, "table_id":1,
"table_name":"COMPILE_ALIAS", "table_name":"GROUP2COMPILE",
"table_type":"compile", "db_tables":["GROUP2COMPILE_DEFAULT", "GROUP2COMPILE_ALIAS"],
"valid_column":8, "table_type":"group2compile",
"associated_compile_table_id":0,
"valid_column":3,
"custom": { "custom": {
"compile_id":1, "group_id":1,
"tags":6, "compile_id":2,
"clause_num":9 "not_flag":4,
"virtual_table_name":5,
"clause_index":6
} }
}, },
{ {
"table_id":2, "table_id":2,
"table_name":"COMPILE_CONJUNCTION", "table_name":"COMPILE_FIREWALL",
"db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"],
"default_compile_table":1,
"table_type":"compile", "table_type":"compile",
"valid_column":8, "valid_column":8,
"custom": { "custom": {
@@ -36,8 +40,7 @@
}, },
{ {
"table_id":3, "table_id":3,
"table_name":"GROUP2COMPILE", "table_name":"GROUP2COMPILE_FIREWALL",
"db_tables":["GROUP2COMPILE_DEFAULT", "GROUP2COMPILE_ALIAS"],
"table_type":"group2compile", "table_type":"group2compile",
"associated_compile_table_id":2, "associated_compile_table_id":2,
"valid_column":3, "valid_column":3,
@@ -51,43 +54,6 @@
}, },
{ {
"table_id":4, "table_id":4,
"table_name":"COMPILE_FIREWALL_DEFAULT",
"table_type":"compile",
"valid_column":8,
"custom": {
"compile_id":1,
"tags":6,
"clause_num":9
}
},
{
"table_id":5,
"table_name":"COMPILE_FIREWALL_CONJUNCTION",
"db_tables":["COMPILE_FIREWALL_DEFAULT"],
"table_type":"compile",
"valid_column":8,
"custom": {
"compile_id":1,
"tags":6,
"clause_num":9
}
},
{
"table_id":6,
"table_name":"GROUP2COMPILE_FIREWALL",
"table_type":"group2compile",
"associated_compile_table_id":5,
"valid_column":3,
"custom": {
"group_id":1,
"compile_id":2,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":7,
"table_name":"GROUP2GROUP", "table_name":"GROUP2GROUP",
"table_type":"group2group", "table_type":"group2group",
"valid_column":4, "valid_column":4,
@@ -98,33 +64,7 @@
} }
}, },
{ {
"table_id":8, "table_id":5,
"table_name":"COMPILE_PLUGIN",
"db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"],
"table_type":"plugin",
"valid_column":8,
"custom": {
"gc_timeout_s":3,
"key_type":"integer",
"key_len":8,
"key":1
}
},
{
"table_id":9,
"table_name":"COMPILE_FIREWALL_PLUGIN",
"db_tables":["COMPILE_FIREWALL_DEFAULT"],
"table_type":"plugin",
"valid_column":8,
"custom": {
"gc_timeout_s":3,
"key_type":"integer",
"key_len":8,
"key":1
}
},
{
"table_id":10,
"table_name":"HTTP_REGION", "table_name":"HTTP_REGION",
"db_tables":["HTTP_URL", "HTTP_HOST"], "db_tables":["HTTP_URL", "HTTP_HOST"],
"table_type":"expr", "table_type":"expr",
@@ -139,7 +79,7 @@
} }
}, },
{ {
"table_id":11, "table_id":6,
"table_name":"KEYWORDS_TABLE", "table_name":"KEYWORDS_TABLE",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
@@ -153,7 +93,7 @@
} }
}, },
{ {
"table_id":12, "table_id":7,
"table_name":"IP_CONFIG", "table_name":"IP_CONFIG",
"table_type":"ip_plus", "table_type":"ip_plus",
"valid_column":11, "valid_column":11,
@@ -171,7 +111,7 @@
} }
}, },
{ {
"table_id":13, "table_id":8,
"table_name":"CONTENT_SIZE", "table_name":"CONTENT_SIZE",
"table_type":"intval", "table_type":"intval",
"valid_column":5, "valid_column":5,
@@ -183,19 +123,18 @@
} }
}, },
{ {
"table_id":14, "table_id":9,
"table_name":"QD_ENTRY_INFO", "table_name":"QD_ENTRY_INFO",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
"custom": { "custom": {
"gc_timeout_s":3,
"key_type":"integer", "key_type":"integer",
"key_len":8, "key_len":8,
"key":1 "key":1
} }
}, },
{ {
"table_id":15, "table_id":10,
"table_name":"HTTP_SIGNATURE", "table_name":"HTTP_SIGNATURE",
"table_type":"expr_plus", "table_type":"expr_plus",
"valid_column":8, "valid_column":8,
@@ -210,7 +149,7 @@
} }
}, },
{ {
"table_id":16, "table_id":11,
"table_name":"IMAGE_FP", "table_name":"IMAGE_FP",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
@@ -224,12 +163,11 @@
} }
}, },
{ {
"table_id":17, "table_id":12,
"table_name":"TEST_EFFECTIVE_RANGE_TABLE", "table_name":"TEST_EFFECTIVE_RANGE_TABLE",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
"custom": { "custom": {
"gc_timeout_s":3,
"key_type":"integer", "key_type":"integer",
"key_len":8, "key_len":8,
"key":1, "key":1,
@@ -237,12 +175,11 @@
} }
}, },
{ {
"table_id":18, "table_id":13,
"table_name":"TEST_FOREIGN_KEY", "table_name":"TEST_FOREIGN_KEY",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
"custom": { "custom": {
"gc_timeout_s":3,
"key_type":"pointer", "key_type":"pointer",
"key":2, "key":2,
"tag":3, "tag":3,
@@ -250,7 +187,7 @@
} }
}, },
{ {
"table_id":19, "table_id":14,
"table_name":"TEST_PLUGIN_EXDATA_TABLE", "table_name":"TEST_PLUGIN_EXDATA_TABLE",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
@@ -262,19 +199,18 @@
} }
}, },
{ {
"table_id":20, "table_id":15,
"table_name":"IR_INTERCEPT_IP", "table_name":"IR_INTERCEPT_IP",
"table_type":"plugin", "table_type":"plugin",
"valid_column":14, "valid_column":14,
"custom": { "custom": {
"gc_timeout_s":3,
"key_type":"pointer", "key_type":"pointer",
"key":2, "key":2,
"tag":18 "tag":18
} }
}, },
{ {
"table_id":21, "table_id":16,
"table_name":"APP_PAYLOAD", "table_name":"APP_PAYLOAD",
"table_type":"expr_plus", "table_type":"expr_plus",
"valid_column":8, "valid_column":8,
@@ -289,7 +225,7 @@
} }
}, },
{ {
"table_id":22, "table_id":17,
"table_name":"TROJAN_PAYLOAD", "table_name":"TROJAN_PAYLOAD",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
@@ -304,7 +240,7 @@
} }
}, },
{ {
"table_id":23, "table_id":18,
"table_name":"MAIL_ADDR", "table_name":"MAIL_ADDR",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
@@ -318,7 +254,7 @@
} }
}, },
{ {
"table_id":24, "table_id":19,
"table_name":"IP_PLUS_CONFIG", "table_name":"IP_PLUS_CONFIG",
"table_type":"ip_plus", "table_type":"ip_plus",
"valid_column":11, "valid_column":11,
@@ -336,32 +272,32 @@
} }
}, },
{ {
"table_id":25, "table_id":20,
"table_name":"HTTP_RESPONSE_KEYWORDS", "table_name":"HTTP_RESPONSE_KEYWORDS",
"table_type":"virtual", "table_type":"virtual",
"physical_table": "KEYWORDS_TABLE" "physical_table": "KEYWORDS_TABLE"
}, },
{ {
"table_id":26, "table_id":21,
"table_name":"HTTP_REQUEST_HEADER", "table_name":"HTTP_REQUEST_HEADER",
"table_type":"virtual", "table_type":"virtual",
"physical_table": "HTTP_SIGNATURE" "physical_table": "HTTP_SIGNATURE"
}, },
{ {
"table_id":27, "table_id":22,
"table_name":"HTTP_RESPONSE_HEADER", "table_name":"HTTP_RESPONSE_HEADER",
"table_type":"virtual", "table_type":"virtual",
"physical_table": "HTTP_SIGNATURE" "physical_table": "HTTP_SIGNATURE"
}, },
{ {
"table_id":28, "table_id":23,
"table_name":"VIRTUAL_IP_PLUS_TABLE", "table_name":"VIRTUAL_IP_PLUS_TABLE",
"db_tables":["VIRTUAL_IP_PLUS_SOURCE", "VIRTUAL_IP_PLUS_DESTINATION"], "db_tables":["VIRTUAL_IP_PLUS_SOURCE", "VIRTUAL_IP_PLUS_DESTINATION"],
"table_type":"virtual", "table_type":"virtual",
"physical_table": "IP_PLUS_CONFIG" "physical_table": "IP_PLUS_CONFIG"
}, },
{ {
"table_id":29, "table_id":24,
"table_name":"TEST_IP_PLUGIN_WITH_EXDATA", "table_name":"TEST_IP_PLUGIN_WITH_EXDATA",
"table_type":"ip_plugin", "table_type":"ip_plugin",
"valid_column":6, "valid_column":6,
@@ -375,7 +311,7 @@
} }
}, },
{ {
"table_id":30, "table_id":25,
"table_name":"AS_NUMBER", "table_name":"AS_NUMBER",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
@@ -389,19 +325,19 @@
} }
}, },
{ {
"table_id":31, "table_id":26,
"table_name":"SOURCE_IP_ASN", "table_name":"SOURCE_IP_ASN",
"table_type":"virtual", "table_type":"virtual",
"physical_table":"AS_NUMBER" "physical_table":"AS_NUMBER"
}, },
{ {
"table_id":32, "table_id":27,
"table_name":"DESTINATION_IP_ASN", "table_name":"DESTINATION_IP_ASN",
"table_type":"virtual", "table_type":"virtual",
"physical_table":"AS_NUMBER" "physical_table":"AS_NUMBER"
}, },
{ {
"table_id":33, "table_id":28,
"table_name":"GeoLocation", "table_name":"GeoLocation",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
@@ -415,13 +351,13 @@
} }
}, },
{ {
"table_id":34, "table_id":29,
"table_name":"SOURCE_IP_GEO", "table_name":"SOURCE_IP_GEO",
"table_type":"virtual", "table_type":"virtual",
"physical_table":"GeoLocation" "physical_table":"GeoLocation"
}, },
{ {
"table_id":35, "table_id":30,
"table_name":"INTERGER_PLUS", "table_name":"INTERGER_PLUS",
"table_type":"intval_plus", "table_type":"intval_plus",
"valid_column":6, "valid_column":6,
@@ -434,7 +370,7 @@
} }
}, },
{ {
"table_id":36, "table_id":31,
"table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA", "table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA",
"table_type":"fqdn_plugin", "table_type":"fqdn_plugin",
"valid_column":5, "valid_column":5,
@@ -446,7 +382,7 @@
} }
}, },
{ {
"table_id":37, "table_id":32,
"table_name":"APP_ID", "table_name":"APP_ID",
"table_type":"intval", "table_type":"intval",
"valid_column":5, "valid_column":5,
@@ -458,7 +394,7 @@
} }
}, },
{ {
"table_id":38, "table_id":33,
"table_name":"EMPTY_KEYWORD", "table_name":"EMPTY_KEYWORD",
"table_type":"expr", "table_type":"expr",
"valid_column":7, "valid_column":7,
@@ -472,7 +408,7 @@
} }
}, },
{ {
"table_id":39, "table_id":34,
"table_name":"EMPTY_INTERGER", "table_name":"EMPTY_INTERGER",
"table_type":"intval", "table_type":"intval",
"valid_column":5, "valid_column":5,
@@ -484,7 +420,7 @@
} }
}, },
{ {
"table_id":40, "table_id":35,
"table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA", "table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA",
"table_type":"bool_plugin", "table_type":"bool_plugin",
"valid_column":4, "valid_column":4,
@@ -495,7 +431,7 @@
} }
}, },
{ {
"table_id":41, "table_id":36,
"table_name":"FLAG_CONFIG", "table_name":"FLAG_CONFIG",
"table_type":"flag", "table_type":"flag",
"valid_column":5, "valid_column":5,
@@ -507,7 +443,7 @@
} }
}, },
{ {
"table_id":42, "table_id":37,
"table_name":"FLAG_PLUS_CONFIG", "table_name":"FLAG_PLUS_CONFIG",
"table_type":"flag_plus", "table_type":"flag_plus",
"valid_column":6, "valid_column":6,
@@ -520,12 +456,11 @@
} }
}, },
{ {
"table_id":43, "table_id":38,
"table_name":"TEST_PLUGIN_LONG_KEY_TYPE_TABLE", "table_name":"TEST_PLUGIN_LONG_KEY_TYPE_TABLE",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
"custom": { "custom": {
"gc_timeout_s":3,
"key_type":"integer", "key_type":"integer",
"key_len":8, "key_len":8,
"key":2, "key":2,
@@ -533,12 +468,11 @@
} }
}, },
{ {
"table_id":44, "table_id":39,
"table_name":"TEST_PLUGIN_INT_KEY_TYPE_TABLE", "table_name":"TEST_PLUGIN_INT_KEY_TYPE_TABLE",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
"custom": { "custom": {
"gc_timeout_s":3,
"key_type":"integer", "key_type":"integer",
"key_len":4, "key_len":4,
"key":2, "key":2,
@@ -546,91 +480,20 @@
} }
}, },
{ {
"table_id":45, "table_id":40,
"table_name":"TEST_PLUGIN_IP_KEY_TYPE_TABLE", "table_name":"TEST_PLUGIN_IP_KEY_TYPE_TABLE",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,
"custom": { "custom": {
"gc_timeout_s":3,
"key_type":"ip_addr", "key_type":"ip_addr",
"addr_type":1, "addr_type":1,
"key":2 "key":2
} }
}, },
{ {
"table_id":46, "table_id":41,
"table_name":"HTTP_URL_FILTER", "table_name":"HTTP_URL_FILTER",
"table_type":"virtual", "table_type":"virtual",
"physical_table": "HTTP_URL" "physical_table": "HTTP_URL"
},
{
"table_id":47,
"table_name":"IP_PERF_CONFIG",
"table_type":"ip_plus",
"valid_column":11,
"custom": {
"item_id":1,
"group_id":2,
"addr_type":3,
"addr_format":4,
"ip1":5,
"ip2":6,
"port_format":7,
"port1":8,
"port2":9,
"protocol":10
}
},
{
"table_id":48,
"table_name":"INTEGER_PERF_CONFIG",
"table_type":"intval",
"valid_column":5,
"custom": {
"item_id":1,
"group_id":2,
"low_bound":3,
"up_bound":4
}
},
{
"table_id":49,
"table_name":"EXPR_LITERAL_PERF_CONFIG",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":50,
"table_name":"EXPR_REGEX_PERF_CONFIG",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":51,
"table_name":"FLAG_PERF_CONFIG",
"table_type":"flag",
"valid_column":5,
"custom": {
"item_id":1,
"group_id":2,
"flag":3,
"flag_mask":4
}
} }
] ]

28
vendor/CMakeLists.txt vendored
View File

@@ -58,34 +58,6 @@ add_dependencies(hyperscan_runtime_static hyperscan)
set_property(TARGET hyperscan_runtime_static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhs_runtime.a) set_property(TARGET hyperscan_runtime_static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhs_runtime.a)
set_property(TARGET hyperscan_runtime_static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) set_property(TARGET hyperscan_runtime_static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)
#pcre-8.45
ExternalProject_Add(pcre PREFIX pcre
URL ${CMAKE_CURRENT_SOURCE_DIR}/pcre-8.45.tar.gz
CONFIGURE_COMMAND ./configure --prefix=${VENDOR_BUILD}
BUILD_COMMAND make
INSTALL_COMMAND make install
BUILD_IN_SOURCE 1)
ExternalProject_Get_Property(pcre INSTALL_DIR)
file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
#rulescan 3.0.1
ExternalProject_Add(rulescan PREFIX rulescan
URL ${CMAKE_CURRENT_SOURCE_DIR}/rulescan-3.0.1.tar.gz
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS="-fPIC")
ExternalProject_Get_Property(rulescan INSTALL_DIR)
file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
#merge librulescan.a and libpcre.a => librs.a
add_custom_command(OUTPUT ${VENDOR_BUILD}/lib/librs.a
COMMAND ar crsT ${VENDOR_BUILD}/lib/librs.a ${VENDOR_BUILD}/lib/libpcre.a ${VENDOR_BUILD}/lib/librulescan.a
DEPENDS pcre rulescan)
add_custom_target(_merge ALL DEPENDS ${VENDOR_BUILD}/lib/librs.a)
add_library(rulescan_static STATIC IMPORTED GLOBAL)
set_property(TARGET rulescan_static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/librs.a)
# hiredis-1.1.0 # hiredis-1.1.0
ExternalProject_Add(hiredis PREFIX hiredis ExternalProject_Add(hiredis PREFIX hiredis
URL ${CMAKE_CURRENT_SOURCE_DIR}/hiredis-1.1.0.tar.gz URL ${CMAKE_CURRENT_SOURCE_DIR}/hiredis-1.1.0.tar.gz

BIN
vendor/pcre-8.45.tar.gz vendored
View File

Binary file not shown.

BIN
vendor/rulescan-3.0.1.tar.gz vendored
View File

Binary file not shown.