diff --git a/include/maat.h b/include/maat.h index 5099e0a..f9afe61 100644 --- a/include/maat.h +++ b/include/maat.h @@ -37,7 +37,6 @@ struct maat_hit_path { }; struct maat_hit_group { - long long item_id; long long group_id; int vtable_id; }; @@ -60,11 +59,6 @@ enum maat_list_type { MAAT_LIST_TYPE_INC }; -enum maat_expr_engine { - MAAT_EXPR_ENGINE_HS = 0, //default engine(hyperscan) - MAAT_EXPR_ENGINE_RS //rulescan -}; - struct ip_addr { int ip_type; //4: IPv4, 6: IPv6 union { @@ -150,8 +144,6 @@ int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filename); -int maat_options_set_expr_engine(struct maat_options *opts, enum maat_expr_engine engine); - /* maat_instance API */ struct maat *maat_new(struct maat_options *opts, const char *table_info_path); void maat_free(struct maat *instance); @@ -279,9 +271,6 @@ int maat_state_set_last_scan(struct maat_state *state); int maat_state_set_scan_compile_table(struct maat_state *state, int compile_table_id); -int maat_state_get_compile_table_ids(struct maat_state *state, long long *compile_ids, - size_t n_compile_ids, int *compile_table_ids); - int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths, size_t n_path); @@ -307,4 +296,4 @@ int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *grou } #endif -#endif +#endif \ No newline at end of file diff --git a/scanner/CMakeLists.txt b/scanner/CMakeLists.txt index 7d45154..429755e 100644 --- a/scanner/CMakeLists.txt +++ b/scanner/CMakeLists.txt @@ -7,9 +7,8 @@ include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) add_subdirectory(ip_matcher/IntervalIndex) -add_library(adapter-static bool_matcher/bool_matcher.cpp expr_matcher/expr_matcher.cpp - expr_matcher/adapter_hs/adapter_hs.cpp expr_matcher/adapter_rs/adapter_rs.cpp +add_library(adapter-static adapter_hs/adapter_hs.cpp bool_matcher/bool_matcher.cpp fqdn_engine/fqdn_engine.cpp ip_matcher/ip_matcher.cpp ip_matcher/ipv4_match.cpp ip_matcher/ipv6_match.cpp flag_matcher/flag_matcher.cpp interval_matcher/cgranges.c interval_matcher/interval_matcher.cpp) -target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static rulescan_static interval_index_static) \ No newline at end of file +target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static interval_index_static) \ No newline at end of file diff --git a/scanner/expr_matcher/adapter_hs/adapter_hs.cpp b/scanner/adapter_hs/adapter_hs.cpp similarity index 77% rename from scanner/expr_matcher/adapter_hs/adapter_hs.cpp rename to scanner/adapter_hs/adapter_hs.cpp index 4d57dcf..d201e11 100644 --- a/scanner/expr_matcher/adapter_hs/adapter_hs.cpp +++ b/scanner/adapter_hs/adapter_hs.cpp @@ -1,10 +1,10 @@ /* ********************************************************************************************** -* File: adapter_hs.c -* Description: -* Authors: Liu wentan +* File: adapter_hs.cpp +* Description: +* Authors: Liu WenTan * Date: 2022-10-31 -* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved. +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. *********************************************************************************************** */ @@ -20,8 +20,9 @@ #include "uthash/utarray.h" #include "uthash/uthash.h" #include "maat_utils.h" -#include "../../bool_matcher/bool_matcher.h" +#include "../bool_matcher/bool_matcher.h" +#define MAX_OFFSET_NUM 1024 #define MAX_HIT_PATTERN_NUM 512 pid_t hs_gettid() @@ -40,7 +41,6 @@ static const char *hs_module_name_str(const char *name) #define MODULE_ADAPTER_HS hs_module_name_str("maat.adapter_hs") struct adpt_hs_compile_data { - enum expr_pattern_type pat_type; unsigned int *ids; unsigned int *flags; char **patterns; @@ -56,6 +56,7 @@ struct adapter_hs_scratch { struct adapter_hs_stream { int thread_id; + size_t n_expr; hs_stream_t *literal_stream; hs_stream_t *regex_stream; struct adapter_hs_runtime *ref_hs_rt; @@ -90,7 +91,7 @@ struct pattern_offset { struct pattern_attribute { long long pattern_id; - enum expr_match_mode match_mode; + enum hs_match_mode match_mode; struct pattern_offset offset; }; @@ -136,12 +137,12 @@ static int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches, static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread, - enum expr_pattern_type pattern_type, + enum hs_pattern_type pattern_type, struct log_handle *logger) { int ret = 0; - if (pattern_type == EXPR_PATTERN_TYPE_STR) { + if (pattern_type == HS_PATTERN_TYPE_STR) { hs_rt->scratch->literal_scratches = ALLOC(hs_scratch_t *, n_worker_thread); ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->scratch->literal_scratches, n_worker_thread, logger); @@ -199,7 +200,7 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, if (regex_cd != NULL) { err = hs_compile_multi((const char *const *)regex_cd->patterns, regex_cd->flags, regex_cd->ids, regex_cd->n_patterns, - HS_MODE_STREAM, + HS_MODE_STREAM | HS_MODE_SOM_HORIZON_SMALL, NULL, &hs_rt->regex_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { @@ -214,11 +215,9 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, return 0; } -static struct adpt_hs_compile_data * -adpt_hs_compile_data_new(enum expr_pattern_type pat_type, size_t n_patterns) +static struct adpt_hs_compile_data *adpt_hs_compile_data_new(size_t n_patterns) { struct adpt_hs_compile_data *hs_cd = ALLOC(struct adpt_hs_compile_data, 1); - hs_cd->pat_type = pat_type; hs_cd->patterns = ALLOC(char *, n_patterns); hs_cd->pattern_lens = ALLOC(size_t, n_patterns); hs_cd->n_patterns = n_patterns; @@ -264,11 +263,8 @@ static void populate_compile_data(struct adpt_hs_compile_data *compile_data, compile_data->ids[index] = pattern_id; /* set flags */ - if (compile_data->pat_type == EXPR_PATTERN_TYPE_STR) { - compile_data->flags[index] |= HS_FLAG_SOM_LEFTMOST; - } - - if (case_sensitive == EXPR_CASE_INSENSITIVE) { + compile_data->flags[index] |= HS_FLAG_SOM_LEFTMOST; + if (case_sensitive == HS_CASE_INSENSITIVE) { compile_data->flags[index] |= HS_FLAG_CASELESS; } @@ -299,14 +295,14 @@ static struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule, pattern_attr[pattern_index].pattern_id = pattern_index; pattern_attr[pattern_index].match_mode = rules[i].patterns[j].match_mode; - if (pattern_attr[pattern_index].match_mode == EXPR_MATCH_MODE_SUB || - pattern_attr[pattern_index].match_mode == EXPR_MATCH_MODE_EXACTLY) { + if (pattern_attr[pattern_index].match_mode == HS_MATCH_MODE_SUB || + pattern_attr[pattern_index].match_mode == HS_MATCH_MODE_EXACTLY) { pattern_attr[pattern_index].offset.start = rules[i].patterns[j].start_offset; pattern_attr[pattern_index].offset.end = rules[i].patterns[j].end_offset; } /* literal pattern */ - if (rules[i].patterns[j].type == EXPR_PATTERN_TYPE_STR) { + if (rules[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) { populate_compile_data(literal_cd, literal_index, pattern_index, rules[i].patterns[j].pat, rules[i].patterns[j].pat_len, rules[i].patterns[j].case_sensitive); @@ -325,7 +321,7 @@ static struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule, bool_exprs[i].expr_id = rules[i].expr_id; bool_exprs[i].item_num = rules[i].n_patterns; - bool_exprs[i].user_tag = rules[i].tag; + bool_exprs[i].user_tag = rules[i].user_tag; } *n_pattern = pattern_index; @@ -349,43 +345,81 @@ static int verify_regex_expression(const char *regex_str, struct log_handle *log FREE(info); hs_free_compile_error(error); - return 0; + return -1; } if (info != NULL) { FREE(info); } - return 1; + return 0; } int adapter_hs_verify_regex_expression(const char *regex_expr, struct log_handle *logger) { if (NULL == regex_expr) { - return 0; + return -1; } return verify_regex_expression(regex_expr, logger); } -void *adapter_hs_new(struct expr_rule *rules, size_t n_rule, - size_t n_literal_pattern, size_t n_regex_pattern, - size_t n_worker_thread, struct log_handle *logger) +struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule, + size_t n_worker_thread, struct log_handle *logger) { + if (0 == n_worker_thread || NULL == rules || 0 == n_rule) { + log_error(logger, MODULE_ADAPTER_HS, + "[%s:%d] input parameters illegal!", __FUNCTION__, __LINE__); + return NULL; + } + /* get the sum of pattern */ - size_t i = 0; + size_t i = 0, j = 0; + size_t literal_pattern_num = 0; + size_t regex_pattern_num = 0; + for (i = 0; i < n_rule; i++) { + if (rules[i].n_patterns > MAX_EXPR_PATTERN_NUM) { + log_error(logger, MODULE_ADAPTER_HS, + "[%s:%d] the number of patterns in one expression " + "should less than %d", __FUNCTION__, __LINE__, + MAX_EXPR_PATTERN_NUM); + return NULL; + } + + for (j = 0; j < rules[i].n_patterns; j++) { + /* pat_len should not 0 */ + if (0 == rules[i].patterns[j].pat_len) { + log_error(logger, MODULE_ADAPTER_HS, + "[%s:%d] expr pattern length should not 0", + __FUNCTION__, __LINE__); + return NULL; + } + + if (rules[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) { + literal_pattern_num++; + } else { + regex_pattern_num++; + } + } + } + + if (0 == literal_pattern_num && 0 == regex_pattern_num) { + log_error(logger, MODULE_ADAPTER_HS, + "[%s:%d] exprs has no valid pattern", __FUNCTION__, __LINE__); + return NULL; + } + struct adpt_hs_compile_data *literal_cd = NULL; struct adpt_hs_compile_data *regex_cd = NULL; - - if (n_literal_pattern > 0) { - literal_cd = adpt_hs_compile_data_new(EXPR_PATTERN_TYPE_STR, n_literal_pattern); + if (literal_pattern_num > 0) { + literal_cd = adpt_hs_compile_data_new(literal_pattern_num); } - if (n_regex_pattern > 0) { - regex_cd = adpt_hs_compile_data_new(EXPR_PATTERN_TYPE_REG, n_regex_pattern); + if (regex_pattern_num > 0) { + regex_cd = adpt_hs_compile_data_new(regex_pattern_num); } - size_t pattern_cnt = n_literal_pattern + n_regex_pattern; + size_t pattern_cnt = literal_pattern_num + regex_pattern_num; struct adapter_hs *hs_inst = ALLOC(struct adapter_hs, 1); hs_inst->hs_attr = ALLOC(struct pattern_attribute, pattern_cnt); hs_inst->logger = logger; @@ -444,21 +478,21 @@ void *adapter_hs_new(struct expr_rule *rules, size_t n_rule, n_worker_thread); for (i = 0; i < n_worker_thread; i++) { hs_inst->hs_rt->scratch->bool_match_buffs[i] = ALLOC(struct bool_expr_match, - MAX_HIT_EXPR_NUM); + hs_inst->n_expr); } /* literal and regex scratch can't reuse */ - if (n_literal_pattern > 0) { + if (literal_pattern_num > 0) { ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread, - EXPR_PATTERN_TYPE_STR, logger); + HS_PATTERN_TYPE_STR, logger); if (ret < 0) { goto error; } } - if (n_regex_pattern > 0) { + if (regex_pattern_num > 0) { ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread, - EXPR_PATTERN_TYPE_REG, logger); + HS_PATTERN_TYPE_REG, logger); if (ret < 0) { goto error; } @@ -466,7 +500,7 @@ void *adapter_hs_new(struct expr_rule *rules, size_t n_rule, hs_inst->hs_rt->streams = ALLOC(struct adapter_hs_stream *, n_worker_thread); for (i = 0; i < n_worker_thread; i++) { - hs_inst->hs_rt->streams[i] = (struct adapter_hs_stream *)adapter_hs_stream_open(hs_inst, i); + hs_inst->hs_rt->streams[i] = adapter_hs_stream_open(hs_inst, i); } return hs_inst; @@ -475,15 +509,13 @@ error: return NULL; } -void adapter_hs_free(void *hs_instance) +void adapter_hs_free(struct adapter_hs *hs_inst) { - if (NULL == hs_instance) { + if (NULL == hs_inst) { return; } - struct adapter_hs *hs_inst = (struct adapter_hs *)hs_instance; size_t i = 0; - if (hs_inst->hs_rt != NULL) { if (hs_inst->hs_rt->literal_db != NULL) { hs_free_database(hs_inst->hs_rt->literal_db); @@ -593,12 +625,12 @@ static int matched_event_cb(unsigned int id, unsigned long long from, int ret = 0; struct pattern_attribute pat_attr = matched_pat->ref_hs_attr[id]; switch (pat_attr.match_mode) { - case EXPR_MATCH_MODE_EXACTLY: + case HS_MATCH_MODE_EXACTLY: if (0 == from && matched_pat->scan_data_len == to) { ret = 1; } break; - case EXPR_MATCH_MODE_SUB: + case HS_MATCH_MODE_SUB: if (pat_attr.offset.start == -1 && pat_attr.offset.end == -1) { ret = 1; @@ -624,12 +656,12 @@ static int matched_event_cb(unsigned int id, unsigned long long from, ret = 1; } break; - case EXPR_MATCH_MODE_PREFIX: + case HS_MATCH_MODE_PREFIX: if (0 == from) { ret = 1; } break; - case EXPR_MATCH_MODE_SUFFIX: + case HS_MATCH_MODE_SUFFIX: if (to == matched_pat->scan_data_len) { ret = 1; } @@ -646,42 +678,43 @@ static int matched_event_cb(unsigned int id, unsigned long long from, return 0; } -UT_icd ut_hs_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; -void *adapter_hs_stream_open(void *hs_instance, int thread_id) +UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; +struct adapter_hs_stream * +adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id) { if (NULL == hs_instance || thread_id < 0) { return NULL; } - struct adapter_hs *hs_inst = (struct adapter_hs *)hs_instance; struct adapter_hs_stream *hs_stream = ALLOC(struct adapter_hs_stream, 1); hs_error_t err; - hs_stream->logger = hs_inst->logger; + hs_stream->logger = hs_instance->logger; hs_stream->thread_id = thread_id; - hs_stream->ref_hs_rt = hs_inst->hs_rt; + hs_stream->n_expr = hs_instance->n_expr; + hs_stream->ref_hs_rt = hs_instance->hs_rt; hs_stream->matched_pat = ALLOC(struct matched_pattern, 1); - hs_stream->matched_pat->ref_hs_attr = hs_inst->hs_attr; - hs_stream->matched_pat->n_patterns = hs_inst->n_patterns; - utarray_new(hs_stream->matched_pat->pattern_ids, &ut_hs_pattern_id_icd); + hs_stream->matched_pat->ref_hs_attr = hs_instance->hs_attr; + hs_stream->matched_pat->n_patterns = hs_instance->n_patterns; + utarray_new(hs_stream->matched_pat->pattern_ids, &ut_pattern_id_icd); utarray_reserve(hs_stream->matched_pat->pattern_ids, MAX_HIT_PATTERN_NUM); int err_count = 0; - if (hs_inst->hs_rt->literal_db != NULL) { - err = hs_open_stream(hs_inst->hs_rt->literal_db, 0, + if (hs_instance->hs_rt->literal_db != NULL) { + err = hs_open_stream(hs_instance->hs_rt->literal_db, 0, &hs_stream->literal_stream); if (err != HS_SUCCESS) { - log_error(hs_inst->logger, MODULE_ADAPTER_HS, + log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err); err_count++; } } - if (hs_inst->hs_rt->regex_db != NULL) { - err = hs_open_stream(hs_inst->hs_rt->regex_db, 0, + if (hs_instance->hs_rt->regex_db != NULL) { + err = hs_open_stream(hs_instance->hs_rt->regex_db, 0, &hs_stream->regex_stream); if (err != HS_SUCCESS) { - log_error(hs_inst->logger, MODULE_ADAPTER_HS, + log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err); err_count++; } @@ -707,37 +740,36 @@ error: return NULL; } -void adapter_hs_stream_close(void *hs_stream) +void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream) { if (NULL == hs_stream) { return; } - struct adapter_hs_stream *stream = (struct adapter_hs_stream *)hs_stream; - if (stream->ref_hs_rt != NULL) { - if (stream->literal_stream != NULL) { - hs_close_stream(stream->literal_stream, NULL, NULL, NULL); - stream->literal_stream = NULL; + if (hs_stream->ref_hs_rt != NULL) { + if (hs_stream->literal_stream != NULL) { + hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL); + hs_stream->literal_stream = NULL; } - if (stream->regex_stream != NULL) { - hs_close_stream(stream->regex_stream, NULL, NULL, NULL); - stream->regex_stream = NULL; + if (hs_stream->regex_stream != NULL) { + hs_close_stream(hs_stream->regex_stream, NULL, NULL, NULL); + hs_stream->regex_stream = NULL; } } - /* stream->hs_rt point to hs_instance->hs_rt which will call free + /* hs_stream->hs_rt point to hs_instance->hs_rt which will call free same as hs_attr */ - stream->ref_hs_rt = NULL; - stream->matched_pat->ref_hs_attr = NULL; + hs_stream->ref_hs_rt = NULL; + hs_stream->matched_pat->ref_hs_attr = NULL; - if (stream->matched_pat->pattern_ids != NULL) { - utarray_free(stream->matched_pat->pattern_ids); - stream->matched_pat->pattern_ids = NULL; + if (hs_stream->matched_pat->pattern_ids != NULL) { + utarray_free(hs_stream->matched_pat->pattern_ids); + hs_stream->matched_pat->pattern_ids = NULL; } - FREE(stream->matched_pat); - FREE(stream); + FREE(hs_stream->matched_pat); + FREE(hs_stream); } static void adapter_hs_stream_reset(struct adapter_hs_stream *hs_stream) @@ -762,9 +794,9 @@ static void adapter_hs_stream_reset(struct adapter_hs_stream *hs_stream) utarray_clear(hs_stream->matched_pat->pattern_ids); } -int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len, - struct expr_scan_result *results, size_t n_result, - size_t *n_hit_result) +int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, + size_t data_len, struct hs_scan_result *results, + size_t n_result, size_t *n_hit_result) { hs_error_t err; @@ -784,37 +816,36 @@ int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len, */ int err_count = 0; - struct adapter_hs_stream *stream = (struct adapter_hs_stream *)hs_stream; - int thread_id = stream->thread_id; - struct adapter_hs_scratch *scratch = stream->ref_hs_rt->scratch; - stream->matched_pat->scan_data_len = data_len; + int thread_id = hs_stream->thread_id; + struct adapter_hs_scratch *scratch = hs_stream->ref_hs_rt->scratch; + hs_stream->matched_pat->scan_data_len = data_len; int err_scratch_flag = 0; - if (stream->literal_stream != NULL) { + if (hs_stream->literal_stream != NULL) { if (scratch->literal_scratches != NULL) { - err = hs_scan_stream(stream->literal_stream, data, data_len, + err = hs_scan_stream(hs_stream->literal_stream, data, data_len, 0, scratch->literal_scratches[thread_id], - matched_event_cb, stream->matched_pat); + matched_event_cb, hs_stream->matched_pat); if (err != HS_SUCCESS) { err_count++; } } else { - log_error(stream->logger, MODULE_ADAPTER_HS, + log_error(hs_stream->logger, MODULE_ADAPTER_HS, "literal_scratches is null, thread_id:%d", thread_id); err_scratch_flag++; } } - if (stream->regex_stream != NULL) { + if (hs_stream->regex_stream != NULL) { if (scratch->regex_scratches != NULL) { - err = hs_scan_stream(stream->regex_stream, data, data_len, + err = hs_scan_stream(hs_stream->regex_stream, data, data_len, 0, scratch->regex_scratches[thread_id], - matched_event_cb, stream->matched_pat); + matched_event_cb, hs_stream->matched_pat); if (err != HS_SUCCESS) { err_count++; } } else { - log_error(stream->logger, MODULE_ADAPTER_HS, + log_error(hs_stream->logger, MODULE_ADAPTER_HS, "regex_scratches is null, thread_id:%d", thread_id); err_scratch_flag++; } @@ -828,7 +859,7 @@ int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len, return -1; } - size_t n_pattern_id = utarray_len(stream->matched_pat->pattern_ids); + size_t n_pattern_id = utarray_len(hs_stream->matched_pat->pattern_ids); if (0 == n_pattern_id) { *n_hit_result = 0; return 0; @@ -837,13 +868,13 @@ int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len, unsigned long long pattern_ids[n_pattern_id]; for (size_t i = 0; i < n_pattern_id; i++) { - pattern_ids[i] = *(unsigned long long *)utarray_eltptr(stream->matched_pat->pattern_ids, i); + pattern_ids[i] = *(unsigned long long *)utarray_eltptr(hs_stream->matched_pat->pattern_ids, i); } int ret = 0; struct bool_expr_match *bool_matcher_results = scratch->bool_match_buffs[thread_id]; - int bool_matcher_ret = bool_matcher_match(stream->ref_hs_rt->bm, pattern_ids, n_pattern_id, - bool_matcher_results, MAX_HIT_EXPR_NUM); + int bool_matcher_ret = bool_matcher_match(hs_stream->ref_hs_rt->bm, pattern_ids, n_pattern_id, + bool_matcher_results, hs_stream->n_expr); if (bool_matcher_ret < 0) { ret = -1; goto next; @@ -860,21 +891,22 @@ int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len, *n_hit_result = bool_matcher_ret; next: - utarray_clear(stream->matched_pat->pattern_ids); + utarray_clear(hs_stream->matched_pat->pattern_ids); return ret; } -int adapter_hs_scan(void *hs_instance, int thread_id, const char *data, size_t data_len, - struct expr_scan_result *results, size_t n_result, size_t *n_hit_result) +int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, + const char *data, size_t data_len, + struct hs_scan_result *results, + size_t n_result, size_t *n_hit_result) { if (NULL == hs_instance || NULL == data || (0 == data_len) || NULL == results || 0 == n_result || NULL == n_hit_result) { return -1; } - struct adapter_hs *hs_inst = (struct adapter_hs *)hs_instance; - struct adapter_hs_stream *hs_stream = hs_inst->hs_rt->streams[thread_id]; + struct adapter_hs_stream *hs_stream = hs_instance->hs_rt->streams[thread_id]; assert(hs_stream != NULL); adapter_hs_stream_reset(hs_stream); diff --git a/scanner/adapter_hs/adapter_hs.h b/scanner/adapter_hs/adapter_hs.h new file mode 100644 index 0000000..672650a --- /dev/null +++ b/scanner/adapter_hs/adapter_hs.h @@ -0,0 +1,134 @@ +/* +********************************************************************************************** +* File: adapter_hs.h +* Description: wrapper for raw hyperscan +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _ADAPTER_HS_H_ +#define _ADAPTER_HS_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include +#include + +#include "log/log.h" + +#define MAX_EXPR_PATTERN_NUM 8 + +struct adapter_hs; + +/* match method */ +enum hs_match_mode { + HS_MATCH_MODE_INVALID = -1, + HS_MATCH_MODE_EXACTLY = 1, /* scan data must match pattern exactly */ + HS_MATCH_MODE_PREFIX, /* pattern must in the head of scan_data */ + HS_MATCH_MODE_SUFFIX, /* pattern must in the end of scan_data */ + HS_MATCH_MODE_SUB /* pattern must in the range[l_offset, r_offset] of scan_data */ +}; + +enum hs_pattern_type { + HS_PATTERN_TYPE_STR = 0, /* pure literal string */ + HS_PATTERN_TYPE_REG /* regex expression */ +}; + +enum hs_case_sensitive { + HS_CASE_SENSITIVE = 0, + HS_CASE_INSENSITIVE +}; + +struct hs_scan_result { + long long rule_id; + void *user_tag; +}; + +struct hs_pattern { + enum hs_case_sensitive case_sensitive; + enum hs_match_mode match_mode; + enum hs_pattern_type pattern_type; + + int is_hexbin; /* 1(yes) 0(no) */ + + /* + * just match in scan_data's range of [start_offset, end_offset], -1 means no limits + * for example: + * [-1, end_offset] means the pattern must in scan_data's [0 ~ start_offset] + * [start_offset, -1] means the pattern must in scan_data's [start_offset ~ data_end] + */ + int start_offset; + int end_offset; + + /* start pointer of pattern */ + char *pat; + /* pattern length */ + size_t pat_len; +}; + +/* logic AND expression, such as (pattern1 & pattern2) */ +struct expr_rule { + long long expr_id; + size_t n_patterns; + struct hs_pattern patterns[MAX_EXPR_PATTERN_NUM]; + void *user_tag; +}; + +int adapter_hs_verify_regex_expression(const char *regex_expr, + struct log_handle *logger); +/** + * @brief new adapter_hs instance + * + * @param rules: logic AND expression's array + * @param n_rule: the number of logic AND expression's array + * @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan() + * + * @retval the pointer to adapter_hs instance +*/ +struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule, + size_t n_worker_thread, struct log_handle *logger); + +/** + * @brief scan input data to match logic AND expression, return all matched expr_id + * + * @param instance: adapter_hs instance obtained by adapter_hs_new() + * @param thread_id: the thread_id of caller + * @param data: data to be scanned + * @param data_len: the length of data to be scanned + * @param results: the array of expr_id + * @param n_results: number of elements in array of expr_id +*/ +int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, + const char *data, size_t data_len, + struct hs_scan_result *results, + size_t n_result, size_t *n_hit_result); + +/** + * @brief destroy adapter_hs instance + * + * @param instance: adapter_hs instance obtained by adapter_hs_new() +*/ +void adapter_hs_free(struct adapter_hs *instance); + +struct adapter_hs_stream; +/** + * @brief open adapter_hs stream after adapter_hs instance initialized for stream scan + * +*/ +struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id); + +int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, size_t data_len, + struct hs_scan_result *results, size_t n_result, size_t *n_hit_result); + +void adapter_hs_stream_close(struct adapter_hs_stream *stream); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/scanner/expr_matcher/adapter_hs/adapter_hs.h b/scanner/expr_matcher/adapter_hs/adapter_hs.h deleted file mode 100644 index c9de7d2..0000000 --- a/scanner/expr_matcher/adapter_hs/adapter_hs.h +++ /dev/null @@ -1,75 +0,0 @@ -/* -********************************************************************************************** -* File: adapter_hs.h -* Description: -* Authors: Liu wentan -* Date: 2022-10-31 -* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved. -*********************************************************************************************** -*/ - -#ifndef _ADAPTER_HS_H_ -#define _ADAPTER_HS_H_ - -#ifdef __cplusplus -extern "C" -{ -#endif - -#include -#include - -#include "log/log.h" -#include "../expr_matcher.h" - -int adapter_hs_verify_regex_expression(const char *regex_expr, struct log_handle *logger); -/** - * @brief new adapter_hs instance - * - * @param rules: logic AND expression's array - * @param n_rule: the number of logic AND expression's array - * @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan() - * - * @retval the pointer to adapter_hs instance -*/ -void *adapter_hs_new(struct expr_rule *rules, size_t n_rule, - size_t n_literal_pattern, size_t n_regex_pattern, - size_t n_worker_thread, struct log_handle *logger); - -/** - * @brief scan input data to match logic AND expression, return all matched expr_id - * - * @param instance: adapter_hs instance obtained by adapter_hs_new() - * @param thread_id: the thread_id of caller - * @param data: data to be scanned - * @param data_len: the length of data to be scanned - * @param results: the array of expr_id - * @param n_results: number of elements in array of expr_id -*/ -int adapter_hs_scan(void *hs_instance, int thread_id, const char *data, size_t data_len, - struct expr_scan_result *results, size_t n_result, size_t *n_hit_result); - -/** - * @brief destroy adapter_hs instance - * - * @param instance: adapter_hs instance obtained by adapter_hs_new() -*/ -void adapter_hs_free(void *instance); - -/** - * @brief open adapter_hs stream after adapter_hs instance initialized for stream scan - * -*/ -void *adapter_hs_stream_open(void *hs_instance, int thread_id); - -int adapter_hs_scan_stream(void *stream, const char *data, size_t data_len, - struct expr_scan_result *results, size_t n_result, - size_t *n_hit_result); - -void adapter_hs_stream_close(void *stream); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/scanner/expr_matcher/adapter_rs/adapter_rs.cpp b/scanner/expr_matcher/adapter_rs/adapter_rs.cpp deleted file mode 100644 index 1459ab1..0000000 --- a/scanner/expr_matcher/adapter_rs/adapter_rs.cpp +++ /dev/null @@ -1,708 +0,0 @@ -/* -********************************************************************************************** -* File: adapter_rs.cpp -* Description: -* Authors: Liu wentan -* Date: 2022-10-31 -* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved. -*********************************************************************************************** -*/ - -#include -#include -#include -#include -#include -#include - -#include "rulescan.h" -#include "adapter_rs.h" -#include "uthash/utarray.h" -#include "uthash/uthash.h" -#include "maat_utils.h" -#include "../../bool_matcher/bool_matcher.h" - -#define MAX_HIT_PATTERN_NUM 512 - -pid_t rs_gettid() -{ - return syscall(SYS_gettid); -} - -static const char *rs_module_name_str(const char *name) -{ - static __thread char module[64]; - snprintf(module, sizeof(module), "%s(%d)", name, rs_gettid()); - - return module; -} - -#define MODULE_ADAPTER_RS rs_module_name_str("maat.adapter_rs") - -struct adpt_rs_compile_data { - struct scan_pattern *patterns; - size_t n_patterns; -}; - -struct adapter_rs_stream { - int thread_id; - size_t offset; /* current stream offset */ - rs_stream_t *literal_stream; - rs_stream_t *regex_stream; - struct adapter_rs_runtime *ref_rs_rt; - - struct log_handle *logger; -}; - -/* adapter_rs runtime */ -struct adapter_rs_runtime { - rs_database_t *literal_db; - rs_database_t *regex_db; - - struct bool_expr_match **bool_match_buffs; /* per thread */ - struct adapter_rs_stream **streams; /* per thread */ - struct matched_pattern **matched_pats; /* per thread */ - struct bool_matcher *bm; -}; - -/* adapter_rs instance */ -struct adapter_rs { - size_t n_worker_thread; - size_t n_expr; - size_t n_patterns; - struct adapter_rs_runtime *rs_rt; - struct pattern_attribute *rs_attr; - struct log_handle *logger; -}; - -struct pattern_offset { - long long start; - long long end; -}; - -struct pattern_attribute { - long long pattern_id; - enum expr_match_mode match_mode; - struct pattern_offset offset; - size_t pattern_len; -}; - -struct matched_pattern { - UT_array *pattern_ids; - size_t n_patterns; - struct pattern_attribute *ref_rs_attr; -}; - -int adapter_rs_verify_regex_expression(const char *regex_expr, - struct log_handle *logger) -{ - int ret = rs_verify_regex(regex_expr); - if (ret == 0) { - log_error(logger, MODULE_ADAPTER_RS, - "[%s:%d] illegal regex expression: \"%s\"", - __FUNCTION__, __LINE__, regex_expr); - } - - return ret; -} -/** - * @brief build rs database for literal string and regex expression respectively - * - * @retval 0(success) -1(failed) -*/ -static int adpt_rs_build_database(struct adapter_rs_runtime *rs_rt, - size_t n_worker_thread, - struct adpt_rs_compile_data *literal_cd, - struct adpt_rs_compile_data *regex_cd, - struct log_handle *logger) -{ - if (NULL == rs_rt) { - return -1; - } - - int ret = 0; - if (literal_cd != NULL) { - ret = rs_compile_lit(literal_cd->patterns, literal_cd->n_patterns, - &rs_rt->literal_db); - if (ret < 0) { - log_error(logger, MODULE_ADAPTER_RS, "[%s:%d] compile error", - __FUNCTION__, __LINE__); - return -1; - } - } - - if (regex_cd != NULL) { - size_t n_failed_pats = 0; - ret = rs_compile_regex(regex_cd->patterns, regex_cd->n_patterns, - n_worker_thread, &rs_rt->regex_db, &n_failed_pats); - if (ret < 0) { - log_error(logger, MODULE_ADAPTER_RS, "[%s:%d] compile error", - __FUNCTION__, __LINE__); - return -1; - } - } - - return 0; -} - -static struct adpt_rs_compile_data *adpt_rs_compile_data_new(size_t n_patterns) -{ - struct adpt_rs_compile_data *rs_cd = ALLOC(struct adpt_rs_compile_data, 1); - rs_cd->patterns = ALLOC(struct scan_pattern, n_patterns); - rs_cd->n_patterns = n_patterns; - - return rs_cd; -} - -static void adpt_rs_compile_data_free(struct adpt_rs_compile_data *rs_cd) -{ - if (NULL == rs_cd) { - return; - } - - if (rs_cd->patterns != NULL) { - for (size_t i = 0; i < rs_cd->n_patterns; i++) { - if (rs_cd->patterns[i].pattern != NULL) { - FREE(rs_cd->patterns[i].pattern); - } - } - - FREE(rs_cd->patterns); - } - - FREE(rs_cd); -} - -static void populate_compile_data(struct adpt_rs_compile_data *compile_data, - size_t index, long long pattern_id, char *pat, - size_t pat_len, int case_sensitive) -{ - compile_data->patterns[index].id = pattern_id; - compile_data->patterns[index].case_sensitive = case_sensitive; - compile_data->patterns[index].pattern = ALLOC(char, pat_len + 1); - memcpy(compile_data->patterns[index].pattern, pat, pat_len); - compile_data->patterns[index].pattern_len = pat_len; -} - -static struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule, - struct pattern_attribute *pattern_attr, - struct adpt_rs_compile_data *literal_cd, - struct adpt_rs_compile_data *regex_cd, - size_t *n_pattern) -{ - long long pattern_idx = 0; - size_t literal_idx = 0; - size_t regex_idx = 0; - - struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_rule); - - /* populate adpt_rs_compile_data and bool_expr */ - for (size_t i = 0; i < n_rule; i++) { - - for (size_t j = 0; j < rules[i].n_patterns; j++) { - pattern_attr[pattern_idx].pattern_id = pattern_idx; - pattern_attr[pattern_idx].match_mode = rules[i].patterns[j].match_mode; - pattern_attr[pattern_idx].pattern_len = rules[i].patterns[j].pat_len; - - if (pattern_attr[pattern_idx].match_mode == EXPR_MATCH_MODE_SUB || - pattern_attr[pattern_idx].match_mode == EXPR_MATCH_MODE_EXACTLY) { - pattern_attr[pattern_idx].offset.start = rules[i].patterns[j].start_offset; - pattern_attr[pattern_idx].offset.end = rules[i].patterns[j].end_offset; - } - - /* literal pattern */ - if (rules[i].patterns[j].type == EXPR_PATTERN_TYPE_STR) { - populate_compile_data(literal_cd, literal_idx, pattern_idx, - rules[i].patterns[j].pat, rules[i].patterns[j].pat_len, - rules[i].patterns[j].case_sensitive); - literal_idx++; - } else { - /* regex pattern */ - populate_compile_data(regex_cd, regex_idx, pattern_idx, - rules[i].patterns[j].pat, rules[i].patterns[j].pat_len, - rules[i].patterns[j].case_sensitive); - regex_idx++; - } - - bool_exprs[i].items[j].item_id = pattern_idx++; - bool_exprs[i].items[j].not_flag = 0; - } - - bool_exprs[i].expr_id = rules[i].expr_id; - bool_exprs[i].item_num = rules[i].n_patterns; - bool_exprs[i].user_tag = rules[i].tag; - } - - *n_pattern = pattern_idx; - - return bool_exprs; -} - -UT_icd ut_rs_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; -void *adapter_rs_new(struct expr_rule *rules, size_t n_rule, - size_t n_literal_pattern, size_t n_regex_pattern, - size_t n_worker_thread, struct log_handle *logger) -{ - /* get the sum of pattern */ - size_t i = 0; - struct adpt_rs_compile_data *literal_cd = NULL; - struct adpt_rs_compile_data *regex_cd = NULL; - - if (n_literal_pattern > 0) { - literal_cd = adpt_rs_compile_data_new(n_literal_pattern); - } - - if (n_regex_pattern > 0) { - regex_cd = adpt_rs_compile_data_new(n_regex_pattern); - } - - size_t pattern_cnt = n_literal_pattern + n_regex_pattern; - struct adapter_rs *rs_inst = ALLOC(struct adapter_rs, 1); - rs_inst->rs_attr = ALLOC(struct pattern_attribute, pattern_cnt); - rs_inst->logger = logger; - rs_inst->n_worker_thread = n_worker_thread; - rs_inst->n_expr = n_rule; - - struct bool_expr *bool_exprs = bool_exprs_new(rules, n_rule, rs_inst->rs_attr, - literal_cd, regex_cd, &pattern_cnt); - if (NULL == bool_exprs) { - return NULL; - } - rs_inst->n_patterns = pattern_cnt; - - /* create bool matcher */ - size_t mem_size = 0; - int rs_ret = 0; - - rs_inst->rs_rt = ALLOC(struct adapter_rs_runtime, 1); - - //rs_rt->bm - rs_inst->rs_rt->bm = bool_matcher_new(bool_exprs, n_rule, &mem_size); - if (rs_inst->rs_rt->bm != NULL) { - log_info(logger, MODULE_ADAPTER_RS, - "Adapter_rs module: build bool matcher of %zu expressions" - " with %zu bytes memory", n_rule, mem_size); - } else { - log_error(logger, MODULE_ADAPTER_RS, - "[%s:%d] Adapter_rs module: build bool matcher failed", - __FUNCTION__, __LINE__); - - rs_ret = -1; - } - FREE(bool_exprs); - - /* build rs database rs_rt->literal_db & rs_rt->regex_db */ - int ret = adpt_rs_build_database(rs_inst->rs_rt, n_worker_thread, - literal_cd, regex_cd, logger); - if (ret < 0) { - rs_ret = -1; - } - - if (literal_cd != NULL) { - adpt_rs_compile_data_free(literal_cd); - } - - if (regex_cd != NULL) { - adpt_rs_compile_data_free(regex_cd); - } - - if (rs_ret < 0) { - goto error; - } - - /* alloc scratch */ - rs_inst->rs_rt->bool_match_buffs = ALLOC(struct bool_expr_match *, n_worker_thread); - for (i = 0; i < n_worker_thread; i++) { - rs_inst->rs_rt->bool_match_buffs[i] = ALLOC(struct bool_expr_match, MAX_HIT_EXPR_NUM); - } - - rs_inst->rs_rt->streams = ALLOC(struct adapter_rs_stream *, n_worker_thread); - for (i = 0; i < n_worker_thread; i++) { - rs_inst->rs_rt->streams[i] = (struct adapter_rs_stream *)adapter_rs_stream_open(rs_inst, i); - } - - rs_inst->rs_rt->matched_pats = ALLOC(struct matched_pattern *, n_worker_thread); - for (i = 0; i < n_worker_thread; i++) { - rs_inst->rs_rt->matched_pats[i] = ALLOC(struct matched_pattern, 1); - rs_inst->rs_rt->matched_pats[i]->ref_rs_attr = rs_inst->rs_attr; - rs_inst->rs_rt->matched_pats[i]->n_patterns = rs_inst->n_patterns; - utarray_new(rs_inst->rs_rt->matched_pats[i]->pattern_ids, &ut_rs_pattern_id_icd); - utarray_reserve(rs_inst->rs_rt->matched_pats[i]->pattern_ids, MAX_HIT_PATTERN_NUM); - } - - return rs_inst; -error: - adapter_rs_free(rs_inst); - return NULL; -} - -void adapter_rs_free(void *rs_instance) -{ - if (NULL == rs_instance) { - return; - } - - size_t i = 0; - struct adapter_rs *rs_inst = (struct adapter_rs *)rs_instance; - - if (rs_inst->rs_rt != NULL) { - if (rs_inst->rs_rt->literal_db != NULL) { - rs_free_database(rs_inst->rs_rt->literal_db); - rs_inst->rs_rt->literal_db = NULL; - } - - if (rs_inst->rs_rt->regex_db != NULL) { - rs_free_database(rs_inst->rs_rt->regex_db); - rs_inst->rs_rt->regex_db = NULL; - } - - if (rs_inst->rs_rt->bool_match_buffs != NULL) { - for (i = 0; i < rs_inst->n_worker_thread; i++) { - if (rs_inst->rs_rt->bool_match_buffs[i] != NULL) { - FREE(rs_inst->rs_rt->bool_match_buffs[i]); - } - } - - FREE(rs_inst->rs_rt->bool_match_buffs); - } - - if (rs_inst->rs_rt->bm != NULL) { - bool_matcher_free(rs_inst->rs_rt->bm); - rs_inst->rs_rt->bm = NULL; - } - - if (rs_inst->rs_rt->streams != NULL) { - for (i = 0; i < rs_inst->n_worker_thread; i++) { - if (rs_inst->rs_rt->streams[i] != NULL) { - adapter_rs_stream_close(rs_inst->rs_rt->streams[i]); - rs_inst->rs_rt->streams[i] = NULL; - } - } - FREE(rs_inst->rs_rt->streams); - } - - if (rs_inst->rs_rt->matched_pats != NULL) { - for (i = 0; i < rs_inst->n_worker_thread; i++) { - if (rs_inst->rs_rt->matched_pats[i] != NULL) { - utarray_free(rs_inst->rs_rt->matched_pats[i]->pattern_ids); - FREE(rs_inst->rs_rt->matched_pats[i]); - } - } - FREE(rs_inst->rs_rt->matched_pats); - } - - FREE(rs_inst->rs_rt); - } - - if (rs_inst->rs_attr != NULL) { - FREE(rs_inst->rs_attr); - } - - FREE(rs_inst); -} - -static inline int compare_pattern_id(const void *a, const void *b) -{ - long long ret = *(const unsigned long long *)a - *(const unsigned long long *)b; - if (ret == 0) { - return 0; - } else if(ret < 0) { - return -1; - } else { - return 1; - } -} - -/** - * @param id: pattern id -*/ -static int matched_event_cb(unsigned int id, int pos_offset, int from, int to, - size_t data_len, void *ctx) -{ - // put id in set - unsigned long long pattern_id = id; - struct matched_pattern *matched_pat = (struct matched_pattern *)ctx; - - if (pattern_id > matched_pat->n_patterns || id < 0) { - return 0; - } - - if (utarray_len(matched_pat->pattern_ids) >= MAX_HIT_PATTERN_NUM) { - return 0; - } - - // duplicate pattern_id - if (utarray_find(matched_pat->pattern_ids, &pattern_id, compare_pattern_id)) { - return 0; - } - - int ret = 0; - struct pattern_attribute pat_attr = matched_pat->ref_rs_attr[id]; - - switch (pat_attr.match_mode) { - case EXPR_MATCH_MODE_EXACTLY: - if (0 == (from + pos_offset) && (int)data_len == (to + pos_offset)) { - ret = 1; - } - break; - case EXPR_MATCH_MODE_SUB: - if (pat_attr.offset.start == -1 && - pat_attr.offset.end == -1) { - ret = 1; - break; - } - - if (pat_attr.offset.start == -1) { - if ((long long)(to + pos_offset - 1) <= pat_attr.offset.end) { - ret = 1; - break; - } - } - - if (pat_attr.offset.end == -1) { - if ((long long)(from + pos_offset) >= pat_attr.offset.start) { - ret = 1; - break; - } - } - - if ((long long)(from + pos_offset) >= pat_attr.offset.start && - (long long)(to + pos_offset - 1) <= pat_attr.offset.end) { - ret = 1; - } - break; - case EXPR_MATCH_MODE_PREFIX: - if (0 == (from + pos_offset)) { - ret = 1; - } - break; - case EXPR_MATCH_MODE_SUFFIX: - if ((to + pos_offset) == (int)data_len) { - ret = 1; - } - break; - default: - break; - } - - if (1 == ret) { - utarray_push_back(matched_pat->pattern_ids, &pattern_id); - utarray_sort(matched_pat->pattern_ids, compare_pattern_id); - } - - return 0; -} - -void *adapter_rs_stream_open(void *rs_instance, int thread_id) -{ - if (NULL == rs_instance || thread_id < 0) { - return NULL; - } - - struct adapter_rs *rs_inst = (struct adapter_rs *)rs_instance; - struct adapter_rs_stream *rs_stream = ALLOC(struct adapter_rs_stream, 1); - - rs_stream->logger = rs_inst->logger; - rs_stream->thread_id = thread_id; - rs_stream->ref_rs_rt = rs_inst->rs_rt; - - int err_count = 0; - if (rs_inst->rs_rt->literal_db != NULL) { - rs_stream->literal_stream = rs_open_stream(rs_inst->rs_rt->literal_db, 0, 128); - if (NULL == rs_stream->literal_stream) { - log_error(rs_inst->logger, MODULE_ADAPTER_RS, "rs_open_stream failed"); - err_count++; - } - } - - if (rs_inst->rs_rt->regex_db != NULL) { - rs_stream->regex_stream = rs_open_stream(rs_inst->rs_rt->regex_db, 0, 128); - if (NULL == rs_stream->regex_stream) { - log_error(rs_inst->logger, MODULE_ADAPTER_RS, "rs_open_stream failed"); - err_count++; - } - } - - if (err_count > 0) { - goto error; - } - - return rs_stream; -error: - if (rs_stream->literal_stream != NULL) { - rs_close_stream(rs_stream->literal_stream); - rs_stream->literal_stream = NULL; - } - - if (rs_stream->regex_stream != NULL) { - rs_close_stream(rs_stream->regex_stream); - rs_stream->regex_stream = NULL; - } - - FREE(rs_stream); - return NULL; -} - -void adapter_rs_stream_close(void *rs_stream) -{ - if (NULL == rs_stream) { - return; - } - - struct adapter_rs_stream *stream = (struct adapter_rs_stream *)rs_stream; - if (stream->ref_rs_rt != NULL) { - if (stream->literal_stream != NULL) { - rs_close_stream(stream->literal_stream); - stream->literal_stream = NULL; - } - - if (stream->regex_stream != NULL) { - rs_close_stream(stream->regex_stream); - stream->regex_stream = NULL; - } - } - - /* rs_stream->rs_rt point to rs_instance->rs_rt which will call free - same as rs_attr */ - stream->ref_rs_rt = NULL; - FREE(stream); -} - -int adapter_rs_scan_stream(void *rs_stream, const char *data, size_t data_len, - struct expr_scan_result *results, size_t n_result, - size_t *n_hit_result) -{ - if (NULL == rs_stream || NULL == data || 0 == data_len || - NULL == results || 0 == n_result || NULL == n_hit_result) { - return -1; - } - - int ret = 0, err_count = 0; - struct adapter_rs_stream *stream = (struct adapter_rs_stream *)rs_stream; - int thread_id = stream->thread_id; - struct adapter_rs_runtime *rs_rt = stream->ref_rs_rt; - struct matched_pattern *matched_pat = rs_rt->matched_pats[thread_id]; - - if (stream->literal_stream != NULL) { - ret = rs_scan_stream(stream->literal_stream, data, data_len, - matched_event_cb, matched_pat); - if (ret < 0) { - err_count++; - } - } - - if (stream->regex_stream != NULL) { - ret = rs_scan_stream(stream->regex_stream, data, data_len, - matched_event_cb, matched_pat); - if (ret < 0) { - err_count++; - } - } - - if (err_count == 2) { - return -1; - } - - size_t n_pattern_id = utarray_len(matched_pat->pattern_ids); - if (0 == n_pattern_id) { - *n_hit_result = 0; - return 0; - } - - unsigned long long pattern_ids[n_pattern_id]; - - for (size_t i = 0; i < n_pattern_id; i++) { - pattern_ids[i] = *(unsigned long long *)utarray_eltptr(matched_pat->pattern_ids, i); - } - - struct bool_expr_match *bool_matcher_results = rs_rt->bool_match_buffs[thread_id]; - int bool_matcher_ret = bool_matcher_match(rs_rt->bm, pattern_ids, n_pattern_id, - bool_matcher_results, MAX_HIT_EXPR_NUM); - if (bool_matcher_ret < 0) { - ret = -1; - goto next; - } - - if (bool_matcher_ret > (int)n_result) { - bool_matcher_ret = n_result; - } - - for (int index = 0; index < bool_matcher_ret; index++) { - results[index].rule_id = bool_matcher_results[index].expr_id; - results[index].user_tag = bool_matcher_results[index].user_tag; - } - *n_hit_result = bool_matcher_ret; - -next: - utarray_clear(matched_pat->pattern_ids); - - return ret; -} - -int adapter_rs_scan(void *rs_instance, int thread_id, const char *data, size_t data_len, - struct expr_scan_result *results, size_t n_result, size_t *n_hit_result) -{ - if (NULL == rs_instance || NULL == data || (0 == data_len) || - NULL == results || 0 == n_result || NULL == n_hit_result) { - return -1; - } - - int ret = 0, err_count = 0; - struct adapter_rs *rs_inst = (struct adapter_rs *)rs_instance; - struct adapter_rs_runtime *rs_rt = rs_inst->rs_rt; - struct matched_pattern *matched_pat = rs_rt->matched_pats[thread_id]; - - if (rs_rt->literal_db != NULL) { - ret = rs_scan(rs_rt->literal_db, thread_id, data, data_len, - 0, matched_event_cb, matched_pat); - if (ret < 0) { - err_count++; - } - } - - if (rs_rt->regex_db != NULL) { - ret = rs_scan(rs_rt->regex_db, thread_id, data, data_len, - 0, matched_event_cb, matched_pat); - if (ret < 0) { - err_count++; - } - } - - if (err_count == 2) { - return -1; - } - - size_t n_pattern_id = utarray_len(matched_pat->pattern_ids); - if (0 == n_pattern_id) { - *n_hit_result = 0; - return 0; - } - - unsigned long long pattern_ids[n_pattern_id]; - for (size_t i = 0; i < n_pattern_id; i++) { - pattern_ids[i] = *(unsigned long long *)utarray_eltptr(matched_pat->pattern_ids, i); - } - - struct bool_expr_match *bool_matcher_results = rs_rt->bool_match_buffs[thread_id]; - int bool_matcher_ret = bool_matcher_match(rs_rt->bm, pattern_ids, n_pattern_id, - bool_matcher_results, MAX_HIT_EXPR_NUM); - if (bool_matcher_ret < 0) { - ret = -1; - goto next; - } - - if (bool_matcher_ret > (int)n_result) { - bool_matcher_ret = n_result; - } - - for (int index = 0; index < bool_matcher_ret; index++) { - results[index].rule_id = bool_matcher_results[index].expr_id; - results[index].user_tag = bool_matcher_results[index].user_tag; - } - *n_hit_result = bool_matcher_ret; - -next: - utarray_clear(matched_pat->pattern_ids); - - return ret; -} \ No newline at end of file diff --git a/scanner/expr_matcher/adapter_rs/adapter_rs.h b/scanner/expr_matcher/adapter_rs/adapter_rs.h deleted file mode 100644 index c43e553..0000000 --- a/scanner/expr_matcher/adapter_rs/adapter_rs.h +++ /dev/null @@ -1,78 +0,0 @@ -/* -********************************************************************************************** -* File: adapter_rs.h -* Description: -* Authors: Liu wentan -* Date: 2023-06-30 -* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved. -*********************************************************************************************** -*/ - -#ifndef _ADAPTER_RS_H_ -#define _ADAPTER_RS_H_ - -#ifdef __cplusplus -extern "C" -{ -#endif - -#include - -#include "log/log.h" - -#include "../expr_matcher.h" - -int adapter_rs_verify_regex_expression(const char *regex_expr, - struct log_handle *logger); - -/** - * @brief new adapter_rs instance - * - * @param rules: logic AND expression's array - * @param n_rule: the number of logic AND expression's array - * @param n_worker_threads: the number of scan threads which will call adapter_rs_scan() - * - * @retval the pointer to adapter_rs instance -*/ -void *adapter_rs_new(struct expr_rule *rules, size_t n_rule, - size_t n_literal_pattern, size_t n_regex_pattern, - size_t n_worker_thread, struct log_handle *logger); - -void adapter_rs_free(void *rs_instance); - -/** - * @brief scan input data to match logic AND expression, return all matched expr_id - * - * @param rs_instance: adapter_rs instance obtained by adapter_rs_new() - * @param thread_id: the thread_id of caller - * @param scan_data: data to be scanned - * @param data_len: the length of data to be scanned - * @param result_array: the array to store hit expr_id which allocated by caller - * @param n_result_array: number of elements in array of expr_id -*/ -int adapter_rs_scan(void *rs_instance, int thread_id, - const char *scan_data, size_t data_len, - struct expr_scan_result *result_array, - size_t n_result_array, size_t *n_hit_results); - -/** - * @brief - */ -void *adapter_rs_stream_open(void *rs_instance, int thread_id); - -/** - * @brief - */ -int adapter_rs_scan_stream(void *rs_stream, const char *scan_data, - size_t data_len, struct expr_scan_result *result_array, - size_t n_result_array, size_t *n_hit_results); -/** - * @brief - */ -void adapter_rs_stream_close(void *rs_stream); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/scanner/expr_matcher/expr_matcher.cpp b/scanner/expr_matcher/expr_matcher.cpp deleted file mode 100644 index 3a37383..0000000 --- a/scanner/expr_matcher/expr_matcher.cpp +++ /dev/null @@ -1,235 +0,0 @@ -/* -********************************************************************************************** -* File: expr_matcher.cpp -* Description: -* Authors: Liu wentan -* Date: 2023-06-30 -* Copyright: (c) Since 2023 Geedge Networks, Ltd. All rights reserved. -*********************************************************************************************** -*/ - -#include -#include -#include - -#include "log/log.h" -#include "expr_matcher.h" -#include "maat_utils.h" -#include "adapter_hs/adapter_hs.h" -#include "adapter_rs/adapter_rs.h" - -pid_t expr_matcher_gettid() -{ - return syscall(SYS_gettid); -} - -static const char *expr_matcher_module_name_str(const char *name) -{ - static __thread char module[64]; - snprintf(module, sizeof(module), "%s(%d)", name, expr_matcher_gettid()); - - return module; -} - -#define MODULE_EXPR_MATCHER expr_matcher_module_name_str("maat.expr_matcher") - -struct expr_matcher { - enum expr_engine_type engine_type; - void *engine; - struct log_handle *logger; -}; - -struct expr_matcher_stream { - enum expr_engine_type engine_type; - void *handle; -}; - -struct expr_engine_operations { - enum expr_engine_type type; - void *(*engine_new)(struct expr_rule *rules, size_t n_rule, - size_t n_literal_pattern, size_t n_regex_pattern, - size_t n_worker_thread, struct log_handle *logger); - void (*engine_free)(void *engine); - int (*engine_scan)(void *engine, int thread_id, const char *scan_data, - size_t data_len, struct expr_scan_result *result_array, - size_t n_result_array, size_t *n_hit_result); - void *(*engine_stream_open)(void *engine, int thread_id); - void (*engine_stream_close)(void *stream); - int (*engine_scan_stream)(void *stream, const char *scan_data, size_t data_len, - struct expr_scan_result *result_array, size_t n_result_array, - size_t *n_hit_result); -}; - -struct expr_engine_operations expr_engine_ops[EXPR_ENGINE_TYPE_MAX] = { - { - .type = EXPR_ENGINE_TYPE_HS, - .engine_new = adapter_hs_new, - .engine_free = adapter_hs_free, - .engine_scan = adapter_hs_scan, - .engine_stream_open = adapter_hs_stream_open, - .engine_stream_close = adapter_hs_stream_close, - .engine_scan_stream = adapter_hs_scan_stream - }, - { - .type = EXPR_ENGINE_TYPE_RS, - .engine_new = adapter_rs_new, - .engine_free = adapter_rs_free, - .engine_scan = adapter_rs_scan, - .engine_stream_open = adapter_rs_stream_open, - .engine_stream_close = adapter_rs_stream_close, - .engine_scan_stream = adapter_rs_scan_stream - } -}; - -int expr_matcher_verify_regex_expression(const char *regex_expr, - struct log_handle *logger) -{ - int ret = adapter_hs_verify_regex_expression(regex_expr, logger); - if (ret == 0) { - return 0; - } - - return adapter_rs_verify_regex_expression(regex_expr, logger); -} - -struct expr_matcher * -expr_matcher_new(struct expr_rule *rules, size_t n_rule, enum expr_engine_type engine_type, - size_t n_worker_thread, struct log_handle *logger) -{ - if (NULL == rules || 0 == n_rule || 0 == n_worker_thread || - (engine_type != EXPR_ENGINE_TYPE_HS && engine_type != EXPR_ENGINE_TYPE_RS)) { - log_error(logger, MODULE_EXPR_MATCHER, "[%s:%d]engine type:%d is illegal", - __FUNCTION__, __LINE__, engine_type); - return NULL; - } - - size_t i = 0, j = 0; - size_t literal_pat_num = 0; - size_t regex_pat_num = 0; - - for (i = 0; i < n_rule; i++) { - if (rules[i].n_patterns > MAX_EXPR_PATTERN_NUM) { - log_error(logger, MODULE_EXPR_MATCHER, - "[%s:%d] the number of patterns in one expression should less than" - " %d", __FUNCTION__, __LINE__, MAX_EXPR_PATTERN_NUM); - return NULL; - } - - for (j = 0; j < rules[i].n_patterns; j++) { - /* pat_len should not 0 */ - if (0 == rules[i].patterns[j].pat_len) { - log_error(logger, MODULE_EXPR_MATCHER, - "[%s:%d] expr pattern length should not 0", - __FUNCTION__, __LINE__); - return NULL; - } - - if (rules[i].patterns[j].type == EXPR_PATTERN_TYPE_STR) { - literal_pat_num++; - } else { - regex_pat_num++; - } - } - } - - if (0 == literal_pat_num && 0 == regex_pat_num) { - log_error(logger, MODULE_EXPR_MATCHER, - "[%s:%d] exprs has no valid pattern", __FUNCTION__, __LINE__); - return NULL; - } - - void *engine = expr_engine_ops[engine_type].engine_new(rules, n_rule, literal_pat_num, - regex_pat_num, n_worker_thread, - logger); - if (NULL == engine) { - log_error(logger, MODULE_EXPR_MATCHER, - "[%s:%d]expr_matcher engine_new failed.", __FUNCTION__, __LINE__); - return NULL; - } - - struct expr_matcher *matcher = ALLOC(struct expr_matcher, 1); - matcher->engine_type = engine_type; - matcher->engine = engine; - matcher->logger = logger; - - return matcher; -} - -void expr_matcher_free(struct expr_matcher *matcher) -{ - if (NULL == matcher) { - return; - } - - if (matcher->engine != NULL) { - expr_engine_ops[matcher->engine_type].engine_free(matcher->engine); - matcher->engine = NULL; - } - - FREE(matcher); -} - -int expr_matcher_match(struct expr_matcher *matcher, int thread_id, const char *scan_data, - size_t data_len, struct expr_scan_result *result_array, - size_t n_result_array, size_t *n_hit_results) -{ - if (NULL == matcher || thread_id < 0 || NULL == scan_data || 0 == data_len - || NULL == result_array || 0 == n_result_array || NULL == n_hit_results) { - return -1; - } - - return expr_engine_ops[matcher->engine_type].engine_scan(matcher->engine, thread_id, - scan_data, data_len, result_array, - n_result_array, n_hit_results); -} - -struct expr_matcher_stream * -expr_matcher_stream_open(struct expr_matcher *matcher, int thread_id) -{ - if (NULL == matcher || thread_id < 0) { - return NULL; - } - - void *s_handle = expr_engine_ops[matcher->engine_type].engine_stream_open(matcher->engine, - thread_id); - if (NULL == s_handle) { - log_error(matcher->logger, MODULE_EXPR_MATCHER, - "[%s:%d] expr_matcher engine_stream_open failed.", - __FUNCTION__, __LINE__); - return NULL; - } - - struct expr_matcher_stream *stream = ALLOC(struct expr_matcher_stream, 1); - stream->engine_type = matcher->engine_type; - stream->handle = s_handle; - - return stream; -} - -int expr_matcher_stream_match(struct expr_matcher_stream *stream, const char *scan_data, - size_t data_len, struct expr_scan_result *result_array, - size_t n_result_array, size_t *n_hit_results) -{ - if (NULL == stream || NULL == scan_data || 0 == data_len || NULL == result_array - || 0 == n_result_array || NULL == n_hit_results) { - return -1; - } - - return expr_engine_ops[stream->engine_type].engine_scan_stream(stream->handle, scan_data, - data_len, result_array, - n_result_array, n_hit_results); -} - -void expr_matcher_stream_close(struct expr_matcher_stream *stream) -{ - if (NULL == stream) { - return; - } - - if (stream->handle != NULL) { - expr_engine_ops[stream->engine_type].engine_stream_close(stream->handle); - stream->handle = NULL; - } - - FREE(stream); -} \ No newline at end of file diff --git a/scanner/expr_matcher/expr_matcher.h b/scanner/expr_matcher/expr_matcher.h deleted file mode 100644 index fb61854..0000000 --- a/scanner/expr_matcher/expr_matcher.h +++ /dev/null @@ -1,134 +0,0 @@ -/* -********************************************************************************************** -* File: expr_matcher.h -* Description: -* Authors: Liu wentan -* Date: 2023-06-30 -* Copyright: (c) Since 2023 Geedge Networks, Ltd. All rights reserved. -*********************************************************************************************** -*/ - -#ifndef _EXPR_MATCHER_H_ -#define _EXPR_MATCHER_H_ - -#ifdef __cplusplus -extern "C" -{ -#endif - -#include - -#include "log/log.h" - -#define MAX_EXPR_PATTERN_NUM 8 /* 每条与表达式最多由MAX_EXPR_ITEM_NUM个规则组成 */ -#define MAX_HIT_EXPR_NUM 1024 - -enum expr_engine_type { - EXPR_ENGINE_TYPE_HS = 0, /* default engine */ - EXPR_ENGINE_TYPE_RS, - EXPR_ENGINE_TYPE_MAX -}; - -enum expr_pattern_type { - EXPR_PATTERN_TYPE_STR = 0, /* pure literal string */ - EXPR_PATTERN_TYPE_REG = 1, /* regex expression */ -}; - -enum expr_case_sensitive { - EXPR_CASE_INSENSITIVE = 0, - EXPR_CASE_SENSITIVE -}; - -enum expr_match_mode { - EXPR_MATCH_MODE_INVALID = -1, - EXPR_MATCH_MODE_EXACTLY = 1, /* scan data must match pattern exactly */ - EXPR_MATCH_MODE_PREFIX, /* pattern must in the head of scan_data */ - EXPR_MATCH_MODE_SUFFIX, /* pattern must in the tail of scan_data */ - EXPR_MATCH_MODE_SUB /* pattern must in the range[l_offset, r_offset] of scan_data */ -}; - -struct expr_pattern { - enum expr_pattern_type type; - enum expr_match_mode match_mode; - enum expr_case_sensitive case_sensitive; - - /* - * just match in scan_data's range of [start_offset, end_offset], -1 means no limits - * for example: - * [-1, end_offset] means the pattern must in scan_data's [0 ~ start_offset] - * [start_offset, -1] means the pattern must in scan_data's [start_offset ~ data_end] - */ - int start_offset; - int end_offset; - - char *pat; - size_t pat_len; -}; - -struct expr_scan_result { - long long rule_id; - void *user_tag; -}; - -/* logic AND expression, such as (rule1 & rule2) */ -struct expr_rule { - long long expr_id; /* AND expression ID */ - size_t n_patterns; - struct expr_pattern patterns[MAX_EXPR_PATTERN_NUM]; - void *tag; /* user defined data, return with hit result */ -}; - -int expr_matcher_verify_regex_expression(const char *regex_expr, - struct log_handle *logger); - -/** - * @brief new expr matcher instance - * - * @param expr_array: logic AND expression's array - * @param n_expr_array: the number of logic AND expression's array - * @param n_worker_threads: the number of scan threads which will call adapter_rs_scan() - * -*/ -struct expr_matcher * -expr_matcher_new(struct expr_rule *rules, size_t n_rule, enum expr_engine_type type, - size_t n_worker_thread, struct log_handle *logger); - -void expr_matcher_free(struct expr_matcher *matcher); - -/** - * @brief scan input data to match logic AND expression, return all matched expr_id - * - * @param matcher: expr_matcher instance obtained by expr_matcher_new() - * @param thread_id: the thread_id of caller - * @param scan_data: data to be scanned - * @param data_len: the length of data to be scanned - * @param result_array: the array to store hit expr_id which allocated by caller - * @param n_result_array: number of elements in array of expr_id -*/ -int expr_matcher_match(struct expr_matcher *matcher, int thread_id, const char *scan_data, - size_t data_len, struct expr_scan_result *result_array, - size_t n_result_array, size_t *n_hit_results); - -/** - * @brief - */ -struct expr_matcher_stream * -expr_matcher_stream_open(struct expr_matcher *matcher, int thread_id); - -/** - * @brief - */ -int expr_matcher_stream_match(struct expr_matcher_stream *stream, const char *scan_data, - size_t data_len, struct expr_scan_result *result_array, - size_t n_result_array, size_t *n_hit_results); - -/** - * @brief - */ -void expr_matcher_stream_close(struct expr_matcher_stream *stream); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e21418..5b56b4a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,15 +23,13 @@ include_directories(/opt/MESA/include/MESA/) include_directories(${PROJECT_SOURCE_DIR}/include/) include_directories(${PROJECT_SOURCE_DIR}/deps/) include_directories(${PROJECT_SOURCE_DIR}/scanner) +include_directories(${PROJECT_SOURCE_DIR}/scanner/adapter_hs) include_directories(${PROJECT_SOURCE_DIR}/scanner/fqdn_engine) include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/flag_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/interval_matcher) include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) -include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher) -include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_hs) -include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_rs) # Static Library Output add_library(maat_frame_static STATIC ${MAAT_SRC} ${LIB_SOURCE_FILES}) diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index 52395c4..e6e588e 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -37,6 +37,12 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, void group2compile_schema_free(void *g2c_schema); int group2compile_associated_compile_table_id(void *g2c_schema); +int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int table_id, + maat_ex_new_func_t *new_func, + maat_ex_free_func_t *free_func, + maat_ex_dup_func_t *dup_func, + long argl, void *argp); + /* compile runtime API */ void *compile_runtime_new(void *compile_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, @@ -64,6 +70,14 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr struct maat_hit_path *hit_path_array, size_t array_size, size_t n_internal_hit_path); +void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt, + struct compile_schema *compile_schema, + long long compile_id); +void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt, + struct compile_schema *compile_schema); + +void compile_runtime_garbage_collect_routine(void *compile_runtime); + /* group2compile runtime API */ void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, @@ -105,13 +119,10 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat struct maat_hit_group *hit_group_array, size_t array_size); -int maat_compile_state_get_compile_table_id(struct maat_compile_state *compile_state, - long long compile_id); - int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); #ifdef __cplusplus } #endif -#endif +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index c6923f6..9f6c9c6 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -52,13 +52,14 @@ long long expr_runtime_get_version(void *expr_runtime); int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len, int vtable_id, struct maat_state *state); -struct expr_matcher_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); +struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); -int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct expr_matcher_stream *s_handle, - const char *data, size_t data_len, int vtable_id, struct maat_state *state); +int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle, + const char *data, size_t data_len, + int vtable_id, struct maat_state *state); void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id, - struct expr_matcher_stream *stream); + struct adapter_hs_stream *s_handle); int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district, size_t district_len, long long *district_id); @@ -83,4 +84,4 @@ long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt); } #endif -#endif +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_kv.h b/src/inc_internal/maat_kv.h index a7ce735..896ec1d 100644 --- a/src/inc_internal/maat_kv.h +++ b/src/inc_internal/maat_kv.h @@ -26,15 +26,12 @@ void maat_kv_store_free(struct maat_kv_store *store); int maat_kv_register(struct maat_kv_store *store, const char *key, long long value); -int maat_kv_read(struct maat_kv_store *store, const char *key, long long *value_array, size_t n_array); +int maat_kv_read(struct maat_kv_store *store, const char *key, long long *value); -int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, - long long *value_array, size_t n_array); +int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, long long *value); int maat_kv_write(struct maat_kv_store *store, const char *key, long long value); -int maat_kv_append(struct maat_kv_store *store, const char *key, long long value); - struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store); #ifdef __cplusplus diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 2c9b9c5..90450ac 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -128,7 +128,6 @@ struct maat_options { int rule_effect_interval_ms; int rule_update_checking_interval_ms; - enum maat_expr_engine expr_engine; enum data_source input_mode; union { struct source_iris_ctx iris_ctx; @@ -226,4 +225,4 @@ void garbage_maat_kv_store_free(void *kv_store, void *arg); } #endif -#endif +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index 7ade1bd..9efdb82 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -19,7 +19,6 @@ extern "C" #include #include -#include "maat.h" #include "log/log.h" #include "maat_garbage_collection.h" @@ -48,8 +47,7 @@ struct table_manager; struct table_manager * table_manager_create(const char *table_info_path, const char *accept_tags, - enum maat_expr_engine expr_engine, struct maat_garbage_bin *garbage_bin, - struct log_handle *logger); + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_thread_num, struct maat_garbage_bin *garbage_bin); @@ -59,23 +57,7 @@ void table_manager_destroy(struct table_manager *tbl_mgr); size_t table_manager_table_size(struct table_manager *tbl_mgr); size_t table_manager_table_count(struct table_manager *tbl_mgr); -int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_name); -/** - * @brief get table_name's all conjunction parents' table_id - * - * for example: "table_id": 1, - * "table_name":"COMPILE_CONJ" - * "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"] - * - * "table_id": 2, - * "table_name": "COMPILE_PLUGIN", - * "db_tables": ["COMPILE_DEFAULT"] - * - * so COMPILE_DEFAULT has two conjunction parents whose table_id is 1 and 2. -*/ -int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name, - long long *table_ids_array, size_t n_table_ids_array); - +int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name); const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id); enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id); @@ -85,8 +67,6 @@ int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr); int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id); -enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr); - size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr); int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags); diff --git a/src/json2iris.c b/src/json2iris.c index de1ec7d..2b3127f 100644 --- a/src/json2iris.c +++ b/src/json2iris.c @@ -377,7 +377,7 @@ static int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, if (1 == cmd[i].str2int_flag) { long long int_value = 0; char *p = item->valuestring; - ret = maat_kv_read(str2int, p, &int_value, 1); + ret = maat_kv_read(str2int, p, &int_value); if (ret < 0) { log_error(logger, MODULE_JSON2IRIS, "[%s:%d] %s's value %s is not valid format", @@ -636,7 +636,7 @@ static int write_region_rule(cJSON *region_json, int compile_id, int group_id, const char *table_type_str = item->valuestring; long long table_type_int; - int ret = maat_kv_read(p_iris->str2int_map, table_type_str, &(table_type_int), 1); + int ret = maat_kv_read(p_iris->str2int_map, table_type_str, &table_type_int); if (ret != 1) { log_error(logger, MODULE_JSON2IRIS, "[%s:%d] compile rule %d table name %s's table_type %s invalid", diff --git a/src/maat_api.c b/src/maat_api.c index 3a5cc5a..0b296a4 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -51,7 +51,7 @@ enum district_flag { struct maat_stream { struct maat *ref_maat_inst; - struct expr_matcher_stream *handle; //each physical table open one stream + struct adapter_hs_stream *handle; //each physical table open one stream long long last_full_version; long long expr_rt_version; struct log_handle *logger; @@ -70,7 +70,6 @@ struct maat_options* maat_options_new(void) options->rule_update_checking_interval_ms = 1 * 1000; options->gc_timeout_ms = 10 * 1000; options->input_mode = DATA_SOURCE_NONE; - options->expr_engine = MAAT_EXPR_ENGINE_HS; options->log_level = 0; return options; @@ -255,19 +254,6 @@ int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filen return 0; } -int maat_options_set_expr_engine(struct maat_options *opts, - enum maat_expr_engine expr_engine) -{ - if (NULL == opts || - (expr_engine != MAAT_EXPR_ENGINE_HS && expr_engine != MAAT_EXPR_ENGINE_RS)) { - return -1; - } - - opts->expr_engine = expr_engine; - - return 0; -} - int maat_options_set_logger(struct maat_options *opts, const char *log_path, enum log_level level) { @@ -371,8 +357,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) pthread_mutex_init(&(maat_inst->background_update_mutex), NULL); maat_inst->tbl_mgr = table_manager_create(table_info_path, maat_inst->opts.accept_tags, - maat_inst->opts.expr_engine, maat_inst->garbage_bin, - maat_inst->logger); + maat_inst->garbage_bin, maat_inst->logger); if (NULL == maat_inst->tbl_mgr) { goto failed; } @@ -425,7 +410,12 @@ int maat_helper_verify_regex_expression(const char *regex_expr) return 0; } - return expr_matcher_verify_regex_expression(regex_expr, NULL); + int ret = adapter_hs_verify_regex_expression(regex_expr, NULL); + if (ret < 0) { + return 0; + } else { + return 1; + } } int maat_get_table_id(struct maat *maat_inst, const char *table_name) @@ -522,6 +512,32 @@ int maat_table_callback_register(struct maat *maat_inst, int table_id, return 0; } +static int compile_table_ex_schema_register(struct maat *maat_inst, int table_id, + maat_ex_new_func_t *new_func, + maat_ex_free_func_t *free_func, + maat_ex_dup_func_t *dup_func, + long argl, void *argp) +{ + void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id); + assert(schema != NULL); + + int ret = compile_table_set_ex_data_schema((struct compile_schema *)schema, table_id, + new_func, free_func, dup_func, + argl, argp); + if (ret < 0) { + return -1; + } + + if (maat_inst->maat_rt != NULL) { + void *runtime = table_manager_get_runtime(maat_inst->tbl_mgr, table_id); + assert(runtime != NULL); + compile_runtime_ex_data_iterate((struct compile_runtime *)runtime, + (struct compile_schema *)schema); + } + + return 0; +} + static int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_id, maat_ex_new_func_t *new_func, maat_ex_free_func_t *free_func, @@ -755,14 +771,16 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_inst, int ret = -1; enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id); - if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_IP_PLUGIN || - table_type == TABLE_TYPE_BOOL_PLUGIN || table_type == TABLE_TYPE_FQDN_PLUGIN ) { + if (TABLE_TYPE_COMPILE == table_type) { + ret = compile_table_ex_schema_register(maat_inst, table_id, new_func, + free_func, dup_func, argl, argp); + } else { ret = generic_plugin_table_ex_schema_register(maat_inst, table_name, table_id, new_func, free_func, dup_func, argl, argp); } - pthread_mutex_unlock(&(maat_inst->background_update_mutex)); + pthread_mutex_unlock(&(maat_inst->background_update_mutex)); return ret; } @@ -793,7 +811,9 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, void *ret = NULL; enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id); - if (TABLE_TYPE_PLUGIN == table_type) { + if (TABLE_TYPE_COMPILE == table_type) { + ret = compile_runtime_get_ex_data(runtime, schema, *(long long *)key); + } else if (TABLE_TYPE_PLUGIN == table_type) { ret = plugin_runtime_get_ex_data(runtime, schema, key, key_len); } @@ -1148,7 +1168,9 @@ int maat_scan_flag(struct maat *maat_inst, int table_id, return MAAT_SCAN_ERR; } + maat_runtime_ref_inc(maat_rt, state->thread_id); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); + int hit_group_cnt = flag_scan(maat_inst->tbl_mgr, state->thread_id, flag, phy_table_id, vtable_id, state); if (hit_group_cnt < 0) { @@ -1156,8 +1178,6 @@ int maat_scan_flag(struct maat *maat_inst, int table_id, return MAAT_SCAN_ERR; } - maat_runtime_ref_inc(maat_rt, state->thread_id); - size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); @@ -1237,7 +1257,9 @@ int maat_scan_integer(struct maat *maat_inst, int table_id, return MAAT_SCAN_ERR; } + maat_runtime_ref_inc(maat_rt, state->thread_id); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); + int hit_group_cnt = interval_scan(maat_inst->tbl_mgr, state->thread_id, integer, phy_table_id, vtable_id, state); if (hit_group_cnt < 0) { @@ -1245,8 +1267,6 @@ int maat_scan_integer(struct maat *maat_inst, int table_id, return MAAT_SCAN_ERR; } - maat_runtime_ref_inc(maat_rt, state->thread_id); - size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); @@ -1326,7 +1346,9 @@ int maat_scan_ipv4(struct maat *maat_inst, int table_id, uint32_t ip_addr, return MAAT_SCAN_ERR; } + maat_runtime_ref_inc(maat_rt, state->thread_id); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); + int hit_group_cnt = ipv4_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr, port, protocol, phy_table_id, vtable_id, state); if (hit_group_cnt < 0) { @@ -1334,8 +1356,6 @@ int maat_scan_ipv4(struct maat *maat_inst, int table_id, uint32_t ip_addr, return MAAT_SCAN_ERR; } - maat_runtime_ref_inc(maat_rt, state->thread_id); - size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); @@ -1416,7 +1436,9 @@ int maat_scan_ipv6(struct maat *maat_inst, int table_id, return MAAT_SCAN_ERR; } + maat_runtime_ref_inc(maat_rt, state->thread_id); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); + int hit_group_cnt = ipv6_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr, port, protocol, phy_table_id, vtable_id, state); if (hit_group_cnt < 0) { @@ -1424,8 +1446,6 @@ int maat_scan_ipv6(struct maat *maat_inst, int table_id, return MAAT_SCAN_ERR; } - maat_runtime_ref_inc(maat_rt, state->thread_id); - size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); @@ -1505,7 +1525,9 @@ int maat_scan_string(struct maat *maat_inst, int table_id, const char *data, return MAAT_SCAN_ERR; } + maat_runtime_ref_inc(maat_rt, state->thread_id); alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1); + int hit_group_cnt = string_scan(maat_inst->tbl_mgr, state->thread_id, data, data_len, phy_table_id, vtable_id, state); if (hit_group_cnt < 0) { @@ -1513,8 +1535,6 @@ int maat_scan_string(struct maat *maat_inst, int table_id, const char *data, return MAAT_SCAN_ERR; } - maat_runtime_ref_inc(maat_rt, state->thread_id); - size_t sum_hit_compile_cnt = 0; if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) { sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state); @@ -1589,8 +1609,8 @@ struct maat_stream *maat_stream_new(struct maat *maat_inst, int table_id, stream->expr_rt_version = expr_runtime_get_version(expr_rt); maat_runtime_ref_inc(maat_inst->maat_rt, state->thread_id); - struct expr_matcher_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt, - state->thread_id); + struct adapter_hs_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt, + state->thread_id); if (NULL == handle) { goto error; } @@ -1869,22 +1889,6 @@ int maat_state_set_scan_compile_table(struct maat_state *state, int compile_tabl return 0; } -int maat_state_get_compile_table_ids(struct maat_state *state, long long *compile_ids, - size_t n_compile_ids, int *compile_table_ids) -{ - if (NULL == state || NULL == compile_ids || 0 == n_compile_ids || - NULL == compile_table_ids) { - return -1; - } - - for (size_t i = 0; i < n_compile_ids; i++) { - compile_table_ids[i] = maat_compile_state_get_compile_table_id(state->compile_state, - compile_ids[i]); - } - - return n_compile_ids; -} - int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths, size_t n_path) { @@ -1956,4 +1960,4 @@ int maat_state_get_hit_groups(struct maat_state *state, enum maat_list_type type int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group) { return 0; -} +} \ No newline at end of file diff --git a/src/maat_compile.c b/src/maat_compile.c index 8c3e23b..4413dba 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -27,12 +27,16 @@ #include "alignment.h" #define MODULE_COMPILE module_name_str("maat.compile") +#define DEFAULT_GC_TIMEOUT_S 10 struct compile_schema { int compile_id_column; int rule_tag_column; int declared_clause_num_column; + int set_flag; + int gc_timeout_s; int table_id; //ugly + struct ex_data_schema ex_schema; struct table_manager *ref_tbl_mgr; struct log_handle *logger; }; @@ -53,6 +57,8 @@ struct compile_item { long long compile_id; char *table_line; size_t table_line_len; + struct compile_schema *ref_schema; + void **ex_data; char table_name[MAX_NAME_STR_LEN]; }; @@ -79,8 +85,7 @@ struct literal_clause { /* compile_runtime and group2compile_runtime share compile_hash_map */ struct compile_runtime { struct bool_matcher *bm; - struct rcu_hash_table *cfg_hash; // - struct rcu_hash_table *tbl_cfg_hash; // + struct rcu_hash_table *cfg_hash; // struct maat_runtime *ref_maat_rt; time_t version; struct literal_clause *literal2clause_hash; @@ -132,15 +137,9 @@ struct maat_internal_hit_path { int vtable_id; }; -struct maat_compile_table { - long long compile_id; - int table_id; -}; - struct maat_compile_state { uint8_t this_scan_hit_item_flag; uint8_t not_clause_hit_flag; - uint8_t inc_hit_path_flag; int Nth_scan; time_t compile_rt_version; @@ -148,14 +147,12 @@ struct maat_compile_state { UT_array *internal_inc_hit_paths; UT_array *all_hit_clauses; UT_array *this_scan_hit_clauses; - UT_array *hit_compile_tables; }; UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL}; UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL}; UT_icd ut_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL}; UT_icd ut_hit_path_icd = {sizeof(struct maat_internal_hit_path), NULL, NULL, NULL}; -UT_icd ut_hit_compile_table_icd = {sizeof(struct maat_compile_table)}; static struct maat_compile *maat_compile_new(long long compile_id) { @@ -188,6 +185,23 @@ static int maat_compile_set(struct maat_compile *compile, const char *table_name return 0; } +static void *rule_ex_data_new(const char *table_name, int table_id, + const char *table_line, + struct ex_data_schema *ex_schema) +{ + void *ex_data = NULL; + + ex_schema->new_func(table_name, table_id, NULL, table_line, &ex_data, + ex_schema->argl, ex_schema->argp); + return ex_data; +} + +static void rule_ex_data_free(int table_id, void **ex_data, + const struct ex_data_schema *ex_schema) +{ + ex_schema->free_func(table_id, ex_data, ex_schema->argl, ex_schema->argp); +} + static int compile_accept_tag_match(struct compile_schema *schema, const char *line, const char *table_name, struct log_handle *logger) { @@ -262,11 +276,18 @@ compile_item_new(const char *table_line, struct compile_schema *schema, } compile_item->declared_clause_num = atoi(table_line + column_offset); + compile_item->ref_schema = schema; + compile_item->ex_data = ALLOC(void *, 1); memcpy(compile_item->table_name, table_name, sizeof(compile_item->table_name)); compile_item->table_line_len = strlen(table_line) + 1; compile_item->table_line = ALLOC(char, compile_item->table_line_len); memcpy(compile_item->table_line, table_line, compile_item->table_line_len); + if (1 == schema->set_flag) { + *(compile_item->ex_data) = rule_ex_data_new(table_name, schema->table_id, + compile_item->table_line, + &(schema->ex_schema)); + } return compile_item; error: FREE(compile_item); @@ -275,6 +296,17 @@ error: static void compile_item_free(struct compile_item *item) { + struct compile_schema *schema = item->ref_schema; + + if (1 == schema->set_flag) { + rule_ex_data_free(schema->table_id, item->ex_data, &(schema->ex_schema)); + *item->ex_data = NULL; + } + + if (item->ex_data != NULL) { + FREE(item->ex_data); + } + item->declared_clause_num = -1; if (item->table_line != NULL) { @@ -315,9 +347,70 @@ static void rcu_compile_cfg_free(void *user_ctx, void *data) maat_compile_free(compile); } -static void rcu_compile_table_cfg_free(void *user_ctx, void *data) +int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int table_id, + maat_ex_new_func_t *new_func, + maat_ex_free_func_t *free_func, + maat_ex_dup_func_t *dup_func, + long argl, void *argp) { - FREE(data); + if (1 == compile_schema->set_flag) { + log_error(compile_schema->logger, MODULE_COMPILE, + "[%s:%d] compile table(table_id:%d)ex schema has been set already," + " can't set again", __FUNCTION__, __LINE__, table_id); + return -1; + } + + compile_schema->ex_schema.new_func = new_func; + compile_schema->ex_schema.free_func = free_func; + compile_schema->ex_schema.dup_func = dup_func; + compile_schema->ex_schema.argl = argl; + compile_schema->ex_schema.argp = argp; + compile_schema->set_flag = 1; + + return 0; +} + +static void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, + long long compile_id) +{ + struct maat_compile *compile = rcu_hash_find(compile_rt->cfg_hash, + (char *)&compile_id, + sizeof(long long)); + void *ret = NULL; + if (compile != NULL) { + ret = compile->user_data; + } + + return ret; +} + +static void rule_ex_data_new_cb(void *user_data, void *param, + const char *table_name, int table_id) +{ + struct ex_data_schema *ex_schema = (struct ex_data_schema *)param; + struct compile_item *compile = (struct compile_item *)user_data; + + void *ad = rule_ex_data_new(table_name, table_id, compile->table_line, ex_schema); + *compile->ex_data = ad; +} + +static void compile_runtime_user_data_iterate(struct compile_runtime *compile_rt, + void (*callback)(void *user_data, void *param, + const char *table_name, int table_id), + void *param, int table_id) +{ + /* I'm in background_update_mutex, config update can't happen, so no need to lock cfg_hash */ + void **data_array = NULL; + size_t data_cnt = rcu_hash_list(compile_rt->cfg_hash, &data_array); + + for (size_t i = 0; i < data_cnt; i++) { + struct maat_compile *compile = (struct maat_compile *)data_array[i]; + if (compile->user_data) { + callback(compile->user_data, param, compile->table_name, table_id); + } + } + + FREE(data_array); } void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -371,6 +464,12 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, goto error; } + //gc_timeout_s is optional + custom_item = cJSON_GetObjectItem(item, "gc_timeout_s"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->gc_timeout_s = custom_item->valueint; + } + schema->ref_tbl_mgr = tbl_mgr; return schema; error: @@ -494,13 +593,14 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num, return NULL; } + struct compile_schema *schema = (struct compile_schema *)compile_schema; struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1); compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM); compile_rt->version = time(NULL); - compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL, 0); - compile_rt->tbl_cfg_hash = rcu_hash_new(rcu_compile_table_cfg_free, NULL, 0); + compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL, + schema->gc_timeout_s + DEFAULT_GC_TIMEOUT_S); compile_rt->literal2clause_hash = NULL; compile_rt->logger = logger; compile_rt->ref_garbage_bin = garbage_bin; @@ -547,11 +647,6 @@ void compile_runtime_free(void *compile_runtime) compile_rt->cfg_hash = NULL; } - if (compile_rt->tbl_cfg_hash != NULL) { - rcu_hash_free(compile_rt->tbl_cfg_hash); - compile_rt->tbl_cfg_hash = NULL; - } - if (compile_rt->literal2clause_hash != NULL) { literal2clause_hash_free(compile_rt->literal2clause_hash); compile_rt->literal2clause_hash = NULL; @@ -892,30 +987,14 @@ static inline int compare_hit_group(const void *pa, const void *pb) struct maat_hit_group *la=(struct maat_hit_group *)pa; struct maat_hit_group *lb=(struct maat_hit_group *)pb; - long long ret = la->item_id - lb->item_id; + long long ret = la->group_id - lb->group_id; if (ret == 0) { - ret = la->group_id - lb->group_id; - if (ret == 0) { - ret = la->vtable_id - lb->vtable_id; - } + ret = la->vtable_id - lb->vtable_id; } return ret; } -static inline int compare_compile_id(const void *a, const void *b) -{ - long long ret = *(const long long *)a - *(const long long *)b; - - if (0 == ret) { - return 0; - } else if (ret < 0) { - return -1; - } else { - return 1; - } -} - static struct literal_clause * maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt) { @@ -996,29 +1075,6 @@ static size_t compile_state_if_new_hit_compile(struct maat_compile_state *compil return r_in_c_cnt; } -static void compile_state_update_hit_compile_table_id(struct maat_compile_state *compile_state, - long long compile_id, int table_id) -{ - if (!utarray_find(compile_state->hit_compile_tables, &compile_id, compare_compile_id)) { - struct maat_compile_table compile_table = {compile_id, table_id}; - utarray_push_back(compile_state->hit_compile_tables, &compile_table); - utarray_sort(compile_state->hit_compile_tables, compare_compile_id); - } -} - -static int compile_runtime_get_compile_table_id(struct compile_runtime *compile_rt, - long long compile_id) -{ - if (NULL == compile_rt || compile_id < 0) { - return -1; - } - - int *table_id = rcu_hash_find(compile_rt->tbl_cfg_hash, (char *)&compile_id, - sizeof(long long)); - - return *table_id; -} - static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int is_last_scan, int thread_id, struct maat_compile_state *compile_state, @@ -1071,11 +1127,6 @@ static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt hit a compile that refer a NOT-logic group in previous scan */ user_data_array[ud_result_cnt] = compile->user_data; ud_result_cnt++; - - int table_id = compile_runtime_get_compile_table_id(compile_rt, compile->compile_id); - if (table_id >= 0) { - compile_state_update_hit_compile_table_id(compile_state, compile->compile_id, table_id); - } } } } @@ -1091,11 +1142,18 @@ static struct compile_item *compile_item_clone(struct compile_item *item) new_item->compile_id = item->compile_id; new_item->declared_clause_num = item->declared_clause_num; + new_item->ref_schema = item->ref_schema; + new_item->ex_data = ALLOC(void *, 1); memcpy(new_item->table_name, item->table_name, sizeof(new_item->table_name)); new_item->table_line_len = item->table_line_len; new_item->table_line = ALLOC(char, new_item->table_line_len); memcpy(new_item->table_line, item->table_line, new_item->table_line_len); + if (1 == item->ref_schema->set_flag) { + *(new_item->ex_data) = rule_ex_data_new(item->table_name, item->ref_schema->table_id, + item->table_line, &(item->ref_schema->ex_schema)); + } + return new_item; } @@ -1231,7 +1289,7 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl, if (NULL == compile) { log_error(logger, MODULE_COMPILE, "[%s:%d] Remove group_id:%lld from compile_id:%lld failed, compile" - " is not existed.", __FUNCTION__, __LINE__, g2c_item->group_id, + " is not exisited.", __FUNCTION__, __LINE__, g2c_item->group_id, compile_id); return -1; } else { @@ -1287,7 +1345,7 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl, } else { log_error(logger, MODULE_COMPILE, "[%s:%d] Remove group_id:%lld from compile_id:%lld failed, " - "compile is not existed.", __FUNCTION__, __LINE__, + "compile is not exisited.", __FUNCTION__, __LINE__, g2c_item->group_id, compile_id); return -1; } @@ -1304,7 +1362,6 @@ struct maat_compile_state *maat_compile_state_new(void) utarray_new(compile_state->internal_inc_hit_paths, &ut_hit_path_icd); utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd); - utarray_new(compile_state->hit_compile_tables, &ut_hit_compile_table_icd); return compile_state; } @@ -1319,14 +1376,11 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state) compile_state->compile_rt_version = 0; compile_state->this_scan_hit_item_flag = 0; compile_state->not_clause_hit_flag = 0; - compile_state->inc_hit_path_flag = 0; utarray_clear(compile_state->internal_hit_paths); utarray_clear(compile_state->internal_inc_hit_paths); utarray_clear(compile_state->all_hit_clauses); utarray_clear(compile_state->this_scan_hit_clauses); - utarray_clear(compile_state->hit_compile_tables); - } void maat_compile_state_free(struct maat_compile_state *compile_state, @@ -1361,12 +1415,6 @@ void maat_compile_state_free(struct maat_compile_state *compile_state, compile_state->this_scan_hit_clauses = NULL; } - if (compile_state->hit_compile_tables != NULL) { - free_bytes += utarray_len(compile_state->hit_compile_tables) * sizeof(struct maat_compile_table); - utarray_free(compile_state->hit_compile_tables); - compile_state->hit_compile_tables = NULL; - } - FREE(compile_state); free_bytes += sizeof(struct maat_compile_state); @@ -1493,14 +1541,10 @@ static void maat_compile_state_update_hit_path(struct maat_compile_state *compil if (compile_state->Nth_scan != Nth_scan) { assert(compile_state->this_scan_hit_item_flag == 0); compile_state->Nth_scan = Nth_scan; + utarray_clear(compile_state->internal_inc_hit_paths); utarray_clear(compile_state->this_scan_hit_clauses); } - if (1 == compile_state->inc_hit_path_flag) { - compile_state->inc_hit_path_flag = 0; - utarray_clear(compile_state->internal_inc_hit_paths); - } - maat_compile_hit_path_add(compile_state->internal_inc_hit_paths, item_id, group_id, vtable_id, Nth_scan, Nth_item_result); @@ -1550,25 +1594,46 @@ static void maat_compile_state_update_hit_clause(struct maat_compile_state *comp } } -int maat_compile_state_get_compile_table_id(struct maat_compile_state *compile_state, - long long compile_id) -{ - struct maat_compile_table *compile_table = NULL; - - compile_table = utarray_find(compile_state->hit_compile_tables, &compile_id, - compare_compile_id); - if (NULL == compile_table) { - return -1; - } - - return compile_table->table_id; -} - int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state) { return compile_state->not_clause_hit_flag; } +void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt, + struct compile_schema *compile_schema) +{ + if (NULL == compile_rt || NULL == compile_schema || + (0 == compile_schema->set_flag)) { + return; + } + + compile_runtime_user_data_iterate(compile_rt, rule_ex_data_new_cb, + &(compile_schema->ex_schema), + compile_schema->table_id); +} + +void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt, + struct compile_schema *schema, + long long compile_id) +{ + if (NULL == compile_rt || NULL == schema || compile_id < 0 + || (0 == schema->set_flag)) { + return NULL; + } + + struct compile_item *item = NULL; + item = (struct compile_item *)compile_runtime_get_user_data(compile_rt, + compile_id); + if (NULL == item) { + return NULL; + } + + void *ex_data = NULL; + schema->ex_schema.dup_func(schema->table_id, &ex_data, item->ex_data, + schema->ex_schema.argl, schema->ex_schema.argp); + return ex_data; +} + static int compile_runtime_add_compile(struct compile_runtime *compile_rt, struct compile_schema *schema, long long compile_id, const char *table_name, @@ -1737,7 +1802,6 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, if (0 == is_valid) { // delete compile_runtime_del_compile(compile_rt, compile_id); - rcu_hash_del(compile_rt->tbl_cfg_hash, (char *)&compile_id, sizeof(long long)); } else { // add int ret = compile_runtime_add_compile(compile_rt, schema, compile_id, @@ -1745,14 +1809,6 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, if (ret < 0) { compile_rt->update_err_cnt++; } - - int *table_id = ALLOC(int, 1); - *table_id = table_manager_get_table_id(schema->ref_tbl_mgr, table_name); - ret = rcu_hash_add(compile_rt->tbl_cfg_hash, (char *)&compile_id, - sizeof(long long), table_id); - if (ret < 0) { - FREE(table_id); - } } return 0; @@ -1905,7 +1961,6 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name, compile_rt->bm = new_bool_matcher; compile_rt->literal2clause_hash = new_literal2clause; rcu_hash_commit(compile_rt->cfg_hash); - rcu_hash_commit(compile_rt->tbl_cfg_hash); maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL, garbage_bool_matcher_free); @@ -2070,7 +2125,6 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat tmp_hit_path = compile_state->internal_hit_paths; } else if (type == MAAT_LIST_TYPE_INC) { tmp_hit_path = compile_state->internal_inc_hit_paths; - compile_state->inc_hit_path_flag = 1; } for (i = 0; i < utarray_len(tmp_hit_path); i++) { @@ -2085,7 +2139,6 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat for (size_t idx = 0; idx < super_group_cnt; idx++) { struct maat_hit_group hit_group; - hit_group.item_id = internal_path->item_id; hit_group.group_id = super_group_ids[idx]; hit_group.vtable_id = internal_path->vtable_id; if (utarray_find(all_hit_groups, &hit_group, compare_hit_group)) { @@ -2167,4 +2220,16 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp } return hit_path_cnt; +} + +void compile_runtime_garbage_collect_routine(void *compile_runtime) +{ + if (NULL == compile_runtime) { + return; + } + + struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; + if (compile_rt->cfg_hash != NULL) { + rcu_hash_garbage_collect_routine(compile_rt->cfg_hash); + } } \ No newline at end of file diff --git a/src/maat_expr.c b/src/maat_expr.c index 8b7e084..3fb7e8a 100644 --- a/src/maat_expr.c +++ b/src/maat_expr.c @@ -60,7 +60,7 @@ struct expr_item { long long group_id; char keywords[MAX_KEYWORDS_STR]; enum expr_type expr_type; - enum expr_match_mode match_mode; + enum hs_match_mode match_mode; int is_hexbin; int is_case_sensitive; void *user_data; @@ -68,7 +68,7 @@ struct expr_item { }; struct expr_runtime { - struct expr_matcher *matcher; + struct adapter_hs *hs; struct rcu_hash_table *item_hash; // long long version; //expr_rt version @@ -79,7 +79,6 @@ struct expr_runtime { struct log_handle *logger; struct maat_garbage_bin *ref_garbage_bin; - enum maat_expr_engine expr_engine; int district_num; struct maat_kv_store *district_map; struct maat_kv_store *tmp_district_map; @@ -115,22 +114,22 @@ static enum expr_type int_to_expr_type(int expr_type) return type; } -static enum expr_match_mode int_to_match_mode(int match_method) +static enum hs_match_mode int_to_match_mode(int match_method) { - enum expr_match_mode mode = EXPR_MATCH_MODE_INVALID; + enum hs_match_mode mode = HS_MATCH_MODE_INVALID; switch (match_method) { case 0: - mode = EXPR_MATCH_MODE_SUB; + mode = HS_MATCH_MODE_SUB; break; case 1: - mode = EXPR_MATCH_MODE_SUFFIX; + mode = HS_MATCH_MODE_SUFFIX; break; case 2: - mode = EXPR_MATCH_MODE_PREFIX; + mode = HS_MATCH_MODE_PREFIX; break; case 3: - mode = EXPR_MATCH_MODE_EXACTLY; + mode = HS_MATCH_MODE_EXACTLY; break; default: break; @@ -144,13 +143,13 @@ static int expr_runtime_get_district_id(struct expr_runtime *expr_rt, { long long district_id = DISTRICT_ANY; - int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id, 1); + int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id); if (map_ret < 0) { if (NULL == expr_rt->tmp_district_map) { expr_rt->tmp_district_map = maat_kv_store_duplicate(expr_rt->district_map); } - map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id, 1); + map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id); if (map_ret < 0) { district_id = expr_rt->district_num; maat_kv_register(expr_rt->tmp_district_map, district, district_id); @@ -168,7 +167,7 @@ int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *dis return -1; } - return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id, 1); + return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id); } static struct expr_item * @@ -235,8 +234,8 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name, __FUNCTION__, __LINE__, table_name, line); goto error; } else if (expr_item->expr_type == EXPR_TYPE_REGEX) { - ret = expr_matcher_verify_regex_expression(expr_item->keywords, expr_rt->logger); - if (0 == ret) { + ret = adapter_hs_verify_regex_expression(expr_item->keywords, expr_rt->logger); + if (ret < 0) { log_error(expr_rt->logger, MODULE_EXPR, "[%s:%d] expr table:<%s> regex expression(item_id:%lld):%s illegal," " will be dropped", __FUNCTION__, __LINE__, table_name, @@ -278,7 +277,7 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name, match_method_type = atoi(line + column_offset); expr_item->match_mode = int_to_match_mode(match_method_type); - if (expr_item->match_mode == EXPR_MATCH_MODE_INVALID) { + if (expr_item->match_mode == HS_MATCH_MODE_INVALID) { log_error(expr_rt->logger, MODULE_EXPR, "[%s:%d] expr table:<%s> has invalid match_method in line:%s", __FUNCTION__, __LINE__, table_name, line); @@ -473,14 +472,12 @@ void *expr_runtime_new(void *expr_schema, size_t max_thread_num, return NULL; } - struct expr_schema *schema = (struct expr_schema *)expr_schema; struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1); expr_rt->item_hash = rcu_hash_new(expr_item_free_cb, NULL, 0); expr_rt->n_worker_thread = max_thread_num; expr_rt->ref_garbage_bin = garbage_bin; expr_rt->logger = logger; - expr_rt->expr_engine = table_manager_get_expr_engine(schema->ref_tbl_mgr); expr_rt->district_map = maat_kv_store_new(); expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); @@ -498,9 +495,9 @@ void expr_runtime_free(void *expr_runtime) } struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; - if (expr_rt->matcher != NULL) { - expr_matcher_free(expr_rt->matcher); - expr_rt->matcher = NULL; + if (expr_rt->hs != NULL) { + adapter_hs_free(expr_rt->hs); + expr_rt->hs = NULL; } if (expr_rt->item_hash != NULL) { @@ -561,18 +558,18 @@ static int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, return 0; } -static enum expr_pattern_type expr_type2pattern_type(enum expr_type expr_type) +static enum hs_pattern_type expr_type2pattern_type(enum expr_type expr_type) { - enum expr_pattern_type pattern_type = EXPR_PATTERN_TYPE_STR; + enum hs_pattern_type pattern_type; switch (expr_type) { case EXPR_TYPE_STRING: case EXPR_TYPE_AND: case EXPR_TYPE_OFFSET: - pattern_type = EXPR_PATTERN_TYPE_STR; + pattern_type = HS_PATTERN_TYPE_STR; break; case EXPR_TYPE_REGEX: - pattern_type = EXPR_PATTERN_TYPE_REG; + pattern_type = HS_PATTERN_TYPE_REG; break; default: break; @@ -689,12 +686,12 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item, } sub_expr_cnt = i; break; - case EXPR_TYPE_STRING: //AND/OFFSET/STRING type expression use \b to represent blank(' ') + case EXPR_TYPE_STRING: sub_expr_cnt = 1; sub_key_array[0] = expr_item->keywords; sub_key_array[0] = str_unescape(sub_key_array[0]); break; - case EXPR_TYPE_REGEX: //only regex type expression use \s to represent blank(' ') + case EXPR_TYPE_REGEX: sub_expr_cnt = 1; sub_key_array[0] = expr_item->keywords; break; @@ -713,15 +710,15 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item, if (TRUE == expr_item->is_case_sensitive) { // insensitive - expr_rule->patterns[i].case_sensitive = EXPR_CASE_SENSITIVE; + expr_rule->patterns[i].case_sensitive = HS_CASE_SENSITIVE; } else { - expr_rule->patterns[i].case_sensitive = EXPR_CASE_INSENSITIVE; + expr_rule->patterns[i].case_sensitive = HS_CASE_INSENSITIVE; } - expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); + expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type); if (TRUE == expr_item->is_hexbin && - expr_rule->patterns[i].type != EXPR_PATTERN_TYPE_REG) { + expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) { region_str_len = strlen(sub_key_array[i]) * 8 + 1; region_string = ALLOC(char, region_str_len); region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]), @@ -741,13 +738,13 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item, } expr_rule->patterns[i].match_mode = expr_item->match_mode; - if (expr_rule->patterns[i].match_mode == EXPR_MATCH_MODE_SUB) { + if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) { expr_rule->patterns[i].start_offset = key_left_offset[i]; expr_rule->patterns[i].end_offset = key_right_offset[i]; } } expr_rule->expr_id = expr_item->item_id; - expr_rule->tag = expr_item->user_data; + expr_rule->user_tag = expr_item->user_data; expr_rule->n_patterns = sub_expr_cnt; return 0; @@ -813,10 +810,10 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, return 0; } -static void garbage_expr_matcher_free(void *expr_matcher, void *arg) +static void garbage_adapter_hs_free(void *adapter_hs, void *arg) { - struct expr_matcher *matcher = (struct expr_matcher *)expr_matcher; - expr_matcher_free(matcher); + struct adapter_hs *hs = (struct adapter_hs *)adapter_hs; + adapter_hs_free(hs); } int expr_runtime_commit(void *expr_runtime, const char *table_name, @@ -867,42 +864,38 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, } } - struct expr_matcher *new_matcher = NULL; - struct expr_matcher *old_matcher = NULL; + struct adapter_hs *new_adapter_hs = NULL; + struct adapter_hs *old_adapter_hs = NULL; if (rule_cnt > 0) { - enum expr_engine_type engine_type = EXPR_ENGINE_TYPE_HS; - if (expr_rt->expr_engine == MAAT_EXPR_ENGINE_RS) { - engine_type = EXPR_ENGINE_TYPE_RS; - } - - new_matcher = expr_matcher_new(rules, real_rule_cnt, engine_type, - expr_rt->n_worker_thread, expr_rt->logger); - if (NULL == new_matcher) { + new_adapter_hs = adapter_hs_new(rules, real_rule_cnt, expr_rt->n_worker_thread, + expr_rt->logger); + if (NULL == new_adapter_hs) { log_error(expr_rt->logger, MODULE_EXPR, - "[%s:%d] table[%s] rebuild expr_matcher failed when update" + "[%s:%d] table[%s] rebuild adapter_hs engine failed when update" " %zu expr rules", __FUNCTION__, __LINE__, table_name, real_rule_cnt); ret = -1; - } else { - log_info(expr_rt->logger, MODULE_EXPR, - "table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) " - "and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt, - real_rule_cnt, real_regex_rule_cnt, maat_rt_version); } } - old_matcher = expr_rt->matcher; - expr_rt->matcher = new_matcher; + old_adapter_hs = expr_rt->hs; + expr_rt->hs = new_adapter_hs; rcu_hash_commit(expr_rt->item_hash); - if (old_matcher != NULL) { - maat_garbage_bagging(expr_rt->ref_garbage_bin, old_matcher, NULL, garbage_expr_matcher_free); + if (old_adapter_hs != NULL) { + maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL, + garbage_adapter_hs_free); } expr_rt->rule_num = real_rule_cnt; expr_rt->regex_rule_num = real_regex_rule_cnt; expr_rt->version = maat_rt_version; + log_info(expr_rt->logger, MODULE_EXPR, + "table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) " + "and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt, + real_rule_cnt, real_regex_rule_cnt, expr_rt->version); + if (rules != NULL) { for (i = 0; i < rule_cnt; i++) { expr_rule_reset(&rules[i]); @@ -956,14 +949,15 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, return 0; } - if (NULL == expr_rt->matcher) { + if (NULL == expr_rt->hs) { return 0; } size_t n_hit_item = 0; - struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM]; - int ret = expr_matcher_match(expr_rt->matcher, thread_id, data, data_len, - hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item); + struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM]; + int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, + hit_results, MAX_SCANNER_HIT_ITEM_NUM, + &n_hit_item); if (ret < 0) { return -1; } @@ -1006,15 +1000,14 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, return real_hit_item_cnt; } -struct expr_matcher_stream * +struct adapter_hs_stream * expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) { if (NULL == expr_rt || thread_id < 0) { return NULL; } - struct expr_matcher_stream *stream = expr_matcher_stream_open(expr_rt->matcher, - thread_id); + struct adapter_hs_stream *stream = adapter_hs_stream_open(expr_rt->hs, thread_id); if (NULL == stream) { return NULL; } @@ -1023,7 +1016,7 @@ expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) } int expr_runtime_stream_scan(struct expr_runtime *expr_rt, - struct expr_matcher_stream *s_handle, + struct adapter_hs_stream *s_handle, const char *data, size_t data_len, int vtable_id, struct maat_state *state) { @@ -1033,10 +1026,10 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, } size_t n_hit_item = 0; - struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM]; + struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM]; - int ret = expr_matcher_stream_match(s_handle, data, data_len, hit_results, - MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item); + int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results, + MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item); if (ret < 0) { return -1; } @@ -1074,13 +1067,13 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, } void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id, - struct expr_matcher_stream *stream) + struct adapter_hs_stream *s_handle) { - if (NULL == expr_rt || thread_id < 0 || NULL == stream) { + if (NULL == expr_rt || thread_id < 0 || NULL == s_handle) { return; } - expr_matcher_stream_close(stream); + adapter_hs_stream_close(s_handle); } void expr_runtime_hit_inc(struct expr_runtime *expr_rt, int thread_id) @@ -1173,4 +1166,4 @@ long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt) alignment_int64_array_reset(expr_rt->scan_bytes, expr_rt->n_worker_thread); return sum; -} +} \ No newline at end of file diff --git a/src/maat_flag.c b/src/maat_flag.c index ecc38dc..da0747f 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -269,13 +269,13 @@ static int flag_runtime_get_district_id(struct flag_runtime *flag_rt, { long long district_id = DISTRICT_ANY; - int map_ret = maat_kv_read(flag_rt->district_map, district, &district_id, 1); + int map_ret = maat_kv_read(flag_rt->district_map, district, &district_id); if (map_ret < 0) { if (NULL == flag_rt->tmp_district_map) { flag_rt->tmp_district_map = maat_kv_store_duplicate(flag_rt->district_map); } - map_ret = maat_kv_read(flag_rt->tmp_district_map, district, &district_id, 1); + map_ret = maat_kv_read(flag_rt->tmp_district_map, district, &district_id); if (map_ret < 0) { district_id = flag_rt->district_num; maat_kv_register(flag_rt->tmp_district_map, district, district_id); @@ -293,7 +293,7 @@ int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *dis return -1; } - return maat_kv_read_unNull(flag_rt->district_map, district, district_len, district_id, 1); + return maat_kv_read_unNull(flag_rt->district_map, district, district_len, district_id); } static struct flag_item * diff --git a/src/maat_interval.c b/src/maat_interval.c index 4e22659..e5a11a8 100644 --- a/src/maat_interval.c +++ b/src/maat_interval.c @@ -242,13 +242,13 @@ static int interval_runtime_get_district_id(struct interval_runtime *interval_rt { long long district_id = DISTRICT_ANY; - int map_ret = maat_kv_read(interval_rt->district_map, district, &district_id, 1); + int map_ret = maat_kv_read(interval_rt->district_map, district, &district_id); if (map_ret < 0) { if (NULL == interval_rt->tmp_district_map) { interval_rt->tmp_district_map = maat_kv_store_duplicate(interval_rt->district_map); } - map_ret = maat_kv_read(interval_rt->tmp_district_map, district, &district_id, 1); + map_ret = maat_kv_read(interval_rt->tmp_district_map, district, &district_id); if (map_ret < 0) { district_id = interval_rt->district_num; maat_kv_register(interval_rt->tmp_district_map, district, district_id); @@ -268,7 +268,7 @@ int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, } return maat_kv_read_unNull(interval_rt->district_map, district, - district_len, district_id, 1); + district_len, district_id); } static struct interval_item * diff --git a/src/maat_kv.c b/src/maat_kv.c index df9938d..2ce1a73 100644 --- a/src/maat_kv.c +++ b/src/maat_kv.c @@ -17,16 +17,11 @@ #define MAAT_KV_MAX_KEY_LEN 512 #define MAX_ENTITY_ID_CNT 4 -struct kv_entity { - long long id[MAX_ENTITY_ID_CNT]; - int id_cnt; //0~4 -}; - struct maat_kv_pair { char* key; //must be lower case. size_t key_len; - struct kv_entity *val; + long long val; UT_hash_handle hh; }; @@ -43,32 +38,15 @@ static void strlowercase(const char* src, size_t src_len, char* dst, size_t dst_ } static struct maat_kv_pair * -maat_kv_pair_new(const char* key, size_t keylen, long long value) +maat_kv_pair_new(const char* key, size_t key_len, long long value) { struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1); - kv->key = ALLOC(char, keylen); - kv->key_len = keylen; - kv->val = ALLOC(struct kv_entity, 1); - - strlowercase(key, keylen, kv->key, kv->key_len); - kv->val->id[0] = value; - kv->val->id_cnt = 1; - - return kv; -} - -static struct maat_kv_pair * -maat_kv_pair_clone(const char *key, size_t key_len, struct kv_entity *entity) -{ - struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1); - kv->key = ALLOC(char, key_len); kv->key_len = key_len; - kv->val = ALLOC(struct kv_entity, 1); + kv->val = value; strlowercase(key, key_len, kv->key, kv->key_len); - *(kv->val) = *entity; return kv; } @@ -83,10 +61,6 @@ static void maat_kv_pair_free(struct maat_kv_pair *kv) FREE(kv->key); } - if (kv->val != NULL) { - FREE(kv->val); - } - FREE(kv); } @@ -114,23 +88,23 @@ void maat_kv_store_free(struct maat_kv_store *store) } static int maat_kv_register_unNull(struct maat_kv_store *store, const char *key, - size_t keylen, long long value) + size_t key_len, long long value) { - if (keylen > MAAT_KV_MAX_KEY_LEN) { + if (key_len > MAAT_KV_MAX_KEY_LEN) { return -1; } struct maat_kv_pair *kv = NULL; struct maat_kv_pair *tmp_kv = NULL; - kv = maat_kv_pair_new(key, keylen, value); - HASH_FIND(hh, store->hash, kv->key, keylen, tmp_kv); + kv = maat_kv_pair_new(key, key_len, value); + HASH_FIND(hh, store->hash, kv->key, key_len, tmp_kv); if (tmp_kv) { maat_kv_pair_free(kv); return -1; } - HASH_ADD_KEYPTR(hh, store->hash, kv->key, keylen, kv); + HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv); return 1; } @@ -142,34 +116,31 @@ int maat_kv_register(struct maat_kv_store* store, const char *key, long long val return ret; } -int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t keylen, - long long *value_array, size_t n_array) +int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_len, + long long *value) { struct maat_kv_pair *kv = NULL; char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0}; - if (keylen > MAAT_KV_MAX_KEY_LEN) { + if (key_len > MAAT_KV_MAX_KEY_LEN) { return -1; } - strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase)); - HASH_FIND(hh, store->hash, key_lowercase, keylen, kv); + strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase)); + HASH_FIND(hh, store->hash, key_lowercase, key_len, kv); int i = 0; if (kv) { - for (i = 0; i < kv->val->id_cnt && i < (int)n_array; i++) { - value_array[i] = kv->val->id[i]; - } - return i; + *value = kv->val; + return 1; } else { return -1; } } -int maat_kv_read(struct maat_kv_store *store, const char * key, - long long *value_array, size_t n_array) +int maat_kv_read(struct maat_kv_store *store, const char * key, long long *value) { - return maat_kv_read_unNull(store, key, strlen(key), value_array, n_array); + return maat_kv_read_unNull(store, key, strlen(key), value); } int maat_kv_write_unNull(struct maat_kv_store* store, const char* key, @@ -185,8 +156,7 @@ int maat_kv_write_unNull(struct maat_kv_store* store, const char* key, strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase)); HASH_FIND(hh, store->hash, key_lowercase, key_len, kv); if (kv) { - kv->val->id[0] = value; - kv->val->id_cnt = 1; + kv->val = value; } else { kv = maat_kv_pair_new(key, key_len, value); HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv); @@ -200,46 +170,13 @@ int maat_kv_write(struct maat_kv_store *store, const char *key, long long value) return maat_kv_write_unNull(store, key, strlen(key), value); } -int maat_kv_append_unNull(struct maat_kv_store* store, const char* key, - size_t key_len, long long value) -{ - struct maat_kv_pair *kv = NULL; - char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0}; - - if (key_len > MAAT_KV_MAX_KEY_LEN) { - return -1; - } - - strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase)); - HASH_FIND(hh, store->hash, key_lowercase, key_len, kv); - if (kv) { - size_t id_cnt = kv->val->id_cnt; - if (id_cnt >= MAX_ENTITY_ID_CNT) { - return -1; - } - - kv->val->id[id_cnt] = value; - kv->val->id_cnt++; - } else { - kv = maat_kv_pair_new(key, key_len, value); - HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv); - } - - return 1; -} - -int maat_kv_append(struct maat_kv_store *store, const char *key, long long value) -{ - return maat_kv_append_unNull(store, key, strlen(key), value); -} - struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map) { struct maat_kv_store *target = maat_kv_store_new(); struct maat_kv_pair *kv = NULL, *tmp_kv = NULL, *copy_kv = NULL; HASH_ITER (hh, origin_map->hash, kv, tmp_kv) { - copy_kv = maat_kv_pair_clone(kv->key, kv->key_len, kv->val); + copy_kv = maat_kv_pair_new(kv->key, kv->key_len, kv->val); HASH_ADD_KEYPTR(hh, target->hash, copy_kv->key, copy_kv->key_len, copy_kv); } diff --git a/src/maat_rule.c b/src/maat_rule.c index 238b614..6223952 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -127,7 +127,8 @@ static int maat_update_cb(const char *table_name, const char *line, void *u_para return 0; } - struct maat *maat_inst =(struct maat *)u_param; + struct maat *maat_inst =(struct maat *)u_param; + struct maat_runtime* maat_rt = NULL; int table_id = table_manager_get_table_id(maat_inst->tbl_mgr, table_name); if (table_id < 0) { log_error(maat_inst->logger, MODULE_MAAT_RULE, @@ -136,36 +137,23 @@ static int maat_update_cb(const char *table_name, const char *line, void *u_para return -1; } - int update_type = MAAT_UPDATE_TYPE_INC; - if (maat_inst->creating_maat_rt != NULL) { // Full update + void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id); + if (NULL == schema) { + log_error(maat_inst->logger, MODULE_MAAT_RULE, + "[%s:%d] update warning, table name %s doesn't have table schema", + __FUNCTION__, __LINE__, table_name); + return -1; + } + + int update_type = MAAT_UPDATE_TYPE_INC; + if (maat_inst->creating_maat_rt != NULL) { //Full update + maat_rt = maat_inst->creating_maat_rt; update_type = MAAT_UPDATE_TYPE_FULL; - } - - // find conjunction id for table_id - long long conj_parent_table_ids[4]; - int conj_parent_table_cnt = table_manager_get_conj_parent_table_ids(maat_inst->tbl_mgr, table_name, - conj_parent_table_ids, 4); - if (conj_parent_table_cnt > 0) { - for (int i = 0; i < conj_parent_table_cnt; i++) { - int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name, - (int)conj_parent_table_ids[i], line, update_type); - if (ret < 0) { - log_error(maat_inst->logger, MODULE_MAAT_RULE, - "[%s:%d] table<%s> update runtime error for rule:%s", - __FUNCTION__, __LINE__, table_name, line); - continue; - } - } - } else { - int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name, - table_id, line, update_type); - if (ret < 0) { - log_error(maat_inst->logger, MODULE_MAAT_RULE, - "[%s:%d] table<%s> update runtime error for rules:%s", - __FUNCTION__, __LINE__, table_name, line); - return -1; - } - } + } else { + maat_rt = maat_inst->maat_rt; + } + + table_manager_update_runtime(maat_rt->ref_tbl_mgr, table_name, table_id, line, update_type); return 0; } @@ -213,6 +201,10 @@ static void maat_plugin_table_garbage_collect_routine(struct table_manager *tbl_ table_type = table_manager_get_table_type(tbl_mgr, i); switch (table_type) { + case TABLE_TYPE_COMPILE: + runtime = table_manager_get_runtime(tbl_mgr, i); + compile_runtime_garbage_collect_routine(runtime); + break; case TABLE_TYPE_PLUGIN: runtime = table_manager_get_runtime(tbl_mgr, i); ex_data_rt = plugin_runtime_get_ex_data_rt(runtime); @@ -353,7 +345,7 @@ long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *ke } long long sequence = 1; - int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence, 1); + int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence); if (map_ret < 0) { maat_kv_register(maat_rt->sequence_map, key, sequence); } else { diff --git a/src/maat_table.c b/src/maat_table.c index 92e4080..e0185ca 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -47,11 +47,9 @@ struct table_manager { struct rule_tag *accept_tags; size_t n_accept_tag; - enum maat_expr_engine expr_engine; int default_compile_table_id; int g2g_table_id; struct maat_kv_store *tablename2id_map; - struct maat_kv_store *conj_tablename2id_map; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -518,7 +516,7 @@ maat_table_new(cJSON *json, struct maat_kv_store *reserved_word_map, } ret = maat_kv_read(reserved_word_map, item->valuestring, - (long long *)&(ptable->table_type), 1); + (long long *)&(ptable->table_type)); if (ret < 0) { log_error(logger, MODULE_TABLE, "[%s:%d] table:%s table_type %s is illegal", @@ -580,159 +578,90 @@ static void maat_table_free(struct maat_table *maat_tbl) FREE(maat_tbl); } -int register_single_tablename2id(struct maat_kv_store *tablename2id_map, const char *table_name, - int table_id, struct log_handle *logger) +static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map, + struct log_handle *logger) { - if (strlen(table_name) >= NAME_MAX) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table:<%s> name length exceed maxium:%d", - __FUNCTION__, __LINE__, table_name, NAME_MAX); - return -1; - } - - long long tmp_table_id = -1; - int ret = maat_kv_read(tablename2id_map, table_name, &tmp_table_id, 1); - if (ret > 0 && tmp_table_id != table_id) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table:<%s>(table_id:%lld) has already been registered" - ", can't register again", - __FUNCTION__, __LINE__, table_name, tmp_table_id); - return -1; - } - - maat_kv_register(tablename2id_map, table_name, table_id); - return 0; -} - -static int register_conjunction_tablename2id(cJSON *root, struct maat_kv_store *conj_tablename2id_map, - struct log_handle *logger) -{ - int json_array_size = cJSON_GetArraySize(root); - - for (int i = 0; i < json_array_size; i++) { - cJSON *json = cJSON_GetArrayItem(root, i); - cJSON *item = cJSON_GetObjectItem(json, "table_id"); - int table_id = item->valueint; - - item = cJSON_GetObjectItem(json, "db_tables"); - if (NULL == item || item->type != cJSON_Array) { - continue; - } - - int n_table_name = cJSON_GetArraySize(item); - for (int j = 0; j < n_table_name; j++) { - cJSON *tmp_item = cJSON_GetArrayItem(item, j); - if (NULL == tmp_item || tmp_item->type != cJSON_String) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) db_tables element format invalid" - ", should be string", __FUNCTION__, __LINE__, table_id); - return -1; - } - - int ret = maat_kv_append(conj_tablename2id_map, tmp_item->valuestring, table_id); - if (ret < 0) { - return -1; - } - } - } - - return 0; -} - -static int register_tablename2id(cJSON *root, struct maat_kv_store *tablename2id_map, - const char *table_info_path, struct log_handle *logger) -{ - int i = 0; - int json_array_size = cJSON_GetArraySize(root); - - //pre register tablename2id - for (i = 0; i < json_array_size; i++) { - cJSON *json = cJSON_GetArrayItem(root, i); - if (NULL == json || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE, - "[%s:%d] %s has invalid json object", - __FUNCTION__, __LINE__, table_info_path); - return -1; - } - - cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (NULL == item || item->type != cJSON_Number) { - log_error(logger, MODULE_TABLE, - "[%s:%d] %s has invalid json object, happened in table_id column ", - __FUNCTION__, __LINE__, table_info_path); - return -1; - } - int table_id = item->valueint; - - item = cJSON_GetObjectItem(json, "table_name"); - if (NULL == item || item->type != cJSON_String) { - log_error(logger, MODULE_TABLE, - "[%s:%d] %s has invalid json object, happened in table_name column", - __FUNCTION__, __LINE__, table_info_path); - return -1; - } - - int ret = register_single_tablename2id(tablename2id_map, item->valuestring, - table_id, logger); - if (ret < 0) { - return -1; - } - } - - //register db_tables's table_name - for (i = 0; i < json_array_size; i++) { - cJSON *json = cJSON_GetArrayItem(root, i); - cJSON *item = cJSON_GetObjectItem(json, "table_id"); - int table_id = item->valueint; - - item = cJSON_GetObjectItem(json, "db_tables"); - if (NULL == item || item->type != cJSON_Array) { - continue; - } - - int n_table_name = cJSON_GetArraySize(item); - for (int j = 0; j < n_table_name; j++) { - cJSON *tmp_item = cJSON_GetArrayItem(item, j); - if (NULL == tmp_item || tmp_item->type != cJSON_String) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) db_tables element format invalid" - ", should be string", __FUNCTION__, __LINE__, table_id); - return -1; - } - - long long tmp_table_id = -1; - int ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &tmp_table_id, 1); - if (ret > 0) { - continue; - } - - ret = register_single_tablename2id(tablename2id_map, tmp_item->valuestring, - table_id, logger); - if (ret < 0) { - return -1; - } - } - } - - return 0; -} - -int maat_default_compile_table_id(cJSON *json, struct log_handle *logger) -{ - cJSON *item = cJSON_GetObjectItem(json, "default_compile_table"); + cJSON *item = cJSON_GetObjectItem(json, "table_id"); if (NULL == item || item->type != cJSON_Number) { return -1; } + int table_id = item->valueint; - item = cJSON_GetObjectItem(json, "table_id"); - //item is cJSON_Number which has been checked in maat_table_new - return item->valueint; + item = cJSON_GetObjectItem(json, "db_tables"); + if (item != NULL && item->type != cJSON_Array) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) has db_tables, but format is invalid" + ", should be array", __FUNCTION__, __LINE__, table_id); + return -1; + } + + int ret = 0; + long long tmp_table_id = 0; + if (item != NULL) { + int n_table_name = cJSON_GetArraySize(item); + + for (int i = 0; i < n_table_name; i++) { + cJSON *tmp_item = cJSON_GetArrayItem(item, i); + if (NULL == tmp_item || tmp_item->type != cJSON_String) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) db_tables element format invalid" + ", should be string", __FUNCTION__, __LINE__, table_id); + return -1; + } + + if (strlen(tmp_item->valuestring) >= NAME_MAX) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) db_tables string %s length " + "exceed maxium:%d", __FUNCTION__, __LINE__, table_id, + tmp_item->valuestring, NAME_MAX); + return -1; + } + + ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &tmp_table_id); + if (ret > 0 && tmp_table_id != table_id) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table:<%s>(table_id:%lld) has already been registered" + ", can't register again", __FUNCTION__, __LINE__, + tmp_item->valuestring, tmp_table_id); + return -1; + } + + maat_kv_register(tablename2id_map, tmp_item->valuestring, table_id); + } + } + + item = cJSON_GetObjectItem(json, "table_name"); + if (NULL == item || item->type != cJSON_String) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) has no table_name", + __FUNCTION__, __LINE__, table_id); + return -1; + } + + if (strlen(item->valuestring) >= NAME_MAX) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) table_name %s length too long", + __FUNCTION__, __LINE__, table_id, item->valuestring); + return -1; + } + + ret = maat_kv_read(tablename2id_map, item->valuestring, &tmp_table_id); + if (ret > 0 && tmp_table_id != table_id) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table:<%s>(table_id:%lld) has already been registered," + " can't register again", __FUNCTION__, __LINE__, item->valuestring, + tmp_table_id); + return -1; + } + + maat_kv_register(tablename2id_map, item->valuestring, table_id); + + return 0; } struct table_manager * table_manager_create(const char *table_info_path, const char *accept_tags, - enum maat_expr_engine expr_engine, struct maat_garbage_bin *garbage_bin, - struct log_handle *logger) + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == table_info_path) { return NULL; @@ -773,28 +702,23 @@ table_manager_create(const char *table_info_path, const char *accept_tags, tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger); tbl_mgr->logger = logger; tbl_mgr->tablename2id_map = maat_kv_store_new(); - tbl_mgr->conj_tablename2id_map = maat_kv_store_new(); - tbl_mgr->expr_engine = expr_engine; tbl_mgr->ref_garbage_bin = garbage_bin; - ret = register_tablename2id(root, tbl_mgr->tablename2id_map, table_info_path, logger); - if (ret < 0) { - log_error(logger, MODULE_TABLE, - "[%s:%d] register_tablename2id failed.", __FUNCTION__, __LINE__); - FREE(json_buff); - cJSON_Delete(root); - table_manager_destroy(tbl_mgr); - return NULL; - } + for (int i = 0; i < json_array_size; i++) { + json = cJSON_GetArrayItem(root, i); - ret = register_conjunction_tablename2id(root, tbl_mgr->conj_tablename2id_map, logger); - if (ret < 0) { - log_error(logger, MODULE_TABLE, - "[%s:%d] register_conjunction_tablename2id failed.", __FUNCTION__, __LINE__); - FREE(json_buff); - cJSON_Delete(root); - table_manager_destroy(tbl_mgr); - return NULL; + if (json != NULL && json->type == cJSON_Object) { + ret = register_tablename2id(json, tbl_mgr->tablename2id_map, logger); + if (ret < 0) { + log_error(logger, MODULE_TABLE, + "[%s:%d] register_tablename2id failed", + __FUNCTION__, __LINE__); + FREE(json_buff); + cJSON_Delete(root); + table_manager_destroy(tbl_mgr); + return NULL; + } + } } int default_compile_table_id = -1; @@ -812,27 +736,22 @@ table_manager_create(const char *table_info_path, const char *accept_tags, goto next; } - long long parent_table_ids[4]; - int parent_table_cnt = table_manager_get_conj_parent_table_ids(tbl_mgr, maat_tbl->table_name, - parent_table_ids, 4); - if (parent_table_cnt <= 0) { - // if table has conjuncion parent, which can share schema and - // runtime with parent. - maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type, - tbl_mgr, logger); - if (NULL == maat_tbl->schema) { - log_error(logger, MODULE_TABLE, - "[%s:%d] Maat table schema new failed, table_name:%s", - __FUNCTION__, __LINE__, maat_tbl->table_name); - ret = -1; - goto next; - } + maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, + maat_tbl->table_type, tbl_mgr, logger); + if (NULL == maat_tbl->schema) { + log_error(logger, MODULE_TABLE, + "[%s:%d] Maat table schema new failed, table_name:%s", + __FUNCTION__, __LINE__, maat_tbl->table_name); + ret = -1; + goto next; } - + if (maat_tbl->table_type == TABLE_TYPE_COMPILE) { if (default_compile_table_id < 0) { - default_compile_table_id = maat_default_compile_table_id(json, logger); - } + default_compile_table_id = maat_tbl->table_id; + } else if (maat_tbl->table_id < default_compile_table_id) { + default_compile_table_id = maat_tbl->table_id; + } } if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) { @@ -997,11 +916,6 @@ void table_manager_destroy(struct table_manager *tbl_mgr) tbl_mgr->tablename2id_map = NULL; } - if (tbl_mgr->conj_tablename2id_map != NULL) { - maat_kv_store_free(tbl_mgr->conj_tablename2id_map); - tbl_mgr->conj_tablename2id_map = NULL; - } - FREE(tbl_mgr); } @@ -1019,14 +933,14 @@ size_t table_manager_table_count(struct table_manager *tbl_mgr) return tbl_mgr->n_table; } -int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_name) +int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) { - if (NULL == tbl_mgr || NULL == table_name) { + if (NULL == tbl_mgr || NULL == name) { return -1; } long long table_id = -1; - int ret = maat_kv_read(tbl_mgr->tablename2id_map, table_name, &table_id, 1); + int ret = maat_kv_read(tbl_mgr->tablename2id_map, name, &table_id); if (ret < 0) { return -1; } @@ -1034,17 +948,6 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_ return (int)table_id; } -int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name, - long long *table_ids_array, size_t n_table_ids_array) -{ - if (NULL == tbl_mgr || NULL == table_name) { - return -1; - } - - return maat_kv_read(tbl_mgr->conj_tablename2id_map, table_name, - table_ids_array, n_table_ids_array); -} - const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id) { if (NULL == tbl_mgr || table_id < 0) { @@ -1115,15 +1018,6 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id) return tbl_mgr->tbl[table_id]->valid_column; } -enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr) -{ - if (NULL == tbl_mgr) { - return MAAT_EXPR_ENGINE_HS; - } - - return tbl_mgr->expr_engine; -} - size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr) { if (NULL == tbl_mgr) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f3be40b..a3bc0e5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,9 +3,7 @@ include_directories(${PROJECT_SOURCE_DIR}/include) include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) include_directories(${PROJECT_SOURCE_DIR}/deps) include_directories(${PROJECT_SOURCE_DIR}/scanner) -include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher) -include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_hs) -include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_rs) +include_directories(${PROJECT_SOURCE_DIR}/scanner/adapter_hs) include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher) include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher) @@ -21,8 +19,8 @@ target_link_libraries(maat_framework_gtest maat_frame_static gtest_static) add_executable(maat_framework_perf_gtest maat_framework_perf_gtest.cpp) target_link_libraries(maat_framework_perf_gtest maat_frame_static gtest_static) -add_executable(expr_matcher_gtest expr_matcher_gtest.cpp) -target_link_libraries(expr_matcher_gtest maat_frame_static gtest_static) +add_executable(adapter_hs_gtest adapter_hs_gtest.cpp) +target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static) add_executable(ip_matcher_gtest ip_matcher_gtest.cpp) target_link_libraries(ip_matcher_gtest maat_frame_static gtest_static) @@ -34,7 +32,6 @@ add_executable(maat_ex_data_gtest maat_ex_data_gtest.cpp) target_link_libraries(maat_ex_data_gtest maat_frame_static gtest_static) add_subdirectory(group_exclude) -add_subdirectory(benchmark) file(COPY table_info.conf DESTINATION ./) file(COPY tsg_table_info.conf DESTINATION ./) @@ -42,13 +39,13 @@ file(COPY file_test_tableinfo.conf DESTINATION ./) file(COPY literal_expr.conf DESTINATION ./) file(COPY regex_expr.conf DESTINATION ./) file(COPY maat_json.json DESTINATION ./) +file(COPY maat_json.json DESTINATION ../tools/) file(COPY ntcrule DESTINATION ./) file(COPY tsgrule DESTINATION ./) file(COPY testdata DESTINATION ./) file(COPY test_streamfiles DESTINATION ./) file(COPY json_update DESTINATION ./) file(COPY group_exclude DESTINATION ./) -file(COPY benchmark DESTINATION ./) include(GoogleTest) -gtest_discover_tests(maat_framework_gtest) +gtest_discover_tests(maat_framework_gtest) \ No newline at end of file diff --git a/test/adapter_hs_gtest.cpp b/test/adapter_hs_gtest.cpp new file mode 100644 index 0000000..b413916 --- /dev/null +++ b/test/adapter_hs_gtest.cpp @@ -0,0 +1,730 @@ +#include + +#include "log/log.h" +#include "adapter_hs.h" +#include "maat_utils.h" +#include "cJSON/cJSON.h" + +struct log_handle *g_logger = NULL; + +enum hs_match_mode match_method_to_match_mode(const char *method) +{ + enum hs_match_mode mode = HS_MATCH_MODE_INVALID; + + if (strcmp(method, "sub") == 0) { + mode = HS_MATCH_MODE_SUB; + } else if (strcmp(method, "exactly") == 0) { + mode = HS_MATCH_MODE_EXACTLY; + } else if (strcmp(method, "prefix") == 0) { + mode = HS_MATCH_MODE_PREFIX; + } else if (strcmp(method, "suffix") == 0) { + mode = HS_MATCH_MODE_SUFFIX; + } else { + assert(0); + } + + return mode; +} + +enum hs_case_sensitive case_sensitive_str_to_enum(const char *str) +{ + enum hs_case_sensitive case_sensitive = HS_CASE_SENSITIVE; + + if (strcmp(str, "yes") == 0) { + case_sensitive = HS_CASE_SENSITIVE; + } else if (strcmp(str, "no") == 0) { + case_sensitive = HS_CASE_INSENSITIVE; + } else { + assert(0); + } + + return case_sensitive; +} + +int is_hexbin_str_to_int(const char *str) +{ + int ret = 0; + + if (strcmp(str, "yes") == 0) { + ret = 1; + } + + return ret; +} + +static int convertHextoint(char srctmp) +{ + if (isdigit(srctmp)) { + return srctmp - '0'; + } else { + char temp = toupper(srctmp); + temp = temp - 'A' + 10; + return temp; + } +} + +static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size) +{ + size_t resultlen = 0; + int high,low; + for (int i = 0; i < hex_len && size > resultlen; i += 2, resultlen++) { + high = convertHextoint(hex[i]); + low = convertHextoint(hex[i+1]); + binary[resultlen] = high * 16 + low; + } + + size = resultlen; + binary[resultlen] = '\0'; + + return resultlen; +} + +enum hs_pattern_type pattern_type_str_to_enum(const char *str) +{ + enum hs_pattern_type pattern_type; + + if (strcmp(str, "regex") == 0) { + pattern_type = HS_PATTERN_TYPE_REG; + } else if (strcmp(str, "literal") == 0) { + pattern_type = HS_PATTERN_TYPE_STR; + } else { + assert(0); + } + + return pattern_type; +} + +int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_expr) +{ + unsigned char *json_buff = NULL; + size_t json_buff_size = 0; + + int ret = load_file_to_memory(filename, &json_buff, &json_buff_size); + if (ret < 0) { + printf("load file:%s to memory failed.\n", filename); + return -1; + } + + size_t rule_cnt = 0; + cJSON *rules_obj = NULL; + cJSON *root = cJSON_Parse((const char *)json_buff); + if (NULL == root) { + printf("Error before: %-200.200s\n", cJSON_GetErrorPtr()); + ret = -1; + goto next; + } + + rules_obj = cJSON_GetObjectItem(root, "expr_rules"); + if (NULL == rules_obj) { + printf("Error before: %-200.200s\n", cJSON_GetErrorPtr()); + ret = -1; + goto next; + } + + rule_cnt = cJSON_GetArraySize(rules_obj); + for (size_t i = 0; i < rule_cnt; i++) { + cJSON *expr_obj = cJSON_GetArrayItem(rules_obj, i); + cJSON *tmp_item = cJSON_GetObjectItem(expr_obj, "expr_id"); + if (tmp_item != NULL && tmp_item->type == cJSON_Number) { + exprs[i].expr_id = tmp_item->valueint; + } + + tmp_item = cJSON_GetObjectItem(expr_obj, "pattern_num"); + if (tmp_item != NULL && tmp_item->type == cJSON_Number) { + exprs[i].n_patterns = tmp_item->valueint; + } + + tmp_item = cJSON_GetObjectItem(expr_obj, "patterns"); + if (NULL == tmp_item || tmp_item->type != cJSON_Array) { + printf("json has no patterns array.\n"); + ret = -1; + goto next; + } + + size_t pattern_cnt = cJSON_GetArraySize(tmp_item); + for (size_t j = 0; j < pattern_cnt; j++) { + cJSON *pat_item = cJSON_GetArrayItem(tmp_item, j); + + cJSON *item = cJSON_GetObjectItem(pat_item, "pattern_type"); + if (item != NULL && item->type == cJSON_String) { + exprs[i].patterns[j].pattern_type = pattern_type_str_to_enum(item->valuestring); + } + + item = cJSON_GetObjectItem(pat_item, "match_method"); + if (item != NULL && item->type == cJSON_String) { + exprs[i].patterns[j].match_mode = match_method_to_match_mode(item->valuestring); + } + + item = cJSON_GetObjectItem(pat_item, "case_sensitive"); + if (item != NULL && item->type == cJSON_String) { + exprs[i].patterns[j].case_sensitive = case_sensitive_str_to_enum(item->valuestring); + } + + item = cJSON_GetObjectItem(pat_item, "is_hexbin"); + if (item != NULL && item->type == cJSON_String) { + exprs[i].patterns[j].is_hexbin = is_hexbin_str_to_int(item->valuestring); + } + + item = cJSON_GetObjectItem(pat_item, "pattern"); + if (item != NULL && item->type == cJSON_String) { + exprs[i].patterns[j].pat = ALLOC(char, strlen(item->valuestring) + 1); + + if (exprs[i].patterns[j].is_hexbin == 1) { + size_t pat_str_len = strlen(item->valuestring) + 1; + char *pat_str = ALLOC(char, pat_str_len); + pat_str_len = hex2bin(item->valuestring, strlen(item->valuestring), + pat_str, pat_str_len); + + memcpy(exprs[i].patterns[j].pat, pat_str, pat_str_len); + free(pat_str); + exprs[i].patterns[j].pat_len = pat_str_len; + } else { + memcpy(exprs[i].patterns[j].pat, item->valuestring, + strlen(item->valuestring)); + exprs[i].patterns[j].pat_len = strlen(item->valuestring); + } + } + + if (exprs[i].patterns->match_mode == HS_MATCH_MODE_SUB) { + item = cJSON_GetObjectItem(pat_item, "offset"); + if (item != NULL && item->type == cJSON_String) { + int key_left_offset = -1; + int key_right_offset = -1; + sscanf(item->valuestring, "%d~%d", &key_left_offset, &key_right_offset); + if (key_left_offset < -1 || key_right_offset < -1) { + printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n", + key_left_offset, key_right_offset); + } + exprs[i].patterns[j].start_offset = key_left_offset; + exprs[i].patterns[j].end_offset = key_right_offset; + } else { + exprs[i].patterns[j].start_offset = -1; + exprs[i].patterns[j].end_offset = -1; + } + } + + if (exprs[i].patterns->match_mode == HS_MATCH_MODE_EXACTLY) { + exprs[i].patterns[j].start_offset = 0; + exprs[i].patterns[j].end_offset = exprs[i].patterns[j].pat_len - 1; + } + } + exprs[i].n_patterns = pattern_cnt; + } + + *n_expr = rule_cnt; +next: + cJSON_Delete(root); + FREE(json_buff); + return ret; +} + +void expr_array_free(struct expr_rule rules[], size_t n_rule) +{ + for (size_t i = 0; i < n_rule; i++) { + for (size_t j = 0; j < rules[i].n_patterns; j++) { + if (rules[i].patterns[j].pat != NULL) { + free(rules[i].patterns[j].pat); + rules[i].patterns[j].pat = NULL; + } + } + } +} + +TEST(adapter_hs_init, invalid_input_parameter) +{ + struct expr_rule rules[64]; + size_t n_rule = 0; + + struct adapter_hs *hs_instance = adapter_hs_new(NULL, 0, 1, g_logger); + EXPECT_TRUE(hs_instance == NULL); + + hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance == NULL); + + n_rule = 1; + rules[0].expr_id = 101; + rules[0].n_patterns = 10; + hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance == NULL); + + memset(rules, 0, sizeof(rules)); + n_rule = 1; + rules[0].expr_id = 101; + rules[0].n_patterns = 1; + hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance == NULL); +} + +TEST(adapter_hs_scan, literal_sub_has_normal_offset) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char scan_data1[64] = "hello aaa"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data2[64] = "Ahello aaa"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 101); + + char scan_data3[64] = "Aahello aaa"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 101); + + char scan_data4[64] = "Aaahello aaa"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char scan_data1[64] = "hello bbb"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 102); + + char scan_data2[64] = "Ahello bbb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 102); + + char scan_data3[64] = "Aahello bbb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 102); + + char scan_data4[64] = "Aaahello bbb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, literal_sub_has_right_unlimit_offset) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char scan_data1[64] = "hello ccc"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data2[64] = "1234hello ccc"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data3[64] = "12345hello ccc"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 103); + + char scan_data4[64] = "12345hello cccAaBb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 103); + + char scan_data5[64] = "123456hello cccAaBb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data5, strlen(scan_data5), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 103); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, literal_sub_with_no_offset) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char scan_data1[64] = "hello ddd"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 104); + + char scan_data2[64] = "123hello ddd"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 104); + + char scan_data3[64] = "123hello ddd456"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 104); + + char scan_data4[64] = "helloddd"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, literal_exactly) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char scan_data1[64] = "hello eee"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 105); + + char scan_data2[64] = "Ahello eee"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data3[64] = "hello eeeB"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, literal_prefix) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char scan_data1[64] = "hello fff"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 106); + + char scan_data2[64] = "Ahello fff"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data3[64] = "Ahello fffBCD"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data4[64] = "hello fffBCD"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 106); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, literal_suffix) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char scan_data1[64] = "hello ggg"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 107); + + char scan_data2[64] = "ABChello ggg"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 107); + + char scan_data3[64] = "ABChello gggDEF"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data4[64] = "hello gggDEF"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, literal_sub_with_hexbin) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char scan_data1[64] = "Content-Type: /html"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 108); + + char scan_data2[64] = " html"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, literal_with_chinese) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char data0[64] = "#中国 你好"; + struct hs_scan_result result0[64] = {0}; + size_t n_result0 = 0; + ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result0, 1); + EXPECT_EQ(result0[0].rule_id, 110); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, same_pattern_different_offset) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + char data[64] = "onetoday,anothertoday"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, data, strlen(data), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 112); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, long_scan_data) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger); + EXPECT_TRUE(hs_instance != NULL); + expr_array_free(rules, n_rule); + + const char* scan_data = "A directed path in a directed graph is a finite or infinite\ +sequence of edges which joins a sequence of distinct vertices, but with the added restriction\ +that the edges be all directed in the same direction."; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data, strlen(scan_data), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].rule_id, 113); + + adapter_hs_free(hs_instance); + hs_instance = NULL; +} + +TEST(adapter_hs_scan, regex_expression_check) +{ + struct expr_rule rules[64] = {0}; + size_t n_rule = 0; + + int ret = parse_config_file("./regex_expr.conf", rules, &n_rule); + EXPECT_EQ(ret, 0); + + for (size_t i = 0; i < n_rule; i++) { + for (size_t j = 0; j < rules[i].n_patterns; j++) { + adapter_hs_verify_regex_expression(rules[i].patterns[j].pat, g_logger); + } + } + + expr_array_free(rules, n_rule); +} + +int main(int argc, char **argv) +{ + int ret = 0; + ::testing::InitGoogleTest(&argc, argv); + g_logger = log_handle_create("./adapter_hs_gtest.log", 0); + + ret = RUN_ALL_TESTS(); + + log_handle_destroy(g_logger); + + return ret; +} \ No newline at end of file diff --git a/test/expr_matcher_gtest.cpp b/test/expr_matcher_gtest.cpp deleted file mode 100644 index 7143e27..0000000 --- a/test/expr_matcher_gtest.cpp +++ /dev/null @@ -1,1330 +0,0 @@ -#include - -#include "log/log.h" -#include "adapter_hs.h" -#include "maat_utils.h" -#include "cJSON/cJSON.h" - -struct log_handle *g_logger = NULL; - -enum expr_match_mode match_method_to_match_mode(const char *method) -{ - enum expr_match_mode mode = EXPR_MATCH_MODE_INVALID; - - if (strcmp(method, "sub") == 0) { - mode = EXPR_MATCH_MODE_SUB; - } else if (strcmp(method, "exactly") == 0) { - mode = EXPR_MATCH_MODE_EXACTLY; - } else if (strcmp(method, "prefix") == 0) { - mode = EXPR_MATCH_MODE_PREFIX; - } else if (strcmp(method, "suffix") == 0) { - mode = EXPR_MATCH_MODE_SUFFIX; - } else { - assert(0); - } - - return mode; -} - -enum expr_case_sensitive case_sensitive_str_to_enum(const char *str) -{ - enum expr_case_sensitive case_sensitive = EXPR_CASE_SENSITIVE; - - if (strcmp(str, "yes") == 0) { - case_sensitive = EXPR_CASE_SENSITIVE; - } else if (strcmp(str, "no") == 0) { - case_sensitive = EXPR_CASE_INSENSITIVE; - } else { - assert(0); - } - - return case_sensitive; -} - -int is_hexbin_str_to_int(const char *str) -{ - int ret = 0; - - if (strcmp(str, "yes") == 0) { - ret = 1; - } - - return ret; -} - -static int convertHextoint(char srctmp) -{ - if (isdigit(srctmp)) { - return srctmp - '0'; - } else { - char temp = toupper(srctmp); - temp = temp - 'A' + 10; - return temp; - } -} - -static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size) -{ - size_t resultlen = 0; - int high,low; - for (int i = 0; i < hex_len && size > resultlen; i += 2, resultlen++) { - high = convertHextoint(hex[i]); - low = convertHextoint(hex[i+1]); - binary[resultlen] = high * 16 + low; - } - - size = resultlen; - binary[resultlen] = '\0'; - - return resultlen; -} - -enum expr_pattern_type pattern_type_str_to_enum(const char *str) -{ - enum expr_pattern_type pattern_type = EXPR_PATTERN_TYPE_STR; - - if (strcmp(str, "regex") == 0) { - pattern_type = EXPR_PATTERN_TYPE_REG; - } else if (strcmp(str, "literal") == 0) { - pattern_type = EXPR_PATTERN_TYPE_STR; - } else { - assert(0); - } - - return pattern_type; -} - -int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_expr) -{ - unsigned char *json_buff = NULL; - size_t json_buff_size = 0; - - int ret = load_file_to_memory(filename, &json_buff, &json_buff_size); - if (ret < 0) { - printf("load file:%s to memory failed.\n", filename); - return -1; - } - - size_t rule_cnt = 0; - cJSON *rules_obj = NULL; - cJSON *root = cJSON_Parse((const char *)json_buff); - if (NULL == root) { - printf("Error before: %-200.200s\n", cJSON_GetErrorPtr()); - ret = -1; - goto next; - } - - rules_obj = cJSON_GetObjectItem(root, "expr_rules"); - if (NULL == rules_obj) { - printf("Error before: %-200.200s\n", cJSON_GetErrorPtr()); - ret = -1; - goto next; - } - - rule_cnt = cJSON_GetArraySize(rules_obj); - for (size_t i = 0; i < rule_cnt; i++) { - cJSON *expr_obj = cJSON_GetArrayItem(rules_obj, i); - cJSON *tmp_item = cJSON_GetObjectItem(expr_obj, "expr_id"); - if (tmp_item != NULL && tmp_item->type == cJSON_Number) { - exprs[i].expr_id = tmp_item->valueint; - } - - tmp_item = cJSON_GetObjectItem(expr_obj, "pattern_num"); - if (tmp_item != NULL && tmp_item->type == cJSON_Number) { - exprs[i].n_patterns = tmp_item->valueint; - } - - tmp_item = cJSON_GetObjectItem(expr_obj, "patterns"); - if (NULL == tmp_item || tmp_item->type != cJSON_Array) { - printf("json has no patterns array.\n"); - ret = -1; - goto next; - } - - size_t pattern_cnt = cJSON_GetArraySize(tmp_item); - for (size_t j = 0; j < pattern_cnt; j++) { - cJSON *pat_item = cJSON_GetArrayItem(tmp_item, j); - - cJSON *item = cJSON_GetObjectItem(pat_item, "pattern_type"); - if (item != NULL && item->type == cJSON_String) { - exprs[i].patterns[j].type = pattern_type_str_to_enum(item->valuestring); - } - - item = cJSON_GetObjectItem(pat_item, "match_method"); - if (item != NULL && item->type == cJSON_String) { - exprs[i].patterns[j].match_mode = match_method_to_match_mode(item->valuestring); - } - - item = cJSON_GetObjectItem(pat_item, "case_sensitive"); - if (item != NULL && item->type == cJSON_String) { - exprs[i].patterns[j].case_sensitive = case_sensitive_str_to_enum(item->valuestring); - } - - int is_hexbin = 0; - item = cJSON_GetObjectItem(pat_item, "is_hexbin"); - if (item != NULL && item->type == cJSON_String) { - is_hexbin = is_hexbin_str_to_int(item->valuestring); - } - - item = cJSON_GetObjectItem(pat_item, "pattern"); - if (item != NULL && item->type == cJSON_String) { - exprs[i].patterns[j].pat = ALLOC(char, strlen(item->valuestring) + 1); - - if (is_hexbin == 1) { - size_t pat_str_len = strlen(item->valuestring) + 1; - char *pat_str = ALLOC(char, pat_str_len); - pat_str_len = hex2bin(item->valuestring, strlen(item->valuestring), - pat_str, pat_str_len); - - memcpy(exprs[i].patterns[j].pat, pat_str, pat_str_len); - free(pat_str); - exprs[i].patterns[j].pat_len = pat_str_len; - } else { - memcpy(exprs[i].patterns[j].pat, item->valuestring, - strlen(item->valuestring)); - exprs[i].patterns[j].pat_len = strlen(item->valuestring); - } - } - - if (exprs[i].patterns->match_mode == EXPR_MATCH_MODE_SUB) { - item = cJSON_GetObjectItem(pat_item, "offset"); - if (item != NULL && item->type == cJSON_String) { - int key_left_offset = -1; - int key_right_offset = -1; - sscanf(item->valuestring, "%d~%d", &key_left_offset, &key_right_offset); - if (key_left_offset < -1 || key_right_offset < -1) { - printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n", - key_left_offset, key_right_offset); - } - exprs[i].patterns[j].start_offset = key_left_offset; - exprs[i].patterns[j].end_offset = key_right_offset; - } else { - exprs[i].patterns[j].start_offset = -1; - exprs[i].patterns[j].end_offset = -1; - } - } - - if (exprs[i].patterns->match_mode == EXPR_MATCH_MODE_EXACTLY) { - exprs[i].patterns[j].start_offset = 0; - exprs[i].patterns[j].end_offset = exprs[i].patterns[j].pat_len - 1; - } - } - exprs[i].n_patterns = pattern_cnt; - } - - *n_expr = rule_cnt; -next: - cJSON_Delete(root); - FREE(json_buff); - return ret; -} - -void expr_array_free(struct expr_rule rules[], size_t n_rule) -{ - for (size_t i = 0; i < n_rule; i++) { - for (size_t j = 0; j < rules[i].n_patterns; j++) { - if (rules[i].patterns[j].pat != NULL) { - free(rules[i].patterns[j].pat); - rules[i].patterns[j].pat = NULL; - } - } - } -} - -TEST(hs_expr_matcher_init, invalid_input_parameter) -{ - struct expr_rule rules[64]; - size_t n_rule = 0; - - struct expr_matcher *matcher = expr_matcher_new(NULL, 0, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher == NULL); - - matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher == NULL); - - n_rule = 1; - rules[0].expr_id = 101; - rules[0].n_patterns = 10; - matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher == NULL); - - memset(rules, 0, sizeof(rules)); - n_rule = 1; - rules[0].expr_id = 101; - rules[0].n_patterns = 1; - matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher == NULL); -} - -TEST(rs_expr_matcher_init, invalid_input_parameter) -{ - struct expr_rule rules[64]; - size_t n_rule = 0; - - struct expr_matcher *matcher = expr_matcher_new(NULL, 0, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher == NULL); - - matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher == NULL); - - n_rule = 1; - rules[0].expr_id = 101; - rules[0].n_patterns = 10; - matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher == NULL); - - memset(rules, 0, sizeof(rules)); - n_rule = 1; - rules[0].expr_id = 101; - rules[0].n_patterns = 1; - matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher == NULL); -} - -TEST(hs_expr_matcher_match, literal_sub_has_normal_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello aaa"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data2[64] = "Ahello aaa"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 101); - - char scan_data3[64] = "Aahello aaa"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 101); - - char scan_data4[64] = "Aaahello aaa"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, literal_sub_has_normal_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello aaa"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data2[64] = "Ahello aaa"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 101); - - char scan_data3[64] = "Aahello aaa"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 101); - - char scan_data4[64] = "Aaahello aaa"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, literal_sub_has_left_unlimit_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello bbb"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 102); - - char scan_data2[64] = "Ahello bbb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 102); - - char scan_data3[64] = "Aahello bbb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 102); - - char scan_data4[64] = "Aaahello bbb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, literal_sub_has_left_unlimit_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello bbb"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 102); - - char scan_data2[64] = "Ahello bbb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 102); - - char scan_data3[64] = "Aahello bbb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 102); - - char scan_data4[64] = "Aaahello bbb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, literal_sub_has_right_unlimit_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello ccc"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data2[64] = "1234hello ccc"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data3[64] = "12345hello ccc"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 103); - - char scan_data4[64] = "12345hello cccAaBb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 103); - - char scan_data5[64] = "123456hello cccAaBb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data5, strlen(scan_data5), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 103); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, literal_sub_has_right_unlimit_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello ccc"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data2[64] = "1234hello ccc"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data3[64] = "12345hello ccc"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 103); - - char scan_data4[64] = "12345hello cccAaBb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 103); - - char scan_data5[64] = "123456hello cccAaBb"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data5, strlen(scan_data5), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 103); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, literal_sub_with_no_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello ddd"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 104); - - char scan_data2[64] = "123hello ddd"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 104); - - char scan_data3[64] = "123hello ddd456"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 104); - - char scan_data4[64] = "helloddd"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, literal_sub_with_no_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello ddd"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 104); - - char scan_data2[64] = "123hello ddd"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 104); - - char scan_data3[64] = "123hello ddd456"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 104); - - char scan_data4[64] = "helloddd"; - memset(result, 0, sizeof(result)); - n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, literal_exactly) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello eee"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 105); - - char scan_data2[64] = "Ahello eee"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data3[64] = "hello eeeB"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, literal_exactly) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello eee"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 105); - - char scan_data2[64] = "Ahello eee"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data3[64] = "hello eeeB"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, literal_prefix) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello fff"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 106); - - char scan_data2[64] = "Ahello fff"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data3[64] = "Ahello fffBCD"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data4[64] = "hello fffBCD"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 106); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, literal_prefix) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello fff"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 106); - - char scan_data2[64] = "Ahello fff"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data3[64] = "Ahello fffBCD"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data4[64] = "hello fffBCD"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 106); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, literal_suffix) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello ggg"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 107); - - char scan_data2[64] = "ABChello ggg"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 107); - - char scan_data3[64] = "ABChello gggDEF"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data4[64] = "hello gggDEF"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, literal_suffix) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "hello ggg"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 107); - - char scan_data2[64] = "ABChello ggg"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 107); - - char scan_data3[64] = "ABChello gggDEF"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - char scan_data4[64] = "hello gggDEF"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, literal_sub_with_hex) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "Content-Type: /html"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 108); - - char scan_data2[64] = " html"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, literal_sub_with_hex) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char scan_data1[64] = "Content-Type: /html"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 108); - - char scan_data2[64] = " html"; - memset(result, 0, sizeof(result)); - n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, literal_with_chinese) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char data0[64] = "#中国 你好"; - struct expr_scan_result result0[64] = {0}; - size_t n_result0 = 0; - ret = expr_matcher_match(matcher, 0, data0, strlen(data0), result0, 64, &n_result0); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result0, 1); - EXPECT_EQ(result0[0].rule_id, 110); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, literal_with_chinese) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char data0[64] = "#中国 你好"; - struct expr_scan_result result0[64] = {0}; - size_t n_result0 = 0; - ret = expr_matcher_match(matcher, 0, data0, strlen(data0), result0, 64, &n_result0); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result0, 1); - EXPECT_EQ(result0[0].rule_id, 110); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, same_pattern_different_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char data[64] = "onetoday,anothertoday"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - ret = expr_matcher_match(matcher, 0, data, strlen(data), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 112); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, same_pattern_different_offset) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - char data[64] = "onetoday,anothertoday"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - ret = expr_matcher_match(matcher, 0, data, strlen(data), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 112); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher_match, long_scan_data) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - const char* scan_data = "A directed path in a directed graph is a finite or infinite\ -sequence of edges which joins a sequence of distinct vertices, but with the added restriction\ -that the edges be all directed in the same direction."; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data, strlen(scan_data), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 113); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_match, long_scan_data) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - const char* scan_data = "A directed path in a directed graph is a finite or infinite\ -sequence of edges which joins a sequence of distinct vertices, but with the added restriction\ -that the edges be all directed in the same direction."; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - ret = expr_matcher_match(matcher, 0, scan_data, strlen(scan_data), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 113); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(expr_matcher_match, regex_expression_check) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./regex_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - for (size_t i = 0; i < n_rule; i++) { - for (size_t j = 0; j < rules[i].n_patterns; j++) { - expr_matcher_verify_regex_expression(rules[i].patterns[j].pat, g_logger); - } - } - - expr_array_free(rules, n_rule); -} - -TEST(hs_expr_matcher_stream, basic) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - const char* scan_data1 = "A directed path in a directed graph is a finite"; - const char *scan_data2 = " or infinite sequence of edges which joins a sequence of distinct vertices"; - - struct expr_scan_result result[64] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - - struct expr_matcher_stream *stream = expr_matcher_stream_open(matcher, thread_id); - EXPECT_TRUE(stream != NULL); - - ret = expr_matcher_stream_match(stream, scan_data1, strlen(scan_data1), result, 64, &n_hit_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_hit_result, 0); - - ret = expr_matcher_stream_match(stream, scan_data2, strlen(scan_data2), result, 64, &n_hit_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(result[0].rule_id, 113); - - expr_matcher_stream_close(stream); - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher_stream, basic) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - const char* scan_data1 = "A directed path in a directed graph is a finite"; - const char *scan_data2 = " or infinite sequence of edges which joins a sequence of distinct vertices"; - - struct expr_scan_result result[64] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - - struct expr_matcher_stream *stream = expr_matcher_stream_open(matcher, thread_id); - EXPECT_TRUE(stream != NULL); - - ret = expr_matcher_stream_match(stream, scan_data1, strlen(scan_data1), result, 64, &n_hit_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_hit_result, 0); - - ret = expr_matcher_stream_match(stream, scan_data2, strlen(scan_data2), result, 64, &n_hit_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(result[0].rule_id, 113); - - expr_matcher_stream_close(stream); - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(hs_expr_matcher, regex_basic) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - ret = expr_matcher_verify_regex_expression("[0-9]rain", g_logger); - EXPECT_EQ(ret, 1); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_HS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567"; - //const char *scan_data2 = "8rain"; - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 114); - - expr_matcher_free(matcher); - matcher = NULL; -} - -TEST(rs_expr_matcher, regex_basic) -{ - struct expr_rule rules[64] = {0}; - size_t n_rule = 0; - - int ret = parse_config_file("./literal_expr.conf", rules, &n_rule); - EXPECT_EQ(ret, 0); - - ret = expr_matcher_verify_regex_expression("[0-9]rain", g_logger); - EXPECT_EQ(ret, 1); - - struct expr_matcher *matcher = expr_matcher_new(rules, n_rule, EXPR_ENGINE_TYPE_RS, 1, g_logger); - EXPECT_TRUE(matcher != NULL); - expr_array_free(rules, n_rule); - - const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567"; - //const char *scan_data2 = "8rain"; - - struct expr_scan_result result[64] = {0}; - size_t n_result = 0; - - ret = expr_matcher_match(matcher, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].rule_id, 114); - - expr_matcher_free(matcher); - matcher = NULL; -} - -int main(int argc, char **argv) -{ - int ret = 0; - ::testing::InitGoogleTest(&argc, argv); - g_logger = log_handle_create("./adapter_hs_gtest.log", 0); - - ret = RUN_ALL_TESTS(); - - log_handle_destroy(g_logger); - - return ret; -} \ No newline at end of file diff --git a/test/json_update/corrupted.json b/test/json_update/corrupted.json index fb491db..5508ec0 100644 --- a/test/json_update/corrupted.json +++ b/test/json_update/corrupted.json @@ -1,5 +1,5 @@ { - "compile_table": "COMPILE_DEFAULT", + "compile_table": "COMPILE", "group_table": "GROUP", "rules": [ { diff --git a/test/json_update/new.json b/test/json_update/new.json index 9c9f9ac..865a873 100644 --- a/test/json_update/new.json +++ b/test/json_update/new.json @@ -1,6 +1,6 @@ { - "compile_table": "COMPILE_DEFAULT", - "group2compile_table": "GROUP2COMPILE_DEFAULT", + "compile_table": "COMPILE", + "group2compile_table": "GROUP2COMPILE", "group2group_table": "GROUP2GROUP", "rules": [ { diff --git a/test/json_update/old.json b/test/json_update/old.json index af32b8e..3ca45e2 100644 --- a/test/json_update/old.json +++ b/test/json_update/old.json @@ -1,6 +1,6 @@ { - "compile_table": "COMPILE_DEFAULT", - "group2compile_table": "GROUP2COMPILE_DEFAULT", + "compile_table": "COMPILE", + "group2compile_table": "GROUP2COMPILE", "group2group_table": "GROUP2GROUP", "rules": [ { diff --git a/test/literal_expr.conf b/test/literal_expr.conf index 7bb2db6..72cec1c 100644 --- a/test/literal_expr.conf +++ b/test/literal_expr.conf @@ -5,7 +5,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", @@ -19,7 +18,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", @@ -33,7 +31,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", @@ -47,7 +44,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", @@ -60,7 +56,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "exactly", "case_sensitive": "yes", "is_hexbin": "no", @@ -73,7 +68,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "prefix", "case_sensitive": "yes", "is_hexbin": "no", @@ -86,7 +80,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "suffix", "case_sensitive": "yes", "is_hexbin": "no", @@ -99,7 +92,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "yes", @@ -113,14 +105,12 @@ "pattern_num": 2, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", "pattern": "multi" }, { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", @@ -133,7 +123,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", @@ -146,7 +135,6 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", @@ -159,7 +147,6 @@ "pattern_num": 2, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", @@ -167,7 +154,6 @@ "offset": "3~7" }, { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", @@ -181,26 +167,12 @@ "pattern_num": 1, "patterns": [ { - "pattern_type": "literal", "match_method": "sub", "case_sensitive": "yes", "is_hexbin": "no", "pattern": "a finite or infinite" } ] - }, - { - "expr_id": 114, - "pattern_num": 1, - "patterns": [ - { - "pattern_type": "regex", - "match_method": "sub", - "case_sensitive": "yes", - "is_hexbin": "no", - "pattern": "query=(.*)" - } - ] } ] -} +} \ No newline at end of file diff --git a/test/maat_ex_data_gtest.cpp b/test/maat_ex_data_gtest.cpp index 84e2848..07d4223 100644 --- a/test/maat_ex_data_gtest.cpp +++ b/test/maat_ex_data_gtest.cpp @@ -163,4 +163,4 @@ int main(int argc, char ** argv) maat_free(g_maat_inst); g_maat_inst = NULL; return ret; -} +} \ No newline at end of file diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 1649f02..36f4f32 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -287,12 +287,12 @@ int test_add_expr_command(struct maat *maat_inst, const char *expr_table, memset(huge_serv_def, 's', sizeof(huge_serv_def) - 1); huge_serv_def[sizeof(huge_serv_def) - 1] = '\0'; - int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, + int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, huge_serv_def, 1, timeout); EXPECT_EQ(ret, 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id, compile_id, 0, "null", 1, timeout); EXPECT_EQ(ret, 1); @@ -306,7 +306,7 @@ int test_add_expr_command(struct maat *maat_inst, const char *expr_table, int del_command(struct maat *maat_inst, int compile_id) { - return compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_DEL, compile_id, "null", 1, 0); + return compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_DEL, compile_id, "null", 1, 0); } static void random_keyword_generate(char *keyword_buf, size_t sz) @@ -461,11 +461,6 @@ void scan_with_old_or_new_cfg(struct maat *maat_inst, int is_old) if (is_old) { EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_TRUE(results[0] == 1); - - int table_id = -1; - int table_cnt = maat_state_get_compile_table_ids(state, results, 1, &table_id); - EXPECT_EQ(table_cnt, 1); - EXPECT_EQ(table_id, 0); } else { EXPECT_EQ(ret, MAAT_SCAN_OK); } @@ -749,8 +744,7 @@ TEST_F(MaatFlagScan, FlagPlus) { state = NULL; } -//hyperscan engine -class MaatHsStringScan : public testing::Test +class MaatStringScan : public testing::Test { protected: static void SetUpTestCase() { @@ -772,7 +766,6 @@ protected: maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_accept_tags(opts, accept_tags); - //maat_options_set_expr_engine(opts, MAAT_EXPR_ENGINE_HS); //default _shared_maat_inst = maat_new(opts, table_info_path); maat_options_free(opts); @@ -792,12 +785,12 @@ protected: static struct maat *_shared_maat_inst; }; -struct maat *MaatHsStringScan::_shared_maat_inst; -struct log_handle *MaatHsStringScan::logger; +struct maat *MaatStringScan::_shared_maat_inst; +struct log_handle *MaatStringScan::logger; -TEST_F(MaatHsStringScan, ScanDataOnlyOneByte) { +TEST_F(MaatStringScan, ScanDataOnlyOneByte) { const char *table_name = "HTTP_URL"; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; int table_id = maat_get_table_id(maat_inst, table_name); ASSERT_GT(table_id, 0); @@ -816,9 +809,9 @@ TEST_F(MaatHsStringScan, ScanDataOnlyOneByte) { state = NULL; } -TEST_F(MaatHsStringScan, Full) { +TEST_F(MaatStringScan, Full) { const char *table_name = "HTTP_URL"; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; int table_id = maat_get_table_id(maat_inst, table_name); ASSERT_GT(table_id, 0); @@ -838,14 +831,14 @@ TEST_F(MaatHsStringScan, Full) { state = NULL; } -TEST_F(MaatHsStringScan, Regex) { +TEST_F(MaatStringScan, Regex) { int ret = 0; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; const char *cookie = "Cookie: Txa123aheadBCAxd"; const char *table_name = "HTTP_URL"; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); int table_id = maat_get_table_id(maat_inst, table_name); @@ -857,7 +850,7 @@ TEST_F(MaatHsStringScan, Regex) { state = NULL; } -TEST_F(MaatHsStringScan, ExprPlus) { +TEST_F(MaatStringScan, ExprPlus) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; @@ -866,7 +859,7 @@ TEST_F(MaatHsStringScan, ExprPlus) { const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567"; const char *scan_data2 = "Addis Sapphire Hotel"; const char *table_name = "HTTP_SIGNATURE"; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); int table_id = maat_get_table_id(maat_inst, table_name); @@ -892,12 +885,12 @@ TEST_F(MaatHsStringScan, ExprPlus) { state = NULL; } -TEST_F(MaatHsStringScan, ExprPlusWithOffset) +TEST_F(MaatStringScan, ExprPlusWithOffset) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); const char *region_name = "Payload"; unsigned char udp_payload_not_hit[] = { /* Stun packet */ @@ -946,11 +939,11 @@ TEST_F(MaatHsStringScan, ExprPlusWithOffset) state = NULL; } -TEST_F(MaatHsStringScan, ExprPlusWithHex) { +TEST_F(MaatStringScan, ExprPlusWithHex) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); const char *scan_data1 = "text/html; charset=UTF-8"; const char *scan_data2 = "Batman\\:Take me Home.Superman/:Fine,stay with me."; @@ -982,11 +975,11 @@ TEST_F(MaatHsStringScan, ExprPlusWithHex) { state = NULL; } -TEST_F(MaatHsStringScan, ExprAndExprPlus) { +TEST_F(MaatStringScan, ExprAndExprPlus) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); const char *expr_table_name = "HTTP_URL"; const char *expr_plus_table_name = "HTTP_SIGNATURE"; @@ -1014,11 +1007,11 @@ TEST_F(MaatHsStringScan, ExprAndExprPlus) { state = NULL; } -TEST_F(MaatHsStringScan, ShouldNotHitExprPlus) { +TEST_F(MaatStringScan, ShouldNotHitExprPlus) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); const char *region_name = "tcp.payload"; unsigned char udp_payload_not_hit[] = { /* Stun packet */ @@ -1048,10 +1041,10 @@ TEST_F(MaatHsStringScan, ShouldNotHitExprPlus) { state = NULL; } -TEST_F(MaatHsStringScan, Expr8) { +TEST_F(MaatStringScan, Expr8) { const char *table_name = "KEYWORDS_TABLE"; int thread_id = 0; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); int table_id = maat_get_table_id(maat_inst, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; @@ -1072,11 +1065,11 @@ TEST_F(MaatHsStringScan, Expr8) { state = NULL; } -TEST_F(MaatHsStringScan, HexBinCaseSensitive) { +TEST_F(MaatStringScan, HexBinCaseSensitive) { const char *table_name = "KEYWORDS_TABLE"; const char *scan_data1 = "String TeST should not hit."; const char *scan_data2 = "String TEST should hit"; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; int thread_id = 0; int table_id = maat_get_table_id(maat_inst, table_name); @@ -1099,7 +1092,7 @@ TEST_F(MaatHsStringScan, HexBinCaseSensitive) { maat_state_free(state); } -TEST_F(MaatHsStringScan, BugReport20190325) { +TEST_F(MaatStringScan, BugReport20190325) { unsigned char scan_data[] = {/* Packet 1 */ 0x01, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, 0x32, 0x00, 0x00, @@ -1119,7 +1112,7 @@ TEST_F(MaatHsStringScan, BugReport20190325) { 0x00, 0x31, 0x3a, 0x47, 0x32, 0x2e, 0x34, 0x30, 0x00}; const char *table_name = "TROJAN_PAYLOAD"; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; int thread_id = 0; int table_id = maat_get_table_id(maat_inst, table_name); @@ -1137,13 +1130,13 @@ TEST_F(MaatHsStringScan, BugReport20190325) { state = NULL; } -TEST_F(MaatHsStringScan, PrefixAndSuffix) { +TEST_F(MaatStringScan, PrefixAndSuffix) { const char *hit_twice = "ceshi3@mailhost.cn"; const char *hit_suffix = "11111111111ceshi3@mailhost.cn"; const char *hit_prefix = "ceshi3@mailhost.cn11111111111"; const char *cont_sz_table_name = "CONTENT_SIZE"; const char *mail_addr_table_name = "MAIL_ADDR"; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; int thread_id = 0; int cont_sz_table_id = maat_get_table_id(maat_inst, cont_sz_table_name); @@ -1183,10 +1176,10 @@ TEST_F(MaatHsStringScan, PrefixAndSuffix) { state = NULL; } -TEST_F(MaatHsStringScan, MaatUnescape) { +TEST_F(MaatStringScan, MaatUnescape) { const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me."; const char *table_name = "KEYWORDS_TABLE"; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; int thread_id = 0; int table_id = maat_get_table_id(maat_inst, table_name); @@ -1204,13 +1197,13 @@ TEST_F(MaatHsStringScan, MaatUnescape) { state = NULL; } -TEST_F(MaatHsStringScan, OffsetChunk64) { +TEST_F(MaatStringScan, OffsetChunk64) { const char *table_name = "IMAGE_FP"; const char *file_name = "./testdata/mesa_logo.jpg"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); FILE *fp = fopen(file_name, "r"); @@ -1243,13 +1236,13 @@ TEST_F(MaatHsStringScan, OffsetChunk64) { state = NULL; } -TEST_F(MaatHsStringScan, OffsetChunk1460) { +TEST_F(MaatStringScan, OffsetChunk1460) { const char *table_name = "IMAGE_FP"; const char *file_name = "./testdata/mesa_logo.jpg"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); FILE *fp = fopen(file_name, "r"); @@ -1282,14 +1275,14 @@ TEST_F(MaatHsStringScan, OffsetChunk1460) { state = NULL; } -TEST_F(MaatHsStringScan, StreamScanUTF8) { +TEST_F(MaatStringScan, StreamScanUTF8) { const char *table_name = "TROJAN_PAYLOAD"; const char* file_name = "./testdata/jd.com.html"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; char scan_data[2048]; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); FILE *fp = fopen(file_name, "r"); @@ -1320,11 +1313,11 @@ TEST_F(MaatHsStringScan, StreamScanUTF8) { state = NULL; } -TEST_F(MaatHsStringScan, StreamInput) { +TEST_F(MaatStringScan, StreamInput) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; const char *table_name = "HTTP_URL"; @@ -1349,13 +1342,13 @@ TEST_F(MaatHsStringScan, StreamInput) { state = NULL; } -TEST_F(MaatHsStringScan, dynamic_config) { +TEST_F(MaatStringScan, dynamic_config) { const char *table_name = "HTTP_URL"; char data[128] = "hello world, welcome to maat version4, it's funny."; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - struct maat *maat_inst = MaatHsStringScan::_shared_maat_inst; + struct maat *maat_inst = MaatStringScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); int table_id = maat_get_table_id(maat_inst, table_name); @@ -1365,8 +1358,8 @@ TEST_F(MaatHsStringScan, dynamic_config) { EXPECT_EQ(n_hit_result, 0); maat_state_reset(state); - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE"; /* compile table add line */ long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); @@ -1386,7 +1379,7 @@ TEST_F(MaatHsStringScan, dynamic_config) { keywords, NULL, 1, 0, 0, 0); /* EXPR_TYPE_AND MATCH_METHOD_SUB */ EXPECT_EQ(ret, 1); - sleep(WAIT_FOR_EFFECTIVE_S * 3); + sleep(WAIT_FOR_EFFECTIVE_S); ret = maat_scan_string(maat_inst, table_id, data, strlen(data), results, ARRAY_SIZE, &n_hit_result, state); @@ -1419,677 +1412,7 @@ TEST_F(MaatHsStringScan, dynamic_config) { state = NULL; } -class MaatRsStringScan : public testing::Test -{ -protected: - static void SetUpTestCase() { - const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," - "{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; - char redis_ip[64] = "127.0.0.1"; - int redis_port = 6379; - int redis_db = 0; - - logger = log_handle_create("./maat_framework_gtest.log", 0); - int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger); - if (ret < 0) { - log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); - } - - struct maat_options *opts = maat_options_new(); - maat_options_set_redis(opts, redis_ip, redis_port, redis_db); - maat_options_set_stat_file(opts, "./stat.log"); - maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); - maat_options_set_accept_tags(opts, accept_tags); - maat_options_set_expr_engine(opts, MAAT_EXPR_ENGINE_RS); - - _shared_maat_inst = maat_new(opts, table_info_path); - maat_options_free(opts); - if (NULL == _shared_maat_inst) { - log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatStringScan failed.", - __FUNCTION__, __LINE__); - } - } - - static void TearDownTestCase() { - maat_free(_shared_maat_inst); - log_handle_destroy(logger); - } - - static struct log_handle *logger; - static struct maat *_shared_maat_inst; -}; - -struct maat *MaatRsStringScan::_shared_maat_inst; -struct log_handle *MaatRsStringScan::logger; - -TEST_F(MaatRsStringScan, ScanDataOnlyOneByte) { - const char *table_name = "HTTP_URL"; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - const char scan_data = 0x20; - - int ret = maat_scan_string(maat_inst, table_id, &scan_data, sizeof(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - EXPECT_EQ(n_hit_result, 0); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, Full) { - const char *table_name = "HTTP_URL"; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567"; - - int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 125); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, Regex) { - int ret = 0; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - const char *cookie = "Cookie: Txa123aheadBCAxd"; - const char *table_name = "HTTP_URL"; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - int table_id = maat_get_table_id(maat_inst, table_name); - ret = maat_scan_string(maat_inst, table_id, cookie, strlen(cookie), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], 146); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, ExprPlus) { - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - const char *region_name1 ="HTTP URL"; - const char *region_name2 ="我的diStricT"; - const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567"; - const char *scan_data2 = "Addis Sapphire Hotel"; - const char *table_name = "HTTP_SIGNATURE"; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - int table_id = maat_get_table_id(maat_inst, table_name); - int ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_ERR);//Should return error for district not setting. - - ret = maat_state_set_scan_district(state, table_id, region_name1, strlen(region_name1)); - ASSERT_EQ(ret, 0); - ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], 128); - maat_state_reset(state); - - ret = maat_state_set_scan_district(state, table_id, region_name2, strlen(region_name2)); - ASSERT_EQ(ret, 0); - ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], 190); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, ExprPlusWithOffset) -{ - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - const char *region_name = "Payload"; - unsigned char udp_payload_not_hit[] = { /* Stun packet */ - 0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42, - 0x4f, 0xc2, 0xc2, 0x70, 0xb3, 0xa8, 0x4e, 0x22, - 0xf5, 0x22, 0x87, 0x4c, 0x40, 0x00, 0x00, 0x46, - 0x03, 0x02, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01, - 0x3a, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a, - 0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9, - 0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b, - 0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2, - 0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f, - 0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64, - 0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a, - 0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a }; - unsigned char udp_payload_hit[] = { /* Stun packet */ //rule:"1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d" - 0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42, //1-1:03 - 0x4f, 0xc2, 0x2d, 0x70, 0xb3, 0xa8, 0x4e, 0x2d, //10-10:2d - 0x34, 0x22, 0x87, 0x4c, 0x2d, 0x00, 0x00, 0x46, //15-16:2d34 - 0x2d, 0x34, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01, //20-20:2d - 0x03, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a, //24-24:2d - 0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9, - 0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b, - 0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2, - 0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f, - 0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64, - 0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a, - 0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a }; - - int table_id = maat_get_table_id(maat_inst, "APP_PAYLOAD"); - ASSERT_GT(table_id, 0); - - int ret = maat_state_set_scan_district(state, table_id, region_name, strlen(region_name)); - EXPECT_EQ(ret, 0); - - ret = maat_scan_string(maat_inst, table_id, (char*)udp_payload_not_hit, sizeof(udp_payload_not_hit), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - - ret = maat_scan_string(maat_inst, table_id, (char*)udp_payload_hit, sizeof(udp_payload_hit), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], 148); - - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, ExprPlusWithHex) { - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - const char *scan_data1 = "text/html; charset=UTF-8"; - const char *scan_data2 = "Batman\\:Take me Home.Superman/:Fine,stay with me."; - const char *region_name1 = "Content-Type"; - const char *region_name2 = "User-Agent"; - - int table_id = maat_get_table_id(maat_inst, "HTTP_SIGNATURE"); - ASSERT_GT(table_id, 0); - - int ret = maat_state_set_scan_district(state, table_id, region_name1, strlen(region_name1)); - ASSERT_EQ(ret, 0); - ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], 156); - - ret = maat_state_set_scan_district(state, table_id, region_name2, strlen(region_name2)); - ASSERT_EQ(ret, 0); - ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); //maat-v3 consider as half hit, it's unreasonable - - table_id = maat_get_table_id(maat_inst, "KEYWORDS_TABLE"); - ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], 132); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, ExprAndExprPlus) { - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - const char *expr_table_name = "HTTP_URL"; - const char *expr_plus_table_name = "HTTP_SIGNATURE"; - const char *region_name = "I love China"; - const char *scan_data = "today is Monday and yesterday is Tuesday"; - - int expr_table_id = maat_get_table_id(maat_inst, expr_table_name); - int expr_plus_table_id = maat_get_table_id(maat_inst, expr_plus_table_name); - - int ret = maat_scan_string(maat_inst, expr_plus_table_id, scan_data, strlen(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_ERR); - - ret = maat_state_set_scan_district(state, expr_plus_table_id, region_name, strlen(region_name)); - ASSERT_EQ(ret, 0); - ret = maat_scan_string(maat_inst, expr_plus_table_id, scan_data, strlen(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - ret = maat_scan_string(maat_inst, expr_table_id, scan_data, strlen(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], 195); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, ShouldNotHitExprPlus) { - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - const char *region_name = "tcp.payload"; - unsigned char udp_payload_not_hit[] = { /* Stun packet */ - 0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42, - 0x4f, 0xc2, 0xc2, 0x70, 0xb3, 0xa8, 0x4e, 0x22, - 0xf5, 0x22, 0x87, 0x4c, 0x40, 0x00, 0x00, 0x46, - 0x03, 0x02, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01, - 0x3a, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a, - 0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9, - 0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b, - 0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2, - 0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f, - 0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64, - 0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a, - 0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a, 0xab, 0x00 }; - - int table_id = maat_get_table_id(maat_inst, "APP_PAYLOAD"); - ASSERT_GT(table_id, 0); - - int ret = maat_state_set_scan_district(state, table_id, region_name, strlen(region_name)); - ASSERT_EQ(ret, 0); - - ret = maat_scan_string(maat_inst, table_id, (char *)udp_payload_not_hit, sizeof(udp_payload_not_hit), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); //maat-v3 consider as half hit, it's unreasonable - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, Expr8) { - const char *table_name = "KEYWORDS_TABLE"; - int thread_id = 0; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - int table_id = maat_get_table_id(maat_inst, table_name); - char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - - int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 182); - - struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; - int n_read = 0; - n_read = maat_state_get_hit_paths(state, hit_path, HIT_PATH_SIZE); - EXPECT_NE(n_read, 0); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, HexBinCaseSensitive) { - const char *table_name = "KEYWORDS_TABLE"; - const char *scan_data1 = "String TeST should not hit."; - const char *scan_data2 = "String TEST should hit"; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - int thread_id = 0; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - int ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - maat_state_reset(state); - - ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 2); - EXPECT_EQ(results[0], 206); - EXPECT_EQ(results[1], 191); - maat_state_free(state); -} - -TEST_F(MaatRsStringScan, BugReport20190325) { - unsigned char scan_data[] = {/* Packet 1 */ - 0x01, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, - 0x00, 0xf4, 0x01, 0x00, 0x00, 0x32, 0x00, 0x00, - 0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2d, 0x3d, 0x3d, 0x20, 0x48, 0x3d, 0x48, 0x20, - 0x3d, 0x3d, 0x2d, 0x3a, 0x00, 0x02, 0x00, 0x00, - 0x00, 0x07, 0x0e, 0x00, 0x00, 0xe8, 0x03, 0x00, - 0x00, 0x4c, 0x69, 0x6e, 0x75, 0x78, 0x20, 0x33, - 0x2e, 0x31, 0x39, 0x2e, 0x30, 0x2d, 0x31, 0x35, - 0x2d, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, - 0x00, 0x31, 0x3a, 0x47, 0x32, 0x2e, 0x34, 0x30, - 0x00}; - const char *table_name = "TROJAN_PAYLOAD"; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - int thread_id = 0; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - int ret = maat_scan_string(maat_inst, table_id, (char *)scan_data, sizeof(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 150); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, PrefixAndSuffix) { - const char *hit_twice = "ceshi3@mailhost.cn"; - const char *hit_suffix = "11111111111ceshi3@mailhost.cn"; - const char *hit_prefix = "ceshi3@mailhost.cn11111111111"; - const char *cont_sz_table_name = "CONTENT_SIZE"; - const char *mail_addr_table_name = "MAIL_ADDR"; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - int thread_id = 0; - - int cont_sz_table_id = maat_get_table_id(maat_inst, cont_sz_table_name); - ASSERT_GT(cont_sz_table_id, 0); - - int mail_addr_table_id = maat_get_table_id(maat_inst, mail_addr_table_name); - ASSERT_GT(mail_addr_table_id, 0); - - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - int ret = maat_scan_integer(maat_inst, cont_sz_table_id, 2015, results, - ARRAY_SIZE, &n_hit_result, state); - - ret = maat_scan_string(maat_inst, mail_addr_table_id, hit_twice, strlen(hit_twice), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 2); - EXPECT_EQ(results[0], 151); - EXPECT_EQ(results[1], 152); - maat_state_reset(state); - - ret = maat_scan_string(maat_inst, mail_addr_table_id, hit_suffix, strlen(hit_suffix), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 151); - - ret = maat_scan_integer(maat_inst, cont_sz_table_id, 2015, results, - ARRAY_SIZE, &n_hit_result, state); - ret = maat_scan_string(maat_inst, mail_addr_table_id, hit_prefix, strlen(hit_prefix), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 152); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, MaatUnescape) { - const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me."; - const char *table_name = "KEYWORDS_TABLE"; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - int thread_id = 0; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 132); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, OffsetChunk64) { - const char *table_name = "IMAGE_FP"; - const char *file_name = "./testdata/mesa_logo.jpg"; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - FILE *fp = fopen(file_name, "r"); - ASSERT_FALSE(fp==NULL); - - char scan_data[64]; - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state); - ASSERT_TRUE(sp != NULL); - - int ret = 0; - int read_size = 0; - int pass_flag = 0; - while (0 == feof(fp)) { - read_size = fread(scan_data, 1, sizeof(scan_data), fp); - ret = maat_stream_scan(sp, scan_data, read_size, - results, ARRAY_SIZE, &n_hit_result, state); - if (ret > 0) { - pass_flag = 1; - break; - } - } - EXPECT_EQ(pass_flag, 1); - EXPECT_EQ(results[0], 136); - maat_stream_free(sp); - fclose(fp); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, OffsetChunk1460) { - const char *table_name = "IMAGE_FP"; - const char *file_name = "./testdata/mesa_logo.jpg"; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - FILE *fp = fopen(file_name, "r"); - ASSERT_FALSE(fp==NULL); - - char scan_data[1460]; - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state); - ASSERT_TRUE(sp != NULL); - - int ret = 0; - int read_size = 0; - int pass_flag = 0; - while (0 == feof(fp)) { - read_size = fread(scan_data, 1, sizeof(scan_data), fp); - ret = maat_stream_scan(sp, scan_data, read_size, - results, ARRAY_SIZE, &n_hit_result, state); - if (ret > 0) { - pass_flag = 1; - break; - } - } - EXPECT_EQ(pass_flag, 1); - EXPECT_EQ(results[0], 136); - maat_stream_free(sp); - fclose(fp); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, StreamScanUTF8) { - const char *table_name = "TROJAN_PAYLOAD"; - const char* file_name = "./testdata/jd.com.html"; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - char scan_data[1500]; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - FILE *fp = fopen(file_name, "r"); - ASSERT_FALSE(fp == NULL); - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state); - ASSERT_FALSE(sp == NULL); - - int pass_flag = 0; - while (0 == feof(fp)) { - size_t read_size = fread(scan_data, 1, sizeof(scan_data), fp); - //read_size can't exceed 1500 - int ret = maat_stream_scan(sp, scan_data, read_size, results, ARRAY_SIZE, - &n_hit_result, state); - if (ret == MAAT_SCAN_HIT) { - pass_flag = 1; - break; - } - } - - EXPECT_EQ(pass_flag, 1); - EXPECT_EQ(results[0], 157); - maat_stream_free(sp); - fclose(fp); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, StreamInput) { - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; - const char *table_name = "HTTP_URL"; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state); - ASSERT_TRUE(sp != NULL); - - int ret = maat_stream_scan(sp, "www.cyberessays.com", strlen("www.cyberessays.com"), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - - ret = maat_stream_scan(sp, scan_data, strlen(scan_data), results, ARRAY_SIZE, - &n_hit_result, state); - maat_stream_free(sp); - - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], 125); - maat_state_free(state); - state = NULL; -} - -TEST_F(MaatRsStringScan, dynamic_config) { - const char *table_name = "HTTP_URL"; - char data[128] = "hello world, welcome to maat version4, it's funny."; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat *maat_inst = MaatRsStringScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - int table_id = maat_get_table_id(maat_inst, table_name); - int ret = maat_scan_string(maat_inst, table_id, data, strlen(data), results, - ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - EXPECT_EQ(n_hit_result, 0); - maat_state_reset(state); - - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; - - /* compile table add line */ - long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_ADD, compile_id, "null", 1, 0); - EXPECT_EQ(ret, 1); - - /* group2compile table add line */ - long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_ADD, group_id, - compile_id, 0, "null", 1, 0); - EXPECT_EQ(ret, 1); - - /* expr table add line */ - long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); - const char *keywords = "welcome to maat"; - ret = expr_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id, - keywords, NULL, 1, 0, 0, 0); /* EXPR_TYPE_AND MATCH_METHOD_SUB */ - EXPECT_EQ(ret, 1); - - sleep(WAIT_FOR_EFFECTIVE_S * 3); - - ret = maat_scan_string(maat_inst, table_id, data, strlen(data), results, - ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], compile_id); - maat_state_reset(state); - - /* expr table del line */ - ret = expr_table_set_line(maat_inst, table_name, MAAT_OP_DEL, item_id, group_id, - keywords, NULL, 1, 0, 0, 0); /* EXPR_TYPE_AND MATCH_METHOD_SUB */ - EXPECT_EQ(ret, 1); - - /* group2compile table del line */ - ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_DEL, group_id, - compile_id, 0, "null", 1, 0); - EXPECT_EQ(ret, 1); - - /* compile table del line */ - ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_DEL, compile_id, "null", 1, 0); - EXPECT_EQ(ret, 1); - - sleep(WAIT_FOR_EFFECTIVE_S); - - ret = maat_scan_string(maat_inst, table_id, data, strlen(data), results, - ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - EXPECT_EQ(n_hit_result, 0); - maat_state_free(state); - state = NULL; -} - -class MaatHsStreamScan : public testing::Test +class MaatStreamScan : public testing::Test { protected: static void SetUpTestCase() { @@ -2122,9 +1445,9 @@ protected: static struct maat *_shared_maat_inst; }; -struct maat *MaatHsStreamScan::_shared_maat_inst; +struct maat *MaatStreamScan::_shared_maat_inst; -TEST_F(MaatHsStreamScan, dynamic_config) { +TEST_F(MaatStreamScan, dynamic_config) { const char *scan_data1 = "hello world cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; const char *table_name = "HTTP_URL"; const char *keywords1 = "hello"; @@ -2132,103 +1455,7 @@ TEST_F(MaatHsStreamScan, dynamic_config) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - struct maat *maat_inst = MaatHsStreamScan::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - // STEP 1: add keywords1 and wait scan stream to hit - long long compile1_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = test_add_expr_command(maat_inst, table_name, compile1_id, 0, keywords1); - EXPECT_EQ(ret, 1); - - sleep(WAIT_FOR_EFFECTIVE_S); - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state); - ASSERT_TRUE(sp != NULL); - - ret = maat_stream_scan(sp, "www.cyberessays.com", strlen("www.cyberessays.com"), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - - ret = maat_stream_scan(sp, scan_data1, strlen(scan_data1), results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], compile1_id); - maat_state_reset(state); - - // STEP 2: Inc config update, use same stream to scan and wait old expr_runtime invalid - random_keyword_generate(keyword_buf, sizeof(keyword_buf)); - long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - ret = test_add_expr_command(maat_inst, table_name, compile_id, 0, keyword_buf); - EXPECT_EQ(ret, 1); - - // Inc config has not yet taken effect, stream scan can hit compile - ret = maat_stream_scan(sp, scan_data1, strlen(scan_data1), results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(results[0], compile1_id); - maat_state_reset(state); - - sleep(WAIT_FOR_EFFECTIVE_S); - - // Inc config has taken effect, stream reference old expr_runtime, should not hit compile - ret = maat_stream_scan(sp, scan_data1, strlen(scan_data1), results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - - maat_stream_free(sp); - maat_state_free(state); - sp = NULL; - state = NULL; -} - -class MaatRsStreamScan : public testing::Test -{ -protected: - static void SetUpTestCase() { - char redis_ip[64] = "127.0.0.1"; - int redis_port = 6379; - int redis_db = 0; - - struct maat_options *opts = maat_options_new(); - maat_options_set_redis(opts, redis_ip, redis_port, redis_db); - maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); - - _shared_maat_inst = maat_new(opts, table_info_path); - assert(_shared_maat_inst != NULL); - - maat_cmd_flushDB(_shared_maat_inst); - maat_free(_shared_maat_inst); - - maat_options_set_foreign_cont_dir(opts, "./foreign_files/"); - maat_options_set_rule_effect_interval_ms(opts, 0); - maat_options_set_gc_timeout_ms(opts, 0); // start GC immediately - maat_options_set_stat_file(opts, "./stat.log"); - maat_options_set_expr_engine(opts, MAAT_EXPR_ENGINE_RS); - _shared_maat_inst = maat_new(opts, table_info_path); - maat_options_free(opts); - } - - static void TearDownTestCase() { - maat_free(_shared_maat_inst); - } - - static struct maat *_shared_maat_inst; -}; - -struct maat *MaatRsStreamScan::_shared_maat_inst; - -TEST_F(MaatRsStreamScan, dynamic_config) { - const char *scan_data1 = "hello world cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; - const char *table_name = "HTTP_URL"; - const char *keywords1 = "hello"; - char keyword_buf[128]; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat *maat_inst = MaatRsStreamScan::_shared_maat_inst; + struct maat *maat_inst = MaatStreamScan::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); // STEP 1: add keywords1 and wait scan stream to hit @@ -2598,8 +1825,8 @@ TEST_F(MaatIPScan, dynamic_config) { EXPECT_EQ(n_hit_result, 0); maat_state_reset(state); - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE"; /* compile table add line */ long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); @@ -4148,7 +3375,7 @@ void compile_ex_param_dup(int table_id, void **to, void **from, long argl, void TEST_F(CompileTable, CompileRuleUpdate) { struct maat *maat_inst = CompileTable::_shared_maat_inst; - const char *compile_table_name = "COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE"; long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); int ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_ADD, compile_id, "null", 1, 0); @@ -4180,12 +3407,6 @@ TEST_F(CompileTable, Conjunction1) { EXPECT_EQ(results[0], 197); EXPECT_EQ(results[1], 141); - int table_ids[2] = {-1, -1}; - ret = maat_state_get_compile_table_ids(state, results, 2, table_ids); - EXPECT_EQ(ret, 2); - EXPECT_EQ(table_ids[0], 0); - EXPECT_EQ(table_ids[1], 1); - struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; int n_read = maat_state_get_hit_paths(state, hit_path, HIT_PATH_SIZE); EXPECT_EQ(n_read, 2); @@ -4214,12 +3435,6 @@ TEST_F(CompileTable, Conjunction2) { EXPECT_EQ(results[0], 197); EXPECT_EQ(results[1], 141); - int table_ids[2] = {-1, -1}; - ret = maat_state_get_compile_table_ids(state, results, 2, table_ids); - EXPECT_EQ(ret, 2); - EXPECT_EQ(table_ids[0], 0); - EXPECT_EQ(table_ids[1], 1); - struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; int n_read = maat_state_get_hit_paths(state, hit_path, HIT_PATH_SIZE); EXPECT_EQ(n_read, 2); @@ -4236,40 +3451,6 @@ TEST_F(CompileTable, Conjunction2) { state = NULL; } -TEST_F(CompileTable, GetHitCompileTableID) { - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - const char *compile_table_name = "COMPILE_FIREWALL_CONJUNCTION"; - const char *scan_data = "This is a firewall engine."; - const char *table_name = "HTTP_URL"; - struct maat *maat_inst = CompileTable::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - int compile_table_id = maat_get_table_id(maat_inst, compile_table_name); - ASSERT_GT(table_id, 0); - - int ret = maat_state_set_scan_compile_table(state, compile_table_id); - EXPECT_EQ(ret, 0); - - ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 198); - - int hit_compile_table_id = -1; - ret = maat_state_get_compile_table_ids(state, results, 1, &hit_compile_table_id); - EXPECT_EQ(ret, 1); - EXPECT_EQ(hit_compile_table_id, 4); - - maat_state_free(state); - state = NULL; -} - class Policy : public testing::Test { protected: @@ -4399,18 +3580,16 @@ TEST_F(Policy, CompileEXData) { int thread_id = 0; const char *url = "firewall should hit"; const char *table_name = "HTTP_URL"; - const char *plugin_table_name = "COMPILE_FIREWALL_PLUGIN"; - const char *compile_table_name = "COMPILE_FIREWALL_CONJUNCTION"; + const char *compile_table_name = "COMPILE_FIREWALL"; const char *expect_name = "I have a name"; struct maat *maat_inst = Policy::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); int table_id = maat_get_table_id(maat_inst, table_name); - int plugin_table_id = maat_get_table_id(maat_inst, plugin_table_name); int compile_table_id = maat_get_table_id(maat_inst, compile_table_name); int ex_data_counter = 0; - int ret = maat_plugin_table_ex_schema_register(maat_inst, plugin_table_name, + int ret = maat_plugin_table_ex_schema_register(maat_inst, compile_table_name, compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, @@ -4427,7 +3606,7 @@ TEST_F(Policy, CompileEXData) { EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 198); - void *ex_data = maat_plugin_table_get_ex_data(maat_inst, plugin_table_id, + void *ex_data = maat_plugin_table_get_ex_data(maat_inst, compile_table_id, (char *)&results[0], sizeof(long long)); ASSERT_TRUE(ex_data!=NULL); struct rule_ex_param *param = (struct rule_ex_param *)ex_data; @@ -4969,8 +4148,8 @@ TEST_F(MaatCmdTest, SetIP) { size_t n_hit_result = 0; int thread_id = 0; const char *ip_table_name = "IP_CONFIG"; - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE"; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); @@ -5023,7 +4202,6 @@ TEST_F(MaatCmdTest, SetExpr) { const char *keywords1 = "Hiredis"; const char *keywords2 = "C Client"; - const char *compile_table_name = "COMPILE_DEFAULT"; char escape_buff1[256], escape_buff2[256]; char keywords[512]; @@ -5053,10 +4231,10 @@ TEST_F(MaatCmdTest, SetExpr) { EXPECT_TRUE(results[0] == compile_id || results[0] == (compile_id - 1)); maat_state_reset(state); - ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_DEL, compile_id-1, + ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_DEL, compile_id-1, "null", 1, 0); EXPECT_EQ(ret, 1); - ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_DEL, compile_id, + ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_DEL, compile_id, "null", 1, 0); EXPECT_EQ(ret, 1); sleep(WAIT_FOR_EFFECTIVE_S); @@ -5081,9 +4259,9 @@ TEST_F(MaatCmdTest, SetExpr8) { const char *scan_data8 = "string1, string2, string3, string4, string5, string6, string7, string8"; const char *scan_data7 = "string1, string2, string3, string4, string5, string6, string7"; - const char *compile_table_name = "COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE"; const char *table_name = "KEYWORDS_TABLE"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE"; const char *keywords8 = "string1&string2&string3&string4&string5&string6&string7&string8"; const char *keywords7 = "string1&string2&string3&string4&string5&string6&string7"; @@ -5150,8 +4328,6 @@ TEST_F(MaatCmdTest, SameFilterRefByOneCompile) { const char *vtable_name = "HTTP_URL_FILTER"; const char *scan_data = "http://filtermenot.com"; const char *keywords = "menot.com"; - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; @@ -5162,17 +4338,17 @@ TEST_F(MaatCmdTest, SameFilterRefByOneCompile) { ASSERT_GT(table_id, 0); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_ADD, compile_id, + int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 2, 0); // compile has two clause EXPECT_EQ(ret, 1); //clause1 & clause2 has same filter => {vtable_id, group_id} long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_ADD, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id, compile_id, 0, vtable_name, 1, 0); EXPECT_EQ(ret, 1); - ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_ADD, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id, compile_id, 0, vtable_name, 2, 0); EXPECT_EQ(ret, 1); @@ -5276,9 +4452,9 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder) { TEST_F(MaatCmdTest, SubGroup) { const char *table_name = "HTTP_URL"; - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE"; const char *g2g_table_name = "GROUP2GROUP"; + const char *compile_table_name = "COMPILE"; const char *scan_data1 = "www.v2ex.com/t/573028#程序员的核心竞争力是什么"; const char *keyword1 = "程序员&核心竞争力"; const char *scan_data2 = "https://ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc" @@ -5432,8 +4608,8 @@ TEST_F(MaatCmdTest, SubGroup) { TEST_F(MaatCmdTest, RefGroup) { const char *table_name = "HTTP_URL"; - const char* compile_table_name = "COMPILE_DEFAULT"; - const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* g2c_table_name = "GROUP2COMPILE"; + const char* compile_table_name = "COMPILE"; const char* scan_data1 = "m.facebook.com/help/2297503110373101?helpref=hc_nav&refid=69"; const char* keyword1 = "something-should-not-hit"; const char* keyword2 = "facebook.com/help/2297503110373101"; @@ -5510,8 +4686,8 @@ TEST_F(MaatCmdTest, RefGroup) { } TEST_F(MaatCmdTest, VirtualTable) { - const char* compile_table_name = "COMPILE_DEFAULT"; - const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* g2c_table_name = "GROUP2COMPILE"; + const char* compile_table_name = "COMPILE"; const char* table_name="HTTP_SIGNATURE"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -5892,12 +5068,11 @@ void plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *arg } TEST_F(MaatCmdTest, CompileEXData) { - const char *plugin_table_name = "COMPILE_FIREWALL_PLUGIN"; - const char *compile_table_name = "COMPILE_FIREWALL_DEFAULT"; + const char *compile_table_name = "COMPILE_FIREWALL"; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; int *ex_data_counter = MaatCmdTest::_ex_data_counter; - int plugin_table_id = maat_get_table_id(maat_inst, plugin_table_name); - EXPECT_GT(plugin_table_id, 0); + int compile_table_id = maat_get_table_id(maat_inst, compile_table_name); + EXPECT_GT(compile_table_id, 0); long long compile1_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); int ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_ADD, @@ -5910,7 +5085,7 @@ TEST_F(MaatCmdTest, CompileEXData) { sleep(WAIT_FOR_EFFECTIVE_S); *ex_data_counter = 0; - ret = maat_plugin_table_ex_schema_register(maat_inst, plugin_table_name, + ret = maat_plugin_table_ex_schema_register(maat_inst, compile_table_name, compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, @@ -5918,13 +5093,13 @@ TEST_F(MaatCmdTest, CompileEXData) { ASSERT_TRUE(ret == 0); EXPECT_EQ(*ex_data_counter, 2); - void *ex_data = maat_plugin_table_get_ex_data(maat_inst, plugin_table_id, + void *ex_data = maat_plugin_table_get_ex_data(maat_inst, compile_table_id, (char *)&compile1_id, sizeof(long long)); ASSERT_TRUE(ex_data != NULL); struct rule_ex_param *param = (struct rule_ex_param *)ex_data; EXPECT_EQ(param->id, 1111); - ex_data = maat_plugin_table_get_ex_data(maat_inst, plugin_table_id, + ex_data = maat_plugin_table_get_ex_data(maat_inst, compile_table_id, (char *)&compile2_id, sizeof(long long)); ASSERT_TRUE(ex_data != NULL); param = (struct rule_ex_param *)ex_data; @@ -5932,10 +5107,10 @@ TEST_F(MaatCmdTest, CompileEXData) { ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_DEL, compile2_id, "test:compile2,2222", 1, 0); - sleep(WAIT_FOR_EFFECTIVE_S); + sleep(WAIT_FOR_EFFECTIVE_S * 5); EXPECT_EQ(param->id, 2222); sleep(2); - //excced gc_timeout_s(3s), the data pointed by param has been freed + //excced gc_timeout_s(11s), the data pointed by param has been freed } TEST_F(MaatCmdTest, PluginEXData) { @@ -6275,8 +5450,8 @@ TEST_F(MaatCmdTest, UpdateBoolPlugin) { #define COMPILE_ID_NUMS 1000 TEST_F(MaatCmdTest, GroupInMassCompiles) { - const char* compile_table_name = "COMPILE_DEFAULT"; - const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* g2c_table_name = "GROUP2COMPILE"; + const char* compile_table_name = "COMPILE"; const char* table_url = "HTTP_URL"; const char* table_appid = "APP_ID"; int thread_id = 0; @@ -6379,339 +5554,10 @@ TEST_F(MaatCmdTest, GroupInMassCompiles) { state = NULL; } -TEST_F(MaatCmdTest, HitGroup) { - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; - const char *g2g_table_name = "GROUP2GROUP"; - const char *http_sig_table_name = "HTTP_SIGNATURE"; - const char *ip_table_name = "IP_CONFIG"; - const char *keywords_table_name = "KEYWORDS_TABLE"; - int thread_id = 0; - struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - /* compile1 */ - long long compile1_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_ADD, compile1_id, - "null", 2, 0); - EXPECT_EQ(ret, 1); - - //group1 -> compile1 - long long group1_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_ADD, group1_id, - compile1_id, 0, "HTTP_REQUEST_HEADER", 1, 0); - EXPECT_EQ(ret, 1); - - //item1 -> group1 -> compile1 - long long item1_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); - ret = expr_table_set_line(maat_inst, http_sig_table_name, MAAT_OP_ADD, item1_id, group1_id, - "hit group item first", "URL", 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/ - EXPECT_EQ(ret, 1); - - /* item1 -> group1 -> compile1 - / - group21_/ - */ - long long group21_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_ADD, group21_id, - compile1_id, 0, "HTTP_RESPONSE_HEADER", 2, 0); - EXPECT_EQ(ret, 1); - - /* item1 -> group1 -> compile1 - / - group2 -> group21 _/ - */ - long long group2_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2group_table_set_line(maat_inst, g2g_table_name, MAAT_OP_ADD, group2_id, - group21_id, 0, 0); - EXPECT_EQ(ret, 1); - - /* item1 -> group1 -> compile1 - / - item2 -> group2 -> group21 _/ - */ - long long item2_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); - ret = expr_table_set_line(maat_inst, http_sig_table_name, MAAT_OP_ADD, item2_id, group2_id, - "hit group item second", "Cookie", 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/ - EXPECT_EQ(ret, 1); - - /* - item1 -> group1 -> group11 - \ - \ -> compile1 - / - item2 -> group2 -> group21 _/ - */ - long long group11_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2group_table_set_line(maat_inst, g2g_table_name, MAAT_OP_ADD, group1_id, - group11_id, 0, 0); - EXPECT_EQ(ret, 1); - - //item3 -> group3, group3 is not referenced by any compile. - long long item3_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); - long long group3_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = ip_table_set_line(maat_inst, ip_table_name, MAAT_OP_ADD, item3_id, group3_id, - IPv4, "220.181.38.150", "220.181.38.151", 0, 65535, 0); - EXPECT_EQ(ret, 1); - - char temp[1024]={0}; - //item4 -> group4, group4 is not referenced by any compile. - long long item4_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); - long long group4_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = expr_table_set_line(maat_inst, keywords_table_name, MAAT_OP_ADD, item4_id, group4_id, - str_escape(temp, sizeof(temp), "hit group item forth"), - NULL, 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/ - EXPECT_EQ(ret, 1); - - /* - item1 -> group1 -> group11 - / \ - item5 -> / \ -> compile1 - / - item2 -> group2 -> group21 _/ - */ - //item5 -> group1 which means group1 has multi items - long long item5_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); - ret = expr_table_set_line(maat_inst, keywords_table_name, MAAT_OP_ADD, item5_id, group1_id, - str_escape(temp, sizeof(temp), "hit group item fifth"), - NULL, 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/ - EXPECT_EQ(ret, 1); - - sleep(WAIT_FOR_EFFECTIVE_S * 2); - - const char* http_url = "en.wikipedia.org hit group item first"; - const char* http_resp_hdr_cookie = "laptop=thinkpad X1 extrem;hit group item second" - "main[XWJOKE]=hoho; Hm_lvt_bbac0322e6ee13093f98d5c4b5a10912=1578874808;"; - - int http_req_table_id = maat_get_table_id(maat_inst, "HTTP_REQUEST_HEADER"); - ASSERT_GT(http_req_table_id, 0); - - ret = maat_state_set_scan_district(state, http_req_table_id, "URL", strlen("URL")); - EXPECT_EQ(ret, 0); - - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - ret = maat_scan_string(maat_inst, http_req_table_id, http_url, strlen(http_url), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - size_t scan_count = maat_state_get_scan_count(state); - EXPECT_EQ(scan_count, 1); - - struct maat_hit_group hit_groups[128]; - memset(hit_groups, 0, sizeof(hit_groups)); - int n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 2); - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item1_id); - EXPECT_EQ(hit_groups[1].group_id, group11_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_req_table_id); - - memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); - EXPECT_EQ(n_hit_group, 2); - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item1_id); - EXPECT_EQ(hit_groups[1].group_id, group11_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_req_table_id); - - int http_res_table_id = maat_get_table_id(maat_inst, "HTTP_RESPONSE_HEADER"); - ASSERT_GT(http_res_table_id, 0); - - ret = maat_state_set_scan_district(state, http_res_table_id, "Cookie", strlen("Cookie")); - EXPECT_EQ(ret, 0); - - ret = maat_scan_string(maat_inst, http_res_table_id, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], compile1_id); - scan_count = maat_state_get_scan_count(state); - EXPECT_EQ(scan_count, 2); - - n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 4); - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item1_id); - EXPECT_EQ(hit_groups[1].group_id, group11_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[2].item_id, item2_id); - EXPECT_EQ(hit_groups[2].group_id, group21_id); - EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[3].item_id, item2_id); - EXPECT_EQ(hit_groups[3].group_id, group2_id); - EXPECT_EQ(hit_groups[3].vtable_id, http_res_table_id); - - memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); - EXPECT_EQ(n_hit_group, 2); - EXPECT_EQ(hit_groups[0].item_id, item2_id); - EXPECT_EQ(hit_groups[0].group_id, group21_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item2_id); - EXPECT_EQ(hit_groups[1].group_id, group2_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); - - - const char* keywords1="In graph theory, hit group item forth"; - const char *keywords2="To test one group hit group item fifth"; - - int keywords_table_id = maat_get_table_id(maat_inst, keywords_table_name); - ASSERT_GT(keywords_table_id, 0); - - struct maat_stream *stream = maat_stream_new(maat_inst, keywords_table_id, state); - ret = maat_stream_scan(stream, keywords1, strlen(keywords1), results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - scan_count = maat_state_get_scan_count(state); - EXPECT_EQ(scan_count, 3); - - n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 5); - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item1_id); - EXPECT_EQ(hit_groups[1].group_id, group11_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[2].item_id, item2_id); - EXPECT_EQ(hit_groups[2].group_id, group21_id); - EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[3].item_id, item2_id); - EXPECT_EQ(hit_groups[3].group_id, group2_id); - EXPECT_EQ(hit_groups[3].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[4].item_id, item4_id); - EXPECT_EQ(hit_groups[4].group_id, group4_id); - EXPECT_EQ(hit_groups[4].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 - - int ip_table_id = maat_get_table_id(maat_inst, ip_table_name); - ASSERT_GT(ip_table_id, 0); - - uint32_t ip_addr; - inet_pton(AF_INET, "220.181.38.150", &ip_addr); - uint16_t port = htons(17272); - ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - scan_count = maat_state_get_scan_count(state); - EXPECT_EQ(scan_count, 4); - - n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 6); - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item1_id); - EXPECT_EQ(hit_groups[1].group_id, group11_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[2].item_id, item2_id); - EXPECT_EQ(hit_groups[2].group_id, group21_id); - EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[3].item_id, item2_id); - EXPECT_EQ(hit_groups[3].group_id, group2_id); - EXPECT_EQ(hit_groups[3].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[4].item_id, item3_id); - EXPECT_EQ(hit_groups[4].group_id, group3_id); - EXPECT_EQ(hit_groups[4].vtable_id, 0); //physical table(ip_table) vtable_id is 0 - - EXPECT_EQ(hit_groups[5].item_id, item4_id); - EXPECT_EQ(hit_groups[5].group_id, group4_id); - EXPECT_EQ(hit_groups[5].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 - - memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); - EXPECT_EQ(n_hit_group, 2); - EXPECT_EQ(hit_groups[0].item_id, item3_id); - EXPECT_EQ(hit_groups[0].group_id, group3_id); - EXPECT_EQ(hit_groups[0].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 - - EXPECT_EQ(hit_groups[1].item_id, item4_id); - EXPECT_EQ(hit_groups[1].group_id, group4_id); - EXPECT_EQ(hit_groups[1].vtable_id, 0); - - ret = maat_stream_scan(stream, keywords2, strlen(keywords2), results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - scan_count = maat_state_get_scan_count(state); - EXPECT_EQ(scan_count, 5); - - memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, 128); - EXPECT_EQ(n_hit_group, 8); - EXPECT_EQ(hit_groups[0].item_id, item1_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); - - EXPECT_EQ(hit_groups[1].item_id, item1_id); - EXPECT_EQ(hit_groups[1].group_id, group11_id); - EXPECT_EQ(hit_groups[1].vtable_id, http_req_table_id); - - - EXPECT_EQ(hit_groups[2].item_id, item2_id); - EXPECT_EQ(hit_groups[2].group_id, group21_id); - EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[3].item_id, item2_id); - EXPECT_EQ(hit_groups[3].group_id, group2_id); - EXPECT_EQ(hit_groups[3].vtable_id, http_res_table_id); - - EXPECT_EQ(hit_groups[4].item_id, item3_id); - EXPECT_EQ(hit_groups[4].group_id, group3_id); - EXPECT_EQ(hit_groups[4].vtable_id, 0); //physical table(ip_table) vtable_id is 0 - - EXPECT_EQ(hit_groups[5].item_id, item4_id); - EXPECT_EQ(hit_groups[5].group_id, group4_id); - EXPECT_EQ(hit_groups[5].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 - - EXPECT_EQ(hit_groups[6].item_id, item5_id); - EXPECT_EQ(hit_groups[6].group_id, group1_id); - EXPECT_EQ(hit_groups[6].vtable_id, 0); - - EXPECT_EQ(hit_groups[7].item_id, item5_id); - EXPECT_EQ(hit_groups[7].group_id, group11_id); - EXPECT_EQ(hit_groups[7].vtable_id, 0); - - memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, 128); - EXPECT_EQ(n_hit_group, 2); - EXPECT_EQ(hit_groups[0].item_id, item5_id); - EXPECT_EQ(hit_groups[0].group_id, group1_id); - EXPECT_EQ(hit_groups[0].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 - - EXPECT_EQ(hit_groups[1].item_id, item5_id); - EXPECT_EQ(hit_groups[1].group_id, group11_id); - EXPECT_EQ(hit_groups[1].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 - - maat_stream_free(stream); - maat_state_free(state); - state = NULL; -} - TEST_F(MaatCmdTest, HitPath) { - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; - const char *g2g_table_name = "GROUP2GROUP"; + const char *g2g_table_name = "GROUP2GROUP"; + const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE"; const char *http_sig_table_name = "HTTP_SIGNATURE"; const char *ip_table_name = "IP_CONFIG"; const char *keywords_table_name = "KEYWORDS_TABLE"; @@ -6792,7 +5638,7 @@ TEST_F(MaatCmdTest, HitPath) { NULL, 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/ EXPECT_EQ(ret, 1); - sleep(WAIT_FOR_EFFECTIVE_S * 2); + sleep(WAIT_FOR_EFFECTIVE_S); const char* http_url = "en.wikipedia.org/wiki/Path_(graph_theory)"; const char* http_resp_hdr_cookie = "laptop=thinkpad X1 extrem;time=2020-02-11T15:34:00;" @@ -6816,6 +5662,23 @@ TEST_F(MaatCmdTest, HitPath) { size_t scan_count = maat_state_get_scan_count(state); EXPECT_EQ(scan_count, 1); + struct maat_hit_group hit_groups[128]; + memset(hit_groups, 0, sizeof(hit_groups)); + int n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 2); + EXPECT_EQ(hit_groups[0].group_id, group1_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[1].group_id, group11_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_req_table_id); + + memset(hit_groups, 0, sizeof(hit_groups)); + n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 2); + EXPECT_EQ(hit_groups[0].group_id, group1_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[1].group_id, group11_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_req_table_id); + struct maat_hit_path hit_path[128]; memset(hit_path, 0, sizeof(hit_path)); int n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); @@ -6852,6 +5715,25 @@ TEST_F(MaatCmdTest, HitPath) { scan_count = maat_state_get_scan_count(state); EXPECT_EQ(scan_count, 2); + n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 4); + EXPECT_EQ(hit_groups[0].group_id, group1_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[1].group_id, group21_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[2].group_id, group2_id); + EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[3].group_id, group11_id); + EXPECT_EQ(hit_groups[3].vtable_id, http_req_table_id); + + memset(hit_groups, 0, sizeof(hit_groups)); + n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 2); + EXPECT_EQ(hit_groups[0].group_id, group21_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[1].group_id, group2_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); + n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); EXPECT_EQ(n_read, 4); @@ -6905,6 +5787,25 @@ that the edges be all directed in the same direction."; scan_count = maat_state_get_scan_count(state); EXPECT_EQ(scan_count, 3); + n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 5); + EXPECT_EQ(hit_groups[0].group_id, group1_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[1].group_id, group21_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[2].group_id, group2_id); + EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[3].group_id, group11_id); + EXPECT_EQ(hit_groups[3].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[4].group_id, group4_id); + EXPECT_EQ(hit_groups[4].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 + + memset(hit_groups, 0, sizeof(hit_groups)); + n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 1); + EXPECT_EQ(hit_groups[0].group_id, group4_id); + EXPECT_EQ(hit_groups[0].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 + n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); EXPECT_EQ(n_read, 5); @@ -6930,6 +5831,27 @@ that the edges be all directed in the same direction."; scan_count = maat_state_get_scan_count(state); EXPECT_EQ(scan_count, 4); + n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_FULL, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 6); + EXPECT_EQ(hit_groups[0].group_id, group1_id); + EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[1].group_id, group21_id); + EXPECT_EQ(hit_groups[1].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[2].group_id, group2_id); + EXPECT_EQ(hit_groups[2].vtable_id, http_res_table_id); + EXPECT_EQ(hit_groups[3].group_id, group11_id); + EXPECT_EQ(hit_groups[3].vtable_id, http_req_table_id); + EXPECT_EQ(hit_groups[4].group_id, group3_id); + EXPECT_EQ(hit_groups[4].vtable_id, 0); //physical table(ip_table) vtable_id is 0 + EXPECT_EQ(hit_groups[5].group_id, group4_id); + EXPECT_EQ(hit_groups[5].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 + + memset(hit_groups, 0, sizeof(hit_groups)); + n_hit_group = maat_state_get_hit_groups(state, MAAT_LIST_TYPE_INC, hit_groups, sizeof(hit_groups)); + EXPECT_EQ(n_hit_group, 1); + EXPECT_EQ(hit_groups[0].group_id, group3_id); + EXPECT_EQ(hit_groups[0].vtable_id, 0); //physical table(keywords_table) vtable_id is 0 + n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path)); EXPECT_EQ(n_read, 6); @@ -6968,9 +5890,9 @@ that the edges be all directed in the same direction."; TEST_F(MaatCmdTest, SameSuperGroupRefByMultiCompile) { char temp[1024]={0}; int thread_id = 0; - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; - const char *g2g_table_name = "GROUP2GROUP"; + const char *g2g_table_name = "GROUP2GROUP"; + const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE"; const char *http_sig_table_name = "HTTP_SIGNATURE"; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -7059,8 +5981,8 @@ TEST_F(MaatCmdTest, SameSuperGroupRefByMultiCompile) { } TEST_F(MaatCmdTest, SameScanStatusWhenClauseUpdate_TSG6419) { - const char* compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE"; + const char* compile_table_name = "COMPILE"; const char* ip_table_name = "IP_PLUS_CONFIG"; const char *app_id_table_name = "APP_ID"; int thread_id = 0; @@ -7153,8 +6075,8 @@ TEST_F(MaatCmdTest, SameScanStatusWhenClauseUpdate_TSG6419) { } TEST_F(MaatCmdTest, GroupEdit) { - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE"; const char *ip_table_name = "IP_PLUS_CONFIG"; const char *app_id_table_name = "APP_ID"; int thread_id = 0; @@ -7272,8 +6194,8 @@ TEST_F(MaatCmdTest, GroupEdit) { } TEST_F(MaatCmdTest, CompileDelete_TSG6548) { - const char* compile_table_name = "COMPILE_DEFAULT"; - const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* g2c_table_name = "GROUP2COMPILE"; + const char* compile_table_name = "COMPILE"; const char* ip_table_name = "IP_PLUS_CONFIG"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -7295,7 +6217,7 @@ TEST_F(MaatCmdTest, CompileDelete_TSG6548) { IPv4, "192.168.73.163", "192.168.73.180", 0, 65535, 0); EXPECT_EQ(ret, 1); - sleep(WAIT_FOR_EFFECTIVE_S * 2); + sleep(WAIT_FOR_EFFECTIVE_S); uint32_t ip_addr; inet_pton(AF_INET, "192.168.73.169", &ip_addr); @@ -7343,8 +6265,8 @@ TEST_F(MaatCmdTest, CompileDelete_TSG6548) { } TEST_F(MaatCmdTest, UpdateDeadLockDetection) { - const char* compile_table_name = "COMPILE_DEFAULT"; - const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* g2c_table_name = "GROUP2COMPILE"; + const char* compile_table_name = "COMPILE"; const char* table_http_url = "HTTP_URL"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -7418,8 +6340,8 @@ TEST_F(MaatCmdTest, UpdateDeadLockDetection) { } TEST_F(MaatCmdTest, StreamScanWhenExprTableIncUpdate) { - const char* compile_table_name = "COMPILE_DEFAULT"; - const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* g2c_table_name = "GROUP2COMPILE"; + const char* compile_table_name = "COMPILE"; const char* scan_table_name = "KEYWORDS_TABLE"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -7480,8 +6402,8 @@ TEST_F(MaatCmdTest, StreamScanWhenExprTableIncUpdate) { } TEST_F(MaatCmdTest, StreamScanSegfaultWhenVersionRollBack_TSG6324) { - const char* compile_table_name = "COMPILE_DEFAULT"; - const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* g2c_table_name = "GROUP2COMPILE"; + const char* compile_table_name = "COMPILE"; const char* scan_table_name = "KEYWORDS_TABLE"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -7523,7 +6445,7 @@ TEST_F(MaatCmdTest, StreamScanSegfaultWhenVersionRollBack_TSG6324) { //DON'T DO THIS!!! //Roll back version, trigger full update. //This operation generates FATAL logs in test_maat_redis.log.yyyy-mm-dd. - //For example: Add group 22 vt_id 0 to clause 2 of compile 979 failed, group is already existed + //For example: Add group 22 vt_id 0 to clause 2 of compile 979 failed, group is already exisited maat_cmd_incrby(maat_inst, "MAAT_VERSION", -100); //Wating for scanner garbage collect expiration. @@ -7539,8 +6461,8 @@ TEST_F(MaatCmdTest, StreamScanSegfaultWhenVersionRollBack_TSG6324) { } TEST_F(MaatCmdTest, IPAndStreamScanWhenIncUpdate) { - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE"; const char *expr_table_name = "KEYWORDS_TABLE"; const char *ip_table_name = "IP_PLUS_CONFIG"; int thread_id = 0; @@ -7631,8 +6553,8 @@ TEST_F(MaatCmdTest, IPAndStreamScanWhenIncUpdate) { } TEST_F(MaatCmdTest, IPAndStreamScanWhenFullUpdate) { - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE"; const char *ip_table_name = "IP_PLUS_CONFIG"; const char *expr_table_name = "KEYWORDS_TABLE"; int thread_id = 0; @@ -7721,8 +6643,8 @@ TEST_F(MaatCmdTest, IPAndStreamScanWhenFullUpdate) { } TEST_F(MaatCmdTest, IPAndStringScanWhenIncUpdate) { - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE"; const char *expr_table_name = "HTTP_URL"; const char *ip_table_name = "IP_PLUS_CONFIG"; const char *keywords = "IP&stringinc"; @@ -7812,8 +6734,8 @@ TEST_F(MaatCmdTest, IPAndStringScanWhenIncUpdate) { } TEST_F(MaatCmdTest, IPAndStringScanWhenFullupdate) { - const char *compile_table_name = "COMPILE_DEFAULT"; - const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE"; const char *ip_table_name = "IP_PLUS_CONFIG"; const char *expr_table_name = "HTTP_URL"; const char *keywords = "IP&string"; @@ -8136,4 +7058,4 @@ int main(int argc, char ** argv) ret=RUN_ALL_TESTS(); return ret; -} +} \ No newline at end of file diff --git a/test/maat_framework_perf_gtest.cpp b/test/maat_framework_perf_gtest.cpp index 7cc237e..4d7204a 100644 --- a/test/maat_framework_perf_gtest.cpp +++ b/test/maat_framework_perf_gtest.cpp @@ -55,6 +55,7 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para) char *buff = ALLOC(char, strlen(line) + 1); memcpy(buff, line, strlen(line) + 1); + while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { buff[strlen(buff) - 1] = '\0'; } @@ -256,79 +257,33 @@ static int ip_table_set_line(struct maat *maat_inst, const char *table_name, enu return maat_cmd_set_line(maat_inst, &line_rule); } -static int integer_table_set_line(struct maat *maat_inst, const char *table_name, - enum maat_operation op, long long item_id, - long long group_id, int low_bound, int up_bound, - int expire_after) -{ - char table_line[1024] = {0}; - int table_id = maat_get_table_id(maat_inst, table_name); - if (table_id < 0) { - return 0; - } - - sprintf(table_line, "%lld\t%lld\t%d\t%d\t%d", - item_id, group_id, low_bound, up_bound, op); - struct maat_cmd_line line_rule; - - line_rule.rule_id = item_id; - line_rule.table_line = table_line; - line_rule.table_name = table_name; - line_rule.expire_after = expire_after; - - return maat_cmd_set_line(maat_inst, &line_rule); -} - -static int flag_table_set_line(struct maat *maat_inst, const char *table_name, - enum maat_operation op, long long item_id, - long long group_id, long long flag, - long long flag_mask, int expire_after) -{ - char table_line[1024] = {0}; - int table_id = maat_get_table_id(maat_inst, table_name); - if (table_id < 0) { - return 0; - } - - sprintf(table_line, "%lld\t%lld\t%lld\t%lld\t%d", - item_id, group_id, flag, flag_mask, op); - struct maat_cmd_line line_rule; - - line_rule.rule_id = item_id; - line_rule.table_line = table_line; - line_rule.table_name = table_name; - line_rule.expire_after = expire_after; - - return maat_cmd_set_line(maat_inst, &line_rule); -} - -static void test_add_expr_command(struct maat *maat_inst, const char *table_name, - const char *keywords) +void test_add_expr_command(struct maat *maat_inst, const char *table_name, + const char *keywords) { long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); + int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0); EXPECT_EQ(ret, 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id, compile_id, 0, "null", 1, 0); EXPECT_EQ(ret, 1); long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); ret = expr_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id, - keywords, "null", 1, 0, 0, 0); + keywords, NULL, 1, 0, 0, 0); EXPECT_EQ(ret, 1); } -static void test_add_ip_command(struct maat *maat_inst, const char *table_name, - const char *ip, uint16_t port) +void test_add_ip_command(struct maat *maat_inst, const char *table_name, + const char *ip, uint16_t port) { long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); + int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0); EXPECT_EQ(ret, 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id, compile_id, 0, "null", 1, 0); EXPECT_EQ(ret, 1); @@ -338,42 +293,6 @@ static void test_add_ip_command(struct maat *maat_inst, const char *table_name, EXPECT_EQ(ret, 1); } -static void test_add_integer_command(struct maat *maat_inst, const char *table_name, - int low_bound, int up_bound) -{ - long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); - EXPECT_EQ(ret, 1); - - long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, - compile_id, 0, "null", 1, 0); - EXPECT_EQ(ret, 1); - - long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); - ret = integer_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id, - low_bound, up_bound, 0); - EXPECT_EQ(ret, 1); -} - -static void test_add_flag_command(struct maat *maat_inst, const char *table_name, - long long flag, long long flag_mask) -{ - long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); - EXPECT_EQ(ret, 1); - - long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, - compile_id, 0, "null", 1, 0); - EXPECT_EQ(ret, 1); - - long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1); - ret = flag_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id, - flag, flag_mask, 0); - EXPECT_EQ(ret, 1); -} - class MaatPerfStringScan : public testing::Test { protected: @@ -426,7 +345,7 @@ void *perf_string_scan_thread(void *arg) struct maat *maat_inst = param->maat_inst; const char *table_name = param->table_name; struct timespec start, end; - const char *scan_data = "today and yesterday should hit"; + const char *scan_data = "String TEST should hit"; long long results[ARRAY_SIZE] = {0}; int hit_times = 0; size_t n_hit_result = 0; @@ -474,67 +393,13 @@ void *perf_string_update_thread(void *arg) return is_all_hit; } -void *perf_regex_scan_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - struct timespec start, end; - const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567"; - long long results[ARRAY_SIZE] = {0}; - int hit_times = 0; - size_t n_hit_result = 0; - struct maat_state *state = maat_state_new(maat_inst, param->thread_id); - - int table_id = maat_get_table_id(maat_inst, table_name); - - clock_gettime(CLOCK_MONOTONIC, &start); - for (int i = 0; i < param->test_count; i++) { - int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - if (ret == MAAT_SCAN_HIT) { - hit_times++; - } - maat_state_reset(state); - } - clock_gettime(CLOCK_MONOTONIC, &end); - - param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000; - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = (hit_times == param->test_count ? 1 : 0); - log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, - "thread_id:%d regex_scan time_elapse:%lldms hit_times:%d", - param->thread_id, param->time_elapse_ms, hit_times); - return is_all_hit; -} - -void *perf_regex_update_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - const int CMD_EXPR_NUM = 10; - char keyword_buf[128]; - - for (int i = 0; i < CMD_EXPR_NUM; i++) { - random_keyword_generate(keyword_buf, sizeof(keyword_buf)); - test_add_expr_command(maat_inst, table_name, keyword_buf); - sleep(1); - } - - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = 1; - - return is_all_hit; -} - void *perf_ip_scan_thread(void *arg) { struct thread_param *param = (struct thread_param *)arg; struct maat *maat_inst = param->maat_inst; const char *table_name = param->table_name; struct timespec start, end; - char ip_str[32] = "10.0.0.1"; + char ip_str[32] = "10.0.7.100"; uint32_t ip_addr; uint16_t port = htons(65530); @@ -597,113 +462,8 @@ void *perf_ip_update_thread(void *arg) return is_all_hit; } -void *perf_integer_scan_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - struct timespec start, end; - int hit_times = 0; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - - struct maat_state *state = maat_state_new(maat_inst, param->thread_id); - int table_id = maat_get_table_id(maat_inst, table_name); - - clock_gettime(CLOCK_MONOTONIC, &start); - for (int i = 0; i < param->test_count; i++) { - int ret = maat_scan_integer(maat_inst, table_id, 3000, results, - ARRAY_SIZE, &n_hit_result, state); - if (ret == MAAT_SCAN_HIT) { - hit_times++; - } - maat_state_reset(state); - } - clock_gettime(CLOCK_MONOTONIC, &end); - - param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + - (end.tv_nsec - start.tv_nsec) / 1000000; - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = (hit_times == param->test_count ? 1 : 0); - log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, - "thread_id:%d integer_scan time_elapse:%lldms hit_times:%d", - param->thread_id, param->time_elapse_ms, hit_times); - return is_all_hit; -} - -void *perf_integer_update_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - const int CMD_EXPR_NUM = 10; - - for (int i = 0; i < CMD_EXPR_NUM; i++) { - test_add_integer_command(maat_inst, table_name, 3001+i, 3001+i); - sleep(1); - } - - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = 1; - - return is_all_hit; -} - -void *perf_flag_scan_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - struct timespec start, end; - int hit_times = 0; - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - long long scan_data = 15; - - struct maat_state *state = maat_state_new(maat_inst, param->thread_id); - int table_id = maat_get_table_id(maat_inst, table_name); - - clock_gettime(CLOCK_MONOTONIC, &start); - for (int i = 0; i < param->test_count; i++) { - int ret = maat_scan_flag(maat_inst, table_id, scan_data, results, - ARRAY_SIZE, &n_hit_result, state); - if (ret == MAAT_SCAN_HIT) { - hit_times++; - } - maat_state_reset(state); - } - clock_gettime(CLOCK_MONOTONIC, &end); - - param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + - (end.tv_nsec - start.tv_nsec) / 1000000; - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = (hit_times == param->test_count ? 1 : 0); - log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, - "thread_id:%d flag_scan time_elapse:%lldms hit_times:%d", - param->thread_id, param->time_elapse_ms, hit_times); - return is_all_hit; -} - -void *perf_flag_update_thread(void *arg) -{ - struct thread_param *param = (struct thread_param *)arg; - struct maat *maat_inst = param->maat_inst; - const char *table_name = param->table_name; - const int CMD_EXPR_NUM = 10; - - for (int i = 0; i < CMD_EXPR_NUM; i++) { - test_add_flag_command(maat_inst, table_name, i, 15); - sleep(1); - } - - int *is_all_hit = ALLOC(int, 1); - *is_all_hit = 1; - - return is_all_hit; -} - -TEST_F(MaatPerfStringScan, LiteralMultiThread) { - const char *table_name = "EXPR_LITERAL_PERF_CONFIG"; +TEST_F(MaatPerfStringScan, MultiThread) { + const char *table_name = "KEYWORDS_TABLE"; struct maat *maat_inst = MaatPerfStringScan::_shared_maat_inst; int table_id = maat_get_table_id(maat_inst, table_name); @@ -743,58 +503,12 @@ TEST_F(MaatPerfStringScan, LiteralMultiThread) { free(is_all_hit); } scan_per_second = scan_count * 1000 / time_elapse_ms; + //EXPECT_GT(scan_per_second, 800 * 1000); log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, "StringScan match rate on %d-threads speed %lld lookups/s/thread", PERF_THREAD_NUM, scan_per_second); } -TEST_F(MaatPerfStringScan, RegexMultiThread) { - const char *table_name = "EXPR_REGEX_PERF_CONFIG"; - struct maat *maat_inst = MaatPerfStringScan::_shared_maat_inst; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - pthread_t threads[PERF_THREAD_NUM + 1]; - struct thread_param thread_params[PERF_THREAD_NUM + 1]; - int i = 0; - int *is_all_hit = NULL; - - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - thread_params[i].maat_inst = maat_inst; - thread_params[i].thread_id = i; - thread_params[i].table_name = table_name; - thread_params[i].test_count = PERF_SCAN_COUNT; - thread_params[i].time_elapse_ms = 0; - thread_params[i].logger = logger; - - if (i < PERF_THREAD_NUM) { - pthread_create(&threads[i], NULL, perf_regex_scan_thread, thread_params+i); - } else { - thread_params[i].test_count = 0; - pthread_create(&threads[i], NULL, perf_regex_update_thread, thread_params+i); - } - } - - long long time_elapse_ms = 0; - long long scan_count = 0; - long long scan_per_second = 0; - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - pthread_join(threads[i], (void **)&is_all_hit); - time_elapse_ms += thread_params[i].time_elapse_ms; - scan_count += thread_params[i].test_count; - - EXPECT_EQ(*is_all_hit, 1); - *is_all_hit = 0; - free(is_all_hit); - } - scan_per_second = scan_count * 1000 / time_elapse_ms; - - log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, - "RegexScan match rate on %d-threads speed %lld lookups/s/thread", - PERF_THREAD_NUM, scan_per_second); -} - class MaatPerfStreamScan : public testing::Test { protected: @@ -847,31 +561,34 @@ void *perf_stream_scan_thread(void *arg) struct maat *maat_inst = param->maat_inst; const char *table_name = param->table_name; struct timespec start, end; - const char *scan_data = "http://www.cyberessays.com/search_results.php?today and yesterday"; + const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; long long results[ARRAY_SIZE] = {0}; int ret = 0, hit_times = 0; size_t n_hit_result = 0; - + struct maat_state *state_array[ARRAY_SIZE]; + struct maat_stream *sp[ARRAY_SIZE]; int table_id = maat_get_table_id(maat_inst, table_name); - struct maat_state *state = maat_state_new(maat_inst, param->thread_id); - struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state); - + clock_gettime(CLOCK_MONOTONIC, &start); for (int i = 0; i < param->test_count; i++) { - ret = maat_stream_scan(sp, scan_data, strlen(scan_data), results, ARRAY_SIZE, - &n_hit_result, state); - if (ret == MAAT_SCAN_HIT) { - hit_times++; + for (int j = 0; j < ARRAY_SIZE; j++) { + state_array[j] = maat_state_new(maat_inst, param->thread_id); + sp[j] = maat_stream_new(maat_inst, table_id, state_array[j]); + + ret = maat_stream_scan(sp[j], scan_data, strlen(scan_data), results, ARRAY_SIZE, + &n_hit_result, state_array[j]); + if (ret == MAAT_SCAN_HIT) { + hit_times++; + } + maat_stream_free(sp[j]); + maat_state_free(state_array[j]); } - maat_state_reset(state); } clock_gettime(CLOCK_MONOTONIC, &end); - maat_stream_free(sp); - maat_state_free(state); param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000; int *is_all_hit = ALLOC(int, 1); - *is_all_hit = ((hit_times == param->test_count) ? 1 : 0); + *is_all_hit = ((hit_times == param->test_count*ARRAY_SIZE) ? 1 : 0); log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, "thread_id:%d stream_scan time_elapse:%lldms hit_times:%d", @@ -879,40 +596,64 @@ void *perf_stream_scan_thread(void *arg) return is_all_hit; } +void *perf_stream_update_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_inst = param->maat_inst; + const char *table_name = param->table_name; + const int CMD_EXPR_NUM = 10; + char keyword_buf[128]; + + for (int i = 0; i < CMD_EXPR_NUM; i++) { + random_keyword_generate(keyword_buf, sizeof(keyword_buf)); + test_add_expr_command(maat_inst, table_name, keyword_buf); + sleep(1); + } + + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = 1; + + return is_all_hit; +} + TEST_F(MaatPerfStreamScan, MultiThread) { - const char *table_name = "EXPR_LITERAL_PERF_CONFIG"; + const char *table_name = "HTTP_URL"; struct maat *maat_inst = MaatPerfStreamScan::_shared_maat_inst; int table_id = maat_get_table_id(maat_inst, table_name); ASSERT_GT(table_id, 0); - pthread_t threads[PERF_THREAD_NUM]; - struct thread_param thread_params[PERF_THREAD_NUM]; + pthread_t threads[PERF_THREAD_NUM + 1]; + struct thread_param thread_params[PERF_THREAD_NUM + 1]; int i = 0; int *is_all_hit = NULL; - for (i = 0; i < PERF_THREAD_NUM; i++) { + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { thread_params[i].maat_inst = maat_inst; thread_params[i].thread_id = i; thread_params[i].table_name = table_name; - thread_params[i].test_count = PERF_SCAN_COUNT; + thread_params[i].test_count = PERF_SCAN_COUNT / 10; thread_params[i].time_elapse_ms = 0; thread_params[i].logger = logger; if (i < PERF_THREAD_NUM) { pthread_create(&threads[i], NULL, perf_stream_scan_thread, thread_params+i); - } + } else { + thread_params[i].test_count = 0; + pthread_create(&threads[i], NULL, perf_stream_update_thread, thread_params+i); + } } long long time_elapse_ms = 0; long long scan_count = 0; long long scan_per_second = 0; - for (i = 0; i < PERF_THREAD_NUM; i++) { + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { pthread_join(threads[i], (void **)&is_all_hit); time_elapse_ms += thread_params[i].time_elapse_ms; scan_count += thread_params[i].test_count; //maybe expr_runtime rebuild in stream_scan, so should not expect is_all_hit always 1 - EXPECT_EQ(*is_all_hit, 1); + //EXPECT_EQ(*is_all_hit, 1); + *is_all_hit = 0; free(is_all_hit); } scan_per_second = scan_count * 1000 / time_elapse_ms; @@ -970,7 +711,7 @@ struct log_handle *MaatPerfIPScan::logger; TEST_F(MaatPerfIPScan, MultiThread) { - const char *table_name = "IP_PERF_CONFIG"; + const char *table_name = "IP_PLUS_CONFIG"; struct maat *maat_inst = MaatPerfIPScan::_shared_maat_inst; int table_id = maat_get_table_id(maat_inst, table_name); @@ -1016,192 +757,6 @@ TEST_F(MaatPerfIPScan, MultiThread) PERF_THREAD_NUM, scan_per_second); } -class MaatPerfIntegerScan : public testing::Test -{ -protected: - static void SetUpTestCase() { - const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," - "{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; - char redis_ip[64] = "127.0.0.1"; - int redis_port = 6379; - int redis_db = 0; - - logger = log_handle_create("./maat_framework_perf_gtest.log", 0); - int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger); - if (ret < 0) { - log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, - "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); - } - - struct maat_options *opts = maat_options_new(); - maat_options_set_stat_file(opts, "./stat.log"); - maat_options_set_perf_on(opts); - maat_options_set_redis(opts, redis_ip, redis_port, redis_db); - maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); - maat_options_set_accept_tags(opts, accept_tags); - maat_options_set_caller_thread_number(opts, 5); - - _shared_maat_inst = maat_new(opts, table_info_path); - maat_options_free(opts); - if (NULL == _shared_maat_inst) { - log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", - __FUNCTION__, __LINE__); - } - } - - static void TearDownTestCase() { - maat_free(_shared_maat_inst); - log_handle_destroy(logger); - } - - static struct log_handle *logger; - static struct maat *_shared_maat_inst; -}; - -struct maat *MaatPerfIntegerScan::_shared_maat_inst; -struct log_handle *MaatPerfIntegerScan::logger; - -TEST_F(MaatPerfIntegerScan, MultiThread) { - const char *table_name = "INTEGER_PERF_CONFIG"; - struct maat *maat_inst = MaatPerfIntegerScan::_shared_maat_inst; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - pthread_t threads[PERF_THREAD_NUM + 1]; - struct thread_param thread_params[PERF_THREAD_NUM + 1]; - int i = 0; - int *is_all_hit = NULL; - - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - thread_params[i].maat_inst = maat_inst; - thread_params[i].thread_id = i; - thread_params[i].table_name = table_name; - thread_params[i].test_count = PERF_SCAN_COUNT; - thread_params[i].time_elapse_ms = 0; - thread_params[i].logger = logger; - - if (i < PERF_THREAD_NUM) { - pthread_create(&threads[i], NULL, perf_integer_scan_thread, thread_params+i); - } else { - thread_params[i].test_count = 0; - pthread_create(&threads[i], NULL, perf_integer_update_thread, thread_params+i); - } - } - - long long time_elapse_ms = 0; - long long scan_count = 0; - long long scan_per_second = 0; - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - pthread_join(threads[i], (void **)&is_all_hit); - time_elapse_ms += thread_params[i].time_elapse_ms; - scan_count += thread_params[i].test_count; - - EXPECT_EQ(*is_all_hit, 1); - *is_all_hit = 0; - free(is_all_hit); - } - scan_per_second = scan_count * 1000 / time_elapse_ms; - - log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, - "IntegerScan match rate on %d-threads speed %lld lookups/s/thread", - PERF_THREAD_NUM, scan_per_second); -} - -class MaatPerfFlagScan : public testing::Test -{ -protected: - static void SetUpTestCase() { - const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," - "{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; - char redis_ip[64] = "127.0.0.1"; - int redis_port = 6379; - int redis_db = 0; - - logger = log_handle_create("./maat_framework_perf_gtest.log", 0); - int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger); - if (ret < 0) { - log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, - "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); - } - - struct maat_options *opts = maat_options_new(); - maat_options_set_stat_file(opts, "./stat.log"); - maat_options_set_perf_on(opts); - maat_options_set_redis(opts, redis_ip, redis_port, redis_db); - maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); - maat_options_set_accept_tags(opts, accept_tags); - maat_options_set_caller_thread_number(opts, 5); - - _shared_maat_inst = maat_new(opts, table_info_path); - maat_options_free(opts); - if (NULL == _shared_maat_inst) { - log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, - "[%s:%d] create maat instance in MaatFlagScan failed.", - __FUNCTION__, __LINE__); - } - } - - static void TearDownTestCase() { - maat_free(_shared_maat_inst); - log_handle_destroy(logger); - } - - static struct log_handle *logger; - static struct maat *_shared_maat_inst; -}; - -struct maat *MaatPerfFlagScan::_shared_maat_inst; -struct log_handle *MaatPerfFlagScan::logger; - -TEST_F(MaatPerfFlagScan, MultiThread) { - const char *table_name = "FLAG_PERF_CONFIG"; - struct maat *maat_inst = MaatPerfFlagScan::_shared_maat_inst; - - int table_id = maat_get_table_id(maat_inst, table_name); - ASSERT_GT(table_id, 0); - - pthread_t threads[PERF_THREAD_NUM + 1]; - struct thread_param thread_params[PERF_THREAD_NUM + 1]; - int i = 0; - int *is_all_hit = NULL; - - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - thread_params[i].maat_inst = maat_inst; - thread_params[i].thread_id = i; - thread_params[i].table_name = table_name; - thread_params[i].test_count = PERF_SCAN_COUNT; - thread_params[i].time_elapse_ms = 0; - thread_params[i].logger = logger; - - if (i < PERF_THREAD_NUM) { - pthread_create(&threads[i], NULL, perf_flag_scan_thread, thread_params+i); - } else { - thread_params[i].test_count = 0; - pthread_create(&threads[i], NULL, perf_flag_update_thread, thread_params+i); - } - } - - long long time_elapse_ms = 0; - long long scan_count = 0; - long long scan_per_second = 0; - for (i = 0; i < PERF_THREAD_NUM + 1; i++) { - pthread_join(threads[i], (void **)&is_all_hit); - time_elapse_ms += thread_params[i].time_elapse_ms; - scan_count += thread_params[i].test_count; - - EXPECT_EQ(*is_all_hit, 1); - *is_all_hit = 0; - free(is_all_hit); - } - scan_per_second = scan_count * 1000 / time_elapse_ms; - - log_info(maat_inst->logger, MODULE_FRAMEWORK_PERF_GTEST, - "FlagScan match rate on %d-threads speed %lld lookups/s/thread", - PERF_THREAD_NUM, scan_per_second); -} - class MaatPerfFQDNPluginScan : public testing::Test { protected: @@ -1777,4 +1332,4 @@ int main(int argc, char ** argv) ret=RUN_ALL_TESTS(); return ret; -} +} \ No newline at end of file diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 2c190e8..058d6c9 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -247,4 +247,4 @@ int main(int argc, char ** argv) log_handle_destroy(g_logger); return ret; -} +} \ No newline at end of file diff --git a/test/maat_json.json b/test/maat_json.json index b2a4eba..d4c65ba 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -1687,6 +1687,28 @@ } ] }, + { + "compile_id": 175, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_composition.match", + "is_valid": "yes", + "groups": [ + { + "group_name": "ipv4_composition.source", + "virtual_table": "COMPOSITION_IP_SOURCE", + "not_flag": 0 + }, + { + "group_name": "ipv4_composition.destination", + "virtual_table": "COMPOSITION_IP_DESTINATION", + "not_flag": 0 + } + ] + }, { "compile_id": 176, "service": 0, @@ -1719,6 +1741,24 @@ } ] }, + { + "compile_id": 177, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_composition.session.match", + "is_valid": "yes", + "groups": [ + { + "group_name": "ipv4_composition.session", + "virtual_table": "COMPOSITION_IP_SESSION", + "not_flag": 0, + "clause_index": 1 + } + ] + }, { "compile_id": 178, "service": 1, @@ -1779,6 +1819,47 @@ } ] }, + { + "compile_id": 180, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "Hierarchy_VirtualWithTwoPhysical", + "is_valid": "yes", + "groups": [ + { + "group_name": "FQDN_OBJ1", + "virtual_table": "VIRTUAL_SSL_SNI", + "not_flag": 0, + "clause_index": 0 + }, + { + "group_name": "FQDN_CAT1", + "virtual_table": "VIRTUAL_SSL_SNI", + "not_flag": 0, + "clause_index": 0 + } + ] + }, + { + "compile_id": 181, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_composition.match", + "is_valid": "yes", + "groups": [ + { + "group_name": "IPv4-composition-source-only", + "virtual_table": "COMPOSITION_IP_SOURCE", + "not_flag": 0 + } + ] + }, { "compile_id": 182, "service": 1, @@ -1861,6 +1942,28 @@ } ] }, + { + "compile_id": 185, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_composition.NOT_match", + "is_valid": "yes", + "groups": [ + { + "group_name": "IPv4-composition-NOT-client-ip", + "virtual_table": "COMPOSITION_IP_SOURCE", + "not_flag": 0 + }, + { + "group_name": "IPv4-composition-NOT-server-ip", + "virtual_table": "COMPOSITION_IP_DESTINATION", + "not_flag": 1 + } + ] + }, { "compile_id": 186, "service": 1, @@ -2258,7 +2361,7 @@ "do_blacklist": 1, "do_log": 1, "user_region": "Something:I\\bhave\\ba\\bname,7799", - "compile_table_name": "COMPILE_FIREWALL_DEFAULT", + "compile_table_name": "COMPILE_FIREWALL", "is_valid": "yes", "groups": [ { @@ -2870,135 +2973,7 @@ "not_flag": 0 } ] - }, - { - "compile_id": 211, - "service": 0, - "action": 0, - "do_blacklist": 0, - "do_log": 0, - "effective_rage": 0, - "user_region": "ip_perf_test", - "is_valid": "yes", - "groups": [ - { - "regions": [ - { - "table_type": "ip_plus", - "table_name": "IP_PERF_CONFIG", - "table_content": { - "addr_type": "ipv4", - "addr_format": "range", - "ip1": "10.0.0.1", - "ip2": "10.0.0.6", - "port_format": "range", - "port1": "65530", - "port2": "65535", - "protocol": 6 - } - } - ], - "not_flag": 0 - } - ] - }, - { - "compile_id": 212, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "user_region": "integer_perf_test", - "is_valid": "yes", - "groups": [ - { - "group_name": "Untitled", - "regions": [ - { - "table_name": "INTEGER_PERF_CONFIG", - "table_type": "interval", - "table_content": { - "low_boundary": 3000, - "up_boundary": 3000 - } - } - ] - } - ] - }, - { - "compile_id": 213, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "user_region": "expr_perf_test", - "is_valid": "yes", - "groups": [ - { - "regions": [ - { - "table_name": "EXPR_LITERAL_PERF_CONFIG", - "table_type": "expr", - "table_content": { - "keywords": "today&yesterday", - "expr_type": "and", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - }, - { - "compile_id": 214, - "service": 0, - "action": 0, - "do_blacklist": 0, - "do_log": 0, - "user_region": "flag_perf_test", - "is_valid": "yes", - "groups": [ - { - "regions": [ - { - "table_type": "flag", - "table_name": "FLAG_PERF_CONFIG", - "table_content": { - "flag": 15, - "flag_mask": 15 - } - } - ] - } - ] - }, - { - "compile_id": 215, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "user_region": "expr_perf_test", - "is_valid": "yes", - "groups": [ - { - "regions": [ - { - "table_name": "EXPR_REGEX_PERF_CONFIG", - "table_type": "expr", - "table_content": { - "keywords": "action=search\\&query=(.*)", - "expr_type": "regex", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - } - ] - } + } ], "plugin_table": [ { @@ -3009,6 +2984,14 @@ "3\t192.168.1.1\t103\t1" ] }, + { + "table_name": "TEST_PLUGIN_TABLE", + "table_content": [ + "1\t3388\t99\t1", + "2\t3355\t66\t1", + "3\tcccc\t11\t1" + ] + }, { "table_name": "TEST_PLUGIN_EXDATA_TABLE", "table_content": [ diff --git a/test/table_info.conf b/test/table_info.conf index 1219396..07702b4 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -1,10 +1,12 @@ [ { "table_id":0, - "table_name":"COMPILE_DEFAULT", + "table_name":"COMPILE", + "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"], "table_type":"compile", "valid_column":8, "custom": { + "gc_timeout_s": 3, "compile_id":1, "tags":6, "clause_num":9 @@ -12,20 +14,22 @@ }, { "table_id":1, - "table_name":"COMPILE_ALIAS", - "table_type":"compile", - "valid_column":8, + "table_name":"GROUP2COMPILE", + "db_tables":["GROUP2COMPILE_DEFAULT", "GROUP2COMPILE_ALIAS"], + "table_type":"group2compile", + "associated_compile_table_id":0, + "valid_column":3, "custom": { - "compile_id":1, - "tags":6, - "clause_num":9 + "group_id":1, + "compile_id":2, + "not_flag":4, + "virtual_table_name":5, + "clause_index":6 } - }, + }, { "table_id":2, - "table_name":"COMPILE_CONJUNCTION", - "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"], - "default_compile_table":1, + "table_name":"COMPILE_FIREWALL", "table_type":"compile", "valid_column":8, "custom": { @@ -36,8 +40,7 @@ }, { "table_id":3, - "table_name":"GROUP2COMPILE", - "db_tables":["GROUP2COMPILE_DEFAULT", "GROUP2COMPILE_ALIAS"], + "table_name":"GROUP2COMPILE_FIREWALL", "table_type":"group2compile", "associated_compile_table_id":2, "valid_column":3, @@ -51,43 +54,6 @@ }, { "table_id":4, - "table_name":"COMPILE_FIREWALL_DEFAULT", - "table_type":"compile", - "valid_column":8, - "custom": { - "compile_id":1, - "tags":6, - "clause_num":9 - } - }, - { - "table_id":5, - "table_name":"COMPILE_FIREWALL_CONJUNCTION", - "db_tables":["COMPILE_FIREWALL_DEFAULT"], - "table_type":"compile", - "valid_column":8, - "custom": { - "compile_id":1, - "tags":6, - "clause_num":9 - } - }, - { - "table_id":6, - "table_name":"GROUP2COMPILE_FIREWALL", - "table_type":"group2compile", - "associated_compile_table_id":5, - "valid_column":3, - "custom": { - "group_id":1, - "compile_id":2, - "not_flag":4, - "virtual_table_name":5, - "clause_index":6 - } - }, - { - "table_id":7, "table_name":"GROUP2GROUP", "table_type":"group2group", "valid_column":4, @@ -98,33 +64,7 @@ } }, { - "table_id":8, - "table_name":"COMPILE_PLUGIN", - "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"], - "table_type":"plugin", - "valid_column":8, - "custom": { - "gc_timeout_s":3, - "key_type":"integer", - "key_len":8, - "key":1 - } - }, - { - "table_id":9, - "table_name":"COMPILE_FIREWALL_PLUGIN", - "db_tables":["COMPILE_FIREWALL_DEFAULT"], - "table_type":"plugin", - "valid_column":8, - "custom": { - "gc_timeout_s":3, - "key_type":"integer", - "key_len":8, - "key":1 - } - }, - { - "table_id":10, + "table_id":5, "table_name":"HTTP_REGION", "db_tables":["HTTP_URL", "HTTP_HOST"], "table_type":"expr", @@ -139,7 +79,7 @@ } }, { - "table_id":11, + "table_id":6, "table_name":"KEYWORDS_TABLE", "table_type":"expr", "valid_column":7, @@ -153,7 +93,7 @@ } }, { - "table_id":12, + "table_id":7, "table_name":"IP_CONFIG", "table_type":"ip_plus", "valid_column":11, @@ -171,7 +111,7 @@ } }, { - "table_id":13, + "table_id":8, "table_name":"CONTENT_SIZE", "table_type":"intval", "valid_column":5, @@ -183,19 +123,18 @@ } }, { - "table_id":14, + "table_id":9, "table_name":"QD_ENTRY_INFO", "table_type":"plugin", "valid_column":4, "custom": { - "gc_timeout_s":3, "key_type":"integer", "key_len":8, "key":1 } }, { - "table_id":15, + "table_id":10, "table_name":"HTTP_SIGNATURE", "table_type":"expr_plus", "valid_column":8, @@ -210,7 +149,7 @@ } }, { - "table_id":16, + "table_id":11, "table_name":"IMAGE_FP", "table_type":"expr", "valid_column":7, @@ -224,12 +163,11 @@ } }, { - "table_id":17, + "table_id":12, "table_name":"TEST_EFFECTIVE_RANGE_TABLE", "table_type":"plugin", "valid_column":4, "custom": { - "gc_timeout_s":3, "key_type":"integer", "key_len":8, "key":1, @@ -237,12 +175,11 @@ } }, { - "table_id":18, + "table_id":13, "table_name":"TEST_FOREIGN_KEY", "table_type":"plugin", "valid_column":4, "custom": { - "gc_timeout_s":3, "key_type":"pointer", "key":2, "tag":3, @@ -250,7 +187,7 @@ } }, { - "table_id":19, + "table_id":14, "table_name":"TEST_PLUGIN_EXDATA_TABLE", "table_type":"plugin", "valid_column":4, @@ -262,19 +199,18 @@ } }, { - "table_id":20, + "table_id":15, "table_name":"IR_INTERCEPT_IP", "table_type":"plugin", "valid_column":14, "custom": { - "gc_timeout_s":3, "key_type":"pointer", "key":2, "tag":18 } }, { - "table_id":21, + "table_id":16, "table_name":"APP_PAYLOAD", "table_type":"expr_plus", "valid_column":8, @@ -289,7 +225,7 @@ } }, { - "table_id":22, + "table_id":17, "table_name":"TROJAN_PAYLOAD", "table_type":"expr", "valid_column":7, @@ -304,7 +240,7 @@ } }, { - "table_id":23, + "table_id":18, "table_name":"MAIL_ADDR", "table_type":"expr", "valid_column":7, @@ -318,7 +254,7 @@ } }, { - "table_id":24, + "table_id":19, "table_name":"IP_PLUS_CONFIG", "table_type":"ip_plus", "valid_column":11, @@ -336,32 +272,32 @@ } }, { - "table_id":25, + "table_id":20, "table_name":"HTTP_RESPONSE_KEYWORDS", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":26, + "table_id":21, "table_name":"HTTP_REQUEST_HEADER", "table_type":"virtual", "physical_table": "HTTP_SIGNATURE" }, { - "table_id":27, + "table_id":22, "table_name":"HTTP_RESPONSE_HEADER", "table_type":"virtual", "physical_table": "HTTP_SIGNATURE" }, { - "table_id":28, + "table_id":23, "table_name":"VIRTUAL_IP_PLUS_TABLE", "db_tables":["VIRTUAL_IP_PLUS_SOURCE", "VIRTUAL_IP_PLUS_DESTINATION"], "table_type":"virtual", "physical_table": "IP_PLUS_CONFIG" }, { - "table_id":29, + "table_id":24, "table_name":"TEST_IP_PLUGIN_WITH_EXDATA", "table_type":"ip_plugin", "valid_column":6, @@ -375,7 +311,7 @@ } }, { - "table_id":30, + "table_id":25, "table_name":"AS_NUMBER", "table_type":"expr", "valid_column":7, @@ -389,19 +325,19 @@ } }, { - "table_id":31, + "table_id":26, "table_name":"SOURCE_IP_ASN", "table_type":"virtual", "physical_table":"AS_NUMBER" }, { - "table_id":32, + "table_id":27, "table_name":"DESTINATION_IP_ASN", "table_type":"virtual", "physical_table":"AS_NUMBER" }, { - "table_id":33, + "table_id":28, "table_name":"GeoLocation", "table_type":"expr", "valid_column":7, @@ -415,13 +351,13 @@ } }, { - "table_id":34, + "table_id":29, "table_name":"SOURCE_IP_GEO", "table_type":"virtual", "physical_table":"GeoLocation" }, { - "table_id":35, + "table_id":30, "table_name":"INTERGER_PLUS", "table_type":"intval_plus", "valid_column":6, @@ -434,7 +370,7 @@ } }, { - "table_id":36, + "table_id":31, "table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA", "table_type":"fqdn_plugin", "valid_column":5, @@ -446,7 +382,7 @@ } }, { - "table_id":37, + "table_id":32, "table_name":"APP_ID", "table_type":"intval", "valid_column":5, @@ -458,7 +394,7 @@ } }, { - "table_id":38, + "table_id":33, "table_name":"EMPTY_KEYWORD", "table_type":"expr", "valid_column":7, @@ -472,7 +408,7 @@ } }, { - "table_id":39, + "table_id":34, "table_name":"EMPTY_INTERGER", "table_type":"intval", "valid_column":5, @@ -484,7 +420,7 @@ } }, { - "table_id":40, + "table_id":35, "table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA", "table_type":"bool_plugin", "valid_column":4, @@ -495,7 +431,7 @@ } }, { - "table_id":41, + "table_id":36, "table_name":"FLAG_CONFIG", "table_type":"flag", "valid_column":5, @@ -507,7 +443,7 @@ } }, { - "table_id":42, + "table_id":37, "table_name":"FLAG_PLUS_CONFIG", "table_type":"flag_plus", "valid_column":6, @@ -520,12 +456,11 @@ } }, { - "table_id":43, + "table_id":38, "table_name":"TEST_PLUGIN_LONG_KEY_TYPE_TABLE", "table_type":"plugin", "valid_column":4, "custom": { - "gc_timeout_s":3, "key_type":"integer", "key_len":8, "key":2, @@ -533,12 +468,11 @@ } }, { - "table_id":44, + "table_id":39, "table_name":"TEST_PLUGIN_INT_KEY_TYPE_TABLE", "table_type":"plugin", "valid_column":4, "custom": { - "gc_timeout_s":3, "key_type":"integer", "key_len":4, "key":2, @@ -546,91 +480,20 @@ } }, { - "table_id":45, + "table_id":40, "table_name":"TEST_PLUGIN_IP_KEY_TYPE_TABLE", "table_type":"plugin", "valid_column":4, "custom": { - "gc_timeout_s":3, "key_type":"ip_addr", "addr_type":1, "key":2 } }, { - "table_id":46, + "table_id":41, "table_name":"HTTP_URL_FILTER", "table_type":"virtual", "physical_table": "HTTP_URL" - }, - { - "table_id":47, - "table_name":"IP_PERF_CONFIG", - "table_type":"ip_plus", - "valid_column":11, - "custom": { - "item_id":1, - "group_id":2, - "addr_type":3, - "addr_format":4, - "ip1":5, - "ip2":6, - "port_format":7, - "port1":8, - "port2":9, - "protocol":10 - } - }, - { - "table_id":48, - "table_name":"INTEGER_PERF_CONFIG", - "table_type":"intval", - "valid_column":5, - "custom": { - "item_id":1, - "group_id":2, - "low_bound":3, - "up_bound":4 - } - }, - { - "table_id":49, - "table_name":"EXPR_LITERAL_PERF_CONFIG", - "table_type":"expr", - "valid_column":7, - "custom": { - "item_id":1, - "group_id":2, - "keywords":3, - "expr_type":4, - "match_method":5, - "is_hexbin":6 - } - }, - { - "table_id":50, - "table_name":"EXPR_REGEX_PERF_CONFIG", - "table_type":"expr", - "valid_column":7, - "custom": { - "item_id":1, - "group_id":2, - "keywords":3, - "expr_type":4, - "match_method":5, - "is_hexbin":6 - } - }, - { - "table_id":51, - "table_name":"FLAG_PERF_CONFIG", - "table_type":"flag", - "valid_column":5, - "custom": { - "item_id":1, - "group_id":2, - "flag":3, - "flag_mask":4 - } } ] \ No newline at end of file diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 7c31d6c..30ea49d 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -58,34 +58,6 @@ add_dependencies(hyperscan_runtime_static hyperscan) set_property(TARGET hyperscan_runtime_static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhs_runtime.a) set_property(TARGET hyperscan_runtime_static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) -#pcre-8.45 -ExternalProject_Add(pcre PREFIX pcre - URL ${CMAKE_CURRENT_SOURCE_DIR}/pcre-8.45.tar.gz - CONFIGURE_COMMAND ./configure --prefix=${VENDOR_BUILD} - BUILD_COMMAND make - INSTALL_COMMAND make install - BUILD_IN_SOURCE 1) - -ExternalProject_Get_Property(pcre INSTALL_DIR) -file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) - -#rulescan 3.0.1 -ExternalProject_Add(rulescan PREFIX rulescan - URL ${CMAKE_CURRENT_SOURCE_DIR}/rulescan-3.0.1.tar.gz - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS="-fPIC") - -ExternalProject_Get_Property(rulescan INSTALL_DIR) -file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) - -#merge librulescan.a and libpcre.a => librs.a -add_custom_command(OUTPUT ${VENDOR_BUILD}/lib/librs.a - COMMAND ar crsT ${VENDOR_BUILD}/lib/librs.a ${VENDOR_BUILD}/lib/libpcre.a ${VENDOR_BUILD}/lib/librulescan.a - DEPENDS pcre rulescan) -add_custom_target(_merge ALL DEPENDS ${VENDOR_BUILD}/lib/librs.a) - -add_library(rulescan_static STATIC IMPORTED GLOBAL) -set_property(TARGET rulescan_static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/librs.a) - # hiredis-1.1.0 ExternalProject_Add(hiredis PREFIX hiredis URL ${CMAKE_CURRENT_SOURCE_DIR}/hiredis-1.1.0.tar.gz diff --git a/vendor/pcre-8.45.tar.gz b/vendor/pcre-8.45.tar.gz deleted file mode 100644 index 19f8dfe..0000000 Binary files a/vendor/pcre-8.45.tar.gz and /dev/null differ diff --git a/vendor/rulescan-3.0.1.tar.gz b/vendor/rulescan-3.0.1.tar.gz deleted file mode 100644 index e137f1c..0000000 Binary files a/vendor/rulescan-3.0.1.tar.gz and /dev/null differ