gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FEATURE]expr_matcher support dual engine(hyperscan & rulescan)

Browse Source
This commit is contained in:
刘文坛
2023-07-28 12:32:25 +00:00
parent c1d413e992
commit bcbb796a7d
55 changed files with 3745 additions and 81813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/maat.h
View File

@@ -60,6 +60,11 @@ enum maat_list_type {
MAAT_LIST_TYPE_INC
};
enum maat_expr_engine {
MAAT_EXPR_ENGINE_HS = 0, //default engine(hyperscan)
MAAT_EXPR_ENGINE_RS //rulescan
};
struct ip_addr {
int ip_type; //4: IPv4, 6: IPv6
union {
@@ -145,6 +150,8 @@ int maat_options_set_redis(struct maat_options *opts, const char *redis_ip,
int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filename);
int maat_options_set_expr_engine(struct maat_options *opts, enum maat_expr_engine engine);
/* maat_instance API */
struct maat *maat_new(struct maat_options *opts, const char *table_info_path);
void maat_free(struct maat *instance);

5
scanner/CMakeLists.txt
View File

@@ -7,8 +7,9 @@ include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
add_subdirectory(ip_matcher/IntervalIndex)
add_library(adapter-static adapter_hs/adapter_hs.cpp bool_matcher/bool_matcher.cpp
add_library(adapter-static bool_matcher/bool_matcher.cpp expr_matcher/expr_matcher.cpp
expr_matcher/adapter_hs/adapter_hs.cpp expr_matcher/adapter_rs/adapter_rs.cpp
fqdn_engine/fqdn_engine.cpp ip_matcher/ip_matcher.cpp ip_matcher/ipv4_match.cpp
ip_matcher/ipv6_match.cpp flag_matcher/flag_matcher.cpp interval_matcher/cgranges.c
interval_matcher/interval_matcher.cpp)
target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static interval_index_static)
target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static rulescan_static interval_index_static)

134
scanner/adapter_hs/adapter_hs.h
View File

@@ -1,134 +0,0 @@
/*
**********************************************************************************************
* File: adapter_hs.h
* Description: wrapper for raw hyperscan
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _ADAPTER_HS_H_
#define _ADAPTER_HS_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include <stdint.h>
#include "log/log.h"
#define MAX_EXPR_PATTERN_NUM 8
struct adapter_hs;
/* match method */
enum hs_match_mode {
HS_MATCH_MODE_INVALID = -1,
HS_MATCH_MODE_EXACTLY = 1, /* scan data must match pattern exactly */
HS_MATCH_MODE_PREFIX, /* pattern must in the head of scan_data */
HS_MATCH_MODE_SUFFIX, /* pattern must in the end of scan_data */
HS_MATCH_MODE_SUB /* pattern must in the range[l_offset, r_offset] of scan_data */
};
enum hs_pattern_type {
HS_PATTERN_TYPE_STR = 0, /* pure literal string */
HS_PATTERN_TYPE_REG /* regex expression */
};
enum hs_case_sensitive {
HS_CASE_SENSITIVE = 0,
HS_CASE_INSENSITIVE
};
struct hs_scan_result {
long long rule_id;
void *user_tag;
};
struct hs_pattern {
enum hs_case_sensitive case_sensitive;
enum hs_match_mode match_mode;
enum hs_pattern_type pattern_type;
int is_hexbin; /* 1(yes) 0(no) */
/*
* just match in scan_data's range of [start_offset, end_offset], -1 means no limits
* for example:
* [-1, end_offset] means the pattern must in scan_data's [0 ~ start_offset]
* [start_offset, -1] means the pattern must in scan_data's [start_offset ~ data_end]
*/
int start_offset;
int end_offset;
/* start pointer of pattern */
char *pat;
/* pattern length */
size_t pat_len;
};
/* logic AND expression, such as (pattern1 & pattern2) */
struct expr_rule {
long long expr_id;
size_t n_patterns;
struct hs_pattern patterns[MAX_EXPR_PATTERN_NUM];
void *user_tag;
};
int adapter_hs_verify_regex_expression(const char *regex_expr,
struct log_handle *logger);
/**
* @brief new adapter_hs instance
*
* @param rules: logic AND expression's array
* @param n_rule: the number of logic AND expression's array
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
*
* @retval the pointer to adapter_hs instance
*/
struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
size_t n_worker_thread, struct log_handle *logger);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
*
* @param instance: adapter_hs instance obtained by adapter_hs_new()
* @param thread_id: the thread_id of caller
* @param data: data to be scanned
* @param data_len: the length of data to be scanned
* @param results: the array of expr_id
* @param n_results: number of elements in array of expr_id
*/
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
const char *data, size_t data_len,
struct hs_scan_result *results,
size_t n_result, size_t *n_hit_result);
/**
* @brief destroy adapter_hs instance
*
* @param instance: adapter_hs instance obtained by adapter_hs_new()
*/
void adapter_hs_free(struct adapter_hs *instance);
struct adapter_hs_stream;
/**
* @brief open adapter_hs stream after adapter_hs instance initialized for stream scan
*
*/
struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id);
int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, size_t data_len,
struct hs_scan_result *results, size_t n_result, size_t *n_hit_result);
void adapter_hs_stream_close(struct adapter_hs_stream *stream);
#ifdef __cplusplus
}
#endif
#endif

224
scanner/adapter_hs/adapter_hs.cpp → scanner/expr_matcher/adapter_hs/adapter_hs.cpp
View File

@@ -1,10 +1,10 @@
/*
**********************************************************************************************
* File: adapter_hs.cpp
* File: adapter_hs.c
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
@@ -20,9 +20,8 @@
#include "uthash/utarray.h"
#include "uthash/uthash.h"
#include "maat_utils.h"
#include "../bool_matcher/bool_matcher.h"
#include "../../bool_matcher/bool_matcher.h"
#define MAX_OFFSET_NUM 1024
#define MAX_HIT_PATTERN_NUM 512
pid_t hs_gettid()
@@ -56,7 +55,6 @@ struct adapter_hs_scratch {
struct adapter_hs_stream {
int thread_id;
size_t n_expr;
hs_stream_t *literal_stream;
hs_stream_t *regex_stream;
struct adapter_hs_runtime *ref_hs_rt;
@@ -91,7 +89,7 @@ struct pattern_offset {
struct pattern_attribute {
long long pattern_id;
enum hs_match_mode match_mode;
enum expr_match_mode match_mode;
struct pattern_offset offset;
};
@@ -137,12 +135,12 @@ static int _hs_alloc_scratch(hs_database_t *db, hs_scratch_t **scratches,
static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt,
size_t n_worker_thread,
enum hs_pattern_type pattern_type,
enum expr_pattern_type pattern_type,
struct log_handle *logger)
{
int ret = 0;
if (pattern_type == HS_PATTERN_TYPE_STR) {
if (pattern_type == EXPR_PATTERN_TYPE_STR) {
hs_rt->scratch->literal_scratches = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->scratch->literal_scratches,
n_worker_thread, logger);
@@ -264,7 +262,7 @@ static void populate_compile_data(struct adpt_hs_compile_data *compile_data,
/* set flags */
compile_data->flags[index] |= HS_FLAG_SOM_LEFTMOST;
if (case_sensitive == HS_CASE_INSENSITIVE) {
if (case_sensitive == EXPR_CASE_INSENSITIVE) {
compile_data->flags[index] |= HS_FLAG_CASELESS;
}
@@ -295,14 +293,14 @@ static struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule,
pattern_attr[pattern_index].pattern_id = pattern_index;
pattern_attr[pattern_index].match_mode = rules[i].patterns[j].match_mode;
if (pattern_attr[pattern_index].match_mode == HS_MATCH_MODE_SUB ||
pattern_attr[pattern_index].match_mode == HS_MATCH_MODE_EXACTLY) {
if (pattern_attr[pattern_index].match_mode == EXPR_MATCH_MODE_SUB ||
pattern_attr[pattern_index].match_mode == EXPR_MATCH_MODE_EXACTLY) {
pattern_attr[pattern_index].offset.start = rules[i].patterns[j].start_offset;
pattern_attr[pattern_index].offset.end = rules[i].patterns[j].end_offset;
}
/* literal pattern */
if (rules[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) {
if (rules[i].patterns[j].type == EXPR_PATTERN_TYPE_STR) {
populate_compile_data(literal_cd, literal_index, pattern_index,
rules[i].patterns[j].pat, rules[i].patterns[j].pat_len,
rules[i].patterns[j].case_sensitive);
@@ -321,7 +319,7 @@ static struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule,
bool_exprs[i].expr_id = rules[i].expr_id;
bool_exprs[i].item_num = rules[i].n_patterns;
bool_exprs[i].user_tag = rules[i].user_tag;
bool_exprs[i].user_tag = rules[i].tag;
}
*n_pattern = pattern_index;
@@ -345,81 +343,43 @@ static int verify_regex_expression(const char *regex_str, struct log_handle *log
FREE(info);
hs_free_compile_error(error);
return -1;
return 0;
}
if (info != NULL) {
FREE(info);
}
return 0;
return 1;
}
int adapter_hs_verify_regex_expression(const char *regex_expr, struct log_handle *logger)
{
if (NULL == regex_expr) {
return -1;
return 0;
}
return verify_regex_expression(regex_expr, logger);
}
struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
void *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern,
size_t n_worker_thread, struct log_handle *logger)
{
if (0 == n_worker_thread || NULL == rules || 0 == n_rule) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] input parameters illegal!", __FUNCTION__, __LINE__);
return NULL;
}
/* get the sum of pattern */
size_t i = 0, j = 0;
size_t literal_pattern_num = 0;
size_t regex_pattern_num = 0;
for (i = 0; i < n_rule; i++) {
if (rules[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] the number of patterns in one expression "
"should less than %d", __FUNCTION__, __LINE__,
MAX_EXPR_PATTERN_NUM);
return NULL;
}
for (j = 0; j < rules[i].n_patterns; j++) {
/* pat_len should not 0 */
if (0 == rules[i].patterns[j].pat_len) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] expr pattern length should not 0",
__FUNCTION__, __LINE__);
return NULL;
}
if (rules[i].patterns[j].pattern_type == HS_PATTERN_TYPE_STR) {
literal_pattern_num++;
} else {
regex_pattern_num++;
}
}
}
if (0 == literal_pattern_num && 0 == regex_pattern_num) {
log_error(logger, MODULE_ADAPTER_HS,
"[%s:%d] exprs has no valid pattern", __FUNCTION__, __LINE__);
return NULL;
}
size_t i = 0;
struct adpt_hs_compile_data *literal_cd = NULL;
struct adpt_hs_compile_data *regex_cd = NULL;
if (literal_pattern_num > 0) {
literal_cd = adpt_hs_compile_data_new(literal_pattern_num);
if (n_literal_pattern > 0) {
literal_cd = adpt_hs_compile_data_new(n_literal_pattern);
}
if (regex_pattern_num > 0) {
regex_cd = adpt_hs_compile_data_new(regex_pattern_num);
if (n_regex_pattern > 0) {
regex_cd = adpt_hs_compile_data_new(n_regex_pattern);
}
size_t pattern_cnt = literal_pattern_num + regex_pattern_num;
size_t pattern_cnt = n_literal_pattern + n_regex_pattern;
struct adapter_hs *hs_inst = ALLOC(struct adapter_hs, 1);
hs_inst->hs_attr = ALLOC(struct pattern_attribute, pattern_cnt);
hs_inst->logger = logger;
@@ -478,21 +438,21 @@ struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
n_worker_thread);
for (i = 0; i < n_worker_thread; i++) {
hs_inst->hs_rt->scratch->bool_match_buffs[i] = ALLOC(struct bool_expr_match,
hs_inst->n_expr);
MAX_HIT_EXPR_NUM);
}
/* literal and regex scratch can't reuse */
if (literal_pattern_num > 0) {
if (n_literal_pattern > 0) {
ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread,
HS_PATTERN_TYPE_STR, logger);
EXPR_PATTERN_TYPE_STR, logger);
if (ret < 0) {
goto error;
}
}
if (regex_pattern_num > 0) {
if (n_regex_pattern > 0) {
ret = adpt_hs_alloc_scratch(hs_inst->hs_rt, n_worker_thread,
HS_PATTERN_TYPE_REG, logger);
EXPR_PATTERN_TYPE_REG, logger);
if (ret < 0) {
goto error;
}
@@ -500,7 +460,7 @@ struct adapter_hs *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
hs_inst->hs_rt->streams = ALLOC(struct adapter_hs_stream *, n_worker_thread);
for (i = 0; i < n_worker_thread; i++) {
hs_inst->hs_rt->streams[i] = adapter_hs_stream_open(hs_inst, i);
hs_inst->hs_rt->streams[i] = (struct adapter_hs_stream *)adapter_hs_stream_open(hs_inst, i);
}
return hs_inst;
@@ -509,13 +469,15 @@ error:
return NULL;
}
void adapter_hs_free(struct adapter_hs *hs_inst)
void adapter_hs_free(void *hs_instance)
{
if (NULL == hs_inst) {
if (NULL == hs_instance) {
return;
}
struct adapter_hs *hs_inst = (struct adapter_hs *)hs_instance;
size_t i = 0;
if (hs_inst->hs_rt != NULL) {
if (hs_inst->hs_rt->literal_db != NULL) {
hs_free_database(hs_inst->hs_rt->literal_db);
@@ -625,12 +587,12 @@ static int matched_event_cb(unsigned int id, unsigned long long from,
int ret = 0;
struct pattern_attribute pat_attr = matched_pat->ref_hs_attr[id];
switch (pat_attr.match_mode) {
case HS_MATCH_MODE_EXACTLY:
case EXPR_MATCH_MODE_EXACTLY:
if (0 == from && matched_pat->scan_data_len == to) {
ret = 1;
}
break;
case HS_MATCH_MODE_SUB:
case EXPR_MATCH_MODE_SUB:
if (pat_attr.offset.start == -1 &&
pat_attr.offset.end == -1) {
ret = 1;
@@ -656,12 +618,12 @@ static int matched_event_cb(unsigned int id, unsigned long long from,
ret = 1;
}
break;
case HS_MATCH_MODE_PREFIX:
case EXPR_MATCH_MODE_PREFIX:
if (0 == from) {
ret = 1;
}
break;
case HS_MATCH_MODE_SUFFIX:
case EXPR_MATCH_MODE_SUFFIX:
if (to == matched_pat->scan_data_len) {
ret = 1;
}
@@ -678,43 +640,42 @@ static int matched_event_cb(unsigned int id, unsigned long long from,
return 0;
}
UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
struct adapter_hs_stream *
adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id)
UT_icd ut_hs_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
void *adapter_hs_stream_open(void *hs_instance, int thread_id)
{
if (NULL == hs_instance || thread_id < 0) {
return NULL;
}
struct adapter_hs *hs_inst = (struct adapter_hs *)hs_instance;
struct adapter_hs_stream *hs_stream = ALLOC(struct adapter_hs_stream, 1);
hs_error_t err;
hs_stream->logger = hs_instance->logger;
hs_stream->logger = hs_inst->logger;
hs_stream->thread_id = thread_id;
hs_stream->n_expr = hs_instance->n_expr;
hs_stream->ref_hs_rt = hs_instance->hs_rt;
hs_stream->ref_hs_rt = hs_inst->hs_rt;
hs_stream->matched_pat = ALLOC(struct matched_pattern, 1);
hs_stream->matched_pat->ref_hs_attr = hs_instance->hs_attr;
hs_stream->matched_pat->n_patterns = hs_instance->n_patterns;
utarray_new(hs_stream->matched_pat->pattern_ids, &ut_pattern_id_icd);
hs_stream->matched_pat->ref_hs_attr = hs_inst->hs_attr;
hs_stream->matched_pat->n_patterns = hs_inst->n_patterns;
utarray_new(hs_stream->matched_pat->pattern_ids, &ut_hs_pattern_id_icd);
utarray_reserve(hs_stream->matched_pat->pattern_ids, MAX_HIT_PATTERN_NUM);
int err_count = 0;
if (hs_instance->hs_rt->literal_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->literal_db, 0,
if (hs_inst->hs_rt->literal_db != NULL) {
err = hs_open_stream(hs_inst->hs_rt->literal_db, 0,
&hs_stream->literal_stream);
if (err != HS_SUCCESS) {
log_error(hs_instance->logger, MODULE_ADAPTER_HS,
log_error(hs_inst->logger, MODULE_ADAPTER_HS,
"hs_open_stream failed, hs err:%d", err);
err_count++;
}
}
if (hs_instance->hs_rt->regex_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->regex_db, 0,
if (hs_inst->hs_rt->regex_db != NULL) {
err = hs_open_stream(hs_inst->hs_rt->regex_db, 0,
&hs_stream->regex_stream);
if (err != HS_SUCCESS) {
log_error(hs_instance->logger, MODULE_ADAPTER_HS,
log_error(hs_inst->logger, MODULE_ADAPTER_HS,
"hs_open_stream failed, hs err:%d", err);
err_count++;
}
@@ -740,36 +701,37 @@ error:
return NULL;
}
void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
void adapter_hs_stream_close(void *hs_stream)
{
if (NULL == hs_stream) {
return;
}
if (hs_stream->ref_hs_rt != NULL) {
if (hs_stream->literal_stream != NULL) {
hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL);
hs_stream->literal_stream = NULL;
struct adapter_hs_stream *stream = (struct adapter_hs_stream *)hs_stream;
if (stream->ref_hs_rt != NULL) {
if (stream->literal_stream != NULL) {
hs_close_stream(stream->literal_stream, NULL, NULL, NULL);
stream->literal_stream = NULL;
}
if (hs_stream->regex_stream != NULL) {
hs_close_stream(hs_stream->regex_stream, NULL, NULL, NULL);
hs_stream->regex_stream = NULL;
if (stream->regex_stream != NULL) {
hs_close_stream(stream->regex_stream, NULL, NULL, NULL);
stream->regex_stream = NULL;
}
}
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free
/* stream->hs_rt point to hs_instance->hs_rt which will call free
same as hs_attr */
hs_stream->ref_hs_rt = NULL;
hs_stream->matched_pat->ref_hs_attr = NULL;
stream->ref_hs_rt = NULL;
stream->matched_pat->ref_hs_attr = NULL;
if (hs_stream->matched_pat->pattern_ids != NULL) {
utarray_free(hs_stream->matched_pat->pattern_ids);
hs_stream->matched_pat->pattern_ids = NULL;
if (stream->matched_pat->pattern_ids != NULL) {
utarray_free(stream->matched_pat->pattern_ids);
stream->matched_pat->pattern_ids = NULL;
}
FREE(hs_stream->matched_pat);
FREE(hs_stream);
FREE(stream->matched_pat);
FREE(stream);
}
static void adapter_hs_stream_reset(struct adapter_hs_stream *hs_stream)
@@ -794,9 +756,9 @@ static void adapter_hs_stream_reset(struct adapter_hs_stream *hs_stream)
utarray_clear(hs_stream->matched_pat->pattern_ids);
}
int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data,
size_t data_len, struct hs_scan_result *results,
size_t n_result, size_t *n_hit_result)
int adapter_hs_scan_stream(void *hs_stream, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result,
size_t *n_hit_result)
{
hs_error_t err;
@@ -816,36 +778,37 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
*/
int err_count = 0;
int thread_id = hs_stream->thread_id;
struct adapter_hs_scratch *scratch = hs_stream->ref_hs_rt->scratch;
hs_stream->matched_pat->scan_data_len = data_len;
struct adapter_hs_stream *stream = (struct adapter_hs_stream *)hs_stream;
int thread_id = stream->thread_id;
struct adapter_hs_scratch *scratch = stream->ref_hs_rt->scratch;
stream->matched_pat->scan_data_len = data_len;
int err_scratch_flag = 0;
if (hs_stream->literal_stream != NULL) {
if (stream->literal_stream != NULL) {
if (scratch->literal_scratches != NULL) {
err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
err = hs_scan_stream(stream->literal_stream, data, data_len,
0, scratch->literal_scratches[thread_id],
matched_event_cb, hs_stream->matched_pat);
matched_event_cb, stream->matched_pat);
if (err != HS_SUCCESS) {
err_count++;
}
} else {
log_error(hs_stream->logger, MODULE_ADAPTER_HS,
log_error(stream->logger, MODULE_ADAPTER_HS,
"literal_scratches is null, thread_id:%d", thread_id);
err_scratch_flag++;
}
}
if (hs_stream->regex_stream != NULL) {
if (stream->regex_stream != NULL) {
if (scratch->regex_scratches != NULL) {
err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
err = hs_scan_stream(stream->regex_stream, data, data_len,
0, scratch->regex_scratches[thread_id],
matched_event_cb, hs_stream->matched_pat);
matched_event_cb, stream->matched_pat);
if (err != HS_SUCCESS) {
err_count++;
}
} else {
log_error(hs_stream->logger, MODULE_ADAPTER_HS,
log_error(stream->logger, MODULE_ADAPTER_HS,
"regex_scratches is null, thread_id:%d", thread_id);
err_scratch_flag++;
}
@@ -859,7 +822,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
return -1;
}
size_t n_pattern_id = utarray_len(hs_stream->matched_pat->pattern_ids);
size_t n_pattern_id = utarray_len(stream->matched_pat->pattern_ids);
if (0 == n_pattern_id) {
*n_hit_result = 0;
return 0;
@@ -868,13 +831,13 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
unsigned long long pattern_ids[n_pattern_id];
for (size_t i = 0; i < n_pattern_id; i++) {
pattern_ids[i] = *(unsigned long long *)utarray_eltptr(hs_stream->matched_pat->pattern_ids, i);
pattern_ids[i] = *(unsigned long long *)utarray_eltptr(stream->matched_pat->pattern_ids, i);
}
int ret = 0;
struct bool_expr_match *bool_matcher_results = scratch->bool_match_buffs[thread_id];
int bool_matcher_ret = bool_matcher_match(hs_stream->ref_hs_rt->bm, pattern_ids, n_pattern_id,
bool_matcher_results, hs_stream->n_expr);
int bool_matcher_ret = bool_matcher_match(stream->ref_hs_rt->bm, pattern_ids, n_pattern_id,
bool_matcher_results, MAX_HIT_EXPR_NUM);
if (bool_matcher_ret < 0) {
ret = -1;
goto next;
@@ -891,22 +854,21 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
*n_hit_result = bool_matcher_ret;
next:
utarray_clear(hs_stream->matched_pat->pattern_ids);
utarray_clear(stream->matched_pat->pattern_ids);
return ret;
}
int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
const char *data, size_t data_len,
struct hs_scan_result *results,
size_t n_result, size_t *n_hit_result)
int adapter_hs_scan(void *hs_instance, int thread_id, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result, size_t *n_hit_result)
{
if (NULL == hs_instance || NULL == data || (0 == data_len) ||
NULL == results || 0 == n_result || NULL == n_hit_result) {
return -1;
}
struct adapter_hs_stream *hs_stream = hs_instance->hs_rt->streams[thread_id];
struct adapter_hs *hs_inst = (struct adapter_hs *)hs_instance;
struct adapter_hs_stream *hs_stream = hs_inst->hs_rt->streams[thread_id];
assert(hs_stream != NULL);
adapter_hs_stream_reset(hs_stream);

75
scanner/expr_matcher/adapter_hs/adapter_hs.h Normal file
View File

@@ -0,0 +1,75 @@
/*
**********************************************************************************************
* File: adapter_hs.h
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#ifndef _ADAPTER_HS_H_
#define _ADAPTER_HS_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include <stdint.h>
#include "log/log.h"
#include "../expr_matcher.h"
int adapter_hs_verify_regex_expression(const char *regex_expr, struct log_handle *logger);
/**
* @brief new adapter_hs instance
*
* @param rules: logic AND expression's array
* @param n_rule: the number of logic AND expression's array
* @param nr_worker_threads: the number of scan threads which will call adapter_hs_scan()
*
* @retval the pointer to adapter_hs instance
*/
void *adapter_hs_new(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern,
size_t n_worker_thread, struct log_handle *logger);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
*
* @param instance: adapter_hs instance obtained by adapter_hs_new()
* @param thread_id: the thread_id of caller
* @param data: data to be scanned
* @param data_len: the length of data to be scanned
* @param results: the array of expr_id
* @param n_results: number of elements in array of expr_id
*/
int adapter_hs_scan(void *hs_instance, int thread_id, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result, size_t *n_hit_result);
/**
* @brief destroy adapter_hs instance
*
* @param instance: adapter_hs instance obtained by adapter_hs_new()
*/
void adapter_hs_free(void *instance);
/**
* @brief open adapter_hs stream after adapter_hs instance initialized for stream scan
*
*/
void *adapter_hs_stream_open(void *hs_instance, int thread_id);
int adapter_hs_scan_stream(void *stream, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result,
size_t *n_hit_result);
void adapter_hs_stream_close(void *stream);
#ifdef __cplusplus
}
#endif
#endif

718
scanner/expr_matcher/adapter_rs/adapter_rs.cpp Normal file
View File

@@ -0,0 +1,718 @@
/*
**********************************************************************************************
* File: adapter_rs.cpp
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#include <stdint.h>
#include <stdio.h>
#include <stddef.h>
#include <assert.h>
#include <unistd.h>
#include <sys/syscall.h>
#include "rulescan.h"
#include "adapter_rs.h"
#include "uthash/utarray.h"
#include "uthash/uthash.h"
#include "maat_utils.h"
#include "../../bool_matcher/bool_matcher.h"
#define MAX_HIT_PATTERN_NUM 512
pid_t rs_gettid()
{
return syscall(SYS_gettid);
}
static const char *rs_module_name_str(const char *name)
{
static __thread char module[64];
snprintf(module, sizeof(module), "%s(%d)", name, rs_gettid());
return module;
}
#define MODULE_ADAPTER_RS rs_module_name_str("maat.adapter_rs")
struct adpt_rs_compile_data {
struct scan_pattern *patterns;
size_t n_patterns;
};
struct adapter_rs_stream {
int thread_id;
size_t offset; /* current stream offset */
rs_stream_t *literal_stream;
rs_stream_t *regex_stream;
struct adapter_rs_runtime *ref_rs_rt;
struct log_handle *logger;
};
/* adapter_rs runtime */
struct adapter_rs_runtime {
rs_database_t *literal_db;
rs_database_t *regex_db;
struct bool_expr_match **bool_match_buffs; /* per thread */
struct adapter_rs_stream **streams; /* per thread */
struct matched_pattern **matched_pats; /* per thread */
struct bool_matcher *bm;
};
/* adapter_hs instance */
struct adapter_rs {
size_t n_worker_thread;
size_t n_expr;
size_t n_patterns;
struct adapter_rs_runtime *rs_rt;
struct pattern_attribute *rs_attr;
struct log_handle *logger;
};
struct pattern_offset {
long long start;
long long end;
};
struct pattern_attribute {
long long pattern_id;
enum expr_match_mode match_mode;
struct pattern_offset offset;
size_t pattern_len;
};
struct matched_pattern {
UT_array *pattern_ids;
size_t n_patterns;
struct pattern_attribute *ref_rs_attr;
};
int adapter_rs_verify_regex_expression(const char *regex_expr,
struct log_handle *logger)
{
int ret = rs_verify_regex(regex_expr);
if (ret == 0) {
log_error(logger, MODULE_ADAPTER_RS,
"[%s:%d] illegal regex expression: \"%s\"",
__FUNCTION__, __LINE__, regex_expr);
}
return ret;
}
/**
* @brief build hs block database for literal string and regex expression respectively
*
* @retval 0(success) -1(failed)
*/
static int adpt_rs_build_database(struct adapter_rs_runtime *rs_rt,
size_t n_worker_thread,
struct adpt_rs_compile_data *literal_cd,
struct adpt_rs_compile_data *regex_cd,
struct log_handle *logger)
{
if (NULL == rs_rt) {
return -1;
}
int ret = 0;
if (literal_cd != NULL) {
ret = rs_compile_lit(literal_cd->patterns, literal_cd->n_patterns,
&rs_rt->literal_db);
if (ret < 0) {
log_error(logger, MODULE_ADAPTER_RS, "[%s:%d] compile error",
__FUNCTION__, __LINE__);
return -1;
}
}
if (regex_cd != NULL) {
size_t n_failed_pats = 0;
ret = rs_compile_regex(regex_cd->patterns, regex_cd->n_patterns,
n_worker_thread, &rs_rt->regex_db, &n_failed_pats);
if (ret < 0) {
log_error(logger, MODULE_ADAPTER_RS, "[%s:%d] compile error",
__FUNCTION__, __LINE__);
return -1;
}
}
return 0;
}
static struct adpt_rs_compile_data *adpt_rs_compile_data_new(size_t n_patterns)
{
struct adpt_rs_compile_data *rs_cd = ALLOC(struct adpt_rs_compile_data, 1);
rs_cd->patterns = ALLOC(struct scan_pattern, n_patterns);
rs_cd->n_patterns = n_patterns;
return rs_cd;
}
static void adpt_rs_compile_data_free(struct adpt_rs_compile_data *rs_cd)
{
if (NULL == rs_cd) {
return;
}
if (rs_cd->patterns != NULL) {
for (size_t i = 0; i < rs_cd->n_patterns; i++) {
if (rs_cd->patterns[i].pattern != NULL) {
FREE(rs_cd->patterns[i].pattern);
}
}
FREE(rs_cd->patterns);
}
FREE(rs_cd);
}
static void populate_compile_data(struct adpt_rs_compile_data *compile_data,
size_t index, long long pattern_id, char *pat,
size_t pat_len, int case_sensitive)
{
compile_data->patterns[index].id = pattern_id;
compile_data->patterns[index].case_sensitive = case_sensitive;
compile_data->patterns[index].pattern = ALLOC(char, pat_len + 1);
memcpy(compile_data->patterns[index].pattern, pat, pat_len);
compile_data->patterns[index].pattern_len = pat_len;
}
static struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule,
struct pattern_attribute *pattern_attr,
struct adpt_rs_compile_data *literal_cd,
struct adpt_rs_compile_data *regex_cd,
size_t *n_pattern)
{
long long pattern_idx = 0;
size_t literal_idx = 0;
size_t regex_idx = 0;
struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_rule);
/* populate adpt_hs_compile_data and bool_expr */
for (size_t i = 0; i < n_rule; i++) {
for (size_t j = 0; j < rules[i].n_patterns; j++) {
pattern_attr[pattern_idx].pattern_id = pattern_idx;
pattern_attr[pattern_idx].match_mode = rules[i].patterns[j].match_mode;
pattern_attr[pattern_idx].pattern_len = rules[i].patterns[j].pat_len;
if (pattern_attr[pattern_idx].match_mode == EXPR_MATCH_MODE_SUB ||
pattern_attr[pattern_idx].match_mode == EXPR_MATCH_MODE_EXACTLY) {
pattern_attr[pattern_idx].offset.start = rules[i].patterns[j].start_offset;
pattern_attr[pattern_idx].offset.end = rules[i].patterns[j].end_offset;
}
/* literal pattern */
if (rules[i].patterns[j].type == EXPR_PATTERN_TYPE_STR) {
populate_compile_data(literal_cd, literal_idx, pattern_idx,
rules[i].patterns[j].pat, rules[i].patterns[j].pat_len,
rules[i].patterns[j].case_sensitive);
literal_idx++;
} else {
/* regex pattern */
populate_compile_data(regex_cd, regex_idx, pattern_idx,
rules[i].patterns[j].pat, rules[i].patterns[j].pat_len,
rules[i].patterns[j].case_sensitive);
regex_idx++;
}
bool_exprs[i].items[j].item_id = pattern_idx++;
bool_exprs[i].items[j].not_flag = 0;
}
bool_exprs[i].expr_id = rules[i].expr_id;
bool_exprs[i].item_num = rules[i].n_patterns;
bool_exprs[i].user_tag = rules[i].tag;
}
*n_pattern = pattern_idx;
return bool_exprs;
}
UT_icd ut_rs_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL};
void *adapter_rs_new(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern,
size_t n_worker_thread, struct log_handle *logger)
{
/* get the sum of pattern */
size_t i = 0;
struct adpt_rs_compile_data *literal_cd = NULL;
struct adpt_rs_compile_data *regex_cd = NULL;
if (n_literal_pattern > 0) {
literal_cd = adpt_rs_compile_data_new(n_literal_pattern);
}
if (n_regex_pattern > 0) {
regex_cd = adpt_rs_compile_data_new(n_regex_pattern);
}
size_t pattern_cnt = n_literal_pattern + n_regex_pattern;
struct adapter_rs *rs_inst = ALLOC(struct adapter_rs, 1);
rs_inst->rs_attr = ALLOC(struct pattern_attribute, pattern_cnt);
rs_inst->logger = logger;
rs_inst->n_worker_thread = n_worker_thread;
rs_inst->n_expr = n_rule;
struct bool_expr *bool_exprs = bool_exprs_new(rules, n_rule, rs_inst->rs_attr,
literal_cd, regex_cd, &pattern_cnt);
if (NULL == bool_exprs) {
return NULL;
}
rs_inst->n_patterns = pattern_cnt;
/* create bool matcher */
size_t mem_size = 0;
int rs_ret = 0;
rs_inst->rs_rt = ALLOC(struct adapter_rs_runtime, 1);
//hs_rt->bm
rs_inst->rs_rt->bm = bool_matcher_new(bool_exprs, n_rule, &mem_size);
if (rs_inst->rs_rt->bm != NULL) {
log_info(logger, MODULE_ADAPTER_RS,
"Adapter_hs module: build bool matcher of %zu expressions"
" with %zu bytes memory", n_rule, mem_size);
} else {
log_error(logger, MODULE_ADAPTER_RS,
"[%s:%d] Adapter_hs module: build bool matcher failed",
__FUNCTION__, __LINE__);
rs_ret = -1;
}
FREE(bool_exprs);
/* build hs database hs_rt->literal_db & hs_rt->regex_db */
int ret = adpt_rs_build_database(rs_inst->rs_rt, n_worker_thread,
literal_cd, regex_cd, logger);
if (ret < 0) {
rs_ret = -1;
}
if (literal_cd != NULL) {
adpt_rs_compile_data_free(literal_cd);
}
if (regex_cd != NULL) {
adpt_rs_compile_data_free(regex_cd);
}
if (rs_ret < 0) {
goto error;
}
/* alloc scratch */
rs_inst->rs_rt->bool_match_buffs = ALLOC(struct bool_expr_match *, n_worker_thread);
for (i = 0; i < n_worker_thread; i++) {
rs_inst->rs_rt->bool_match_buffs[i] = ALLOC(struct bool_expr_match, MAX_HIT_EXPR_NUM);
}
rs_inst->rs_rt->streams = ALLOC(struct adapter_rs_stream *, n_worker_thread);
for (i = 0; i < n_worker_thread; i++) {
rs_inst->rs_rt->streams[i] = (struct adapter_rs_stream *)adapter_rs_stream_open(rs_inst, i);
}
rs_inst->rs_rt->matched_pats = ALLOC(struct matched_pattern *, n_worker_thread);
for (i = 0; i < n_worker_thread; i++) {
rs_inst->rs_rt->matched_pats[i] = ALLOC(struct matched_pattern, 1);
rs_inst->rs_rt->matched_pats[i]->ref_rs_attr = rs_inst->rs_attr;
rs_inst->rs_rt->matched_pats[i]->n_patterns = rs_inst->n_patterns;
utarray_new(rs_inst->rs_rt->matched_pats[i]->pattern_ids, &ut_rs_pattern_id_icd);
utarray_reserve(rs_inst->rs_rt->matched_pats[i]->pattern_ids, MAX_HIT_PATTERN_NUM);
}
return rs_inst;
error:
adapter_rs_free(rs_inst);
return NULL;
}
void adapter_rs_free(void *rs_instance)
{
if (NULL == rs_instance) {
return;
}
size_t i = 0;
struct adapter_rs *rs_inst = (struct adapter_rs *)rs_instance;
if (rs_inst->rs_rt != NULL) {
if (rs_inst->rs_rt->literal_db != NULL) {
rs_free_database(rs_inst->rs_rt->literal_db);
rs_inst->rs_rt->literal_db = NULL;
}
if (rs_inst->rs_rt->regex_db != NULL) {
rs_free_database(rs_inst->rs_rt->regex_db);
rs_inst->rs_rt->regex_db = NULL;
}
if (rs_inst->rs_rt->bool_match_buffs != NULL) {
for (i = 0; i < rs_inst->n_worker_thread; i++) {
if (rs_inst->rs_rt->bool_match_buffs[i] != NULL) {
FREE(rs_inst->rs_rt->bool_match_buffs[i]);
}
}
FREE(rs_inst->rs_rt->bool_match_buffs);
}
if (rs_inst->rs_rt->bm != NULL) {
bool_matcher_free(rs_inst->rs_rt->bm);
rs_inst->rs_rt->bm = NULL;
}
if (rs_inst->rs_rt->streams != NULL) {
for (i = 0; i < rs_inst->n_worker_thread; i++) {
if (rs_inst->rs_rt->streams[i] != NULL) {
adapter_rs_stream_close(rs_inst->rs_rt->streams[i]);
rs_inst->rs_rt->streams[i] = NULL;
}
}
FREE(rs_inst->rs_rt->streams);
}
if (rs_inst->rs_rt->matched_pats != NULL) {
for (i = 0; i < rs_inst->n_worker_thread; i++) {
if (rs_inst->rs_rt->matched_pats[i] != NULL) {
utarray_free(rs_inst->rs_rt->matched_pats[i]->pattern_ids);
FREE(rs_inst->rs_rt->matched_pats[i]);
}
}
FREE(rs_inst->rs_rt->matched_pats);
}
FREE(rs_inst->rs_rt);
}
if (rs_inst->rs_attr != NULL) {
FREE(rs_inst->rs_attr);
}
FREE(rs_inst);
}
static inline int compare_pattern_id(const void *a, const void *b)
{
long long ret = *(const unsigned long long *)a - *(const unsigned long long *)b;
if (ret == 0) {
return 0;
} else if(ret < 0) {
return -1;
} else {
return 1;
}
}
/**
* @param id: pattern id
*/
static int matched_event_cb(unsigned int id, int pos_offset, int from, int to,
size_t data_len, void *ctx)
{
// put id in set
unsigned long long pattern_id = id;
struct matched_pattern *matched_pat = (struct matched_pattern *)ctx;
if (pattern_id > matched_pat->n_patterns || id < 0) {
return 0;
}
if (utarray_len(matched_pat->pattern_ids) >= MAX_HIT_PATTERN_NUM) {
return 0;
}
// duplicate pattern_id
if (utarray_find(matched_pat->pattern_ids, &pattern_id, compare_pattern_id)) {
return 0;
}
int ret = 0;
struct pattern_attribute pat_attr = matched_pat->ref_rs_attr[id];
switch (pat_attr.match_mode) {
case EXPR_MATCH_MODE_EXACTLY:
if (0 == (from + pos_offset) && (int)data_len == (to + pos_offset)) {
ret = 1;
}
break;
case EXPR_MATCH_MODE_SUB:
if (pat_attr.offset.start == -1 &&
pat_attr.offset.end == -1) {
ret = 1;
break;
}
if (pat_attr.offset.start == -1) {
if ((long long)(to + pos_offset - 1) <= pat_attr.offset.end) {
ret = 1;
break;
}
}
if (pat_attr.offset.end == -1) {
if ((long long)(from + pos_offset) >= pat_attr.offset.start) {
ret = 1;
break;
}
}
if ((long long)(from + pos_offset) >= pat_attr.offset.start &&
(long long)(to + pos_offset - 1) <= pat_attr.offset.end) {
ret = 1;
}
break;
case EXPR_MATCH_MODE_PREFIX:
if (0 == (from + pos_offset)) {
ret = 1;
}
break;
case EXPR_MATCH_MODE_SUFFIX:
if ((to + pos_offset) == (int)data_len) {
ret = 1;
}
break;
default:
break;
}
if (1 == ret) {
utarray_push_back(matched_pat->pattern_ids, &pattern_id);
utarray_sort(matched_pat->pattern_ids, compare_pattern_id);
}
return 0;
}
void *adapter_rs_stream_open(void *rs_instance, int thread_id)
{
if (NULL == rs_instance || thread_id < 0) {
return NULL;
}
struct adapter_rs *rs_inst = (struct adapter_rs *)rs_instance;
struct adapter_rs_stream *rs_stream = ALLOC(struct adapter_rs_stream, 1);
rs_stream->logger = rs_inst->logger;
rs_stream->thread_id = thread_id;
rs_stream->ref_rs_rt = rs_inst->rs_rt;
int err_count = 0;
if (rs_inst->rs_rt->literal_db != NULL) {
rs_stream->literal_stream = rs_open_stream(rs_inst->rs_rt->literal_db, 0, 128);
if (NULL == rs_stream->literal_stream) {
log_error(rs_inst->logger, MODULE_ADAPTER_RS, "rs_open_stream failed");
err_count++;
}
}
if (rs_inst->rs_rt->regex_db != NULL) {
rs_stream->regex_stream = rs_open_stream(rs_inst->rs_rt->regex_db, 0, 128);
if (NULL == rs_stream->regex_stream) {
log_error(rs_inst->logger, MODULE_ADAPTER_RS, "rs_open_stream failed");
err_count++;
}
}
if (err_count > 0) {
goto error;
}
return rs_stream;
error:
if (rs_stream->literal_stream != NULL) {
rs_close_stream(rs_stream->literal_stream);
rs_stream->literal_stream = NULL;
}
if (rs_stream->regex_stream != NULL) {
rs_close_stream(rs_stream->regex_stream);
rs_stream->regex_stream = NULL;
}
FREE(rs_stream);
return NULL;
}
void adapter_rs_stream_close(void *rs_stream)
{
if (NULL == rs_stream) {
return;
}
struct adapter_rs_stream *stream = (struct adapter_rs_stream *)rs_stream;
if (stream->ref_rs_rt != NULL) {
if (stream->literal_stream != NULL) {
rs_close_stream(stream->literal_stream);
stream->literal_stream = NULL;
}
if (stream->regex_stream != NULL) {
rs_close_stream(stream->regex_stream);
stream->regex_stream = NULL;
}
}
/* rs_stream->rs_rt point to rs_instance->rs_rt which will call free
same as rs_attr */
stream->ref_rs_rt = NULL;
FREE(stream);
}
int adapter_rs_scan_stream(void *rs_stream, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result,
size_t *n_hit_result)
{
if (NULL == rs_stream || NULL == data || 0 == data_len ||
NULL == results || 0 == n_result || NULL == n_hit_result) {
return -1;
}
/*
In streaming mode, a non-zero return from the user-specified event-handler
function has consequences for the rest of that stream's lifetime: when a
non-zero return occurs, it signals that no more of the stream should be
scanned. Consequently if the user makes a subsequent call to
`hs_scan_stream` on a stream whose processing was terminated in this way,
hs_scan_stream will return `HS_SCAN_TERMINATED`. This case has not been
demonstrated in pcapscan, as its callback always returns 0.
*/
int ret = 0, err_count = 0;
struct adapter_rs_stream *stream = (struct adapter_rs_stream *)rs_stream;
int thread_id = stream->thread_id;
struct adapter_rs_runtime *rs_rt = stream->ref_rs_rt;
struct matched_pattern *matched_pat = rs_rt->matched_pats[thread_id];
if (stream->literal_stream != NULL) {
ret = rs_scan_stream(stream->literal_stream, data, data_len,
matched_event_cb, matched_pat);
if (ret < 0) {
err_count++;
}
}
if (stream->regex_stream != NULL) {
ret = rs_scan_stream(stream->regex_stream, data, data_len,
matched_event_cb, matched_pat);
if (ret < 0) {
err_count++;
}
}
if (err_count == 2) {
return -1;
}
size_t n_pattern_id = utarray_len(matched_pat->pattern_ids);
if (0 == n_pattern_id) {
*n_hit_result = 0;
return 0;
}
unsigned long long pattern_ids[n_pattern_id];
for (size_t i = 0; i < n_pattern_id; i++) {
pattern_ids[i] = *(unsigned long long *)utarray_eltptr(matched_pat->pattern_ids, i);
}
struct bool_expr_match *bool_matcher_results = rs_rt->bool_match_buffs[thread_id];
int bool_matcher_ret = bool_matcher_match(rs_rt->bm, pattern_ids, n_pattern_id,
bool_matcher_results, MAX_HIT_EXPR_NUM);
if (bool_matcher_ret < 0) {
ret = -1;
goto next;
}
if (bool_matcher_ret > (int)n_result) {
bool_matcher_ret = n_result;
}
for (int index = 0; index < bool_matcher_ret; index++) {
results[index].rule_id = bool_matcher_results[index].expr_id;
results[index].user_tag = bool_matcher_results[index].user_tag;
}
*n_hit_result = bool_matcher_ret;
next:
utarray_clear(matched_pat->pattern_ids);
return ret;
}
int adapter_rs_scan(void *rs_instance, int thread_id, const char *data, size_t data_len,
struct expr_scan_result *results, size_t n_result, size_t *n_hit_result)
{
if (NULL == rs_instance || NULL == data || (0 == data_len) ||
NULL == results || 0 == n_result || NULL == n_hit_result) {
return -1;
}
int ret = 0, err_count = 0;
struct adapter_rs *rs_inst = (struct adapter_rs *)rs_instance;
struct adapter_rs_runtime *rs_rt = rs_inst->rs_rt;
struct matched_pattern *matched_pat = rs_rt->matched_pats[thread_id];
if (rs_rt->literal_db != NULL) {
ret = rs_scan(rs_rt->literal_db, thread_id, data, data_len,
0, matched_event_cb, matched_pat);
if (ret < 0) {
err_count++;
}
}
if (rs_rt->regex_db != NULL) {
ret = rs_scan(rs_rt->regex_db, thread_id, data, data_len,
0, matched_event_cb, matched_pat);
if (ret < 0) {
err_count++;
}
}
if (err_count == 2) {
return -1;
}
size_t n_pattern_id = utarray_len(matched_pat->pattern_ids);
if (0 == n_pattern_id) {
*n_hit_result = 0;
return 0;
}
unsigned long long pattern_ids[n_pattern_id];
for (size_t i = 0; i < n_pattern_id; i++) {
pattern_ids[i] = *(unsigned long long *)utarray_eltptr(matched_pat->pattern_ids, i);
}
struct bool_expr_match *bool_matcher_results = rs_rt->bool_match_buffs[thread_id];
int bool_matcher_ret = bool_matcher_match(rs_rt->bm, pattern_ids, n_pattern_id,
bool_matcher_results, MAX_HIT_EXPR_NUM);
if (bool_matcher_ret < 0) {
ret = -1;
goto next;
}
if (bool_matcher_ret > (int)n_result) {
bool_matcher_ret = n_result;
}
for (int index = 0; index < bool_matcher_ret; index++) {
results[index].rule_id = bool_matcher_results[index].expr_id;
results[index].user_tag = bool_matcher_results[index].user_tag;
}
*n_hit_result = bool_matcher_ret;
next:
utarray_clear(matched_pat->pattern_ids);
return ret;
}

78
scanner/expr_matcher/adapter_rs/adapter_rs.h Normal file
View File

@@ -0,0 +1,78 @@
/*
**********************************************************************************************
* File: adapter_rs.h
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2023-06-30
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#ifndef _ADAPTER_RS_H_
#define _ADAPTER_RS_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include "log/log.h"
#include "../expr_matcher.h"
int adapter_rs_verify_regex_expression(const char *regex_expr,
struct log_handle *logger);
/**
* @brief new adapter_rs instance
*
* @param rules: logic AND expression's array
* @param n_rule: the number of logic AND expression's array
* @param n_worker_threads: the number of scan threads which will call adapter_rs_scan()
*
* @retval the pointer to adapter_rs instance
*/
void *adapter_rs_new(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern,
size_t n_worker_thread, struct log_handle *logger);
void adapter_rs_free(void *rs_instance);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
*
* @param rs_instance: adapter_rs instance obtained by adapter_rs_new()
* @param thread_id: the thread_id of caller
* @param scan_data: data to be scanned
* @param data_len: the length of data to be scanned
* @param result_array: the array to store hit expr_id which allocated by caller
* @param n_result_array: number of elements in array of expr_id
*/
int adapter_rs_scan(void *rs_instance, int thread_id,
const char *scan_data, size_t data_len,
struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results);
/**
* @brief
*/
void *adapter_rs_stream_open(void *rs_instance, int thread_id);
/**
* @brief
*/
int adapter_rs_scan_stream(void *rs_stream, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results);
/**
* @brief
*/
void adapter_rs_stream_close(void *rs_stream);
#ifdef __cplusplus
}
#endif
#endif

235
scanner/expr_matcher/expr_matcher.cpp Normal file
View File

@@ -0,0 +1,235 @@
/*
**********************************************************************************************
* File: expr_matcher.cpp
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2023-06-30
* Copyright: (c) Since 2023 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#include <unistd.h>
#include <assert.h>
#include <sys/syscall.h>
#include "log/log.h"
#include "expr_matcher.h"
#include "maat_utils.h"
#include "adapter_hs/adapter_hs.h"
#include "adapter_rs/adapter_rs.h"
pid_t expr_matcher_gettid()
{
return syscall(SYS_gettid);
}
static const char *expr_matcher_module_name_str(const char *name)
{
static __thread char module[64];
snprintf(module, sizeof(module), "%s(%d)", name, expr_matcher_gettid());
return module;
}
#define MODULE_EXPR_MATCHER expr_matcher_module_name_str("maat.expr_matcher")
struct expr_matcher {
enum expr_engine_type engine_type;
void *engine;
struct log_handle *logger;
};
struct expr_matcher_stream {
enum expr_engine_type engine_type;
void *handle;
};
struct expr_engine_operations {
enum expr_engine_type type;
void *(*engine_new)(struct expr_rule *rules, size_t n_rule,
size_t n_literal_pattern, size_t n_regex_pattern,
size_t n_worker_thread, struct log_handle *logger);
void (*engine_free)(void *engine);
int (*engine_scan)(void *engine, int thread_id, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_result);
void *(*engine_stream_open)(void *engine, int thread_id);
void (*engine_stream_close)(void *stream);
int (*engine_scan_stream)(void *stream, const char *scan_data, size_t data_len,
struct expr_scan_result *result_array, size_t n_result_array,
size_t *n_hit_result);
};
struct expr_engine_operations expr_engine_ops[EXPR_ENGINE_TYPE_MAX] = {
{
.type = EXPR_ENGINE_TYPE_HS,
.engine_new = adapter_hs_new,
.engine_free = adapter_hs_free,
.engine_scan = adapter_hs_scan,
.engine_stream_open = adapter_hs_stream_open,
.engine_stream_close = adapter_hs_stream_close,
.engine_scan_stream = adapter_hs_scan_stream
},
{
.type = EXPR_ENGINE_TYPE_RS,
.engine_new = adapter_rs_new,
.engine_free = adapter_rs_free,
.engine_scan = adapter_rs_scan,
.engine_stream_open = adapter_rs_stream_open,
.engine_stream_close = adapter_rs_stream_close,
.engine_scan_stream = adapter_rs_scan_stream
}
};
int expr_matcher_verify_regex_expression(const char *regex_expr,
struct log_handle *logger)
{
int ret = adapter_hs_verify_regex_expression(regex_expr, logger);
if (ret == 0) {
return 0;
}
return adapter_rs_verify_regex_expression(regex_expr, logger);
}
struct expr_matcher *
expr_matcher_new(struct expr_rule *rules, size_t n_rule, enum expr_engine_type engine_type,
size_t n_worker_thread, struct log_handle *logger)
{
if (NULL == rules || 0 == n_rule || 0 == n_worker_thread ||
(engine_type != EXPR_ENGINE_TYPE_HS && engine_type != EXPR_ENGINE_TYPE_RS)) {
log_error(logger, MODULE_EXPR_MATCHER, "[%s:%d]engine type:%d is illegal",
__FUNCTION__, __LINE__, engine_type);
return NULL;
}
size_t i = 0, j = 0;
size_t literal_pat_num = 0;
size_t regex_pat_num = 0;
for (i = 0; i < n_rule; i++) {
if (rules[i].n_patterns > MAX_EXPR_PATTERN_NUM) {
log_error(logger, MODULE_EXPR_MATCHER,
"[%s:%d] the number of patterns in one expression should less than"
" %d", __FUNCTION__, __LINE__, MAX_EXPR_PATTERN_NUM);
return NULL;
}
for (j = 0; j < rules[i].n_patterns; j++) {
/* pat_len should not 0 */
if (0 == rules[i].patterns[j].pat_len) {
log_error(logger, MODULE_EXPR_MATCHER,
"[%s:%d] expr pattern length should not 0",
__FUNCTION__, __LINE__);
return NULL;
}
if (rules[i].patterns[j].type == EXPR_PATTERN_TYPE_STR) {
literal_pat_num++;
} else {
regex_pat_num++;
}
}
}
if (0 == literal_pat_num && 0 == regex_pat_num) {
log_error(logger, MODULE_EXPR_MATCHER,
"[%s:%d] exprs has no valid pattern", __FUNCTION__, __LINE__);
return NULL;
}
void *engine = expr_engine_ops[engine_type].engine_new(rules, n_rule, literal_pat_num,
regex_pat_num, n_worker_thread,
logger);
if (NULL == engine) {
log_error(logger, MODULE_EXPR_MATCHER,
"[%s:%d]expr_matcher engine_new failed.", __FUNCTION__, __LINE__);
return NULL;
}
struct expr_matcher *matcher = ALLOC(struct expr_matcher, 1);
matcher->engine_type = engine_type;
matcher->engine = engine;
matcher->logger = logger;
return matcher;
}
void expr_matcher_free(struct expr_matcher *matcher)
{
if (NULL == matcher) {
return;
}
if (matcher->engine != NULL) {
expr_engine_ops[matcher->engine_type].engine_free(matcher->engine);
matcher->engine = NULL;
}
FREE(matcher);
}
int expr_matcher_match(struct expr_matcher *matcher, int thread_id, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results)
{
if (NULL == matcher || thread_id < 0 || NULL == scan_data || 0 == data_len
|| NULL == result_array || 0 == n_result_array || NULL == n_hit_results) {
return -1;
}
return expr_engine_ops[matcher->engine_type].engine_scan(matcher->engine, thread_id,
scan_data, data_len, result_array,
n_result_array, n_hit_results);
}
struct expr_matcher_stream *
expr_matcher_stream_open(struct expr_matcher *matcher, int thread_id)
{
if (NULL == matcher || thread_id < 0) {
return NULL;
}
void *s_handle = expr_engine_ops[matcher->engine_type].engine_stream_open(matcher->engine,
thread_id);
if (NULL == s_handle) {
log_error(matcher->logger, MODULE_EXPR_MATCHER,
"[%s:%d] expr_matcher engine_stream_open failed.",
__FUNCTION__, __LINE__);
return NULL;
}
struct expr_matcher_stream *stream = ALLOC(struct expr_matcher_stream, 1);
stream->engine_type = matcher->engine_type;
stream->handle = s_handle;
return stream;
}
int expr_matcher_stream_match(struct expr_matcher_stream *stream, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results)
{
if (NULL == stream || NULL == scan_data || 0 == data_len || NULL == result_array
|| 0 == n_result_array || NULL == n_hit_results) {
return -1;
}
return expr_engine_ops[stream->engine_type].engine_scan_stream(stream->handle, scan_data,
data_len, result_array,
n_result_array, n_hit_results);
}
void expr_matcher_stream_close(struct expr_matcher_stream *stream)
{
if (NULL == stream) {
return;
}
if (stream->handle != NULL) {
expr_engine_ops[stream->engine_type].engine_stream_close(stream->handle);
stream->handle = NULL;
}
FREE(stream);
}

134
scanner/expr_matcher/expr_matcher.h Normal file
View File

@@ -0,0 +1,134 @@
/*
**********************************************************************************************
* File: expr_matcher.h
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2023-06-30
* Copyright: (c) Since 2023 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#ifndef _EXPR_MATCHER_H_
#define _EXPR_MATCHER_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stddef.h>
#include "log/log.h"
#define MAX_EXPR_PATTERN_NUM 8 /* 每条与表达式最多由MAX_EXPR_ITEM_NUM个规则组成 */
#define MAX_HIT_EXPR_NUM 1024
enum expr_engine_type {
EXPR_ENGINE_TYPE_HS = 0, /* default engine */
EXPR_ENGINE_TYPE_RS,
EXPR_ENGINE_TYPE_MAX
};
enum expr_pattern_type {
EXPR_PATTERN_TYPE_STR = 0, /* pure literal string */
EXPR_PATTERN_TYPE_REG = 1, /* regex expression */
};
enum expr_case_sensitive {
EXPR_CASE_INSENSITIVE = 0,
EXPR_CASE_SENSITIVE
};
enum expr_match_mode {
EXPR_MATCH_MODE_INVALID = -1,
EXPR_MATCH_MODE_EXACTLY = 1, /* scan data must match pattern exactly */
EXPR_MATCH_MODE_PREFIX, /* pattern must in the head of scan_data */
EXPR_MATCH_MODE_SUFFIX, /* pattern must in the tail of scan_data */
EXPR_MATCH_MODE_SUB /* pattern must in the range[l_offset, r_offset] of scan_data */
};
struct expr_pattern {
enum expr_pattern_type type;
enum expr_match_mode match_mode;
enum expr_case_sensitive case_sensitive;
/*
* just match in scan_data's range of [start_offset, end_offset], -1 means no limits
* for example:
* [-1, end_offset] means the pattern must in scan_data's [0 ~ start_offset]
* [start_offset, -1] means the pattern must in scan_data's [start_offset ~ data_end]
*/
int start_offset;
int end_offset;
char *pat;
size_t pat_len;
};
struct expr_scan_result {
long long rule_id;
void *user_tag;
};
/* logic AND expression, such as (rule1 & rule2) */
struct expr_rule {
long long expr_id; /* AND expression ID */
size_t n_patterns;
struct expr_pattern patterns[MAX_EXPR_PATTERN_NUM];
void *tag; /* user defined data, return with hit result */
};
int expr_matcher_verify_regex_expression(const char *regex_expr,
struct log_handle *logger);
/**
* @brief new expr matcher instance
*
* @param expr_array: logic AND expression's array
* @param n_expr_array: the number of logic AND expression's array
* @param n_worker_threads: the number of scan threads which will call adapter_rs_scan()
*
*/
struct expr_matcher *
expr_matcher_new(struct expr_rule *rules, size_t n_rule, enum expr_engine_type type,
size_t n_worker_thread, struct log_handle *logger);
void expr_matcher_free(struct expr_matcher *matcher);
/**
* @brief scan input data to match logic AND expression, return all matched expr_id
*
* @param matcher: expr_matcher instance obtained by expr_matcher_new()
* @param thread_id: the thread_id of caller
* @param scan_data: data to be scanned
* @param data_len: the length of data to be scanned
* @param result_array: the array to store hit expr_id which allocated by caller
* @param n_result_array: number of elements in array of expr_id
*/
int expr_matcher_match(struct expr_matcher *matcher, int thread_id, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results);
/**
* @brief
*/
struct expr_matcher_stream *
expr_matcher_stream_open(struct expr_matcher *matcher, int thread_id);
/**
* @brief
*/
int expr_matcher_stream_match(struct expr_matcher_stream *stream, const char *scan_data,
size_t data_len, struct expr_scan_result *result_array,
size_t n_result_array, size_t *n_hit_results);
/**
* @brief
*/
void expr_matcher_stream_close(struct expr_matcher_stream *stream);
#ifdef __cplusplus
}
#endif
#endif

4
scanner/fqdn_engine/fqdn_engine.cpp
View File

@@ -37,12 +37,12 @@
#define FOR(i, n) for(int i=0, _n=(int)(n); i<_n; i++)
struct packedRT_t
typedef struct packedRT
{
unsigned long long bitmap[4];
unsigned int A;
unsigned char B[4];
};
}packedRT_t;
static void * aligned_malloc(size_t size, size_t align)
{

37
scanner/ip_matcher/IntervalIndex/SuccinctHash.cpp
View File

@@ -36,6 +36,43 @@ inline unsigned int myhash(unsigned int key)
return h;
}
static void * aligned_malloc(size_t size, size_t align)
{
void * malloc_ptr;
void * aligned_ptr;
/* Error if align is not a power of two. */
if (align & (align - 1))
{
return ((void*) 0);
}
if (align==0 || size == 0)
{
return ((void *) 0);
}
malloc_ptr = malloc (sizeof(void *) + align - 1 + size);
if (!malloc_ptr)
{
return ((void *) 0);
}
aligned_ptr = (void *) (((size_t)malloc_ptr + sizeof(void *) + align-1) & ~(align-1));
((void **) aligned_ptr) [-1] = malloc_ptr;
return aligned_ptr;
}
static void aligned_free(void * aligned_ptr)
{
if (aligned_ptr)
{
free (((void **) aligned_ptr) [-1]);
}
}
CSuccinctHash::CSuccinctHash()
{
m_RT=NULL;

7
scanner/ip_matcher/IntervalIndex/SuccinctHash.h
View File

@@ -20,6 +20,13 @@
#include "sigmastar_tools.h"
struct packedRT_t
{
unsigned long long bitmap[4];
unsigned int A;
unsigned char B[4];
};
class CSuccinctHash
{
public:

59
scanner/ip_matcher/IntervalIndex/sigmastar_tools.cpp
View File

@@ -2,54 +2,7 @@
#include <stdlib.h>
#include <stdio.h>
void * aligned_malloc(size_t size, size_t align)
{
void * malloc_ptr;
void * aligned_ptr;
/* Error if align is not a power of two. */
if (align & (align - 1))
{
return ((void*) 0);
}
if (align==0 || size == 0)
{
return ((void *) 0);
}
malloc_ptr = malloc (sizeof(void *) + align - 1 + size);
if (!malloc_ptr)
{
return ((void *) 0);
}
aligned_ptr = (void *) (((size_t)malloc_ptr + sizeof(void *) + align-1) & ~(align-1));
((void **) aligned_ptr) [-1] = malloc_ptr;
return aligned_ptr;
}
void aligned_free(void * aligned_ptr)
{
if (aligned_ptr)
{
free (((void **) aligned_ptr) [-1]);
}
}
unsigned char popcnt_u16[65536];
unsigned char popcnt_u32(unsigned int x)
{
char c=0;
while(x)
{
if(x&1) c++;
x>>=1;
}
return c;
}
static unsigned char popcnt_u16[65536];
#ifndef USE_SSE_INSTR
int popcnt_u64(unsigned long long x)
@@ -60,13 +13,3 @@ int popcnt_u64(unsigned long long x)
+popcnt_u16[(x>>48)&0xFFFF];
}
#endif
int initialize_sigmastar_tools()
{
for(unsigned int i=0; i<65536; i++)
{
popcnt_u16[i]=popcnt_u32(i);
}
return 1;
}

13
scanner/ip_matcher/IntervalIndex/sigmastar_tools.h
View File

@@ -27,19 +27,6 @@ using namespace std;
//#define DEBUG_PARTITION
int initialize_sigmastar_tools();
struct packedRT_t
{
unsigned long long bitmap[4];
unsigned int A;
unsigned char B[4];
};
void * aligned_malloc(size_t size, size_t align);
void aligned_free(void * aligned_ptr);
#if (defined __linux__) && (defined __SSE4_2__)
#define USE_SSE_INSTR
#endif

4
src/CMakeLists.txt
View File

@@ -23,13 +23,15 @@ include_directories(/opt/MESA/include/MESA/)
include_directories(${PROJECT_SOURCE_DIR}/include/)
include_directories(${PROJECT_SOURCE_DIR}/deps/)
include_directories(${PROJECT_SOURCE_DIR}/scanner)
include_directories(${PROJECT_SOURCE_DIR}/scanner/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/fqdn_engine)
include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/flag_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/interval_matcher)
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_rs)
# Static Library Output
add_library(maat_frame_static STATIC ${MAAT_SRC} ${LIB_SOURCE_FILES})

9
src/inc_internal/maat_expr.h
View File

@@ -52,14 +52,13 @@ long long expr_runtime_get_version(void *expr_runtime);
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int vtable_id, struct maat_state *state);
struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
struct expr_matcher_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct expr_matcher_stream *s_handle,
const char *data, size_t data_len, int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,
struct adapter_hs_stream *s_handle);
struct expr_matcher_stream *stream);
int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district,
size_t district_len, long long *district_id);

1
src/inc_internal/maat_rule.h
View File

@@ -128,6 +128,7 @@ struct maat_options {
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
enum maat_expr_engine expr_engine;
enum data_source input_mode;
union {
struct source_iris_ctx iris_ctx;

6
src/inc_internal/maat_table.h
View File

@@ -19,6 +19,7 @@ extern "C"
#include <stddef.h>
#include <cJSON/cJSON.h>
#include "maat.h"
#include "log/log.h"
#include "maat_garbage_collection.h"
@@ -47,7 +48,8 @@ struct table_manager;
struct table_manager *
table_manager_create(const char *table_info_path, const char *accept_tags,
struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
enum maat_expr_engine expr_engine, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin);
@@ -67,6 +69,8 @@ int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr);
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr);
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr);
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags);

48
src/maat_api.c
View File

@@ -51,7 +51,7 @@ enum district_flag {
struct maat_stream {
struct maat *ref_maat_inst;
struct adapter_hs_stream *handle; //each physical table open one stream
struct expr_matcher_stream *handle; //each physical table open one stream
long long last_full_version;
long long expr_rt_version;
struct log_handle *logger;
@@ -70,6 +70,7 @@ struct maat_options* maat_options_new(void)
options->rule_update_checking_interval_ms = 1 * 1000;
options->gc_timeout_ms = 10 * 1000;
options->input_mode = DATA_SOURCE_NONE;
options->expr_engine = MAAT_EXPR_ENGINE_HS;
options->log_level = 0;
return options;
@@ -254,6 +255,19 @@ int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filen
return 0;
}
int maat_options_set_expr_engine(struct maat_options *opts,
enum maat_expr_engine expr_engine)
{
if (NULL == opts ||
(expr_engine != MAAT_EXPR_ENGINE_HS && expr_engine != MAAT_EXPR_ENGINE_RS)) {
return -1;
}
opts->expr_engine = expr_engine;
return 0;
}
int maat_options_set_logger(struct maat_options *opts, const char *log_path,
enum log_level level)
{
@@ -357,7 +371,8 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
pthread_mutex_init(&(maat_inst->background_update_mutex), NULL);
maat_inst->tbl_mgr = table_manager_create(table_info_path, maat_inst->opts.accept_tags,
maat_inst->garbage_bin, maat_inst->logger);
maat_inst->opts.expr_engine, maat_inst->garbage_bin,
maat_inst->logger);
if (NULL == maat_inst->tbl_mgr) {
goto failed;
}
@@ -410,12 +425,7 @@ int maat_helper_verify_regex_expression(const char *regex_expr)
return 0;
}
int ret = adapter_hs_verify_regex_expression(regex_expr, NULL);
if (ret < 0) {
return 0;
} else {
return 1;
}
return expr_matcher_verify_regex_expression(regex_expr, NULL);
}
int maat_get_table_id(struct maat *maat_inst, const char *table_name)
@@ -1168,9 +1178,7 @@ int maat_scan_flag(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = flag_scan(maat_inst->tbl_mgr, state->thread_id, flag,
phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1178,6 +1186,8 @@ int maat_scan_flag(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1257,9 +1267,7 @@ int maat_scan_integer(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = interval_scan(maat_inst->tbl_mgr, state->thread_id, integer,
phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1267,6 +1275,8 @@ int maat_scan_integer(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1346,9 +1356,7 @@ int maat_scan_ipv4(struct maat *maat_inst, int table_id, uint32_t ip_addr,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = ipv4_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr,
port, protocol, phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1356,6 +1364,8 @@ int maat_scan_ipv4(struct maat *maat_inst, int table_id, uint32_t ip_addr,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1436,9 +1446,7 @@ int maat_scan_ipv6(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = ipv6_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr,
port, protocol, phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1446,6 +1454,8 @@ int maat_scan_ipv6(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1525,9 +1535,7 @@ int maat_scan_string(struct maat *maat_inst, int table_id, const char *data,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = string_scan(maat_inst->tbl_mgr, state->thread_id, data,
data_len, phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1535,6 +1543,8 @@ int maat_scan_string(struct maat *maat_inst, int table_id, const char *data,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1609,7 +1619,7 @@ struct maat_stream *maat_stream_new(struct maat *maat_inst, int table_id,
stream->expr_rt_version = expr_runtime_get_version(expr_rt);
maat_runtime_ref_inc(maat_inst->maat_rt, state->thread_id);
struct adapter_hs_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt,
struct expr_matcher_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt,
state->thread_id);
if (NULL == handle) {
goto error;

8
src/maat_bool_plugin.c
View File

@@ -476,6 +476,10 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
"update %zu bool_plugin rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
} else {
log_info(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"table[%s] commit %zu bool_plugin rules and rebuild bool_matcher"
" completed, version:%lld", table_name, rule_cnt, maat_rt_version);
}
}
@@ -488,10 +492,6 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
bool_plugin_rt->rule_num = rule_cnt;
log_info(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"table[%s] commit %zu bool_plugin rules and rebuild bool_matcher completed"
", version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) {
FREE(rules);
}

1
src/maat_config_monitor.c
View File

@@ -151,7 +151,6 @@ static int cm_read_table_file(struct cm_table_info_t *index,
size_t file_offset = 0;
char line[MAX_CONFIG_LINE] = {0};
read_nxt_line_from_buff(file_buff, file_sz, &file_offset, line, sizeof(line));
int cfg_num = 0;
sscanf(line, "%d\n", &cfg_num);

123
src/maat_expr.c
View File

@@ -60,7 +60,7 @@ struct expr_item {
long long group_id;
char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type;
enum hs_match_mode match_mode;
enum expr_match_mode match_mode;
int is_hexbin;
int is_case_sensitive;
void *user_data;
@@ -68,7 +68,7 @@ struct expr_item {
};
struct expr_runtime {
struct adapter_hs *hs;
struct expr_matcher *matcher;
struct rcu_hash_table *item_hash; // <item_id, struct expr_item>
long long version; //expr_rt version
@@ -79,6 +79,7 @@ struct expr_runtime {
struct log_handle *logger;
struct maat_garbage_bin *ref_garbage_bin;
enum maat_expr_engine expr_engine;
int district_num;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
@@ -114,22 +115,22 @@ static enum expr_type int_to_expr_type(int expr_type)
return type;
}
static enum hs_match_mode int_to_match_mode(int match_method)
static enum expr_match_mode int_to_match_mode(int match_method)
{
enum hs_match_mode mode = HS_MATCH_MODE_INVALID;
enum expr_match_mode mode = EXPR_MATCH_MODE_INVALID;
switch (match_method) {
case 0:
mode = HS_MATCH_MODE_SUB;
mode = EXPR_MATCH_MODE_SUB;
break;
case 1:
mode = HS_MATCH_MODE_SUFFIX;
mode = EXPR_MATCH_MODE_SUFFIX;
break;
case 2:
mode = HS_MATCH_MODE_PREFIX;
mode = EXPR_MATCH_MODE_PREFIX;
break;
case 3:
mode = HS_MATCH_MODE_EXACTLY;
mode = EXPR_MATCH_MODE_EXACTLY;
break;
default:
break;
@@ -234,8 +235,8 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name,
__FUNCTION__, __LINE__, table_name, line);
goto error;
} else if (expr_item->expr_type == EXPR_TYPE_REGEX) {
ret = adapter_hs_verify_regex_expression(expr_item->keywords, expr_rt->logger);
if (ret < 0) {
ret = expr_matcher_verify_regex_expression(expr_item->keywords, expr_rt->logger);
if (0 == ret) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> regex expression(item_id:%lld):%s illegal,"
" will be dropped", __FUNCTION__, __LINE__, table_name,
@@ -277,7 +278,7 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name,
match_method_type = atoi(line + column_offset);
expr_item->match_mode = int_to_match_mode(match_method_type);
if (expr_item->match_mode == HS_MATCH_MODE_INVALID) {
if (expr_item->match_mode == EXPR_MATCH_MODE_INVALID) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> has invalid match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line);
@@ -472,12 +473,14 @@ void *expr_runtime_new(void *expr_schema, size_t max_thread_num,
return NULL;
}
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->item_hash = rcu_hash_new(expr_item_free_cb, NULL, 0);
expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger;
expr_rt->expr_engine = table_manager_get_expr_engine(schema->ref_tbl_mgr);
expr_rt->district_map = maat_kv_store_new();
expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
@@ -495,9 +498,9 @@ void expr_runtime_free(void *expr_runtime)
}
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
if (expr_rt->hs != NULL) {
adapter_hs_free(expr_rt->hs);
expr_rt->hs = NULL;
if (expr_rt->matcher != NULL) {
expr_matcher_free(expr_rt->matcher);
expr_rt->matcher = NULL;
}
if (expr_rt->item_hash != NULL) {
@@ -558,18 +561,18 @@ static int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
return 0;
}
static enum hs_pattern_type expr_type2pattern_type(enum expr_type expr_type)
static enum expr_pattern_type expr_type2pattern_type(enum expr_type expr_type)
{
enum hs_pattern_type pattern_type;
enum expr_pattern_type pattern_type = EXPR_PATTERN_TYPE_STR;
switch (expr_type) {
case EXPR_TYPE_STRING:
case EXPR_TYPE_AND:
case EXPR_TYPE_OFFSET:
pattern_type = HS_PATTERN_TYPE_STR;
pattern_type = EXPR_PATTERN_TYPE_STR;
break;
case EXPR_TYPE_REGEX:
pattern_type = HS_PATTERN_TYPE_REG;
pattern_type = EXPR_PATTERN_TYPE_REG;
break;
default:
break;
@@ -686,12 +689,12 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
}
sub_expr_cnt = i;
break;
case EXPR_TYPE_STRING:
case EXPR_TYPE_STRING: //AND/OFFSET/STRING type expression use \b to represent blank(' ')
sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords;
sub_key_array[0] = str_unescape(sub_key_array[0]);
break;
case EXPR_TYPE_REGEX:
case EXPR_TYPE_REGEX: //only regex type expression use \s to represent blank(' ')
sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords;
break;
@@ -710,15 +713,15 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
if (TRUE == expr_item->is_case_sensitive) {
// insensitive
expr_rule->patterns[i].case_sensitive = HS_CASE_SENSITIVE;
expr_rule->patterns[i].case_sensitive = EXPR_CASE_SENSITIVE;
} else {
expr_rule->patterns[i].case_sensitive = HS_CASE_INSENSITIVE;
expr_rule->patterns[i].case_sensitive = EXPR_CASE_INSENSITIVE;
}
expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type);
expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type);
if (TRUE == expr_item->is_hexbin &&
expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) {
expr_rule->patterns[i].type != EXPR_PATTERN_TYPE_REG) {
region_str_len = strlen(sub_key_array[i]) * 8 + 1;
region_string = ALLOC(char, region_str_len);
region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]),
@@ -738,13 +741,13 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
}
expr_rule->patterns[i].match_mode = expr_item->match_mode;
if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) {
if (expr_rule->patterns[i].match_mode == EXPR_MATCH_MODE_SUB) {
expr_rule->patterns[i].start_offset = key_left_offset[i];
expr_rule->patterns[i].end_offset = key_right_offset[i];
}
}
expr_rule->expr_id = expr_item->item_id;
expr_rule->user_tag = expr_item->user_data;
expr_rule->tag = expr_item->user_data;
expr_rule->n_patterns = sub_expr_cnt;
return 0;
@@ -810,10 +813,10 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
return 0;
}
static void garbage_adapter_hs_free(void *adapter_hs, void *arg)
static void garbage_expr_matcher_free(void *expr_matcher, void *arg)
{
struct adapter_hs *hs = (struct adapter_hs *)adapter_hs;
adapter_hs_free(hs);
struct expr_matcher *matcher = (struct expr_matcher *)expr_matcher;
expr_matcher_free(matcher);
}
int expr_runtime_commit(void *expr_runtime, const char *table_name,
@@ -864,38 +867,42 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
}
}
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
struct expr_matcher *new_matcher = NULL;
struct expr_matcher *old_matcher = NULL;
if (rule_cnt > 0) {
new_adapter_hs = adapter_hs_new(rules, real_rule_cnt, expr_rt->n_worker_thread,
expr_rt->logger);
if (NULL == new_adapter_hs) {
enum expr_engine_type engine_type = EXPR_ENGINE_TYPE_HS;
if (expr_rt->expr_engine == MAAT_EXPR_ENGINE_RS) {
engine_type = EXPR_ENGINE_TYPE_RS;
}
new_matcher = expr_matcher_new(rules, real_rule_cnt, engine_type,
expr_rt->n_worker_thread, expr_rt->logger);
if (NULL == new_matcher) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild adapter_hs engine failed when update"
"[%s:%d] table[%s] rebuild expr_matcher failed when update"
" %zu expr rules", __FUNCTION__, __LINE__, table_name, real_rule_cnt);
ret = -1;
} else {
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, maat_rt_version);
}
}
old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs;
old_matcher = expr_rt->matcher;
expr_rt->matcher = new_matcher;
rcu_hash_commit(expr_rt->item_hash);
if (old_adapter_hs != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL,
garbage_adapter_hs_free);
if (old_matcher != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_matcher, NULL, garbage_expr_matcher_free);
}
expr_rt->rule_num = real_rule_cnt;
expr_rt->regex_rule_num = real_regex_rule_cnt;
expr_rt->version = maat_rt_version;
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, expr_rt->version);
if (rules != NULL) {
for (i = 0; i < rule_cnt; i++) {
expr_rule_reset(&rules[i]);
@@ -949,15 +956,14 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
return 0;
}
if (NULL == expr_rt->hs) {
if (NULL == expr_rt->matcher) {
return 0;
}
size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item);
struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = expr_matcher_match(expr_rt->matcher, thread_id, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
if (ret < 0) {
return -1;
}
@@ -1000,14 +1006,15 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
return real_hit_item_cnt;
}
struct adapter_hs_stream *
struct expr_matcher_stream *
expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
{
if (NULL == expr_rt || thread_id < 0) {
return NULL;
}
struct adapter_hs_stream *stream = adapter_hs_stream_open(expr_rt->hs, thread_id);
struct expr_matcher_stream *stream = expr_matcher_stream_open(expr_rt->matcher,
thread_id);
if (NULL == stream) {
return NULL;
}
@@ -1016,7 +1023,7 @@ expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
}
int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
struct adapter_hs_stream *s_handle,
struct expr_matcher_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state)
{
@@ -1026,9 +1033,9 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
}
size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results,
int ret = expr_matcher_stream_match(s_handle, data, data_len, hit_results,
MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
if (ret < 0) {
return -1;
@@ -1067,13 +1074,13 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,
struct adapter_hs_stream *s_handle)
struct expr_matcher_stream *stream)
{
if (NULL == expr_rt || thread_id < 0 || NULL == s_handle) {
if (NULL == expr_rt || thread_id < 0 || NULL == stream) {
return;
}
adapter_hs_stream_close(s_handle);
expr_matcher_stream_close(stream);
}
void expr_runtime_hit_inc(struct expr_runtime *expr_rt, int thread_id)

8
src/maat_flag.c
View File

@@ -497,6 +497,10 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
"when update %zu flag rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
} else {
log_info(flag_rt->logger, MODULE_FLAG,
"table[%s] commit %zu flag rules and rebuild flag_matcher completed,"
" version:%lld", table_name, rule_cnt, maat_rt_version);
}
}
@@ -511,10 +515,6 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
flag_rt->rule_num = rule_cnt;
log_info(flag_rt->logger, MODULE_FLAG,
"table[%s] commit %zu flag rules and rebuild flag_matcher completed,"
" version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) {
FREE(rules);
}

8
src/maat_fqdn_plugin.c
View File

@@ -481,6 +481,10 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
" %zu fqdn_plugin rules", __FUNCTION__, __LINE__, table_name,
rule_cnt);
ret = -1;
} else {
log_info(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"table[%s] commit %zu fqdn_plugin rules and rebuild FQDN engine"
" completed, version:%lld", table_name, rule_cnt, maat_rt_version);
}
}
@@ -493,10 +497,6 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
fqdn_plugin_rt->rule_num = rule_cnt;
log_info(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"table[%s] commit %zu fqdn_plugin rules and rebuild FQDN engine completed"
", version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) {
FREE(rules);
}

8
src/maat_interval.c
View File

@@ -498,6 +498,10 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name,
"when update %zu interval rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
} else {
log_info(interval_rt->logger, MODULE_INTERVAL,
"table[%s] commit %zu interval rules and rebuild interval_matcher "
"completed, version:%lld", table_name, rule_cnt, maat_rt_version);
}
}
@@ -512,10 +516,6 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name,
interval_rt->rule_num = rule_cnt;
log_info(interval_rt->logger, MODULE_INTERVAL,
"table[%s] commit %zu interval rules and rebuild interval_matcher "
"completed, version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) {
FREE(rules);
}

8
src/maat_ip.c
View File

@@ -612,6 +612,10 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
"when update %zu ip rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
} else {
log_info(ip_rt->logger, MODULE_IP,
"table[%s] commit %zu ip rules and rebuild ip_matcher completed"
", version:%lld", table_name, rule_cnt, maat_rt_version);
}
new_intval_matcher = interval_matcher_new(intval_rules, rule_cnt);
@@ -642,10 +646,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
ip_rt->rule_num = rule_cnt;
log_info(ip_rt->logger, MODULE_IP,
"table[%s] commit %zu ip rules and rebuild ip_matcher completed"
", version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) {
FREE(rules);
}

8
src/maat_ip_plugin.c
View File

@@ -520,6 +520,10 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
"[%s:%d] ip_plugin table[%s] rebuild ip_matcher failed when "
"update %zu rules", __FUNCTION__, __LINE__, table_name, rule_cnt);
ret = -1;
} else {
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] commit %zu ip_plugin rules and rebuild ip_matcher "
"completed, version:%lld", table_name, rule_cnt, maat_rt_version);
}
}
@@ -532,10 +536,6 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
ip_plugin_rt->rule_num = rule_cnt;
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] commit %zu ip_plugin rules and rebuild ip_matcher "
"completed, version:%lld", table_name, rule_cnt, maat_rt_version);
if (rules != NULL) {
FREE(rules);
}

14
src/maat_table.c
View File

@@ -47,6 +47,7 @@ struct table_manager {
struct rule_tag *accept_tags;
size_t n_accept_tag;
enum maat_expr_engine expr_engine;
int default_compile_table_id;
int g2g_table_id;
struct maat_kv_store *tablename2id_map;
@@ -661,7 +662,8 @@ static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id
struct table_manager *
table_manager_create(const char *table_info_path, const char *accept_tags,
struct maat_garbage_bin *garbage_bin, struct log_handle *logger)
enum maat_expr_engine expr_engine, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == table_info_path) {
return NULL;
@@ -702,6 +704,7 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger);
tbl_mgr->logger = logger;
tbl_mgr->tablename2id_map = maat_kv_store_new();
tbl_mgr->expr_engine = expr_engine;
tbl_mgr->ref_garbage_bin = garbage_bin;
for (int i = 0; i < json_array_size; i++) {
@@ -1018,6 +1021,15 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->valid_column;
}
enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr)
{
if (NULL == tbl_mgr) {
return MAAT_EXPR_ENGINE_HS;
}
return tbl_mgr->expr_engine;
}
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr)
{
if (NULL == tbl_mgr) {

9
test/CMakeLists.txt
View File

@@ -3,7 +3,9 @@ include_directories(${PROJECT_SOURCE_DIR}/include)
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
include_directories(${PROJECT_SOURCE_DIR}/deps)
include_directories(${PROJECT_SOURCE_DIR}/scanner)
include_directories(${PROJECT_SOURCE_DIR}/scanner/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_rs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher)
@@ -19,8 +21,8 @@ target_link_libraries(maat_framework_gtest maat_frame_static gtest_static)
add_executable(maat_framework_perf_gtest maat_framework_perf_gtest.cpp)
target_link_libraries(maat_framework_perf_gtest maat_frame_static gtest_static)
add_executable(adapter_hs_gtest adapter_hs_gtest.cpp)
target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static)
add_executable(expr_matcher_gtest expr_matcher_gtest.cpp)
target_link_libraries(expr_matcher_gtest maat_frame_static gtest_static)
add_executable(ip_matcher_gtest ip_matcher_gtest.cpp)
target_link_libraries(ip_matcher_gtest maat_frame_static gtest_static)
@@ -39,7 +41,6 @@ file(COPY file_test_tableinfo.conf DESTINATION ./)
file(COPY literal_expr.conf DESTINATION ./)
file(COPY regex_expr.conf DESTINATION ./)
file(COPY maat_json.json DESTINATION ./)
file(COPY maat_json.json DESTINATION ../tools/)
file(COPY ntcrule DESTINATION ./)
file(COPY tsgrule DESTINATION ./)
file(COPY testdata DESTINATION ./)

730
test/adapter_hs_gtest.cpp
View File

@@ -1,730 +0,0 @@
#include <gtest/gtest.h>
#include "log/log.h"
#include "adapter_hs.h"
#include "maat_utils.h"
#include "cJSON/cJSON.h"
struct log_handle *g_logger = NULL;
enum hs_match_mode match_method_to_match_mode(const char *method)
{
enum hs_match_mode mode = HS_MATCH_MODE_INVALID;
if (strcmp(method, "sub") == 0) {
mode = HS_MATCH_MODE_SUB;
} else if (strcmp(method, "exactly") == 0) {
mode = HS_MATCH_MODE_EXACTLY;
} else if (strcmp(method, "prefix") == 0) {
mode = HS_MATCH_MODE_PREFIX;
} else if (strcmp(method, "suffix") == 0) {
mode = HS_MATCH_MODE_SUFFIX;
} else {
assert(0);
}
return mode;
}
enum hs_case_sensitive case_sensitive_str_to_enum(const char *str)
{
enum hs_case_sensitive case_sensitive = HS_CASE_SENSITIVE;
if (strcmp(str, "yes") == 0) {
case_sensitive = HS_CASE_SENSITIVE;
} else if (strcmp(str, "no") == 0) {
case_sensitive = HS_CASE_INSENSITIVE;
} else {
assert(0);
}
return case_sensitive;
}
int is_hexbin_str_to_int(const char *str)
{
int ret = 0;
if (strcmp(str, "yes") == 0) {
ret = 1;
}
return ret;
}
static int convertHextoint(char srctmp)
{
if (isdigit(srctmp)) {
return srctmp - '0';
} else {
char temp = toupper(srctmp);
temp = temp - 'A' + 10;
return temp;
}
}
static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size)
{
size_t resultlen = 0;
int high,low;
for (int i = 0; i < hex_len && size > resultlen; i += 2, resultlen++) {
high = convertHextoint(hex[i]);
low = convertHextoint(hex[i+1]);
binary[resultlen] = high * 16 + low;
}
size = resultlen;
binary[resultlen] = '\0';
return resultlen;
}
enum hs_pattern_type pattern_type_str_to_enum(const char *str)
{
enum hs_pattern_type pattern_type;
if (strcmp(str, "regex") == 0) {
pattern_type = HS_PATTERN_TYPE_REG;
} else if (strcmp(str, "literal") == 0) {
pattern_type = HS_PATTERN_TYPE_STR;
} else {
assert(0);
}
return pattern_type;
}
int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_expr)
{
unsigned char *json_buff = NULL;
size_t json_buff_size = 0;
int ret = load_file_to_memory(filename, &json_buff, &json_buff_size);
if (ret < 0) {
printf("load file:%s to memory failed.\n", filename);
return -1;
}
size_t rule_cnt = 0;
cJSON *rules_obj = NULL;
cJSON *root = cJSON_Parse((const char *)json_buff);
if (NULL == root) {
printf("Error before: %-200.200s\n", cJSON_GetErrorPtr());
ret = -1;
goto next;
}
rules_obj = cJSON_GetObjectItem(root, "expr_rules");
if (NULL == rules_obj) {
printf("Error before: %-200.200s\n", cJSON_GetErrorPtr());
ret = -1;
goto next;
}
rule_cnt = cJSON_GetArraySize(rules_obj);
for (size_t i = 0; i < rule_cnt; i++) {
cJSON *expr_obj = cJSON_GetArrayItem(rules_obj, i);
cJSON *tmp_item = cJSON_GetObjectItem(expr_obj, "expr_id");
if (tmp_item != NULL && tmp_item->type == cJSON_Number) {
exprs[i].expr_id = tmp_item->valueint;
}
tmp_item = cJSON_GetObjectItem(expr_obj, "pattern_num");
if (tmp_item != NULL && tmp_item->type == cJSON_Number) {
exprs[i].n_patterns = tmp_item->valueint;
}
tmp_item = cJSON_GetObjectItem(expr_obj, "patterns");
if (NULL == tmp_item || tmp_item->type != cJSON_Array) {
printf("json has no patterns array.\n");
ret = -1;
goto next;
}
size_t pattern_cnt = cJSON_GetArraySize(tmp_item);
for (size_t j = 0; j < pattern_cnt; j++) {
cJSON *pat_item = cJSON_GetArrayItem(tmp_item, j);
cJSON *item = cJSON_GetObjectItem(pat_item, "pattern_type");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].pattern_type = pattern_type_str_to_enum(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "match_method");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].match_mode = match_method_to_match_mode(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "case_sensitive");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].case_sensitive = case_sensitive_str_to_enum(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "is_hexbin");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].is_hexbin = is_hexbin_str_to_int(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "pattern");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].pat = ALLOC(char, strlen(item->valuestring) + 1);
if (exprs[i].patterns[j].is_hexbin == 1) {
size_t pat_str_len = strlen(item->valuestring) + 1;
char *pat_str = ALLOC(char, pat_str_len);
pat_str_len = hex2bin(item->valuestring, strlen(item->valuestring),
pat_str, pat_str_len);
memcpy(exprs[i].patterns[j].pat, pat_str, pat_str_len);
free(pat_str);
exprs[i].patterns[j].pat_len = pat_str_len;
} else {
memcpy(exprs[i].patterns[j].pat, item->valuestring,
strlen(item->valuestring));
exprs[i].patterns[j].pat_len = strlen(item->valuestring);
}
}
if (exprs[i].patterns->match_mode == HS_MATCH_MODE_SUB) {
item = cJSON_GetObjectItem(pat_item, "offset");
if (item != NULL && item->type == cJSON_String) {
int key_left_offset = -1;
int key_right_offset = -1;
sscanf(item->valuestring, "%d~%d", &key_left_offset, &key_right_offset);
if (key_left_offset < -1 || key_right_offset < -1) {
printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n",
key_left_offset, key_right_offset);
}
exprs[i].patterns[j].start_offset = key_left_offset;
exprs[i].patterns[j].end_offset = key_right_offset;
} else {
exprs[i].patterns[j].start_offset = -1;
exprs[i].patterns[j].end_offset = -1;
}
}
if (exprs[i].patterns->match_mode == HS_MATCH_MODE_EXACTLY) {
exprs[i].patterns[j].start_offset = 0;
exprs[i].patterns[j].end_offset = exprs[i].patterns[j].pat_len - 1;
}
}
exprs[i].n_patterns = pattern_cnt;
}
*n_expr = rule_cnt;
next:
cJSON_Delete(root);
FREE(json_buff);
return ret;
}
void expr_array_free(struct expr_rule rules[], size_t n_rule)
{
for (size_t i = 0; i < n_rule; i++) {
for (size_t j = 0; j < rules[i].n_patterns; j++) {
if (rules[i].patterns[j].pat != NULL) {
free(rules[i].patterns[j].pat);
rules[i].patterns[j].pat = NULL;
}
}
}
}
TEST(adapter_hs_init, invalid_input_parameter)
{
struct expr_rule rules[64];
size_t n_rule = 0;
struct adapter_hs *hs_instance = adapter_hs_new(NULL, 0, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
n_rule = 1;
rules[0].expr_id = 101;
rules[0].n_patterns = 10;
hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
memset(rules, 0, sizeof(rules));
n_rule = 1;
rules[0].expr_id = 101;
rules[0].n_patterns = 1;
hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance == NULL);
}
TEST(adapter_hs_scan, literal_sub_has_normal_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello aaa";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data2[64] = "Ahello aaa";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 101);
char scan_data3[64] = "Aahello aaa";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 101);
char scan_data4[64] = "Aaahello aaa";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_sub_has_left_unlimit_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello bbb";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 102);
char scan_data2[64] = "Ahello bbb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 102);
char scan_data3[64] = "Aahello bbb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 102);
char scan_data4[64] = "Aaahello bbb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_sub_has_right_unlimit_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello ccc";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data2[64] = "1234hello ccc";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data3[64] = "12345hello ccc";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 103);
char scan_data4[64] = "12345hello cccAaBb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 103);
char scan_data5[64] = "123456hello cccAaBb";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data5, strlen(scan_data5), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 103);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_sub_with_no_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello ddd";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 104);
char scan_data2[64] = "123hello ddd";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 104);
char scan_data3[64] = "123hello ddd456";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 104);
char scan_data4[64] = "helloddd";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_exactly)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello eee";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 105);
char scan_data2[64] = "Ahello eee";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data3[64] = "hello eeeB";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_prefix)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello fff";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 106);
char scan_data2[64] = "Ahello fff";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data3[64] = "Ahello fffBCD";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data4[64] = "hello fffBCD";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 106);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_suffix)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "hello ggg";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 107);
char scan_data2[64] = "ABChello ggg";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 107);
char scan_data3[64] = "ABChello gggDEF";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
char scan_data4[64] = "hello gggDEF";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_sub_with_hexbin)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char scan_data1[64] = "Content-Type: /html";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 108);
char scan_data2[64] = " html";
memset(result, 0, sizeof(result));
n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 0);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, literal_with_chinese)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char data0[64] = "#中国 你好";
struct hs_scan_result result0[64] = {0};
size_t n_result0 = 0;
ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result0, 1);
EXPECT_EQ(result0[0].rule_id, 110);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, same_pattern_different_offset)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
char data[64] = "onetoday,anothertoday";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, data, strlen(data), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 112);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, long_scan_data)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./literal_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(rules, n_rule, 1, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(rules, n_rule);
const char* scan_data = "A directed path in a directed graph is a finite or infinite\
sequence of edges which joins a sequence of distinct vertices, but with the added restriction\
that the edges be all directed in the same direction.";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data, strlen(scan_data), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].rule_id, 113);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
TEST(adapter_hs_scan, regex_expression_check)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./regex_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < n_rule; i++) {
for (size_t j = 0; j < rules[i].n_patterns; j++) {
adapter_hs_verify_regex_expression(rules[i].patterns[j].pat, g_logger);
}
}
expr_array_free(rules, n_rule);
}
int main(int argc, char **argv)
{
int ret = 0;
::testing::InitGoogleTest(&argc, argv);
g_logger = log_handle_create("./adapter_hs_gtest.log", 0);
ret = RUN_ALL_TESTS();
log_handle_destroy(g_logger);
return ret;
}

1330
test/expr_matcher_gtest.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

28
test/literal_expr.conf
View File

@@ -5,6 +5,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -18,6 +19,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -31,6 +33,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -44,6 +47,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -56,6 +60,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "exactly",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -68,6 +73,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "prefix",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -80,6 +86,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "suffix",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -92,6 +99,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "yes",
@@ -105,12 +113,14 @@
"pattern_num": 2,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
"pattern": "multi"
},
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -123,6 +133,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -135,6 +146,7 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -147,6 +159,7 @@
"pattern_num": 2,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -154,6 +167,7 @@
"offset": "3~7"
},
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
@@ -167,12 +181,26 @@
"pattern_num": 1,
"patterns": [
{
"pattern_type": "literal",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
"pattern": "a finite or infinite"
}
]
},
{
"expr_id": 114,
"pattern_num": 1,
"patterns": [
{
"pattern_type": "regex",
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
"pattern": "query=(.*)"
}
]
}
]
}

22
test/maat_demo/CMakeLists.txt
View File

@@ -1,22 +0,0 @@
include_directories(./)
add_library(gtest-static STATIC IMPORTED GLOBAL)
add_dependencies(gtest-static gtest)
set_property(TARGET gtest-static PROPERTY IMPORTED_LOCATION ${PROJECT_SOURCE_DIR}/lib/libgtest.a)
set_property(TARGET gtest-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PROJECT_SOURCE_DIR}/include)
add_library(maat-shared SHARED IMPORTED GLOBAL)
add_dependencies(maat-shared maat)
set_property(TARGET maat-shared PROPERTY IMPORTED_LOCATION ${PROJECT_SOURCE_DIR}/lib/libmaat4.so)
set_property(TARGET maat-shared PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${PROJECT_SOURCE_DIR}/include)
add_executable(maat_demo_gtest maat_demo_gtest.cpp)
target_link_libraries(maat_demo_gtest maat-shared gtest-static pthread)
file(COPY demo_table_info.conf DESTINATION ./)
file(COPY maat_demo.json DESTINATION ./)
file(COPY testdata DESTINATION ./)

129
test/maat_demo/demo_table_info.conf
View File

@@ -1,129 +0,0 @@
[
{
"table_id":0,
"table_name":"COMPILE",
"table_type":"compile",
"valid_column":8,
"custom": {
"gc_timeout_s": 3,
"compile_id":1,
"tags":6,
"clause_num":9
}
},
{
"table_id":1,
"table_name":"GROUP2COMPILE",
"table_type":"group2compile",
"associated_compile_table_id":0,
"valid_column":3,
"custom": {
"group_id":1,
"compile_id":2,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":2,
"table_name":"HTTP_URL",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":3,
"table_name":"KEYWORDS_TABLE",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":4,
"table_name":"HTTP_SIGNATURE",
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"item_id":1,
"group_id":2,
"district":3,
"keywords":4,
"expr_type":5,
"match_method":6,
"is_hexbin":7
}
},
{
"table_id":5,
"table_name":"IMAGE_FP",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":6,
"table_name":"APP_PAYLOAD",
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"item_id":1,
"group_id":2,
"district":3,
"keywords":4,
"expr_type":5,
"match_method":6,
"is_hexbin":7
}
},
{
"table_id":7,
"table_name":"TROJAN_PAYLOAD",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":8,
"table_name":"MAIL_ADDR",
"table_type":"expr",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
}
]

289
test/maat_demo/include/maat.h
View File

@@ -1,289 +0,0 @@
/*
**********************************************************************************************
* Maat: Deep Packet Inspection Policy Framework
* Maat is the Goddess of truth and justice in ancient Egyptian concept.
* Her feather was the measure that determined whether the souls (considered
* to reside in the heart) of the departed would reach the paradise of afterlife
* successfully.
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2023 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_H_
#define _MAAT_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <stdint.h>
#include <netinet/in.h>
/* maat instance handle */
struct maat;
struct maat_hit_path {
int Nth_scan;
int vtable_id; // 0 is not a virtual table.
long long item_id;
long long sub_group_id;
long long top_group_id;
long long compile_id;
};
struct maat_hit_group {
long long group_id;
int vtable_id;
};
enum maat_scan_status {
MAAT_SCAN_ERR = -1, //scan error
MAAT_SCAN_OK, //scan but not hit(group or compile)
MAAT_SCAN_HALF_HIT, //half hit: hit group, not hit compile
MAAT_SCAN_HIT //scan hit compile
};
enum maat_update_type {
MAAT_UPDATE_TYPE_INVALID = 0,
MAAT_UPDATE_TYPE_FULL,
MAAT_UPDATE_TYPE_INC
};
enum maat_list_type {
MAAT_LIST_TYPE_FULL = 1,
MAAT_LIST_TYPE_INC
};
struct ip_addr {
int ip_type; //4: IPv4, 6: IPv6
union {
unsigned int ipv4; //network order
unsigned int ipv6[4];
};
};
enum log_level {
LOG_LEVEL_TRACE,
LOG_LEVEL_DEBUG,
LOG_LEVEL_INFO,
LOG_LEVEL_WARN,
LOG_LEVEL_ERROR,
LOG_LEVEL_FATAL
};
/* update_type: MAAT_UPDATE_TYPE_FULL or MAAT_UPDATE_TYPE_INC */
typedef void maat_start_callback_t(int update_type, void *u_param);
typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para);
typedef void maat_finish_callback_t(void *u_para);
typedef void maat_ex_new_func_t(const char *table_name, int table_id, const char *key,
const char *table_line, void **ad, long argl, void *argp);
typedef void maat_ex_free_func_t(int table_id, void **ad, long argl, void *argp);
typedef void maat_ex_dup_func_t(int table_id, void **to, void **from, long argl, void *argp);
/* maat_instance options API */
struct maat_options;
struct maat_options *maat_options_new(void);
void maat_options_free(struct maat_options *opts);
/**
* @brief set maat instance name
*
* @note The maximum length of instance_name is 15 bytes
*/
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name);
int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread);
int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags);
int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms);
int maat_options_set_deferred_load_on(struct maat_options *opts);
int maat_options_set_stat_on(struct maat_options *opts);
int maat_options_set_perf_on(struct maat_options *opts);
int maat_options_set_foreign_cont_dir(struct maat_options *opts, const char *dir);
int maat_options_set_logger(struct maat_options *opts, const char *log_path,
enum log_level level);
int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
const char *increment_directory);
int maat_options_set_json_file(struct maat_options *opts, const char *json_filename);
/**
* Indicate whether the JSON file is compressed by gzip
* flag: 1(compressed) 0(uncompressed)
* */
int maat_options_set_json_file_gzip_flag(struct maat_options *opts, int flag);
/* Specify the decryption key for the JSON file to be decrypted */
int maat_options_set_json_file_decrypt_key(struct maat_options *opts, const char *decrypt_key);
int maat_options_set_redis(struct maat_options *opts, const char *redis_ip,
uint16_t redis_port, int redis_db);
int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filename);
/* maat_instance API */
struct maat *maat_new(struct maat_options *opts, const char *table_info_path);
void maat_free(struct maat *instance);
/* maat helper API */
int maat_helper_read_column(const char *table_line, int Nth_column,
size_t *column_offset, size_t *column_len);
/**
* verify if regex expression is legal
*
* @param The NULL-terminated expression to parse.
* @retval 1(legal) 0(illegal)
**/
int maat_helper_verify_regex_expression(const char *expression);
/* maat table API */
int maat_get_table_id(struct maat *instance, const char *table_name);
/* return 0 if success, otherwise return -1 */
int maat_table_callback_register(struct maat *instance, int table_id,
maat_start_callback_t *start,
maat_update_callback_t *update,
maat_finish_callback_t *finish,
void *u_para);
/* maat plugin table API */
int maat_plugin_table_ex_schema_register(struct maat *instance, const char *table_name,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp);
/**
* xx_plugin_table_get_ex_data
* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register,
* caller is responsible to free the data.
*
* free_func support gargbage collection(gc), gc timeout(default 0) can be configured
* in table_info which means maat will not call free_func until the timeout expires
*/
/**
* NOTE: only plugin table support three key type(integer, pointer, ip_addr)
* specified in table_info.conf. If use ip_addr key type, then key should be
* ip address in network order.
*/
void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id,
const char *key, size_t key_len);
int maat_ip_plugin_table_get_ex_data(struct maat *instance, int table_id,
const struct ip_addr *ip, void **ex_data_array,
size_t n_ex_data);
int maat_fqdn_plugin_table_get_ex_data(struct maat *instance, int table_id,
const char *fqdn, void **ex_data_array,
size_t n_ex_data);
int maat_bool_plugin_table_get_ex_data(struct maat *instance, int table_id,
unsigned long long *item_ids, size_t n_item,
void **ex_data_array, size_t n_ex_data);
/* maat scan API */
struct maat_state;
/**
* @param instance: maat instance created by maat_new()
* @param table_id: the id of table which to be scanned
* @param thread_id: thread index
* @param results: array to store hit compile id
* @param n_result: the array size
* @param n_hit_result: the number of hit compile id
* @param state: scan mid status
*
* @retval MAAT_SCAN_ERR
* MAAT_SCAN_OK
* MAAT_SCAN_HALF_HIT
* MAAT_SCAN_HIT
*/
int maat_scan_flag(struct maat *instance, int table_id,
long long flag, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state *state);
int maat_scan_integer(struct maat *instance, int table_id,
long long integer, long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state *state);
/**
* @param ip_addr: network ipv4 address
* @param port: network port
* @param protocol: -1(ANY protocol) 1(ICMP) 6(TCP) 17(UDP)
*/
int maat_scan_ipv4(struct maat *instance, int table_id,
uint32_t ip_addr, uint16_t port, int protocol,
long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state *state);
int maat_scan_ipv6(struct maat *instance, int table_id,
uint8_t *ip_addr, uint16_t port, int protocol,
long long *results, size_t n_result,
size_t *n_hit_result, struct maat_state *state);
int maat_scan_string(struct maat *instance, int table_id,
const char *data, size_t data_len, long long *results,
size_t n_result, size_t *n_hit_result,
struct maat_state *state);
struct maat_stream;
struct maat_stream *maat_stream_new(struct maat *instance, int table_id,
struct maat_state *state);
int maat_stream_scan(struct maat_stream *stream, const char *data, int data_len,
long long *results, size_t n_result, size_t *n_hit_result,
struct maat_state *state);
void maat_stream_free(struct maat_stream *stream);
/* maat state API */
struct maat_state *maat_state_new(struct maat *instance, int thread_id);
void maat_state_reset(struct maat_state *state);
void maat_state_free(struct maat_state *state);
int maat_state_set_scan_district(struct maat_state *state, int table_id,
const char *district, size_t district_len);
int maat_state_set_last_scan(struct maat_state *state);
int maat_state_set_scan_compile_table(struct maat_state *state, int compile_table_id);
int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths,
size_t n_path);
size_t maat_state_get_scan_count(struct maat_state *state);
int maat_state_get_hit_groups(struct maat_state *state, enum maat_list_type type,
struct maat_hit_group *groups, size_t n_group);
/* return hit object compile_id */
int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group);
#ifdef __cplusplus
}
#endif
#endif

56
test/maat_demo/include/maat_command.h
View File

@@ -1,56 +0,0 @@
/*
**********************************************************************************************
* File: maat_command.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_COMMAND_H_
#define _MAAT_COMMAND_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <limits.h>
#include "maat.h"
enum maat_operation {
MAAT_OP_DEL = 0,
MAAT_OP_ADD,
MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after
};
struct maat_cmd_line {
const char *table_name;
const char *table_line;
long long rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary.
int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout.
};
/**
* @brief write one line to redis
*
* @retval
* success: number of successfully updated rule.
* failed: -1
*/
int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule);
int maat_cmd_set_file(struct maat *maat_instance, const char *key, const char *value,
size_t size, enum maat_operation op);
long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int increment);
int maat_cmd_flushDB(struct maat *maat_instance);
#ifdef __cplusplus
}
#endif
#endif

BIN
test/maat_demo/lib/libgtest.a
View File

Binary file not shown.

1
test/maat_demo/lib/libmaat4.so
View File

@@ -1 +0,0 @@
libmaat4.so.4

1
test/maat_demo/lib/libmaat4.so.4
View File

@@ -1 +0,0 @@
libmaat4.so.4.0

BIN
test/maat_demo/lib/libmaat4.so.4.0
View File

Binary file not shown.

432
test/maat_demo/maat_demo.json
View File

@@ -1,432 +0,0 @@
{
"compile_table": "COMPILE",
"group2compile_table": "GROUP2COMPILE",
"group2group_table": "GROUP2GROUP",
"rules": [
{
"compile_id": 125,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "action=search\\&query=(.*)",
"expr_type": "regex",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 128,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "StringScan.ExprPlus",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_SIGNATURE",
"table_type": "expr_plus",
"table_content": {
"district": "HtTP\\bUrL",
"keywords": "abckkk&123",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 132,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "string\\bunescape",
"is_valid": "yes",
"groups": [
{
"group_name": "TakeMeHome",
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "expr",
"table_content": {
"keywords": "Take\\bme\\bHome&Batman\\",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 136,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "offset_string",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "IMAGE_FP",
"table_type": "expr",
"table_content": {
"keywords": "4362-4458:323031333A30333A30372032333A35363A313000323031333A30333A30372032333A35363A3130000000FFE20C584943435F50524F46494C4500010100000C484C696E6F021000006D6E74725247422058595A2007CE00020009000600310000",
"expr_type": "offset",
"match_method": "none",
"format": "hexbin"
}
}
]
}
]
},
{
"compile_id": 146,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "StringScan.Regex",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "Cookie:\\s.*head",
"expr_type": "regex",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 148,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"user_region": "StringScan.ExprPlusWithOffset",
"effective_rage": 0,
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "APP_PAYLOAD",
"table_type": "expr_plus",
"table_content": {
"format": "hexbin",
"match_method": "sub",
"district": "Payload",
"keywords": "1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d",
"expr_type": "offset"
}
}
]
}
]
},
{
"compile_id": 150,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "StringScan.BugReport20190325",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "TROJAN_PAYLOAD",
"table_content": {
"keywords": "0-4:01000000",
"expr_type": "offset",
"format": "hexbin",
"match_method": "sub"
}
}
],
"group_name": "billgates_regist1"
},
{
"regions": [
{
"table_type": "expr",
"table_name": "TROJAN_PAYLOAD",
"table_content": {
"keywords": "1:G2.40",
"expr_type": "none",
"format": "uncase plain",
"match_method": "sub"
}
}
],
"group_name": "billgates_regist2"
}
]
},
{
"compile_id": 151,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "StringScan.PrefixAndSuffix",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "MAIL_ADDR",
"table_content": {
"keywords": "ceshi3@mailhost.cn",
"expr_type": "none",
"format": "uncase plain",
"match_method": "suffix"
}
}
],
"group_name": "Untitled"
}
]
},
{
"compile_id": 156,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "ExprPlusWithHex",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_SIGNATURE",
"table_type": "expr_plus",
"table_content": {
"district": "Content-Type",
"keywords": "2f68746d6c",
"expr_type": "none",
"match_method": "sub",
"format": "hexbin"
}
}
]
}
]
},
{
"compile_id": 157,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "StringScan.StreamScanUTF8",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "TROJAN_PAYLOAD",
"table_content": {
"keywords": "我的订单",
"expr_type": "none",
"format": "none",
"match_method": "sub"
}
}
]
}
]
},
{
"compile_id": 182,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "8-expr",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "expr",
"table_content": {
"keywords": "string1&string2&string3&string4&string5&string6&string7&string8",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 190,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "StringScan.ExprPlus",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "HTTP_SIGNATURE",
"table_type": "expr_plus",
"table_content": {
"district": "我的DistrIct",
"keywords": "addis&sapphire",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 191,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "StringScan.HexBinCaseSensitive",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "KEYWORDS_TABLE",
"table_content": {
"keywords": "54455354",
"expr_type": "none",
"format": "hexbin",
"match_method": "sub"
}
}
],
"group_name": "Untitled"
}
]
},
{
"compile_id": 195,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_name": "HTTP_SIGNATURE",
"table_type": "expr_plus",
"table_content": {
"district": "I love China",
"keywords": "today&yesterday",
"expr_type": "and",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "Monday",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 206,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "duplicateRuleFor191",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "KEYWORDS_TABLE",
"table_content": {
"keywords": "54455354",
"expr_type": "none",
"format": "hexbin",
"match_method": "sub"
}
}
],
"group_name": "Untitled"
}
]
}
]
}

718
test/maat_demo/maat_demo_gtest.cpp
View File

@@ -1,718 +0,0 @@
#include <gtest/gtest.h>
#include <dirent.h>
#include <openssl/md5.h>
#include "include/maat.h"
#include "include/maat_command.h"
#define MODULE_FRAMEWORK_GTEST module_name_str("maat.framework_gtest")
#define ARRAY_SIZE 10
#define HIT_PATH_SIZE 128
#define WAIT_FOR_EFFECTIVE_S 2
#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
const char *table_info_path = "./demo_table_info.conf";
const char *maat_json_file = "./maat_demo.json";
int compile_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long compile_id,
const char *user_region, int clause_num,
int expire_after)
{
char table_line[1024 * 16] = {0};
sprintf(table_line, "%lld\t0\t0\t0\t0\t0\t%s\t%d\t%d\t0.0",
compile_id, user_region, op, clause_num);
struct maat_cmd_line line_rule;
line_rule.rule_id = compile_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
#define TO_GROUP2X_KEY(group_id, parent_id, clause_index) (((unsigned long)group_id<<32|parent_id) + clause_index)
int group2compile_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long group_id,
long long compile_id, int not_flag,
const char *vtable_name, int clause_index,
int expire_after)
{
char table_line[128] = {0};
sprintf(table_line, "%lld\t%lld\t%d\t%d\t%s\t%d",
group_id, compile_id, op, not_flag, vtable_name, clause_index);
struct maat_cmd_line line_rule;
line_rule.rule_id = TO_GROUP2X_KEY(group_id, compile_id, clause_index);
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
int expr_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, const char *keywords,
int expr_type, int match_method, int is_hexbin,
int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) {
return 0;
}
sprintf(table_line, "%lld\t%lld\t%s\t%d\t%d\t%d\t%d", item_id, group_id,
keywords, expr_type, match_method, is_hexbin, op);
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
class JsonMode : public testing::Test
{
protected:
static void SetUpTestCase() {
struct maat_options *opts = maat_options_new();
maat_options_set_json_file(opts, maat_json_file);
maat_options_set_logger(opts, "./maat_sample_gtest.log", LOG_LEVEL_INFO);
_shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
assert(0);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_inst);
}
static struct maat *_shared_maat_inst;
};
struct maat *JsonMode::_shared_maat_inst;
TEST_F(JsonMode, ScanDataOnlyOneByte) {
const char *table_name = "HTTP_URL";
struct maat *maat_inst = JsonMode::_shared_maat_inst;
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
const char scan_data = 0x20;
int ret = maat_scan_string(maat_inst, table_id, &scan_data, sizeof(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
EXPECT_EQ(n_hit_result, 0);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, literal) {
const char *table_name = "HTTP_URL";
struct maat *maat_inst = JsonMode::_shared_maat_inst;
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567";
int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 125);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, Regex) {
int ret = 0;
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
const char *cookie = "Cookie: Txa123aheadBCAxd";
const char *table_name = "HTTP_URL";
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int table_id = maat_get_table_id(maat_inst, table_name);
ret = maat_scan_string(maat_inst, table_id, cookie, strlen(cookie),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 146);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, ExprPlus) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
const char *region_name1 ="HTTP URL";
const char *region_name2 ="我的diStricT";
const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567";
const char *scan_data2 = "Addis Sapphire Hotel";
const char *table_name = "HTTP_SIGNATURE";
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int table_id = maat_get_table_id(maat_inst, table_name);
int ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_ERR);//Should return error for district not setting.
ret = maat_state_set_scan_district(state, table_id, region_name1, strlen(region_name1));
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 128);
maat_state_reset(state);
ret = maat_state_set_scan_district(state, table_id, region_name2, strlen(region_name2));
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 190);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, ExprPlusWithOffset)
{
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
const char *region_name = "Payload";
unsigned char udp_payload_not_hit[] = { /* Stun packet */
0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42,
0x4f, 0xc2, 0xc2, 0x70, 0xb3, 0xa8, 0x4e, 0x22,
0xf5, 0x22, 0x87, 0x4c, 0x40, 0x00, 0x00, 0x46,
0x03, 0x02, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01,
0x3a, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a,
0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9,
0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b,
0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2,
0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f,
0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64,
0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a,
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a };
unsigned char udp_payload_hit[] = { /* Stun packet */ //rule:"1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d"
0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42, //1-1:03
0x4f, 0xc2, 0x2d, 0x70, 0xb3, 0xa8, 0x4e, 0x2d, //10-10:2d
0x34, 0x22, 0x87, 0x4c, 0x2d, 0x00, 0x00, 0x46, //15-16:2d34
0x2d, 0x34, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01, //20-20:2d
0x03, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a, //24-24:2d
0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9,
0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b,
0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2,
0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f,
0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64,
0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a,
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a };
int table_id = maat_get_table_id(maat_inst, "APP_PAYLOAD");
ASSERT_GT(table_id, 0);
int ret = maat_state_set_scan_district(state, table_id, region_name, strlen(region_name));
EXPECT_EQ(ret, 0);
ret = maat_scan_string(maat_inst, table_id, (char*)udp_payload_not_hit, sizeof(udp_payload_not_hit),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_scan_string(maat_inst, table_id, (char*)udp_payload_hit, sizeof(udp_payload_hit),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 148);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, ExprPlusWithHex) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
const char *scan_data1 = "text/html; charset=UTF-8";
const char *scan_data2 = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char *region_name1 = "Content-Type";
const char *region_name2 = "User-Agent";
int table_id = maat_get_table_id(maat_inst, "HTTP_SIGNATURE");
ASSERT_GT(table_id, 0);
int ret = maat_state_set_scan_district(state, table_id, region_name1, strlen(region_name1));
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 156);
ret = maat_state_set_scan_district(state, table_id, region_name2, strlen(region_name2));
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK); //maat-v3 consider as half hit, it's unreasonable
table_id = maat_get_table_id(maat_inst, "KEYWORDS_TABLE");
ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 132);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, ExprAndExprPlus) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
const char *expr_table_name = "HTTP_URL";
const char *expr_plus_table_name = "HTTP_SIGNATURE";
const char *region_name = "I love China";
const char *scan_data = "today is Monday and yesterday is Tuesday";
int expr_table_id = maat_get_table_id(maat_inst, expr_table_name);
int expr_plus_table_id = maat_get_table_id(maat_inst, expr_plus_table_name);
int ret = maat_scan_string(maat_inst, expr_plus_table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_ERR);
ret = maat_state_set_scan_district(state, expr_plus_table_id, region_name, strlen(region_name));
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_inst, expr_plus_table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
ret = maat_scan_string(maat_inst, expr_table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 195);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, ShouldNotHitExprPlus) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
const char *region_name = "tcp.payload";
unsigned char udp_payload_not_hit[] = { /* Stun packet */
0x00, 0x03, 0x00, 0x4a, 0x21, 0x12, 0xa4, 0x42,
0x4f, 0xc2, 0xc2, 0x70, 0xb3, 0xa8, 0x4e, 0x22,
0xf5, 0x22, 0x87, 0x4c, 0x40, 0x00, 0x00, 0x46,
0x03, 0x02, 0xab, 0x39, 0xbb, 0x97, 0xe5, 0x01,
0x3a, 0x46, 0x1c, 0x28, 0x5b, 0xab, 0xfa, 0x9a,
0xab, 0x2e, 0x71, 0x39, 0x66, 0xa0, 0xd7, 0xb9,
0xd8, 0x41, 0xa7, 0xa0, 0x84, 0xa9, 0xf3, 0x1b,
0x03, 0x7f, 0xa8, 0x28, 0xa2, 0xd3, 0x64, 0xc2,
0x3d, 0x20, 0xe0, 0xb1, 0x41, 0x12, 0x6c, 0x2f,
0xc5, 0xbb, 0xc3, 0xba, 0x69, 0x73, 0x52, 0x64,
0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a,
0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a, 0xab, 0x00 };
int table_id = maat_get_table_id(maat_inst, "APP_PAYLOAD");
ASSERT_GT(table_id, 0);
int ret = maat_state_set_scan_district(state, table_id, region_name, strlen(region_name));
ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_inst, table_id, (char *)udp_payload_not_hit, sizeof(udp_payload_not_hit),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK); //maat-v3 consider as half hit, it's unreasonable
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, Expr8) {
const char *table_name = "KEYWORDS_TABLE";
int thread_id = 0;
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 182);
struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0};
int n_read = 0;
n_read = maat_state_get_hit_paths(state, hit_path, HIT_PATH_SIZE);
EXPECT_NE(n_read, 0);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, HexBinCaseSensitive) {
const char *table_name = "KEYWORDS_TABLE";
const char *scan_data1 = "String TeST should not hit.";
const char *scan_data2 = "String TEST should hit";
struct maat *maat_inst = JsonMode::_shared_maat_inst;
int thread_id = 0;
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_inst, table_id, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
maat_state_reset(state);
ret = maat_scan_string(maat_inst, table_id, scan_data2, strlen(scan_data2),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 206);
EXPECT_EQ(results[1], 191);
maat_state_free(state);
}
TEST_F(JsonMode, BugReport20190325) {
unsigned char scan_data[] = {/* Packet 1 */
0x01, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00,
0x00, 0xf4, 0x01, 0x00, 0x00, 0x32, 0x00, 0x00,
0x00, 0xe8, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xff, 0xff, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00,
0x2d, 0x3d, 0x3d, 0x20, 0x48, 0x3d, 0x48, 0x20,
0x3d, 0x3d, 0x2d, 0x3a, 0x00, 0x02, 0x00, 0x00,
0x00, 0x07, 0x0e, 0x00, 0x00, 0xe8, 0x03, 0x00,
0x00, 0x4c, 0x69, 0x6e, 0x75, 0x78, 0x20, 0x33,
0x2e, 0x31, 0x39, 0x2e, 0x30, 0x2d, 0x31, 0x35,
0x2d, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63,
0x00, 0x31, 0x3a, 0x47, 0x32, 0x2e, 0x34, 0x30,
0x00};
const char *table_name = "TROJAN_PAYLOAD";
struct maat *maat_inst = JsonMode::_shared_maat_inst;
int thread_id = 0;
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_inst, table_id, (char *)scan_data, sizeof(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 150);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, MaatUnescape) {
const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char *table_name = "KEYWORDS_TABLE";
struct maat *maat_inst = JsonMode::_shared_maat_inst;
int thread_id = 0;
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 132);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, OffsetChunk64) {
const char *table_name = "IMAGE_FP";
const char *file_name = "./testdata/mesa_logo.jpg";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
FILE *fp = fopen(file_name, "r");
ASSERT_FALSE(fp==NULL);
char scan_data[64];
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state);
ASSERT_TRUE(sp != NULL);
int ret = 0;
int read_size = 0;
int pass_flag = 0;
while (0 == feof(fp)) {
read_size = fread(scan_data, 1, sizeof(scan_data), fp);
ret = maat_stream_scan(sp, scan_data, read_size,
results, ARRAY_SIZE, &n_hit_result, state);
if (ret > 0) {
pass_flag = 1;
break;
}
}
EXPECT_EQ(pass_flag, 1);
EXPECT_EQ(results[0], 136);
maat_stream_free(sp);
fclose(fp);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, OffsetChunk1460) {
const char *table_name = "IMAGE_FP";
const char *file_name = "./testdata/mesa_logo.jpg";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
FILE *fp = fopen(file_name, "r");
ASSERT_FALSE(fp==NULL);
char scan_data[1460];
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state);
ASSERT_TRUE(sp != NULL);
int ret = 0;
int read_size = 0;
int pass_flag = 0;
while (0 == feof(fp)) {
read_size = fread(scan_data, 1, sizeof(scan_data), fp);
ret = maat_stream_scan(sp, scan_data, read_size,
results, ARRAY_SIZE, &n_hit_result, state);
if (ret > 0) {
pass_flag = 1;
break;
}
}
EXPECT_EQ(pass_flag, 1);
EXPECT_EQ(results[0], 136);
maat_stream_free(sp);
fclose(fp);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, StreamScanUTF8) {
const char *table_name = "TROJAN_PAYLOAD";
const char* file_name = "./testdata/jd.com.html";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
char scan_data[2048];
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
FILE *fp = fopen(file_name, "r");
ASSERT_FALSE(fp == NULL);
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state);
ASSERT_FALSE(sp == NULL);
int pass_flag = 0;
while (0 == feof(fp)) {
size_t read_size = fread(scan_data, 1, sizeof(scan_data), fp);
int ret = maat_stream_scan(sp, scan_data, read_size, results, ARRAY_SIZE,
&n_hit_result, state);
if (ret == MAAT_SCAN_HIT) {
pass_flag = 1;
break;
}
}
EXPECT_EQ(pass_flag, 1);
EXPECT_EQ(results[0], 157);
maat_stream_free(sp);
fclose(fp);
maat_state_free(state);
state = NULL;
}
TEST_F(JsonMode, StreamInput) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_inst = JsonMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
const char *table_name = "HTTP_URL";
int table_id = maat_get_table_id(maat_inst, table_name);
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_stream_new(maat_inst, table_id, state);
ASSERT_TRUE(sp != NULL);
int ret = maat_stream_scan(sp, "www.cyberessays.com", strlen("www.cyberessays.com"),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
ret = maat_stream_scan(sp, scan_data, strlen(scan_data), results, ARRAY_SIZE,
&n_hit_result, state);
maat_stream_free(sp);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 125);
maat_state_free(state);
state = NULL;
}
class RedisMode : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *redis_ip = "127.0.0.1";
uint16_t redis_port = 6379;
int redis_db = 0;
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_sample_gtest.log", LOG_LEVEL_INFO);
_shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
assert(0);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_inst);
}
static struct maat *_shared_maat_inst;
};
struct maat *RedisMode::_shared_maat_inst;
TEST_F(RedisMode, dynamic_config) {
const char *table_name = "HTTP_URL";
char data[128] = "welcome to maat version4, it's funny.";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_inst = RedisMode::_shared_maat_inst;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
int table_id = maat_get_table_id(maat_inst, table_name);
int ret = maat_scan_string(maat_inst, table_id, data, strlen(data), results,
ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
maat_state_reset(state);
const char *compile_table_name = "COMPILE";
const char *g2c_table_name = "GROUP2COMPILE";
/* compile table add line */
long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_ADD, compile_id, "null", 1, 0);
EXPECT_EQ(ret, 1);
/* group2compile table add line */
long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1);
/* expr table add line */
long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1);
const char *keywords = "welcome to maat";
ret = expr_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id, group_id,
keywords, 1, 0, 0, 0); /* EXPR_TYPE_AND MATCH_METHOD_SUB */
EXPECT_EQ(ret, 1);
sleep(WAIT_FOR_EFFECTIVE_S);
ret = maat_scan_string(maat_inst, table_id, data, strlen(data), results,
ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], compile_id);
maat_state_reset(state);
/* expr table del line */
ret = expr_table_set_line(maat_inst, table_name, MAAT_OP_DEL, item_id, group_id,
keywords, 1, 0, 0, 0); /* EXPR_TYPE_AND MATCH_METHOD_SUB */
EXPECT_EQ(ret, 1);
/* group2compile table del line */
ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_DEL, group_id,
compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1);
/* compile table del line */
ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_DEL, compile_id, "null", 1, 0);
EXPECT_EQ(ret, 1);
sleep(WAIT_FOR_EFFECTIVE_S);
ret = maat_scan_string(maat_inst, table_id, data, strlen(data), results,
ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
maat_state_free(state);
state = NULL;
}
int main(int argc, char ** argv)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
ret=RUN_ALL_TESTS();
return ret;
}

18
test/maat_demo/readme.md
View File

@@ -1,18 +0,0 @@
依赖 crypto库，需提前安装
安装 redis-server 并按默认配置启动即可，maat_demo_gtest 会用到 redis
## 编译 & 运行单测
1. 当前目录，mkdir build
2. cd build
3. cmake ..
4. make
5. ./maat_demo_gtest
## 文件说明：
- include 目录存放 maat 库头文件
- lib 目录存放 maat 动态库及 gtest 静态库
- testdata 为单测所需测试数据
- maat_demo.json 为json 格式的匹配规则，运行时会转为 iris 格式，位于maat_demo.json_iris_tmp目录(运行时生成)
- demo_table_info.conf用于表示 iris 格式规则每列代表的含义，maat解析对应列的数据
- maat_demo_gtest.cpp 为单测文件，字符串匹配相关测试用例可供参考

77893
test/maat_demo/testdata/bool-matcher-test-exprs.txt vendored
View File

File diff suppressed because it is too large Load Diff

48
test/maat_demo/testdata/charsetWindows1251.txt vendored
View File

@@ -1,48 +0,0 @@
<EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>!','JS_CORE_WINDOW_AUTH':'<27><><EFBFBD><EFBFBD><EFBFBD>','JS_CORE_IMAGE_FULL':'<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>'});</script>
<script type="text/javascript">(window.BX||top.BX).message({'LANGUAGE_ID':'ru','FORMAT_DATE':'DD.MM.YYYY','FORMAT_DATETIME':'DD.MM.YYYY HH:MI:SS','COOKIE_PREFIX':'BITRIX_SM','USER_ID':'','SERVER_TIME':'1578340589','SERVER_TZ_OFFSET':'18000','USER_TZ_OFFSET':'0','USER_TZ_AUTO':'Y','bitrix_sessid':'fadf964e9f5bb819f212e5abf5ffb255','SITE_ID':'s1'});</script>
<script type="text/javascript" src="/bitrix/cache/js/s1/web20/kernel_main/kernel_main.js?1402043622360126"></script>
<script type="text/javascript" src="/bitrix/js/main/rsasecurity.js?136876011925044"></script>
<script type="text/javascript">BX.setCSSList(['/bitrix/js/main/core/css/core.css','/bitrix/js/main/core/css/core_popup.css','/bitrix/js/main/core/css/core_date.css','/bitrix/js/main/core/css/core.css','/bitrix/js/main/core/css/core_popup.css','/bitrix/js/main/core/css/core_date.css']); </script>
<script type="text/javascript">BX.setJSList(['/bitrix/js/main/core/core.js','/bitrix/js/main/core/core_ajax.js','/bitrix/js/main/session.js','/bitrix/js/main/json/json2.min.js','/bitrix/js/main/core/core_ls.js','/bitrix/js/main/core/core_window.js','/bitrix/js/main/utils.js','/bitrix/js/main/core/core_popup.js','/bitrix/js/main/core/core_date.js','/bitrix/js/main/dd.js']); </script>
<script type="text/javascript">
bxSession.Expand(1440, 'fadf964e9f5bb819f212e5abf5ffb255', false, '35a74b06af8f9ea55ffbda20075b0894');
</script>
<script>new Image().src='http://www.sgaice.ru/bitrix/spread.php?s=QklUUklYX1NNX0dVRVNUX0lEATY4MTg5NQExNjA5NDQ0NTg5AS8BAQECQklUUklYX1NNX0xBU1RfVklTSVQBMDcuMDEuMjAyMCAwMDo1NjoyOQExNjA5NDQ0NTg5AS8BAQEC&k=71d3b79b44f9716b27b47feab4a206cf';
</script>
<script type="text/javascript" src="/bitrix/cache/js/s1/web20/template_1e341eb2f86845c7519566374f51d35a/template_1e341eb2f86845c7519566374f51d35a_368c1a68876fd1c32b307a10695f3654.js?14010848191120"></script>
<script type="text/javascript" src="/bitrix/js/imgzoom/thumb.js"></script>
<meta name="google-site-verification" content="gL_64SaiDgQcX5z-pvPZmBJ-exN-wS6KZNoDMcPtYtM" />
<title><3E><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><>ѻ</title>
</head>
<body>
<div id="maintop">
<table align="left1" width="100%" border="0" cellpadding="0" cellspacing="0">
<tr><td valign="top">
<script type="text/javascript">
top.BX.defer(top.rsasec_form_bind)({'formid':'system_auth_form6zOUGO','key':{'M':'HazQxsgvQCIFPf30iHR40R22fp7P9YLPXFhQu6uus68RZxf2IpMo9v0KDpxkgg43WXaZaXrTRvjg1e2126IOo66vH5bphkMP/69MSPlEoaXYzWjTokd+Yzy30WR6HEOyB9tJwADGyjysqoE4+jUfHZQv2JMaVZS0U4SHWOUPwNU=','E':'AQAB','chunk':'128'},'rsa_rand':'5e1390ed8a8e19.17355178','params':['USER_PASSWORD']});
</script>
<div id="login-form-window">
<a href="" onclick="return CloseLoginForm()" style="float:right;"><3E><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD></a>
<form method="post" target="_top" action="/index.php?login=yes">
<input type="hidden" name="backurl" value="/index.php" />
<input type="hidden" name="AUTH_FORM" value="Y" />
<input type="hidden" name="TYPE" value="AUTH" />
<table width="95%">
<tr>
<td colspan="2">
<09><><EFBFBD><EFBFBD><EFBFBD>:<br />
<input type="text" name="USER_LOGIN" maxlength="50" value="

BIN
test/maat_demo/testdata/digest_test.data vendored
View File

Binary file not shown.

968
test/maat_demo/testdata/jd.com.html vendored
View File

File diff suppressed because one or more lines are too long

BIN
test/maat_demo/testdata/mesa_logo.jpg vendored
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 105 KiB

857
test/maat_framework_gtest.cpp
View File

File diff suppressed because it is too large Load Diff

1
test/maat_framework_perf_gtest.cpp
View File

@@ -55,7 +55,6 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') {
buff[strlen(buff) - 1] = '\0';
}

28
vendor/CMakeLists.txt vendored
View File

@@ -58,6 +58,34 @@ add_dependencies(hyperscan_runtime_static hyperscan)
set_property(TARGET hyperscan_runtime_static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhs_runtime.a)
set_property(TARGET hyperscan_runtime_static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)
#pcre-8.45
ExternalProject_Add(pcre PREFIX pcre
URL ${CMAKE_CURRENT_SOURCE_DIR}/pcre-8.45.tar.gz
CONFIGURE_COMMAND ./configure --prefix=${VENDOR_BUILD}
BUILD_COMMAND make
INSTALL_COMMAND make install
BUILD_IN_SOURCE 1)
ExternalProject_Get_Property(pcre INSTALL_DIR)
file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
#rulescan 3.0.1
ExternalProject_Add(rulescan PREFIX rulescan
URL ${CMAKE_CURRENT_SOURCE_DIR}/rulescan-3.0.1.tar.gz
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS="-fPIC")
ExternalProject_Get_Property(rulescan INSTALL_DIR)
file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
#merge librulescan.a and libpcre.a => librs.a
add_custom_command(OUTPUT ${VENDOR_BUILD}/lib/librs.a
COMMAND ar crsT ${VENDOR_BUILD}/lib/librs.a ${VENDOR_BUILD}/lib/libpcre.a ${VENDOR_BUILD}/lib/librulescan.a
DEPENDS pcre rulescan)
add_custom_target(_merge ALL DEPENDS ${VENDOR_BUILD}/lib/librs.a)
add_library(rulescan_static STATIC IMPORTED GLOBAL)
set_property(TARGET rulescan_static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/librs.a)
# hiredis-1.1.0
ExternalProject_Add(hiredis PREFIX hiredis
URL ${CMAKE_CURRENT_SOURCE_DIR}/hiredis-1.1.0.tar.gz

BIN
vendor/pcre-8.45.tar.gz vendored Normal file
View File

Binary file not shown.

BIN
vendor/rulescan-3.0.1.tar.gz vendored Normal file
View File

Binary file not shown.