gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[PATCH]add expr_matcher hit pattern statistics

Browse Source
This commit is contained in:
liuwentan
2023-12-27 12:04:15 +08:00
parent 102c8ac0f8
commit 6d5fea298a
36 changed files with 1643 additions and 1080 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
scanner/expr_matcher/adapter_hs/adapter_hs.cpp
View File

@@ -404,6 +404,9 @@ static void hs_lit_stream_reset(struct hs_lit_stream *hs_lit_stream)
scratches[hs_lit_stream->thread_id],
matched_event_cb, hs_lit_stream->matched_pat);
}
utarray_clear(hs_lit_stream->matched_pat->pattern_ids);
bloom_reset(hs_lit_stream->matched_pat->ref_bloom);
}
static void hs_regex_stream_reset(struct hs_regex_stream *hs_regex_stream)
@@ -418,6 +421,9 @@ static void hs_regex_stream_reset(struct hs_regex_stream *hs_regex_stream)
scratches[hs_regex_stream->thread_id],
matched_event_cb, hs_regex_stream->matched_pat);
}
utarray_clear(hs_regex_stream->matched_pat->pattern_ids);
bloom_reset(hs_regex_stream->matched_pat->ref_bloom);
}
static int gather_hit_pattern_id(struct matched_pattern *matched_pat,
@@ -436,8 +442,6 @@ static int gather_hit_pattern_id(struct matched_pattern *matched_pat,
}
*n_pattern_id = array_index;
utarray_clear(matched_pat->pattern_ids);
bloom_reset(matched_pat->ref_bloom);
return 0;
}

8
scanner/expr_matcher/adapter_rs/adapter_rs.cpp
View File

@@ -282,8 +282,6 @@ static int gather_hit_pattern_id(struct matched_pattern *matched_pat,
}
*n_pattern_id = array_index;
utarray_clear(matched_pat->pattern_ids);
bloom_reset(matched_pat->ref_bloom);
return 0;
}
@@ -365,6 +363,9 @@ int rs_lit_engine_scan(void *rs_lit_engine, int thread_id,
struct rs_lit_stream *rs_lit_stream = rs_lit_inst->streams[thread_id];
assert(rs_lit_stream != NULL);
utarray_clear(rs_lit_stream->matched_pat->pattern_ids);
bloom_reset(rs_lit_stream->matched_pat->ref_bloom);
if (rs_lit_inst->rs_db != NULL) {
int ret = rs_scan(rs_lit_inst->rs_db, thread_id, data, data_len,
0, matched_event_cb, rs_lit_stream->matched_pat);
@@ -536,6 +537,9 @@ int rs_regex_engine_scan(void *rs_regex_engine, int thread_id,
struct rs_regex_stream *rs_regex_stream = rs_regex_inst->streams[thread_id];
assert(rs_regex_stream != NULL);
utarray_clear(rs_regex_stream->matched_pat->pattern_ids);
bloom_reset(rs_regex_stream->matched_pat->ref_bloom);
if (rs_regex_inst->rs_db != NULL) {
int ret = rs_scan(rs_regex_inst->rs_db, thread_id, data, data_len,
0, matched_event_cb, rs_regex_stream->matched_pat);

81
scanner/expr_matcher/expr_matcher.cpp
View File

@@ -270,9 +270,10 @@ void expr_matcher_free(struct expr_matcher *matcher)
FREE(matcher);
}
struct expr_matcher *expr_matcher_new(struct expr_rule *rules, size_t n_rule,
enum expr_engine_type engine_type,
size_t n_thread, struct log_handle *logger)
struct expr_matcher *
expr_matcher_new(struct expr_rule *rules, size_t n_rule,
enum expr_engine_type engine_type,
size_t n_thread, struct log_handle *logger)
{
if (NULL == rules || 0 == n_rule || 0 == n_thread ||
(engine_type != EXPR_ENGINE_TYPE_HS &&
@@ -443,7 +444,8 @@ next:
int expr_matcher_match(struct expr_matcher *matcher, int thread_id,
const char *data, size_t data_len,
struct expr_scan_result *result_array,
size_t array_size, size_t *n_hit_result)
size_t array_size, size_t *n_hit_result,
size_t *n_hit_pattern)
{
if (NULL == matcher || thread_id < 0 || NULL == data || 0 == data_len
|| NULL == result_array || 0 == array_size || NULL == n_hit_result) {
@@ -451,21 +453,21 @@ int expr_matcher_match(struct expr_matcher *matcher, int thread_id,
}
int err_count = 0;
unsigned long long lit_pattern_ids[MAX_HIT_PATTERN_NUM];
unsigned long long regex_pattern_ids[MAX_HIT_PATTERN_NUM];
size_t n_lit_pattern = 0;
size_t n_regex_pattern = 0;
size_t n_pattern = 0;
unsigned long long lit_pat_ids[MAX_HIT_PATTERN_NUM];
unsigned long long regex_pat_ids[MAX_HIT_PATTERN_NUM];
size_t lit_pat_cnt = 0;
size_t regex_pat_cnt = 0;
size_t pat_cnt = 0;
int ret = engine_ops[matcher->engine_type].engine_scan(matcher->lit_runtime, thread_id,
data, data_len, lit_pattern_ids,
MAX_HIT_PATTERN_NUM, &n_lit_pattern);
data, data_len, lit_pat_ids,
MAX_HIT_PATTERN_NUM, &lit_pat_cnt);
if (ret < 0) {
err_count++;
}
ret = hs_regex_engine_scan(matcher->regex_runtime, thread_id, data, data_len,
regex_pattern_ids, MAX_HIT_PATTERN_NUM, &n_regex_pattern);
regex_pat_ids, MAX_HIT_PATTERN_NUM, &regex_pat_cnt);
if (ret < 0) {
err_count++;
}
@@ -474,20 +476,22 @@ int expr_matcher_match(struct expr_matcher *matcher, int thread_id,
return -1;
}
n_pattern = n_lit_pattern + n_regex_pattern;
if (n_pattern > MAX_HIT_PATTERN_NUM) {
n_pattern = MAX_HIT_PATTERN_NUM;
pat_cnt = lit_pat_cnt + regex_pat_cnt;
*n_hit_pattern = pat_cnt;
if (pat_cnt > MAX_HIT_PATTERN_NUM) {
pat_cnt = MAX_HIT_PATTERN_NUM;
}
size_t j = 0;
for (size_t i = n_lit_pattern; i < n_pattern; i++, j++) {
lit_pattern_ids[i] = regex_pattern_ids[j];
for (size_t i = lit_pat_cnt; i < pat_cnt; i++, j++) {
lit_pat_ids[i] = regex_pat_ids[j];
}
struct bool_expr_match *match_buff = matcher->bool_match_buffs[thread_id];
return expr_matcher_bool_matcher_match(matcher->bm, match_buff, MAX_HIT_PATTERN_NUM,
lit_pattern_ids, n_pattern, result_array,
lit_pat_ids, pat_cnt, result_array,
array_size, n_hit_result);
}
@@ -534,29 +538,31 @@ expr_matcher_stream_open(struct expr_matcher *matcher, int thread_id)
int expr_matcher_stream_match(struct expr_matcher_stream *stream,
const char *data, size_t data_len,
struct expr_scan_result *result_array,
size_t array_size, size_t *n_hit_result)
size_t array_size, size_t *n_hit_result,
size_t *n_hit_pattern)
{
if (NULL == stream || NULL == data || 0 == data_len || NULL == result_array
|| 0 == array_size || NULL == n_hit_result) {
if (NULL == stream || NULL == data || 0 == data_len ||
NULL == result_array || 0 == array_size ||
NULL == n_hit_result) {
return -1;
}
int err_count = 0;
unsigned long long lit_pattern_ids[MAX_HIT_PATTERN_NUM];
unsigned long long regex_pattern_ids[MAX_HIT_PATTERN_NUM];
size_t n_lit_pattern = 0;
size_t n_regex_pattern = 0;
size_t n_pattern = 0;
unsigned long long lit_pat_ids[MAX_HIT_PATTERN_NUM];
unsigned long long regex_pat_ids[MAX_HIT_PATTERN_NUM];
size_t lit_pat_cnt = 0;
size_t regex_pat_cnt = 0;
size_t pat_cnt = 0;
int ret = engine_ops[stream->engine_type].scan_stream(stream->lit_stream, data, data_len,
lit_pattern_ids, MAX_HIT_PATTERN_NUM,
&n_lit_pattern);
lit_pat_ids, MAX_HIT_PATTERN_NUM,
&lit_pat_cnt);
if (ret < 0) {
err_count++;
}
ret = hs_regex_stream_scan(stream->regex_stream, data, data_len, regex_pattern_ids,
MAX_HIT_PATTERN_NUM, &n_regex_pattern);
ret = hs_regex_stream_scan(stream->regex_stream, data, data_len, regex_pat_ids,
MAX_HIT_PATTERN_NUM, &regex_pat_cnt);
if (ret < 0) {
err_count++;
}
@@ -565,22 +571,23 @@ int expr_matcher_stream_match(struct expr_matcher_stream *stream,
return -1;
}
n_pattern = n_lit_pattern + n_regex_pattern;
if (n_pattern > MAX_HIT_PATTERN_NUM) {
n_pattern = MAX_HIT_PATTERN_NUM;
pat_cnt = lit_pat_cnt + regex_pat_cnt;
*n_hit_pattern = pat_cnt;
if (pat_cnt > MAX_HIT_PATTERN_NUM) {
pat_cnt = MAX_HIT_PATTERN_NUM;
}
size_t j = 0;
for (size_t i = n_lit_pattern; i < n_pattern; i++, j++) {
lit_pattern_ids[i] = regex_pattern_ids[j];
for (size_t i = lit_pat_cnt; i < pat_cnt; i++, j++) {
lit_pat_ids[i] = regex_pat_ids[j];
}
struct expr_matcher *matcher = stream->ref_matcher;
struct bool_expr_match *match_buff = matcher->bool_match_buffs[stream->thread_id];
return expr_matcher_bool_matcher_match(matcher->bm, match_buff, MAX_HIT_PATTERN_NUM,
lit_pattern_ids, n_pattern, result_array,
array_size, n_hit_result);
lit_pat_ids, pat_cnt, result_array, array_size,
n_hit_result);
}
void expr_matcher_stream_close(struct expr_matcher_stream *stream)

13
scanner/expr_matcher/expr_matcher.h
View File

@@ -88,9 +88,10 @@ int expr_matcher_verify_regex_expression(const char *regex_expr,
* @param n_worker_threads: the number of scan threads which will call adapter_rs_scan()
*
*/
struct expr_matcher *expr_matcher_new(struct expr_rule *rules, size_t n_rule,
enum expr_engine_type type, size_t n_thread,
struct log_handle *logger);
struct expr_matcher *
expr_matcher_new(struct expr_rule *rules, size_t n_rule,
enum expr_engine_type type, size_t n_thread,
struct log_handle *logger);
void expr_matcher_free(struct expr_matcher *matcher);
@@ -107,7 +108,8 @@ void expr_matcher_free(struct expr_matcher *matcher);
int expr_matcher_match(struct expr_matcher *matcher, int thread_id,
const char *data, size_t data_len,
struct expr_scan_result *result_array,
size_t array_size, size_t *n_hit_result);
size_t array_size, size_t *n_hit_result,
size_t *n_hit_pattern);
/**
* @brief
@@ -121,7 +123,8 @@ expr_matcher_stream_open(struct expr_matcher *matcher, int thread_id);
int expr_matcher_stream_match(struct expr_matcher_stream *stream,
const char *data, size_t data_len,
struct expr_scan_result *result_array,
size_t array_size, size_t *n_hit_result);
size_t array_size, size_t *n_hit_result,
size_t *n_hit_pattern);
/**
* @brief