gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[OPTIMIZE]optimize alloc in adapter_hs_scan_stream

Browse Source
This commit is contained in:
liuwentan
2023-06-07 11:53:08 +08:00
parent ac8a0a512c
commit 75e23c7d27
3 changed files with 70 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -48,14 +48,20 @@ struct adpt_hs_compile_data {
unsigned int n_patterns; unsigned int n_patterns;
}; };
struct adapter_hs_scratch {
hs_scratch_t **literal_scratches;
hs_scratch_t **regex_scratches;
struct bool_expr_match **bool_match_buffs;
};
/* adapter_hs runtime */ /* adapter_hs runtime */
struct adapter_hs_runtime { struct adapter_hs_runtime {
hs_database_t *literal_db; hs_database_t *literal_db;
hs_database_t *regex_db; hs_database_t *regex_db;
hs_scratch_t **literal_scratches; // hs_scratch_t **literal_scratches;
hs_scratch_t **regex_scratches; // hs_scratch_t **regex_scratches;
struct adapter_hs_scratch *scratch;
struct bool_matcher *bm; struct bool_matcher *bm;
}; };
@@ -134,17 +140,19 @@ static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_work
int ret = 0; int ret = 0;
if (pattern_type == HS_PATTERN_TYPE_STR) { if (pattern_type == HS_PATTERN_TYPE_STR) {
hs_rt->literal_scratches = ALLOC(hs_scratch_t *, n_worker_thread); hs_rt->scratch->literal_scratches = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->literal_scratches, n_worker_thread, logger); ret = _hs_alloc_scratch(hs_rt->literal_db, hs_rt->scratch->literal_scratches,
n_worker_thread, logger);
if (ret < 0) { if (ret < 0) {
FREE(hs_rt->literal_scratches); FREE(hs_rt->scratch->literal_scratches);
return -1; return -1;
} }
} else { } else {
hs_rt->regex_scratches = ALLOC(hs_scratch_t *, n_worker_thread); hs_rt->scratch->regex_scratches = ALLOC(hs_scratch_t *, n_worker_thread);
ret = _hs_alloc_scratch(hs_rt->regex_db, hs_rt->regex_scratches, n_worker_thread, logger); ret = _hs_alloc_scratch(hs_rt->regex_db, hs_rt->scratch->regex_scratches,
n_worker_thread, logger);
if (ret < 0) { if (ret < 0) {
FREE(hs_rt->regex_scratches); FREE(hs_rt->scratch->regex_scratches);
return -1; return -1;
} }
} }
@@ -448,6 +456,12 @@ struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
goto error; goto error;
} }
hs_instance->hs_rt->scratch = ALLOC(struct adapter_hs_scratch, 1);
hs_instance->hs_rt->scratch->bool_match_buffs = ALLOC(struct bool_expr_match *, n_worker_thread);
for (size_t i = 0; i < n_worker_thread; i++) {
hs_instance->hs_rt->scratch->bool_match_buffs[i] = ALLOC(struct bool_expr_match, hs_instance->n_expr);
}
/* literal and regex scratch can't reuse */ /* literal and regex scratch can't reuse */
if (literal_pattern_num > 0) { if (literal_pattern_num > 0) {
ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_STR, logger); ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, HS_PATTERN_TYPE_STR, logger);
@@ -486,27 +500,38 @@ void adapter_hs_free(struct adapter_hs *hs_instance)
hs_instance->hs_rt->regex_db = NULL; hs_instance->hs_rt->regex_db = NULL;
} }
if (hs_instance->hs_rt->literal_scratches != NULL) { if (hs_instance->hs_rt->scratch != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { if (hs_instance->hs_rt->scratch->literal_scratches != NULL) {
if (hs_instance->hs_rt->literal_scratches[i] != NULL) { for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
hs_free_scratch(hs_instance->hs_rt->literal_scratches[i]); if (hs_instance->hs_rt->scratch->literal_scratches[i] != NULL) {
hs_instance->hs_rt->literal_scratches[i] = NULL; hs_free_scratch(hs_instance->hs_rt->scratch->literal_scratches[i]);
hs_instance->hs_rt->scratch->literal_scratches[i] = NULL;
}
} }
FREE(hs_instance->hs_rt->scratch->literal_scratches);
} }
FREE(hs_instance->hs_rt->literal_scratches); if (hs_instance->hs_rt->scratch->regex_scratches != NULL) {
} for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->scratch->regex_scratches[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->scratch->regex_scratches[i]);
if (hs_instance->hs_rt->regex_scratches != NULL) { hs_instance->hs_rt->scratch->regex_scratches[i] = NULL;
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { }
if (hs_instance->hs_rt->regex_scratches[i] != NULL) {
hs_free_scratch(hs_instance->hs_rt->regex_scratches[i]);
hs_instance->hs_rt->regex_scratches[i] = NULL;
} }
FREE(hs_instance->hs_rt->scratch->regex_scratches);
} }
FREE(hs_instance->hs_rt->regex_scratches); if (hs_instance->hs_rt->scratch->bool_match_buffs != NULL) {
for (size_t i = 0; i < hs_instance->n_worker_thread; i++) {
if (hs_instance->hs_rt->scratch->bool_match_buffs[i] != NULL) {
FREE(hs_instance->hs_rt->scratch->bool_match_buffs[i]);
}
}
FREE(hs_instance->hs_rt->scratch->bool_match_buffs);
}
FREE(hs_instance->hs_rt->scratch);
} }
if (hs_instance->hs_rt->bm != NULL) { if (hs_instance->hs_rt->bm != NULL) {
@@ -658,7 +683,6 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
return hs_stream; return hs_stream;
error: error:
//TODO: hs_stream->hs_rt->scratches[thread_id] may be free twice
if (hs_stream->literal_stream != NULL) { if (hs_stream->literal_stream != NULL) {
hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL); hs_close_stream(hs_stream->literal_stream, NULL, NULL, NULL);
hs_stream->literal_stream = NULL; hs_stream->literal_stream = NULL;
@@ -731,9 +755,9 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
int err_scratch_flag = 0; int err_scratch_flag = 0;
if (hs_stream->literal_stream != NULL) { if (hs_stream->literal_stream != NULL) {
if (hs_stream->ref_hs_rt->literal_scratches != NULL) { if (hs_stream->ref_hs_rt->scratch->literal_scratches != NULL) {
err = hs_scan_stream(hs_stream->literal_stream, data, data_len, err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
0, hs_stream->ref_hs_rt->literal_scratches[thread_id], 0, hs_stream->ref_hs_rt->scratch->literal_scratches[thread_id],
matched_event_cb, hs_stream->matched_pat); matched_event_cb, hs_stream->matched_pat);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
err_count++; err_count++;
@@ -746,9 +770,9 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
} }
if (hs_stream->regex_stream != NULL) { if (hs_stream->regex_stream != NULL) {
if (hs_stream->ref_hs_rt->regex_scratches != NULL) { if (hs_stream->ref_hs_rt->scratch->regex_scratches != NULL) {
err = hs_scan_stream(hs_stream->regex_stream, data, data_len, err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
0, hs_stream->ref_hs_rt->regex_scratches[thread_id], 0, hs_stream->ref_hs_rt->scratch->regex_scratches[thread_id],
matched_event_cb, hs_stream->matched_pat); matched_event_cb, hs_stream->matched_pat);
if (err != HS_SUCCESS) { if (err != HS_SUCCESS) {
err_count++; err_count++;
@@ -781,7 +805,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
} }
int ret = 0; int ret = 0;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); struct bool_expr_match *bool_matcher_results = hs_stream->ref_hs_rt->scratch->bool_match_buffs[thread_id];
int bool_matcher_ret = bool_matcher_match(hs_stream->ref_hs_rt->bm, pattern_ids, n_pattern_id, int bool_matcher_ret = bool_matcher_match(hs_stream->ref_hs_rt->bm, pattern_ids, n_pattern_id,
bool_matcher_results, hs_stream->n_expr); bool_matcher_results, hs_stream->n_expr);
if (bool_matcher_ret < 0) { if (bool_matcher_ret < 0) {
@@ -800,7 +824,6 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
*n_hit_result = bool_matcher_ret; *n_hit_result = bool_matcher_ret;
next: next:
FREE(bool_matcher_results);
utarray_clear(hs_stream->matched_pat->pattern_ids); utarray_clear(hs_stream->matched_pat->pattern_ids);
return ret; return ret;

20
src/maat_compile.c
View File

@@ -1311,7 +1311,7 @@ int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, struct group2comp
g2c_item->not_flag); g2c_item->not_flag);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d" "[%s:%d] add literal_id{group_id:%lld, vtable_id:%d} to clause_index: %d"
" of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id, " of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id,
g2c_item->vtable_id, g2c_item->clause_index, compile_id); g2c_item->vtable_id, g2c_item->clause_index, compile_id);
} }
@@ -1323,7 +1323,7 @@ int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, struct group2comp
g2c_item->not_flag); g2c_item->not_flag);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d" "[%s:%d] add literal_id{group_id:%lld, vtable_id:%d} to clause_index: %d"
" of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id, " of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id,
g2c_item->vtable_id, g2c_item->clause_index, compile_id); g2c_item->vtable_id, g2c_item->clause_index, compile_id);
} }
@@ -1351,7 +1351,7 @@ int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, struct group2comp
g2c_item->not_flag); g2c_item->not_flag);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d" "[%s:%d] add literal_id{group_id:%lld, vtable_id:%d} to clause_index: %d"
" of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id, " of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id,
g2c_item->vtable_id, g2c_item->clause_index, compile_id); g2c_item->vtable_id, g2c_item->clause_index, compile_id);
} }
@@ -1364,7 +1364,7 @@ int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, struct group2comp
g2c_item->not_flag); g2c_item->not_flag);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d" "[%s:%d] add literal_id{group_id:%lld, vtable_id:%d} to clause_index: %d"
" of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id, " of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id,
g2c_item->vtable_id, g2c_item->clause_index, compile_id); g2c_item->vtable_id, g2c_item->clause_index, compile_id);
} }
@@ -1390,7 +1390,7 @@ int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
sizeof(long long)); sizeof(long long));
if (NULL == compile) { if (NULL == compile) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group %d from compile %d failed, compile" "[%s:%d] Remove group_id:%lld from compile_id:%lld failed, compile"
" is not exisited.", __FUNCTION__, __LINE__, g2c_item->group_id, " is not exisited.", __FUNCTION__, __LINE__, g2c_item->group_id,
compile_id); compile_id);
return -1; return -1;
@@ -1400,8 +1400,8 @@ int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
g2c_item->clause_index); g2c_item->clause_index);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group %d vtable_id %d from clause %d of " "[%s:%d] Remove group_id:%lld vtable_id %d from clause %d of "
"compile %d failed, literal is not in compile.", __FUNCTION__, "compile_id:%lld failed, literal is not in compile.", __FUNCTION__,
__LINE__, g2c_item->group_id, g2c_item->vtable_id, __LINE__, g2c_item->group_id, g2c_item->vtable_id,
g2c_item->clause_index, compile_id); g2c_item->clause_index, compile_id);
} }
@@ -1433,8 +1433,8 @@ int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
g2c_item->clause_index); g2c_item->clause_index);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group %d vtable_id %d from clause %d of compile" "[%s:%d] Remove group_id:%lld vtable_id %d from clause %d of compile_id:"
" %d failed, literal is not in compile.", __FUNCTION__, __LINE__, "%lld failed, literal is not in compile.", __FUNCTION__, __LINE__,
g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index,
compile_id); compile_id);
} }
@@ -1446,7 +1446,7 @@ int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
} }
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group %d from compile %d failed, compile is not exisited.", "[%s:%d] Remove group_id:%lld from compile_id:%lld failed, compile is not exisited.",
__FUNCTION__, __LINE__, g2c_item->group_id, compile_id); __FUNCTION__, __LINE__, g2c_item->group_id, compile_id);
return -1; return -1;
} }

6
src/maat_group.c
View File

@@ -973,6 +973,12 @@ void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_g
if (depth >= MAX_RECURSION_DEPTH) { if (depth >= MAX_RECURSION_DEPTH) {
log_error(group_topo->logger, MODULE_GROUP, log_error(group_topo->logger, MODULE_GROUP,
"[%s:%d]exceed max recursion depth(5)", __FUNCTION__, __LINE__); "[%s:%d]exceed max recursion depth(5)", __FUNCTION__, __LINE__);
for (int i = 0; i < utarray_len(hit_group_ids); i++) {
long long *p = (long long *)utarray_eltptr(hit_group_ids, i);
log_error(group_topo->logger, MODULE_GROUP,
"[%s:%d]group_id:%lld can't recursively get super group_id",
__FUNCTION__, __LINE__, *p);
}
return; return;
} }