gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rollback to v4.0.31

Browse Source
This commit is contained in:
liuwentan
2023-08-09 19:22:09 +08:00
parent d29eef0423
commit fb0cb5405d
40 changed files with 1907 additions and 5448 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
src/maat_expr.c
View File

@@ -60,7 +60,7 @@ struct expr_item {
long long group_id;
char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type;
enum expr_match_mode match_mode;
enum hs_match_mode match_mode;
int is_hexbin;
int is_case_sensitive;
void *user_data;
@@ -68,7 +68,7 @@ struct expr_item {
};
struct expr_runtime {
struct expr_matcher *matcher;
struct adapter_hs *hs;
struct rcu_hash_table *item_hash; // <item_id, struct expr_item>
long long version; //expr_rt version
@@ -79,7 +79,6 @@ struct expr_runtime {
struct log_handle *logger;
struct maat_garbage_bin *ref_garbage_bin;
enum maat_expr_engine expr_engine;
int district_num;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
@@ -115,22 +114,22 @@ static enum expr_type int_to_expr_type(int expr_type)
return type;
}
static enum expr_match_mode int_to_match_mode(int match_method)
static enum hs_match_mode int_to_match_mode(int match_method)
{
enum expr_match_mode mode = EXPR_MATCH_MODE_INVALID;
enum hs_match_mode mode = HS_MATCH_MODE_INVALID;
switch (match_method) {
case 0:
mode = EXPR_MATCH_MODE_SUB;
mode = HS_MATCH_MODE_SUB;
break;
case 1:
mode = EXPR_MATCH_MODE_SUFFIX;
mode = HS_MATCH_MODE_SUFFIX;
break;
case 2:
mode = EXPR_MATCH_MODE_PREFIX;
mode = HS_MATCH_MODE_PREFIX;
break;
case 3:
mode = EXPR_MATCH_MODE_EXACTLY;
mode = HS_MATCH_MODE_EXACTLY;
break;
default:
break;
@@ -144,13 +143,13 @@ static int expr_runtime_get_district_id(struct expr_runtime *expr_rt,
{
long long district_id = DISTRICT_ANY;
int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id, 1);
int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id);
if (map_ret < 0) {
if (NULL == expr_rt->tmp_district_map) {
expr_rt->tmp_district_map = maat_kv_store_duplicate(expr_rt->district_map);
}
map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id, 1);
map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id);
if (map_ret < 0) {
district_id = expr_rt->district_num;
maat_kv_register(expr_rt->tmp_district_map, district, district_id);
@@ -168,7 +167,7 @@ int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *dis
return -1;
}
return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id, 1);
return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id);
}
static struct expr_item *
@@ -235,8 +234,8 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name,
__FUNCTION__, __LINE__, table_name, line);
goto error;
} else if (expr_item->expr_type == EXPR_TYPE_REGEX) {
ret = expr_matcher_verify_regex_expression(expr_item->keywords, expr_rt->logger);
if (0 == ret) {
ret = adapter_hs_verify_regex_expression(expr_item->keywords, expr_rt->logger);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> regex expression(item_id:%lld):%s illegal,"
" will be dropped", __FUNCTION__, __LINE__, table_name,
@@ -278,7 +277,7 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name,
match_method_type = atoi(line + column_offset);
expr_item->match_mode = int_to_match_mode(match_method_type);
if (expr_item->match_mode == EXPR_MATCH_MODE_INVALID) {
if (expr_item->match_mode == HS_MATCH_MODE_INVALID) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> has invalid match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line);
@@ -473,14 +472,12 @@ void *expr_runtime_new(void *expr_schema, size_t max_thread_num,
return NULL;
}
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->item_hash = rcu_hash_new(expr_item_free_cb, NULL, 0);
expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger;
expr_rt->expr_engine = table_manager_get_expr_engine(schema->ref_tbl_mgr);
expr_rt->district_map = maat_kv_store_new();
expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
@@ -498,9 +495,9 @@ void expr_runtime_free(void *expr_runtime)
}
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
if (expr_rt->matcher != NULL) {
expr_matcher_free(expr_rt->matcher);
expr_rt->matcher = NULL;
if (expr_rt->hs != NULL) {
adapter_hs_free(expr_rt->hs);
expr_rt->hs = NULL;
}
if (expr_rt->item_hash != NULL) {
@@ -561,18 +558,18 @@ static int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
return 0;
}
static enum expr_pattern_type expr_type2pattern_type(enum expr_type expr_type)
static enum hs_pattern_type expr_type2pattern_type(enum expr_type expr_type)
{
enum expr_pattern_type pattern_type = EXPR_PATTERN_TYPE_STR;
enum hs_pattern_type pattern_type;
switch (expr_type) {
case EXPR_TYPE_STRING:
case EXPR_TYPE_AND:
case EXPR_TYPE_OFFSET:
pattern_type = EXPR_PATTERN_TYPE_STR;
pattern_type = HS_PATTERN_TYPE_STR;
break;
case EXPR_TYPE_REGEX:
pattern_type = EXPR_PATTERN_TYPE_REG;
pattern_type = HS_PATTERN_TYPE_REG;
break;
default:
break;
@@ -689,12 +686,12 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
}
sub_expr_cnt = i;
break;
case EXPR_TYPE_STRING: //AND/OFFSET/STRING type expression use \b to represent blank(' ')
case EXPR_TYPE_STRING:
sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords;
sub_key_array[0] = str_unescape(sub_key_array[0]);
break;
case EXPR_TYPE_REGEX: //only regex type expression use \s to represent blank(' ')
case EXPR_TYPE_REGEX:
sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords;
break;
@@ -713,15 +710,15 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
if (TRUE == expr_item->is_case_sensitive) {
// insensitive
expr_rule->patterns[i].case_sensitive = EXPR_CASE_SENSITIVE;
expr_rule->patterns[i].case_sensitive = HS_CASE_SENSITIVE;
} else {
expr_rule->patterns[i].case_sensitive = EXPR_CASE_INSENSITIVE;
expr_rule->patterns[i].case_sensitive = HS_CASE_INSENSITIVE;
}
expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type);
expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type);
if (TRUE == expr_item->is_hexbin &&
expr_rule->patterns[i].type != EXPR_PATTERN_TYPE_REG) {
expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) {
region_str_len = strlen(sub_key_array[i]) * 8 + 1;
region_string = ALLOC(char, region_str_len);
region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]),
@@ -741,13 +738,13 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
}
expr_rule->patterns[i].match_mode = expr_item->match_mode;
if (expr_rule->patterns[i].match_mode == EXPR_MATCH_MODE_SUB) {
if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) {
expr_rule->patterns[i].start_offset = key_left_offset[i];
expr_rule->patterns[i].end_offset = key_right_offset[i];
}
}
expr_rule->expr_id = expr_item->item_id;
expr_rule->tag = expr_item->user_data;
expr_rule->user_tag = expr_item->user_data;
expr_rule->n_patterns = sub_expr_cnt;
return 0;
@@ -813,10 +810,10 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
return 0;
}
static void garbage_expr_matcher_free(void *expr_matcher, void *arg)
static void garbage_adapter_hs_free(void *adapter_hs, void *arg)
{
struct expr_matcher *matcher = (struct expr_matcher *)expr_matcher;
expr_matcher_free(matcher);
struct adapter_hs *hs = (struct adapter_hs *)adapter_hs;
adapter_hs_free(hs);
}
int expr_runtime_commit(void *expr_runtime, const char *table_name,
@@ -867,42 +864,38 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
}
}
struct expr_matcher *new_matcher = NULL;
struct expr_matcher *old_matcher = NULL;
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
if (rule_cnt > 0) {
enum expr_engine_type engine_type = EXPR_ENGINE_TYPE_HS;
if (expr_rt->expr_engine == MAAT_EXPR_ENGINE_RS) {
engine_type = EXPR_ENGINE_TYPE_RS;
}
new_matcher = expr_matcher_new(rules, real_rule_cnt, engine_type,
expr_rt->n_worker_thread, expr_rt->logger);
if (NULL == new_matcher) {
new_adapter_hs = adapter_hs_new(rules, real_rule_cnt, expr_rt->n_worker_thread,
expr_rt->logger);
if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild expr_matcher failed when update"
"[%s:%d] table[%s] rebuild adapter_hs engine failed when update"
" %zu expr rules", __FUNCTION__, __LINE__, table_name, real_rule_cnt);
ret = -1;
} else {
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, maat_rt_version);
}
}
old_matcher = expr_rt->matcher;
expr_rt->matcher = new_matcher;
old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs;
rcu_hash_commit(expr_rt->item_hash);
if (old_matcher != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_matcher, NULL, garbage_expr_matcher_free);
if (old_adapter_hs != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL,
garbage_adapter_hs_free);
}
expr_rt->rule_num = real_rule_cnt;
expr_rt->regex_rule_num = real_regex_rule_cnt;
expr_rt->version = maat_rt_version;
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, expr_rt->version);
if (rules != NULL) {
for (i = 0; i < rule_cnt; i++) {
expr_rule_reset(&rules[i]);
@@ -956,14 +949,15 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
return 0;
}
if (NULL == expr_rt->matcher) {
if (NULL == expr_rt->hs) {
return 0;
}
size_t n_hit_item = 0;
struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = expr_matcher_match(expr_rt->matcher, thread_id, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item);
if (ret < 0) {
return -1;
}
@@ -1006,15 +1000,14 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
return real_hit_item_cnt;
}
struct expr_matcher_stream *
struct adapter_hs_stream *
expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
{
if (NULL == expr_rt || thread_id < 0) {
return NULL;
}
struct expr_matcher_stream *stream = expr_matcher_stream_open(expr_rt->matcher,
thread_id);
struct adapter_hs_stream *stream = adapter_hs_stream_open(expr_rt->hs, thread_id);
if (NULL == stream) {
return NULL;
}
@@ -1023,7 +1016,7 @@ expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
}
int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
struct expr_matcher_stream *s_handle,
struct adapter_hs_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state)
{
@@ -1033,10 +1026,10 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
}
size_t n_hit_item = 0;
struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = expr_matcher_stream_match(s_handle, data, data_len, hit_results,
MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results,
MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
if (ret < 0) {
return -1;
}
@@ -1074,13 +1067,13 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,
struct expr_matcher_stream *stream)
struct adapter_hs_stream *s_handle)
{
if (NULL == expr_rt || thread_id < 0 || NULL == stream) {
if (NULL == expr_rt || thread_id < 0 || NULL == s_handle) {
return;
}
expr_matcher_stream_close(stream);
adapter_hs_stream_close(s_handle);
}
void expr_runtime_hit_inc(struct expr_runtime *expr_rt, int thread_id)
@@ -1173,4 +1166,4 @@ long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt)
alignment_int64_array_reset(expr_rt->scan_bytes, expr_rt->n_worker_thread);
return sum;
}
}