gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feature]verify regex expression

Browse Source
This commit is contained in:
liuwentan
2023-05-09 17:45:43 +08:00
parent 4540321998
commit e97adb8b97
11 changed files with 262 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/maat.h
View File

@@ -123,6 +123,13 @@ void maat_free(struct maat *instance);
/* maat helper API */ /* maat helper API */
int maat_helper_read_column(const char *table_line, int Nth_column, int maat_helper_read_column(const char *table_line, int Nth_column,
size_t *column_offset, size_t *column_len); size_t *column_offset, size_t *column_len);
/**
* verify if regex expression is legal
*
* @param The NULL-terminated expression to parse.
* @retval 0(legal) -1(illegal)
**/
int maat_helper_verify_regex_expression(const char *expression);
/* maat table API */ /* maat table API */
int maat_get_table_id(struct maat *instance, const char *table_name); int maat_get_table_id(struct maat *instance, const char *table_name);

35
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -304,6 +304,40 @@ struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule, struct
return bool_exprs; return bool_exprs;
} }
int verify_regex_expression(const char *regex_str, struct log_handle *logger)
{
hs_expr_info_t *info = NULL;
hs_compile_error_t *error = NULL;
hs_error_t err = hs_expression_info(regex_str, HS_FLAG_CASELESS, &info, &error);
if (err != HS_SUCCESS) {
// Expression will fail compilation and report error elsewhere.
if (logger != NULL) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] illegal regex expression: \"%s\": %s",
__FUNCTION__, __LINE__, regex_str, error->message);
}
FREE(info);
hs_free_compile_error(error);
return -1;
}
if (info != NULL) {
FREE(info);
}
return 0;
}
int adapter_hs_verify_regex_expression(const char *regex_expr, struct log_handle *logger)
{
if (NULL == regex_expr) {
return -1;
}
return verify_regex_expression(regex_expr, logger);
}
struct adapter_hs *adapter_hs_new(size_t n_worker_thread, struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
struct expr_rule *rules, size_t n_rule, struct expr_rule *rules, size_t n_rule,
struct log_handle *logger) struct log_handle *logger)
@@ -666,6 +700,7 @@ void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
same as hs_attr */ same as hs_attr */
hs_stream->ref_hs_rt = NULL; hs_stream->ref_hs_rt = NULL;
hs_stream->matched_pat->ref_hs_attr = NULL; hs_stream->matched_pat->ref_hs_attr = NULL;
if (hs_stream->matched_pat->pattern_ids != NULL) { if (hs_stream->matched_pat->pattern_ids != NULL) {
utarray_free(hs_stream->matched_pat->pattern_ids); utarray_free(hs_stream->matched_pat->pattern_ids);
hs_stream->matched_pat->pattern_ids = NULL; hs_stream->matched_pat->pattern_ids = NULL;

2
scanner/adapter_hs/adapter_hs.h
View File

@@ -81,6 +81,8 @@ struct expr_rule {
void *user_tag; void *user_tag;
}; };
int adapter_hs_verify_regex_expression(const char *regex_expr,
struct log_handle *logger);
/** /**
* @brief new adapter_hs instance * @brief new adapter_hs instance
* *

10
src/maat_api.c
View File

@@ -27,6 +27,7 @@
#include "maat_compile.h" #include "maat_compile.h"
#include "alignment.h" #include "alignment.h"
#include "ip_matcher.h" #include "ip_matcher.h"
#include "adapter_hs.h"
#include "maat_garbage_collection.h" #include "maat_garbage_collection.h"
#include "maat_group.h" #include "maat_group.h"
#include "maat_expr.h" #include "maat_expr.h"
@@ -471,6 +472,15 @@ int maat_helper_read_column(const char *table_line, int Nth_column,
return get_column_pos(table_line, Nth_column, column_offset, column_len); return get_column_pos(table_line, Nth_column, column_offset, column_len);
} }
int maat_helper_verify_regex_expression(const char *regex_expr)
{
if (NULL == regex_expr) {
return -1;
}
return adapter_hs_verify_regex_expression(regex_expr, NULL);
}
int maat_get_table_id(struct maat *maat_instance, const char *table_name) int maat_get_table_id(struct maat *maat_instance, const char *table_name)
{ {
int table_id = -1; int table_id = -1;

100
src/maat_expr.c
View File

@@ -70,7 +70,7 @@ struct expr_item {
struct expr_runtime { struct expr_runtime {
struct adapter_hs *hs; struct adapter_hs *hs;
struct rcu_hash_table *expr_item_hash; // store hs_expr rule for rebuild adapter_hs instance struct rcu_hash_table *item_hash; // store hs_expr rule for rebuild adapter_hs instance
long long version; //expr_rt version long long version; //expr_rt version
long long rule_num; long long rule_num;
@@ -200,6 +200,47 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
} }
expr_item->group_id = atoll(line + column_offset); expr_item->group_id = atoll(line + column_offset);
ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no keywords",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
goto error;
}
if (column_len >= MAX_KEYWORDS_STR) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s keywords length too long",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
goto error;
}
memcpy(expr_item->keywords, (line + column_offset), column_len);
ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no expr_type",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
goto error;
}
expr_type = atoi(line + column_offset);
expr_item->expr_type = int_to_expr_type(expr_type);
if (expr_item->expr_type == EXPR_TYPE_INVALID) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has invalid expr_type",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
goto error;
} else if (expr_item->expr_type == EXPR_TYPE_REGEX) {
ret = adapter_hs_verify_regex_expression(expr_item->keywords, expr_rt->logger);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) regex expression(item_id:%lld):%s illegal, will be dropped",
__FUNCTION__, __LINE__, expr_schema->table_id, expr_item->item_id, expr_item->keywords);
goto error;
}
}
table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id); table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id);
if (table_type == TABLE_TYPE_EXPR_PLUS) { if (table_type == TABLE_TYPE_EXPR_PLUS) {
ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len);
@@ -223,23 +264,6 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
expr_item->district_id = DISTRICT_ANY; expr_item->district_id = DISTRICT_ANY;
} }
ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no expr_type",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
goto error;
}
expr_type = atoi(line + column_offset);
expr_item->expr_type = int_to_expr_type(expr_type);
if (expr_item->expr_type == EXPR_TYPE_INVALID) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has invalid expr_type",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
goto error;
}
ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
@@ -286,22 +310,6 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
goto error; goto error;
} }
ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no keywords",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
goto error;
}
if (column_len >= MAX_KEYWORDS_STR) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s keywords length too long",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
goto error;
}
memcpy(expr_item->keywords, (line + column_offset), column_len);
return expr_item; return expr_item;
error: error:
FREE(expr_item); FREE(expr_item);
@@ -461,7 +469,7 @@ void *expr_runtime_new(void *expr_schema, size_t max_thread_num,
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1); struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->expr_item_hash = rcu_hash_new(expr_item_free_cb, NULL); expr_rt->item_hash = rcu_hash_new(expr_item_free_cb, NULL);
expr_rt->n_worker_thread = max_thread_num; expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin; expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger; expr_rt->logger = logger;
@@ -488,9 +496,9 @@ void expr_runtime_free(void *expr_runtime)
expr_rt->hs = NULL; expr_rt->hs = NULL;
} }
if (expr_rt->expr_item_hash != NULL) { if (expr_rt->item_hash != NULL) {
rcu_hash_free(expr_rt->expr_item_hash); rcu_hash_free(expr_rt->item_hash);
expr_rt->expr_item_hash = NULL; expr_rt->item_hash = NULL;
} }
assert(expr_rt->tmp_district_map == NULL); assert(expr_rt->tmp_district_map == NULL);
@@ -535,13 +543,13 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_
if (0 == is_valid) { if (0 == is_valid) {
//delete //delete
rcu_hash_del(expr_rt->expr_item_hash, key, key_len); rcu_hash_del(expr_rt->item_hash, key, key_len);
} else { } else {
//add //add
ret = rcu_hash_add(expr_rt->expr_item_hash, key, key_len, (void *)item); ret = rcu_hash_add(expr_rt->item_hash, key, key_len, (void *)item);
if (ret < 0) { if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr item(item_id:%lld) add to expr_item_hash failed", "[%s:%d] expr item(item_id:%lld) add to item_hash failed",
__FUNCTION__, __LINE__, item->item_id); __FUNCTION__, __LINE__, item->item_id);
return -1; return -1;
} }
@@ -809,7 +817,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
int updating_flag = rcu_hash_is_updating(expr_rt->expr_item_hash); int updating_flag = rcu_hash_is_updating(expr_rt->item_hash);
if (0 == updating_flag) { if (0 == updating_flag) {
return 0; return 0;
} }
@@ -828,7 +836,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
struct expr_rule *rules = NULL; struct expr_rule *rules = NULL;
void **ex_data_array = NULL; void **ex_data_array = NULL;
size_t rule_cnt = rcu_updating_hash_list(expr_rt->expr_item_hash, &ex_data_array); size_t rule_cnt = rcu_updating_hash_list(expr_rt->item_hash, &ex_data_array);
if (rule_cnt > 0) { if (rule_cnt > 0) {
rules = ALLOC(struct expr_rule, rule_cnt); rules = ALLOC(struct expr_rule, rule_cnt);
for (i = 0; i < rule_cnt; i++) { for (i = 0; i < rule_cnt; i++) {
@@ -859,7 +867,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
old_adapter_hs = expr_rt->hs; old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs; expr_rt->hs = new_adapter_hs;
rcu_hash_commit(expr_rt->expr_item_hash); rcu_hash_commit(expr_rt->item_hash);
if (old_adapter_hs != NULL) { if (old_adapter_hs != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL, garbage_adapter_hs_free); maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL, garbage_adapter_hs_free);
@@ -954,7 +962,7 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag); inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) { if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id; long long item_id = hit_results[i].rule_id;
struct expr_item *expr_item = (struct expr_item *)rcu_hash_find(expr_rt->expr_item_hash, struct expr_item *expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
(char *)&item_id, (char *)&item_id,
sizeof(long long)); sizeof(long long));
if (!expr_item) { if (!expr_item) {
@@ -1019,7 +1027,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_str
for (size_t i = 0; i < n_hit_item; i++) { for (size_t i = 0; i < n_hit_item; i++) {
long long item_id = hit_results[i].rule_id; long long item_id = hit_results[i].rule_id;
struct expr_item *expr_item = (struct expr_item *)rcu_hash_find(expr_rt->expr_item_hash, struct expr_item *expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
(char *)&item_id, (char *)&item_id,
sizeof(long long)); sizeof(long long));
if (!expr_item) { if (!expr_item) {

24
src/maat_flag.c
View File

@@ -46,7 +46,7 @@ struct flag_item {
struct flag_runtime { struct flag_runtime {
struct flag_matcher *matcher; struct flag_matcher *matcher;
struct rcu_hash_table *flag_item_hash; struct rcu_hash_table *item_hash;
long long rule_num; long long rule_num;
long long version; long long version;
@@ -189,7 +189,7 @@ void *flag_runtime_new(void *flag_schema, size_t max_thread_num,
struct flag_runtime *flag_rt = ALLOC(struct flag_runtime, 1); struct flag_runtime *flag_rt = ALLOC(struct flag_runtime, 1);
flag_rt->flag_item_hash = rcu_hash_new(flag_item_free_cb, NULL); flag_rt->item_hash = rcu_hash_new(flag_item_free_cb, NULL);
flag_rt->n_worker_thread = max_thread_num; flag_rt->n_worker_thread = max_thread_num;
flag_rt->ref_garbage_bin = garbage_bin; flag_rt->ref_garbage_bin = garbage_bin;
flag_rt->logger = logger; flag_rt->logger = logger;
@@ -209,9 +209,9 @@ void flag_runtime_free(void *flag_runtime)
} }
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
if (flag_rt->flag_item_hash != NULL) { if (flag_rt->item_hash != NULL) {
rcu_hash_free(flag_rt->flag_item_hash); rcu_hash_free(flag_rt->item_hash);
flag_rt->flag_item_hash = NULL; flag_rt->item_hash = NULL;
} }
if (flag_rt->matcher != NULL) { if (flag_rt->matcher != NULL) {
@@ -251,13 +251,13 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_
if (0 == is_valid) { if (0 == is_valid) {
//delete //delete
rcu_hash_del(flag_rt->flag_item_hash, key, key_len); rcu_hash_del(flag_rt->item_hash, key, key_len);
} else { } else {
//add //add
ret = rcu_hash_add(flag_rt->flag_item_hash, key, key_len, (void *)item); ret = rcu_hash_add(flag_rt->item_hash, key, key_len, (void *)item);
if (ret < 0) { if (ret < 0) {
log_error(flag_rt->logger, MODULE_FLAG, log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag item(item_id:%lld) add to flag_item_hash failed", "[%s:%d] flag item(item_id:%lld) add to item_hash failed",
__FUNCTION__, __LINE__, item->item_id); __FUNCTION__, __LINE__, item->item_id);
return -1; return -1;
} }
@@ -449,7 +449,7 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
int updating_flag = rcu_hash_is_updating(flag_rt->flag_item_hash); int updating_flag = rcu_hash_is_updating(flag_rt->item_hash);
if (0 == updating_flag) { if (0 == updating_flag) {
return 0; return 0;
} }
@@ -465,7 +465,7 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
struct flag_rule *rules = NULL; struct flag_rule *rules = NULL;
void **ex_data_array = NULL; void **ex_data_array = NULL;
size_t rule_cnt = rcu_updating_hash_list(flag_rt->flag_item_hash, &ex_data_array); size_t rule_cnt = rcu_updating_hash_list(flag_rt->item_hash, &ex_data_array);
if (rule_cnt > 0) { if (rule_cnt > 0) {
rules = ALLOC(struct flag_rule, rule_cnt); rules = ALLOC(struct flag_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) { for (size_t i = 0; i < rule_cnt; i++) {
@@ -490,7 +490,7 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
old_flag_matcher = flag_rt->matcher; old_flag_matcher = flag_rt->matcher;
flag_rt->matcher = new_flag_matcher; flag_rt->matcher = new_flag_matcher;
rcu_hash_commit(flag_rt->flag_item_hash); rcu_hash_commit(flag_rt->item_hash);
if (old_flag_matcher != NULL) { if (old_flag_matcher != NULL) {
maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher, NULL, maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher, NULL,
@@ -554,7 +554,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag); inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) { if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id; long long item_id = hit_results[i].rule_id;
struct flag_item *flag_item = (struct flag_item *)rcu_hash_find(flag_rt->flag_item_hash, struct flag_item *flag_item = (struct flag_item *)rcu_hash_find(flag_rt->item_hash,
(char *)&item_id, (char *)&item_id,
sizeof(long long)); sizeof(long long));
if (!flag_item) { if (!flag_item) {

22
src/maat_interval.c
View File

@@ -43,7 +43,7 @@ struct interval_item {
struct interval_runtime { struct interval_runtime {
struct interval_matcher *matcher; struct interval_matcher *matcher;
struct rcu_hash_table *int_item_hash; //store interval rule for rebuild interval_matcher instance struct rcu_hash_table *item_hash; //store interval rule for rebuild interval_matcher instance
long long version; long long version;
long long rule_num; long long rule_num;
@@ -185,7 +185,7 @@ void *interval_runtime_new(void *interval_schema, size_t max_thread_num,
struct interval_runtime *interval_rt = ALLOC(struct interval_runtime, 1); struct interval_runtime *interval_rt = ALLOC(struct interval_runtime, 1);
interval_rt->int_item_hash = rcu_hash_new(interval_item_free_cb, NULL); interval_rt->item_hash = rcu_hash_new(interval_item_free_cb, NULL);
interval_rt->n_worker_thread = max_thread_num; interval_rt->n_worker_thread = max_thread_num;
interval_rt->ref_garbage_bin = garbage_bin; interval_rt->ref_garbage_bin = garbage_bin;
interval_rt->logger = logger; interval_rt->logger = logger;
@@ -205,9 +205,9 @@ void interval_runtime_free(void *interval_runtime)
} }
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
if (interval_rt->int_item_hash != NULL) { if (interval_rt->item_hash != NULL) {
rcu_hash_free(interval_rt->int_item_hash); rcu_hash_free(interval_rt->item_hash);
interval_rt->int_item_hash = NULL; interval_rt->item_hash = NULL;
} }
if (interval_rt->matcher != NULL) { if (interval_rt->matcher != NULL) {
@@ -363,10 +363,10 @@ int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key,
if (0 == is_valid) { if (0 == is_valid) {
//delete //delete
rcu_hash_del(interval_rt->int_item_hash, key, key_len); rcu_hash_del(interval_rt->item_hash, key, key_len);
} else { } else {
//add //add
ret = rcu_hash_add(interval_rt->int_item_hash, key, key_len, (void *)item); ret = rcu_hash_add(interval_rt->item_hash, key, key_len, (void *)item);
if (ret < 0) { if (ret < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL, log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval item(item_id:%lld) add to interavl_item_hash failed", "[%s:%d] interval item(item_id:%lld) add to interavl_item_hash failed",
@@ -446,7 +446,7 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name, long
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
int updating_flag = rcu_hash_is_updating(interval_rt->int_item_hash); int updating_flag = rcu_hash_is_updating(interval_rt->item_hash);
if (0 == updating_flag) { if (0 == updating_flag) {
return 0; return 0;
} }
@@ -462,7 +462,7 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name, long
void **ex_data_array = NULL; void **ex_data_array = NULL;
struct interval_rule *rules = NULL; struct interval_rule *rules = NULL;
size_t rule_cnt = rcu_updating_hash_list(interval_rt->int_item_hash, &ex_data_array); size_t rule_cnt = rcu_updating_hash_list(interval_rt->item_hash, &ex_data_array);
if (rule_cnt > 0) { if (rule_cnt > 0) {
rules = ALLOC(struct interval_rule, rule_cnt); rules = ALLOC(struct interval_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) { for (size_t i = 0; i < rule_cnt; i++) {
@@ -487,7 +487,7 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name, long
old_interval_matcher = interval_rt->matcher; old_interval_matcher = interval_rt->matcher;
interval_rt->matcher = new_interval_matcher; interval_rt->matcher = new_interval_matcher;
rcu_hash_commit(interval_rt->int_item_hash); rcu_hash_commit(interval_rt->item_hash);
if (old_interval_matcher != NULL) { if (old_interval_matcher != NULL) {
maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher, NULL, maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher, NULL,
@@ -551,7 +551,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag); inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) { if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id; long long item_id = hit_results[i].rule_id;
struct interval_item *int_item = (struct interval_item *)rcu_hash_find(interval_rt->int_item_hash, struct interval_item *int_item = (struct interval_item *)rcu_hash_find(interval_rt->item_hash,
(char *)&item_id, (char *)&item_id,
sizeof(long long)); sizeof(long long));
if (!int_item) { if (!int_item) {

24
src/maat_ip.c
View File

@@ -69,7 +69,7 @@ struct ip_item {
struct ip_runtime { struct ip_runtime {
struct ip_matcher *ip_matcher; struct ip_matcher *ip_matcher;
struct interval_matcher *intval_matcher; struct interval_matcher *intval_matcher;
struct rcu_hash_table *ip_item_hash; struct rcu_hash_table *item_hash;
long long version; long long version;
long long rule_num; long long rule_num;
@@ -394,7 +394,7 @@ void *ip_runtime_new(void *ip_schema, size_t max_thread_num,
struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1); struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1);
ip_rt->ip_item_hash = rcu_hash_new(ip_item_free_cb, NULL); ip_rt->item_hash = rcu_hash_new(ip_item_free_cb, NULL);
ip_rt->n_worker_thread = max_thread_num; ip_rt->n_worker_thread = max_thread_num;
ip_rt->ref_garbage_bin = garbage_bin; ip_rt->ref_garbage_bin = garbage_bin;
ip_rt->logger = logger; ip_rt->logger = logger;
@@ -423,9 +423,9 @@ void ip_runtime_free(void *ip_runtime)
ip_rt->intval_matcher = NULL; ip_rt->intval_matcher = NULL;
} }
if (ip_rt->ip_item_hash != NULL) { if (ip_rt->item_hash != NULL) {
rcu_hash_free(ip_rt->ip_item_hash); rcu_hash_free(ip_rt->item_hash);
ip_rt->ip_item_hash = NULL; ip_rt->item_hash = NULL;
} }
if (ip_rt->hit_cnt != NULL) { if (ip_rt->hit_cnt != NULL) {
@@ -476,10 +476,10 @@ int ip_runtime_update_row(struct ip_runtime *ip_rt, char *key, size_t key_len,
if (0 == is_valid) { if (0 == is_valid) {
// delete // delete
rcu_hash_del(ip_rt->ip_item_hash, key, key_len); rcu_hash_del(ip_rt->item_hash, key, key_len);
} else { } else {
// add // add
ret = rcu_hash_add(ip_rt->ip_item_hash, key, key_len, (void *)item); ret = rcu_hash_add(ip_rt->item_hash, key, key_len, (void *)item);
if (ret < 0) { if (ret < 0) {
log_error(ip_rt->logger, MODULE_IP, log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] ip item(item_id:%lld) add to ip runtime htable failed", "[%s:%d] ip item(item_id:%lld) add to ip runtime htable failed",
@@ -558,7 +558,7 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
int updating_flag = rcu_hash_is_updating(ip_rt->ip_item_hash); int updating_flag = rcu_hash_is_updating(ip_rt->item_hash);
if (0 == updating_flag) { if (0 == updating_flag) {
return 0; return 0;
} }
@@ -569,7 +569,7 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
struct interval_rule *intval_rules = NULL; struct interval_rule *intval_rules = NULL;
void **ex_data_array = NULL; void **ex_data_array = NULL;
size_t rule_cnt = rcu_updating_hash_list(ip_rt->ip_item_hash, &ex_data_array); size_t rule_cnt = rcu_updating_hash_list(ip_rt->item_hash, &ex_data_array);
if (rule_cnt > 0) { if (rule_cnt > 0) {
rules = ALLOC(struct ip_rule, rule_cnt); rules = ALLOC(struct ip_rule, rule_cnt);
intval_rules = ALLOC(struct interval_rule, rule_cnt); intval_rules = ALLOC(struct interval_rule, rule_cnt);
@@ -612,7 +612,7 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
old_ip_matcher = ip_rt->ip_matcher; old_ip_matcher = ip_rt->ip_matcher;
ip_rt->ip_matcher = new_ip_matcher; ip_rt->ip_matcher = new_ip_matcher;
rcu_hash_commit(ip_rt->ip_item_hash); rcu_hash_commit(ip_rt->item_hash);
if (old_ip_matcher != NULL) { if (old_ip_matcher != NULL) {
maat_garbage_bagging(ip_rt->ref_garbage_bin, old_ip_matcher, NULL, maat_garbage_bagging(ip_rt->ref_garbage_bin, old_ip_matcher, NULL,
@@ -745,7 +745,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
for (int i = 0; i < n_hit_port_item; i++) { for (int i = 0; i < n_hit_port_item; i++) {
long long item_id = port_results[i].rule_id; long long item_id = port_results[i].rule_id;
struct ip_item *ip_item = (struct ip_item *)rcu_hash_find(ip_rt->ip_item_hash, struct ip_item *ip_item = (struct ip_item *)rcu_hash_find(ip_rt->item_hash,
(char *)&item_id, (char *)&item_id,
sizeof(long long)); sizeof(long long));
if (!ip_item) { if (!ip_item) {
@@ -775,7 +775,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
for (size_t i = 0; i < n_hit_ip_item; i++) { for (size_t i = 0; i < n_hit_ip_item; i++) {
long long item_id = ip_results[i].rule_id; long long item_id = ip_results[i].rule_id;
struct ip_item *ip_item = (struct ip_item *)rcu_hash_find(ip_rt->ip_item_hash, struct ip_item *ip_item = (struct ip_item *)rcu_hash_find(ip_rt->item_hash,
(char *)&item_id, (char *)&item_id,
sizeof(long long)); sizeof(long long));
if (!ip_item) { if (!ip_item) {

42
test/adapter_hs_gtest.cpp
View File

@@ -79,6 +79,21 @@ static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size)
return resultlen; return resultlen;
} }
enum hs_pattern_type pattern_type_str_to_enum(const char *str)
{
enum hs_pattern_type pattern_type = HS_PATTERN_TYPE_MAX;
if (strcmp(str, "regex") == 0) {
pattern_type = HS_PATTERN_TYPE_REG;
} else if (strcmp(str, "literal") == 0) {
pattern_type = HS_PATTERN_TYPE_STR;
} else {
assert(0);
}
return pattern_type;
}
int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_expr) int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_expr)
{ {
unsigned char *json_buff = NULL; unsigned char *json_buff = NULL;
@@ -129,7 +144,13 @@ int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_
size_t pattern_cnt = cJSON_GetArraySize(tmp_item); size_t pattern_cnt = cJSON_GetArraySize(tmp_item);
for (size_t j = 0; j < pattern_cnt; j++) { for (size_t j = 0; j < pattern_cnt; j++) {
cJSON *pat_item = cJSON_GetArrayItem(tmp_item, j); cJSON *pat_item = cJSON_GetArrayItem(tmp_item, j);
cJSON *item = cJSON_GetObjectItem(pat_item, "match_method");
cJSON *item = cJSON_GetObjectItem(pat_item, "pattern_type");
if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].pattern_type = pattern_type_str_to_enum(item->valuestring);
}
item = cJSON_GetObjectItem(pat_item, "match_method");
if (item != NULL && item->type == cJSON_String) { if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].match_mode = match_method_to_match_mode(item->valuestring); exprs[i].patterns[j].match_mode = match_method_to_match_mode(item->valuestring);
} }
@@ -146,7 +167,7 @@ int parse_config_file(const char *filename, struct expr_rule exprs[], size_t *n_
item = cJSON_GetObjectItem(pat_item, "pattern"); item = cJSON_GetObjectItem(pat_item, "pattern");
if (item != NULL && item->type == cJSON_String) { if (item != NULL && item->type == cJSON_String) {
exprs[i].patterns[j].pat = ALLOC(char, strlen(item->valuestring)); exprs[i].patterns[j].pat = ALLOC(char, strlen(item->valuestring) + 1);
if (exprs[i].patterns[j].is_hexbin == 1) { if (exprs[i].patterns[j].is_hexbin == 1) {
size_t pat_str_len = strlen(item->valuestring) + 1; size_t pat_str_len = strlen(item->valuestring) + 1;
@@ -678,6 +699,23 @@ that the edges be all directed in the same direction.";
hs_instance = NULL; hs_instance = NULL;
} }
TEST(adapter_hs_scan, regex_expression_check)
{
struct expr_rule rules[64] = {0};
size_t n_rule = 0;
int ret = parse_config_file("./regex_expr.conf", rules, &n_rule);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < n_rule; i++) {
for (size_t j = 0; j < rules[i].n_patterns; j++) {
adapter_hs_verify_regex_expression(rules[i].patterns[j].pat, g_logger);
}
}
expr_array_free(rules, n_rule);
}
int main(int argc, char **argv) int main(int argc, char **argv)
{ {
int ret = 0; int ret = 0;

27
test/maat_json.json
View File

@@ -2809,6 +2809,33 @@
] ]
} }
] ]
},
{
"compile_id": 205,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "StringScan.RegexExpressionIllegal",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "KEYWORDS_TABLE",
"table_content": {
"keywords": "123^456",
"expr_type": "regex",
"format": "uncase plain",
"match_method": "sub"
}
}
],
"group_name": "Untitled"
}
]
} }
], ],
"plugin_table": [ "plugin_table": [

51
test/regex_expr.conf
View File

@@ -1,10 +1,11 @@
{ {
"expr_rules": [ "expr_rules": [
{ {
"expr_id": 301, "rule_id": 301,
"pattern_num": 1, "pattern_num": 1,
"patterns": [ "patterns": [
{ {
"pattern_type": "regex",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -13,16 +14,18 @@
] ]
}, },
{ {
"expr_id": 302, "rule_id": 302,
"pattern_num": 2, "pattern_num": 2,
"patterns": [ "patterns": [
{ {
"pattern_type": "regex",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
"pattern": "[0-9]today" "pattern": "[0-9]today"
}, },
{ {
"pattern_type": "regex",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -31,16 +34,18 @@
] ]
}, },
{ {
"expr_id": 303, "rule_id": 303,
"pattern_num": 2, "pattern_num": 2,
"patterns": [ "patterns": [
{ {
"pattern_type": "regex",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
"pattern": "Cookie:\\s" "pattern": "Cookie:\\s"
}, },
{ {
"pattern_type": "regex",
"match_method": "sub", "match_method": "sub",
"case_sensitive": "yes", "case_sensitive": "yes",
"is_hexbin": "no", "is_hexbin": "no",
@@ -48,5 +53,45 @@
} }
] ]
}, },
{
"rule_id": 304,
"pattern_num": 2,
"patterns": [
{
"pattern_type": "regex",
"match_method": "sub",
"case_sensitive": "no",
"is_hexbin": "no",
"pattern": "123^abc"
},
{
"pattern_type": "regex",
"match_method": "sub",
"case_sensitive": "no",
"is_hexbin": "no",
"pattern": "^123"
}
]
},
{
"rule_id": 305,
"pattern_num": 2,
"patterns": [
{
"pattern_type": "regex",
"match_method": "sub",
"case_sensitive": "no",
"is_hexbin": "no",
"pattern": "^123"
},
{
"pattern_type": "regex",
"match_method": "sub",
"case_sensitive": "no",
"is_hexbin": "no",
"pattern": "123^abc"
}
]
}
] ]
} }