gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[BUGFIX]fix hyperscan-5.4.2 literal empty string check bug

Browse Source
This commit is contained in:
刘文坛
2023-06-14 09:10:16 +00:00
parent d48a6e2390
commit 5a18084eaa
13 changed files with 94 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/inc_internal/maat_compile.h
View File

@@ -49,7 +49,8 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
struct log_handle *logger);
void compile_runtime_free(void *compile_runtime);
void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt);
void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt,
void *g2c_runtime);
int compile_runtime_update(void *compile_runtime, void *compile_schema,
const char *table_name, const char *line,
@@ -80,8 +81,7 @@ void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime,
void *g2g_runtime);
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime);
void group2compile_runtime_free(void *g2c_runtime);
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,

14
src/inc_internal/maat_rule.h
View File

@@ -183,18 +183,10 @@ struct maat {
struct log_handle *logger;
struct maat_garbage_bin *garbage_bin;
int default_compile_table_id;
int g2g_table_id; //group2group table id
/* statistics */
struct maat_stat *stat;
};
enum district_flag {
DISTRICT_FLAG_UNSET,
DISTRICT_FLAG_SET
};
enum last_scan_flag {
LAST_SCAN_UNSET,
LAST_SCAN_SET,
@@ -216,12 +208,6 @@ int my_scandir(const char *dir, struct dirent ***namelist,
int(*filter)(const struct dirent *),
int(*compar)(const void *, const void *));
struct item_district *item_district_new(int district_id);
void item_district_free(void *item_district);
int item_district_id(struct item_district *item_dist);
void *rule_monitor_loop(void *arg);
long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *key);

2
src/inc_internal/maat_table.h
View File

@@ -62,7 +62,7 @@ const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int tabl
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id);
int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr);
int table_manager_get_default_compile_table_id(struct table_manager *tbl_mgr);
int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr);
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);

22
src/maat_api.c
View File

@@ -44,6 +44,11 @@
#define MODULE_MAAT_API module_name_str("maat.api")
enum district_flag {
DISTRICT_FLAG_UNSET,
DISTRICT_FLAG_SET
};
struct maat_stream {
struct maat *ref_maat_instance;
struct adapter_hs_stream *s_handle; //each physical table open one stream
@@ -354,9 +359,6 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
goto failed;
}
maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
if (0 == maat_instance->opts.deferred_load_on) {
maat_read_full_config(maat_instance);
}
@@ -1072,7 +1074,7 @@ size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n
if (state->compile_table_id > 0) {
compile_table_id = state->compile_table_id;
} else {
compile_table_id = maat_instance->default_compile_table_id;
compile_table_id = table_manager_get_default_compile_table_id(maat_instance->tbl_mgr);
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
@@ -1869,7 +1871,7 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat
if (state->compile_table_id > 0) {
compile_table_id = state->compile_table_id;
} else {
compile_table_id = maat_instance->default_compile_table_id;
compile_table_id = table_manager_get_default_compile_table_id(maat_instance->tbl_mgr);
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr,
@@ -1878,8 +1880,9 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat
return -1;
}
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr,
maat_instance->g2g_table_id);
int g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, g2g_table_id);
size_t internal_hit_path_cnt = maat_compile_state_get_internal_hit_paths(state->compile_state,
(struct compile_runtime *)compile_rt,
(struct group2group_runtime *)g2g_runtime,
@@ -1896,8 +1899,9 @@ int maat_state_get_hit_groups(struct maat_state *state, struct maat_hit_group *g
return -1;
}
void *g2g_runtime = table_manager_get_runtime(state->maat_instance->tbl_mgr,
state->maat_instance->g2g_table_id);
int g2g_table_id = table_manager_get_group2group_table_id(state->maat_instance->tbl_mgr);
void *g2g_runtime = table_manager_get_runtime(state->maat_instance->tbl_mgr, g2g_table_id);
return maat_compile_state_get_hit_groups(state->compile_state,
(struct group2group_runtime *)g2g_runtime,
groups, n_group);

30
src/maat_compile.c
View File

@@ -45,7 +45,7 @@ struct group2compile_schema {
int not_flag_column;
int vtable_name_column;
int clause_index_column;
char associated_compile_table_id;
int asso_compile_table_id; //asso is abbreviation for associated
int table_id;//ugly
struct table_manager *ref_tbl_mgr;
};
@@ -61,7 +61,7 @@ struct group2compile_item {
int not_flag;
int vtable_id;
int clause_index;
int associated_compile_table_id;
int asso_compile_table_id;
};
struct maat_literal_id {
@@ -108,6 +108,7 @@ struct compile_runtime {
struct maat_clause *clause_by_literals_hash;
struct literal_clause *literal2clause_hash;
struct group_reference *group_ref_hash;
struct group2compile_runtime *ref_g2c_rt;
pthread_mutex_t mutex;
long long rule_num;
long long update_err_cnt;
@@ -392,7 +393,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "associated_compile_table_id");
if (item != NULL && item->type == cJSON_Number) {
g2c_schema->associated_compile_table_id = item->valueint;
g2c_schema->asso_compile_table_id = item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no associated_compile_table_id column",
@@ -474,7 +475,7 @@ int group2compile_associated_compile_table_id(void *g2c_schema)
{
struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema;
return schema->associated_compile_table_id;
return schema->asso_compile_table_id;
}
int compile_accept_tag_match(struct compile_schema *schema, const char *line,
@@ -677,16 +678,23 @@ void compile_runtime_free(void *compile_runtime)
FREE(compile_rt);
}
void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt)
void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt,
void *g2c_runtime)
{
if (NULL == compile_runtime) {
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
if (maat_rt != NULL) {
compile_rt->ref_maat_rt = maat_rt;
}
if (g2c_runtime != NULL) {
compile_rt->ref_g2c_rt = g2c_runtime;
}
}
void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
@@ -700,8 +708,7 @@ void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
return g2c_rt;
}
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime,
void *g2g_runtime)
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime)
{
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
@@ -1648,7 +1655,6 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
struct maat_literal_id literal_id = {group_id, vtable_id};
struct literal_clause *l2c_val = NULL;
long long *clause_id = 0;
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
HASH_FIND(hh, compile_rt->literal2clause_hash, &literal_id, sizeof(literal_id), l2c_val);
@@ -1657,6 +1663,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
}
size_t i = 0;
long long *clause_id = 0;
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
for (i = 0; i < utarray_len(l2c_val->clause_ids); i++) {
clause_id = (long long *)utarray_eltptr(l2c_val->clause_ids, i);
@@ -2180,7 +2187,7 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
if (state->compile_table_id > 0) {
compile_table_id = state->compile_table_id;
} else {
compile_table_id = maat_instance->default_compile_table_id;
compile_table_id = table_manager_get_default_compile_table_id(maat_instance->tbl_mgr);
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr,
@@ -2189,13 +2196,12 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
return;
}
void *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr,
maat_instance->g2g_table_id);
int g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
void *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, g2g_table_id);
if (NULL == g2g_rt) {
return;
}
long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids,
hit_cnt, super_group_ids,

27
src/maat_expr.c
View File

@@ -797,9 +797,11 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
return -1;
}
struct item_district *item_dist = item_district_new(expr_item->district_id);
expr_item->user_data = item_dist;
expr_item->user_data_free = item_district_free;
int *item_district_id = ALLOC(int, 1);
*item_district_id = expr_item->district_id;
expr_item->user_data = item_district_id;
expr_item->user_data_free = free;
}
int ret = expr_runtime_update_row(expr_rt, (char *)&item_id, sizeof(long long),
@@ -975,18 +977,14 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct item_district *item_dist = NULL;
struct expr_item *expr_item = NULL;
struct maat_item hit_maat_items[n_hit_item];
size_t real_hit_item_cnt = 0;
int district_id = state->district_id;
for (size_t i = 0; i < n_hit_item; i++) {
item_dist = (struct item_district *)(hit_results[i].user_tag);
int tag_district_id = item_district_id(item_dist);
if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) {
int tag_district_id = *(int *)(hit_results[i].user_tag);
if (tag_district_id == state->district_id || tag_district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
struct expr_item *expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
(char *)&item_id,
sizeof(long long));
if (!expr_item) {
@@ -994,13 +992,8 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
continue;
}
if (real_hit_item_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
real_hit_item_cnt = MAX_SCANNER_HIT_GROUP_NUM;
// Prevent group_id_array out of bounds
} else {
hit_maat_items[real_hit_item_cnt].item_id = item_id;
hit_maat_items[real_hit_item_cnt].group_id = expr_item->group_id;
}
real_hit_item_cnt++;
}
@@ -1050,7 +1043,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item hit_maat_items[n_hit_item];
struct expr_item *expr_item = NULL;
size_t real_hit_item_cnt = 0;

20
src/maat_flag.c
View File

@@ -425,9 +425,11 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table
return -1;
}
struct item_district *item_dist = item_district_new(flag_item->district_id);
flag_item->user_data = item_dist;
flag_item->user_data_free = item_district_free;
int *item_district_id = ALLOC(int, 1);
*item_district_id = flag_item->district_id;
flag_item->user_data = item_district_id;
flag_item->user_data_free = free;
}
int ret = flag_runtime_update_row(flag_rt, (char *)&item_id, sizeof(long long),
@@ -555,18 +557,14 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct item_district *item_dist = NULL;
struct flag_item *flag_item = NULL;
struct maat_item hit_maat_items[n_hit_item];
size_t real_hit_item_cnt = 0;
int district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) {
item_dist = (struct item_district *)(hit_results[i].user_tag);
int tag_district_id = item_district_id(item_dist);
if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) {
int tag_district_id = *(int *)(hit_results[i].user_tag);
if (tag_district_id == state->district_id || tag_district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
flag_item = (struct flag_item *)rcu_hash_find(flag_rt->item_hash,
struct flag_item *flag_item = (struct flag_item *)rcu_hash_find(flag_rt->item_hash,
(char *)&item_id,
sizeof(long long));
if (!flag_item) {

20
src/maat_interval.c
View File

@@ -429,9 +429,11 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
return -1;
}
struct item_district *item_dist = item_district_new(interval_item->district_id);
interval_item->user_data = item_dist;
interval_item->user_data_free = item_district_free;
int *item_district_id = ALLOC(int, 1);
*item_district_id = interval_item->district_id;
interval_item->user_data = item_district_id;
interval_item->user_data_free = free;
}
int ret = interval_runtime_update_row(interval_rt, (char *)&item_id, sizeof(long long),
@@ -559,18 +561,14 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct item_district *item_dist = NULL;
struct interval_item *int_item = NULL;
struct maat_item hit_maat_items[n_hit_item];
size_t real_hit_item_cnt = 0;
int district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) {
item_dist = (struct item_district *)(hit_results[i].user_tag);
int tag_district_id = item_district_id(item_dist);
if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) {
int tag_district_id = *(int *)(hit_results[i].user_tag);
if (tag_district_id == state->district_id || tag_district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
int_item = (struct interval_item *)rcu_hash_find(interval_rt->item_hash,
struct interval_item *int_item = (struct interval_item *)rcu_hash_find(interval_rt->item_hash,
(char *)&item_id,
sizeof(long long));
if (!int_item) {

29
src/maat_rule.c
View File

@@ -32,33 +32,6 @@
#define MODULE_MAAT_RULE module_name_str("maat.rule")
struct item_district {
int district_id;
};
struct item_district *
item_district_new(int district_id)
{
struct item_district *item_dist = ALLOC(struct item_district, 1);
item_dist->district_id = district_id;
return item_dist;
}
void item_district_free(void *item_district)
{
if (NULL == item_district) {
return;
}
FREE(item_district);
}
int item_district_id(struct item_district *item_dist)
{
return item_dist->district_id;
}
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)
{
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
@@ -107,7 +80,7 @@ void maat_start_cb(long long new_version, int update_type, void *u_param)
if (table_type == TABLE_TYPE_COMPILE) {
// compile runtime need a reference to maat runtime
void *compile_rt = table_manager_get_updating_runtime(maat_instance->tbl_mgr, i);
compile_runtime_init(compile_rt, maat_instance->creating_maat_rt);
compile_runtime_init(compile_rt, maat_instance->creating_maat_rt, NULL);
}
}
} else {

22
src/maat_stat.c
View File

@@ -164,6 +164,11 @@ int maat_fieldstat_table_row_register(struct maat_stat *stat, struct table_manag
continue;
}
enum table_type table_type = table_manager_get_table_type(tbl_mgr, i);
if (table_type == TABLE_TYPE_VIRTUAL) {
continue;
}
const char *table_name = table_manager_get_table_name(tbl_mgr, i);
assert(table_name != NULL);
@@ -376,7 +381,7 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
break;
}
if (table_type == TABLE_TYPE_PLUGIN) {
if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_VIRTUAL) {
continue;
}
@@ -388,23 +393,28 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
total_rule_num += rule_num;
if (table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS) {
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_REGEX_NUM], regex_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_REGEX_NUM],
regex_rule_num);
long long stream_num = expr_runtime_stream_num(runtime);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_STREAM_NUM], stream_num);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_STREAM_NUM],
stream_num);
total_stream_num += stream_num;
long long input_bytes = expr_runtime_scan_bytes(runtime);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_BYTES], input_bytes);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_SCAN_BYTES],
input_bytes);
total_input_bytes += input_bytes;
}
if (table_type == TABLE_TYPE_IP_PLUS) {
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_REGEX_NUM], ipv6_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_REGEX_NUM],
ipv6_rule_num);
}
if (1 == perf_on) {
long long scan_cpu_time = table_manager_runtime_scan_cpu_time(stat->ref_tbl_mgr, i);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_CPU_TIME], scan_cpu_time);
fieldstat_value_set(stat->fs_handle, stat->fs_column_id[i][COLUMN_CPU_TIME],
scan_cpu_time);
total_scan_cpu_time += scan_cpu_time;
}

11
src/maat_table.c
View File

@@ -856,12 +856,9 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_threa
continue;
}
int associated_compile_table_id = group2compile_associated_compile_table_id(schema);
void *compile_updating_rt = table_manager_get_updating_runtime(tbl_mgr, associated_compile_table_id);
int g2g_group_id = table_manager_get_group2group_table_id(tbl_mgr);
void *g2g_updating_rt = table_manager_get_updating_runtime(tbl_mgr, g2g_group_id);
group2compile_runtime_init(g2c_updating_rt, compile_updating_rt, g2g_updating_rt);
int asso_compile_table_id = group2compile_associated_compile_table_id(schema);
void *compile_updating_rt = table_manager_get_updating_runtime(tbl_mgr, asso_compile_table_id);
group2compile_runtime_init(g2c_updating_rt, compile_updating_rt);
}
return 0;
@@ -973,7 +970,7 @@ enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int
return tbl_mgr->tbl[table_id]->table_type;
}
int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr)
int table_manager_get_default_compile_table_id(struct table_manager *tbl_mgr)
{
return tbl_mgr->default_compile_table_id;
}

4
test/maat_framework_gtest.cpp
View File

@@ -23,8 +23,6 @@ const char *json_filename = "maat_json.json";
size_t g_thread_num = 4;
extern int system_cmd_rmdir(const char *dir);
void wait_for_cmd_effective(struct maat *maat_instance, long long version_before)
{
long long version_after = version_before;
@@ -355,7 +353,7 @@ protected:
maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
const char *table_info_path = "./tsg_static_tableinfo.json";
const char *table_info_path = "./test_table_info.conf";
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {

BIN
vendor/hyperscan-5.4.2.tar.gz vendored
View File

Binary file not shown.