gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[OPTIMIZE]delete update_hit_groups to reduce cpu consumption & reduce maat_state memory usage

Browse Source
This commit is contained in:
liuwentan
2023-06-09 16:44:47 +08:00
parent daf9e96e10
commit 48397f754e
12 changed files with 119 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/inc_internal/maat_command.h
View File

@@ -24,6 +24,7 @@ extern "C"
#include "log/log.h" #include "log/log.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
#include "hiredis/hiredis.h" #include "hiredis/hiredis.h"
#include "maat_limits.h"
enum maat_operation { enum maat_operation {
MAAT_OP_DEL = 0, MAAT_OP_DEL = 0,
@@ -50,7 +51,7 @@ struct serial_rule {
enum maat_operation op;//0: delete, 1: add. enum maat_operation op;//0: delete, 1: add.
long long rule_id; long long rule_id;
long long timeout; // absolute unix time. long long timeout; // absolute unix time.
char table_name[NAME_MAX]; char table_name[MAX_NAME_STR_LEN];
char *table_line; char *table_line;
int n_foreign; int n_foreign;
struct foreign_key *f_keys; struct foreign_key *f_keys;

5
src/inc_internal/maat_compile.h
View File

@@ -65,7 +65,7 @@ long long compile_runtime_update_err_count(void *compile_runtime);
int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile_ids, int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile_ids,
size_t compile_ids_size, struct maat_state *state); size_t compile_ids_size, struct maat_state *state);
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thread_id,
struct maat_compile_state *compile_state, struct maat_compile_state *compile_state,
struct maat_hit_path *hit_path_array, struct maat_hit_path *hit_path_array,
size_t array_size, size_t n_internal_hit_path); size_t array_size, size_t n_internal_hit_path);
@@ -96,7 +96,7 @@ long long group2compile_runtime_update_err_count(void *g2c_runtime);
/* maat compile state API */ /* maat compile state API */
struct maat_compile_state; struct maat_compile_state;
struct maat_compile_state *maat_compile_state_new(int thread_id); struct maat_compile_state *maat_compile_state_new(void);
void maat_compile_state_reset(struct maat_compile_state *compile_state); void maat_compile_state_reset(struct maat_compile_state *compile_state);
void maat_compile_state_free(struct maat_compile_state *compile_state); void maat_compile_state_free(struct maat_compile_state *compile_state);
@@ -109,6 +109,7 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp
struct maat_hit_path *hit_path_array, struct maat_hit_path *hit_path_array,
size_t array_size); size_t array_size);
size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state, size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state,
struct group2group_runtime *g2g_rt,
struct maat_hit_group *hit_group_array, struct maat_hit_group *hit_group_array,
size_t array_size); size_t array_size);

1
src/inc_internal/maat_limits.h
View File

@@ -18,6 +18,7 @@ extern "C"
#define MAX_KEYWORDS_STR 1024 #define MAX_KEYWORDS_STR 1024
#define MAX_MAAT_STAT_NUM 64 #define MAX_MAAT_STAT_NUM 64
#define MAX_NAME_STR_LEN 64
#ifdef __cplusplus #ifdef __cplusplus
} }

16
src/inc_internal/maat_rule.h
View File

@@ -201,24 +201,24 @@ enum last_scan_flag {
struct maat_state { struct maat_state {
struct maat *maat_instance; struct maat *maat_instance;
int thread_id;
int compile_table_id;
enum district_flag is_set_district;
enum last_scan_flag is_last_scan;
long long district_id; //-1: Any District; -2: Unkonwn District;
int scan_cnt;
struct maat_compile_state *compile_state; struct maat_compile_state *compile_state;
int scan_cnt;
int district_id; //-1: Any District; -2: Unkonwn District;
uint16_t thread_id;
int16_t compile_table_id;
uint8_t is_set_district;
uint8_t is_last_scan;
}; };
int my_scandir(const char *dir, struct dirent ***namelist, int my_scandir(const char *dir, struct dirent ***namelist,
int(*filter)(const struct dirent *), int(*filter)(const struct dirent *),
int(*compar)(const void *, const void *)); int(*compar)(const void *, const void *));
struct item_district *item_district_new(long long district_id); struct item_district *item_district_new(int district_id);
void item_district_free(void *item_district); void item_district_free(void *item_district);
long long item_district_id(struct item_district *item_dist); int item_district_id(struct item_district *item_dist);
void *rule_monitor_loop(void *arg); void *rule_monitor_loop(void *arg);

2
src/json2iris.c
View File

@@ -42,7 +42,7 @@ struct group_info {
}; };
struct iris_table { struct iris_table {
char table_name[NAME_MAX]; char table_name[MAX_NAME_STR_LEN];
char table_path[PATH_MAX]; char table_path[PATH_MAX];
int line_count; int line_count;
enum table_type table_type; enum table_type table_type;

23
src/maat_api.c
View File

@@ -1772,6 +1772,7 @@ int maat_state_set_scan_district(struct maat_state *state, int vtable_id,
} }
int ret = -1; int ret = -1;
long long district_id;
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_type = table_manager_get_table_type(maat_instance->tbl_mgr,
physical_table_id); physical_table_id);
void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr,
@@ -1781,18 +1782,15 @@ int maat_state_set_scan_district(struct maat_state *state, int vtable_id,
switch (table_type) { switch (table_type) {
case TABLE_TYPE_FLAG_PLUS: case TABLE_TYPE_FLAG_PLUS:
ret = flag_runtime_set_scan_district((struct flag_runtime *)runtime, ret = flag_runtime_set_scan_district((struct flag_runtime *)runtime,
district, district_len, district, district_len, &district_id);
&(state->district_id));
break; break;
case TABLE_TYPE_EXPR_PLUS: case TABLE_TYPE_EXPR_PLUS:
ret = expr_runtime_set_scan_district((struct expr_runtime *)runtime, ret = expr_runtime_set_scan_district((struct expr_runtime *)runtime,
district, district_len, district, district_len, &district_id);
&(state->district_id));
break; break;
case TABLE_TYPE_INTERVAL_PLUS: case TABLE_TYPE_INTERVAL_PLUS:
ret = interval_runtime_set_scan_district((struct interval_runtime *)runtime, ret = interval_runtime_set_scan_district((struct interval_runtime *)runtime,
district, district_len, district, district_len, &district_id);
&(state->district_id));
break; break;
default: default:
log_error(maat_instance->logger, MODULE_MAAT_API, log_error(maat_instance->logger, MODULE_MAAT_API,
@@ -1803,6 +1801,8 @@ int maat_state_set_scan_district(struct maat_state *state, int vtable_id,
if (ret < 0) { if (ret < 0) {
state->district_id = DISTRICT_UNKNOWN; state->district_id = DISTRICT_UNKNOWN;
} else {
state->district_id = (int)district_id;
} }
state->is_set_district = DISTRICT_FLAG_SET; state->is_set_district = DISTRICT_FLAG_SET;
@@ -1881,9 +1881,8 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat
(struct group2group_runtime *)g2g_runtime, (struct group2group_runtime *)g2g_runtime,
paths, n_path); paths, n_path);
return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, state->thread_id,
state->compile_state, paths, n_path, state->compile_state, paths, n_path, internal_hit_path_cnt);
internal_hit_path_cnt);
} }
int maat_state_get_hit_groups(struct maat_state *state, struct maat_hit_group *groups, int maat_state_get_hit_groups(struct maat_state *state, struct maat_hit_group *groups,
@@ -1893,7 +1892,11 @@ int maat_state_get_hit_groups(struct maat_state *state, struct maat_hit_group *g
return -1; return -1;
} }
return maat_compile_state_get_hit_groups(state->compile_state, groups, n_group); void *g2g_runtime = table_manager_get_runtime(state->maat_instance->tbl_mgr,
state->maat_instance->g2g_table_id);
return maat_compile_state_get_hit_groups(state->compile_state,
(struct group2group_runtime *)g2g_runtime,
groups, n_group);
} }
int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group) int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group)

136
src/maat_compile.c
View File

@@ -38,7 +38,6 @@ struct compile_schema {
struct table_manager *ref_tbl_mgr; struct table_manager *ref_tbl_mgr;
struct log_handle *logger; struct log_handle *logger;
int table_id; //ugly int table_id; //ugly
char table_name[NAME_MAX];
}; };
struct group2compile_schema { struct group2compile_schema {
@@ -85,14 +84,14 @@ struct literal_clause {
}; };
struct compile_rule { struct compile_rule {
long long magic_num; uint32_t magic_num;
int declared_clause_num;
long long compile_id; long long compile_id;
char *table_line; char *table_line;
size_t table_line_len; size_t table_line_len;
struct compile_schema *ref_schema; struct compile_schema *ref_schema;
void **ex_data; void **ex_data;
int declared_clause_num; char table_name[MAX_NAME_STR_LEN];
char table_name[NAME_MAX];
}; };
struct group_reference { struct group_reference {
@@ -140,38 +139,35 @@ struct compile_sort_para {
#define MAAT_COMPILE_MAGIC 0x4a5b6c7d #define MAAT_COMPILE_MAGIC 0x4a5b6c7d
struct maat_compile { struct maat_compile {
unsigned int magic; uint32_t magic;
long long compile_id;
char table_name[NAME_MAX];
int actual_clause_num; int actual_clause_num;
int declared_clause_num; int declared_clause_num;
int not_clause_cnt; int not_clause_cnt;
long long compile_id;
char table_name[MAX_NAME_STR_LEN];
void *user_data; void *user_data;
void (*user_data_free)(void *); void (*user_data_free)(void *);
UT_hash_handle hh;
struct maat_clause_state clause_states[MAX_ITEMS_PER_BOOL_EXPR]; struct maat_clause_state clause_states[MAX_ITEMS_PER_BOOL_EXPR];
}; };
struct maat_internal_hit_path { struct maat_internal_hit_path {
int Nth_scan;
int Nth_hit_item;
long long item_id; long long item_id;
long long group_id; long long group_id;
int Nth_scan;
int Nth_hit_item;
int vtable_id; int vtable_id;
}; };
struct maat_compile_state { struct maat_compile_state {
int thread_id; uint8_t this_scan_hit_item_flag;
uint8_t not_clause_hit_flag;
int Nth_scan; int Nth_scan;
time_t compile_rt_version; time_t compile_rt_version;
size_t this_scan_hit_item_cnt;
int not_clause_hitted_flag;
size_t hit_path_cnt;
UT_array *internal_hit_paths; UT_array *internal_hit_paths;
UT_array *all_hit_clauses; UT_array *all_hit_clauses;
UT_array *this_scan_hit_clauses; UT_array *this_scan_hit_clauses;
UT_array *all_hit_groups; // UT_array *all_hit_groups;
}; };
UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL}; UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL};
@@ -746,7 +742,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
{ {
size_t column_offset = 0; size_t column_offset = 0;
size_t column_len = 0; size_t column_len = 0;
char vtable_name[NAME_MAX] = {0}; char vtable_name[MAX_NAME_STR_LEN] = {0};
struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1); struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1);
int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset,
@@ -788,11 +784,11 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
goto error; goto error;
} }
if (column_len > NAME_MAX) { if (column_len > MAX_NAME_STR_LEN) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> virtual_table_name length exceed " "[%s:%d] g2c table:<%s> virtual_table_name length exceed "
"maxium:%d in line:%s", __FUNCTION__, __LINE__, table_name, "maxium:%d in line:%s", __FUNCTION__, __LINE__, table_name,
NAME_MAX, line); MAX_NAME_STR_LEN, line);
goto error; goto error;
} }
@@ -1127,14 +1123,14 @@ static size_t compile_state_if_new_hit_compile(struct maat_compile_state *compil
} }
size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int is_last_scan, size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int is_last_scan,
struct maat_compile_state *compile_state, struct maat_compile_state *compile_state, int thread_id,
void **user_data_array, size_t ud_array_size) void **user_data_array, size_t ud_array_size)
{ {
size_t ud_result_cnt = 0; size_t ud_result_cnt = 0;
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + struct bool_expr_match *expr_match = compile_rt->expr_match_buff +
(compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM); (thread_id * MAX_SCANNER_HIT_COMPILE_NUM);
assert(compile_state->thread_id >= 0); assert(thread_id >= 0);
if (0 == compile_state->compile_rt_version) { if (0 == compile_state->compile_rt_version) {
compile_state->compile_rt_version = compile_rt->version; compile_state->compile_rt_version = compile_rt->version;
@@ -1142,7 +1138,7 @@ size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int i
if (NULL == compile_rt->bm || 0 == utarray_len(compile_state->all_hit_clauses) if (NULL == compile_rt->bm || 0 == utarray_len(compile_state->all_hit_clauses)
|| compile_state->compile_rt_version != compile_rt->version) { || compile_state->compile_rt_version != compile_rt->version) {
compile_state->this_scan_hit_item_cnt = 0; compile_state->this_scan_hit_item_flag = 0;
return 0; return 0;
} }
@@ -1168,11 +1164,11 @@ size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int i
} }
size_t n_new_hit_compile = compile_state_if_new_hit_compile(compile_state, compile); size_t n_new_hit_compile = compile_state_if_new_hit_compile(compile_state, compile);
size_t n_this_scan_hit_item = compile_state->this_scan_hit_item_cnt; int this_scan_hit_item_flag = compile_state->this_scan_hit_item_flag;
if ((compile->not_clause_cnt > 0) && (LAST_SCAN_UNSET == is_last_scan)) { if ((compile->not_clause_cnt > 0) && (LAST_SCAN_UNSET == is_last_scan)) {
compile_state->not_clause_hitted_flag = 1; compile_state->not_clause_hit_flag = 1;
} else if (compile->user_data) { } else if (compile->user_data) {
if (n_new_hit_compile > 0 || 0 == n_this_scan_hit_item) { if (n_new_hit_compile > 0 || 0 == this_scan_hit_item_flag) {
/* compile hit because of new item or /* compile hit because of new item or
hit a compile that refer a NOT-logic group in previous scan */ hit a compile that refer a NOT-logic group in previous scan */
user_data_array[ud_result_cnt] = compile->user_data; user_data_array[ud_result_cnt] = compile->user_data;
@@ -1181,7 +1177,7 @@ size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int i
} }
} }
compile_state->this_scan_hit_item_cnt = 0; compile_state->this_scan_hit_item_flag = 0;
return ud_result_cnt; return ud_result_cnt;
} }
@@ -1455,15 +1451,13 @@ int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
return ret; return ret;
} }
struct maat_compile_state *maat_compile_state_new(int thread_id) struct maat_compile_state *maat_compile_state_new(void)
{ {
struct maat_compile_state *compile_state = ALLOC(struct maat_compile_state, 1); struct maat_compile_state *compile_state = ALLOC(struct maat_compile_state, 1);
compile_state->thread_id = thread_id;
utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd); utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd);
utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->all_hit_groups, &ut_hit_group_icd);
return compile_state; return compile_state;
} }
@@ -1476,14 +1470,12 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state)
compile_state->Nth_scan = 0; compile_state->Nth_scan = 0;
compile_state->compile_rt_version = 0; compile_state->compile_rt_version = 0;
compile_state->this_scan_hit_item_cnt = 0; compile_state->this_scan_hit_item_flag = 0;
compile_state->not_clause_hitted_flag = 0; compile_state->not_clause_hit_flag = 0;
compile_state->hit_path_cnt = 0;
utarray_clear(compile_state->internal_hit_paths); utarray_clear(compile_state->internal_hit_paths);
utarray_clear(compile_state->all_hit_clauses); utarray_clear(compile_state->all_hit_clauses);
utarray_clear(compile_state->this_scan_hit_clauses); utarray_clear(compile_state->this_scan_hit_clauses);
utarray_clear(compile_state->all_hit_groups);
} }
void maat_compile_state_free(struct maat_compile_state *compile_state) void maat_compile_state_free(struct maat_compile_state *compile_state)
@@ -1507,11 +1499,6 @@ void maat_compile_state_free(struct maat_compile_state *compile_state)
compile_state->this_scan_hit_clauses = NULL; compile_state->this_scan_hit_clauses = NULL;
} }
if (compile_state->all_hit_groups != NULL) {
utarray_free(compile_state->all_hit_groups);
compile_state->all_hit_groups = NULL;
}
FREE(compile_state); FREE(compile_state);
} }
@@ -1569,7 +1556,7 @@ static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_path
return 0; return 0;
} }
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thread_id,
struct maat_compile_state *compile_state, struct maat_compile_state *compile_state,
struct maat_hit_path *hit_path_array, struct maat_hit_path *hit_path_array,
size_t array_size, size_t n_internal_hit_path) size_t array_size, size_t n_internal_hit_path)
@@ -1579,8 +1566,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
struct maat_literal_id literal_id = {0, 0}; struct maat_literal_id literal_id = {0, 0};
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + struct bool_expr_match *expr_match = compile_rt->expr_match_buff +
(compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM); (thread_id * MAX_SCANNER_HIT_COMPILE_NUM);
assert(compile_state->thread_id >= 0); assert(thread_id >= 0);
if (compile_state->compile_rt_version != compile_rt->version) { if (compile_state->compile_rt_version != compile_rt->version) {
return 0; return 0;
@@ -1634,7 +1621,7 @@ void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state
int Nth_scan, int Nth_item_result) int Nth_scan, int Nth_item_result)
{ {
if (compile_state->Nth_scan != Nth_scan) { if (compile_state->Nth_scan != Nth_scan) {
assert(compile_state->this_scan_hit_item_cnt == 0); assert(compile_state->this_scan_hit_item_flag == 0);
compile_state->Nth_scan = Nth_scan; compile_state->Nth_scan = Nth_scan;
utarray_clear(compile_state->this_scan_hit_clauses); utarray_clear(compile_state->this_scan_hit_clauses);
} }
@@ -1642,8 +1629,7 @@ void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state
maat_compile_hit_path_add(compile_state->internal_hit_paths, item_id, group_id, maat_compile_hit_path_add(compile_state->internal_hit_paths, item_id, group_id,
vtable_id, Nth_scan, Nth_item_result); vtable_id, Nth_scan, Nth_item_result);
compile_state->hit_path_cnt++; compile_state->this_scan_hit_item_flag = 1;
compile_state->this_scan_hit_item_cnt++;
} }
void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state,
@@ -1686,25 +1672,9 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
} }
} }
void maat_compile_state_update_hit_group(struct maat_compile_state *compile_state,
long long group_id, int vtable_id)
{
if (NULL == compile_state) {
return;
}
struct maat_hit_group hit_group = {group_id, vtable_id};
if (utarray_find(compile_state->all_hit_groups, &hit_group, compare_hit_group)) {
return;
}
utarray_push_back(compile_state->all_hit_groups, &hit_group);
utarray_sort(compile_state->all_hit_groups, compare_hit_group);
}
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state) int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state)
{ {
return compile_state->not_clause_hitted_flag; return compile_state->not_clause_hit_flag;
} }
void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt, void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
@@ -2160,7 +2130,7 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
// all hit clause_id -> compile_id // all hit clause_id -> compile_id
size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, is_last_scan, size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, is_last_scan,
compile_state, compile_state, state->thread_id,
(void **)compile_rules, (void **)compile_rules,
compile_ids_size); compile_ids_size);
if (bool_match_ret > 0) { if (bool_match_ret > 0) {
@@ -2188,7 +2158,7 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
struct maat *maat_instance = state->maat_instance; struct maat *maat_instance = state->maat_instance;
if (NULL == state->compile_state) { if (NULL == state->compile_state) {
state->compile_state = maat_compile_state_new(state->thread_id); state->compile_state = maat_compile_state_new();
alignment_int64_array_add(maat_instance->stat->maat_compile_state_cnt, alignment_int64_array_add(maat_instance->stat->maat_compile_state_cnt,
state->thread_id, 1); state->thread_id, 1);
} }
@@ -2231,19 +2201,16 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
for (int j = 0; j < super_group_cnt; j++) { for (int j = 0; j < super_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt, maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
super_group_ids[j], vtable_id); super_group_ids[j], vtable_id);
maat_compile_state_update_hit_group(state->compile_state, super_group_ids[j],
vtable_id);
} }
for (int j = 0; j < hit_cnt; j++) { for (int j = 0; j < hit_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt, maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
hit_group_ids[j], vtable_id); hit_group_ids[j], vtable_id);
maat_compile_state_update_hit_group(state->compile_state, hit_group_ids[j],
vtable_id);
} }
} }
size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state, size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_state,
struct group2group_runtime *g2g_rt,
struct maat_hit_group *hit_group_array, struct maat_hit_group *hit_group_array,
size_t array_size) size_t array_size)
{ {
@@ -2252,14 +2219,39 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
} }
size_t i = 0; size_t i = 0;
struct maat_hit_group *hit_group = NULL; UT_array *all_hit_groups;
utarray_new(all_hit_groups, &ut_hit_group_icd);
struct maat_internal_hit_path *internal_path = NULL;
for (i = 0; i < utarray_len(compile_state->all_hit_groups) && i < array_size; i++) { for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) {
hit_group = (struct maat_hit_group *)utarray_eltptr(compile_state->all_hit_groups, i); internal_path = (struct maat_internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i);
hit_group_array[i].group_id = hit_group->group_id;
hit_group_array[i].vtable_id = hit_group->vtable_id; long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, &(internal_path->group_id), 1,
super_group_ids, MAX_SCANNER_HIT_GROUP_NUM);
if (super_group_cnt + 1 <= MAX_SCANNER_HIT_GROUP_NUM) {
super_group_ids[super_group_cnt++] = internal_path->group_id;
} }
for (size_t idx = 0; idx < super_group_cnt; idx++) {
struct maat_hit_group hit_group;
hit_group.group_id = super_group_ids[idx];
hit_group.vtable_id = internal_path->vtable_id;
if (utarray_find(all_hit_groups, &hit_group, compare_hit_group)) {
continue;
}
utarray_push_back(all_hit_groups, &hit_group);
utarray_sort(all_hit_groups, compare_hit_group);
}
}
struct maat_hit_group *tmp = NULL;
for (i = 0; i < utarray_len(all_hit_groups) && i < array_size; i++) {
tmp = (struct maat_hit_group *)utarray_eltptr(all_hit_groups, i);
hit_group_array[i] = *tmp;
}
utarray_free(all_hit_groups);
return i; return i;
} }

10
src/maat_expr.c
View File

@@ -58,7 +58,6 @@ enum match_method {
struct expr_item { struct expr_item {
long long item_id; long long item_id;
long long group_id; long long group_id;
long long district_id;
char keywords[MAX_KEYWORDS_STR]; char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type; enum expr_type expr_type;
enum hs_match_mode match_mode; enum hs_match_mode match_mode;
@@ -66,6 +65,7 @@ struct expr_item {
int is_case_sensitive; int is_case_sensitive;
void *user_data; void *user_data;
void (*user_data_free)(void *data); void (*user_data_free)(void *data);
int district_id;
}; };
struct expr_runtime { struct expr_runtime {
@@ -140,7 +140,7 @@ enum hs_match_mode int_to_match_mode(int match_method)
return mode; return mode;
} }
long long expr_runtime_get_district_id(struct expr_runtime *expr_rt, const char *district) int expr_runtime_get_district_id(struct expr_runtime *expr_rt, const char *district)
{ {
long long district_id = DISTRICT_ANY; long long district_id = DISTRICT_ANY;
@@ -158,7 +158,7 @@ long long expr_runtime_get_district_id(struct expr_runtime *expr_rt, const char
} }
} }
return district_id; return (int)district_id;
} }
int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district, int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district,
@@ -979,11 +979,11 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
struct item_district *item_dist = NULL; struct item_district *item_dist = NULL;
struct expr_item *expr_item = NULL; struct expr_item *expr_item = NULL;
size_t real_hit_item_cnt = 0; size_t real_hit_item_cnt = 0;
long long district_id = state->district_id; int district_id = state->district_id;
for (size_t i = 0; i < n_hit_item; i++) { for (size_t i = 0; i < n_hit_item; i++) {
item_dist = (struct item_district *)(hit_results[i].user_tag); item_dist = (struct item_district *)(hit_results[i].user_tag);
long long tag_district_id = item_district_id(item_dist); int tag_district_id = item_district_id(item_dist);
if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) { if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id; long long item_id = hit_results[i].rule_id;
expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash, expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,

10
src/maat_flag.c
View File

@@ -37,11 +37,11 @@ struct flag_schema {
struct flag_item { struct flag_item {
long long item_id; long long item_id;
long long group_id; long long group_id;
long long district_id;
long long flag; long long flag;
long long flag_mask; long long flag_mask;
void *user_data; void *user_data;
void (*user_data_free)(void *data); void (*user_data_free)(void *data);
int district_id;
}; };
struct flag_runtime { struct flag_runtime {
@@ -266,7 +266,7 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_
return 0; return 0;
} }
long long flag_runtime_get_district_id(struct flag_runtime *flag_rt, const char *district) int flag_runtime_get_district_id(struct flag_runtime *flag_rt, const char *district)
{ {
long long district_id = DISTRICT_ANY; long long district_id = DISTRICT_ANY;
@@ -284,7 +284,7 @@ long long flag_runtime_get_district_id(struct flag_runtime *flag_rt, const char
} }
} }
return district_id; return (int)district_id;
} }
int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *district, int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *district,
@@ -559,11 +559,11 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
struct item_district *item_dist = NULL; struct item_district *item_dist = NULL;
struct flag_item *flag_item = NULL; struct flag_item *flag_item = NULL;
size_t real_hit_item_cnt = 0; size_t real_hit_item_cnt = 0;
long long district_id = state->district_id; int district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) { for (int i = 0; i < n_hit_item; i++) {
item_dist = (struct item_district *)(hit_results[i].user_tag); item_dist = (struct item_district *)(hit_results[i].user_tag);
long long tag_district_id = item_district_id(item_dist); int tag_district_id = item_district_id(item_dist);
if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) { if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id; long long item_id = hit_results[i].rule_id;
flag_item = (struct flag_item *)rcu_hash_find(flag_rt->item_hash, flag_item = (struct flag_item *)rcu_hash_find(flag_rt->item_hash,

14
src/maat_interval.c
View File

@@ -34,11 +34,11 @@ struct interval_schema {
struct interval_item { struct interval_item {
long long item_id; long long item_id;
long long group_id; long long group_id;
long long district_id;
int low_bound; int low_bound;
int up_bound; int up_bound;
void *user_data; void *user_data;
void (*user_data_free)(void *data); void (*user_data_free)(void *data);
int district_id;
}; };
struct interval_runtime { struct interval_runtime {
@@ -240,7 +240,7 @@ void interval_runtime_free(void *interval_runtime)
FREE(interval_rt); FREE(interval_rt);
} }
long long interval_runtime_get_district_id(struct interval_runtime *interval_rt, int interval_runtime_get_district_id(struct interval_runtime *interval_rt,
const char *district) const char *district)
{ {
long long district_id = DISTRICT_ANY; long long district_id = DISTRICT_ANY;
@@ -259,7 +259,7 @@ long long interval_runtime_get_district_id(struct interval_runtime *interval_rt,
} }
} }
return district_id; return (int)district_id;
} }
int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, int interval_runtime_set_scan_district(struct interval_runtime *interval_rt,
@@ -270,8 +270,8 @@ int interval_runtime_set_scan_district(struct interval_runtime *interval_rt,
return -1; return -1;
} }
return maat_kv_read_unNull(interval_rt->district_map, district, district_len, return maat_kv_read_unNull(interval_rt->district_map, district,
district_id); district_len, district_id);
} }
struct interval_item * struct interval_item *
@@ -563,11 +563,11 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
struct item_district *item_dist = NULL; struct item_district *item_dist = NULL;
struct interval_item *int_item = NULL; struct interval_item *int_item = NULL;
size_t real_hit_item_cnt = 0; size_t real_hit_item_cnt = 0;
long long district_id = state->district_id; int district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) { for (int i = 0; i < n_hit_item; i++) {
item_dist = (struct item_district *)(hit_results[i].user_tag); item_dist = (struct item_district *)(hit_results[i].user_tag);
long long tag_district_id = item_district_id(item_dist); int tag_district_id = item_district_id(item_dist);
if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) { if (tag_district_id == district_id || tag_district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id; long long item_id = hit_results[i].rule_id;
int_item = (struct interval_item *)rcu_hash_find(interval_rt->item_hash, int_item = (struct interval_item *)rcu_hash_find(interval_rt->item_hash,

15
src/maat_rule.c
View File

@@ -33,16 +33,13 @@
#define MODULE_MAAT_RULE module_name_str("maat.rule") #define MODULE_MAAT_RULE module_name_str("maat.rule")
struct item_district { struct item_district {
long long magic_num; int district_id;
long long district_id;
}; };
#define ITEM_RULE_MAGIC 0x4d3c2b1a
struct item_district * struct item_district *
item_district_new(long long district_id) item_district_new(int district_id)
{ {
struct item_district *item_dist = ALLOC(struct item_district, 1); struct item_district *item_dist = ALLOC(struct item_district, 1);
item_dist->magic_num = ITEM_RULE_MAGIC;
item_dist->district_id = district_id; item_dist->district_id = district_id;
return item_dist; return item_dist;
@@ -54,14 +51,10 @@ void item_district_free(void *item_district)
return; return;
} }
struct item_district *item_dist = (struct item_district *)item_district; FREE(item_district);
assert(item_dist->magic_num == ITEM_RULE_MAGIC);
item_dist->magic_num = 0;
FREE(item_dist);
} }
long long item_district_id(struct item_district *item_dist) int item_district_id(struct item_district *item_dist)
{ {
return item_dist->district_id; return item_dist->district_id;
} }

2
src/maat_table.c
View File

@@ -32,7 +32,7 @@
struct maat_table { struct maat_table {
int table_id; int table_id;
char table_name[NAME_MAX]; char table_name[MAX_NAME_STR_LEN];
enum table_type table_type; enum table_type table_type;
int valid_column; int valid_column;
void *schema; void *schema;