gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FEATURE]Compile table must register plugin table to get compile ex_data

Browse Source
This commit is contained in:
刘文坛
2023-10-30 08:00:49 +00:00
parent 732c709ac6
commit 5e907a171f
24 changed files with 920 additions and 709 deletions
Download Patch File Download Diff File Expand all files Collapse all files

339
src/maat_compile.c
View File

@@ -28,7 +28,6 @@
#define MODULE_COMPILE module_name_str("maat.compile")
#define DEFAULT_GC_TIMEOUT_S 10
#define MAX_SUPER_GROUP_CNT 128
#define MAX_NOT_CLAUSE_NUM 8
#define VTABLE_MAX_NOT_GROUP_NUM 8
@@ -42,10 +41,7 @@ struct compile_schema {
int compile_id_column;
int rule_tag_column;
int declared_clause_num_column;
int set_flag;
int gc_timeout_s;
int table_id; //ugly
struct ex_data_schema ex_schema;
struct table_manager *ref_tbl_mgr;
struct log_handle *logger;
};
@@ -66,8 +62,6 @@ struct compile_item {
long long compile_id;
char *table_line;
size_t table_line_len;
struct compile_schema *ref_schema;
void **ex_data;
char table_name[MAX_NAME_STR_LEN];
};
@@ -100,7 +94,8 @@ struct table_not_clause {
/* compile_runtime and group2compile_runtime share compile_hash_map */
struct compile_runtime {
struct bool_matcher *bm;
struct rcu_hash_table *cfg_hash; // <compile_id, struct maat_compile>
struct rcu_hash_table *cfg_hash; // <compile_id, struct maat_compile>
struct rcu_hash_table *tbl_cfg_hash; // <compile_id, table_id>
struct maat_runtime *ref_maat_rt;
time_t version;
struct literal_clause *literal2clause_hash; //store clause_ids(not_flag == 0)
@@ -153,6 +148,11 @@ struct internal_hit_path {
int NOT_flag; // 1 means NOT clause
};
struct compile2table_id {
long long compile_id;
int table_id;
};
struct compile_state {
int Nth_scan;
time_t compile_rt_version;
@@ -162,12 +162,14 @@ struct compile_state {
UT_array *this_scan_hit_clauses;
UT_array *direct_hit_groups;
UT_array *indirect_hit_groups;
UT_array *hit_compile_table_ids;
};
UT_icd ut_literal_id_icd = {sizeof(struct literal_id), NULL, NULL, NULL};
UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL};
UT_icd ut_maat_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL};
UT_icd ut_hit_path_icd = {sizeof(struct internal_hit_path), NULL, NULL, NULL};
UT_icd ut_hit_compile_table_id_icd = {sizeof(struct compile2table_id), NULL, NULL, NULL};
static struct maat_compile *maat_compile_new(long long compile_id)
{
@@ -200,23 +202,6 @@ static int maat_compile_set(struct maat_compile *compile, const char *table_name
return 0;
}
static void *rule_ex_data_new(const char *table_name, int table_id,
const char *table_line,
struct ex_data_schema *ex_schema)
{
void *ex_data = NULL;
ex_schema->new_func(table_name, table_id, NULL, table_line, &ex_data,
ex_schema->argl, ex_schema->argp);
return ex_data;
}
static void rule_ex_data_free(int table_id, void **ex_data,
const struct ex_data_schema *ex_schema)
{
ex_schema->free_func(table_id, ex_data, ex_schema->argl, ex_schema->argp);
}
static int compile_accept_tag_match(struct compile_schema *schema, const char *line,
const char *table_name, struct log_handle *logger)
{
@@ -300,18 +285,11 @@ compile_item_new(const char *table_line, struct compile_schema *schema,
goto error;
}
compile_item->ref_schema = schema;
compile_item->ex_data = ALLOC(void *, 1);
memcpy(compile_item->table_name, table_name, sizeof(compile_item->table_name));
compile_item->table_line_len = strlen(table_line);
compile_item->table_line = ALLOC(char, compile_item->table_line_len + 1);
memcpy(compile_item->table_line, table_line, compile_item->table_line_len);
if (1 == schema->set_flag) {
*(compile_item->ex_data) = rule_ex_data_new(table_name, schema->table_id,
compile_item->table_line,
&(schema->ex_schema));
}
return compile_item;
error:
FREE(compile_item);
@@ -320,17 +298,6 @@ error:
static void compile_item_free(struct compile_item *item)
{
struct compile_schema *schema = item->ref_schema;
if (1 == schema->set_flag) {
rule_ex_data_free(schema->table_id, item->ex_data, &(schema->ex_schema));
*item->ex_data = NULL;
}
if (item->ex_data != NULL) {
FREE(item->ex_data);
}
item->declared_clause_num = 0;
if (item->table_line != NULL) {
@@ -371,70 +338,9 @@ static void rcu_compile_cfg_free(void *user_ctx, void *data)
maat_compile_free(compile);
}
int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp)
static void rcu_compile_table_cfg_free(void *user_ctx, void *data)
{
if (1 == compile_schema->set_flag) {
log_error(compile_schema->logger, MODULE_COMPILE,
"[%s:%d] compile table(table_id:%d)ex schema has been set already,"
" can't set again", __FUNCTION__, __LINE__, table_id);
return -1;
}
compile_schema->ex_schema.new_func = new_func;
compile_schema->ex_schema.free_func = free_func;
compile_schema->ex_schema.dup_func = dup_func;
compile_schema->ex_schema.argl = argl;
compile_schema->ex_schema.argp = argp;
compile_schema->set_flag = 1;
return 0;
}
static void *compile_runtime_get_user_data(struct compile_runtime *compile_rt,
long long compile_id)
{
struct maat_compile *compile = rcu_hash_find(compile_rt->cfg_hash,
(char *)&compile_id,
sizeof(long long));
void *ret = NULL;
if (compile != NULL) {
ret = compile->user_data;
}
return ret;
}
static void rule_ex_data_new_cb(void *user_data, void *param,
const char *table_name, int table_id)
{
struct ex_data_schema *ex_schema = (struct ex_data_schema *)param;
struct compile_item *compile = (struct compile_item *)user_data;
void *ad = rule_ex_data_new(table_name, table_id, compile->table_line, ex_schema);
*compile->ex_data = ad;
}
static void compile_runtime_user_data_iterate(struct compile_runtime *compile_rt,
void (*callback)(void *user_data, void *param,
const char *table_name, int table_id),
void *param, int table_id)
{
/* I'm in background_update_mutex, config update can't happen, so no need to lock cfg_hash */
void **data_array = NULL;
size_t data_cnt = rcu_hash_list(compile_rt->cfg_hash, &data_array);
for (size_t i = 0; i < data_cnt; i++) {
struct maat_compile *compile = (struct maat_compile *)data_array[i];
if (compile->user_data) {
callback(compile->user_data, param, compile->table_name, table_id);
}
}
FREE(data_array);
FREE(data);
}
void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
@@ -488,12 +394,6 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
goto error;
}
//gc_timeout_s is optional
custom_item = cJSON_GetObjectItem(item, "gc_timeout_s");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->gc_timeout_s = custom_item->valueint;
}
schema->ref_tbl_mgr = tbl_mgr;
return schema;
error:
@@ -617,14 +517,13 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
return NULL;
}
struct compile_schema *schema = (struct compile_schema *)compile_schema;
struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1);
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match,
max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM);
compile_rt->version = time(NULL);
compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL,
schema->gc_timeout_s + DEFAULT_GC_TIMEOUT_S);
compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL, 0);
compile_rt->tbl_cfg_hash = rcu_hash_new(rcu_compile_table_cfg_free, NULL, 0);
compile_rt->literal2clause_hash = NULL;
compile_rt->literal2not_clause_hash = NULL;
compile_rt->logger = logger;
@@ -672,6 +571,11 @@ void compile_runtime_free(void *compile_runtime)
compile_rt->cfg_hash = NULL;
}
if (compile_rt->tbl_cfg_hash != NULL) {
rcu_hash_free(compile_rt->tbl_cfg_hash);
compile_rt->tbl_cfg_hash = NULL;
}
if (compile_rt->literal2clause_hash != NULL) {
literal2clause_hash_free(compile_rt->literal2clause_hash);
compile_rt->literal2clause_hash = NULL;
@@ -1040,6 +944,35 @@ static inline int compare_clause_id(const void *a, const void *b)
}
}
static inline int compare_hit_group(const void *pa, const void *pb)
{
struct maat_hit_group *la=(struct maat_hit_group *)pa;
struct maat_hit_group *lb=(struct maat_hit_group *)pb;
long long ret = la->item_id - lb->item_id;
if (0 == ret) {
ret = la->group_id - lb->group_id;
if (0 == ret) {
ret = la->vtable_id - lb->vtable_id;
}
}
return ret;
}
static inline int compare_compile_id(const void *a, const void *b)
{
long long ret = *(const long long *)a - *(const long long *)b;
if (0 == ret) {
return 0;
} else if (ret < 0) {
return -1;
} else {
return 1;
}
}
/**
* @brief build <literal, clause_id_array> hash for clause or not_clause
*
@@ -1140,10 +1073,33 @@ static size_t compile_state_if_new_hit_compile(struct compile_state *compile_sta
return r_in_c_cnt;
}
size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt,
struct compile_state *compile_state,
int thread_id, void **user_data_array,
size_t ud_array_size)
static void compile_state_update_hit_compile_table_id(struct compile_state *compile_state,
long long compile_id, int table_id)
{
if (!utarray_find(compile_state->hit_compile_table_ids, &compile_id, compare_compile_id)) {
struct compile2table_id compile_table_id = {compile_id, table_id};
utarray_push_back(compile_state->hit_compile_table_ids, &compile_table_id);
utarray_sort(compile_state->hit_compile_table_ids, compare_compile_id);
}
}
static int compile_runtime_get_compile_table_id(struct compile_runtime *compile_rt,
long long compile_id)
{
if (NULL == compile_rt || compile_id < 0) {
return -1;
}
int *table_id = rcu_hash_find(compile_rt->tbl_cfg_hash, (char *)&compile_id,
sizeof(long long));
return *table_id;
}
static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt,
struct compile_state *compile_state,
int thread_id, void **user_data_array,
size_t ud_array_size)
{
size_t ud_result_cnt = 0;
struct maat_compile *compile = NULL;
@@ -1186,6 +1142,11 @@ size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt,
if (compile->user_data != NULL && n_new_hit_compile > 0) {
user_data_array[ud_result_cnt] = compile->user_data;
ud_result_cnt++;
int table_id = compile_runtime_get_compile_table_id(compile_rt, compile->compile_id);
if (table_id >= 0) {
compile_state_update_hit_compile_table_id(compile_state, compile->compile_id, table_id);
}
}
}
@@ -1198,18 +1159,11 @@ static struct compile_item *compile_item_clone(struct compile_item *item)
new_item->compile_id = item->compile_id;
new_item->declared_clause_num = item->declared_clause_num;
new_item->ref_schema = item->ref_schema;
new_item->ex_data = ALLOC(void *, 1);
memcpy(new_item->table_name, item->table_name, sizeof(new_item->table_name));
new_item->table_line_len = item->table_line_len;
new_item->table_line = ALLOC(char, new_item->table_line_len + 1);
memcpy(new_item->table_line, item->table_line, new_item->table_line_len);
if (1 == item->ref_schema->set_flag) {
*(new_item->ex_data) = rule_ex_data_new(item->table_name, item->ref_schema->table_id,
item->table_line, &(item->ref_schema->ex_schema));
}
return new_item;
}
@@ -1420,6 +1374,7 @@ struct compile_state *compile_state_new(void)
utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->direct_hit_groups, &ut_maat_hit_group_icd);
utarray_new(compile_state->indirect_hit_groups, &ut_maat_hit_group_icd);
utarray_new(compile_state->hit_compile_table_ids, &ut_hit_compile_table_id_icd);
return compile_state;
}
@@ -1438,6 +1393,7 @@ void compile_state_reset(struct compile_state *compile_state)
utarray_clear(compile_state->this_scan_hit_clauses);
utarray_clear(compile_state->direct_hit_groups);
utarray_clear(compile_state->indirect_hit_groups);
utarray_clear(compile_state->hit_compile_table_ids);
}
void compile_state_free(struct compile_state *compile_state,
@@ -1479,6 +1435,12 @@ void compile_state_free(struct compile_state *compile_state,
compile_state->indirect_hit_groups = NULL;
}
if (compile_state->hit_compile_table_ids != NULL) {
free_bytes += utarray_size(compile_state->hit_compile_table_ids) * sizeof(struct compile2table_id);
utarray_free(compile_state->hit_compile_table_ids);
compile_state->hit_compile_table_ids = NULL;
}
FREE(compile_state);
free_bytes += sizeof(struct compile_state);
@@ -1486,9 +1448,9 @@ void compile_state_free(struct compile_state *compile_state,
thread_id, free_bytes);
}
static void maat_compile_hit_path_add(UT_array *hit_paths, long long item_id,
long long group_id, int vtable_id, int NOT_flag,
int Nth_scan)
static void add_internal_hit_path(UT_array *hit_paths, long long item_id,
long long group_id, int vtable_id, int NOT_flag,
int Nth_scan)
{
if (NULL == hit_paths || utarray_len(hit_paths) >= MAX_HIT_PATH_NUM) {
return;
@@ -1601,9 +1563,9 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
return (n_hit_path + new_hit_path_cnt);
}
static void compile_state_update_direct_hit_groups(UT_array *hit_group_array,
struct maat_item *hit_items,
size_t n_hit_items, int vtable_id)
static void update_direct_hit_groups(UT_array *hit_group_array,
struct maat_item *hit_items,
size_t n_hit_items, int vtable_id)
{
if (NULL == hit_group_array) {
return;
@@ -1618,9 +1580,9 @@ static void compile_state_update_direct_hit_groups(UT_array *hit_group_array,
}
}
static void compile_state_update_indirect_hit_groups(UT_array *hit_group_array,
long long *group_ids,
size_t n_group_ids, int vtable_id)
static void update_indirect_hit_groups(UT_array *hit_group_array,
long long *group_ids,
size_t n_group_ids, int vtable_id)
{
if (NULL == hit_group_array) {
return;
@@ -1696,10 +1658,10 @@ static inline int compare_group_id(const void *a, const void *b)
}
static size_t compile_state_update_hit_not_clauses(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
long long *group_ids, size_t n_group_ids,
int vtable_id, long long *NOT_group_ids_array,
size_t NOT_group_ids_array_size)
struct compile_runtime *compile_rt,
long long *group_ids, size_t n_group_ids,
int vtable_id, long long *NOT_group_ids_array,
size_t NOT_group_ids_array_size)
{
if (NULL == compile_state || NULL == compile_rt) {
return 0;
@@ -1734,39 +1696,18 @@ static size_t compile_state_update_hit_not_clauses(struct compile_state *compile
return hit_NOT_group_cnt;
}
void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema)
int compile_state_get_compile_table_id(struct compile_state *compile_state,
long long compile_id)
{
if (NULL == compile_rt || NULL == compile_schema ||
(0 == compile_schema->set_flag)) {
return;
}
struct compile2table_id *tmp = NULL;
compile_runtime_user_data_iterate(compile_rt, rule_ex_data_new_cb,
&(compile_schema->ex_schema),
compile_schema->table_id);
}
void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt,
struct compile_schema *schema,
long long compile_id)
{
if (NULL == compile_rt || NULL == schema || compile_id < 0
|| (0 == schema->set_flag)) {
return NULL;
}
struct compile_item *item = NULL;
item = (struct compile_item *)compile_runtime_get_user_data(compile_rt,
compile_id);
if (NULL == item) {
return NULL;
tmp = utarray_find(compile_state->hit_compile_table_ids, &compile_id,
compare_compile_id);
if (NULL == tmp) {
return -1;
}
void *ex_data = NULL;
schema->ex_schema.dup_func(schema->table_id, &ex_data, item->ex_data,
schema->ex_schema.argl, schema->ex_schema.argp);
return ex_data;
return tmp->table_id;
}
static int compile_runtime_add_compile(struct compile_runtime *compile_rt,
@@ -1836,7 +1777,7 @@ static int compile_runtime_add_compile(struct compile_runtime *compile_rt,
return 0;
}
void garbage_compile_item_free(void *data, void *arg)
static void garbage_compile_item_free(void *data, void *arg)
{
if (NULL == data) {
return;
@@ -1937,6 +1878,7 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
if (0 == is_valid) {
// delete
compile_runtime_del_compile(compile_rt, compile_id);
rcu_hash_del(compile_rt->tbl_cfg_hash, (char *)&compile_id, sizeof(long long));
} else {
// add
int ret = compile_runtime_add_compile(compile_rt, schema, compile_id,
@@ -1944,6 +1886,14 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
if (ret < 0) {
compile_rt->update_err_cnt++;
}
int *table_id = ALLOC(int, 1);
*table_id = table_manager_get_table_id(schema->ref_tbl_mgr, table_name);
ret = rcu_hash_add(compile_rt->tbl_cfg_hash, (char *)&compile_id,
sizeof(long long), table_id);
if (ret < 0) {
FREE(table_id);
}
}
return 0;
@@ -2157,6 +2107,7 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
compile_rt->literal2clause_hash = new_literal2clause;
compile_rt->literal2not_clause_hash = new_literal2not_clause;
rcu_hash_commit(compile_rt->cfg_hash);
rcu_hash_commit(compile_rt->tbl_cfg_hash);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL,
garbage_bool_matcher_free);
@@ -2274,16 +2225,15 @@ int compile_state_update(int vtable_id, struct maat_item *hit_items,
MAX_SCANNER_HIT_GROUP_NUM);
if (1 == maat_inst->opts.hit_path_on && hit_cnt > 0) {
for (i = 0; i < hit_cnt; i++) {
maat_compile_hit_path_add(compile_state->internal_hit_paths, hit_items[i].item_id,
hit_items[i].group_id, vtable_id, 0, state->scan_cnt);
add_internal_hit_path(compile_state->internal_hit_paths, hit_items[i].item_id,
hit_items[i].group_id, vtable_id, 0, state->scan_cnt);
}
}
if (1 == maat_inst->opts.hit_group_on) {
compile_state_update_direct_hit_groups(compile_state->direct_hit_groups,
hit_items, hit_cnt, vtable_id);
compile_state_update_indirect_hit_groups(compile_state->indirect_hit_groups,
super_group_ids, super_group_cnt, vtable_id);
update_direct_hit_groups(compile_state->direct_hit_groups, hit_items, hit_cnt, vtable_id);
update_indirect_hit_groups(compile_state->indirect_hit_groups,
super_group_ids, super_group_cnt, vtable_id);
}
/* update hit clause */
@@ -2314,8 +2264,8 @@ int compile_state_update(int vtable_id, struct maat_item *hit_items,
if (1 == maat_inst->opts.hit_path_on && hit_not_cnt > 0) {
for (i = 0; i < hit_not_cnt; i++) {
maat_compile_hit_path_add(compile_state->internal_hit_paths, -1, hit_NOT_group_ids[i],
vtable_id, 1, state->scan_cnt);
add_internal_hit_path(compile_state->internal_hit_paths, -1, hit_NOT_group_ids[i],
vtable_id, 1, state->scan_cnt);
}
}
@@ -2344,10 +2294,18 @@ size_t compile_state_get_indirect_hit_groups(struct compile_state *compile_state
return i;
}
size_t compile_state_get_indirect_hit_group_cnt(struct compile_state *compile_state)
{
if (NULL == compile_state) {
return 0;
}
return utarray_len(compile_state->indirect_hit_groups);
}
size_t compile_state_get_direct_hit_groups(struct compile_state *compile_state,
enum maat_list_type type,
struct maat_hit_group *group_array,
size_t array_size)
struct maat_hit_group *group_array,
size_t array_size)
{
if (NULL == compile_state) {
return 0;
@@ -2369,6 +2327,15 @@ size_t compile_state_get_direct_hit_groups(struct compile_state *compile_state,
return i;
}
size_t compile_state_get_direct_hit_group_cnt(struct compile_state *compile_state)
{
if (NULL == compile_state) {
return 0;
}
return utarray_len(compile_state->direct_hit_groups);
}
UT_icd ut_compile_group_id_icd = {sizeof(long long), NULL, NULL, NULL};
size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
@@ -2431,16 +2398,4 @@ size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state,
}
return hit_path_cnt;
}
void compile_runtime_garbage_collect_routine(void *compile_runtime)
{
if (NULL == compile_runtime) {
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
if (compile_rt->cfg_hash != NULL) {
rcu_hash_garbage_collect_routine(compile_rt->cfg_hash);
}
}