gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix rule_monitor_loop bug

Browse Source
This commit is contained in:
liuwentan
2023-02-16 16:45:06 +08:00
parent d1b015226e
commit b5b47837d2
12 changed files with 160 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/maat_api.c
View File

@@ -331,6 +331,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
maat_instance->last_full_version = 0;
maat_instance->nr_worker_thread = opts->nr_worker_threads;
maat_instance->rule_effect_interval_ms = opts->rule_effect_interval_ms;
maat_instance->rule_update_checking_interval_ms = opts->rule_update_checking_interval_ms;
maat_instance->gc_timeout_ms = opts->gc_timeout_ms;
maat_instance->deferred_load = opts->deferred_load_on;
garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) +
@@ -856,6 +857,14 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id,
}
maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
if ((table_type == TABLE_TYPE_INTERVAL_PLUS) &&
(NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
maat_instance->scan_err_cnt++;
return MAAT_SCAN_ERR;
}
alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};

100
src/maat_compile.c
View File

@@ -91,6 +91,7 @@ struct compile_runtime {
uint32_t rule_num;
uint32_t updating_rule_num;
pthread_rwlock_t rwlock; /* TODO: replaced with mutex? */
struct bool_expr_match *expr_match_buff;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
@@ -195,7 +196,7 @@ void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, int comp
struct maat_compile *compile = NULL;
void *ret = NULL;
//TODO: add mutex
pthread_rwlock_rdlock(&compile_rt->rwlock);
HASH_FIND_INT(compile_rt->compile_hash, &compile_id, compile);
if (compile != NULL) {
ret = compile->user_data;
@@ -203,6 +204,7 @@ void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, int comp
compile->user_data = NULL;
}
}
pthread_rwlock_unlock(&compile_rt->rwlock);
return ret;
}
@@ -578,6 +580,7 @@ void *compile_runtime_new(void *compile_schema, int max_thread_num,
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM);
compile_rt->logger = logger;
compile_rt->ref_garbage_bin = garbage_bin;
pthread_rwlock_init(&compile_rt->rwlock, NULL);
return compile_rt;
}
@@ -617,6 +620,9 @@ void compile_runtime_free(void *compile_runtime)
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
pthread_rwlock_wrlock(&compile_rt->rwlock);
if (compile_rt->bm != NULL) {
bool_matcher_free(compile_rt->bm);
}
@@ -624,6 +630,7 @@ void compile_runtime_free(void *compile_runtime)
if (compile_rt->compile_hash != NULL) {
maat_compile_hash_free(&(compile_rt->compile_hash));
}
pthread_rwlock_unlock(&compile_rt->rwlock);
if (compile_rt->expr_match_buff != NULL) {
FREE(compile_rt->expr_match_buff);
@@ -797,22 +804,9 @@ int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id,
struct maat_compile *compile)
{
int ret = 0;
struct maat_compile *tmp_compile = NULL;
HASH_FIND_INT(*compile_hash, &compile_id, tmp_compile);
if (!tmp_compile) {
// not found
assert(compile->declared_clause_num >= 0);
HASH_ADD_INT(*compile_hash, compile_id, compile);
} else {
// already exist
if (tmp_compile->user_data != NULL) {
ret = -1;
} else {
maat_compile_set(tmp_compile, compile->declared_clause_num,
compile->user_data, compile->user_data_free);
}
}
assert(compile->declared_clause_num >= 0);
HASH_ADD_INT(*compile_hash, compile_id, compile);
//TODO:mytest need to delete
#if 0
@@ -825,16 +819,23 @@ int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id,
return ret;
}
int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id,
struct maat_garbage_bin *garbage_bin)
void maat_compile_hash_set(struct maat_compile **compile_hash, int compile_id,
struct maat_compile *compile)
{
struct maat_compile *compile = NULL;
struct maat_compile *tmp_compile = NULL;
HASH_FIND_INT(*compile_hash, &compile_id, compile);
if (!compile) {
return -1;
}
HASH_FIND_INT(*compile_hash, &compile_id, tmp_compile);
assert(tmp_compile != NULL);
assert(tmp_compile->user_data == NULL);
maat_compile_set(tmp_compile, compile->declared_clause_num,
compile->user_data, compile->user_data_free);
}
int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id,
struct maat_compile *compile, struct maat_garbage_bin *garbage_bin)
{
if (0 == compile->actual_clause_num) {
HASH_DEL(*compile_hash, compile);
maat_garbage_bagging(garbage_bin, compile, (void (*)(void *))maat_compile_free);
@@ -1045,6 +1046,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
j++;
}
}
//some compile may have zero groups, e.g. default policy.
if (j == (size_t)compile->declared_clause_num && j > 0) {
bool_expr_array[expr_cnt].expr_id = compile->compile_id;
@@ -1420,16 +1422,16 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
assert(compile_rt->compile_hash != NULL);
struct maat_compile *compile = NULL, *tmp_compile = NULL;
struct maat_clause_state *clause_state = NULL;
struct maat_literal_id literal_id = {group_id, vtable_id};
struct maat_literal_id *tmp = NULL;
unsigned long long *clause_id = 0;
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
pthread_rwlock_wrlock(&compile_rt->rwlock);
assert(compile_rt->compile_hash != NULL);
HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) {
for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
clause_state = compile->clause_states + i;
@@ -1463,6 +1465,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
}
}
}
pthread_rwlock_unlock(&compile_rt->rwlock);
}
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state)
@@ -1562,10 +1565,20 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
int compile_id = get_column_value(line, schema->compile_id_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
}
if (0 == is_valid) {
//delete
pthread_rwlock_wrlock(&compile_rt->rwlock);
compile = maat_compile_hash_find(&(compile_rt->compile_hash), compile_id);
if (NULL == compile) {
pthread_rwlock_unlock(&compile_rt->rwlock);
return -1;
}
ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id,
compile_rt->ref_garbage_bin);
compile, compile_rt->ref_garbage_bin);
pthread_rwlock_unlock(&compile_rt->rwlock);
if (ret < 0) {
log_error(compile_rt->logger, MODULE_COMPILE,
"remove compile table(table_id:%d) compile(compile_id:%d) from compile_hash failed",
@@ -1574,8 +1587,11 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
}
} else {
//add
pthread_rwlock_wrlock(&compile_rt->rwlock);
compile_item = compile_item_new(line, schema, compile_rt->logger);
if (NULL == compile_item) {
pthread_rwlock_unlock(&compile_rt->rwlock);
return -1;
}
@@ -1587,6 +1603,7 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
compile = maat_compile_new(compile_rule->compile_id);
if (NULL == compile) {
destroy_compile_rule(compile_rule);
pthread_rwlock_unlock(&compile_rt->rwlock);
log_error(compile_rt->logger, MODULE_COMPILE,
"maat_compile_new failed, compile_table(table_id:%d) compile_id:%d",
schema->table_id, compile_item->compile_id);
@@ -1595,14 +1612,15 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule,
(void (*)(void *))destroy_compile_rule);
ret = maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile);
if (ret < 0) {
maat_compile_free(compile);
log_error(compile_rt->logger, MODULE_COMPILE,
"add compile table(table_id:%d) compile(compile_id:%d) to compile_hash failed",
schema->table_id, compile_id);
return -1;
struct maat_compile *tmp_compile = maat_compile_hash_find(&(compile_rt->compile_hash), compile_id);
if (tmp_compile != NULL) {
maat_compile_hash_set(&(compile_rt->compile_hash), compile_id, compile);
} else {
maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile);
}
pthread_rwlock_unlock(&compile_rt->rwlock);
}
return 0;
@@ -1630,6 +1648,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
return -1;
}
pthread_rwlock_wrlock(&compile_rt->rwlock);
if (0 == is_valid) {
//delete
ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item,
@@ -1649,6 +1668,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
}
}
}
pthread_rwlock_unlock(&compile_rt->rwlock);
group2compile_item_free(g2c_item);
return ret;
@@ -1661,6 +1681,7 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name)
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
//TODO: no need add rdlock?
size_t compile_cnt = maat_compile_in_use_count(compile_rt->compile_hash);
if (0 == compile_cnt) {
return 0;
@@ -1684,8 +1705,11 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name)
ret = -1;
}
pthread_rwlock_wrlock(&compile_rt->rwlock);
old_bool_matcher = compile_rt->bm;
compile_rt->bm = new_bool_matcher;
pthread_rwlock_unlock(&compile_rt->rwlock);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher,
(void (*)(void*))maat_compile_bool_matcher_free);
@@ -1748,10 +1772,12 @@ int compile_runtime_match(struct compile_runtime *compile_rt,
struct compile_rule *compile_rules[compile_ids_size];
// all hit clause_id -> compile_id
pthread_rwlock_rdlock(&compile_rt->rwlock);
size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt->bm,
is_last_scan, compile_state,
(void **)compile_rules,
compile_ids_size);
pthread_rwlock_unlock(&compile_rt->rwlock);
if (bool_match_ret > 0) {
qsort(compile_rules, bool_match_ret, sizeof(struct compile_rule *),
compare_compile_rule);

4
src/maat_expr.c
View File

@@ -866,11 +866,13 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
int district_id = state->district_id;
memset(hit_item_ids, -1, sizeof(hit_item_ids));
for (size_t i = 0; i < n_hit_item; i++) {
item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (item->district_id == district_id || district_id == DISTRICT_ANY) {

4
src/maat_flag.c
View File

@@ -429,7 +429,9 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
memset(hit_item_ids, -1, sizeof(hit_item_ids));
for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].rule_id;
}

22
src/maat_fqdn.c
View File

@@ -10,9 +10,31 @@
#include "maat_fqdn.h"
#include "log/log.h"
#include "fqdn_engine.h"
struct fqdn_schema {
int item_id_column;
int group_id_column;
int fqdn_column;
int match_method_column;
int table_id;
struct table_manager *ref_tbl_mgr;
};
struct fqdn_runtime {
struct FQDN_engine *engine;
struct rcu_hash_table *htable;
uint32_t rule_num;
uint32_t updating_rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long *scan_cnt;
long long *hit_cnt;
};
void *fqdn_schema_new(cJSON *json, struct table_manager *tbl_mgr,

15
src/maat_interval.c
View File

@@ -354,7 +354,7 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
return -1;
}
u_para = maat_item_inner_new(interval_item->group_id, item_id, 0);
u_para = maat_item_inner_new(interval_item->group_id, item_id, interval_item->district_id);
item = maat_item_new(item_id, interval_item->group_id, u_para);
HASH_ADD_INT(interval_rt->item_hash, item_id, item);
@@ -459,9 +459,18 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
int district_id = state->district_id;
memset(hit_item_ids, -1, sizeof(hit_item_ids));
for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].rule_id;
item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (item->district_id == district_id || district_id == DISTRICT_ANY) {
hit_item_ids[real_hit_item_cnt++] = hit_results[i].rule_id;
}
}
size_t group_hit_cnt = 0;

4
src/maat_ip.c
View File

@@ -509,7 +509,9 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
memset(hit_item_ids, -1, sizeof(hit_item_ids));
for (int i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = scan_results[i].rule_id;
}

4
src/maat_rule.c
View File

@@ -417,7 +417,9 @@ void *rule_monitor_loop(void *arg)
struct stat attrib;
while (maat_instance->is_running) {
usleep(maat_instance->rule_update_checking_interval_ms * 1000);
if( 0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"rule_monitor_loop.................%d", maat_instance->rule_update_checking_interval_ms * 1000);
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
redis_monitor_traverse(maat_instance->maat_version,