gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplify table rcu hash

Browse Source
This commit is contained in:
liuwentan
2023-05-07 23:09:33 +08:00
parent 8a3683fa30
commit 7ba644ce66
11 changed files with 519 additions and 620 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/inc_internal/maat_compile.h
View File

@@ -96,9 +96,8 @@ struct maat_compile_state *maat_compile_state_new(int thread_id);
void maat_compile_state_reset(struct maat_compile_state *compile_state);
void maat_compile_state_free(struct maat_compile_state *compile_state);
int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
long long *hit_item_ids, size_t hit_item_cnt,
size_t *n_hit_group_id, struct maat_state *state);
void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
size_t n_hit_item, struct maat_state *state);
size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *compile_state,
struct group2group_runtime *g2g_rt,

10
src/inc_internal/maat_rule.h
View File

@@ -69,9 +69,6 @@ struct maat_item_inner {
struct maat_item {
long long item_id;
long long group_id;
void *user_data;
void (*user_data_free)(void *data);
};
#define COMPILE_RULE_MAGIC 0x1a2b3c4d
@@ -282,11 +279,8 @@ size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag);
struct maat_item *maat_item_new(long long item_id, long long group_id, void *user_data, void (*user_data_free)(void *));
void maat_item_free(void *maat_item);
struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, long long district_id);
struct maat_item_inner *maat_item_inner_new(long long item_id, long long group_id,
long long district_id);
void maat_item_inner_free(void *item_inner);

48
src/maat_compile.c
View File

@@ -2012,35 +2012,22 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
return MIN(bool_match_ret, compile_ids_size);
}
int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
long long *hit_item_ids, size_t hit_item_cnt,
size_t *n_hit_group_id, struct maat_state *state)
void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
size_t n_hit_item, struct maat_state *state)
{
struct maat_item *item = NULL;
size_t i = 0;
size_t hit_cnt = n_hit_item;
long long hit_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
size_t hit_group_cnt = 0;
for (size_t i = 0; i < hit_item_cnt; i++) {
item = (struct maat_item *)rcu_hash_find(item_htable, (char *)&(hit_item_ids[i]), sizeof(long long));
//assert(item != NULL);
if (!item) {
// item config has been deleted
continue;
}
if (hit_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
hit_group_cnt = MAX_SCANNER_HIT_GROUP_NUM;
//Prevent group_id_array out of bounds
} else {
hit_group_ids[hit_group_cnt++] = item->group_id;
}
// update hit path
maat_compile_state_update_hit_path(state->compile_state, hit_item_ids[i],
item->group_id, vtable_id, state->scan_cnt, i);
if (hit_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
hit_cnt = MAX_SCANNER_HIT_GROUP_NUM;
}
*n_hit_group_id = hit_group_cnt;
for (i = 0; i < hit_cnt; i++) {
maat_compile_state_update_hit_path(state->compile_state, hit_items[i].item_id,
hit_items[i].group_id, vtable_id, state->scan_cnt, i);
hit_group_ids[i] = hit_items[i].group_id;
}
/* update hit clause */
int compile_table_id = -1;
@@ -2053,20 +2040,19 @@ int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
compile_table_id);
if (NULL == compile_rt) {
return 0;
return;
}
void *g2g_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
state->maat_instance->g2g_table_id);
if (NULL == g2g_rt) {
return 0;
return;
}
long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
//hit_group_ids also will be added to super_group_ids
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids,
hit_group_cnt,
super_group_ids,
hit_cnt, super_group_ids,
MAX_SCANNER_HIT_GROUP_NUM);
if (super_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
super_group_cnt = MAX_SCANNER_HIT_GROUP_NUM;
@@ -2077,12 +2063,10 @@ int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
super_group_ids[j], vtable_id);
}
for (int j = 0; j < hit_group_cnt; j++) {
for (int j = 0; j < hit_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
hit_group_ids[j], vtable_id);
}
return 0;
}
size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *compile_state,

257
src/maat_expr.c
View File

@@ -64,20 +64,22 @@ struct expr_item {
enum hs_match_mode match_mode;
int is_hexbin;
int is_case_sensitive;
void *user_data;
void (*user_data_free)(void *data);
};
struct expr_runtime {
struct adapter_hs *hs;
struct rcu_hash_table *htable; // store hs_expr rule for rebuild adapter_hs instance
struct rcu_hash_table *item_htable; // store this expr table's all maat_item which will be used in expr_runtime_scan
struct rcu_hash_table *expr_item_hash; // store hs_expr rule for rebuild adapter_hs instance
long long version; //expr_rt version
long long regex_rule_num;
long long rule_num;
long long update_err_cnt;
long long regex_rule_num;
size_t n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
struct maat_garbage_bin *ref_garbage_bin;
int district_num;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
@@ -85,6 +87,7 @@ struct expr_runtime {
long long *scan_cnt;
long long *scan_cpu_time;
long long *hit_cnt;
long long update_err_cnt;
long long *stream_num;
long long *scan_bytes;
};
@@ -305,11 +308,6 @@ error:
return NULL;
}
void expr_item_free(struct expr_item *expr_item)
{
FREE(expr_item);
}
void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
@@ -425,29 +423,32 @@ void expr_schema_free(void *expr_schema)
FREE(expr_schema);
}
void expr_rule_free(struct hs_expr *expr_rule)
void expr_rule_reset(struct expr_rule *rule)
{
if (NULL == expr_rule) {
if (NULL == rule) {
return;
}
for (size_t i = 0; i < expr_rule->n_patterns; i++) {
FREE(expr_rule->patterns[i].pat);
for (size_t i = 0; i < rule->n_patterns; i++) {
FREE(rule->patterns[i].pat);
}
}
void expr_item_free(struct expr_item *item)
{
if (item->user_data != NULL &&
item->user_data_free != NULL) {
item->user_data_free(item->user_data);
item->user_data = NULL;
}
FREE(expr_rule);
FREE(item);
}
void expr_ex_data_free(void *user_ctx, void *data)
void expr_item_free_cb(void *user_ctx, void *data)
{
struct hs_expr *expr_rule = (struct hs_expr *)data;
expr_rule_free(expr_rule);
}
void expr_maat_item_free(void *user_ctx, void *data)
{
struct maat_item *item = (struct maat_item *)data;
maat_item_free(item);
struct expr_item *item = (struct expr_item *)data;
expr_item_free(item);
}
void *expr_runtime_new(void *expr_schema, size_t max_thread_num,
@@ -460,8 +461,7 @@ void *expr_runtime_new(void *expr_schema, size_t max_thread_num,
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->htable = rcu_hash_new(expr_ex_data_free, NULL);
expr_rt->item_htable = rcu_hash_new(expr_maat_item_free, NULL);
expr_rt->expr_item_hash = rcu_hash_new(expr_item_free_cb, NULL);
expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger;
@@ -488,14 +488,9 @@ void expr_runtime_free(void *expr_runtime)
expr_rt->hs = NULL;
}
if (expr_rt->htable != NULL) {
rcu_hash_free(expr_rt->htable);
expr_rt->htable = NULL;
}
if (expr_rt->item_htable != NULL) {
rcu_hash_free(expr_rt->item_htable);
expr_rt->item_htable = NULL;
if (expr_rt->expr_item_hash != NULL) {
rcu_hash_free(expr_rt->expr_item_hash);
expr_rt->expr_item_hash = NULL;
}
assert(expr_rt->tmp_district_map == NULL);
@@ -534,20 +529,20 @@ void expr_runtime_free(void *expr_runtime)
}
int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len,
long long item_id, struct hs_expr *expr_rule, int is_valid)
struct expr_item *item, int is_valid)
{
int ret = -1;
if (0 == is_valid) {
//delete
rcu_hash_del(expr_rt->htable, key, key_len);
rcu_hash_del(expr_rt->expr_item_hash, key, key_len);
} else {
//add
ret = rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule);
ret = rcu_hash_add(expr_rt->expr_item_hash, key, key_len, (void *)item);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr rule(rule_id:%lld) added to expr runtime htable failed",
__FUNCTION__, __LINE__, item_id);
"[%s:%d] expr item(item_id:%lld) add to expr_item_hash failed",
__FUNCTION__, __LINE__, item->item_id);
return -1;
}
}
@@ -603,8 +598,8 @@ static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size)
}
#define MAAT_MAX_EXPR_ITEM_NUM 8
struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data,
struct log_handle *logger)
int expr_item_to_expr_rule(struct expr_item *expr_item, struct expr_rule *expr_rule,
struct log_handle *logger)
{
size_t i = 0;
size_t sub_expr_cnt = 0;
@@ -619,8 +614,6 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
memset(key_left_offset, -1, sizeof(key_left_offset));
memset(key_right_offset, -1, sizeof(key_right_offset));
struct hs_expr *expr_rule = ALLOC(struct hs_expr, 1);
switch (expr_item->expr_type) {
case EXPR_TYPE_AND:
case EXPR_TYPE_REGEX:
@@ -634,7 +627,7 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%d) too many patterns",
__FUNCTION__, __LINE__, expr_item->item_id);
goto error;
return -1;
}
sub_key_array[i] = tmp;
@@ -657,7 +650,7 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%d) too many patterns",
__FUNCTION__, __LINE__, expr_item->item_id);
goto error;
return -1;
}
sub_key_array[i] = tmp;
@@ -667,7 +660,7 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%d) has invalid offset.",
__FUNCTION__, __LINE__, expr_item->item_id);
goto error;
return -1;
}
sub_key_array[i] = (char *)memchr(sub_key_array[i], ':', strlen(sub_key_array[i]));
@@ -675,7 +668,7 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%d) has invalid offset keyword format.",
__FUNCTION__, __LINE__, expr_item->item_id);
goto error;
return -1;
}
sub_key_array[i]++;//jump over ':'
@@ -692,7 +685,7 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%lld) has invalid expr type=%d",
__FUNCTION__, __LINE__, expr_item->item_id, expr_item->expr_type);
goto error;
return -1;
}
for (i = 0; i < sub_expr_cnt; i++) {
@@ -700,9 +693,11 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
char *region_string = NULL;
size_t sub_key_len = 0;
if (FALSE == expr_item->is_case_sensitive) {
if (TRUE == expr_item->is_case_sensitive) {
// insensitive
expr_rule->patterns[i].case_sensitive = HS_CASE_INSESITIVE;
expr_rule->patterns[i].case_sensitive = HS_CASE_SENSITIVE;
} else {
expr_rule->patterns[i].case_sensitive = HS_CASE_INSENSITIVE;
}
expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type);
@@ -732,7 +727,7 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
}
}
expr_rule->expr_id = expr_item->item_id;
expr_rule->user_tag = user_data;
expr_rule->user_tag = expr_item->user_data;
expr_rule->n_patterns = sub_expr_cnt;
//mytest
@@ -744,10 +739,7 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
// printf("expr_rule->patterns[%zu].l_offset:%d\n", i, expr_rule->patterns[i].l_offset);
// printf("expr_rule->patterns[%zu].r_offset:%d\n", i, expr_rule->patterns[i].r_offset);
// }
return expr_rule;
error:
FREE(expr_rule);
return NULL;
return 0;
}
int expr_runtime_update(void *expr_runtime, void *expr_schema,
@@ -759,10 +751,6 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
return -1;
}
int ret = -1;
struct maat_item_inner *u_para = NULL;
struct maat_item *item = NULL;
struct hs_expr *expr_rule = NULL;
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
@@ -776,47 +764,29 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
if (is_valid < 0) {
expr_rt->update_err_cnt++;
return -1;
} else if (0 == is_valid) {
//delete
rcu_hash_del(expr_rt->item_htable, (char *)&item_id, sizeof(item_id));
} else {
}
struct expr_item *expr_item = NULL;
if (1 == is_valid) {
//add
struct expr_item *expr_item = expr_item_new(line, schema, expr_rt);
expr_item = expr_item_new(line, schema, expr_rt);
if (NULL == expr_item) {
expr_rt->update_err_cnt++;
return -1;
}
u_para = maat_item_inner_new(expr_item->group_id, item_id, expr_item->district_id);
item = maat_item_new(item_id, expr_item->group_id, u_para, maat_item_inner_free);
ret = rcu_hash_add(expr_rt->item_htable, (char *)&item_id, sizeof(item_id), item);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] [table:%s] expr runtime add item(item_id:%lld) failed",
__FUNCTION__, __LINE__, table_name, item_id);
expr_item_free(expr_item);
maat_item_free(item);
expr_rt->update_err_cnt++;
return -1;
}
expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger);
expr_item_free(expr_item);
if (NULL == expr_rule) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] [table:%s] transform expr_item(item_id:%lld) to expr_rule failed",
__FUNCTION__, __LINE__, table_name, item_id);
expr_rt->update_err_cnt++;
return -1;
}
struct maat_item_inner *inner_item = maat_item_inner_new(expr_item->item_id,
expr_item->group_id,
expr_item->district_id);
expr_item->user_data = inner_item;
expr_item->user_data_free = maat_item_inner_free;
}
char *key = (char *)&item_id;
ret = expr_runtime_update_row(expr_rt, key, sizeof(item_id), item_id,
expr_rule, is_valid);
int ret = expr_runtime_update_row(expr_rt, (char *)&item_id, sizeof(long long),
expr_item, is_valid);
if (ret < 0) {
if (expr_rule != NULL) {
expr_rule_free(expr_rule);
if (expr_item != NULL) {
expr_item_free(expr_item);
}
expr_rt->update_err_cnt++;
return -1;
@@ -839,13 +809,11 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
int updating_flag = rcu_hash_is_updating(expr_rt->htable);
int updating_flag = rcu_hash_is_updating(expr_rt->expr_item_hash);
if (0 == updating_flag) {
return 0;
}
rcu_hash_commit(expr_rt->htable);
if (expr_rt->tmp_district_map != NULL) {
struct maat_kv_store *tmp_map = expr_rt->district_map;
expr_rt->district_map = expr_rt->tmp_district_map;
@@ -854,18 +822,28 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
garbage_maat_kv_store_free);
}
struct hs_expr *rules = NULL;
int ret = 0;
size_t i = 0;
size_t real_rule_cnt = 0;
struct expr_rule *rules = NULL;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list(expr_rt->htable, &ex_data_array);
size_t rule_cnt = rcu_updating_hash_list(expr_rt->expr_item_hash, &ex_data_array);
if (rule_cnt > 0) {
rules = ALLOC(struct hs_expr, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct hs_expr *)ex_data_array[i];
rules = ALLOC(struct expr_rule, rule_cnt);
for (i = 0; i < rule_cnt; i++) {
struct expr_item *expr_item = (struct expr_item *)ex_data_array[i];
struct expr_rule tmp_rule;
ret = expr_item_to_expr_rule(expr_item, &tmp_rule, expr_rt->logger);
if (ret < 0) {
continue;
}
rules[i] = tmp_rule;
real_rule_cnt++;
}
}
int ret = 0;
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
@@ -881,11 +859,12 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs;
rcu_hash_commit(expr_rt->expr_item_hash);
if (old_adapter_hs != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL, garbage_adapter_hs_free);
}
rcu_hash_commit(expr_rt->item_htable);
expr_rt->rule_num = rule_cnt;
expr_rt->version = maat_rt_version;
@@ -894,7 +873,10 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
table_name, rule_cnt, expr_rt->version);
if (rules != NULL) {
FREE(rules);
for (i = 0; i < rule_cnt; i++) {
expr_rule_reset(&rules[i]);
}
FREE(rules);
}
if (ex_data_array != NULL) {
@@ -963,27 +945,38 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *inner_item = NULL;
size_t real_hit_item_cnt = 0;
long long district_id = state->district_id;
for (size_t i = 0; i < n_hit_item; i++) {
item = (struct maat_item_inner *)(hit_results[i].user_tag);
assert(item != NULL);
if (item->district_id == district_id || item->district_id == DISTRICT_ANY) {
hit_item_ids[real_hit_item_cnt++] = hit_results[i].item_id;
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
struct expr_item *expr_item = (struct expr_item *)rcu_hash_find(expr_rt->expr_item_hash,
(char *)&item_id,
sizeof(long long));
if (!expr_item) {
// item config has been deleted
continue;
}
if (real_hit_item_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
real_hit_item_cnt = MAX_SCANNER_HIT_GROUP_NUM;
// Prevent group_id_array out of bounds
} else {
hit_maat_items[real_hit_item_cnt].item_id = item_id;
hit_maat_items[real_hit_item_cnt].group_id = expr_item->group_id;
}
real_hit_item_cnt++;
}
}
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_htable, vtable_id, hit_item_ids,
real_hit_item_cnt, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return group_hit_cnt;
return real_hit_item_cnt;
}
struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
@@ -1018,26 +1011,30 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_str
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(expr_rt->logger, MODULE_EXPR,
"hit expr item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
size_t real_hit_item_cnt = 0;
for (size_t i = 0; i < n_hit_item; i++) {
hit_item_ids[i] = hit_results[i].item_id;
long long item_id = hit_results[i].rule_id;
struct expr_item *expr_item = (struct expr_item *)rcu_hash_find(expr_rt->expr_item_hash,
(char *)&item_id,
sizeof(long long));
if (!expr_item) {
// item config has been deleted
continue;
}
hit_maat_items[real_hit_item_cnt].item_id = item_id;
hit_maat_items[real_hit_item_cnt].group_id = expr_item->group_id;
real_hit_item_cnt++;
}
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_htable, vtable_id, hit_item_ids,
n_hit_item, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return group_hit_cnt;
return real_hit_item_cnt;
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,

181
src/maat_flag.c
View File

@@ -40,17 +40,20 @@ struct flag_item {
long long district_id;
long long flag;
long long flag_mask;
void *user_data;
void (*user_data_free)(void *data);
};
struct flag_runtime {
struct flag_matcher *matcher;
struct rcu_hash_table *htable; //store flag rule for rebuild flag_matcher instance
struct rcu_hash_table *item_htable; //store this flag table's all maat_item which will be used in flag_runtime_scan
struct rcu_hash_table *flag_item_hash;
long long rule_num;
long long version;
size_t n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
struct maat_garbage_bin *ref_garbage_bin;
int district_num;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
@@ -155,16 +158,25 @@ void flag_schema_free(void *flag_schema)
FREE(flag_schema);
}
void flag_ex_data_free(void *user_ctx, void *data)
void flag_item_free(struct flag_item *item)
{
struct flag_item *flag_item = (struct flag_item *)data;
FREE(flag_item);
if (NULL == item) {
return;
}
if (item->user_data != NULL && item->user_data_free != NULL) {
item->user_data_free(item->user_data);
item->user_data = NULL;
}
FREE(item);
}
void flag_maat_item_free(void *user_ctx, void *data)
void flag_item_free_cb(void *user_ctx, void *data)
{
struct maat_item *item = (struct maat_item *)data;
maat_item_free(item);
struct flag_item *item = (struct flag_item *)data;
flag_item_free(item);
}
void *flag_runtime_new(void *flag_schema, size_t max_thread_num,
@@ -177,8 +189,7 @@ void *flag_runtime_new(void *flag_schema, size_t max_thread_num,
struct flag_runtime *flag_rt = ALLOC(struct flag_runtime, 1);
flag_rt->htable = rcu_hash_new(flag_ex_data_free, NULL);
flag_rt->item_htable = rcu_hash_new(flag_maat_item_free, NULL);
flag_rt->flag_item_hash = rcu_hash_new(flag_item_free_cb, NULL);
flag_rt->n_worker_thread = max_thread_num;
flag_rt->ref_garbage_bin = garbage_bin;
flag_rt->logger = logger;
@@ -198,14 +209,9 @@ void flag_runtime_free(void *flag_runtime)
}
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
if (flag_rt->htable != NULL) {
rcu_hash_free(flag_rt->htable);
flag_rt->htable = NULL;
}
if (flag_rt->item_htable != NULL) {
rcu_hash_free(flag_rt->item_htable);
flag_rt->item_htable = NULL;
if (flag_rt->flag_item_hash != NULL) {
rcu_hash_free(flag_rt->flag_item_hash);
flag_rt->flag_item_hash = NULL;
}
if (flag_rt->matcher != NULL) {
@@ -239,20 +245,20 @@ void flag_runtime_free(void *flag_runtime)
}
int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len,
long long item_id, struct flag_rule *rule, int is_valid)
struct flag_item *item, int is_valid)
{
int ret = -1;
if (0 == is_valid) {
//delete
rcu_hash_del(flag_rt->htable, key, key_len);
rcu_hash_del(flag_rt->flag_item_hash, key, key_len);
} else {
//add
ret = rcu_hash_add(flag_rt->htable, key, key_len, (void *)rule);
ret = rcu_hash_add(flag_rt->flag_item_hash, key, key_len, (void *)item);
if (ret < 0) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag rule(rule_id:%lld) add to flag runtime htable failed",
__FUNCTION__, __LINE__, item_id);
"[%s:%d] flag item(item_id:%lld) add to flag_item_hash failed",
__FUNCTION__, __LINE__, item->item_id);
return -1;
}
}
@@ -365,28 +371,18 @@ error:
return NULL;
}
void flag_item_free(struct flag_item *item)
struct flag_rule flag_item_to_flag_rule(struct flag_item *item)
{
FREE(item);
}
struct flag_rule rule;
struct flag_rule *flag_item_to_flag_rule(struct flag_item *item, void *user_data)
{
struct flag_rule *rule = ALLOC(struct flag_rule, 1);
rule->rule_id = item->item_id;
rule->flag = item->flag;
rule->mask = item->flag_mask;
rule->user_tag = user_data;
rule.rule_id = item->item_id;
rule.flag = item->flag;
rule.mask = item->flag_mask;
rule.user_tag = item->user_data;
return rule;
}
void flag_rule_free(struct flag_rule *rule)
{
FREE(rule);
}
int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table_name,
const char *line, int valid_column)
{
@@ -395,10 +391,6 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table
return -1;
}
int ret = -1;
struct maat_item_inner *u_para = NULL;
struct maat_item *item = NULL;
struct flag_rule *flag_rule = NULL;
struct flag_schema *schema = (struct flag_schema *)flag_schema;
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
@@ -411,47 +403,29 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table
if (is_valid < 0) {
flag_rt->update_err_cnt++;
return -1;
} else if (0 == is_valid) {
//delete
rcu_hash_del(flag_rt->item_htable, (char *)&item_id, sizeof(item_id));
} else {
//add
struct flag_item *flag_item = flag_item_new(line, schema, flag_rt);
}
struct flag_item *flag_item = NULL;
if (1 == is_valid) {
flag_item = flag_item_new(line, schema, flag_rt);
if (NULL == flag_item) {
flag_rt->update_err_cnt++;
return -1;
}
u_para = maat_item_inner_new(flag_item->group_id, item_id, flag_item->district_id);
item = maat_item_new(item_id, flag_item->group_id, u_para, maat_item_inner_free);
ret = rcu_hash_add(flag_rt->item_htable, (char *)&(item_id), sizeof(item_id), item);
if (ret < 0) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] [table:%s] flag runtime add item(item_id:%lld) failed",
__FUNCTION__, __LINE__, table_name, item_id);
flag_item_free(flag_item);
maat_item_free(item);
flag_rt->update_err_cnt++;
return -1;
}
flag_rule = flag_item_to_flag_rule(flag_item, u_para);
flag_item_free(flag_item);
if (NULL == flag_rule) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] [table:%s] transform flag_item(item_id:%lld) to flag_rule failed",
__FUNCTION__, __LINE__, table_name, item_id);
flag_rt->update_err_cnt++;
return -1;
}
struct maat_item_inner *inner_item = maat_item_inner_new(flag_item->item_id,
flag_item->group_id,
flag_item->district_id);
flag_item->user_data = inner_item;
flag_item->user_data_free = maat_item_inner_free;
}
char *key = (char *)&item_id;
ret = flag_runtime_update_row(flag_rt, key, sizeof(long long), item_id, flag_rule, is_valid);
int ret = flag_runtime_update_row(flag_rt, (char *)&item_id, sizeof(long long),
flag_item, is_valid);
if (ret < 0) {
if (flag_rule != NULL) {
flag_rule_free(flag_rule);
flag_rule = NULL;
if (flag_item != NULL) {
flag_item_free(flag_item);
flag_item = NULL;
}
flag_rt->update_err_cnt++;
return -1;
@@ -466,7 +440,8 @@ void garbage_flag_matcher_free(void *flag_matcher, void *arg)
flag_matcher_free(matcher);
}
int flag_runtime_commit(void *flag_runtime, const char *table_name, long long maat_rt_version)
int flag_runtime_commit(void *flag_runtime, const char *table_name,
long long maat_rt_version)
{
if (NULL == flag_runtime) {
return -1;
@@ -474,12 +449,10 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name, long long ma
struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
int updating_flag = rcu_hash_is_updating(flag_rt->htable);
int updating_flag = rcu_hash_is_updating(flag_rt->flag_item_hash);
if (0 == updating_flag) {
return 0;
}
rcu_hash_commit(flag_rt->htable);
if (flag_rt->tmp_district_map != NULL) {
struct maat_kv_store *tmp_map = flag_rt->district_map;
@@ -491,11 +464,13 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name, long long ma
struct flag_rule *rules = NULL;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list(flag_rt->htable, &ex_data_array);
size_t rule_cnt = rcu_updating_hash_list(flag_rt->flag_item_hash, &ex_data_array);
if (rule_cnt > 0) {
rules = ALLOC(struct flag_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct flag_rule *)ex_data_array[i];
struct flag_item *flag_item = (struct flag_item *)ex_data_array[i];
rules[i] = flag_item_to_flag_rule(flag_item);
}
}
@@ -515,12 +490,13 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name, long long ma
old_flag_matcher = flag_rt->matcher;
flag_rt->matcher = new_flag_matcher;
rcu_hash_commit(flag_rt->flag_item_hash);
if (old_flag_matcher != NULL) {
maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher, NULL,
garbage_flag_matcher_free);
}
rcu_hash_commit(flag_rt->item_htable);
flag_rt->rule_num = rule_cnt;
flag_rt->version = maat_rt_version;
@@ -566,32 +542,35 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(flag_rt->logger, MODULE_FLAG,
"hit flag item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *inner_item = NULL;
size_t real_hit_item_cnt = 0;
long long district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) {
item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (item->district_id == district_id || item->district_id == DISTRICT_ANY) {
hit_item_ids[real_hit_item_cnt++] = hit_results[i].rule_id;
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
struct flag_item *flag_item = (struct flag_item *)rcu_hash_find(flag_rt->flag_item_hash,
(char *)&item_id,
sizeof(long long));
if (!flag_item) {
// item config has been deleted
continue;
}
hit_maat_items[real_hit_item_cnt].item_id = item_id;
hit_maat_items[real_hit_item_cnt].group_id = flag_item->group_id;
real_hit_item_cnt++;
}
}
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(flag_rt->item_htable, vtable_id, hit_item_ids,
n_hit_item, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return group_hit_cnt;
return real_hit_item_cnt;
}
void flag_runtime_hit_inc(struct flag_runtime *flag_rt, int thread_id)

178
src/maat_interval.c
View File

@@ -37,22 +37,25 @@ struct interval_item {
long long district_id;
int low_bound;
int up_bound;
void *user_data;
void (*user_data_free)(void *data);
};
struct interval_runtime {
struct interval_matcher *matcher;
struct rcu_hash_table *htable; //store interval rule for rebuild interval_matcher instance
struct rcu_hash_table *item_htable; //store this interval table's all maat_item which will be used in interval_runtime_scan
struct rcu_hash_table *int_item_hash; //store interval rule for rebuild interval_matcher instance
long long version;
long long rule_num;
long long update_err_cnt;
size_t n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
struct maat_garbage_bin *ref_garbage_bin;
int district_num;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
long long update_err_cnt;
long long *scan_cnt;
long long *scan_cpu_time;
long long *hit_cnt;
@@ -152,16 +155,24 @@ void interval_schema_free(void *interval_schema)
FREE(interval_schema);
}
void interval_ex_data_free(void *user_ctx, void *data)
void interval_item_free(struct interval_item *item)
{
struct interval_item *item = (struct interval_item *)data;
if (NULL == item) {
return;
}
if (item->user_data != NULL && item->user_data_free != NULL) {
item->user_data_free(item->user_data);
item->user_data = NULL;
}
FREE(item);
}
void interval_maat_item_free(void *user_ctx, void *data)
void interval_item_free_cb(void *user_ctx, void *data)
{
struct maat_item *item = (struct maat_item *)data;
maat_item_free(item);
struct interval_item *item = (struct interval_item *)data;
interval_item_free(item);
}
void *interval_runtime_new(void *interval_schema, size_t max_thread_num,
@@ -174,8 +185,7 @@ void *interval_runtime_new(void *interval_schema, size_t max_thread_num,
struct interval_runtime *interval_rt = ALLOC(struct interval_runtime, 1);
interval_rt->htable = rcu_hash_new(interval_ex_data_free, NULL);
interval_rt->item_htable = rcu_hash_new(interval_maat_item_free, NULL);
interval_rt->int_item_hash = rcu_hash_new(interval_item_free_cb, NULL);
interval_rt->n_worker_thread = max_thread_num;
interval_rt->ref_garbage_bin = garbage_bin;
interval_rt->logger = logger;
@@ -195,14 +205,9 @@ void interval_runtime_free(void *interval_runtime)
}
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
if (interval_rt->htable != NULL) {
rcu_hash_free(interval_rt->htable);
interval_rt->htable = NULL;
}
if (interval_rt->item_htable != NULL) {
rcu_hash_free(interval_rt->item_htable);
interval_rt->item_htable = NULL;
if (interval_rt->int_item_hash != NULL) {
rcu_hash_free(interval_rt->int_item_hash);
interval_rt->int_item_hash = NULL;
}
if (interval_rt->matcher != NULL) {
@@ -339,43 +344,33 @@ error:
return NULL;
}
void interval_item_free(void *interval_item)
struct interval_rule interval_item_to_interval_rule(struct interval_item *item)
{
FREE(interval_item);
}
struct interval_rule rule;
struct interval_rule *interval_item_to_interval_rule(struct interval_item *item, void *user_data)
{
struct interval_rule *rule = ALLOC(struct interval_rule, 1);
rule->start = item->low_bound;
rule->end = item->up_bound;
rule->result.rule_id = item->item_id;
rule->result.user_tag = user_data;
rule.start = item->low_bound;
rule.end = item->up_bound;
rule.result.rule_id = item->item_id;
rule.result.user_tag = item->user_data;
return rule;
}
void interval_rule_free(struct interval_rule *rule)
{
FREE(rule);
}
int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, size_t key_len,
long long item_id, struct interval_rule *rule, int is_valid)
struct interval_item *item, int is_valid)
{
int ret = -1;
if (0 == is_valid) {
//delete
rcu_hash_del(interval_rt->htable, key, key_len);
rcu_hash_del(interval_rt->int_item_hash, key, key_len);
} else {
//add
ret = rcu_hash_add(interval_rt->htable, key, key_len, (void *)rule);
ret = rcu_hash_add(interval_rt->int_item_hash, key, key_len, (void *)item);
if (ret < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval rule(rule_id:%lld) add to interval runtime htable failed",
__FUNCTION__, __LINE__, item_id);
"[%s:%d] interval item(item_id:%lld) add to interavl_item_hash failed",
__FUNCTION__, __LINE__, item->item_id);
return -1;
}
}
@@ -392,10 +387,6 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
return -1;
}
int ret = -1;
struct maat_item_inner *u_para = NULL;
struct maat_item *item = NULL;
struct interval_rule *interval_rule = NULL;
struct interval_schema *schema = (struct interval_schema *)interval_schema;
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
@@ -409,47 +400,30 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
if (is_valid < 0) {
interval_rt->update_err_cnt++;
return -1;
} else if (0 == is_valid) {
//delete
rcu_hash_del(interval_rt->item_htable, (char *)&item_id, sizeof(item_id));
} else {
}
struct interval_item *interval_item = NULL;
if (1 == is_valid) {
//add
struct interval_item *interval_item = interval_item_new(line, schema, interval_rt);
interval_item = interval_item_new(line, schema, interval_rt);
if (NULL == interval_item) {
interval_rt->update_err_cnt++;
return -1;
}
u_para = maat_item_inner_new(interval_item->group_id, item_id, interval_item->district_id);
item = maat_item_new(item_id, interval_item->group_id, u_para, maat_item_inner_free);
ret = rcu_hash_add(interval_rt->item_htable, (char *)&(item_id), sizeof(item_id), item);
if (ret < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] [table:%s] interval runtime add item(item_id:%lld) failed",
__FUNCTION__, __LINE__, table_name, item_id);
interval_item_free(interval_item);
maat_item_free(item);
interval_rt->update_err_cnt++;
return -1;
}
interval_rule = interval_item_to_interval_rule(interval_item, u_para);
interval_item_free(interval_item);
if (NULL == interval_rule) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] [table:%s] transform interval_item(item_id:%lld) to interval_rule failed",
__FUNCTION__, __LINE__, table_name, item_id);
interval_rt->update_err_cnt++;
return -1;
}
struct maat_item_inner *inner_item = maat_item_inner_new(interval_item->item_id,
interval_item->group_id,
interval_item->district_id);
interval_item->user_data = inner_item;
interval_item->user_data_free = maat_item_inner_free;
}
char *key = (char *)&item_id;
ret = interval_runtime_update_row(interval_rt, key, sizeof(long long), item_id, interval_rule, is_valid);
int ret = interval_runtime_update_row(interval_rt, (char *)&item_id, sizeof(long long),
interval_item, is_valid);
if (ret < 0) {
if (interval_rule != NULL) {
interval_rule_free(interval_rule);
interval_rule = NULL;
if (interval_item != NULL) {
interval_item_free(interval_item);
interval_item = NULL;
}
interval_rt->update_err_cnt++;
return -1;
@@ -472,13 +446,11 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name, long
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
int updating_flag = rcu_hash_is_updating(interval_rt->htable);
int updating_flag = rcu_hash_is_updating(interval_rt->int_item_hash);
if (0 == updating_flag) {
return 0;
}
rcu_hash_commit(interval_rt->htable);
if (interval_rt->tmp_district_map != NULL) {
struct maat_kv_store *tmp_map = interval_rt->district_map;
interval_rt->district_map = interval_rt->tmp_district_map;
@@ -489,11 +461,13 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name, long
void **ex_data_array = NULL;
struct interval_rule *rules = NULL;
size_t rule_cnt = rcu_hash_list(interval_rt->htable, &ex_data_array);
size_t rule_cnt = rcu_updating_hash_list(interval_rt->int_item_hash, &ex_data_array);
if (rule_cnt > 0) {
rules = ALLOC(struct interval_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct interval_rule *)ex_data_array[i];
struct interval_item *interval_item = (struct interval_item *)ex_data_array[i];
rules[i] = interval_item_to_interval_rule(interval_item);
}
}
@@ -513,12 +487,13 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name, long
old_interval_matcher = interval_rt->matcher;
interval_rt->matcher = new_interval_matcher;
rcu_hash_commit(interval_rt->int_item_hash);
if (old_interval_matcher != NULL) {
maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher, NULL,
garbage_interval_matcher_free);
}
rcu_hash_commit(interval_rt->item_htable);
interval_rt->rule_num = rule_cnt;
interval_rt->version = maat_rt_version;
@@ -564,32 +539,35 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(interval_rt->logger, MODULE_INTERVAL,
"hit interval item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *item = NULL;
int real_hit_item_cnt = 0;
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *inner_item = NULL;
size_t real_hit_item_cnt = 0;
long long district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) {
item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (item->district_id == district_id || item->district_id == DISTRICT_ANY) {
hit_item_ids[real_hit_item_cnt++] = hit_results[i].rule_id;
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
struct interval_item *int_item = (struct interval_item *)rcu_hash_find(interval_rt->int_item_hash,
(char *)&item_id,
sizeof(long long));
if (!int_item) {
// item config has been deleted
continue;
}
hit_maat_items[real_hit_item_cnt].item_id = item_id;
hit_maat_items[real_hit_item_cnt].group_id = int_item->group_id;
real_hit_item_cnt++;
}
}
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(interval_rt->item_htable, vtable_id, hit_item_ids,
n_hit_item, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return group_hit_cnt;
return real_hit_item_cnt;
}
void interval_runtime_hit_inc(struct interval_runtime *interval_rt, int thread_id)

171
src/maat_ip.c
View File

@@ -62,21 +62,23 @@ struct ip_item {
uint16_t min_port;
uint16_t max_port;
int proto;
void *user_data;
void (*user_data_free)(void *data);
};
struct ip_runtime {
struct ip_matcher *ip_matcher;
struct interval_matcher *intval_matcher;
struct rcu_hash_table *htable; //store ip rule for rebuild ip_matcher instance
struct rcu_hash_table *item_htable; //store this ip table's all maat_item which will be used in ip_runtime_scan
struct rcu_hash_table *ip_item_hash;
long long version;
long long rule_num;
long long ipv6_rule_num;
long long update_err_cnt;
size_t n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
struct maat_garbage_bin *ref_garbage_bin;
long long update_err_cnt;
long long *scan_cnt;
long long *scan_cpu_time;
long long *hit_cnt;
@@ -362,23 +364,26 @@ error:
return NULL;
}
void ip_item_free(struct ip_item *ip_item)
void ip_item_free(struct ip_item *item)
{
FREE(ip_item);
if (NULL == item) {
return;
}
if (item->user_data != NULL && item->user_data_free != NULL) {
item->user_data_free(item->user_data);
item->user_data = NULL;
}
FREE(item);
}
void ip_ex_data_free(void *user_ctx, void *data)
void ip_item_free_cb(void *user_ctx, void *data)
{
struct ip_item *item = (struct ip_item *)data;
ip_item_free(item);
}
void ip_maat_item_free(void *user_ctx, void *data)
{
struct maat_item *item = (struct maat_item *)data;
maat_item_free(item);
}
void *ip_runtime_new(void *ip_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
@@ -389,8 +394,7 @@ void *ip_runtime_new(void *ip_schema, size_t max_thread_num,
struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1);
ip_rt->htable = rcu_hash_new(ip_ex_data_free, NULL);
ip_rt->item_htable = rcu_hash_new(ip_maat_item_free, NULL);
ip_rt->ip_item_hash = rcu_hash_new(ip_item_free_cb, NULL);
ip_rt->n_worker_thread = max_thread_num;
ip_rt->ref_garbage_bin = garbage_bin;
ip_rt->logger = logger;
@@ -419,14 +423,9 @@ void ip_runtime_free(void *ip_runtime)
ip_rt->intval_matcher = NULL;
}
if (ip_rt->htable != NULL) {
rcu_hash_free(ip_rt->htable);
ip_rt->htable = NULL;
}
if (ip_rt->item_htable != NULL) {
rcu_hash_free(ip_rt->item_htable);
ip_rt->item_htable = NULL;
if (ip_rt->ip_item_hash != NULL) {
rcu_hash_free(ip_rt->ip_item_hash);
ip_rt->ip_item_hash = NULL;
}
if (ip_rt->hit_cnt != NULL) {
@@ -477,13 +476,13 @@ int ip_runtime_update_row(struct ip_runtime *ip_rt, char *key, size_t key_len,
if (0 == is_valid) {
// delete
rcu_hash_del(ip_rt->htable, key, key_len);
rcu_hash_del(ip_rt->ip_item_hash, key, key_len);
} else {
// add
ret = rcu_hash_add(ip_rt->htable, key, key_len, (void *)item);
ret = rcu_hash_add(ip_rt->ip_item_hash, key, key_len, (void *)item);
if (ret < 0) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] ip rule(rule_id:%lld) add to ip runtime htable failed",
"[%s:%d] ip item(item_id:%lld) add to ip runtime htable failed",
__FUNCTION__, __LINE__, item->item_id);
return -1;
}
@@ -500,12 +499,9 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
return -1;
}
int ret = -1;
struct maat_item *item = NULL;
struct ip_item *ip_item = NULL;
struct maat_item_inner *u_para = NULL;
struct ip_schema *schema = (struct ip_schema *)ip_schema;
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
ip_rt->update_err_cnt++;
@@ -516,10 +512,10 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
if (is_valid < 0) {
ip_rt->update_err_cnt++;
return -1;
} else if (0 == is_valid) {
//delete
rcu_hash_del(ip_rt->item_htable, (char *)&item_id, sizeof(item_id));
} else {
}
struct ip_item *ip_item = NULL;
if (1 == is_valid) {
//add
ip_item = ip_item_new(line, schema, ip_rt->logger);
if (NULL == ip_item) {
@@ -527,22 +523,15 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
return -1;
}
u_para = maat_item_inner_new(ip_item->group_id, item_id, 0);
item = maat_item_new(item_id, ip_item->group_id, u_para, maat_item_inner_free);
ret = rcu_hash_add(ip_rt->item_htable, (char *)&(item_id), sizeof(item_id), item);
if (ret < 0) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] [table:%s] ip runtime add item(item_id:%lld) failed",
__FUNCTION__, __LINE__, table_name, item_id);
ip_item_free(ip_item);
maat_item_free(item);
ip_rt->update_err_cnt++;
return -1;
}
struct maat_item_inner *inner_item = maat_item_inner_new(ip_item->item_id,
ip_item->group_id,
0);
ip_item->user_data = inner_item;
ip_item->user_data_free = maat_item_inner_free;
}
char *key = (char *)&item_id;
ret = ip_runtime_update_row(ip_rt, key, sizeof(long long), ip_item, is_valid);
int ret = ip_runtime_update_row(ip_rt, (char *)&item_id, sizeof(long long),
ip_item, is_valid);
if (ret < 0) {
if (ip_item != NULL) {
ip_item_free(ip_item);
@@ -569,18 +558,18 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
int updating_flag = rcu_hash_is_updating(ip_rt->htable);
int updating_flag = rcu_hash_is_updating(ip_rt->ip_item_hash);
if (0 == updating_flag) {
return 0;
}
rcu_hash_commit(ip_rt->htable);
ip_rt->ipv6_rule_num = 0;
struct ip_rule *rules = NULL;
struct interval_rule *intval_rules = NULL;
void **ex_data_array = NULL;
size_t rule_cnt = rcu_hash_list(ip_rt->htable, &ex_data_array);
size_t rule_cnt = rcu_updating_hash_list(ip_rt->ip_item_hash, &ex_data_array);
if (rule_cnt > 0) {
rules = ALLOC(struct ip_rule, rule_cnt);
intval_rules = ALLOC(struct interval_rule, rule_cnt);
@@ -602,6 +591,7 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
struct interval_matcher *new_intval_matcher = NULL;
struct interval_matcher *old_intval_matcher = NULL;
if (rule_cnt > 0) {
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
@@ -622,6 +612,8 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
old_ip_matcher = ip_rt->ip_matcher;
ip_rt->ip_matcher = new_ip_matcher;
rcu_hash_commit(ip_rt->ip_item_hash);
if (old_ip_matcher != NULL) {
maat_garbage_bagging(ip_rt->ref_garbage_bin, old_ip_matcher, NULL,
garbage_ip_matcher_free);
@@ -634,8 +626,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
garbage_interval_matcher_free);
}
rcu_hash_commit(ip_rt->item_htable);
ip_rt->rule_num = rule_cnt;
ip_rt->version = maat_rt_version;
@@ -678,34 +668,24 @@ long long ip_runtime_ipv6_rule_count(void *ip_runtime)
return ip_rt->ipv6_rule_num;
}
int validate_port(struct rcu_hash_table *htable, const char *key, size_t key_len,
uint16_t port, int proto)
int validate_port(struct ip_item *item, uint16_t port, int proto)
{
struct ip_item *ip_item = (struct ip_item *)rcu_hash_find(htable, key, key_len);
if (NULL == ip_item) {
return -1;
}
uint16_t host_port = ntohs(port);
if (ip_item->min_port > host_port || ip_item->max_port < host_port) {
if (item->min_port > host_port || item->max_port < host_port) {
return -1;
}
if (ip_item->proto != -1 && ip_item->proto != proto) {
if (item->proto != -1 && item->proto != proto) {
return -1;
}
return 0;
}
int validate_proto(struct rcu_hash_table *htable, const char *key, size_t key_len, int proto)
int validate_proto(struct ip_item *item, int proto)
{
struct ip_item *ip_item = (struct ip_item *)rcu_hash_find(htable, key, key_len);
if (NULL == ip_item) {
return -1;
}
if (ip_item->proto != -1 && ip_item->proto != proto) {
if (item->proto != -1 && item->proto != proto) {
return -1;
}
@@ -746,8 +726,8 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
}
int ret = 0;
size_t real_hit_index = 0;
long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM];
size_t real_hit_item_cnt = 0;
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
// any ip, so scan port+proto
if (1 == any_ip_flag) {
@@ -760,19 +740,27 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
}
if (n_hit_port_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(ip_rt->logger, MODULE_IP,
"hit port item count:%d exceed maxium:%d",
n_hit_port_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_port_item = MAX_SCANNER_HIT_ITEM_NUM;
}
for (size_t i = 0; i < n_hit_port_item; i++) {
for (int i = 0; i < n_hit_port_item; i++) {
long long item_id = port_results[i].rule_id;
ret = validate_proto(ip_rt->htable, (char *)&item_id, sizeof(item_id), proto);
struct ip_item *ip_item = (struct ip_item *)rcu_hash_find(ip_rt->ip_item_hash,
(char *)&item_id,
sizeof(long long));
if (!ip_item) {
// item config has been deleted
continue;
}
ret = validate_proto(ip_item, proto);
if (ret < 0) {
continue;
}
hit_item_ids[real_hit_index++] = port_results[i].rule_id;
hit_maat_items[real_hit_item_cnt].item_id = port_results[i].rule_id;
hit_maat_items[real_hit_item_cnt].group_id = ip_item->group_id;
real_hit_item_cnt++;
}
} else {
int n_hit_ip_item = ip_matcher_match(ip_rt->ip_matcher, &scan_data,
@@ -782,30 +770,33 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
}
if (n_hit_ip_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(ip_rt->logger, MODULE_IP,
"hit ip item count:%d exceed maxium:%d",
n_hit_ip_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_ip_item = MAX_SCANNER_HIT_ITEM_NUM;
}
for (size_t i = 0; i < n_hit_ip_item; i++) {
long long item_id = ip_results[i].rule_id;
ret = validate_port(ip_rt->htable, (char *)&item_id, sizeof(item_id), port, proto);
struct ip_item *ip_item = (struct ip_item *)rcu_hash_find(ip_rt->ip_item_hash,
(char *)&item_id,
sizeof(long long));
if (!ip_item) {
// item config has been deleted
continue;
}
ret = validate_port(ip_item, port, proto);
if (ret < 0) {
continue;
}
hit_item_ids[real_hit_index++] = ip_results[i].rule_id;
hit_maat_items[real_hit_item_cnt].item_id = ip_results[i].rule_id;
hit_maat_items[real_hit_item_cnt].group_id = ip_item->group_id;
real_hit_item_cnt++;
}
}
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(ip_rt->item_htable, vtable_id, hit_item_ids,
real_hit_index, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return group_hit_cnt;
return real_hit_item_cnt;
}
void ip_runtime_hit_inc(struct ip_runtime *ip_rt, int thread_id)

27
src/maat_rule.c
View File

@@ -32,30 +32,6 @@
#define MODULE_MAAT_RULE module_name_str("maat.rule")
struct maat_item *maat_item_new(long long item_id, long long group_id,
void *user_data, void (*user_data_free)(void *))
{
struct maat_item *item = NULL;
item = ALLOC(struct maat_item, 1);
item->group_id = group_id;
item->item_id = item_id;
item->user_data = user_data;
item->user_data_free = user_data_free;
return item;
}
void maat_item_free(void *maat_item)
{
struct maat_item *item = (struct maat_item *)maat_item;
if (item->user_data_free && item->user_data) {
item->user_data_free(item->user_data);
item->user_data = NULL;
}
FREE(item);
}
static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, size_t n_accept_tag)
{
if (NULL == tag_obj || NULL == accept_tags) {
@@ -210,7 +186,8 @@ error:
return ret;
}
struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, long long district_id)
struct maat_item_inner *maat_item_inner_new(long long item_id, long long group_id,
long long district_id)
{
struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1);
item->magic_num = ITEM_RULE_MAGIC;