gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FEATURE]Refactor NOT clause, NOTE:forward incompatibility!!!

Browse Source
This commit is contained in:
刘文坛
2023-10-18 03:32:53 +00:00
parent 48af7e7aac
commit 613b5b3dcf
19 changed files with 1609 additions and 498 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/inc_internal/maat_compile.h
View File

@@ -91,7 +91,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
const char *table_name, const char *line,
int valid_column);
long long group2compile_runtime_not_group_count(void *g2c_runtime);
long long group2compile_runtime_not_clause_count(void *g2c_runtime);
long long group2compile_runtime_rule_count(void *g2c_runtime);
@@ -104,7 +104,7 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state);
void maat_compile_state_free(struct maat_compile_state *compile_state,
struct maat *maat_instance, int thread_id);
void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
int maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
size_t n_hit_item, struct maat_state *state);
size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *compile_state,

7
src/inc_internal/maat_rule.h
View File

@@ -155,7 +155,6 @@ struct maat_stat {
long long *thread_call_cnt;
long long *stream_num;
long long *hit_cnt;
long long *not_grp_hit_cnt;
long long *maat_state_cnt;
long long *maat_compile_state_cnt;
long long *maat_state_free_cnt;
@@ -190,11 +189,6 @@ struct maat {
struct maat_stat *stat;
};
enum compile_not_flag {
COMPILE_NOT_UNSET,
COMPILE_NOT_SET
};
struct maat_state {
struct maat *maat_inst;
struct maat_compile_state *compile_state;
@@ -203,7 +197,6 @@ struct maat_state {
uint16_t thread_id;
int16_t compile_table_id;
uint8_t is_set_district;
uint8_t compile_NOT_flag;
};
int my_scandir(const char *dir, struct dirent ***namelist,

85
src/maat_api.c
View File

@@ -1016,17 +1016,6 @@ int maat_bool_plugin_table_get_ex_data(struct maat *maat_inst, int table_id,
ex_data_array, array_size);
}
static inline int scan_status_should_compile_NOT(struct maat_state *state)
{
if (state && (COMPILE_NOT_SET == state->compile_NOT_flag) &&
state->compile_state &&
maat_compile_state_has_NOT_clause(state->compile_state)) {
return 1;
}
return 0;
}
static size_t hit_group_to_compile(void *compile_runtime, long long *compile_ids,
size_t compile_ids_size, struct maat_state *state)
{
@@ -1287,18 +1276,13 @@ int maat_scan_flag(struct maat *maat_inst, int table_id,
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
if (hit_group_cnt > 0) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
*n_hit_result = sum_hit_compile_cnt;
}
if (sum_hit_compile_cnt > 0) {
alignment_int64_array_add(maat_inst->stat->hit_cnt, state->thread_id, 1);
if (0 == hit_group_cnt) {
//hit NOT group
alignment_int64_array_add(maat_inst->stat->not_grp_hit_cnt,
state->thread_id, 1);
}
}
void *flag_rt = table_manager_get_runtime(maat_inst->tbl_mgr, phy_table_id);
@@ -1376,18 +1360,13 @@ int maat_scan_integer(struct maat *maat_inst, int table_id,
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
if (hit_group_cnt > 0) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
*n_hit_result = sum_hit_compile_cnt;
}
if (sum_hit_compile_cnt > 0) {
alignment_int64_array_add(maat_inst->stat->hit_cnt, state->thread_id, 1);
if (0 == hit_group_cnt) {
//hit NOT group
alignment_int64_array_add(maat_inst->stat->not_grp_hit_cnt,
state->thread_id, 1);
}
}
void *interval_rt = table_manager_get_runtime(maat_inst->tbl_mgr, phy_table_id);
@@ -1465,18 +1444,13 @@ int maat_scan_ipv4(struct maat *maat_inst, int table_id, uint32_t ip_addr,
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
if (hit_group_cnt > 0) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
*n_hit_result = sum_hit_compile_cnt;
}
if (sum_hit_compile_cnt > 0) {
alignment_int64_array_add(maat_inst->stat->hit_cnt, state->thread_id, 1);
if (0 == hit_group_cnt) {
//hit NOT group
alignment_int64_array_add(maat_inst->stat->not_grp_hit_cnt,
state->thread_id, 1);
}
}
void *ip_rt = table_manager_get_runtime(maat_inst->tbl_mgr, phy_table_id);
@@ -1555,18 +1529,13 @@ int maat_scan_ipv6(struct maat *maat_inst, int table_id,
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
if (hit_group_cnt > 0) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
*n_hit_result = sum_hit_compile_cnt;
}
if (sum_hit_compile_cnt > 0) {
alignment_int64_array_add(maat_inst->stat->hit_cnt, state->thread_id, 1);
if (0 == hit_group_cnt) {
//hit NOT group
alignment_int64_array_add(maat_inst->stat->not_grp_hit_cnt,
state->thread_id, 1);
}
}
void *ip_rt = table_manager_get_runtime(maat_inst->tbl_mgr, phy_table_id);
@@ -1644,18 +1613,13 @@ int maat_scan_string(struct maat *maat_inst, int table_id, const char *data,
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
if (hit_group_cnt > 0) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
*n_hit_result = sum_hit_compile_cnt;
}
if (sum_hit_compile_cnt > 0) {
alignment_int64_array_add(maat_inst->stat->hit_cnt, state->thread_id, 1);
if (0 == hit_group_cnt) {
//hit NOT group
alignment_int64_array_add(maat_inst->stat->not_grp_hit_cnt,
state->thread_id, 1);
}
}
void *expr_rt = table_manager_get_runtime(maat_inst->tbl_mgr, phy_table_id);
@@ -1772,18 +1736,13 @@ int maat_stream_scan(struct maat_stream *maat_stream, const char *data, int data
}
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
if (hit_group_cnt > 0) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
*n_hit_result = sum_hit_compile_cnt;
}
if (sum_hit_compile_cnt > 0) {
alignment_int64_array_add(maat_inst->stat->hit_cnt, maat_stream->thread_id, 1);
if (0 == hit_group_cnt) {
//hit NOT group
alignment_int64_array_add(maat_inst->stat->not_grp_hit_cnt,
maat_stream->thread_id, 1);
}
}
if (1 == maat_inst->opts.perf_on) {
@@ -1836,7 +1795,6 @@ struct maat_state *maat_state_new(struct maat *maat_inst, int thread_id)
state->maat_inst = maat_inst;
state->is_set_district = DISTRICT_FLAG_UNSET;
state->compile_NOT_flag = COMPILE_NOT_UNSET;
state->district_id = DISTRICT_ANY;
state->thread_id = thread_id;
@@ -1854,7 +1812,6 @@ void maat_state_reset(struct maat_state *state)
state->compile_table_id = 0;
state->is_set_district = DISTRICT_FLAG_UNSET;
state->compile_NOT_flag = COMPILE_NOT_UNSET;
state->district_id = DISTRICT_ANY;
state->scan_cnt = 0;
@@ -1960,36 +1917,6 @@ int maat_state_set_scan_district(struct maat_state *state, int table_id,
return 0;
}
int maat_state_enable_compile_NOT(struct maat_state *state)
{
if (NULL == state) {
return -1;
}
struct maat *maat_inst = state->maat_inst;
assert(maat_inst != NULL);
if (NULL == maat_inst->maat_rt) {
return -1;
}
assert(state->compile_NOT_flag == COMPILE_NOT_UNSET);
state->compile_NOT_flag = COMPILE_NOT_SET;
return 0;
}
int maat_state_disable_compile_NOT(struct maat_state *state)
{
if (NULL == state) {
return -1;
}
state->compile_NOT_flag = COMPILE_NOT_UNSET;
return 0;
}
int maat_state_set_scan_compile_table(struct maat_state *state, int compile_table_id)
{
if (NULL == state || compile_table_id < 0) {

1
src/maat_bool_plugin.c
View File

@@ -333,6 +333,7 @@ bool_plugin_expr_new(struct bool_plugin_schema *schema, const char *table_name,
goto error;
}
memset(expr_buffer, 0, sizeof(expr_buffer));
memcpy(expr_buffer, line + column_offset, column_len);
for (token = expr_buffer; ; token = NULL) {
sub_token = strtok_r(token, "&", &saveptr);

361
src/maat_compile.c
View File

@@ -27,9 +27,11 @@
#include "alignment.h"
#define MODULE_COMPILE module_name_str("maat.compile")
#define DEFAULT_GC_TIMEOUT_S 10
#define MAX_SUPER_GROUP_CNT 128
#define MAX_NOT_CLAUSE_NUM 8
#define DEFAULT_GC_TIMEOUT_S 10
#define MAX_SUPER_GROUP_CNT 128
#define MAX_NOT_CLAUSE_NUM 8
#define VTABLE_MAX_NOT_GROUP_NUM 8
enum clause_not_flag {
CLAUSE_NOT_FLAG_UNSET = 0,
@@ -75,12 +77,12 @@ struct group2compile_item {
int not_flag;
int vtable_id;
int clause_index;
int asso_compile_table_id;
};
struct maat_literal_id {
long long group_id;
long long vtable_id;
int vtable_id;
int not_flag;
};
struct literal_clause {
@@ -89,13 +91,20 @@ struct literal_clause {
UT_hash_handle hh;
};
struct vtable_not_clause {
int vtable_id;
int not_clause_num;
UT_hash_handle hh;
};
/* compile_runtime and group2compile_runtime share compile_hash_map */
struct compile_runtime {
struct bool_matcher *bm;
struct rcu_hash_table *cfg_hash; // <compile_id, struct maat_compile>
struct maat_runtime *ref_maat_rt;
time_t version;
struct literal_clause *literal2clause_hash;
struct literal_clause *literal2clause_hash; //store clause_ids(not_flag == 0)
struct literal_clause *literal2not_clause_hash; //store NOT_clause_ids(not_flag == 1)
long long rule_num;
long long update_err_cnt;
@@ -105,10 +114,11 @@ struct compile_runtime {
};
struct group2compile_runtime {
long long not_flag_group;
long long not_clause_cnt;
long long rule_num;
long long update_err_cnt;
struct compile_runtime *ref_compile_rt;
struct vtable_not_clause *not_clause_hash;
};
struct maat_clause {
@@ -129,7 +139,6 @@ struct maat_compile {
uint32_t magic_num;
int actual_clause_num;
int declared_clause_num;
int not_clause_cnt;
long long compile_id;
char table_name[MAX_NAME_STR_LEN];
void *user_data; // compile_item
@@ -141,11 +150,10 @@ struct maat_internal_hit_path {
long long group_id;
int Nth_scan;
int vtable_id;
int NOT_flag; // 1 means NOT clause
};
struct maat_compile_state {
uint8_t this_scan_hit_item_flag;
uint8_t not_clause_hit_flag;
int Nth_scan;
time_t compile_rt_version;
@@ -618,6 +626,7 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL,
schema->gc_timeout_s + DEFAULT_GC_TIMEOUT_S);
compile_rt->literal2clause_hash = NULL;
compile_rt->literal2not_clause_hash = NULL;
compile_rt->logger = logger;
compile_rt->ref_garbage_bin = garbage_bin;
@@ -668,6 +677,11 @@ void compile_runtime_free(void *compile_runtime)
compile_rt->literal2clause_hash = NULL;
}
if (compile_rt->literal2not_clause_hash != NULL) {
literal2clause_hash_free(compile_rt->literal2not_clause_hash);
compile_rt->literal2not_clause_hash = NULL;
}
if (compile_rt->expr_match_buff != NULL) {
FREE(compile_rt->expr_match_buff);
}
@@ -694,6 +708,7 @@ void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
}
struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1);
g2c_rt->not_clause_hash = NULL;
return g2c_rt;
}
@@ -711,6 +726,16 @@ void group2compile_runtime_free(void *g2c_runtime)
return;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
if (g2c_rt->not_clause_hash != NULL) {
struct vtable_not_clause *not_clause = NULL, *tmp_not_clause = NULL;
HASH_ITER(hh, g2c_rt->not_clause_hash, not_clause, tmp_not_clause) {
HASH_DEL(g2c_rt->not_clause_hash, not_clause);
FREE(not_clause);
}
}
assert(g2c_rt->not_clause_hash == NULL);
FREE(g2c_runtime);
}
@@ -795,6 +820,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
goto error;
}
memset(vtable_name, 0, sizeof(vtable_name));
memcpy(vtable_name, (line + column_offset), column_len);
if (is_valid_table_name(vtable_name)) {
@@ -849,6 +875,9 @@ static int compare_literal_id(const void *pa, const void *pb)
long long ret = la->vtable_id - lb->vtable_id;
if (0 == ret) {
ret = la->group_id - lb->group_id;
if (0 == ret) {
ret = la->not_flag - lb->not_flag;
}
}
return ret;
@@ -872,6 +901,7 @@ static int maat_compile_clause_add_literal(struct maat_compile *compile,
if (tmp) {
assert(tmp->group_id == literal_id->group_id);
assert(tmp->vtable_id == literal_id->vtable_id);
assert(tmp->not_flag == literal_id->not_flag);
return -1;
} else {
utarray_push_back(clause->ut_literal_ids, literal_id);
@@ -947,20 +977,17 @@ maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, size_t *compil
iter_compile = (struct maat_compile *)data_array[idx];
for (i = 0, j = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
if (iter_compile->clauses[i].in_use) {
if (iter_compile->clauses[i].not_flag) {
iter_compile->not_clause_cnt++;
}
// TODO:mytest need to delete
#if 0
struct maat_literal_id *p = NULL;
for(p = (struct maat_literal_id *)utarray_front(iter_compile->clauses[i].ut_literal_ids); p!=NULL;
p = (struct maat_literal_id *)utarray_next(iter_compile->clauses[i].ut_literal_ids, p)) {
printf("<before bool_matcher_new> compile_rt:%p compile_id:%lld, clause_id:%llu, literal{%lld: %lld}\n",
compile_rt, iter_compile->compile_id, iter_compile->clauses[i].clause_id, p->group_id, p->vtable_id);
printf("<before bool_matcher_new> compile_rt:%p compile_id:%lld, clause_id:%llu, literal{%lld: %d, %d}\n",
compile_rt, iter_compile->compile_id, iter_compile->clauses[i].clause_id, p->group_id, p->vtable_id, p->not_flag);
}
#endif
bool_expr_array[expr_cnt].items[j].item_id = iter_compile->clauses[i].clause_id;
bool_expr_array[expr_cnt].items[j].not_flag = iter_compile->clauses[i].not_flag;
bool_expr_array[expr_cnt].items[j].not_flag = 0;
j++;
}
}
@@ -1013,8 +1040,18 @@ static inline int compare_clause_id(const void *a, const void *b)
}
}
/**
* @brief build <literal, clause_id_array> hash for clause or not_clause
*
* @param compile_rt: compile runtime handle
* @param not_flag: specify whether to build clause or NOT_clause hash for compile runtime
* 0 -> clause hash
* 1 -> NOT_clause hash
*
* @retval generated literal clause hash
*/
static struct literal_clause *
maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt)
maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt, int not_flag)
{
if (NULL == compile_rt) {
return NULL;
@@ -1034,6 +1071,16 @@ maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt)
continue;
}
if (0 == not_flag) {
if (CLAUSE_NOT_FLAG_SET == clause->not_flag) {
continue;
}
} else {
if (CLAUSE_NOT_FLAG_UNSET == clause->not_flag) {
continue;
}
}
for (size_t j = 0; j < utarray_len(clause->ut_literal_ids); j++) {
tmp_literal_id = (struct maat_literal_id *)utarray_eltptr(clause->ut_literal_ids, j);
HASH_FIND(hh, literal2clause_hash, tmp_literal_id, sizeof(struct maat_literal_id), l2c_value);
@@ -1093,9 +1140,10 @@ static size_t compile_state_if_new_hit_compile(struct maat_compile_state *compil
return r_in_c_cnt;
}
size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int compile_not_flag,
struct maat_compile_state *compile_state, int thread_id,
void **user_data_array, size_t ud_array_size)
size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt,
struct maat_compile_state *compile_state,
int thread_id, void **user_data_array,
size_t ud_array_size)
{
size_t ud_result_cnt = 0;
struct maat_compile *compile = NULL;
@@ -1109,7 +1157,6 @@ size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int c
if (NULL == compile_rt->bm || 0 == utarray_len(compile_state->all_hit_clauses)
|| compile_state->compile_rt_version != compile_rt->version) {
compile_state->this_scan_hit_item_flag = 0;
return 0;
}
@@ -1135,21 +1182,13 @@ size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int c
}
size_t n_new_hit_compile = compile_state_if_new_hit_compile(compile_state, compile);
int this_scan_hit_item_flag = compile_state->this_scan_hit_item_flag;
if ((compile->not_clause_cnt > 0) && (COMPILE_NOT_UNSET == compile_not_flag)) {
compile_state->not_clause_hit_flag = 1;
} else if (compile->user_data) {
if (n_new_hit_compile > 0 || 0 == this_scan_hit_item_flag) {
/* compile hit because of new item or
hit a compile that refer a NOT-logic group in previous scan */
user_data_array[ud_result_cnt] = compile->user_data;
ud_result_cnt++;
}
if (compile->user_data != NULL && n_new_hit_compile > 0) {
user_data_array[ud_result_cnt] = compile->user_data;
ud_result_cnt++;
}
}
compile_state->this_scan_hit_item_flag = 0;
return ud_result_cnt;
}
@@ -1184,7 +1223,6 @@ maat_compile_clone(struct maat_compile *compile, int deep_copy)
new_compile->actual_clause_num = compile->actual_clause_num;
new_compile->declared_clause_num = compile->declared_clause_num;
memcpy(new_compile->table_name, compile->table_name, sizeof(new_compile->table_name));
new_compile->not_clause_cnt = compile->not_clause_cnt;
if (1 == deep_copy && compile->user_data != NULL) {
new_compile->user_data = compile_item_clone((struct compile_item *)compile->user_data);
}
@@ -1215,7 +1253,8 @@ static int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl,
int ret = -1;
long long compile_id = g2c_item->compile_id;
struct maat_compile *compile = NULL;
struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vtable_id};
struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vtable_id,
g2c_item->not_flag};
int updating_flag = rcu_hash_is_updating(hash_tbl);
if (1 == updating_flag) {
@@ -1297,7 +1336,8 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
int ret = -1;
long long compile_id = g2c_item->compile_id;
struct maat_compile *compile = NULL;
struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vtable_id};
struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vtable_id,
g2c_item->not_flag};
int updating_flag = rcu_hash_is_updating(hash_tbl);
if (1 == updating_flag) {
@@ -1392,8 +1432,6 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state)
compile_state->Nth_scan = 0;
compile_state->compile_rt_version = 0;
compile_state->this_scan_hit_item_flag = 0;
compile_state->not_clause_hit_flag = 0;
utarray_clear(compile_state->internal_hit_paths);
utarray_clear(compile_state->all_hit_clauses);
@@ -1449,18 +1487,20 @@ void maat_compile_state_free(struct maat_compile_state *compile_state,
}
static void maat_compile_hit_path_add(UT_array *hit_paths, long long item_id,
long long group_id, int vtable_id, int Nth_scan)
long long group_id, int vtable_id, int NOT_flag,
int Nth_scan)
{
if (NULL == hit_paths || utarray_len(hit_paths) >= MAX_HIT_PATH_NUM) {
return;
}
struct maat_internal_hit_path new_path;
new_path.item_id = item_id;
new_path.Nth_scan = Nth_scan;
new_path.group_id = group_id;
new_path.vtable_id = vtable_id;
new_path.NOT_flag = NOT_flag;
utarray_push_back(hit_paths, &new_path);
}
@@ -1537,6 +1577,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
}
literal_id.vtable_id = hit_path_array[j].vtable_id;
literal_id.not_flag = hit_path_array[j].NOT_flag;
if (maat_compile_has_literal(compile, &literal_id)) {
if (hit_path_array[j].top_group_id < 0) {
hit_path_array[j].top_group_id = hit_path_array[j].sub_group_id;
@@ -1594,42 +1635,15 @@ static void maat_compile_state_update_indirect_hit_groups(UT_array *hit_group_ar
}
}
static void maat_compile_state_update_hit_paths(struct maat_compile_state *compile_state,
struct maat_item *hit_items, size_t n_hit_items,
int vtable_id, int Nth_scan)
static void exec_update_hit_clauses(struct maat_compile_state *compile_state,
UT_array *clause_id_array)
{
if (NULL == compile_state) {
return;
}
for (size_t i = 0; i < n_hit_items; i++) {
maat_compile_hit_path_add(compile_state->internal_hit_paths, hit_items[i].item_id,
hit_items[i].group_id, vtable_id, Nth_scan);
}
}
static void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state,
struct compile_runtime *compile_rt,
long long group_id, int vtable_id)
{
if (NULL == compile_state || NULL == compile_rt) {
return;
}
struct maat_literal_id literal_id = {group_id, vtable_id};
struct literal_clause *l2c_val = NULL;
HASH_FIND(hh, compile_rt->literal2clause_hash, &literal_id, sizeof(literal_id), l2c_val);
if (!l2c_val) {
return;
}
size_t i = 0;
long long *clause_id = 0;
long long *clause_id = NULL;
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
for (i = 0; i < utarray_len(l2c_val->clause_ids); i++) {
clause_id = (long long *)utarray_eltptr(l2c_val->clause_ids, i);
for (i = 0; i < utarray_len(clause_id_array); i++) {
clause_id = (long long *)utarray_eltptr(clause_id_array, i);
if (utarray_find(compile_state->all_hit_clauses, clause_id, compare_clause_id)) {
continue;
}
@@ -1639,7 +1653,7 @@ static void maat_compile_state_update_hit_clause(struct maat_compile_state *comp
if ((utarray_len(compile_state->this_scan_hit_clauses) - new_clause_idx) > 0) {
utarray_reserve(compile_state->all_hit_clauses,
utarray_len(compile_state->this_scan_hit_clauses) - new_clause_idx);
for (i = new_clause_idx; i < utarray_len(compile_state->this_scan_hit_clauses); i++) {
clause_id = (long long *)utarray_eltptr(compile_state->this_scan_hit_clauses, i);
utarray_push_back(compile_state->all_hit_clauses, clause_id);
@@ -1648,9 +1662,76 @@ static void maat_compile_state_update_hit_clause(struct maat_compile_state *comp
}
}
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state)
{
return compile_state->not_clause_hit_flag;
static void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state,
struct compile_runtime *compile_rt,
long long group_id, int vtable_id)
{
if (NULL == compile_state || NULL == compile_rt) {
return;
}
struct maat_literal_id literal_id = {group_id, vtable_id, 0};
struct literal_clause *l2c_val = NULL;
HASH_FIND(hh, compile_rt->literal2clause_hash, &literal_id, sizeof(literal_id), l2c_val);
if (!l2c_val) {
return;
}
exec_update_hit_clauses(compile_state, l2c_val->clause_ids);
}
static inline int compare_group_id(const void *a, const void *b)
{
long long ret = *(const long long *)a - *(const long long *)b;
if (0 == ret) {
return 0;
} else if(ret < 0) {
return -1;
} else {
return 1;
}
}
static size_t maat_compile_state_update_hit_not_clauses(struct maat_compile_state *compile_state,
struct compile_runtime *compile_rt,
long long *group_ids, size_t n_group_ids,
int vtable_id, long long *NOT_group_ids_array,
size_t NOT_group_ids_array_size)
{
if (NULL == compile_state || NULL == compile_rt) {
return 0;
}
if (n_group_ids != 0) {
qsort(group_ids, n_group_ids, sizeof(long long *), compare_group_id);
}
size_t hit_NOT_group_cnt = 0;
struct literal_clause *l2c_val = NULL, *tmp_l2c_val = NULL;
//NOTE: Each virtual table can reference up to VTABLE_MAX_NOT_GROUP_NUM groups
HASH_ITER(hh, compile_rt->literal2not_clause_hash, l2c_val, tmp_l2c_val) {
if (l2c_val->key.vtable_id != vtable_id) {
continue;
}
long long *tmp_group_id = bsearch(&(l2c_val->key.group_id), group_ids, n_group_ids,
sizeof(long long), compare_group_id);
if (tmp_group_id != NULL) {
continue;
}
if (hit_NOT_group_cnt < NOT_group_ids_array_size) {
NOT_group_ids_array[hit_NOT_group_cnt++] = l2c_val->key.group_id;
}
exec_update_hit_clauses(compile_state, l2c_val->clause_ids);
}
return hit_NOT_group_cnt;
}
void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
@@ -1868,6 +1949,50 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
return 0;
}
int validate_vtable_not_clause(struct group2compile_runtime *g2c_rt,
struct table_manager *tbl_mgr, int vtable_id,
int is_valid, struct log_handle *logger)
{
enum table_type table_type = table_manager_get_table_type(tbl_mgr, vtable_id);
if (table_type != TABLE_TYPE_VIRTUAL) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]table(table_id:%d) is not virtual table, can't own NOT clause.",
__FUNCTION__, __LINE__, vtable_id);
return -1;
}
struct vtable_not_clause *not_clause = NULL;
HASH_FIND_INT(g2c_rt->not_clause_hash, &vtable_id, not_clause);
if (0 == is_valid) {
//delete
if (NULL == not_clause || 0 == not_clause->not_clause_num) {
return 0;
} else {
not_clause->not_clause_num--;
}
} else {
//add
if (NULL == not_clause) {
not_clause = ALLOC(struct vtable_not_clause, 1);
not_clause->vtable_id = vtable_id;
not_clause->not_clause_num++;
HASH_ADD_INT(g2c_rt->not_clause_hash, vtable_id, not_clause);
} else {
if (not_clause->not_clause_num >= MAX_NOT_CLAUSE_NUM) {
const char *table_name = table_manager_get_table_name(tbl_mgr, vtable_id);
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]virtual table:<%s> NOT clause num exceed maximum:%d",
__FUNCTION__, __LINE__, table_name, MAX_NOT_CLAUSE_NUM);
return -1;
}
not_clause->not_clause_num++;
}
}
return 0;
}
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
const char *table_name, const char *line,
int valid_column)
@@ -1898,13 +2023,24 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
return -1;
}
if (1 == g2c_item->not_flag) {
ret = validate_vtable_not_clause(g2c_rt, schema->ref_tbl_mgr, g2c_item->vtable_id,
is_valid, compile_rt->logger);
if (ret < 0) {
log_fatal(compile_rt->logger, MODULE_COMPILE,
"[%s:%d]validate NOT clause failed, abandon config:%s",
__FUNCTION__, __LINE__, line);
goto next;
}
}
if (0 == is_valid) {
//delete
ret = maat_remove_group_from_compile(compile_rt->cfg_hash, g2c_item,
compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
g2c_rt->not_flag_group--;
g2c_rt->not_clause_cnt--;
}
g2c_rt->rule_num--;
} else {
@@ -1916,26 +2052,27 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
g2c_rt->not_flag_group++;
g2c_rt->not_clause_cnt++;
}
g2c_rt->rule_num++;
} else {
g2c_rt->update_err_cnt++;
}
}
next:
group2compile_item_free(g2c_item);
return ret;
}
long long group2compile_runtime_not_group_count(void *g2c_runtime)
long long group2compile_runtime_not_clause_count(void *g2c_runtime)
{
if (NULL == g2c_runtime) {
return 0;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
return g2c_rt->not_flag_group;
return g2c_rt->not_clause_cnt;
}
long long group2compile_runtime_rule_count(void *g2c_runtime)
@@ -1998,10 +2135,14 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
struct literal_clause *old_literal2clause = NULL;
struct literal_clause *new_literal2clause = NULL;
struct literal_clause *old_literal2not_clause = NULL;
struct literal_clause *new_literal2not_clause = NULL;
new_literal2clause = maat_compile_build_literal2clause_hash(compile_rt);
new_literal2clause = maat_compile_build_literal2clause_hash(compile_rt, 0);
new_literal2not_clause = maat_compile_build_literal2clause_hash(compile_rt, 1);
old_literal2clause = compile_rt->literal2clause_hash;
old_literal2not_clause = compile_rt->literal2not_clause_hash;
old_bool_matcher = compile_rt->bm;
/*
@@ -2021,12 +2162,15 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
compile_rt->bm = new_bool_matcher;
compile_rt->literal2clause_hash = new_literal2clause;
compile_rt->literal2not_clause_hash = new_literal2not_clause;
rcu_hash_commit(compile_rt->cfg_hash);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL,
garbage_bool_matcher_free);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_literal2clause, NULL,
garbage_literal2clause_hash_free);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_literal2not_clause, NULL,
garbage_literal2clause_hash_free);
compile_rt->rule_num = rcu_hash_count(compile_rt->cfg_hash);
@@ -2087,12 +2231,11 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
size_t compile_ids_size, struct maat_state *state)
{
struct maat_compile_state *compile_state = state->compile_state;
int compile_NOT_flag = state->compile_NOT_flag;
struct compile_item *compile_items[compile_ids_size];
// all hit clause_id -> compile_id
size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, compile_NOT_flag,
compile_state, state->thread_id,
size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, compile_state,
state->thread_id,
(void **)compile_items,
compile_ids_size);
if (bool_match_ret > 0) {
@@ -2107,17 +2250,13 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
return MIN(bool_match_ret, compile_ids_size);
}
void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
int maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
size_t n_hit_item, struct maat_state *state)
{
size_t i = 0;
size_t i = 0, j = 0;
size_t hit_cnt = n_hit_item;
long long hit_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
if (hit_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
hit_cnt = MAX_SCANNER_HIT_GROUP_NUM;
}
struct maat *maat_inst = state->maat_inst;
if (NULL == state->compile_state) {
state->compile_state = maat_compile_state_new();
@@ -2128,11 +2267,9 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
struct maat_compile_state *compile_state = state->compile_state;
if (hit_cnt > 0) {
if (compile_state->Nth_scan != state->scan_cnt) {
assert(compile_state->this_scan_hit_item_flag == 0);
compile_state->Nth_scan = state->scan_cnt;
utarray_clear(compile_state->this_scan_hit_clauses);
}
compile_state->this_scan_hit_item_flag = 1;
}
for (i = 0; i < hit_cnt; i++) {
@@ -2146,10 +2283,12 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids,
hit_cnt, super_group_ids,
MAX_SCANNER_HIT_GROUP_NUM);
if (1 == maat_inst->opts.hit_path_on) {
maat_compile_state_update_hit_paths(compile_state, hit_items, hit_cnt,
vtable_id, state->scan_cnt);
}
if (1 == maat_inst->opts.hit_path_on && hit_cnt > 0) {
for (i = 0; i < hit_cnt; i++) {
maat_compile_hit_path_add(compile_state->internal_hit_paths, hit_items[i].item_id,
hit_items[i].group_id, vtable_id, 0, state->scan_cnt);
}
}
if (1 == maat_inst->opts.hit_group_on) {
maat_compile_state_update_direct_hit_groups(compile_state->direct_hit_groups,
@@ -2167,18 +2306,31 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items,
struct compile_runtime *compile_rt = table_manager_get_runtime(maat_inst->tbl_mgr,
compile_table_id);
if (NULL == compile_rt) {
return;
return 0;
}
for (int j = 0; j < super_group_cnt; j++) {
maat_compile_state_update_hit_clause(compile_state, compile_rt,
super_group_ids[j], vtable_id);
for (j = 0; j < super_group_cnt && hit_cnt < MAX_SCANNER_HIT_GROUP_NUM; j++) {
hit_group_ids[hit_cnt++] = super_group_ids[j];
}
for (int j = 0; j < hit_cnt; j++) {
maat_compile_state_update_hit_clause(compile_state, compile_rt,
hit_group_ids[j], vtable_id);
for (i = 0; i < hit_cnt; i++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
hit_group_ids[i], vtable_id);
}
long long hit_NOT_group_ids[VTABLE_MAX_NOT_GROUP_NUM];
size_t hit_not_cnt = maat_compile_state_update_hit_not_clauses(state->compile_state, compile_rt,
hit_group_ids, hit_cnt, vtable_id,
hit_NOT_group_ids, VTABLE_MAX_NOT_GROUP_NUM);
if (1 == maat_inst->opts.hit_path_on && hit_not_cnt > 0) {
for (i = 0; i < hit_not_cnt; i++) {
maat_compile_hit_path_add(compile_state->internal_hit_paths, -1, hit_NOT_group_ids[i],
vtable_id, 1, state->scan_cnt);
}
}
return (hit_cnt + hit_not_cnt);
}
size_t maat_compile_state_get_indirect_hit_groups(struct maat_compile_state *compile_state,
@@ -2272,6 +2424,7 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp
tmp_path.sub_group_id = internal_path->group_id;
tmp_path.top_group_id = *p;
tmp_path.vtable_id = internal_path->vtable_id;
tmp_path.NOT_flag = internal_path->NOT_flag;
tmp_path.compile_id = -1;
/* check if internal_path is duplicated from hit_path_array[] element */

39
src/maat_expr.c
View File

@@ -975,20 +975,13 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
return -1;
}
if (0 == n_hit_item) {
return 0;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
log_info(expr_rt->logger, MODULE_EXPR,
"hit expr item count:%d exceed maxium:%d",
n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_item hit_maat_items[n_hit_item];
size_t real_hit_item_cnt = 0;
if (0 == n_hit_item) {
goto next;
}
for (size_t i = 0; i < n_hit_item; i++) {
int tag_district_id = *(int *)(hit_results[i].user_tag);
if (tag_district_id == state->district_id || tag_district_id == DISTRICT_ANY) {
@@ -1008,9 +1001,8 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
}
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return real_hit_item_cnt;
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
struct expr_matcher_stream *
@@ -1048,18 +1040,14 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
return -1;
}
if (0 == n_hit_item) {
return 0;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_item hit_maat_items[n_hit_item];
struct expr_item *expr_item = NULL;
size_t real_hit_item_cnt = 0;
if (0 == n_hit_item) {
goto next;
}
for (size_t i = 0; i < n_hit_item; i++) {
long long item_id = hit_results[i].rule_id;
expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
@@ -1075,9 +1063,8 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
real_hit_item_cnt++;
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return real_hit_item_cnt;
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,

19
src/maat_flag.c
View File

@@ -558,17 +558,17 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int n_hit_item = flag_matcher_match(flag_rt->matcher, flag,
hit_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_item <= 0) {
return n_hit_item;
if (n_hit_item < 0) {
return -1;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_item hit_maat_items[n_hit_item];
size_t real_hit_item_cnt = 0;
if (0 == n_hit_item) {
goto next;
}
for (int i = 0; i < n_hit_item; i++) {
int tag_district_id = *(int *)(hit_results[i].user_tag);
if (tag_district_id == state->district_id || tag_district_id == DISTRICT_ANY) {
@@ -587,9 +587,8 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
}
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return real_hit_item_cnt;
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
void flag_runtime_hit_inc(struct flag_runtime *flag_rt, int thread_id)

7
src/maat_group.c
View File

@@ -363,6 +363,13 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
goto error;
}
if (g2g_item->is_exclude != 0 && g2g_item->is_exclude != 1) {
log_error(logger, MODULE_GROUP,
"[%s:%d] g2g table:<%s> is_exclude:%d is illegal in line:%s",
__FUNCTION__, __LINE__, table_name, g2g_item->is_exclude, line);
goto error;
}
return g2g_item;
error:
FREE(g2g_item);

17
src/maat_interval.c
View File

@@ -559,17 +559,17 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int n_hit_item = interval_matcher_match(interval_rt->matcher, integer,
hit_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_item <= 0) {
return n_hit_item;
if (n_hit_item < 0) {
return -1;
}
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
}
struct maat_item hit_maat_items[n_hit_item];
size_t real_hit_item_cnt = 0;
if (0 == n_hit_item) {
goto next;
}
for (int i = 0; i < n_hit_item; i++) {
int tag_district_id = *(int *)(hit_results[i].user_tag);
if (tag_district_id == state->district_id || tag_district_id == DISTRICT_ANY) {
@@ -588,9 +588,8 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
}
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return real_hit_item_cnt;
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
void interval_runtime_hit_inc(struct interval_runtime *interval_rt, int thread_id)

21
src/maat_ip.c
View File

@@ -385,6 +385,14 @@ ip_item_new(struct ip_schema *ip_schema, const char *table_name,
goto error;
}
if (ip_item->proto != IP_PROTO_ANY && ip_item->proto != IP_PROTO_ICMP &&
ip_item->proto != IP_PROTO_TCP && ip_item->proto != IP_PROTO_UDP) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table:<%s> protocol:%d is illegal in line:%s",
__FUNCTION__, __LINE__, table_name, ip_item->proto, line);
goto error;
}
return ip_item;
error:
FREE(ip_item);
@@ -724,12 +732,12 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
int n_hit_ip_item = ip_matcher_match(ip_rt->ip_matcher, &scan_data,
ip_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_ip_item <= 0) {
return n_hit_ip_item;
if (n_hit_ip_item < 0) {
return -1;
}
if (n_hit_ip_item > MAX_SCANNER_HIT_ITEM_NUM) {
n_hit_ip_item = MAX_SCANNER_HIT_ITEM_NUM;
if (0 == n_hit_ip_item) {
goto next;
}
for (size_t i = 0; i < n_hit_ip_item; i++) {
@@ -752,9 +760,8 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
real_hit_item_cnt++;
}
maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
return real_hit_item_cnt;
next:
return maat_compile_state_update(vtable_id, hit_maat_items, real_hit_item_cnt, state);
}
void ip_runtime_hit_inc(struct ip_runtime *ip_rt, int thread_id)

68
src/maat_redis_monitor.c
View File

@@ -43,11 +43,11 @@ const char *foreign_key_prefix = "__FILE_";
const char *mr_op_str[] = {"DEL", "ADD", "RENEW_TIMEOUT"};
#define POSSIBLE_REDIS_REPLY_SIZE 2
#define REDIS_REPLY_SIZE 2
struct expected_reply {
int s_rule_seq;
int possible_reply_num;
redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE];
redisReply possible_replies[REDIS_REPLY_SIZE];
};
static char *get_foreign_cont_filename(const char *table_name, long long rule_id,
@@ -867,7 +867,7 @@ static void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get %s,%lld foreign key %s content failed, redis server error",
__FUNCTION__, __LINE__,
rule_list[track[i].rule_idx].table_name,
rule_list[track[i].rule_idx].table_name,
rule_list[track[i].rule_idx].rule_id,
rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key);
break;
@@ -886,15 +886,15 @@ static void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
FILE *fp = fopen(s_rule->f_keys[track[i].foreign_idx].filename, "w");
if (NULL == fp) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Write foreign content failed: fopen %s error",
__FUNCTION__, __LINE__, s_rule->f_keys[track[i].foreign_idx].filename);
"[%s:%d] Write foreign content failed: fopen %s error", __FUNCTION__,
__LINE__, s_rule->f_keys[track[i].foreign_idx].filename);
} else {
fwrite(reply->str, 1, reply->len, fp);
fclose(fp);
fp = NULL;
if (1 == print_fn) {
printf("[%s:%d] Written foreign content %s\n",
__FUNCTION__, __LINE__, s_rule->f_keys[track[i].foreign_idx].filename);
printf("[%s:%d] Written foreign content %s\n", __FUNCTION__,
__LINE__, s_rule->f_keys[track[i].foreign_idx].filename);
}
}
}
@@ -920,7 +920,7 @@ void maat_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
}
}
static int invalidate_line(char *line, int column_seq)
static int validate_line(char *line, int column_seq)
{
if (NULL == line || column_seq < 0) {
return -1;
@@ -957,7 +957,8 @@ void maat_rewrite_table_line_with_foreign(struct serial_rule *s_rule)
strncat(pos_rewrite_line, pos_origin_line, origin_column - pos_origin_line);
pos_rewrite_line += origin_column - pos_origin_line;
pos_origin_line = origin_column+origin_column_size;
strncat(pos_rewrite_line, s_rule->f_keys[i].filename, strlen(s_rule->f_keys[i].filename));
strncat(pos_rewrite_line, s_rule->f_keys[i].filename,
strlen(s_rule->f_keys[i].filename));
pos_rewrite_line += strlen(s_rule->f_keys[i].filename);
}
@@ -971,7 +972,7 @@ static void expected_reply_add(struct expected_reply* expected, int s_rule_seq,
int type, long long integer)
{
int i = expected->possible_reply_num;
assert(i < POSSIBLE_REDIS_REPLY_SIZE);
assert(i < REDIS_REPLY_SIZE);
expected->s_rule_seq = s_rule_seq;
expected->possible_replies[i].type = type;
expected->possible_replies[i].integer = integer;
@@ -1049,7 +1050,7 @@ const char* lua_exec_done=
"return maat_version;";
static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list,
long long server_time, int renew_allowed,
struct expected_reply *expect_reply, size_t *cnt)
struct expected_reply *expect_reply, size_t *cnt)
{
redisReply *data_reply = NULL;
@@ -1061,11 +1062,8 @@ static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction
if (strlen(transaction_list) > 0) {
data_reply = maat_wrap_redis_command(c, "eval %s 4 MAAT_VERSION %s %s %s %lld",
lua_exec_done,
mr_status_sset,
mr_version_sset,
transaction_list,
server_time);
lua_exec_done, mr_status_sset, mr_version_sset,
transaction_list, server_time);
freeReplyObject(data_reply);
data_reply = NULL;
expected_reply_add(expect_reply + *cnt, -1, REDIS_REPLY_INTEGER, 0);
@@ -1077,10 +1075,10 @@ static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction
return data_reply;
}
static void exec_serial_rule(redisContext *c, const char *transaction_list,
struct serial_rule *s_rule, size_t rule_num,
struct expected_reply *expect_reply, size_t *cnt,
size_t offset, int renew_allowed)
static void exec_serial_rule(redisContext *c, const char *transaction_list,
struct serial_rule *s_rule, size_t rule_num,
struct expected_reply *expect_reply, size_t *cnt,
size_t offset, int renew_allowed)
{
size_t i = 0;
size_t append_cmd_cnt = 0;
@@ -1235,7 +1233,7 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
char transaction_list[NAME_MAX * 2] = {0};
long long transaction_version = 0;
long long transaction_finished_version = 0;
size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2;// 2 for operation in exec_serial_rule_end()
size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2; // 2 for operation in exec_serial_rule_end()
struct expected_reply *expected_reply = ALLOC(struct expected_reply, max_multi_cmd_num);
for (i = 0; i < serial_rule_num; i++) {
@@ -1456,7 +1454,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
//authorized to write
if (mr_ctx->write_ctx != NULL && mr_ctx->write_ctx->err == 0) {
//For thread safe, deliberately use redis_read_ctx but not redis_write_ctx.
if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout_ms)) {
if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock,
mr_expire_lock_timeout_ms)) {
check_maat_expiration(mr_ctx->read_ctx, maat_inst->logger);
cleanup_update_status(mr_ctx->read_ctx, maat_inst->logger);
redlock_unlock(mr_ctx->read_ctx, mr_expire_lock);
@@ -1464,7 +1463,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
}
if (NULL == mr_ctx->read_ctx || mr_ctx->read_ctx->err) {
if (time(NULL) - mr_ctx->last_reconnect_time < MAAT_REDIS_RECONNECT_INTERVAL_S) {
if ((time(NULL) - mr_ctx->last_reconnect_time) <
MAAT_REDIS_RECONNECT_INTERVAL_S) {
return;
}
@@ -1476,9 +1476,9 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
log_info(maat_inst->logger, MODULE_REDIS_MONITOR, "Reconnecting...");
mr_ctx->read_ctx = maat_connect_redis(mr_ctx->redis_ip,
mr_ctx->redis_port,
mr_ctx->redis_db,
maat_inst->logger);
mr_ctx->redis_port,
mr_ctx->redis_db,
maat_inst->logger);
if (NULL == mr_ctx->read_ctx) {
return;
} else {
@@ -1491,11 +1491,11 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
int update_type = MAAT_UPDATE_TYPE_INC;
int rule_num = maat_get_rm_key_list(mr_ctx->read_ctx, version,
maat_inst->load_specific_version,
&new_version, maat_inst->tbl_mgr,
&rule_list, &update_type,
maat_inst->opts.cumulative_update_off,
maat_inst->logger);
maat_inst->load_specific_version,
&new_version, maat_inst->tbl_mgr,
&rule_list, &update_type,
maat_inst->opts.cumulative_update_off,
maat_inst->logger);
//redis communication error
if (rule_num < 0) {
redisFree(mr_ctx->read_ctx);
@@ -1538,7 +1538,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
maat_inst, maat_inst->opts.foreign_cont_dir);
if (ret > 0) {
maat_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0,
maat_inst->logger);
maat_inst->logger);
}
}
@@ -1562,10 +1562,10 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
if (rule_list[i].op == MAAT_OP_DEL) {
valid_column = table_manager_get_valid_column(maat_inst->tbl_mgr, table_id);
ret = invalidate_line(rule_list[i].table_line, valid_column);
ret = validate_line(rule_list[i].table_line, valid_column);
if (ret < 0) {
log_error(maat_inst->logger, MODULE_REDIS_MONITOR,
"[%s:%d] Invalidate line failed, invaid format %s",
"[%s:%d] Validate line failed, invaid format %s",
__FUNCTION__, __LINE__, rule_list[i].table_line);
continue;
}

27
src/maat_stat.c
View File

@@ -28,7 +28,7 @@ enum MAAT_FS_STATUS {
STATUS_TABLE_NUM,
STATUS_PLUGIN_CACHE_NUM,
STATUS_PLUGIN_ACC_NUM,
STATUS_GROUP_REF_NOT_NUM,
STATUS_CLAUSE_REF_NOT_NUM,
STATUS_GROUP_REF_EXCL_NUM, //group reference exclude group num
STATUS_MAAT_STATE_NUM,
STATUS_MAAT_PER_STATE_MEM,
@@ -38,7 +38,6 @@ enum MAAT_FS_STATUS {
STATUS_SCAN_ERR_CNT,
STATUS_STREAM_NUM,
STATUS_ZOMBIE_STREAM_NUM,
STATUS_NOT_GROUP_HIT,
STATUS_CMD_LINE_NUM
};
@@ -83,10 +82,8 @@ static void maat_fieldstat_register(struct maat_stat *stat)
"plug_cached", NULL, 0);
stat->fs_status_id[STATUS_PLUGIN_ACC_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"plug_acc", NULL, 0);
stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"not_grp", NULL, 0);
stat->fs_status_id[STATUS_NOT_GROUP_HIT] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"not_grp_hit", NULL, 0);
stat->fs_status_id[STATUS_CLAUSE_REF_NOT_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"not_clause", NULL, 0);
stat->fs_status_id[STATUS_GROUP_REF_EXCL_NUM] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
"excl_grp", NULL, 0);
stat->fs_status_id[STATUS_GARBAGE_QSIZE] = fieldstat_register(stat->fs_handle, FIELD_TYPE_GAUGE,
@@ -211,7 +208,6 @@ struct maat_stat *maat_stat_new(const char *stat_file, size_t max_thread_num,
stat->stream_num = alignment_int64_array_alloc(max_thread_num);
stat->thread_call_cnt = alignment_int64_array_alloc(max_thread_num);
stat->hit_cnt = alignment_int64_array_alloc(max_thread_num);
stat->not_grp_hit_cnt = alignment_int64_array_alloc(max_thread_num);
stat->maat_state_cnt = alignment_int64_array_alloc(max_thread_num);
stat->maat_compile_state_cnt = alignment_int64_array_alloc(max_thread_num);
stat->maat_state_free_cnt = alignment_int64_array_alloc(max_thread_num);
@@ -241,11 +237,6 @@ void maat_stat_free(struct maat_stat *stat)
stat->hit_cnt = NULL;
}
if (stat->not_grp_hit_cnt != NULL) {
alignment_int64_array_free(stat->not_grp_hit_cnt);
stat->not_grp_hit_cnt = NULL;
}
if (stat->maat_state_cnt != NULL) {
alignment_int64_array_free(stat->maat_state_cnt);
stat->maat_state_cnt = NULL;
@@ -330,7 +321,7 @@ static void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
long long total_rule_num = 0, total_input_bytes = 0;
long long total_scan_cnt = 0, total_scan_cpu_time = 0, total_regex_num = 0;
long long total_ipv6_num = 0, total_hit_cnt = 0, total_update_err = 0;
long long g2c_not_group_num = 0, g2g_excl_rule_num = 0;
long long g2c_not_clause_num = 0, g2g_excl_rule_num = 0;
size_t max_table_count = table_manager_table_size(stat->ref_tbl_mgr);
for (size_t i = 0; i < max_table_count; i++) {
@@ -352,7 +343,7 @@ static void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
plugin_rule_num += plugin_runtime_rule_count(runtime);
break;
case TABLE_TYPE_GROUP2COMPILE:
g2c_not_group_num += group2compile_runtime_not_group_count(runtime);
g2c_not_clause_num += group2compile_runtime_not_clause_count(runtime);
break;
case TABLE_TYPE_GROUP2GROUP:
g2g_excl_rule_num += group2group_runtime_exclude_rule_count(runtime);
@@ -441,8 +432,8 @@ static void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
plugin_cache_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_PLUGIN_ACC_NUM],
plugin_rule_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_NOT_NUM],
g2c_not_group_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CLAUSE_REF_NOT_NUM],
g2c_not_clause_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GROUP_REF_EXCL_NUM],
g2g_excl_rule_num);
}
@@ -465,8 +456,6 @@ void maat_stat_output(struct maat_stat *stat, long long maat_version, int perf_o
stat->nr_worker_thread);
long long maat_compile_state_cnt = alignment_int64_array_sum(stat->maat_compile_state_cnt,
stat->nr_worker_thread);
long long not_grp_hit_cnt = alignment_int64_array_sum(stat->not_grp_hit_cnt,
stat->nr_worker_thread);
size_t table_cnt = table_manager_table_count(stat->ref_tbl_mgr);
size_t garbage_q_len = maat_garbage_bin_get_size(stat->ref_garbage_bin);
@@ -487,8 +476,6 @@ void maat_stat_output(struct maat_stat *stat, long long maat_version, int perf_o
per_state_mem);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_COMPILE_STATE_NUM],
maat_compile_state_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_NOT_GROUP_HIT],
not_grp_hit_cnt);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_CMD_LINE_NUM],
stat->line_cmd_acc_num);
fieldstat_value_set(stat->fs_handle, stat->fs_status_id[STATUS_GARBAGE_QSIZE],

1
src/maat_virtual.c
View File

@@ -13,7 +13,6 @@
#include "maat_kv.h"
#include "maat_utils.h"
#include "log/log.h"
#include "maat_virtual.h"
#include "maat_rule.h"
#include "maat_table.h"