gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[FEATURE]plugin table support ip_addr key type

Browse Source
This commit is contained in:
liuwentan
2023-05-30 16:16:18 +08:00
parent 1c2aa3c3b7
commit 51e29f0b95
18 changed files with 751 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
include/maat.h
View File

@@ -147,9 +147,14 @@ int maat_plugin_table_ex_schema_register(struct maat *instance, const char *tabl
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp);
/* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register,
caller is responsible to free the data. */
void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id, const char *key);
/**
* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register,
* caller is responsible to free the data.
* NOTE: support three key type(integer, pointer, ip_addr) specified in table_info.conf
* if use ip_addr key type, then key should be ip address in network order.
*/
void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id,
const char *key, size_t key_len);
int maat_ip_plugin_table_get_ex_data(struct maat *instance, int table_id,
const struct ip_addr *ip, void **ex_data_array,

1
src/inc_internal/maat_limits.h
View File

@@ -17,7 +17,6 @@ extern "C"
#endif
#define MAX_KEYWORDS_STR 1024
#define MAX_MAAT_STAT_NUM 64
#ifdef __cplusplus

5
src/maat_api.c
View File

@@ -747,7 +747,7 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance,
}
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
const char *key)
const char *key, size_t key_len)
{
if (NULL == maat_instance || table_id < 0 || table_id >= MAX_TABLE_NUM
@@ -773,9 +773,10 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
void *ret = NULL;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (TABLE_TYPE_COMPILE == table_type) {
assert(key_len == sizeof(long long));
ret = compile_runtime_get_ex_data(runtime, schema, *(long long *)key);
} else if (TABLE_TYPE_PLUGIN == table_type) {
ret = plugin_runtime_get_ex_data(runtime, schema, key, strlen(key));
ret = plugin_runtime_get_ex_data(runtime, schema, key, key_len);
} else {
return NULL;
}

58
src/maat_bool_plugin.c
View File

@@ -56,7 +56,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> schema has no table_id column",
"[%s:%d] bool_plugin table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -64,7 +64,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> schema has no custom column",
"[%s:%d] bool_plugin table:<%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -74,7 +74,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->item_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> schema has no item_id column",
"[%s:%d] bool_plugin table:<%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -84,7 +84,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->bool_expr_column = custom_item->valueint;
} else {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> schema has no bool_expr column",
"[%s:%d] bool_plugin table:<%s> schema has no bool_expr column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -122,8 +122,8 @@ int bool_plugin_table_set_ex_container_schema(void *bool_plugin_schema, int tabl
if (1 == schema->container_schema.set_flag) {
log_error(schema->logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table(table_id:%d) ex_container_schema has been set, can't set again",
__FUNCTION__, __LINE__, table_id);
"[%s:%d] bool_plugin table(table_id:%d) ex_container_schema"
" has been set, can't set again", __FUNCTION__, __LINE__, table_id);
return -1;
}
@@ -139,7 +139,8 @@ int bool_plugin_table_set_ex_container_schema(void *bool_plugin_schema, int tabl
return 0;
}
struct ex_container_schema *bool_plugin_table_get_ex_container_schema(void *bool_plugin_schema)
struct ex_container_schema *
bool_plugin_table_get_ex_container_schema(void *bool_plugin_schema)
{
struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema;
@@ -244,7 +245,8 @@ int bool_plugin_runtime_update_row(struct bool_plugin_runtime *bool_plugin_rt,
}
} else {
// add
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, table_name, row, key, key_len);
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, table_name, row,
key, key_len);
struct ex_container *ex_container = ex_container_new(ex_data, (void *)expr);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
@@ -267,7 +269,7 @@ int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> has no rule_tag in line:%s",
"[%s:%d] bool_plugin table:<%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
@@ -279,14 +281,14 @@ int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> has invalid tag format in line:%s",
"[%s:%d] bool_plugin table:<%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
"[%s:%d] bool_plugin table:<%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
@@ -297,8 +299,8 @@ int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *
}
struct bool_expr *
bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
const char *table_name, struct log_handle *logger)
bool_plugin_expr_new(struct bool_plugin_schema *schema, const char *table_name,
const char *line, struct log_handle *logger)
{
int ret = bool_plugin_accept_tag_match(schema, table_name, line, logger);
if (ret == TAG_MATCH_UNMATCHED) {
@@ -316,7 +318,7 @@ bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> has no item_id in line:%s",
"[%s:%d] bool_plugin table:<%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -325,7 +327,7 @@ bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
ret = get_column_pos(line, schema->bool_expr_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> has no bool_expr in line:%s",
"[%s:%d] bool_plugin table:<%s> has no bool_expr in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -341,8 +343,8 @@ bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
n_item++;
if (ret != 1 || n_item > MAX_ITEMS_PER_BOOL_EXPR) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> has invalid format of bool_expr in line:%s",
__FUNCTION__, __LINE__, table_name, line);
"[%s:%d] bool_plugin table:<%s> has invalid format of "
"bool_expr in line:%s", __FUNCTION__, __LINE__, table_name, line);
goto error;
}
}
@@ -381,12 +383,20 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
bool_plugin_rt->update_err_cnt++;
return -1;
}
int ret = get_column_pos(line, schema->item_id_column, &item_id_offset, &item_id_len);
if (ret < 0) {
log_error(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table:<%s> has no item_id(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->item_id_column, line);
bool_plugin_rt->update_err_cnt++;
return -1;
}
@@ -394,7 +404,7 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche
if (1 == schema->container_schema.set_flag) {
if (1 == is_valid) {
// add
bool_expr = bool_plugin_expr_new(line, schema, table_name, bool_plugin_rt->logger);
bool_expr = bool_plugin_expr_new(schema, table_name, line, bool_plugin_rt->logger);
if (NULL == bool_expr) {
bool_plugin_rt->update_err_cnt++;
return -1;
@@ -468,8 +478,9 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
new_bool_matcher = bool_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_bool_matcher) {
log_error(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table[%s] rebuild bool_matcher engine failed when update %zu bool_plugin rules",
__FUNCTION__, __LINE__, table_name, rule_cnt);
"[%s:%d] table[%s] rebuild bool_matcher engine failed when "
"update %zu bool_plugin rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
}
}
@@ -487,8 +498,8 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
}
log_info(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"table[%s] commit %zu bool_plugin rules and rebuild bool_matcher completed, version:%lld",
table_name, rule_cnt, bool_plugin_rt->version);
"table[%s] commit %zu bool_plugin rules and rebuild bool_matcher completed"
", version:%lld", table_name, rule_cnt, bool_plugin_rt->version);
if (rules != NULL) {
FREE(rules);
@@ -559,7 +570,8 @@ void bool_plugin_runtime_perf_stat(void *bool_plugin_runtime, struct timespec *s
alignment_int64_array_add(bool_plugin_rt->scan_cnt, thread_id, 1);
if (start != NULL && end != NULL) {
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + (end->tv_nsec - start->tv_nsec);
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 +
(end->tv_nsec - start->tv_nsec);
alignment_int64_array_add(bool_plugin_rt->scan_cpu_time, thread_id, consume_time);
}
}

134
src/maat_compile.c
View File

@@ -746,64 +746,71 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
char vtable_name[NAME_MAX] = {0};
struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1);
int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len);
int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has no group_id in line:%s",
"[%s:%d] g2c table:<%s> has no group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
g2c_item->group_id = atoll(line + column_offset);
ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset, &column_len);
ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has no compile_id in line:%s",
"[%s:%d] g2c table:<%s> has no compile_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
g2c_item->compile_id = atoll(line + column_offset);
ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset, &column_len);
ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has no NOT_flag in line:%s ",
"[%s:%d] g2c table:<%s> has no NOT_flag in line:%s ",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
g2c_item->not_flag = atoi(line + column_offset);
ret = get_column_pos(line, g2c_schema->vtable_name_column, &column_offset, &column_len);
ret = get_column_pos(line, g2c_schema->vtable_name_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has no virtual_table_name in line:%s",
"[%s:%d] g2c table:<%s>has no virtual_table_name in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
if (column_len > NAME_MAX) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> virtual_table_name length too long in line:%s",
__FUNCTION__, __LINE__, table_name, line);
"[%s:%d] g2c table:<%s> virtual_table_name length exceed "
"maxium:%d in line:%s", __FUNCTION__, __LINE__, table_name,
NAME_MAX, line);
goto error;
}
memcpy(vtable_name, (line + column_offset), column_len);
if (is_valid_table_name(vtable_name)) {
g2c_item->vtable_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr, vtable_name);
g2c_item->vtable_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr,
vtable_name);
if (g2c_item->vtable_id < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has unknown virtual table:%s in line:%s",
"[%s:%d] g2c table:<%s> has unknown virtual table:%s in line:%s",
__FUNCTION__, __LINE__, table_name, vtable_name, line);
goto error;
}
}
ret = get_column_pos(line, g2c_schema->clause_index_column, &column_offset, &column_len);
ret = get_column_pos(line, g2c_schema->clause_index_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has no clause_index in line:%s",
"[%s:%d] g2c table:<%s> has no clause_index in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -913,7 +920,8 @@ maat_clause_hash_fetch_clause(struct compile_runtime *compile_rt,
return clause;
}
struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, size_t *compile_cnt)
struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compile_rt,
size_t *compile_cnt)
{
if (NULL == compile_rt) {
return NULL;
@@ -1248,26 +1256,22 @@ struct maat_compile *maat_compile_clone(struct maat_compile *compile, int deep_c
memcpy(new_compile->table_name, compile->table_name, sizeof(new_compile->table_name));
new_compile->not_clause_cnt = compile->not_clause_cnt;
new_compile->user_data_free = compile->user_data_free;
if (1 == deep_copy && compile->user_data != NULL)
{
if (1 == deep_copy && compile->user_data != NULL) {
new_compile->user_data = compile_rule_clone((struct compile_rule *)compile->user_data);
}
struct maat_literal_id *literal_id = NULL;
for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++)
{
for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
new_compile->clause_states[i].clause_id = compile->clause_states[i].clause_id;
new_compile->clause_states[i].in_use = compile->clause_states[i].in_use;
new_compile->clause_states[i].not_flag = compile->clause_states[i].not_flag;
utarray_new(new_compile->clause_states[i].ut_literal_ids, &ut_literal_id_icd);
for (int j = 0; j < utarray_len(compile->clause_states[i].ut_literal_ids); j++)
{
for (int j = 0; j < utarray_len(compile->clause_states[i].ut_literal_ids); j++) {
literal_id = (struct maat_literal_id *)utarray_eltptr(compile->clause_states[i].ut_literal_ids, j);
utarray_push_back(new_compile->clause_states[i].ut_literal_ids, literal_id);
}
for (int k = 0; k < utarray_len(new_compile->clause_states[i].ut_literal_ids); k++)
{
for (int k = 0; k < utarray_len(new_compile->clause_states[i].ut_literal_ids); k++) {
literal_id = (struct maat_literal_id *)utarray_eltptr(new_compile->clause_states[i].ut_literal_ids, k);
}
}
@@ -1288,23 +1292,25 @@ int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, struct group2comp
compile = rcu_updating_hash_find(hash_tbl, (char *)&compile_id, sizeof(long long));
if (compile != NULL) {
/* compile found in updating hash(added by compile runtime), it can be modified directly */
ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, g2c_item->not_flag);
ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index,
g2c_item->not_flag);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d of compile %d failed",
__FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index,
compile_id);
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d"
" of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id,
g2c_item->vtable_id, g2c_item->clause_index, compile_id);
}
} else {
/* compile neither in effective hash nor in updating hash, so new one */
compile = maat_compile_new(compile_id);
assert(compile != NULL);
ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, g2c_item->not_flag);
ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index,
g2c_item->not_flag);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d of compile %d failed",
__FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index,
compile_id);
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d"
" of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id,
g2c_item->vtable_id, g2c_item->clause_index, compile_id);
}
rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), compile);
}
@@ -1330,9 +1336,9 @@ int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, struct group2comp
g2c_item->not_flag);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d of compile %d failed",
__FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index,
compile_id);
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d"
" of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id,
g2c_item->vtable_id, g2c_item->clause_index, compile_id);
}
rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), copy_compile);
@@ -1343,9 +1349,9 @@ int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, struct group2comp
g2c_item->not_flag);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d of compile %d failed",
__FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index,
compile_id);
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d"
" of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id,
g2c_item->vtable_id, g2c_item->clause_index, compile_id);
}
rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), compile);
}
@@ -1365,20 +1371,24 @@ int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
int updating_flag = rcu_hash_is_updating(hash_tbl);
if (1 == updating_flag) {
compile = rcu_updating_hash_find(hash_tbl, (char *)&compile_id, sizeof(long long));
compile = rcu_updating_hash_find(hash_tbl, (char *)&compile_id,
sizeof(long long));
if (NULL == compile) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group %d from compile %d failed, compile is not exisited.",
__FUNCTION__, __LINE__, g2c_item->group_id, compile_id);
"[%s:%d] Remove group %d from compile %d failed, compile"
" is not exisited.", __FUNCTION__, __LINE__, g2c_item->group_id,
compile_id);
return -1;
} else {
/* compile found in updating hash, it can be modified directly */
ret = maat_compile_clause_remove_literal(compile, &literal_id, g2c_item->clause_index);
ret = maat_compile_clause_remove_literal(compile, &literal_id,
g2c_item->clause_index);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group %d vtable_id %d from clause %d of compile %d failed, literal is not in compile.",
__FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index,
compile_id);
"[%s:%d] Remove group %d vtable_id %d from clause %d of "
"compile %d failed, literal is not in compile.", __FUNCTION__,
__LINE__, g2c_item->group_id, g2c_item->vtable_id,
g2c_item->clause_index, compile_id);
}
if (0 == compile->actual_clause_num && NULL == compile->user_data) {
@@ -1404,11 +1414,13 @@ int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
/* delete compile from rcu hash */
rcu_hash_del(hash_tbl, (char *)&compile_id, sizeof(long long));
ret = maat_compile_clause_remove_literal(copy_compile, &literal_id, g2c_item->clause_index);
ret = maat_compile_clause_remove_literal(copy_compile, &literal_id,
g2c_item->clause_index);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group %d vtable_id %d from clause %d of compile %d failed, literal is not in compile.",
__FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index,
"[%s:%d] Remove group %d vtable_id %d from clause %d of compile"
" %d failed, literal is not in compile.", __FUNCTION__, __LINE__,
g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index,
compile_id);
}
@@ -1481,8 +1493,9 @@ void maat_compile_state_free(struct maat_compile_state *compile_state)
FREE(compile_state);
}
static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id, long long group_id,
int vtable_id, int Nth_scan, int Nth_item_result)
static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id,
long long group_id, int vtable_id,
int Nth_scan, int Nth_item_result)
{
struct maat_internal_hit_path new_path;
@@ -1833,12 +1846,20 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] compile table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
compile_rt->update_err_cnt++;
return -1;
}
long long compile_id = get_column_value(line, schema->compile_id_column);
if (compile_id < 0) {
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] compile table:<%s> has no compile_id(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->compile_id_column, line);
compile_rt->update_err_cnt++;
return -1;
}
@@ -1872,6 +1893,10 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
g2c_rt->update_err_cnt++;
return -1;
}
@@ -1984,7 +2009,8 @@ long long group2compile_runtime_update_err_count(void *g2c_runtime)
return g2c_rt->update_err_cnt;
}
int compile_runtime_commit(void *compile_runtime, const char *table_name, long long maat_rt_version)
int compile_runtime_commit(void *compile_runtime, const char *table_name,
long long maat_rt_version)
{
if (NULL == compile_runtime) {
return -1;
@@ -2005,13 +2031,13 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name, long l
new_bool_matcher = maat_compile_bool_matcher_new(compile_rt, &compile_cnt);
if (NULL == new_bool_matcher) {
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] table[%s] rebuild compile bool_matcher failed, compile rules count:%zu",
__FUNCTION__, __LINE__, table_name, compile_cnt);
"[%s:%d] table[%s] rebuild compile bool_matcher failed, compile"
" rules count:%zu", __FUNCTION__, __LINE__, table_name, compile_cnt);
ret = -1;
} else {
log_info(compile_rt->logger, MODULE_COMPILE,
"table[%s] commit %zu compile rules and rebuild compile bool_matcher completed, version:%lld",
table_name, compile_cnt, maat_rt_version);
"table[%s] commit %zu compile rules and rebuild compile bool_matcher"
" completed, version:%lld", table_name, compile_cnt, maat_rt_version);
}
struct literal_clause *old_literal2clause = NULL;

172
src/maat_expr.c
View File

@@ -171,8 +171,8 @@ int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *dis
return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id);
}
struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schema,
struct expr_runtime *expr_rt)
struct expr_item *expr_item_new(struct expr_schema *expr_schema, const char *table_name,
const char *line, struct expr_runtime *expr_rt)
{
size_t column_offset = 0;
size_t column_len = 0;
@@ -182,20 +182,22 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
enum table_type table_type = TABLE_TYPE_INVALID;
struct expr_item *expr_item = ALLOC(struct expr_item, 1);
int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len);
int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no item_id",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
expr_item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len);
ret = get_column_pos(line, expr_schema->group_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no group_id",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> has no group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
expr_item->group_id = atoll(line + column_offset);
@@ -203,15 +205,15 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no keywords",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> has no keywords in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
if (column_len >= MAX_KEYWORDS_STR) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s keywords length too long",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> keywords length too long in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
memcpy(expr_item->keywords, (line + column_offset), column_len);
@@ -219,8 +221,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no expr_type",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> has no expr_type in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -228,15 +230,16 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
expr_item->expr_type = int_to_expr_type(expr_type);
if (expr_item->expr_type == EXPR_TYPE_INVALID) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has invalid expr_type",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> has invalid expr_type in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
} else if (expr_item->expr_type == EXPR_TYPE_REGEX) {
ret = adapter_hs_verify_regex_expression(expr_item->keywords, expr_rt->logger);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) regex expression(item_id:%lld):%s illegal, will be dropped",
__FUNCTION__, __LINE__, expr_schema->table_id, expr_item->item_id, expr_item->keywords);
"[%s:%d] expr table:<%s> regex expression(item_id:%lld):%s illegal,"
" will be dropped", __FUNCTION__, __LINE__, table_name,
expr_item->item_id, expr_item->keywords);
goto error;
}
}
@@ -250,8 +253,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
if (column_len >= MAX_DISTRICT_STR) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s district length too long",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> district length exceed maxium:%d in line:%s",
__FUNCTION__, __LINE__, table_name, MAX_DISTRICT_STR, line);
goto error;
}
@@ -267,8 +270,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no match_method",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> has no match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -276,16 +279,16 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
expr_item->match_mode = int_to_match_mode(match_method_type);
if (expr_item->match_mode == HS_MATCH_MODE_INVALID) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has invalid match_method",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> has invalid match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has no is_hexbin",
__FUNCTION__, __LINE__, expr_schema->table_id, line);
"[%s:%d] expr table:<%s> has no is_hexbin in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
db_hexbin = atoi(line + column_offset);
@@ -305,8 +308,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
break;
default:
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table(table_id:%d) line:%s has invalid hexbin value:%d",
__FUNCTION__, __LINE__, expr_schema->table_id, line, db_hexbin);
"[%s:%d] expr table:<%s> has invalid hexbin value:%d in line:%s",
__FUNCTION__, __LINE__, table_name, db_hexbin, line);
goto error;
}
@@ -328,7 +331,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
expr_schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no table_id column",
"[%s:%d] expr table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -340,7 +343,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no custom column",
"[%s:%d] expr table:<%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -350,7 +353,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
expr_schema->item_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no item_id column",
"[%s:%d] expr table:<%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -360,7 +363,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
expr_schema->group_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no group_id column",
"[%s:%d] expr table:<%s> schema has no group_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -370,7 +373,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
expr_schema->keywords_column = custom_item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no keywords column",
"[%s:%d] expr table:<%s> schema has no keywords column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -382,7 +385,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
expr_schema->district_column = custom_item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] expr_plus table %s has no district column",
"[%s:%d] expr_plus table:<%s> schema has no district column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -393,7 +396,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
expr_schema->expr_type_column = custom_item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no expr_type column",
"[%s:%d] expr table:<%s> schema has no expr_type column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -403,7 +406,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
expr_schema->match_method_column = custom_item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no match_method column",
"[%s:%d] expr table:<%s> schema has no match_method column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -413,7 +416,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
expr_schema->is_hexbin_column = custom_item->valueint;
} else {
log_error(logger, MODULE_EXPR,
"[%s:%d] table %s has no is_hexbin column",
"[%s:%d] expr table:<%s> schema has no is_hexbin column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -636,8 +639,9 @@ int expr_item_to_expr_rule(struct expr_item *expr_item, struct expr_rule *expr_r
if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%d) too many patterns",
__FUNCTION__, __LINE__, expr_item->item_id);
"[%s:%d]abandon config expr_item(item_id:%d) "
"too many patterns", __FUNCTION__, __LINE__,
expr_item->item_id);
return -1;
}
@@ -655,25 +659,30 @@ int expr_item_to_expr_rule(struct expr_item *expr_item, struct expr_rule *expr_r
if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%d) too many patterns",
__FUNCTION__, __LINE__, expr_item->item_id);
"[%s:%d]abandon config expr_item(item_id:%d) "
"too many patterns", __FUNCTION__, __LINE__,
expr_item->item_id);
return -1;
}
sub_key_array[i] = tmp;
sscanf(sub_key_array[i], "%d-%d:", &(key_left_offset[i]), &(key_right_offset[i]));
sscanf(sub_key_array[i], "%d-%d:", &(key_left_offset[i]),
&(key_right_offset[i]));
if (!(key_left_offset[i] >= 0 && key_right_offset[i] > 0
&& key_left_offset[i] <= key_right_offset[i])) {
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%d) has invalid offset.",
__FUNCTION__, __LINE__, expr_item->item_id);
"[%s:%d]abandon config expr_item(item_id:%d) "
"has invalid offset.", __FUNCTION__, __LINE__,
expr_item->item_id);
return -1;
}
sub_key_array[i] = (char *)memchr(sub_key_array[i], ':', strlen(sub_key_array[i]));
sub_key_array[i] = (char *)memchr(sub_key_array[i], ':',
strlen(sub_key_array[i]));
if (NULL == sub_key_array[i]) {
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%d) has invalid offset keyword format.",
"[%s:%d]abandon config expr_item(item_id:%d) "
"has invalid offset keyword format.",
__FUNCTION__, __LINE__, expr_item->item_id);
return -1;
}
@@ -694,8 +703,9 @@ int expr_item_to_expr_rule(struct expr_item *expr_item, struct expr_rule *expr_r
break;
default:
log_error(logger, MODULE_EXPR,
"[%s:%d]abandon config expr_item(item_id:%lld) has invalid expr type=%d",
__FUNCTION__, __LINE__, expr_item->item_id, expr_item->expr_type);
"[%s:%d]abandon config expr_item(item_id:%lld) has "
"invalid expr type=%d", __FUNCTION__, __LINE__,
expr_item->item_id, expr_item->expr_type);
return -1;
}
@@ -713,10 +723,12 @@ int expr_item_to_expr_rule(struct expr_item *expr_item, struct expr_rule *expr_r
expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type);
if (TRUE == expr_item->is_hexbin && expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) {
if (TRUE == expr_item->is_hexbin &&
expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) {
region_str_len = strlen(sub_key_array[i]) * 8 + 1;
region_string = ALLOC(char, region_str_len);
region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]), region_string, region_str_len);
region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]),
region_string, region_str_len);
}
if (region_string != NULL) {
@@ -758,12 +770,20 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> has no item_id(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->item_id_column, line);
expr_rt->update_err_cnt++;
return -1;
}
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
expr_rt->update_err_cnt++;
return -1;
}
@@ -771,7 +791,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
struct expr_item *expr_item = NULL;
if (1 == is_valid) {
//add
expr_item = expr_item_new(line, schema, expr_rt);
expr_item = expr_item_new(schema, table_name, line, expr_rt);
if (NULL == expr_item) {
expr_rt->update_err_cnt++;
return -1;
@@ -803,7 +823,8 @@ void garbage_adapter_hs_free(void *adapter_hs, void *arg)
adapter_hs_free(hs);
}
int expr_runtime_commit(void *expr_runtime, const char *table_name, long long maat_rt_version)
int expr_runtime_commit(void *expr_runtime, const char *table_name,
long long maat_rt_version)
{
if (NULL == expr_runtime) {
return -1;
@@ -850,11 +871,12 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
struct adapter_hs *old_adapter_hs = NULL;
if (rule_cnt > 0) {
new_adapter_hs = adapter_hs_new(expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger);
new_adapter_hs = adapter_hs_new(expr_rt->n_worker_thread, rules, rule_cnt,
expr_rt->logger);
if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild adapter_hs engine failed when update %zu expr rules",
__FUNCTION__, __LINE__, table_name, rule_cnt);
"[%s:%d] table[%s] rebuild adapter_hs engine failed when update"
" %zu expr rules", __FUNCTION__, __LINE__, table_name, rule_cnt);
ret = -1;
}
}
@@ -864,15 +886,17 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
rcu_hash_commit(expr_rt->item_hash);
if (old_adapter_hs != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL, garbage_adapter_hs_free);
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL,
garbage_adapter_hs_free);
}
expr_rt->rule_num = rule_cnt;
expr_rt->version = maat_rt_version;
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules and rebuild adapter_hs completed, version:%lld",
table_name, rule_cnt, real_rule_cnt, expr_rt->version);
"table[%s] has %zu rules, commit %zu expr rules and rebuild adapter_hs"
" completed, version:%lld", table_name, rule_cnt, real_rule_cnt,
expr_rt->version);
if (rules != NULL) {
for (i = 0; i < rule_cnt; i++) {
@@ -918,8 +942,9 @@ long long expr_runtime_get_version(void *expr_runtime)
return expr_rt->version;
}
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int vtable_id, struct maat_state *state)
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len, int vtable_id,
struct maat_state *state)
{
if (0 == expr_rt->rule_num) {
//empty expr table
@@ -949,14 +974,16 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *inner_item = NULL;
struct expr_item *expr_item = NULL;
size_t real_hit_item_cnt = 0;
long long district_id = state->district_id;
for (size_t i = 0; i < n_hit_item; i++) {
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) {
if (inner_item->district_id == district_id ||
inner_item->district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
struct expr_item *expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
(char *)&item_id,
sizeof(long long));
if (!expr_item) {
@@ -981,7 +1008,8 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
return real_hit_item_cnt;
}
struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
struct adapter_hs_stream *
expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
{
if (NULL == expr_rt || thread_id < 0) {
return NULL;
@@ -992,8 +1020,10 @@ struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt,
return adapter_hs_stream_open(expr_rt->hs, thread_id);
}
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle,
const char *data, size_t data_len, int vtable_id, struct maat_state *state)
int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
struct adapter_hs_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state)
{
if (0 == expr_rt->rule_num) {
//empty expr table
@@ -1003,7 +1033,8 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_str
size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results,
MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
if (ret < 0) {
return -1;
}
@@ -1017,11 +1048,12 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_str
}
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct expr_item *expr_item = NULL;
size_t real_hit_item_cnt = 0;
for (size_t i = 0; i < n_hit_item; i++) {
long long item_id = hit_results[i].rule_id;
struct expr_item *expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
expr_item = (struct expr_item *)rcu_hash_find(expr_rt->item_hash,
(char *)&item_id,
sizeof(long long));
if (!expr_item) {
@@ -1067,7 +1099,8 @@ void expr_runtime_perf_stat(struct expr_runtime *expr_rt, size_t scan_len,
alignment_int64_array_add(expr_rt->scan_bytes, thread_id, scan_len);
if (start != NULL && end != NULL) {
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + (end->tv_nsec - start->tv_nsec);
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 +
(end->tv_nsec - start->tv_nsec);
alignment_int64_array_add(expr_rt->scan_cpu_time, thread_id, consume_time);
}
}
@@ -1143,7 +1176,8 @@ long long expr_runtime_stream_num(struct expr_runtime *expr_rt)
return 0;
}
long long sum = alignment_int64_array_sum(expr_rt->stream_num, expr_rt->n_worker_thread);
long long sum = alignment_int64_array_sum(expr_rt->stream_num,
expr_rt->n_worker_thread);
alignment_int64_array_reset(expr_rt->stream_num, expr_rt->n_worker_thread);
return sum;

73
src/maat_flag.c
View File

@@ -76,7 +76,7 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_FLAG,
"[%s:%d] table %s has no table_id column",
"[%s:%d] flag table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -88,7 +88,7 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_FLAG,
"[%s:%d] table %s has no custom column",
"[%s:%d] flag table:<%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -98,7 +98,7 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->item_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FLAG,
"[%s:%d] table %s has no item_id column",
"[%s:%d] flag table:<%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -108,7 +108,7 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->group_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FLAG,
"[%s:%d] table %s has no group_id column",
"[%s:%d] flag table:<%s> schema has no group_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -120,7 +120,7 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->district_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FLAG,
"[%s:%d] flag_plus table %s has no district column",
"[%s:%d] flag_plus table:<%s> schema has no district column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -131,7 +131,7 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->flag_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FLAG,
"[%s:%d] table %s has no flag column",
"[%s:%d] flag table:<%s> schema has no flag column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -141,7 +141,7 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->flag_mask_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FLAG,
"[%s:%d] table %s has no flag_mask column",
"[%s:%d] flag table:<%s> schema has no flag_mask column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -297,43 +297,47 @@ int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *dis
return maat_kv_read_unNull(flag_rt->district_map, district, district_len, district_id);
}
struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
struct flag_runtime *flag_rt)
struct flag_item *flag_item_new(struct flag_schema *schema, const char *table_name,
const char *line, struct flag_runtime *flag_rt)
{
size_t column_offset = 0;
size_t column_len = 0;
enum table_type table_type = TABLE_TYPE_INVALID;
struct flag_item *item = ALLOC(struct flag_item, 1);
int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
int ret = get_column_pos(line, schema->item_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag table(table_id:%d) line:%s has no item_id",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] flag table:<%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->group_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag table(table_id:%d) line:%s has no group_id",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] flag table:<%s> has no group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
item->group_id = atoll(line + column_offset);
table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id);
if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
ret = get_column_pos(line, schema->district_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->district_column, &column_offset,
&column_len);
if (ret < 0) {
goto error;
}
if (column_len >= MAX_DISTRICT_STR) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag_plus table(table_id:%d) line:%s district length too long",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] flag_plus table:<%s> district length exceed "
"maxium:%d in line:%s", __FUNCTION__, __LINE__, table_name,
MAX_DISTRICT_STR, line);
goto error;
}
@@ -349,8 +353,8 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
ret = get_column_pos(line, schema->flag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag table(table_id:%d) line:%s has no flag",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] flag table:<%s> has no flag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -359,8 +363,8 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
ret = get_column_pos(line, schema->flag_mask_column, &column_offset, &column_len);
if (ret < 0) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag table(table_id:%d) line:%s has no flag_mask",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] flag table:<%s> has no flag_mask in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
item->flag_mask = strtoull(line + column_offset, NULL, 0);
@@ -396,18 +400,26 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table
long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag table:<%s> has no item_id(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->item_id_column, line);
return -1;
}
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
flag_rt->update_err_cnt++;
return -1;
}
struct flag_item *flag_item = NULL;
if (1 == is_valid) {
flag_item = flag_item_new(line, schema, flag_rt);
flag_item = flag_item_new(schema, table_name, line, flag_rt);
if (NULL == flag_item) {
flag_rt->update_err_cnt++;
return -1;
@@ -481,8 +493,9 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
new_flag_matcher = flag_matcher_new(rules, rule_cnt);
if (NULL == new_flag_matcher) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] table[%s] rebuild flag_matcher engine failed when update %zu flag rules",
__FUNCTION__, __LINE__, table_name, rule_cnt);
"[%s:%d] table[%s] rebuild flag_matcher engine failed "
"when update %zu flag rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
}
}
@@ -500,8 +513,8 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
flag_rt->version = maat_rt_version;
log_info(flag_rt->logger, MODULE_FLAG,
"table[%s] commit %zu flag rules and rebuild flag_matcher completed, version:%lld",
table_name, rule_cnt, flag_rt->version);
"table[%s] commit %zu flag rules and rebuild flag_matcher completed,"
" version:%lld", table_name, rule_cnt, flag_rt->version);
if (rules != NULL) {
FREE(rules);
@@ -546,14 +559,16 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *inner_item = NULL;
struct flag_item *flag_item = NULL;
size_t real_hit_item_cnt = 0;
long long district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) {
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) {
if (inner_item->district_id == district_id ||
inner_item->district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
struct flag_item *flag_item = (struct flag_item *)rcu_hash_find(flag_rt->item_hash,
flag_item = (struct flag_item *)rcu_hash_find(flag_rt->item_hash,
(char *)&item_id,
sizeof(long long));
if (!flag_item) {

58
src/maat_fqdn_plugin.c
View File

@@ -56,7 +56,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> schema has no table_id column",
"[%s:%d] fqdn_plugin table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -64,7 +64,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> schema has no custom column",
"[%s:%d] fqdn_plugin table:<%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -74,7 +74,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->item_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> schema has no item_id column",
"[%s:%d] fqdn_plugin table:<%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -84,7 +84,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->suffix_flag_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> schema has no suffix_match_method column",
"[%s:%d] fqdn_plugin table:<%s> schema has no suffix_match_method column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -95,7 +95,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->fqdn_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> schema has no fqdn column",
"[%s:%d] fqdn_plugin table:<%s> schema has no fqdn column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -133,8 +133,8 @@ int fqdn_plugin_table_set_ex_container_schema(void *fqdn_plugin_schema, int tabl
if (1 == schema->container_schema.set_flag) {
log_error(schema->logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table(table_id:%d) ex_container_schema has been set, can't set again",
__FUNCTION__, __LINE__, table_id);
"[%s:%d] fqdn_plugin table(table_id:%d) ex_container_schema "
"has been set, can't set again", __FUNCTION__, __LINE__, table_id);
return -1;
}
@@ -150,7 +150,8 @@ int fqdn_plugin_table_set_ex_container_schema(void *fqdn_plugin_schema, int tabl
return 0;
}
struct ex_container_schema *fqdn_plugin_table_get_ex_container_schema(void *fqdn_plugin_schema)
struct ex_container_schema *
fqdn_plugin_table_get_ex_container_schema(void *fqdn_plugin_schema)
{
struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema;
@@ -184,7 +185,8 @@ void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, size_t max_thread_num,
fqdn_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger);
if (1 == schema->container_schema.set_flag) {
ex_data_runtime_set_ex_container_schema(fqdn_plugin_rt->ex_data_rt, &(schema->container_schema));
ex_data_runtime_set_ex_container_schema(fqdn_plugin_rt->ex_data_rt,
&(schema->container_schema));
}
fqdn_plugin_rt->n_worker_thread = max_thread_num;
@@ -238,7 +240,7 @@ int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> has no rule_tag in line:%s",
"[%s:%d] fqdn_plugin table:<%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
@@ -250,14 +252,14 @@ int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> has invalid tag format in line:%s",
"[%s:%d] fqdn_plugin table:<%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
"[%s:%d] fqdn_plugin table:<%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
@@ -285,7 +287,7 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> has no item_id in line:%s",
"[%s:%d] fqdn_plugin table:<%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -294,7 +296,7 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
ret = get_column_pos(line, schema->suffix_flag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> has no suffix_match_method in line:%s",
"[%s:%d] fqdn_plugin table:<%s> has no suffix_match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -303,7 +305,7 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
ret = get_column_pos(line, schema->fqdn_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> has no fqdn in line:%s",
"[%s:%d] fqdn_plugin table:<%s> has no fqdn in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -360,7 +362,8 @@ int fqdn_plugin_runtime_update_row(struct fqdn_plugin_runtime *fqdn_plugin_rt,
}
} else {
// add
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, table_name, row, key, key_len);
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, table_name, row,
key, key_len);
struct ex_container *ex_container = ex_container_new(ex_data, (void *)fqdn_plugin_rule);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container);
if (ret < 0) {
@@ -386,12 +389,20 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
fqdn_plugin_rt->update_err_cnt++;
return -1;
}
int ret = get_column_pos(line, schema->item_id_column, &item_id_offset, &item_id_len);
if (ret < 0) {
log_error(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:<%s> has no item_id(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->item_id_column, line);
fqdn_plugin_rt->update_err_cnt++;
return -1;
}
@@ -473,8 +484,9 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
new_fqdn_engine = FQDN_engine_new(rules, rule_cnt);
if (NULL == new_fqdn_engine) {
log_error(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table[%s] rebuild FQDN engine failed when update %zu fqdn_plugin rules",
__FUNCTION__, __LINE__, table_name, rule_cnt);
"[%s:%d] table[%s] rebuild FQDN engine failed when update"
" %zu fqdn_plugin rules", __FUNCTION__, __LINE__, table_name,
rule_cnt);
ret = -1;
}
}
@@ -492,8 +504,8 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
}
log_info(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"table[%s] commit %zu fqdn_plugin rules and rebuild FQDN engine completed, version:%lld",
table_name, rule_cnt, fqdn_plugin_rt->version);
"table[%s] commit %zu fqdn_plugin rules and rebuild FQDN engine completed"
", version:%lld", table_name, rule_cnt, fqdn_plugin_rt->version);
if (rules != NULL) {
FREE(rules);
@@ -544,7 +556,8 @@ int fqdn_plugin_runtime_get_ex_data(void *fqdn_plugin_runtime, const char *query
struct FQDN_match results[n_ex_data];
assert(fqdn_plugin_rt->engine != NULL);
int n_result = FQDN_engine_search(fqdn_plugin_rt->engine, query_fqdn, strlen(query_fqdn), results, n_ex_data);
int n_result = FQDN_engine_search(fqdn_plugin_rt->engine, query_fqdn, strlen(query_fqdn),
results, n_ex_data);
for (int i = 0; i < n_result; i++) {
ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(fqdn_plugin_rt->ex_data_rt,
(struct ex_container *)results[i].user_tag);
@@ -564,7 +577,8 @@ void fqdn_plugin_runtime_perf_stat(void *fqdn_plugin_runtime, struct timespec *s
alignment_int64_array_add(fqdn_plugin_rt->scan_cnt, thread_id, 1);
if (start != NULL && end != NULL) {
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + (end->tv_nsec - start->tv_nsec);
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 +
(end->tv_nsec - start->tv_nsec);
alignment_int64_array_add(fqdn_plugin_rt->scan_cpu_time, thread_id, consume_time);
}
}

25
src/maat_group.c
View File

@@ -97,14 +97,15 @@ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2g_schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_GROUP,
"[%s:%d] table %s has no table_id column", table_name);
"[%s:%d] g2g table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_GROUP,
"[%s:%d] table %s has no custom column",
"[%s:%d] g2g table:<%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -114,7 +115,7 @@ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2g_schema->group_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_GROUP,
"[%s:%d] table %s has no group_id column",
"[%s:%d] g2g table:<%s> schema has no group_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -124,7 +125,7 @@ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2g_schema->super_group_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_GROUP,
"[%s:%d] table %s has no super_group_id column",
"[%s:%d] g2g table:<%s> schema has no super_group_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -134,7 +135,7 @@ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2g_schema->is_exclude_column = custom_item->valueint;
} else {
log_error(logger, MODULE_GROUP,
"[%s:%d] table %s has no is_exclude column",
"[%s:%d] g2g table:<%s> schema has no is_exclude column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -327,7 +328,7 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_GROUP,
"[%s:%d] group2group table:%s line:%s has no group_id",
"[%s:%d] g2g table:<%s> has no group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -337,7 +338,7 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_GROUP,
"[%s:%d] group2group table:%s line:%s has no super_group_id",
"[%s:%d] g2 table:<%s> has no super_group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -347,7 +348,7 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_GROUP,
"[%s:%d] group2group table:%s line:%s has no is_exclude",
"[%s:%d] g2g table:<%s> has no is_exclude in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -684,6 +685,10 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(g2g_rt->logger, MODULE_GROUP,
"[%s:%d] g2g table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
g2g_rt->update_err_cnt++;
return -1;
}
@@ -765,8 +770,8 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name,
g2g_rt->version = maat_rt_version;
log_info(g2g_rt->logger, MODULE_GROUP,
"table[%s] commit %zu g2g rules and rebuild super_groups completed, version:%lld",
table_name, g2g_rt->rule_num, g2g_rt->version);
"table[%s] commit %zu g2g rules and rebuild super_groups completed,"
" version:%lld", table_name, g2g_rt->rule_num, g2g_rt->version);
return 0;
}

112
src/maat_interval.c
View File

@@ -73,7 +73,7 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_INTERVAL,
"[%s:%d] table %s has no table_id column",
"[%s:%d] interval table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -85,7 +85,7 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_INTERVAL,
"[%s:%d] table %s has no custom column",
"[%s:%d] interval table:<%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -95,7 +95,7 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->item_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_INTERVAL,
"[%s:%d] table %s has no item_id column",
"[%s:%d] interval table:<%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -105,7 +105,7 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->group_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_INTERVAL,
"[%s:%d] table %s has no group_id column",
"[%s:%d] interval table:<%s> schema has no group_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -117,7 +117,7 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->district_column = custom_item->valueint;
} else {
log_error(logger, MODULE_INTERVAL,
"[%s:%d] interval_plus table %s has no district column",
"[%s:%d] interval_plus table:<%s> schema has no district column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -128,7 +128,7 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->low_bound_column = custom_item->valueint;
} else {
log_error(logger, MODULE_INTERVAL,
"[%s:%d] table %s has no low_bound column",
"[%s:%d] interval table:<%s> schema has no low_bound column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -138,7 +138,7 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->up_bound_column = custom_item->valueint;
} else {
log_error(logger, MODULE_INTERVAL,
"[%s:%d] table %s has no up_bound column",
"[%s:%d] interval table:<%s> schema has no up_bound column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -240,7 +240,8 @@ void interval_runtime_free(void *interval_runtime)
FREE(interval_rt);
}
long long interval_runtime_get_district_id(struct interval_runtime *interval_rt, const char *district)
long long interval_runtime_get_district_id(struct interval_runtime *interval_rt,
const char *district)
{
long long district_id = DISTRICT_ANY;
@@ -261,53 +262,60 @@ long long interval_runtime_get_district_id(struct interval_runtime *interval_rt,
return district_id;
}
int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, const char *district,
size_t district_len, long long *district_id)
int interval_runtime_set_scan_district(struct interval_runtime *interval_rt,
const char *district, size_t district_len,
long long *district_id)
{
if (NULL == interval_rt || NULL == district || 0 == district_len) {
return -1;
}
return maat_kv_read_unNull(interval_rt->district_map, district, district_len, district_id);
return maat_kv_read_unNull(interval_rt->district_map, district, district_len,
district_id);
}
struct interval_item *interval_item_new(const char *line, struct interval_schema *schema,
struct interval_runtime *interval_rt)
struct interval_item *
interval_item_new(struct interval_schema *schema, const char *table_name,
const char *line, struct interval_runtime *interval_rt)
{
size_t column_offset = 0;
size_t column_len = 0;
enum table_type table_type = TABLE_TYPE_INVALID;
struct interval_item *item = ALLOC(struct interval_item, 1);
int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
int ret = get_column_pos(line, schema->item_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval table(table_id:%d) line:%s has no item_id",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] interval table:<%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->group_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval table(table_id:%d) line:%s has no group_id",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] interval table:<%s> has no group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
item->group_id = atoll(line + column_offset);
table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id);
if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
ret = get_column_pos(line, schema->district_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->district_column, &column_offset,
&column_len);
if (ret < 0) {
goto error;
}
if (column_len >= MAX_DISTRICT_STR) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval_plus table(table_id:%d) line:%s district length too long",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] interval_plus table:<%s> district length exceed "
"maxium:%d in line:%s", __FUNCTION__, __LINE__, table_name,
MAX_DISTRICT_STR, line);
goto error;
}
@@ -323,8 +331,8 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
ret = get_column_pos(line, schema->low_bound_column, &column_offset, &column_len);
if (ret < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval table(table_id:%d) line:%s has no low_bound",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] interval table:<%s> has no low_bound in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
item->low_bound = atoi(line + column_offset);
@@ -332,8 +340,8 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
ret = get_column_pos(line, schema->up_bound_column, &column_offset, &column_len);
if (ret < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval table(table_id:%d) line:%s has no up_bound",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] interval table:<%s> has no up_bound in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
item->up_bound = atoi(line + column_offset);
@@ -356,8 +364,9 @@ struct interval_rule interval_item_to_interval_rule(struct interval_item *item)
return rule;
}
int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, size_t key_len,
struct interval_item *item, int is_valid)
int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key,
size_t key_len, struct interval_item *item,
int is_valid)
{
int ret = -1;
@@ -369,8 +378,9 @@ int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key,
ret = rcu_hash_add(interval_rt->item_hash, key, key_len, (void *)item);
if (ret < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval item(item_id:%lld) add to interavl_item_hash failed",
__FUNCTION__, __LINE__, item->item_id);
"[%s:%d] interval item(item_id:%lld) add to "
"interavl_item_hash failed", __FUNCTION__, __LINE__,
item->item_id);
return -1;
}
}
@@ -392,12 +402,20 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval table:<%s> has no item_id(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->item_id_column, line);
interval_rt->update_err_cnt++;
return -1;
}
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] interval table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
interval_rt->update_err_cnt++;
return -1;
}
@@ -405,7 +423,7 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
struct interval_item *interval_item = NULL;
if (1 == is_valid) {
//add
interval_item = interval_item_new(line, schema, interval_rt);
interval_item = interval_item_new(schema, table_name, line, interval_rt);
if (NULL == interval_item) {
interval_rt->update_err_cnt++;
return -1;
@@ -437,7 +455,8 @@ void garbage_interval_matcher_free(void *interval_matcher, void *arg)
interval_matcher_free(matcher);
}
int interval_runtime_commit(void *interval_runtime, const char *table_name, long long maat_rt_version)
int interval_runtime_commit(void *interval_runtime, const char *table_name,
long long maat_rt_version)
{
if (NULL == interval_runtime) {
return -1;
@@ -478,8 +497,9 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name, long
new_interval_matcher = interval_matcher_new(rules, rule_cnt);
if (NULL == new_interval_matcher) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] table[%s]rebuild interval_matcher engine failed when update %zu interval rules",
__FUNCTION__, __LINE__, table_name, rule_cnt);
"[%s:%d] table[%s]rebuild interval_matcher engine failed "
"when update %zu interval rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
}
}
@@ -497,8 +517,8 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name, long
interval_rt->version = maat_rt_version;
log_info(interval_rt->logger, MODULE_INTERVAL,
"table[%s] commit %zu interval rules and rebuild interval_matcher completed, version:%lld",
table_name, rule_cnt, interval_rt->version);
"table[%s] commit %zu interval rules and rebuild interval_matcher "
"completed, version:%lld", table_name, rule_cnt, interval_rt->version);
if (rules != NULL) {
FREE(rules);
@@ -543,14 +563,16 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
struct maat_item hit_maat_items[MAX_SCANNER_HIT_ITEM_NUM];
struct maat_item_inner *inner_item = NULL;
struct interval_item *int_item = NULL;
size_t real_hit_item_cnt = 0;
long long district_id = state->district_id;
for (int i = 0; i < n_hit_item; i++) {
inner_item = (struct maat_item_inner *)(hit_results[i].user_tag);
if (inner_item->district_id == district_id || inner_item->district_id == DISTRICT_ANY) {
if (inner_item->district_id == district_id ||
inner_item->district_id == DISTRICT_ANY) {
long long item_id = hit_results[i].rule_id;
struct interval_item *int_item = (struct interval_item *)rcu_hash_find(interval_rt->item_hash,
int_item = (struct interval_item *)rcu_hash_find(interval_rt->item_hash,
(char *)&item_id,
sizeof(long long));
if (!int_item) {
@@ -578,8 +600,9 @@ void interval_runtime_hit_inc(struct interval_runtime *interval_rt, int thread_i
alignment_int64_array_add(interval_rt->hit_cnt, thread_id, 1);
}
void interval_runtime_perf_stat(struct interval_runtime *interval_rt, struct timespec *start,
struct timespec *end, int thread_id)
void interval_runtime_perf_stat(struct interval_runtime *interval_rt,
struct timespec *start, struct timespec *end,
int thread_id)
{
if (NULL == interval_rt || thread_id < 0) {
return;
@@ -588,7 +611,8 @@ void interval_runtime_perf_stat(struct interval_runtime *interval_rt, struct tim
alignment_int64_array_add(interval_rt->scan_cnt, thread_id, 1);
if (start != NULL && end != NULL) {
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + end->tv_nsec - start->tv_nsec;
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 +
(end->tv_nsec - start->tv_nsec);
alignment_int64_array_add(interval_rt->scan_cpu_time, thread_id, consume_time);
}
}
@@ -600,7 +624,8 @@ long long interval_runtime_scan_count(void *interval_runtime)
}
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
long long sum = alignment_int64_array_sum(interval_rt->scan_cnt, interval_rt->n_worker_thread);
long long sum = alignment_int64_array_sum(interval_rt->scan_cnt,
interval_rt->n_worker_thread);
alignment_int64_array_reset(interval_rt->scan_cnt, interval_rt->n_worker_thread);
return sum;
@@ -627,7 +652,8 @@ long long interval_runtime_hit_count(void *interval_runtime)
}
struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
long long sum = alignment_int64_array_sum(interval_rt->hit_cnt, interval_rt->n_worker_thread);
long long sum = alignment_int64_array_sum(interval_rt->hit_cnt,
interval_rt->n_worker_thread);
alignment_int64_array_reset(interval_rt->hit_cnt, interval_rt->n_worker_thread);
return sum;

164
src/maat_ip.c
View File

@@ -93,12 +93,17 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
ip_schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] ip table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table %s has no custom column",
"[%s:%d] ip table:<%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -108,7 +113,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->item_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no item_id column",
"[%s:%d] ip table:<%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -118,7 +123,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->group_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no group_id column",
"[%s:%d] ip table:<%s> schema has no group_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -128,7 +133,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->addr_type_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no add_type column",
"[%s:%d] ip table:<%s> schema has no add_type column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -138,7 +143,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->addr_format_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no addr_format column",
"[%s:%d] ip table:<%s> schema has no addr_format column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -148,7 +153,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->ip1_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no ip1 column",
"[%s:%d] ip table:<%s> schema has no ip1 column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -158,7 +163,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->ip2_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no ip2 column",
"[%s:%d] ip table:<%s> schema has no ip2 column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -168,7 +173,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->port_format_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no port_format column",
"[%s:%d] ip table:<%s> schema has no port_format column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -178,7 +183,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->port1_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no port1 column",
"[%s:%d] ip table:<%s> schema has no port1 column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -188,7 +193,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->port2_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no port2 column",
"[%s:%d] ip table:<%s> schema has no port2 column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -198,7 +203,7 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
ip_schema->protocol_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP,
"[%s:%d] table %s has no protocol column",
"[%s:%d] ip table:<%s> schema has no protocol column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -215,8 +220,8 @@ void ip_schema_free(void *ip_schema)
FREE(ip_schema);
}
struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema,
struct log_handle *logger)
struct ip_item *ip_item_new(struct ip_schema *ip_schema, const char *table_name,
const char *line, struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
@@ -226,134 +231,146 @@ struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema,
char ip2_str[40] = {0};
struct ip_item *ip_item = ALLOC(struct ip_item, 1);
int ret = get_column_pos(line, ip_schema->item_id_column, &column_offset, &column_len);
int ret = get_column_pos(line, ip_schema->item_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip plus table(table_id:%d) line:%s has no item_id",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_item->item_id = atoll(line + column_offset);
ret = get_column_pos(line, ip_schema->group_id_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->group_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip plus table(table_id:%d) line:%s has no group_id",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has no group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_item->group_id = atoll(line + column_offset);
ret = get_column_pos(line, ip_schema->addr_type_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->addr_type_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip plus table(table_id:%d) line:%s has no addr_type",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has no addr_type in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_item->addr_type = atoi(line + column_offset);
if (ip_item->addr_type != IPv4 && ip_item->addr_type != IPv6) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has invalid addr type:%d",
__FUNCTION__, __LINE__, ip_schema->table_id, line, ip_item->addr_type);
"[%s:%d] ip table:<%s> has invalid addr type:%d in line:%s",
__FUNCTION__, __LINE__, table_name, ip_item->addr_type, line);
goto error;
}
ret = get_column_pos(line, ip_schema->addr_format_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->addr_format_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has no addr_format",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has no addr_format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
memcpy(addr_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(addr_format)) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has invalid saddr_format, should be single/range/CIDR",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has invalid addr_format, "
"should be single/range/CIDR/mask in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ret = get_column_pos(line, ip_schema->ip1_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->ip1_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has no ip1",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has no ip1 in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
memcpy(ip1_str, (line + column_offset), column_len);
ret = get_column_pos(line, ip_schema->ip2_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->ip2_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has no ip2",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has no ip2 in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
memcpy(ip2_str, (line + column_offset), column_len);
if (IPv4 == ip_item->addr_type) {
ret = ip_format2range(ip_item->addr_type, ip_format_str2int(addr_format), ip1_str, ip2_str,
&ip_item->ipv4.min_ip, &ip_item->ipv4.max_ip);
ret = ip_format2range(ip_item->addr_type, ip_format_str2int(addr_format),
ip1_str, ip2_str, &ip_item->ipv4.min_ip, &ip_item->ipv4.max_ip);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s ip_format2range(ip4) failed",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> ip_format2range(ip4) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
} else {
//ipv6
ret = ip_format2range(ip_item->addr_type, ip_format_str2int(addr_format), ip1_str, ip2_str,
ip_item->ipv6.min_ip, ip_item->ipv6.max_ip);
ret = ip_format2range(ip_item->addr_type, ip_format_str2int(addr_format),
ip1_str, ip2_str, ip_item->ipv6.min_ip, ip_item->ipv6.max_ip);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s ip_format2range(ip6) failed",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> ip_format2range(ip6) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
}
ret = get_column_pos(line, ip_schema->port_format_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->port_format_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has no port_format",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has no port_format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
memcpy(port_format, (line + column_offset), column_len);
if (PORT_FORMAT_UNKNOWN == port_format_str2int(port_format)) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has invalid port_format, should be single/range",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has invalid port_format, "
"should be single/range in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_item->port_format = port_format_str2int(port_format);
ret = get_column_pos(line, ip_schema->port1_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->port1_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has no port1",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s>) has no port1 in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_item->min_port = atoi(line + column_offset);
ret = get_column_pos(line, ip_schema->port2_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->port2_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has no port2",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has no port2 in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_item->max_port = atoi(line + column_offset);
ret = get_column_pos(line, ip_schema->protocol_column, &column_offset, &column_len);
ret = get_column_pos(line, ip_schema->protocol_column, &column_offset,
&column_len);
if (ret < 0) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table(table_id:%d) line:%s has no protocol",
__FUNCTION__, __LINE__, ip_schema->table_id, line);
"[%s:%d] ip table:<%s> has no protocol in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_item->proto = atoi(line + column_offset);
@@ -504,12 +521,20 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
long long item_id = get_column_value(line, schema->item_id_column);
if (item_id < 0) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] ip table:<%s> has no item_id(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->item_id_column, line);
ip_rt->update_err_cnt++;
return -1;
}
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] ip table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
ip_rt->update_err_cnt++;
return -1;
}
@@ -517,7 +542,7 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
struct ip_item *ip_item = NULL;
if (1 == is_valid) {
//add
ip_item = ip_item_new(line, schema, ip_rt->logger);
ip_item = ip_item_new(schema, table_name, line, ip_rt->logger);
if (NULL == ip_item) {
ip_rt->update_err_cnt++;
return -1;
@@ -549,7 +574,8 @@ void garbage_ip_matcher_free(void *ip_matcher, void *arg)
ip_matcher_free(matcher);
}
int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_rt_version)
int ip_runtime_commit(void *ip_runtime, const char *table_name,
long long maat_rt_version)
{
if (NULL == ip_runtime) {
return -1;
@@ -595,16 +621,18 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] table[%s] rebuild ip_matcher engine failed when update %zu ip rules",
__FUNCTION__, __LINE__, table_name, rule_cnt);
"[%s:%d] table[%s] rebuild ip_matcher engine failed "
"when update %zu ip rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
}
new_intval_matcher = interval_matcher_new(intval_rules, rule_cnt);
if (NULL == new_intval_matcher) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] table[%s] rebuild interval_matcher engine failed when update %zu ip rules",
__FUNCTION__, __LINE__, table_name, rule_cnt);
"[%s:%d] table[%s] rebuild interval_matcher engine "
"failed when update %zu ip rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
}
}
@@ -629,8 +657,8 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_r
ip_rt->version = maat_rt_version;
log_info(ip_rt->logger, MODULE_IP,
"table[%s] commit %zu ip rules and rebuild ip_matcher completed, version:%lld",
table_name, rule_cnt, ip_rt->version);
"table[%s] commit %zu ip rules and rebuild ip_matcher completed"
", version:%lld", table_name, rule_cnt, ip_rt->version);
if (rules != NULL) {
FREE(rules);
@@ -817,7 +845,8 @@ void ip_runtime_perf_stat(struct ip_runtime *ip_rt, struct timespec *start,
alignment_int64_array_add(ip_rt->scan_cnt, thread_id, 1);
if (start != NULL && end != NULL) {
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + end->tv_nsec - start->tv_nsec;
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 +
(end->tv_nsec - start->tv_nsec);
alignment_int64_array_add(ip_rt->scan_cpu_time, thread_id, consume_time);
}
}
@@ -856,7 +885,8 @@ long long ip_runtime_hit_count(void *ip_runtime)
}
struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
long long sum = alignment_int64_array_sum(ip_rt->hit_cnt, ip_rt->n_worker_thread);
long long sum = alignment_int64_array_sum(ip_rt->hit_cnt,
ip_rt->n_worker_thread);
alignment_int64_array_reset(ip_rt->hit_cnt, ip_rt->n_worker_thread);
return sum;

53
src/maat_ip_plugin.c
View File

@@ -60,7 +60,7 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no table_id column",
"[%s:%d] ip_plugin table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -68,7 +68,7 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no custom column",
"[%s:%d] ip_plugin table:<%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -78,7 +78,7 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->item_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no item_id column",
"[%s:%d] ip_plugin table:<%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -88,7 +88,7 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->ip_type_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no ip_type column",
"[%s:%d] ip_plugin table:<%s> schema has no ip_type column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -98,7 +98,7 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->start_ip_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no start_ip column",
"[%s:%d] ip_plugin table:<%s> schema has no start_ip column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -108,7 +108,7 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->end_ip_column = custom_item->valueint;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no end_ip column",
"[%s:%d] ip_plugin table:<%s> schema has no end_ip column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -160,7 +160,7 @@ int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *tabl
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> has no rule_tag in line:%s",
"[%s:%d] ip_plugin table:<%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
@@ -172,14 +172,14 @@ int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *tabl
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> has invalid tag format in line:%s",
"[%s:%d] ip_plugin table:<%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
"[%s:%d] ip_plugin table:<%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
@@ -190,8 +190,8 @@ int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *tabl
}
struct ip_rule *
ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
const char *table_name, struct log_handle *logger)
ip_plugin_rule_new(struct ip_plugin_schema *schema, const char *table_name,
const char *line, struct log_handle *logger)
{
int ret = ip_plugin_accept_tag_match(schema, table_name, line, logger);
if (ret == TAG_MATCH_UNMATCHED) {
@@ -208,7 +208,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> has no item_id in line:%s",
"[%s:%d] ip_plugin table:<%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -217,14 +217,14 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> has no ip_type in line:%s",
"[%s:%d] ip_plugin table:<%s> has no ip_type in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_plugin_rule->type = atoi(line + column_offset);
if (ip_plugin_rule->type != IPv4 && ip_plugin_rule->type != IPv6) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> ip_type[%d] invalid in line:%s",
"[%s:%d] ip_plugin table:<%s> ip_type[%d] invalid in line:%s",
__FUNCTION__, __LINE__, table_name, ip_plugin_rule->type, line);
goto error;
}
@@ -253,7 +253,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> has no start_ip in line:%s",
"[%s:%d] ip_plugin table:<%s> has no start_ip in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -262,7 +262,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> has no end_ip in line:%s",
"[%s:%d] ip_plugin table:<%s> has no end_ip in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -275,7 +275,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
&ip_plugin_rule->ipv4_rule.end_ip);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> ip_format2range(ip4) failed in line:%s",
"[%s:%d] ip_plugin table:<%s>> ip_format2range(ip4) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -287,7 +287,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ip_plugin_rule->ipv6_rule.end_ip);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> ip_format2range(ip6) failed in line:%s",
"[%s:%d] ip_plugin table:<%s> ip_format2range(ip6) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -446,7 +446,8 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
return -1;
}
int ret = get_column_pos(line, schema->item_id_column, &item_id_offset, &item_id_len);
int ret = get_column_pos(line, schema->item_id_column, &item_id_offset,
&item_id_len);
if (ret < 0) {
ip_plugin_rt->update_err_cnt++;
return -1;
@@ -455,7 +456,8 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
if (1 == schema->container_schema.set_flag) {
if (1 == is_valid) {
// add
ip_plugin_rule = ip_plugin_rule_new(line, schema, table_name, ip_plugin_rt->logger);
ip_plugin_rule = ip_plugin_rule_new(schema, table_name, line,
ip_plugin_rt->logger);
if (NULL == ip_plugin_rule) {
ip_plugin_rt->update_err_cnt++;
return -1;
@@ -523,8 +525,8 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table[%s] rebuild ip_matcher failed when update %zu rules",
__FUNCTION__, __LINE__, table_name, rule_cnt);
"[%s:%d] ip_plugin table[%s] rebuild ip_matcher failed when "
"update %zu rules", __FUNCTION__, __LINE__, table_name, rule_cnt);
ret = -1;
}
}
@@ -542,8 +544,8 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
}
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] commit %zu ip_plugin rules and rebuild ip_matcher completed, version:%lld",
table_name, rule_cnt, ip_plugin_rt->version);
"table[%s] commit %zu ip_plugin rules and rebuild ip_matcher "
"completed, version:%lld", table_name, rule_cnt, ip_plugin_rt->version);
if (rules != NULL) {
FREE(rules);
@@ -618,7 +620,8 @@ void ip_plugin_runtime_perf_stat(void *ip_plugin_runtime, struct timespec *start
alignment_int64_array_add(ip_plugin_rt->scan_cnt, thread_id, 1);
if (start != NULL && end != NULL) {
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 + (end->tv_nsec - start->tv_nsec);
long long consume_time = (end->tv_sec - start->tv_sec) * 1000000000 +
(end->tv_nsec - start->tv_nsec);
alignment_int64_array_add(ip_plugin_rt->scan_cpu_time, thread_id, consume_time);
}
}

146
src/maat_plugin.c
View File

@@ -20,6 +20,8 @@
#include "maat_table.h"
#define MODULE_PLUGIN module_name_str("maat.plugin")
#define IPV4 4
#define IPV6 6
struct plugin_callback_schema {
maat_start_callback_t *start;
@@ -41,12 +43,14 @@ struct plugin_runtime {
enum plugin_key_type {
PLUGIN_KEY_TYPE_INVALID = 0,
PLUGIN_KEY_TYPE_POINTER,
PLUGIN_KEY_TYPE_INTEGER
PLUGIN_KEY_TYPE_INTEGER,
PLUGIN_KEY_TYPE_IP_ADDR
};
#define MAX_PLUGIN_PER_TABLE 32
struct plugin_schema {
enum plugin_key_type key_type;
int addr_type_column;
int key_column;
int rule_tag_column;
int n_foreign;
@@ -87,7 +91,7 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] table: <%s> schema has no table_id column",
"[%s:%d]plugin table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -99,7 +103,7 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
custom_item = cJSON_GetObjectItem(item, "key");
if (NULL == custom_item || custom_item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] table: <%s> schema has no key column",
"[%s:%d]plugin table:<%s> schema has no key column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -108,7 +112,7 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
custom_item = cJSON_GetObjectItem(item, "key_type");
if (NULL == custom_item || custom_item->type != cJSON_String) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] table: <%s> schema has no key_type column",
"[%s:%d]plugin table:<%s> schema has no key_type column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -117,10 +121,21 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->key_type = PLUGIN_KEY_TYPE_POINTER;
} else if (strcmp(custom_item->valuestring, "integer") == 0) {
schema->key_type = PLUGIN_KEY_TYPE_INTEGER;
} else if (strcmp(custom_item->valuestring, "ip_addr") == 0) {
schema->key_type = PLUGIN_KEY_TYPE_IP_ADDR;
custom_item = cJSON_GetObjectItem(item, "addr_type");
if (NULL == custom_item || custom_item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d]plugin table:<%s> schema ip_addr key must have addr_type column",
__FUNCTION__, __LINE__, table_name);
}
schema->addr_type_column = custom_item->valueint;
} else {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] table: <%s> schema key_type:%s is illegal, just allow {pointer} or {integer}",
__FUNCTION__, __LINE__, table_name, custom_item->valuestring);
"[%s:%d]plugin table:<%s> schema key_type:%s is illegal, "
"just allow {pointer}, {integer} or {ip_addr}",
__FUNCTION__, __LINE__, table_name,
custom_item->valuestring);
goto error;
}
@@ -354,8 +369,8 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *table_name
int ret = get_column_pos(line, schema->rule_tag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
"[%s:%d] table: <%s> has no rule_tag(column_seq:%d) in table_line:%s",
__FUNCTION__, __LINE__, table_name, schema->rule_tag_column, line);
return TAG_MATCH_ERR;
}
@@ -366,14 +381,14 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *table_name
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] table: <%s> has invalid tag format in line:%s",
"[%s:%d] table: <%s> has invalid tag format in table_line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
"[%s:%d] table: <%s> has unmatched tag in table_line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
@@ -383,6 +398,94 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *table_name
return TAG_MATCH_MATCHED;
}
int plugin_table_line_get_key(struct plugin_schema *schema, const char *table_name,
const char *line, char *dst_key, size_t *dst_key_len,
struct log_handle *logger)
{
size_t key_offset = 0, key_len = 0;
int ret = get_column_pos(line, schema->key_column, &key_offset, &key_len);
if (ret < 0) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:<%s> has no key(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->key_column, line);
return -1;
}
long long key_int = 0;
const char *common_key = line + key_offset;
if (schema->key_type == PLUGIN_KEY_TYPE_POINTER) {
memcpy(dst_key, common_key, key_len);
*dst_key_len = key_len;
} else if (schema->key_type == PLUGIN_KEY_TYPE_INTEGER) {
key_int = atoll(common_key);
memcpy(dst_key, (char *)&key_int, sizeof(long long));
*dst_key_len = sizeof(long long);
} else if (schema->key_type == PLUGIN_KEY_TYPE_IP_ADDR) {
if (key_len >= INET6_ADDRSTRLEN) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:<%s> ip_key too long(illegal) in "
"table_line:%s", __FUNCTION__, __LINE__, table_name, line);
return -1;
}
size_t addr_type_offset = 0, addr_type_len = 0;
ret = get_column_pos(line, schema->addr_type_column, &addr_type_offset,
&addr_type_len);
if (ret < 0) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:<%s> has no addr_type(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->addr_type_column, line);
return -1;
}
char ip_key[INET6_ADDRSTRLEN] = {0};
//snprintf() write at most (key_len+1) bytes (including the terminating null{'\0}) to ip_key.
snprintf(ip_key, key_len + 1, "%s", common_key);
int addr_type = atoi(line + addr_type_offset);
if (IPV4 == addr_type) {
uint32_t ipv4_addr;
ret = inet_pton(AF_INET, ip_key, &ipv4_addr);
if (ret <= 0) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:<%s> ipv4 key(column seq:%d)"
" illegal in table_line:%s", __FUNCTION__, __LINE__,
table_name, schema->key_column, line);
return -1;
}
memcpy(dst_key, (char *)&ipv4_addr, sizeof(ipv4_addr));
*dst_key_len = sizeof(ipv4_addr);
} else if (IPV6 == addr_type) {
uint8_t ipv6_addr[16];
ret = inet_pton(AF_INET6, ip_key, ipv6_addr);
if (ret <= 0) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:<%s> ipv6 key(column seq:%d)"
" illegal in table_line:%s", __FUNCTION__, __LINE__,
table_name, schema->key_column, line);
return -1;
}
memcpy(dst_key, (char *)&ipv6_addr, sizeof(ipv6_addr));
*dst_key_len = sizeof(ipv6_addr);
} else {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:<%s> addr_type:%d illegal, just"
" allow{4, 6}, table_line:%s", __FUNCTION__, __LINE__,
table_name, addr_type, line);
return -1;
}
}
return 0;
}
int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
const char *table_name, const char *line,
int valid_column)
@@ -396,6 +499,10 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_error(plugin_rt->logger, MODULE_PLUGIN,
"[%s:%d] plugin table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
plugin_rt->update_err_cnt++;
return -1;
}
@@ -406,21 +513,15 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
return -1;
}
size_t key_offset = 0, key_len = 0;
ret = get_column_pos(line, schema->key_column, &key_offset, &key_len);
char key[MAX_KEYWORDS_STR] = {0};
size_t key_len = 0;
ret = plugin_table_line_get_key(schema, table_name, line, key, &key_len,
plugin_rt->logger);
if (ret < 0) {
plugin_rt->update_err_cnt++;
return -1;
}
long long key_int = 0;
const char *key = line + key_offset;
if (schema->key_type == PLUGIN_KEY_TYPE_INTEGER) {
key_int = atoll(key);
key = (char *)&key_int;
key_len = sizeof(long long);
}
ret = plugin_runtime_update_row(plugin_rt, schema, table_name, line,
key, key_len, is_valid);
if (ret < 0) {
@@ -529,11 +630,6 @@ void *plugin_runtime_get_ex_data(void *plugin_runtime, void *plugin_schema,
}
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
if (schema->key_type == PLUGIN_KEY_TYPE_INTEGER) {
key_len = sizeof(long long);
}
return ex_data_runtime_get_ex_data_by_key(plugin_rt->ex_data_rt, key, key_len);
}

4
src/maat_virtual.c
View File

@@ -34,7 +34,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
log_error(logger, MODULE_VIRTUAL,
"[%s:%d] virtual table %s has no table_id column",
"[%s:%d] virtual table:<%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -43,7 +43,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_VIRTUAL,
"[%s:%d] virtual table %s has no physical_table column",
"[%s:%d] virtual table:<%s> schema has no physical_table column",
__FUNCTION__, __LINE__, table_name);
goto error;
}

85
test/maat_framework_gtest.cpp
View File

@@ -453,6 +453,7 @@ protected:
logger = log_handle_create("./maat_framework_gtest.log", 0);
struct maat_options *opts = maat_options_new();
maat_options_set_instance_name(opts, "firewall");
maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
maat_options_set_json_file(opts, watched_json);
maat_options_set_json_file_gzip_flag(opts, 1);
maat_options_set_json_file_decrypt_key(opts, json_decrypt_key);
@@ -2592,14 +2593,16 @@ TEST_F(PluginTable, EX_DATA) {
const char *key1 = "HeBei";
struct plugin_ud *ud = NULL;
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id, key1);
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id,
key1, strlen(key1));
ASSERT_TRUE(ud != NULL);
EXPECT_STREQ(ud->value, "Shijiazhuang");
EXPECT_EQ(ud->id, 1);
plugin_EX_free_cb(table_id, (void **)&ud, 0, NULL);
const char *key2 = "ShanDong";
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id, key2);
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id,
key2, strlen(key2));
ASSERT_TRUE(ud != NULL);
EXPECT_STREQ(ud->value, "Jinan");
EXPECT_EQ(ud->id, 3);
@@ -2624,20 +2627,84 @@ TEST_F(PluginTable, KEY_TYPE) {
long long key1 = 11111111;
struct plugin_ud *ud = NULL;
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id, (char *)&key1);
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id,
(char *)&key1, sizeof(long long));
ASSERT_TRUE(ud != NULL);
EXPECT_STREQ(ud->value, "Shijiazhuang");
EXPECT_EQ(ud->id, 1);
plugin_EX_free_cb(table_id, (void **)&ud, 0, NULL);
long long key2 = 33333333;
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id, (char *)&key2);
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id,
(char *)&key2, sizeof(long long));
ASSERT_TRUE(ud != NULL);
EXPECT_STREQ(ud->value, "Jinan");
EXPECT_EQ(ud->id, 3);
plugin_EX_free_cb(table_id, (void **)&ud, 0, NULL);
}
TEST_F(PluginTable, IP_KEY_TYPE) {
const char *table_name = "TEST_PLUGIN_IP_KEY_TYPE_TABLE";
struct maat *maat_instance = PluginTable::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
int plugin_ex_data_counter = 0;
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name,
plugin_EX_new_cb,
plugin_EX_free_cb,
plugin_EX_dup_cb,
0, &plugin_ex_data_counter);
EXPECT_EQ(ret, 0);
EXPECT_EQ(plugin_ex_data_counter, 4);
uint32_t ipv4_addr1;
ret = inet_pton(AF_INET, "100.64.1.1", &ipv4_addr1);
EXPECT_EQ(ret, 1);
struct plugin_ud *ud = NULL;
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id,
(char *)&ipv4_addr1, sizeof(ipv4_addr1));
ASSERT_TRUE(ud != NULL);
EXPECT_STREQ(ud->value, "XiZang");
EXPECT_EQ(ud->id, 4);
plugin_EX_free_cb(table_id, (void **)&ud, 0, NULL);
uint32_t ipv4_addr2;
ret = inet_pton(AF_INET, "100.64.1.2", &ipv4_addr2);
EXPECT_EQ(ret, 1);
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id,
(char *)&ipv4_addr2, sizeof(ipv4_addr2));
ASSERT_TRUE(ud != NULL);
EXPECT_STREQ(ud->value, "XinJiang");
EXPECT_EQ(ud->id, 4);
plugin_EX_free_cb(table_id, (void **)&ud, 0, NULL);
uint8_t ipv6_addr1[16];
ret = inet_pton(AF_INET6, "2001:da8:205:1::101", ipv6_addr1);
EXPECT_EQ(ret, 1);
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id,
(char *)ipv6_addr1, sizeof(ipv6_addr1));
ASSERT_TRUE(ud != NULL);
EXPECT_STREQ(ud->value, "GuiZhou");
EXPECT_EQ(ud->id, 6);
plugin_EX_free_cb(table_id, (void **)&ud, 0, NULL);
uint8_t ipv6_addr2[16];
ret = inet_pton(AF_INET6, "1001:da8:205:1::101", ipv6_addr2);
EXPECT_EQ(ret, 1);
ud = (struct plugin_ud *)maat_plugin_table_get_ex_data(maat_instance, table_id,
(char *)ipv6_addr2, sizeof(ipv6_addr2));
ASSERT_TRUE(ud != NULL);
EXPECT_STREQ(ud->value, "SiChuan");
EXPECT_EQ(ud->id, 6);
plugin_EX_free_cb(table_id, (void **)&ud, 0, NULL);
}
class IPPluginTable : public testing::Test
{
protected:
@@ -3438,7 +3505,7 @@ TEST_F(Policy, CompileEXData) {
EXPECT_EQ(results[0], 198);
void *ex_data = maat_plugin_table_get_ex_data(maat_instance, compile_table_id,
(char *)&results[0]);
(char *)&results[0], sizeof(long long));
ASSERT_TRUE(ex_data!=NULL);
struct rule_ex_param *param = (struct rule_ex_param *)ex_data;
EXPECT_EQ(param->id, 7799);
@@ -4892,8 +4959,8 @@ TEST_F(MaatCmdTest, PluginEXData) {
struct user_info *uinfo = NULL;
const char *key1 = "192.168.0.2";
uinfo = (struct user_info *)maat_plugin_table_get_ex_data(maat_instance,
table_id, key1);
uinfo = (struct user_info *)maat_plugin_table_get_ex_data(maat_instance, table_id,
key1, strlen(key1));
ASSERT_TRUE(uinfo != NULL);
EXPECT_EQ(0, strcmp(uinfo->name, "liuqiangdong"));
EXPECT_EQ(uinfo->id, 2);
@@ -4910,8 +4977,8 @@ TEST_F(MaatCmdTest, PluginEXData) {
sleep(WAIT_FOR_EFFECTIVE_S);
const char *key2 = "192.168.0.2";
uinfo = (struct user_info *)maat_plugin_table_get_ex_data(maat_instance,
table_id, key2);
uinfo = (struct user_info *)maat_plugin_table_get_ex_data(maat_instance, table_id,
key2, strlen(key2));
ASSERT_TRUE(uinfo == NULL);
}

11
test/maat_json.json
View File

@@ -3027,6 +3027,17 @@
"305\t0&1&2&3&4&5&6&7\ttunnel5\t1",
"306\t101&101\tinvalid\t1"
]
},
{
"table_name": "TEST_PLUGIN_IP_KEY_TYPE_TABLE",
"table_content": [
"4\t100.64.1.1\tXiZang\t1\t0",
"4\t100.64.1.2\tXinJiang\t1\t0",
"6\t2001:da8:205:1::101\tGuiZhou\t1\t0",
"6\t1001:da8:205:1::101\tSiChuan\t1\t0",
"7\t100.64.1.3\tQingHai\t1\t0",
"6\t100.64.1.4\tGanSu\t1\t0"
]
}
]
}

11
test/table_info.conf
View File

@@ -458,5 +458,16 @@
"key":2,
"tag":5
}
},
{
"table_id":39,
"table_name":"TEST_PLUGIN_IP_KEY_TYPE_TABLE",
"table_type":"plugin",
"valid_column":4,
"custom": {
"key_type":"ip_addr",
"addr_type":1,
"key":2
}
}
]