gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix group_exclude logic miss & add some corner case

Browse Source
This commit is contained in:
刘文坛
2023-05-23 03:23:39 +00:00
parent b58ecc09e6
commit 464dc43cc4
29 changed files with 3317 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -223,9 +223,18 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd)
FREE(hs_cd->patterns[i]); FREE(hs_cd->patterns[i]);
} }
FREE(hs_cd->patterns); FREE(hs_cd->patterns);
}
if (hs_cd->pattern_lens != NULL) {
FREE(hs_cd->pattern_lens); FREE(hs_cd->pattern_lens);
}
if (hs_cd->ids != NULL) {
FREE(hs_cd->ids); FREE(hs_cd->ids);
}
if (hs_cd->flags != NULL) {
FREE(hs_cd->flags); FREE(hs_cd->flags);
} }
@@ -290,8 +299,6 @@ struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule, struct
bool_exprs[i].items[j].item_id = pattern_index++; bool_exprs[i].items[j].item_id = pattern_index++;
bool_exprs[i].items[j].not_flag = 0; bool_exprs[i].items[j].not_flag = 0;
// printf("item_id:%llu, pat:%s pat_len:%zu\n",
// bool_exprs[i].items[j].item_id, exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len);
} }
bool_exprs[i].expr_id = rules[i].expr_id; bool_exprs[i].expr_id = rules[i].expr_id;
@@ -405,21 +412,6 @@ struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
} }
hs_instance->n_patterns = pattern_cnt; hs_instance->n_patterns = pattern_cnt;
//mytest
// for (size_t i = 0; i < n_expr; i++) {
// {
// printf("<before bool_matcher_new> exprs[%zu] expr_id:%llu, item_num:%zu\n",
// i, bool_exprs[i].expr_id, bool_exprs[i].item_num);
// printf("item_id: ");
// for (size_t j = 0; j < bool_exprs[i].item_num; j++)
// {
// printf("%llu ", bool_exprs[i].items[j].item_id);
// }
// }
// printf("\n");
// }
/* create bool matcher */ /* create bool matcher */
size_t mem_size = 0; size_t mem_size = 0;
int hs_ret = 0; int hs_ret = 0;
@@ -502,7 +494,6 @@ void adapter_hs_free(struct adapter_hs *hs_instance)
} }
FREE(hs_instance->hs_rt->literal_scratches); FREE(hs_instance->hs_rt->literal_scratches);
hs_instance->hs_rt->literal_scratches = NULL;
} }
@@ -515,7 +506,6 @@ void adapter_hs_free(struct adapter_hs *hs_instance)
} }
FREE(hs_instance->hs_rt->regex_scratches); FREE(hs_instance->hs_rt->regex_scratches);
hs_instance->hs_rt->regex_scratches = NULL;
} }
if (hs_instance->hs_rt->bm != NULL) { if (hs_instance->hs_rt->bm != NULL) {

31
src/maat_api.c
View File

@@ -283,7 +283,6 @@ void _maat_free(struct maat *maat_instance)
if (maat_instance->opts.accept_tags != NULL) { if (maat_instance->opts.accept_tags != NULL) {
FREE(maat_instance->opts.accept_tags); FREE(maat_instance->opts.accept_tags);
maat_instance->opts.accept_tags = NULL;
} }
pthread_mutex_destroy(&(maat_instance->background_update_mutex)); pthread_mutex_destroy(&(maat_instance->background_update_mutex));
@@ -529,20 +528,21 @@ int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
switch (table_type) { switch (table_type) {
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
ret = plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func, dup_func, ret = plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func,
free, argl, argp); dup_func, free, argl, argp);
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
ret = ip_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func, dup_func, ret = ip_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func,
free, argl, argp); dup_func, free, argl, argp);
break; break;
case TABLE_TYPE_FQDN_PLUGIN: case TABLE_TYPE_FQDN_PLUGIN:
ret = fqdn_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func, dup_func, ret = fqdn_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func,
(void (*)(void *))fqdn_rule_free, argl, argp); dup_func, (void (*)(void *))fqdn_rule_free,
argl, argp);
break; break;
case TABLE_TYPE_BOOL_PLUGIN: case TABLE_TYPE_BOOL_PLUGIN:
ret = bool_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func, dup_func, ret = bool_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func,
free, argl, argp); dup_func, free, argl, argp);
break; break;
default: default:
log_error(logger, MODULE_MAAT_API, log_error(logger, MODULE_MAAT_API,
@@ -720,18 +720,19 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance,
int ret = -1; int ret = -1;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (TABLE_TYPE_COMPILE == table_type) { if (TABLE_TYPE_COMPILE == table_type) {
ret = compile_table_ex_schema_register(maat_instance, table_id, ret = compile_table_ex_schema_register(maat_instance, table_id, new_func, free_func,
new_func, free_func, dup_func, argl, argp); dup_func, argl, argp);
} else { } else {
ret = generic_plugin_table_ex_schema_register(maat_instance, table_name, table_id, ret = generic_plugin_table_ex_schema_register(maat_instance, table_name, table_id, new_func,
new_func, free_func, dup_func, argl, argp); free_func, dup_func, argl, argp);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return ret; return ret;
} }
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, const char *key) void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
const char *key)
{ {
if (NULL == maat_instance || table_id < 0 || table_id >= MAX_TABLE_NUM if (NULL == maat_instance || table_id < 0 || table_id >= MAX_TABLE_NUM
@@ -768,7 +769,7 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, co
} }
int maat_ip_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, int maat_ip_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
const struct ip_addr *ip_addr, void **ex_data_array, const struct ip_addr *ip_addr, void **ex_data_array,
size_t n_ex_data) size_t n_ex_data)
{ {
if (NULL == maat_instance || table_id < 0 || table_id >= MAX_TABLE_NUM if (NULL == maat_instance || table_id < 0 || table_id >= MAX_TABLE_NUM

34
src/maat_bool_plugin.c
View File

@@ -56,7 +56,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint; schema->table_id = item->valueint;
} else { } else {
log_error(logger, MODULE_BOOL_PLUGIN, log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table %s has no table_id column", "[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -64,7 +64,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) { if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_BOOL_PLUGIN, log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table %s has no custom column", "[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -74,7 +74,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->item_id_column = custom_item->valueint; schema->item_id_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_BOOL_PLUGIN, log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table %s has no item_id column", "[%s:%d] table: <%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -84,7 +84,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->bool_expr_column = custom_item->valueint; schema->bool_expr_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_BOOL_PLUGIN, log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table %s has no bool_expr column", "[%s:%d] table: <%s> schema has no bool_expr column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -185,7 +185,8 @@ void *bool_plugin_runtime_new(void *bool_plugin_schema, size_t max_thread_num,
bool_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger); bool_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger);
if (1 == schema->container_schema.set_flag) { if (1 == schema->container_schema.set_flag) {
ex_data_runtime_set_ex_container_schema(bool_plugin_rt->ex_data_rt, &(schema->container_schema)); ex_data_runtime_set_ex_container_schema(bool_plugin_rt->ex_data_rt,
&(schema->container_schema));
} }
bool_plugin_rt->n_worker_thread = max_thread_num; bool_plugin_rt->n_worker_thread = max_thread_num;
@@ -254,8 +255,8 @@ int bool_plugin_runtime_update_row(struct bool_plugin_runtime *bool_plugin_rt,
return 0; return 0;
} }
int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *line, int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *table_name,
struct log_handle *logger) const char *line, struct log_handle *logger)
{ {
size_t column_offset = 0; size_t column_offset = 0;
size_t column_len = 0; size_t column_len = 0;
@@ -266,8 +267,8 @@ int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *
&column_offset, &column_len); &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_BOOL_PLUGIN, log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table(table_id:%d) has no rule_tag, line:%s", "[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, schema->table_id, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
@@ -278,12 +279,15 @@ int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *
FREE(tag_str); FREE(tag_str);
if (TAG_MATCH_ERR == ret) { if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_BOOL_PLUGIN, log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table(table_id:%d) has invalid tag format, line:%s", "[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, schema->table_id, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
if (TAG_MATCH_UNMATCHED == ret) { if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED; return TAG_MATCH_UNMATCHED;
} }
} }
@@ -296,7 +300,7 @@ struct bool_expr *
bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema, bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
const char *table_name, struct log_handle *logger) const char *table_name, struct log_handle *logger)
{ {
int ret = bool_plugin_accept_tag_match(schema, line, logger); int ret = bool_plugin_accept_tag_match(schema, table_name, line, logger);
if (ret == TAG_MATCH_UNMATCHED) { if (ret == TAG_MATCH_UNMATCHED) {
return NULL; return NULL;
} }
@@ -312,7 +316,7 @@ bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len); ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_BOOL_PLUGIN, log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table:%s line:%s has no item_id column", "[%s:%d] table: <%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -321,7 +325,7 @@ bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
ret = get_column_pos(line, schema->bool_expr_column, &column_offset, &column_len); ret = get_column_pos(line, schema->bool_expr_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_BOOL_PLUGIN, log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table:%s line:%s has no bool_expr column", "[%s:%d] table: <%s> has no bool_expr in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -337,7 +341,7 @@ bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
n_item++; n_item++;
if (ret != 1 || n_item > MAX_ITEMS_PER_BOOL_EXPR) { if (ret != 1 || n_item > MAX_ITEMS_PER_BOOL_EXPR) {
log_error(logger, MODULE_BOOL_PLUGIN, log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table:%s line:%s has invalid format of bool_expr column", "[%s:%d] table: <%s> has invalid format of bool_expr in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }

95
src/maat_compile.c
View File

@@ -118,7 +118,6 @@ struct compile_runtime {
struct group2compile_runtime { struct group2compile_runtime {
long long not_flag_group; long long not_flag_group;
long long version;
long long rule_num; long long rule_num;
long long update_err_cnt; long long update_err_cnt;
struct compile_runtime *ref_compile_rt; struct compile_runtime *ref_compile_rt;
@@ -229,7 +228,7 @@ void maat_compile_free(struct maat_compile *compile)
clause_state->in_use = 0; clause_state->in_use = 0;
} }
compile->magic = 0; compile->magic = 0;
free(compile); FREE(compile);
} }
void rcu_maat_compile_free(void *user_ctx, void *data) void rcu_maat_compile_free(void *user_ctx, void *data)
@@ -328,14 +327,15 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
compile_schema->table_id = item->valueint; compile_schema->table_id = item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no table_id column", table_name); "[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no custom column", "[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -345,7 +345,8 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
compile_schema->compile_id_column = custom_item->valueint; compile_schema->compile_id_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no compile_id column", table_name); "[%s:%d] table: <%s> schema has no compile_id column",
__FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -359,7 +360,8 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
compile_schema->declared_clause_num_column = custom_item->valueint; compile_schema->declared_clause_num_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no clause_num column", table_name); "[%s:%d] table: <%s> schema has no clause_num column",
__FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -386,7 +388,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->table_id = item->valueint; g2c_schema->table_id = item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no table_id column", "[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -396,7 +398,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->associated_compile_table_id = item->valueint; g2c_schema->associated_compile_table_id = item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no associated_compile_table_id column", "[%s:%d] table: <%s> schema has no associated_compile_table_id column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -404,7 +406,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no custom column", "[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -414,7 +416,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->group_id_column = custom_item->valueint; g2c_schema->group_id_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no group_id column", "[%s:%d] table: <%s> schema has no group_id column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -424,7 +426,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->compile_id_column = custom_item->valueint; g2c_schema->compile_id_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no compile_id column", "[%s:%d] table: <%s> schema has no compile_id column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -434,7 +436,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->not_flag_column = custom_item->valueint; g2c_schema->not_flag_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no not_flag column", "[%s:%d] table: <%s> schema has no not_flag column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -444,7 +446,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->vtable_name_column = custom_item->valueint; g2c_schema->vtable_name_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no virtual_table_name column", "[%s:%d] table: <%s> schema has no virtual_table_name column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -454,7 +456,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->clause_index_column = custom_item->valueint; g2c_schema->clause_index_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no clause_index column", "[%s:%d] table: <%s> schema has no clause_index column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -490,7 +492,7 @@ int compile_accept_tag_match(struct compile_schema *schema, const char *line,
&column_offset, &column_len); &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s has no rule_tag in line:%s", "[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
@@ -502,12 +504,15 @@ int compile_accept_tag_match(struct compile_schema *schema, const char *line,
FREE(tag_str); FREE(tag_str);
if (TAG_MATCH_ERR == ret) { if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s has invalid tag format in line:%s", "[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
if (TAG_MATCH_UNMATCHED == ret) { if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED; return TAG_MATCH_UNMATCHED;
} }
} }
@@ -522,9 +527,6 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
{ {
int ret = compile_accept_tag_match(compile_schema, line, table_name, logger); int ret = compile_accept_tag_match(compile_schema, line, table_name, logger);
if (ret == TAG_MATCH_UNMATCHED) { if (ret == TAG_MATCH_UNMATCHED) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s accept tag unmatched in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return NULL; return NULL;
} }
@@ -536,7 +538,7 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
&column_offset, &column_len); &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s has no compile_id in line:%s", "[%s:%d] table: <%s> has no compile_id in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -546,7 +548,7 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
&column_offset, &column_len); &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s has no clause_num in line:%s", "[%s:%d] table: <%s> has no clause_num in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -577,7 +579,8 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1); struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1);
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM); compile_rt->expr_match_buff = ALLOC(struct bool_expr_match,
max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM);
compile_rt->version = time(NULL); compile_rt->version = time(NULL);
compile_rt->cfg_hash_tbl = rcu_hash_new(rcu_maat_compile_free, NULL); compile_rt->cfg_hash_tbl = rcu_hash_new(rcu_maat_compile_free, NULL);
compile_rt->clause_by_literals_hash = NULL; compile_rt->clause_by_literals_hash = NULL;
@@ -746,7 +749,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len); int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no group_id", "[%s:%d] table: <%s> has no group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -755,7 +758,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset, &column_len); ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no compile_id", "[%s:%d] table: <%s> has no compile_id in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -764,7 +767,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset, &column_len); ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no NOT_flag", "[%s:%d] table: <%s> has no NOT_flag in line:%s ",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -773,14 +776,14 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
ret = get_column_pos(line, g2c_schema->vtable_name_column, &column_offset, &column_len); ret = get_column_pos(line, g2c_schema->vtable_name_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no virtual_table_name", "[%s:%d] table: <%s> has no virtual_table_name in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
if (column_len > NAME_MAX) { if (column_len > NAME_MAX) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s virtual_table_name length too long", "[%s:%d] table: <%s> virtual_table_name length too long in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -791,8 +794,8 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
g2c_item->vtable_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr, vtable_name); g2c_item->vtable_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr, vtable_name);
if (g2c_item->vtable_id < 0) { if (g2c_item->vtable_id < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s unknown virtual table:%s", "[%s:%d] table: <%s> has unknown virtual table:%s in line:%s",
__FUNCTION__, __LINE__, table_name, line, vtable_name); __FUNCTION__, __LINE__, table_name, vtable_name, line);
goto error; goto error;
} }
} }
@@ -800,7 +803,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
ret = get_column_pos(line, g2c_schema->clause_index_column, &column_offset, &column_len); ret = get_column_pos(line, g2c_schema->clause_index_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no clause_index", "[%s:%d] table: <%s> has no clause_index in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -960,7 +963,7 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
#if 0 #if 0
struct maat_literal_id *p = NULL; struct maat_literal_id *p = NULL;
for(p = (struct maat_literal_id *)utarray_front(iter_compile->clause_states[i].ut_literal_ids); p!=NULL; for(p = (struct maat_literal_id *)utarray_front(iter_compile->clause_states[i].ut_literal_ids); p!=NULL;
p=(struct maat_literal_id *)utarray_next(iter_compile->clause_states[i].ut_literal_ids,p)) { p = (struct maat_literal_id *)utarray_next(iter_compile->clause_states[i].ut_literal_ids, p)) {
printf("<before bool_matcher_new> compile_rt:%p compile_id:%lld, clause_id:%llu, literal{%lld: %lld}\n", printf("<before bool_matcher_new> compile_rt:%p compile_id:%lld, clause_id:%llu, literal{%lld: %lld}\n",
compile_rt, iter_compile->compile_id, iter_compile->clause_states[i].clause_id, p->group_id, p->vtable_id); compile_rt, iter_compile->compile_id, iter_compile->clause_states[i].clause_id, p->group_id, p->vtable_id);
} }
@@ -986,8 +989,8 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
// STEP 3, build bool matcher // STEP 3, build bool matcher
size_t mem_size = 0; size_t mem_size = 0;
if (0 == expr_cnt) { if (0 == expr_cnt) {
log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] No bool expression to build bool matcher.", log_error(compile_rt->logger, MODULE_COMPILE,
__FUNCTION__, __LINE__); "[%s:%d] No bool expression to build bool matcher.", __FUNCTION__, __LINE__);
FREE(bool_expr_array); FREE(bool_expr_array);
return NULL; return NULL;
} }
@@ -995,7 +998,8 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
struct bool_matcher *bm = bool_matcher_new(bool_expr_array, expr_cnt, &mem_size); struct bool_matcher *bm = bool_matcher_new(bool_expr_array, expr_cnt, &mem_size);
if (bm != NULL) { if (bm != NULL) {
log_info(compile_rt->logger, MODULE_COMPILE, log_info(compile_rt->logger, MODULE_COMPILE,
"Build bool matcher of %zu expressions with %zu bytes memory.", expr_cnt, mem_size); "Build bool matcher of %zu expressions with %zu bytes memory.",
expr_cnt, mem_size);
} else { } else {
log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] Build bool matcher failed!", log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] Build bool matcher failed!",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
@@ -1215,14 +1219,21 @@ void compile_rule_free(struct compile_rule *compile_rule)
struct compile_schema *schema = compile_rule->ref_schema; struct compile_schema *schema = compile_rule->ref_schema;
assert(compile_rule->magic_num == COMPILE_RULE_MAGIC); assert(compile_rule->magic_num == COMPILE_RULE_MAGIC);
if (1 == schema->set_flag) if (1 == schema->set_flag) {
{
rule_ex_data_free(schema->table_id, compile_rule->ex_data, &(schema->ex_schema)); rule_ex_data_free(schema->table_id, compile_rule->ex_data, &(schema->ex_schema));
*compile_rule->ex_data = NULL; *compile_rule->ex_data = NULL;
} }
FREE(compile_rule->ex_data);
if (compile_rule->ex_data != NULL) {
FREE(compile_rule->ex_data);
}
compile_rule->declared_clause_num = -1; compile_rule->declared_clause_num = -1;
FREE(compile_rule->table_line);
if (compile_rule->table_line != NULL) {
FREE(compile_rule->table_line);
}
FREE(compile_rule); FREE(compile_rule);
} }
@@ -1468,7 +1479,7 @@ void maat_compile_state_free(struct maat_compile_state *compile_state)
compile_state->this_scan_hit_clauses = NULL; compile_state->this_scan_hit_clauses = NULL;
} }
free(compile_state); FREE(compile_state);
} }
static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id, long long group_id, static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id, long long group_id,
@@ -1533,7 +1544,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
size_t new_hit_path_cnt = 0; size_t new_hit_path_cnt = 0;
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
struct maat_literal_id literal_id = {0, 0}; struct maat_literal_id literal_id = {0, 0};
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM; struct bool_expr_match *expr_match = compile_rt->expr_match_buff +
(compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM);
assert(compile_state->thread_id >= 0); assert(compile_state->thread_id >= 0);
if (compile_state->compile_rt_version != compile_rt->version) { if (compile_state->compile_rt_version != compile_rt->version) {
@@ -1910,7 +1922,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
group_ref = ALLOC(struct group_reference, 1); group_ref = ALLOC(struct group_reference, 1);
group_ref->group_id = g2c_item->group_id; group_ref->group_id = g2c_item->group_id;
HASH_ADD_KEYPTR(hh, compile_rt->group_ref_hash, (char *)&(group_ref->group_id), HASH_ADD_KEYPTR(hh, compile_rt->group_ref_hash, (char *)&(group_ref->group_id),
sizeof(long long), group_ref); sizeof(long long), group_ref);
} }
group_ref->ref_by_compile_cnt++; group_ref->ref_by_compile_cnt++;
pthread_mutex_unlock(&(compile_rt->mutex)); pthread_mutex_unlock(&(compile_rt->mutex));
@@ -1933,6 +1945,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
int group_referenced_by_compile(struct compile_runtime *compile_rt, long long group_id) int group_referenced_by_compile(struct compile_runtime *compile_rt, long long group_id)
{ {
struct group_reference *group_ref = NULL; struct group_reference *group_ref = NULL;
pthread_mutex_lock(&(compile_rt->mutex)); pthread_mutex_lock(&(compile_rt->mutex));
HASH_FIND(hh, compile_rt->group_ref_hash, (char *)&group_id, sizeof(long long), group_ref); HASH_FIND(hh, compile_rt->group_ref_hash, (char *)&group_id, sizeof(long long), group_ref);
if (group_ref != NULL) { if (group_ref != NULL) {

17
src/maat_expr.c
View File

@@ -444,6 +444,10 @@ void expr_rule_reset(struct expr_rule *rule)
void expr_item_free(struct expr_item *item) void expr_item_free(struct expr_item *item)
{ {
if (NULL == item) {
return;
}
if (item->user_data != NULL && if (item->user_data != NULL &&
item->user_data_free != NULL) { item->user_data_free != NULL) {
item->user_data_free(item->user_data); item->user_data_free(item->user_data);
@@ -737,15 +741,6 @@ int expr_item_to_expr_rule(struct expr_item *expr_item, struct expr_rule *expr_r
expr_rule->user_tag = expr_item->user_data; expr_rule->user_tag = expr_item->user_data;
expr_rule->n_patterns = sub_expr_cnt; expr_rule->n_patterns = sub_expr_cnt;
//mytest
// printf("expr_rule->expr_id:%u\n", expr_rule->expr_id);
// printf("expr_rule->n_patterns:%zu\n", expr_rule->n_patterns);
// for (size_t i = 0; i < expr_rule->n_patterns; i++) {
// printf("expr_rule->patterns[%zu].pat:%s\n", i, expr_rule->patterns[i].pat);
// printf("expr_rule->patterns[%zu].pat_len:%zu\n", i, expr_rule->patterns[i].pat_len);
// printf("expr_rule->patterns[%zu].l_offset:%d\n", i, expr_rule->patterns[i].l_offset);
// printf("expr_rule->patterns[%zu].r_offset:%d\n", i, expr_rule->patterns[i].r_offset);
// }
return 0; return 0;
} }
@@ -876,8 +871,8 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
expr_rt->version = maat_rt_version; expr_rt->version = maat_rt_version;
log_info(expr_rt->logger, MODULE_EXPR, log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] commit %zu expr rules and rebuild adapter_hs completed, version:%lld", "table[%s] has %zu rules, commit %zu expr rules and rebuild adapter_hs completed, version:%lld",
table_name, rule_cnt, expr_rt->version); table_name, rule_cnt, real_rule_cnt, expr_rt->version);
if (rules != NULL) { if (rules != NULL) {
for (i = 0; i < rule_cnt; i++) { for (i = 0; i < rule_cnt; i++) {

1
src/maat_flag.c
View File

@@ -425,7 +425,6 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table
if (ret < 0) { if (ret < 0) {
if (flag_item != NULL) { if (flag_item != NULL) {
flag_item_free(flag_item); flag_item_free(flag_item);
flag_item = NULL;
} }
flag_rt->update_err_cnt++; flag_rt->update_err_cnt++;
return -1; return -1;

55
src/maat_fqdn_plugin.c
View File

@@ -56,7 +56,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint; schema->table_id = item->valueint;
} else { } else {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no table_id column", "[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -64,7 +64,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) { if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no custom column", "[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -74,7 +74,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->item_id_column = custom_item->valueint; schema->item_id_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no item_id column", "[%s:%d] table: <%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -84,7 +84,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->suffix_flag_column = custom_item->valueint; schema->suffix_flag_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no suffix_match_method column", "[%s:%d] table: <%s> schema has no suffix_match_method column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -95,7 +95,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->fqdn_column = custom_item->valueint; schema->fqdn_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no fqdn column", "[%s:%d] table: <%s> schema has no fqdn column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -159,9 +159,15 @@ struct ex_container_schema *fqdn_plugin_table_get_ex_container_schema(void *fqdn
void fqdn_rule_free(struct FQDN_rule *fqdn_rule) void fqdn_rule_free(struct FQDN_rule *fqdn_rule)
{ {
if (NULL == fqdn_rule) {
return;
}
assert(fqdn_rule->user_tag == NULL); assert(fqdn_rule->user_tag == NULL);
free(fqdn_rule->FQDN); if (fqdn_rule->FQDN != NULL) {
fqdn_rule->FQDN = NULL; FREE(fqdn_rule->FQDN);
}
free(fqdn_rule); free(fqdn_rule);
} }
@@ -220,8 +226,8 @@ void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime)
FREE(fqdn_plugin_rt); FREE(fqdn_plugin_rt);
} }
int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *line, int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *table_name,
struct log_handle *logger) const char *line, struct log_handle *logger)
{ {
size_t column_offset = 0; size_t column_offset = 0;
size_t column_len = 0; size_t column_len = 0;
@@ -232,8 +238,8 @@ int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *
&column_offset, &column_len); &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table(table_id:%d) has no rule_tag, line:%s", "[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, schema->table_id, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
@@ -244,12 +250,15 @@ int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *
FREE(tag_str); FREE(tag_str);
if (TAG_MATCH_ERR == ret) { if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table(table_id:%d) has invalid tag format, line:%s", "[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, schema->table_id, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
if (TAG_MATCH_UNMATCHED == ret) { if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED; return TAG_MATCH_UNMATCHED;
} }
} }
@@ -262,7 +271,7 @@ struct FQDN_rule *
fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema, fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
const char *table_name, struct log_handle *logger) const char *table_name, struct log_handle *logger)
{ {
int ret = fqdn_plugin_accept_tag_match(schema, line, logger); int ret = fqdn_plugin_accept_tag_match(schema, table_name, line, logger);
if (ret == TAG_MATCH_UNMATCHED) { if (ret == TAG_MATCH_UNMATCHED) {
return NULL; return NULL;
} }
@@ -276,7 +285,7 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len); ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:%s line:%s has no item_id column", "[%s:%d] table: <%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -285,7 +294,7 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
ret = get_column_pos(line, schema->suffix_flag_column, &column_offset, &column_len); ret = get_column_pos(line, schema->suffix_flag_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:%s line:%s has no suffix_match_method column", "[%s:%d] table: <%s> has no suffix_match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -294,7 +303,7 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
ret = get_column_pos(line, schema->fqdn_column, &column_offset, &column_len); ret = get_column_pos(line, schema->fqdn_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN, log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:%s line:%s has no fqdn column", "[%s:%d] table: <%s> has no fqdn in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -323,6 +332,10 @@ error:
void fqdn_plugin_rule_free(struct FQDN_rule *rule) void fqdn_plugin_rule_free(struct FQDN_rule *rule)
{ {
if (NULL == rule) {
return;
}
if (rule->FQDN != NULL) { if (rule->FQDN != NULL) {
FREE(rule->FQDN); FREE(rule->FQDN);
} }
@@ -386,7 +399,8 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche
if (1 == schema->container_schema.set_flag) { if (1 == schema->container_schema.set_flag) {
if (1 == is_valid) { if (1 == is_valid) {
// add // add
fqdn_plugin_rule = fqdn_plugin_rule_new(line, schema, table_name, fqdn_plugin_rt->logger); fqdn_plugin_rule = fqdn_plugin_rule_new(line, schema, table_name,
fqdn_plugin_rt->logger);
if (NULL == fqdn_plugin_rule) { if (NULL == fqdn_plugin_rule) {
fqdn_plugin_rt->update_err_cnt++; fqdn_plugin_rt->update_err_cnt++;
return -1; return -1;
@@ -396,7 +410,7 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche
const char *key = line + item_id_offset; const char *key = line + item_id_offset;
size_t key_len = item_id_len; size_t key_len = item_id_len;
ret = fqdn_plugin_runtime_update_row(fqdn_plugin_rt, table_name, line, key, key_len, ret = fqdn_plugin_runtime_update_row(fqdn_plugin_rt, table_name, line, key, key_len,
fqdn_plugin_rule, is_valid); fqdn_plugin_rule, is_valid);
if (ret < 0) { if (ret < 0) {
if (fqdn_plugin_rule != NULL) { if (fqdn_plugin_rule != NULL) {
fqdn_plugin_rule_free(fqdn_plugin_rule); fqdn_plugin_rule_free(fqdn_plugin_rule);
@@ -419,7 +433,8 @@ void garbage_fqdn_engine_free(void *fqdn_engine, void *arg)
FQDN_engine_free(engine); FQDN_engine_free(engine);
} }
int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name, long long maat_rt_version) int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name,
long long maat_rt_version)
{ {
if (NULL == fqdn_plugin_runtime) { if (NULL == fqdn_plugin_runtime) {
return -1; return -1;

21
src/maat_garbage_collection.c
View File

@@ -42,8 +42,12 @@ struct maat_garbage_bin *maat_garbage_bin_new(int default_timeout)
return bin; return bin;
} }
void maat_garbage_bin_free(struct maat_garbage_bin* bin) void maat_garbage_bin_free(struct maat_garbage_bin *bin)
{ {
if (NULL == bin) {
return;
}
struct maat_garbage_bag *p = NULL; struct maat_garbage_bag *p = NULL;
while ((p = TAILQ_FIRST(&bin->garbage_q)) != NULL) { while ((p = TAILQ_FIRST(&bin->garbage_q)) != NULL) {
@@ -51,6 +55,7 @@ void maat_garbage_bin_free(struct maat_garbage_bin* bin)
if (p->arg != NULL) { if (p->arg != NULL) {
FREE(p->arg); FREE(p->arg);
} }
TAILQ_REMOVE(&bin->garbage_q, p, entries); TAILQ_REMOVE(&bin->garbage_q, p, entries);
FREE(p); FREE(p);
bin->bag_cnt--; bin->bag_cnt--;
@@ -59,12 +64,16 @@ void maat_garbage_bin_free(struct maat_garbage_bin* bin)
FREE(bin); FREE(bin);
} }
size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin) size_t maat_garbage_bin_get_size(struct maat_garbage_bin *bin)
{ {
if (NULL == bin) {
return 0;
}
return bin->bag_cnt; return bin->bag_cnt;
} }
void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void *arg, void maat_garbage_bagging(struct maat_garbage_bin *bin, void *garbage, void *arg,
void (* func)(void *, void *)) void (* func)(void *, void *))
{ {
struct maat_garbage_bag *bag = ALLOC(struct maat_garbage_bag, 1); struct maat_garbage_bag *bag = ALLOC(struct maat_garbage_bag, 1);
@@ -77,7 +86,8 @@ void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void *arg
TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries); TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries);
bin->bag_cnt++; bin->bag_cnt++;
} }
void maat_garbage_collect_routine(struct maat_garbage_bin* bin)
void maat_garbage_collect_routine(struct maat_garbage_bin *bin)
{ {
struct maat_garbage_bag *p = NULL, *tmp = NULL; struct maat_garbage_bag *p = NULL, *tmp = NULL;
size_t n_clollected = 0, n_bag = 0; size_t n_clollected = 0, n_bag = 0;
@@ -101,9 +111,10 @@ void maat_garbage_collect_routine(struct maat_garbage_bin* bin)
bin->bag_cnt -= n_clollected; bin->bag_cnt -= n_clollected;
} }
void maat_garbage_collect_by_force(struct maat_garbage_bin* bin) void maat_garbage_collect_by_force(struct maat_garbage_bin *bin)
{ {
struct maat_garbage_bag *p = NULL; struct maat_garbage_bag *p = NULL;
while ((p = TAILQ_FIRST(&bin->garbage_q)) != NULL) { while ((p = TAILQ_FIRST(&bin->garbage_q)) != NULL) {
p->garbage_free(p->garbage, p->arg); p->garbage_free(p->garbage, p->arg);
if (p->arg != NULL) { if (p->arg != NULL) {

129
src/maat_group.c
View File

@@ -153,6 +153,10 @@ void group2group_schema_free(void *g2g_schema)
void group_vertex_free(struct maat_group *group) void group_vertex_free(struct maat_group *group)
{ {
if (NULL == group) {
return;
}
if (group->incl_super_group_ids != NULL) { if (group->incl_super_group_ids != NULL) {
utarray_free(group->incl_super_group_ids); utarray_free(group->incl_super_group_ids);
group->incl_super_group_ids = NULL; group->incl_super_group_ids = NULL;
@@ -194,6 +198,10 @@ struct maat_group_topology *maat_group_topology_new(struct log_handle *logger)
void maat_group_topology_free(struct maat_group_topology *group_topo) void maat_group_topology_free(struct maat_group_topology *group_topo)
{ {
if (NULL == group_topo) {
return;
}
struct maat_group *group = NULL, *tmp_group = NULL; struct maat_group *group = NULL, *tmp_group = NULL;
HASH_CLEAR(hh_vertex_id, group_topo->hash_by_vertex_id);//No need group memory clean up. HASH_CLEAR(hh_vertex_id, group_topo->hash_by_vertex_id);//No need group memory clean up.
@@ -770,7 +778,7 @@ void get_candidate_super_group_ids(struct maat_group_topology *group_topo,
{ {
long long *p = NULL; long long *p = NULL;
//Find super candidates //Find super candidates
for (p = (long long *)utarray_front(hit_group_ids); p != NULL; for (p = (long long *)utarray_front(hit_group_ids); p != NULL;
p = (long long *)utarray_next(hit_group_ids, p)) { p = (long long *)utarray_next(hit_group_ids, p)) {
struct maat_group *group = group_topology_find_group(group_topo, *p); struct maat_group *group = group_topology_find_group(group_topo, *p);
@@ -792,6 +800,74 @@ void get_candidate_super_group_ids(struct maat_group_topology *group_topo,
} }
} }
void verify_group_by_sub_include_groups(struct maat_group *group, UT_array *candidate_group_ids,
UT_array *kept_super_group_ids, UT_array *all_hit_group_ids)
{
size_t remove_idx = 0;
long long *tmp_id = NULL;
// delete groups whose all incl sub not in all_hit_group_ids
if (utarray_len(group->incl_sub_group_ids) != 0) {
int sub_incl_flag = 0;
for (tmp_id = (long long *)utarray_front(group->incl_sub_group_ids); tmp_id != NULL;
tmp_id = (long long *)utarray_next(group->incl_sub_group_ids, tmp_id)) {
if (utarray_find(candidate_group_ids, tmp_id, compare_group_id)) {
sub_incl_flag = 1;
break;
}
}
if (0 == sub_incl_flag) {
tmp_id = utarray_find(all_hit_group_ids, &(group->group_id), compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(all_hit_group_ids, tmp_id);
utarray_erase(all_hit_group_ids, remove_idx, 1);
}
tmp_id = utarray_find(kept_super_group_ids, &(group->group_id), compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(kept_super_group_ids, tmp_id);
utarray_erase(kept_super_group_ids, remove_idx, 1);
}
}
}
}
void verify_group_by_sub_exclude_groups(struct maat_group *group, UT_array *candidate_group_ids,
UT_array *kept_super_group_ids, UT_array *all_hit_group_ids)
{
if (0 == utarray_len(group->excl_sub_group_ids)) {
return;
}
// delete groups whose excl sub in all_hit_group_ids
int sub_excl_flag = 0;
long long *tmp_id = NULL;
for (tmp_id = (long long *)utarray_front(group->excl_sub_group_ids); tmp_id != NULL;
tmp_id = (long long *)utarray_next(group->excl_sub_group_ids, tmp_id)) {
if (utarray_find(candidate_group_ids, tmp_id, compare_group_id)) {
sub_excl_flag = 1;
break;
}
}
if (1 == sub_excl_flag) {
size_t remove_idx = 0;
tmp_id = utarray_find(all_hit_group_ids, &(group->group_id), compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(all_hit_group_ids, tmp_id);
utarray_erase(all_hit_group_ids, remove_idx, 1);
}
tmp_id = utarray_find(kept_super_group_ids, &(group->group_id), compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(kept_super_group_ids, tmp_id);
utarray_erase(kept_super_group_ids, remove_idx, 1);
}
}
}
void verify_candidate_super_group_ids(struct maat_group_topology *group_topo, void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
UT_array *candidate_super_group_ids, UT_array *candidate_super_group_ids,
UT_array *all_hit_group_ids, UT_array *all_hit_group_ids,
@@ -802,6 +878,7 @@ void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
utarray_new(candidate_group_ids, &ut_group_id_icd); utarray_new(candidate_group_ids, &ut_group_id_icd);
/* merge this round of candidate super groups with hit groups from the previous round */
for (p = (long long *)utarray_front(candidate_super_group_ids); p != NULL; for (p = (long long *)utarray_front(candidate_super_group_ids); p != NULL;
p = (long long *)utarray_next(candidate_super_group_ids, p)) { p = (long long *)utarray_next(candidate_super_group_ids, p)) {
utarray_push_back(candidate_group_ids, p); utarray_push_back(candidate_group_ids, p);
@@ -832,10 +909,11 @@ void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
} }
} }
//delete group_id from kept_group_ids //kept super groups should not store this group
if (1 == sub_excl_flag) { if (1 == sub_excl_flag) {
continue; continue;
} }
utarray_push_back(kept_super_group_ids, p); utarray_push_back(kept_super_group_ids, p);
utarray_sort(kept_super_group_ids, compare_group_id); utarray_sort(kept_super_group_ids, compare_group_id);
@@ -855,7 +933,8 @@ void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
} }
/** /**
* delete groups whose all incl sub is non-exist in all_hit_group_ids * 1. delete groups whose excl sub in all_hit_group_ids
* 2. delete groups whose all incl sub is non-exist in all_hit_group_ids
*/ */
for (p = (long long *)utarray_front(candidate_group_ids); p != NULL; for (p = (long long *)utarray_front(candidate_group_ids); p != NULL;
p = (long long *)utarray_next(candidate_group_ids, p)) { p = (long long *)utarray_next(candidate_group_ids, p)) {
@@ -864,38 +943,15 @@ void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
continue; continue;
} }
if (0 == utarray_len(group->incl_sub_group_ids)) { verify_group_by_sub_exclude_groups(group, candidate_group_ids,
continue; kept_super_group_ids, all_hit_group_ids);
} verify_group_by_sub_include_groups(group, candidate_group_ids,
kept_super_group_ids, all_hit_group_ids);
int sub_incl_flag = 0;
long long *tmp_id = NULL;
for (tmp_id = (long long *)utarray_front(group->incl_sub_group_ids); tmp_id != NULL;
tmp_id = (long long *)utarray_next(group->incl_sub_group_ids, tmp_id)) {
if (utarray_find(candidate_group_ids, tmp_id, compare_group_id)) {
sub_incl_flag = 1;
break;
}
}
if (0 == sub_incl_flag) {
tmp_id = utarray_find(all_hit_group_ids, p, compare_group_id);
assert(tmp_id != NULL);
size_t remove_idx = utarray_eltidx(all_hit_group_ids, tmp_id);
utarray_erase(all_hit_group_ids, remove_idx, 1);
tmp_id = utarray_find(kept_super_group_ids, p, compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(kept_super_group_ids, tmp_id);
utarray_erase(kept_super_group_ids, remove_idx, 1);
}
}
} }
utarray_free(candidate_group_ids); utarray_free(candidate_group_ids);
} }
#define MAX_RECURSION_DEPTH 5
void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_group_ids, void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_group_ids,
UT_array *all_hit_group_ids, size_t depth) UT_array *all_hit_group_ids, size_t depth)
{ {
@@ -903,6 +959,8 @@ void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_g
UT_array *kept_super_group_ids; UT_array *kept_super_group_ids;
if (depth >= MAX_RECURSION_DEPTH) { if (depth >= MAX_RECURSION_DEPTH) {
log_error(group_topo->logger, MODULE_GROUP,
"[%s:%d]exceed max recursion depth(5)", __FUNCTION__, __LINE__);
return; return;
} }
@@ -911,7 +969,7 @@ void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_g
/** /**
candidate super groups means all hit groups' super include group, candidate super groups means all hit groups' super include group,
no need to consider super exclude groups don't consider super exclude groups
for example: for example:
hit_groups = {g4, g11} hit_groups = {g4, g11}
g4's super include groups = {g7, g8} g4's super include groups = {g7, g8}
@@ -941,9 +999,12 @@ void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_g
x \ / \ \ x \ / \ \
x \/ \ \ x \/ \ \
g3 g4 g5 g11 g3 g4 g5 g11
candidates = {g4, g11, g7, g8, g12} candidate super groups = {g7, g8, g12}
g12's sub_exclude g8 in candidates, so g12 should be dropped verify logic:
after verify candidates, all hit super groups = {g7, g8}, 1. g12's sub_exclude g8 in candidates, so g12 should be dropped
2. g7 & g8, their sub_include in hit groups, so kept them
if their all sub_include not exist in hit groups, they should be dropped
after verify candidates, kept super groups = {g7, g8},
all hit groups = {g4, g11, g7, g8} all hit groups = {g4, g11, g7, g8}
*/ */
verify_candidate_super_group_ids(group_topo, candidate_super_group_ids, all_hit_group_ids, verify_candidate_super_group_ids(group_topo, candidate_super_group_ids, all_hit_group_ids,

1
src/maat_interval.c
View File

@@ -423,7 +423,6 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
if (ret < 0) { if (ret < 0) {
if (interval_item != NULL) { if (interval_item != NULL) {
interval_item_free(interval_item); interval_item_free(interval_item);
interval_item = NULL;
} }
interval_rt->update_err_cnt++; interval_rt->update_err_cnt++;
return -1; return -1;

1
src/maat_ip.c
View File

@@ -535,7 +535,6 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
if (ret < 0) { if (ret < 0) {
if (ip_item != NULL) { if (ip_item != NULL) {
ip_item_free(ip_item); ip_item_free(ip_item);
ip_item = NULL;
} }
ip_rt->update_err_cnt++; ip_rt->update_err_cnt++;
return -1; return -1;

87
src/maat_ip_plugin.c
View File

@@ -51,7 +51,6 @@ struct ip_plugin_runtime {
void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger) const char *table_name, struct log_handle *logger)
{ {
size_t read_cnt = 0;
struct ip_plugin_schema *schema = ALLOC(struct ip_plugin_schema, 1); struct ip_plugin_schema *schema = ALLOC(struct ip_plugin_schema, 1);
schema->logger = logger; schema->logger = logger;
@@ -59,13 +58,17 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) { if (item != NULL && item->type == cJSON_Number) {
schema->table_id = item->valueint; schema->table_id = item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
} }
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) { if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table %s has no custom column", "[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -73,33 +76,54 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
custom_item = cJSON_GetObjectItem(item, "item_id"); custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->item_id_column = custom_item->valueint; schema->item_id_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
} }
custom_item = cJSON_GetObjectItem(item, "ip_type"); custom_item = cJSON_GetObjectItem(item, "ip_type");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->ip_type_column = custom_item->valueint; schema->ip_type_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no ip_type column",
__FUNCTION__, __LINE__, table_name);
goto error;
} }
custom_item = cJSON_GetObjectItem(item, "start_ip"); custom_item = cJSON_GetObjectItem(item, "start_ip");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->start_ip_column = custom_item->valueint; schema->start_ip_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no start_ip column",
__FUNCTION__, __LINE__, table_name);
goto error;
} }
custom_item = cJSON_GetObjectItem(item, "end_ip"); custom_item = cJSON_GetObjectItem(item, "end_ip");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->end_ip_column = custom_item->valueint; schema->end_ip_column = custom_item->valueint;
read_cnt++; } else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no end_ip column",
__FUNCTION__, __LINE__, table_name);
goto error;
} }
custom_item = cJSON_GetObjectItem(item, "addr_format"); custom_item = cJSON_GetObjectItem(item, "addr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->addr_format_column = custom_item->valueint; schema->addr_format_column = custom_item->valueint;
//TODO: just because test table not add this column }
//read_cnt++; //TODO: just because test table has no addr_format
} // else {
// log_error(logger, MODULE_IP_PLUGIN,
// "[%s:%d] table: <%s> schema has no addr_format column",
// __FUNCTION__, __LINE__, table_name);
// goto error;
// }
// rule_tag is optional // rule_tag is optional
custom_item = cJSON_GetObjectItem(item, "rule_tag"); custom_item = cJSON_GetObjectItem(item, "rule_tag");
@@ -108,10 +132,6 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
} }
schema->ref_tbl_mgr = tbl_mgr; schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 5) {
goto error;
}
return schema; return schema;
error: error:
@@ -128,8 +148,8 @@ void ip_plugin_schema_free(void *ip_plugin_schema)
FREE(ip_plugin_schema); FREE(ip_plugin_schema);
} }
int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line, int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *table_name,
struct log_handle *logger) const char *line, struct log_handle *logger)
{ {
size_t column_offset = 0; size_t column_offset = 0;
size_t column_len = 0; size_t column_len = 0;
@@ -140,8 +160,8 @@ int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line
&column_offset, &column_len); &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table(table_id:%d) has no rule_tag, line:%s", "[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, schema->table_id, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
@@ -152,12 +172,15 @@ int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line
FREE(tag_str); FREE(tag_str);
if (TAG_MATCH_ERR == ret) { if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table(table_id:%d) has invalid tag format, line:%s", "[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, schema->table_id, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
if (TAG_MATCH_UNMATCHED == ret) { if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED; return TAG_MATCH_UNMATCHED;
} }
} }
@@ -170,7 +193,7 @@ struct ip_rule *
ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema, ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
const char *table_name, struct log_handle *logger) const char *table_name, struct log_handle *logger)
{ {
int ret = ip_plugin_accept_tag_match(schema, line, logger); int ret = ip_plugin_accept_tag_match(schema, table_name, line, logger);
if (ret == TAG_MATCH_UNMATCHED) { if (ret == TAG_MATCH_UNMATCHED) {
return NULL; return NULL;
} }
@@ -185,7 +208,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len); ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip plugin table:%s line:%s has no item_id", "[%s:%d] table: <%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -194,15 +217,15 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len); ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip plugin table:%s line:%s has no ip_type", "[%s:%d] table: <%s> has no ip_type in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
ip_plugin_rule->type = atoi(line + column_offset); ip_plugin_rule->type = atoi(line + column_offset);
if (ip_plugin_rule->type != IPv4 && ip_plugin_rule->type != IPv6) { if (ip_plugin_rule->type != IPv4 && ip_plugin_rule->type != IPv6) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s ip_type[%d] invalid", "[%s:%d] table: <%s> ip_type[%d] invalid in line:%s",
__FUNCTION__, __LINE__, table_name, line, ip_plugin_rule->type); __FUNCTION__, __LINE__, table_name, ip_plugin_rule->type, line);
goto error; goto error;
} }
@@ -230,7 +253,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len); ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s has no start_ip", "[%s:%d] table: <%s> has no start_ip in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -239,7 +262,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len); ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s has no end_ip", "[%s:%d] table: <%s> has no end_ip in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -252,7 +275,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
&ip_plugin_rule->ipv4_rule.end_ip); &ip_plugin_rule->ipv4_rule.end_ip);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s ip_format2range(ip4) failed", "[%s:%d] table: <%s> ip_format2range(ip4) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -264,7 +287,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ip_plugin_rule->ipv6_rule.end_ip); ip_plugin_rule->ipv6_rule.end_ip);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN, log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s ip_format2range(ip6) failed", "[%s:%d] table: <%s> ip_format2range(ip6) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
goto error; goto error;
} }
@@ -359,7 +382,8 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, size_t max_thread_num,
ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger); ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger);
if (1 == schema->container_schema.set_flag) { if (1 == schema->container_schema.set_flag) {
ex_data_runtime_set_ex_container_schema(ip_plugin_rt->ex_data_rt, &(schema->container_schema)); ex_data_runtime_set_ex_container_schema(ip_plugin_rt->ex_data_rt,
&(schema->container_schema));
} }
ip_plugin_rt->n_worker_thread = max_thread_num; ip_plugin_rt->n_worker_thread = max_thread_num;
@@ -458,7 +482,8 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
return 0; return 0;
} }
int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name, long long maat_rt_version) int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
long long maat_rt_version)
{ {
if (NULL == ip_plugin_runtime) { if (NULL == ip_plugin_runtime) {
return -1; return -1;

15
src/maat_kv.c
View File

@@ -48,10 +48,15 @@ struct maat_kv_pair* maat_kv_pair_new(const char* key, size_t keylen, long long
return kv; return kv;
} }
void maat_kv_pair_free(struct maat_kv_pair* kv) void maat_kv_pair_free(struct maat_kv_pair *kv)
{ {
FREE(kv->key); if (NULL == kv) {
kv->key = NULL; return;
}
if (kv->key != NULL) {
FREE(kv->key);
}
FREE(kv); FREE(kv);
} }
@@ -62,7 +67,7 @@ struct maat_kv_store* maat_kv_store_new(void)
return store; return store;
} }
void maat_kv_store_free(struct maat_kv_store* store) void maat_kv_store_free(struct maat_kv_store *store)
{ {
if (NULL == store) { if (NULL == store) {
return; return;
@@ -79,7 +84,7 @@ void maat_kv_store_free(struct maat_kv_store* store)
FREE(store); FREE(store);
} }
int maat_kv_register_unNull(struct maat_kv_store* store, const char *key, size_t keylen, long long value) int maat_kv_register_unNull(struct maat_kv_store *store, const char *key, size_t keylen, long long value)
{ {
if (keylen > MAAT_KV_MAX_KEY_LEN) { if (keylen > MAAT_KV_MAX_KEY_LEN) {
return -1; return -1;

27
src/maat_plugin.c
View File

@@ -87,7 +87,7 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) { if (NULL == item || item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN, log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table %s has no table_id column", "[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -99,7 +99,7 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
custom_item = cJSON_GetObjectItem(item, "key"); custom_item = cJSON_GetObjectItem(item, "key");
if (NULL == custom_item || custom_item->type != cJSON_Number) { if (NULL == custom_item || custom_item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN, log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s has no key column", "[%s:%d] table: <%s> schema has no key column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -108,7 +108,7 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
custom_item = cJSON_GetObjectItem(item, "key_type"); custom_item = cJSON_GetObjectItem(item, "key_type");
if (NULL == custom_item || custom_item->type != cJSON_String) { if (NULL == custom_item || custom_item->type != cJSON_String) {
log_error(logger, MODULE_PLUGIN, log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s has no key_type column", "[%s:%d] table: <%s> schema has no key_type column",
__FUNCTION__, __LINE__, table_name); __FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -119,9 +119,8 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->key_type = PLUGIN_KEY_TYPE_INTEGER; schema->key_type = PLUGIN_KEY_TYPE_INTEGER;
} else { } else {
log_error(logger, MODULE_PLUGIN, log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s key_type:%s illegal", "[%s:%d] table: <%s> schema key_type:%s is illegal, just allow {pointer} or {integer}",
__FUNCTION__, __LINE__, table_name, __FUNCTION__, __LINE__, table_name, custom_item->valuestring);
custom_item->valuestring);
goto error; goto error;
} }
@@ -274,7 +273,8 @@ void *plugin_runtime_new(void *plugin_schema, size_t max_thread_num,
plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger); plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger);
if (1 == schema->container_schema.set_flag) { if (1 == schema->container_schema.set_flag) {
ex_data_runtime_set_ex_container_schema(plugin_rt->ex_data_rt, &(schema->container_schema)); ex_data_runtime_set_ex_container_schema(plugin_rt->ex_data_rt,
&(schema->container_schema));
} }
plugin_rt->ref_garbage_bin = garbage_bin; plugin_rt->ref_garbage_bin = garbage_bin;
@@ -319,7 +319,8 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, table_name, void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, table_name,
row, key, key_len); row, key, key_len);
struct ex_container *ex_container = ex_container_new(ex_data, NULL); struct ex_container *ex_container = ex_container_new(ex_data, NULL);
ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container); ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len,
ex_container);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -353,7 +354,7 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *table_name
int ret = get_column_pos(line, schema->rule_tag_column, &column_offset, &column_len); int ret = get_column_pos(line, schema->rule_tag_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_PLUGIN, log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:%s has no rule_tag, line:%s", "[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
@@ -365,12 +366,15 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *table_name
FREE(tag_str); FREE(tag_str);
if (TAG_MATCH_ERR == ret) { if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_PLUGIN, log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:%s has invalid tag format, line:%s", "[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line); __FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR; return TAG_MATCH_ERR;
} }
if (TAG_MATCH_UNMATCHED == ret) { if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED; return TAG_MATCH_UNMATCHED;
} }
} }
@@ -427,7 +431,8 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
return 0; return 0;
} }
int plugin_runtime_commit(void *plugin_runtime, const char *table_name, long long maat_rt_version) int plugin_runtime_commit(void *plugin_runtime, const char *table_name,
long long maat_rt_version)
{ {
if (NULL == plugin_runtime) { if (NULL == plugin_runtime) {
return -1; return -1;

18
src/maat_redis_monitor.c
View File

@@ -411,16 +411,28 @@ int get_inc_key_list(long long instance_version, long long target_version,
void serial_rule_free(struct serial_rule *s_rule) void serial_rule_free(struct serial_rule *s_rule)
{ {
if (NULL == s_rule) {
return;
}
if (s_rule->table_line != NULL) { if (s_rule->table_line != NULL) {
FREE(s_rule->table_line); FREE(s_rule->table_line);
} }
if (s_rule->n_foreign > 0) { if (s_rule->n_foreign > 0) {
for (int i = 0; i < s_rule->n_foreign; i++) { for (int i = 0; i < s_rule->n_foreign; i++) {
FREE(s_rule->f_keys[i].filename); if (s_rule->f_keys[i].filename != NULL) {
FREE(s_rule->f_keys[i].key); FREE(s_rule->f_keys[i].filename);
}
if (s_rule->f_keys[i].key != NULL) {
FREE(s_rule->f_keys[i].key);
}
} }
FREE(s_rule->f_keys);
if (s_rule->f_keys != NULL) {
FREE(s_rule->f_keys);
}
} }
FREE(s_rule); FREE(s_rule);

2
src/maat_rule.c
View File

@@ -55,7 +55,7 @@ void maat_item_inner_free(void *item_inner)
assert(item->magic_num == ITEM_RULE_MAGIC); assert(item->magic_num == ITEM_RULE_MAGIC);
item->magic_num = 0; item->magic_num = 0;
free(item); FREE(item);
} }
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance) struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)

5
src/maat_stat.c
View File

@@ -185,7 +185,7 @@ void maat_stat_free(struct maat_stat *stat)
stat->fs_handle = NULL; stat->fs_handle = NULL;
} }
free(stat); FREE(stat);
} }
int maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr, int maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr,
@@ -287,8 +287,7 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
break; break;
} }
if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_COMPILE || if (table_type == TABLE_TYPE_PLUGIN ) {
table_type == TABLE_TYPE_GROUP2COMPILE || table_type == TABLE_TYPE_GROUP2GROUP) {
continue; continue;
} }

2
test/CMakeLists.txt
View File

@@ -30,12 +30,14 @@ target_link_libraries(maat_ex_data_gtest maat_frame_static gtest_static)
add_subdirectory(group_exclude) add_subdirectory(group_exclude)
file(COPY table_info.conf DESTINATION ./) file(COPY table_info.conf DESTINATION ./)
file(COPY tsg_table_info.conf DESTINATION ./)
file(COPY file_test_tableinfo.conf DESTINATION ./) file(COPY file_test_tableinfo.conf DESTINATION ./)
file(COPY literal_expr.conf DESTINATION ./) file(COPY literal_expr.conf DESTINATION ./)
file(COPY regex_expr.conf DESTINATION ./) file(COPY regex_expr.conf DESTINATION ./)
file(COPY maat_json.json DESTINATION ./) file(COPY maat_json.json DESTINATION ./)
file(COPY maat_json.json DESTINATION ../tools/) file(COPY maat_json.json DESTINATION ../tools/)
file(COPY ntcrule DESTINATION ./) file(COPY ntcrule DESTINATION ./)
file(COPY tsgrule DESTINATION ./)
file(COPY testdata DESTINATION ./) file(COPY testdata DESTINATION ./)
file(COPY test_streamfiles DESTINATION ./) file(COPY test_streamfiles DESTINATION ./)
file(COPY json_update DESTINATION ./) file(COPY json_update DESTINATION ./)

136
test/group_exclude/group_exclude_L4.conf Normal file
View File

@@ -0,0 +1,136 @@
{
"add_items": [
{
"group_id": 14,
"super_group_id": 8,
"is_exclude": 0
},
{
"group_id": 14,
"super_group_id": 9,
"is_exclude": 0
},
{
"group_id": 8,
"super_group_id": 5,
"is_exclude": 0
},
{
"group_id": 9,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 16,
"super_group_id": 9,
"is_exclude": 0
},
{
"group_id": 16,
"super_group_id": 10,
"is_exclude": 0
},
{
"group_id": 10,
"super_group_id": 7,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 6,
"is_exclude": 0
},
{
"group_id": 6,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 17,
"super_group_id": 11,
"is_exclude": 0
},
{
"group_id": 17,
"super_group_id": 12,
"is_exclude": 0
},
{
"group_id": 17,
"super_group_id":13,
"is_exclude": 0
},
{
"group_id": 11,
"super_group_id": 6,
"is_exclude": 0
},
{
"group_id": 12,
"super_group_id": 7,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 2,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 4,
"is_exclude": 0
},
{
"group_id": 13,
"super_group_id": 3,
"is_exclude": 0
},
{
"group_id": 15,
"super_group_id": 8,
"is_exclude": 1
},
{
"group_id": 9,
"super_group_id": 5,
"is_exclude": 1
},
{
"group_id": 5,
"super_group_id": 1,
"is_exclude": 1
},
{
"group_id": 10,
"super_group_id": 6,
"is_exclude": 1
},
{
"group_id": 6,
"super_group_id": 2,
"is_exclude": 1
},
{
"group_id": 6,
"super_group_id": 3,
"is_exclude": 1
},
{
"group_id": 18,
"super_group_id":13,
"is_exclude": 1
},
{
"group_id": 13,
"super_group_id": 4,
"is_exclude": 1
}
],
"del_items": [
{
"group_id": 10,
"super_group_id": 6,
"is_exclude": 1
}
]
}

BIN
test/group_exclude/group_exclude_L4.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 74 KiB

76
test/group_exclude/group_exclude_exceed.conf Normal file
View File

@@ -0,0 +1,76 @@
{
"add_items": [
{
"group_id": 6,
"super_group_id": 3,
"is_exclude": 0
},
{
"group_id": 3,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 3,
"is_exclude": 1
},
{
"group_id": 13,
"super_group_id": 10,
"is_exclude": 0
},
{
"group_id": 10,
"super_group_id": 11,
"is_exclude": 0
},
{
"group_id": 11,
"super_group_id": 8,
"is_exclude": 0
},
{
"group_id": 8,
"super_group_id": 5,
"is_exclude": 0
},
{
"group_id": 5,
"super_group_id": 4,
"is_exclude": 0
},
{
"group_id": 5,
"super_group_id": 12,
"is_exclude": 0
},
{
"group_id": 4,
"super_group_id": 2,
"is_exclude": 0
},
{
"group_id": 4,
"super_group_id": 1,
"is_exclude": 1
},
{
"group_id": 2,
"super_group_id": 12,
"is_exclude": 1
}
],
"del_items": [
{
"group_id": 13,
"super_group_id": 10,
"is_exclude": 0
}
]
}

BIN
test/group_exclude/group_exclude_exceed.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 46 KiB

303
test/group_exclude/group_exclude_gtest.cpp
View File

@@ -11,6 +11,7 @@
#define MAX_ITEM_NUM 64 #define MAX_ITEM_NUM 64
#define WAIT_FOR_EFFECTIVE_S 2 #define WAIT_FOR_EFFECTIVE_S 2
#define MAX_G2G_SCAN_TIMES (1000 * 1000)
const char *table_info_path = "./table_info.conf"; const char *table_info_path = "./table_info.conf";
@@ -138,38 +139,34 @@ protected:
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_schema_new failed."); log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_schema_new failed.");
assert(0); assert(0);
} }
g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
} }
static void TearDownTestCase() { static void TearDownTestCase() {
log_handle_destroy(logger); log_handle_destroy(logger);
maat_garbage_bin_free(garbage_bin); maat_garbage_bin_free(garbage_bin);
group2group_runtime_free(g2g_runtime);
group2group_schema_free(g2g_schema); group2group_schema_free(g2g_schema);
} }
static void *g2g_schema; static void *g2g_schema;
static void *g2g_runtime;
static struct log_handle *logger; static struct log_handle *logger;
static struct maat_garbage_bin *garbage_bin; static struct maat_garbage_bin *garbage_bin;
}; };
void *MaatGroupExclude::g2g_schema; void *MaatGroupExclude::g2g_schema;
void *MaatGroupExclude::g2g_runtime;
struct log_handle *MaatGroupExclude::logger; struct log_handle *MaatGroupExclude::logger;
struct maat_garbage_bin *MaatGroupExclude::garbage_bin; struct maat_garbage_bin *MaatGroupExclude::garbage_bin;
TEST_F(MaatGroupExclude, level_3_basic) { TEST_F(MaatGroupExclude, level_3_function) {
char table_line[1024]; char table_line[1024];
struct group2group_rule rules; struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP"; const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_L3.conf", &rules); int ret = parse_config_file("group_exclude_L3.conf", &rules);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
@@ -199,7 +196,7 @@ TEST_F(MaatGroupExclude, level_3_basic) {
group2group_runtime_commit(g2g_runtime, table_name, 2); group2group_runtime_commit(g2g_runtime, table_name, 2);
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2, super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM); super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 3); EXPECT_EQ(super_group_cnt, 3);
EXPECT_EQ(super_group_ids[0], 1); EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 6); EXPECT_EQ(super_group_ids[1], 6);
@@ -213,12 +210,290 @@ TEST_F(MaatGroupExclude, level_3_basic) {
group2group_runtime_commit(g2g_runtime, table_name, 3); group2group_runtime_commit(g2g_runtime, table_name, 3);
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2, super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM); super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 4); EXPECT_EQ(super_group_cnt, 4);
EXPECT_EQ(super_group_ids[0], 1); EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 4); EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 6); EXPECT_EQ(super_group_ids[2], 6);
EXPECT_EQ(super_group_ids[3], 7); EXPECT_EQ(super_group_ids[3], 7);
group2group_runtime_free(g2g_runtime);
}
TEST_F(MaatGroupExclude, level_3_perf) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_L3.conf", &rules);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < rules.n_add_item; i++) {
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.add_items[i].group_id,
rules.add_items[i].super_group_id, rules.add_items[i].is_exclude, 1);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
}
group2group_runtime_commit(g2g_runtime, table_name, 4);
long long hit_group_ids[2] = {11, 13};
long long super_group_ids[MAX_ITEM_NUM];
size_t super_group_cnt = 0;
struct timespec start, end;
struct log_handle *logger = MaatGroupExclude::logger;
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 2);
EXPECT_EQ(super_group_ids[0], 2);
EXPECT_EQ(super_group_ids[1], 7);
}
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_3_basic hit 2 super_groups scan consume time %lldms",
time_elapse_ms);
//delete group_id = 7, super_group_id = 6, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[0].group_id,
rules.del_items[0].super_group_id, rules.del_items[0].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 5);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 3);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 6);
EXPECT_EQ(super_group_ids[2], 7);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_3_basic hit 3 super_groups scan consume time %lldms",
time_elapse_ms);
//delete group_id = 13, super_group_id = 4, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[1].group_id,
rules.del_items[1].super_group_id, rules.del_items[1].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 6);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 4);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 6);
EXPECT_EQ(super_group_ids[3], 7);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_3_basic hit 4 super_groups scan consume time %lldms",
time_elapse_ms);
group2group_runtime_free(g2g_runtime);
}
TEST_F(MaatGroupExclude, level_4_function) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_L4.conf", &rules);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < rules.n_add_item; i++) {
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.add_items[i].group_id,
rules.add_items[i].super_group_id, rules.add_items[i].is_exclude, 1);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
}
group2group_runtime_commit(g2g_runtime, table_name, 7);
long long hit_group_ids[2] = {14, 16};
long long super_group_ids[MAX_ITEM_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 5);
EXPECT_EQ(super_group_ids[0], 4);
EXPECT_EQ(super_group_ids[1], 7);
EXPECT_EQ(super_group_ids[2], 8);
EXPECT_EQ(super_group_ids[3], 9);
EXPECT_EQ(super_group_ids[4], 10);
//delete group_id = 10, super_group_id = 6, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[0].group_id,
rules.del_items[0].super_group_id, rules.del_items[0].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 8);
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 7);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 6);
EXPECT_EQ(super_group_ids[3], 7);
EXPECT_EQ(super_group_ids[4], 8);
EXPECT_EQ(super_group_ids[5], 9);
EXPECT_EQ(super_group_ids[6], 10);
group2group_runtime_free(g2g_runtime);
}
TEST_F(MaatGroupExclude, level_4_perf) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_L4.conf", &rules);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < rules.n_add_item; i++) {
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.add_items[i].group_id,
rules.add_items[i].super_group_id, rules.add_items[i].is_exclude, 1);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
}
group2group_runtime_commit(g2g_runtime, table_name, 7);
long long hit_group_ids[2] = {14, 16};
long long super_group_ids[MAX_ITEM_NUM];
size_t super_group_cnt = 0;
struct timespec start, end;
struct log_handle *logger = MaatGroupExclude::logger;
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 5);
EXPECT_EQ(super_group_ids[0], 4);
EXPECT_EQ(super_group_ids[1], 7);
EXPECT_EQ(super_group_ids[2], 8);
EXPECT_EQ(super_group_ids[3], 9);
EXPECT_EQ(super_group_ids[4], 10);
}
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_4_basic hit 5 super_groups scan consume time %lldms",
time_elapse_ms);
//delete group_id = 10, super_group_id = 6, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[0].group_id,
rules.del_items[0].super_group_id, rules.del_items[0].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 8);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 7);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 6);
EXPECT_EQ(super_group_ids[3], 7);
EXPECT_EQ(super_group_ids[4], 8);
EXPECT_EQ(super_group_ids[5], 9);
EXPECT_EQ(super_group_ids[6], 10);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_4_basic hit 7 super_groups scan consume time %lldms",
time_elapse_ms);
group2group_runtime_free(g2g_runtime);
}
TEST_F(MaatGroupExclude, level_exceed_function) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_exceed.conf", &rules);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < rules.n_add_item; i++) {
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.add_items[i].group_id,
rules.add_items[i].super_group_id, rules.add_items[i].is_exclude, 1);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
}
group2group_runtime_commit(g2g_runtime, table_name, 8);
long long hit_group_ids[2] = {7, 13};
long long super_group_ids[MAX_ITEM_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 6);
EXPECT_EQ(super_group_ids[0], 4);
EXPECT_EQ(super_group_ids[1], 5);
EXPECT_EQ(super_group_ids[2], 8);
EXPECT_EQ(super_group_ids[3], 10);
EXPECT_EQ(super_group_ids[4], 11);
EXPECT_EQ(super_group_ids[5], 12);
//delete group_id = 13, super_group_id = 10, is_exclude = 0
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[0].group_id,
rules.del_items[0].super_group_id, rules.del_items[0].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 9);
hit_group_ids[0] = 7;
hit_group_ids[1] = 10;
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 5);
EXPECT_EQ(super_group_ids[0], 2);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 5);
EXPECT_EQ(super_group_ids[3], 8);
EXPECT_EQ(super_group_ids[4], 11);
group2group_runtime_free(g2g_runtime);
} }
int main(int argc, char ** argv) int main(int argc, char ** argv)
@@ -227,4 +502,4 @@ int main(int argc, char ** argv)
::testing::InitGoogleTest(&argc, argv); ::testing::InitGoogleTest(&argc, argv);
ret=RUN_ALL_TESTS(); ret=RUN_ALL_TESTS();
return ret; return ret;
} }

796
test/maat_framework_gtest.cpp
View File

File diff suppressed because it is too large Load Diff

758
test/maat_framework_perf_gtest.cpp
View File

@@ -294,7 +294,8 @@ class MaatPerfStringScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
@@ -308,7 +309,7 @@ protected:
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -362,7 +363,7 @@ void *perf_string_scan_thread(void *arg)
int *is_all_hit = ALLOC(int, 1); int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0); *is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d time_elapse:%lldms hit_times:%d", "thread_id:%d string_scan time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times); param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit; return is_all_hit;
} }
@@ -421,7 +422,7 @@ void *perf_ip_scan_thread(void *arg)
int *is_all_hit = ALLOC(int, 1); int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0); *is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d time_elapse:%lldms hit_times:%d", "thread_id:%d ip_scan time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times); param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit; return is_all_hit;
} }
@@ -456,7 +457,7 @@ void *perf_ip_update_thread(void *arg)
return is_all_hit; return is_all_hit;
} }
TEST_F(MaatPerfStringScan, basic) { TEST_F(MaatPerfStringScan, MultiThread) {
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = MaatPerfStringScan::_shared_maat_instance; struct maat *maat_instance = MaatPerfStringScan::_shared_maat_instance;
@@ -498,14 +499,169 @@ TEST_F(MaatPerfStringScan, basic) {
} }
scan_per_second = scan_count * 1000 / time_elapse_ms; scan_per_second = scan_count * 1000 / time_elapse_ms;
//EXPECT_GT(scan_per_second, 800 * 1000); //EXPECT_GT(scan_per_second, 800 * 1000);
printf("High match rate on %d-threads speed %lld lookups/s/thread\n", PERF_THREAD_NUM, scan_per_second); log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"StringScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfStreamScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfStreamScan::_shared_maat_instance;
struct log_handle *MaatPerfStreamScan::logger;
void *perf_stream_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const char *table_name = param->table_name;
struct timespec start, end;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
long long results[ARRAY_SIZE] = {0};
int ret = 0, hit_times = 0;
size_t n_hit_result = 0;
struct maat_state *state_array[ARRAY_SIZE];
struct maat_stream *sp[ARRAY_SIZE];
int table_id = maat_get_table_id(maat_instance, table_name);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) {
for (int j = 0; j < ARRAY_SIZE; j++) {
state_array[j] = maat_state_new(maat_instance, param->thread_id);
sp[j] = maat_stream_new(maat_instance, table_id, state_array[j]);
ret = maat_stream_scan(sp[j], scan_data, strlen(scan_data), results, ARRAY_SIZE,
&n_hit_result, state_array[j]);
if (ret == MAAT_SCAN_HIT) {
hit_times++;
}
maat_stream_free(sp[j]);
maat_state_free(state_array[j]);
}
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = ((hit_times == param->test_count*ARRAY_SIZE) ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d stream_scan time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void *perf_stream_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10;
char keyword_buf[128];
for (int i = 0; i < CMD_EXPR_NUM; i++) {
random_keyword_generate(keyword_buf, sizeof(keyword_buf));
test_add_expr_command(maat_instance, table_name, keyword_buf);
sleep(1);
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;
return is_all_hit;
}
TEST_F(MaatPerfStreamScan, MultiThread) {
const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatPerfStreamScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = 100 * 1000;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_stream_scan_thread, thread_params+i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_stream_update_thread, thread_params+i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
//maybe expr_runtime rebuild in stream_scan, so should not expect is_all_hit always 1
//EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"StreamScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
} }
class MaatPerfIPScan : public testing::Test class MaatPerfIPScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
@@ -544,7 +700,7 @@ protected:
struct maat *MaatPerfIPScan::_shared_maat_instance; struct maat *MaatPerfIPScan::_shared_maat_instance;
struct log_handle *MaatPerfIPScan::logger; struct log_handle *MaatPerfIPScan::logger;
TEST_F(MaatPerfIPScan, basic) TEST_F(MaatPerfIPScan, MultiThread)
{ {
const char *table_name = "IP_PLUS_CONFIG"; const char *table_name = "IP_PLUS_CONFIG";
struct maat *maat_instance = MaatPerfIPScan::_shared_maat_instance; struct maat *maat_instance = MaatPerfIPScan::_shared_maat_instance;
@@ -561,7 +717,7 @@ TEST_F(MaatPerfIPScan, basic)
thread_params[i].maat_instance = maat_instance; thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i; thread_params[i].thread_id = i;
thread_params[i].table_name = table_name; thread_params[i].table_name = table_name;
thread_params[i].test_count = 1 * 1000 * 1000; thread_params[i].test_count = 10 * 1000 * 1000;
thread_params[i].time_elapse_ms = 0; thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger; thread_params[i].logger = logger;
@@ -586,8 +742,588 @@ TEST_F(MaatPerfIPScan, basic)
free(is_all_hit); free(is_all_hit);
} }
scan_per_second = scan_count * 1000 / time_elapse_ms; scan_per_second = scan_count * 1000 / time_elapse_ms;
//EXPECT_GT(scan_per_second, 800 * 1000);
printf("High match rate on %d-threads speed %lld lookups/s/thread\n", PERF_THREAD_NUM, scan_per_second); log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"IPScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfFQDNPluginScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfFQDNPluginScan::_shared_maat_instance;
struct log_handle *MaatPerfFQDNPluginScan::logger;
struct perf_fqdn_plugin_ud {
long long rule_id;
int catid;
int ref_cnt;
};
void perf_fqdn_plugin_EX_new_cb(const char *table_name, int table_id, const char *key,
const char *table_line, void **ad, long argl, void *argp)
{
int *counter=(int *)argp;
size_t column_offset=0, column_len=0;
struct perf_fqdn_plugin_ud *ud = ALLOC(struct perf_fqdn_plugin_ud, 1);
int ret = maat_helper_read_column(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->rule_id = atoll(table_line+column_offset);
ret = maat_helper_read_column(table_line, 4, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
sscanf(table_line+column_offset, "catid=%d",&ud->catid);
ud->ref_cnt = 1;
*ad = ud;
(*counter)++;
}
void perf_fqdn_plugin_EX_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct perf_fqdn_plugin_ud *u = (struct perf_fqdn_plugin_ud *)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) {
free(u);
*ad = NULL;
}
}
void perf_fqdn_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct perf_fqdn_plugin_ud *u = (struct perf_fqdn_plugin_ud *)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
void* perf_fqdn_plugin_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
struct perf_fqdn_plugin_ud *result[ARRAY_SIZE];
int i=0, j=0, ret=0, hit_times=0;
int table_id = maat_get_table_id(maat_instance, param->table_name);
memset(&result, 0, sizeof(result));
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < param->test_count; i++) {
ret = maat_fqdn_plugin_table_get_ex_data(maat_instance, table_id,
"r3---sn-i3belne6.example2.com",
(void**)result, ARRAY_SIZE);
if (ret == 2) {
hit_times++;
}
for (j = 0; j < ret; j++) {
perf_fqdn_plugin_EX_free_cb(0, (void**)&(result[j]), 0, NULL);
}
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec-start.tv_sec)*1000 + (end.tv_nsec-start.tv_nsec)/1000000;
int* is_all_hit = (int*)malloc(sizeof(int));
*is_all_hit = (hit_times == param->test_count) ? 1 : 0;
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d fqdn_plugin_get_ex_data time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void random_fqdn_generate(char *fqdn_buff, int sz)
{
int min_fqdn_len = 6, max_fqdn_len = 32;
int i=0, j=0;
int len = random() % (max_fqdn_len - min_fqdn_len) + min_fqdn_len;
if (len >= sz) {
len = sz - 1;
}
for (i = 0; i < len-4; i++) {
fqdn_buff[i] = 'a' + random() % ('z' - 'a');
if (j > 5) {
if (random() % 3 == 0) {
fqdn_buff[i] = '.';
j = 0;
}
}
j++;
}
fqdn_buff[i] = '\0';
}
void *perf_fqdn_plugin_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const int CMD_EXPR_NUM = 20;
int i = 0;
struct maat_cmd_line line_rule;
char line_buff[1024], fqdn_buff[256];
for (i = 0; i < CMD_EXPR_NUM; i++) {
line_rule.rule_id = (int)maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1);
line_rule.table_name = param->table_name;
random_fqdn_generate(fqdn_buff, sizeof(fqdn_buff));
snprintf(line_buff, 1024, "%lld\t1\t%s\tcatid=4\t1", line_rule.rule_id, fqdn_buff);
line_rule.table_line = line_buff;
line_rule.expire_after = 0;
maat_cmd_set_line(maat_instance, &line_rule);
sleep(WAIT_FOR_EFFECTIVE_S);
}
int *is_all_hit = (int *)malloc(sizeof(int));
*is_all_hit = 1;
return is_all_hit;
}
TEST_F(MaatPerfFQDNPluginScan, MultiThread) {
struct maat *maat_instance = MaatPerfFQDNPluginScan::_shared_maat_instance;
const char *table_name = "TEST_FQDN_PLUGIN_WITH_EXDATA";
int fqdn_plugin_ex_data_counter = 0;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name,
perf_fqdn_plugin_EX_new_cb,
perf_fqdn_plugin_EX_free_cb,
perf_fqdn_plugin_EX_dup_cb,
0, &fqdn_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = 1 * 1000 * 1000;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_fqdn_plugin_scan_thread, thread_params + i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_fqdn_plugin_update_thread, thread_params + i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"FQDNPluginScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfBoolPluginScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in BoolPluginScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfBoolPluginScan::_shared_maat_instance;
struct log_handle *MaatPerfBoolPluginScan::logger;
struct bool_plugin_ud {
int id;
char *name;
int ref_cnt;
};
void perf_bool_plugin_ex_new_cb(const char *table_name, int table_id, const char *key,
const char *table_line, void **ad, long argl, void *argp)
{
int *counter=(int *)argp;
size_t column_offset=0, column_len=0;
struct bool_plugin_ud *ud = ALLOC(struct bool_plugin_ud, 1);
int ret = get_column_pos(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->id = atoi(table_line + column_offset);
ret = get_column_pos(table_line, 3, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->name = (char *)malloc(column_len+1);
memcpy(ud->name, table_line+column_offset, column_len);
ud->ref_cnt = 1;
*ad = ud;
(*counter)++;
}
void perf_bool_plugin_ex_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct bool_plugin_ud *u = (struct bool_plugin_ud *)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
{
free(u->name);
free(u);
*ad = NULL;
}
}
void perf_bool_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct bool_plugin_ud *u = (struct bool_plugin_ud *)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
void* perf_bool_plugin_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
struct perf_fqdn_plugin_ud *result[ARRAY_SIZE];
int i=0, j=0, ret=0, hit_times=0;
int table_id = maat_get_table_id(maat_instance, param->table_name);
memset(&result, 0, sizeof(result));
struct timespec start, end;
unsigned long long items_4[]={7, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7};
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < param->test_count; i++) {
ret = maat_bool_plugin_table_get_ex_data(maat_instance, table_id, items_4,
sizeof(items_4)/sizeof(unsigned long long),
(void**)result, 6);
if (ret == 1) {
hit_times++;
}
for (j = 0; j < ret; j++) {
perf_bool_plugin_ex_free_cb(0, (void**)&(result[j]), 0, NULL);
}
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec-start.tv_sec)*1000 + (end.tv_nsec-start.tv_nsec)/1000000;
int* is_all_hit = (int*)malloc(sizeof(int));
*is_all_hit = (hit_times == param->test_count) ? 1 : 0;
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d bool_plugin_get_ex_data time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void *perf_bool_plugin_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const int CMD_EXPR_NUM = 20;
int i = 0;
struct maat_cmd_line line_rule;
char line_buff[1024];
for (i = 0; i < CMD_EXPR_NUM; i++) {
line_rule.rule_id = (int)maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1);
line_rule.table_name = param->table_name;
snprintf(line_buff, 1024, "%lld\t1&%d\ttunnel2\t1", line_rule.rule_id, i);
line_rule.table_line = line_buff;
line_rule.expire_after = 0;
maat_cmd_set_line(maat_instance, &line_rule);
sleep(WAIT_FOR_EFFECTIVE_S);
}
int *is_all_hit = (int *)malloc(sizeof(int));
*is_all_hit = 1;
return is_all_hit;
}
TEST_F(MaatPerfBoolPluginScan, MultiThread) {
int ex_data_counter = 0;
const char *table_name = "TEST_BOOL_PLUGIN_WITH_EXDATA";
struct maat *maat_instance = MaatPerfBoolPluginScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name,
perf_bool_plugin_ex_new_cb,
perf_bool_plugin_ex_free_cb,
perf_bool_plugin_ex_dup_cb,
0, &ex_data_counter);
ASSERT_TRUE(ret >= 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = 1 * 1000 * 1000;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_bool_plugin_scan_thread, thread_params + i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_bool_plugin_update_thread, thread_params + i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"BoolPluginScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfFileScan : public testing::Test
{
protected:
static void SetUpTestCase() {
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
const char *rule_folder = "./tsgrule/full/index";
const char *table_info = "./tsg_table_info.conf";
struct maat_options *opts = maat_options_new();
maat_options_set_iris(opts, rule_folder, rule_folder);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatPerfFileScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfFileScan::_shared_maat_instance;
struct log_handle *MaatPerfFileScan::logger;
struct perf_ip_plugin_ud {
long long rule_id;
int ref_cnt;
};
void perf_ip_plugin_EX_new_cb(const char *table_name, int table_id, const char *key,
const char *table_line, void **ad, long argl, void *argp)
{
int *counter=(int *)argp, ret=0;
size_t column_offset=0, column_len=0;
struct perf_ip_plugin_ud *ud = ALLOC(struct perf_ip_plugin_ud, 1);
ret = maat_helper_read_column(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->rule_id = atoll(table_line+column_offset);
ret = maat_helper_read_column(table_line, 5, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->ref_cnt = 1;
*ad = ud;
(*counter)++;
}
void perf_ip_plugin_EX_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct perf_ip_plugin_ud *u = (struct perf_ip_plugin_ud*)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) {
free(u);
*ad = NULL;
}
}
void perf_ip_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct perf_ip_plugin_ud *u = (struct perf_ip_plugin_ud*)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
static void *ip_plugin_get_thread(void *arg)
{
const char *table_name = "TSG_IP_LOCATION_BUILT_IN";
int test_times = 1000*1000, hit_times = 0;
int ret = 0, i=0, j=0;
struct maat *maat_instance = (struct maat *)arg;
int table_id = maat_get_table_id(maat_instance, table_name);
struct timespec start, end;
struct ip_addr ipv4;
ipv4.ip_type = IPv4;
inet_pton(AF_INET, "191.70.72.1", &ipv4.ipv4);
clock_gettime(CLOCK_MONOTONIC, &start);
struct perf_ip_plugin_ud *results[ARRAY_SIZE];
for (i = 0; i < test_times; i++) {
ret = maat_ip_plugin_table_get_ex_data(maat_instance, table_id, &ipv4, (void**)results, 4);
if (ret > 0) {
hit_times++;
}
for (j = 0; j < ret; j++) {
perf_ip_plugin_EX_free_cb(table_id, (void**)&(results[j]), 0, NULL);
}
}
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"ip_plugin_get_ex_data time_elapse:%lldms hit_times:%d",
time_elapse_ms, hit_times);
int *is_all_hit = (int *)malloc(sizeof(int));
*is_all_hit = (hit_times == test_times) ? 1 : 0;
return is_all_hit;
}
TEST_F(MaatPerfFileScan, IPPlugin) {
struct maat *maat_instance = MaatPerfFileScan::_shared_maat_instance;
const char* table_name = "TSG_IP_LOCATION_BUILT_IN";
int ip_plugin_ex_data_counter = 0;
int* is_all_hit = NULL;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name,
perf_ip_plugin_EX_new_cb,
perf_ip_plugin_EX_free_cb,
perf_ip_plugin_EX_dup_cb,
0, &ip_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
int i = 0;
pthread_t threads[PERF_THREAD_NUM];
for (i = 0; i < PERF_THREAD_NUM; i++) {
pthread_create(&(threads[i]), NULL, ip_plugin_get_thread, maat_instance);
}
for (i = 0; i < PERF_THREAD_NUM; i++) {
pthread_join(threads[i], (void**)&is_all_hit);
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit=0;
free(is_all_hit);
}
} }
int main(int argc, char ** argv) int main(int argc, char ** argv)

106
test/maat_json.json
View File

@@ -2836,6 +2836,112 @@
"group_name": "Untitled" "group_name": "Untitled"
} }
] ]
},
{
"compile_id": 206,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "duplicateRuleFor191",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "KEYWORDS_TABLE",
"table_content": {
"keywords": "54455354",
"expr_type": "none",
"format": "hexbin",
"match_method": "sub"
}
}
],
"group_name": "Untitled"
}
]
},
{
"compile_id": 207,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"user_region": "duplicateRuleFor192",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "flag",
"table_name": "FLAG_CONFIG",
"table_content": {
"flag": 1,
"flag_mask": 3
}
}
]
}
]
},
{
"compile_id": 208,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "duplicateRuleFor154",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv4",
"addr_format": "range",
"ip1": "10.0.7.100",
"ip2": "10.0.7.106",
"port_format": "range",
"port1": "65530",
"port2": "65535",
"protocol": 6
}
}
],
"not_flag": 0
}
]
},
{
"compile_id": 209,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "duplicateRuleFor179",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "INTERGER_PLUS",
"table_type": "interval_plus",
"table_content": {
"district": "interval.plus",
"low_boundary": 2020,
"up_boundary": 2020
}
}
]
}
]
} }
], ],
"plugin_table": [ "plugin_table": [

15
test/tsg_table_info.conf Normal file
View File

@@ -0,0 +1,15 @@
[
{
"table_id": 42,
"table_name":"TSG_IP_LOCATION_BUILT_IN",
"table_type":"ip_plugin",
"valid_column":18,
"custom": {
"item_id":1,
"ip_type":3,
"start_ip":4,
"end_ip":5,
"addr_format":7
}
}
]

1001
test/tsgrule/TSG_IP_LOCATION_BUILT_IN.head_1k Normal file
View File

File diff suppressed because it is too large Load Diff

2
test/tsgrule/full/index/full_config_index.00000000000000106967 Normal file
View File

@@ -0,0 +1,2 @@
TSG_IP_LOCATION_BUILT_IN 1000 ./tsgrule/TSG_IP_LOCATION_BUILT_IN.head_1k