gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix group_exclude logic miss & add some corner case

Browse Source
This commit is contained in:
刘文坛
2023-05-23 03:23:39 +00:00
parent b58ecc09e6
commit 464dc43cc4
29 changed files with 3317 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -223,9 +223,18 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd)
FREE(hs_cd->patterns[i]);
}
FREE(hs_cd->patterns);
FREE(hs_cd->patterns);
}
if (hs_cd->pattern_lens != NULL) {
FREE(hs_cd->pattern_lens);
}
if (hs_cd->ids != NULL) {
FREE(hs_cd->ids);
}
if (hs_cd->flags != NULL) {
FREE(hs_cd->flags);
}
@@ -290,8 +299,6 @@ struct bool_expr *bool_exprs_new(struct expr_rule *rules, size_t n_rule, struct
bool_exprs[i].items[j].item_id = pattern_index++;
bool_exprs[i].items[j].not_flag = 0;
// printf("item_id:%llu, pat:%s pat_len:%zu\n",
// bool_exprs[i].items[j].item_id, exprs[i].patterns[j].pat, exprs[i].patterns[j].pat_len);
}
bool_exprs[i].expr_id = rules[i].expr_id;
@@ -405,21 +412,6 @@ struct adapter_hs *adapter_hs_new(size_t n_worker_thread,
}
hs_instance->n_patterns = pattern_cnt;
//mytest
// for (size_t i = 0; i < n_expr; i++) {
// {
// printf("<before bool_matcher_new> exprs[%zu] expr_id:%llu, item_num:%zu\n",
// i, bool_exprs[i].expr_id, bool_exprs[i].item_num);
// printf("item_id: ");
// for (size_t j = 0; j < bool_exprs[i].item_num; j++)
// {
// printf("%llu ", bool_exprs[i].items[j].item_id);
// }
// }
// printf("\n");
// }
/* create bool matcher */
size_t mem_size = 0;
int hs_ret = 0;
@@ -502,7 +494,6 @@ void adapter_hs_free(struct adapter_hs *hs_instance)
}
FREE(hs_instance->hs_rt->literal_scratches);
hs_instance->hs_rt->literal_scratches = NULL;
}
@@ -515,7 +506,6 @@ void adapter_hs_free(struct adapter_hs *hs_instance)
}
FREE(hs_instance->hs_rt->regex_scratches);
hs_instance->hs_rt->regex_scratches = NULL;
}
if (hs_instance->hs_rt->bm != NULL) {

31
src/maat_api.c
View File

@@ -283,7 +283,6 @@ void _maat_free(struct maat *maat_instance)
if (maat_instance->opts.accept_tags != NULL) {
FREE(maat_instance->opts.accept_tags);
maat_instance->opts.accept_tags = NULL;
}
pthread_mutex_destroy(&(maat_instance->background_update_mutex));
@@ -529,20 +528,21 @@ int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
switch (table_type) {
case TABLE_TYPE_PLUGIN:
ret = plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func, dup_func,
free, argl, argp);
ret = plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func,
dup_func, free, argl, argp);
break;
case TABLE_TYPE_IP_PLUGIN:
ret = ip_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func, dup_func,
free, argl, argp);
ret = ip_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func,
dup_func, free, argl, argp);
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret = fqdn_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func, dup_func,
(void (*)(void *))fqdn_rule_free, argl, argp);
ret = fqdn_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func,
dup_func, (void (*)(void *))fqdn_rule_free,
argl, argp);
break;
case TABLE_TYPE_BOOL_PLUGIN:
ret = bool_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func, dup_func,
free, argl, argp);
ret = bool_plugin_table_set_ex_container_schema(schema, table_id, new_func, free_func,
dup_func, free, argl, argp);
break;
default:
log_error(logger, MODULE_MAAT_API,
@@ -720,18 +720,19 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance,
int ret = -1;
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (TABLE_TYPE_COMPILE == table_type) {
ret = compile_table_ex_schema_register(maat_instance, table_id,
new_func, free_func, dup_func, argl, argp);
ret = compile_table_ex_schema_register(maat_instance, table_id, new_func, free_func,
dup_func, argl, argp);
} else {
ret = generic_plugin_table_ex_schema_register(maat_instance, table_name, table_id,
new_func, free_func, dup_func, argl, argp);
ret = generic_plugin_table_ex_schema_register(maat_instance, table_name, table_id, new_func,
free_func, dup_func, argl, argp);
}
pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return ret;
}
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, const char *key)
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
const char *key)
{
if (NULL == maat_instance || table_id < 0 || table_id >= MAX_TABLE_NUM
@@ -768,7 +769,7 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, co
}
int maat_ip_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
const struct ip_addr *ip_addr, void **ex_data_array,
const struct ip_addr *ip_addr, void **ex_data_array,
size_t n_ex_data)
{
if (NULL == maat_instance || table_id < 0 || table_id >= MAX_TABLE_NUM

34
src/maat_bool_plugin.c
View File

@@ -56,7 +56,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table %s has no table_id column",
"[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -64,7 +64,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table %s has no custom column",
"[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -74,7 +74,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->item_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table %s has no item_id column",
"[%s:%d] table: <%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -84,7 +84,7 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->bool_expr_column = custom_item->valueint;
} else {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table %s has no bool_expr column",
"[%s:%d] table: <%s> schema has no bool_expr column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -185,7 +185,8 @@ void *bool_plugin_runtime_new(void *bool_plugin_schema, size_t max_thread_num,
bool_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger);
if (1 == schema->container_schema.set_flag) {
ex_data_runtime_set_ex_container_schema(bool_plugin_rt->ex_data_rt, &(schema->container_schema));
ex_data_runtime_set_ex_container_schema(bool_plugin_rt->ex_data_rt,
&(schema->container_schema));
}
bool_plugin_rt->n_worker_thread = max_thread_num;
@@ -254,8 +255,8 @@ int bool_plugin_runtime_update_row(struct bool_plugin_runtime *bool_plugin_rt,
return 0;
}
int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *line,
struct log_handle *logger)
int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *table_name,
const char *line, struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
@@ -266,8 +267,8 @@ int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table(table_id:%d) has no rule_tag, line:%s",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
@@ -278,12 +279,15 @@ int bool_plugin_accept_tag_match(struct bool_plugin_schema *schema, const char *
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table(table_id:%d) has invalid tag format, line:%s",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
}
@@ -296,7 +300,7 @@ struct bool_expr *
bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
const char *table_name, struct log_handle *logger)
{
int ret = bool_plugin_accept_tag_match(schema, line, logger);
int ret = bool_plugin_accept_tag_match(schema, table_name, line, logger);
if (ret == TAG_MATCH_UNMATCHED) {
return NULL;
}
@@ -312,7 +316,7 @@ bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table:%s line:%s has no item_id column",
"[%s:%d] table: <%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -321,7 +325,7 @@ bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
ret = get_column_pos(line, schema->bool_expr_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table:%s line:%s has no bool_expr column",
"[%s:%d] table: <%s> has no bool_expr in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -337,7 +341,7 @@ bool_plugin_expr_new(const char *line, struct bool_plugin_schema *schema,
n_item++;
if (ret != 1 || n_item > MAX_ITEMS_PER_BOOL_EXPR) {
log_error(logger, MODULE_BOOL_PLUGIN,
"[%s:%d] bool_plugin table:%s line:%s has invalid format of bool_expr column",
"[%s:%d] table: <%s> has invalid format of bool_expr in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}

95
src/maat_compile.c
View File

@@ -118,7 +118,6 @@ struct compile_runtime {
struct group2compile_runtime {
long long not_flag_group;
long long version;
long long rule_num;
long long update_err_cnt;
struct compile_runtime *ref_compile_rt;
@@ -229,7 +228,7 @@ void maat_compile_free(struct maat_compile *compile)
clause_state->in_use = 0;
}
compile->magic = 0;
free(compile);
FREE(compile);
}
void rcu_maat_compile_free(void *user_ctx, void *data)
@@ -328,14 +327,15 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
compile_schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no table_id column", table_name);
"[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no custom column",
"[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -345,7 +345,8 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
compile_schema->compile_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no compile_id column", table_name);
"[%s:%d] table: <%s> schema has no compile_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -359,7 +360,8 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
compile_schema->declared_clause_num_column = custom_item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no clause_num column", table_name);
"[%s:%d] table: <%s> schema has no clause_num column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -386,7 +388,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no table_id column",
"[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -396,7 +398,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->associated_compile_table_id = item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no associated_compile_table_id column",
"[%s:%d] table: <%s> schema has no associated_compile_table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -404,7 +406,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no custom column",
"[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -414,7 +416,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->group_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no group_id column",
"[%s:%d] table: <%s> schema has no group_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -424,7 +426,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->compile_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no compile_id column",
"[%s:%d] table: <%s> schema has no compile_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -434,7 +436,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->not_flag_column = custom_item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no not_flag column",
"[%s:%d] table: <%s> schema has no not_flag column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -444,7 +446,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->vtable_name_column = custom_item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no virtual_table_name column",
"[%s:%d] table: <%s> schema has no virtual_table_name column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -454,7 +456,7 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2c_schema->clause_index_column = custom_item->valueint;
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table %s has no clause_index column",
"[%s:%d] table: <%s> schema has no clause_index column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -490,7 +492,7 @@ int compile_accept_tag_match(struct compile_schema *schema, const char *line,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s has no rule_tag in line:%s",
"[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
@@ -502,12 +504,15 @@ int compile_accept_tag_match(struct compile_schema *schema, const char *line,
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s has invalid tag format in line:%s",
"[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
}
@@ -522,9 +527,6 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
{
int ret = compile_accept_tag_match(compile_schema, line, table_name, logger);
if (ret == TAG_MATCH_UNMATCHED) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s accept tag unmatched in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return NULL;
}
@@ -536,7 +538,7 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s has no compile_id in line:%s",
"[%s:%d] table: <%s> has no compile_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -546,7 +548,7 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] compile table:%s has no clause_num in line:%s",
"[%s:%d] table: <%s> has no clause_num in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -577,7 +579,8 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1);
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM);
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match,
max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM);
compile_rt->version = time(NULL);
compile_rt->cfg_hash_tbl = rcu_hash_new(rcu_maat_compile_free, NULL);
compile_rt->clause_by_literals_hash = NULL;
@@ -746,7 +749,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no group_id",
"[%s:%d] table: <%s> has no group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -755,7 +758,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no compile_id",
"[%s:%d] table: <%s> has no compile_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -764,7 +767,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no NOT_flag",
"[%s:%d] table: <%s> has no NOT_flag in line:%s ",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -773,14 +776,14 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
ret = get_column_pos(line, g2c_schema->vtable_name_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no virtual_table_name",
"[%s:%d] table: <%s> has no virtual_table_name in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
if (column_len > NAME_MAX) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s virtual_table_name length too long",
"[%s:%d] table: <%s> virtual_table_name length too long in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -791,8 +794,8 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
g2c_item->vtable_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr, vtable_name);
if (g2c_item->vtable_id < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s unknown virtual table:%s",
__FUNCTION__, __LINE__, table_name, line, vtable_name);
"[%s:%d] table: <%s> has unknown virtual table:%s in line:%s",
__FUNCTION__, __LINE__, table_name, vtable_name, line);
goto error;
}
}
@@ -800,7 +803,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
ret = get_column_pos(line, g2c_schema->clause_index_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] group2compile table:%s line:%s has no clause_index",
"[%s:%d] table: <%s> has no clause_index in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -960,7 +963,7 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
#if 0
struct maat_literal_id *p = NULL;
for(p = (struct maat_literal_id *)utarray_front(iter_compile->clause_states[i].ut_literal_ids); p!=NULL;
p=(struct maat_literal_id *)utarray_next(iter_compile->clause_states[i].ut_literal_ids,p)) {
p = (struct maat_literal_id *)utarray_next(iter_compile->clause_states[i].ut_literal_ids, p)) {
printf("<before bool_matcher_new> compile_rt:%p compile_id:%lld, clause_id:%llu, literal{%lld: %lld}\n",
compile_rt, iter_compile->compile_id, iter_compile->clause_states[i].clause_id, p->group_id, p->vtable_id);
}
@@ -986,8 +989,8 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
// STEP 3, build bool matcher
size_t mem_size = 0;
if (0 == expr_cnt) {
log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] No bool expression to build bool matcher.",
__FUNCTION__, __LINE__);
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] No bool expression to build bool matcher.", __FUNCTION__, __LINE__);
FREE(bool_expr_array);
return NULL;
}
@@ -995,7 +998,8 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
struct bool_matcher *bm = bool_matcher_new(bool_expr_array, expr_cnt, &mem_size);
if (bm != NULL) {
log_info(compile_rt->logger, MODULE_COMPILE,
"Build bool matcher of %zu expressions with %zu bytes memory.", expr_cnt, mem_size);
"Build bool matcher of %zu expressions with %zu bytes memory.",
expr_cnt, mem_size);
} else {
log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] Build bool matcher failed!",
__FUNCTION__, __LINE__);
@@ -1215,14 +1219,21 @@ void compile_rule_free(struct compile_rule *compile_rule)
struct compile_schema *schema = compile_rule->ref_schema;
assert(compile_rule->magic_num == COMPILE_RULE_MAGIC);
if (1 == schema->set_flag)
{
if (1 == schema->set_flag) {
rule_ex_data_free(schema->table_id, compile_rule->ex_data, &(schema->ex_schema));
*compile_rule->ex_data = NULL;
}
FREE(compile_rule->ex_data);
if (compile_rule->ex_data != NULL) {
FREE(compile_rule->ex_data);
}
compile_rule->declared_clause_num = -1;
FREE(compile_rule->table_line);
if (compile_rule->table_line != NULL) {
FREE(compile_rule->table_line);
}
FREE(compile_rule);
}
@@ -1468,7 +1479,7 @@ void maat_compile_state_free(struct maat_compile_state *compile_state)
compile_state->this_scan_hit_clauses = NULL;
}
free(compile_state);
FREE(compile_state);
}
static int maat_compile_hit_path_add(UT_array *hit_paths, long long item_id, long long group_id,
@@ -1533,7 +1544,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
size_t new_hit_path_cnt = 0;
struct maat_compile *compile = NULL;
struct maat_literal_id literal_id = {0, 0};
struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM;
struct bool_expr_match *expr_match = compile_rt->expr_match_buff +
(compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM);
assert(compile_state->thread_id >= 0);
if (compile_state->compile_rt_version != compile_rt->version) {
@@ -1910,7 +1922,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
group_ref = ALLOC(struct group_reference, 1);
group_ref->group_id = g2c_item->group_id;
HASH_ADD_KEYPTR(hh, compile_rt->group_ref_hash, (char *)&(group_ref->group_id),
sizeof(long long), group_ref);
sizeof(long long), group_ref);
}
group_ref->ref_by_compile_cnt++;
pthread_mutex_unlock(&(compile_rt->mutex));
@@ -1933,6 +1945,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
int group_referenced_by_compile(struct compile_runtime *compile_rt, long long group_id)
{
struct group_reference *group_ref = NULL;
pthread_mutex_lock(&(compile_rt->mutex));
HASH_FIND(hh, compile_rt->group_ref_hash, (char *)&group_id, sizeof(long long), group_ref);
if (group_ref != NULL) {

17
src/maat_expr.c
View File

@@ -444,6 +444,10 @@ void expr_rule_reset(struct expr_rule *rule)
void expr_item_free(struct expr_item *item)
{
if (NULL == item) {
return;
}
if (item->user_data != NULL &&
item->user_data_free != NULL) {
item->user_data_free(item->user_data);
@@ -737,15 +741,6 @@ int expr_item_to_expr_rule(struct expr_item *expr_item, struct expr_rule *expr_r
expr_rule->user_tag = expr_item->user_data;
expr_rule->n_patterns = sub_expr_cnt;
//mytest
// printf("expr_rule->expr_id:%u\n", expr_rule->expr_id);
// printf("expr_rule->n_patterns:%zu\n", expr_rule->n_patterns);
// for (size_t i = 0; i < expr_rule->n_patterns; i++) {
// printf("expr_rule->patterns[%zu].pat:%s\n", i, expr_rule->patterns[i].pat);
// printf("expr_rule->patterns[%zu].pat_len:%zu\n", i, expr_rule->patterns[i].pat_len);
// printf("expr_rule->patterns[%zu].l_offset:%d\n", i, expr_rule->patterns[i].l_offset);
// printf("expr_rule->patterns[%zu].r_offset:%d\n", i, expr_rule->patterns[i].r_offset);
// }
return 0;
}
@@ -876,8 +871,8 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name, long long ma
expr_rt->version = maat_rt_version;
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] commit %zu expr rules and rebuild adapter_hs completed, version:%lld",
table_name, rule_cnt, expr_rt->version);
"table[%s] has %zu rules, commit %zu expr rules and rebuild adapter_hs completed, version:%lld",
table_name, rule_cnt, real_rule_cnt, expr_rt->version);
if (rules != NULL) {
for (i = 0; i < rule_cnt; i++) {

1
src/maat_flag.c
View File

@@ -425,7 +425,6 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table
if (ret < 0) {
if (flag_item != NULL) {
flag_item_free(flag_item);
flag_item = NULL;
}
flag_rt->update_err_cnt++;
return -1;

55
src/maat_fqdn_plugin.c
View File

@@ -56,7 +56,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->table_id = item->valueint;
} else {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no table_id column",
"[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -64,7 +64,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no custom column",
"[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -74,7 +74,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->item_id_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no item_id column",
"[%s:%d] table: <%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -84,7 +84,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->suffix_flag_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no suffix_match_method column",
"[%s:%d] table: <%s> schema has no suffix_match_method column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -95,7 +95,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->fqdn_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table %s has no fqdn column",
"[%s:%d] table: <%s> schema has no fqdn column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -159,9 +159,15 @@ struct ex_container_schema *fqdn_plugin_table_get_ex_container_schema(void *fqdn
void fqdn_rule_free(struct FQDN_rule *fqdn_rule)
{
if (NULL == fqdn_rule) {
return;
}
assert(fqdn_rule->user_tag == NULL);
free(fqdn_rule->FQDN);
fqdn_rule->FQDN = NULL;
if (fqdn_rule->FQDN != NULL) {
FREE(fqdn_rule->FQDN);
}
free(fqdn_rule);
}
@@ -220,8 +226,8 @@ void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime)
FREE(fqdn_plugin_rt);
}
int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *line,
struct log_handle *logger)
int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *table_name,
const char *line, struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
@@ -232,8 +238,8 @@ int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table(table_id:%d) has no rule_tag, line:%s",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
@@ -244,12 +250,15 @@ int fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema, const char *
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table(table_id:%d) has invalid tag format, line:%s",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
}
@@ -262,7 +271,7 @@ struct FQDN_rule *
fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
const char *table_name, struct log_handle *logger)
{
int ret = fqdn_plugin_accept_tag_match(schema, line, logger);
int ret = fqdn_plugin_accept_tag_match(schema, table_name, line, logger);
if (ret == TAG_MATCH_UNMATCHED) {
return NULL;
}
@@ -276,7 +285,7 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:%s line:%s has no item_id column",
"[%s:%d] table: <%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -285,7 +294,7 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
ret = get_column_pos(line, schema->suffix_flag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:%s line:%s has no suffix_match_method column",
"[%s:%d] table: <%s> has no suffix_match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -294,7 +303,7 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
ret = get_column_pos(line, schema->fqdn_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:%s line:%s has no fqdn column",
"[%s:%d] table: <%s> has no fqdn in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -323,6 +332,10 @@ error:
void fqdn_plugin_rule_free(struct FQDN_rule *rule)
{
if (NULL == rule) {
return;
}
if (rule->FQDN != NULL) {
FREE(rule->FQDN);
}
@@ -386,7 +399,8 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche
if (1 == schema->container_schema.set_flag) {
if (1 == is_valid) {
// add
fqdn_plugin_rule = fqdn_plugin_rule_new(line, schema, table_name, fqdn_plugin_rt->logger);
fqdn_plugin_rule = fqdn_plugin_rule_new(line, schema, table_name,
fqdn_plugin_rt->logger);
if (NULL == fqdn_plugin_rule) {
fqdn_plugin_rt->update_err_cnt++;
return -1;
@@ -396,7 +410,7 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche
const char *key = line + item_id_offset;
size_t key_len = item_id_len;
ret = fqdn_plugin_runtime_update_row(fqdn_plugin_rt, table_name, line, key, key_len,
fqdn_plugin_rule, is_valid);
fqdn_plugin_rule, is_valid);
if (ret < 0) {
if (fqdn_plugin_rule != NULL) {
fqdn_plugin_rule_free(fqdn_plugin_rule);
@@ -419,7 +433,8 @@ void garbage_fqdn_engine_free(void *fqdn_engine, void *arg)
FQDN_engine_free(engine);
}
int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name, long long maat_rt_version)
int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name,
long long maat_rt_version)
{
if (NULL == fqdn_plugin_runtime) {
return -1;

21
src/maat_garbage_collection.c
View File

@@ -42,8 +42,12 @@ struct maat_garbage_bin *maat_garbage_bin_new(int default_timeout)
return bin;
}
void maat_garbage_bin_free(struct maat_garbage_bin* bin)
void maat_garbage_bin_free(struct maat_garbage_bin *bin)
{
if (NULL == bin) {
return;
}
struct maat_garbage_bag *p = NULL;
while ((p = TAILQ_FIRST(&bin->garbage_q)) != NULL) {
@@ -51,6 +55,7 @@ void maat_garbage_bin_free(struct maat_garbage_bin* bin)
if (p->arg != NULL) {
FREE(p->arg);
}
TAILQ_REMOVE(&bin->garbage_q, p, entries);
FREE(p);
bin->bag_cnt--;
@@ -59,12 +64,16 @@ void maat_garbage_bin_free(struct maat_garbage_bin* bin)
FREE(bin);
}
size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin)
size_t maat_garbage_bin_get_size(struct maat_garbage_bin *bin)
{
if (NULL == bin) {
return 0;
}
return bin->bag_cnt;
}
void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void *arg,
void maat_garbage_bagging(struct maat_garbage_bin *bin, void *garbage, void *arg,
void (* func)(void *, void *))
{
struct maat_garbage_bag *bag = ALLOC(struct maat_garbage_bag, 1);
@@ -77,7 +86,8 @@ void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void *arg
TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries);
bin->bag_cnt++;
}
void maat_garbage_collect_routine(struct maat_garbage_bin* bin)
void maat_garbage_collect_routine(struct maat_garbage_bin *bin)
{
struct maat_garbage_bag *p = NULL, *tmp = NULL;
size_t n_clollected = 0, n_bag = 0;
@@ -101,9 +111,10 @@ void maat_garbage_collect_routine(struct maat_garbage_bin* bin)
bin->bag_cnt -= n_clollected;
}
void maat_garbage_collect_by_force(struct maat_garbage_bin* bin)
void maat_garbage_collect_by_force(struct maat_garbage_bin *bin)
{
struct maat_garbage_bag *p = NULL;
while ((p = TAILQ_FIRST(&bin->garbage_q)) != NULL) {
p->garbage_free(p->garbage, p->arg);
if (p->arg != NULL) {

129
src/maat_group.c
View File

@@ -153,6 +153,10 @@ void group2group_schema_free(void *g2g_schema)
void group_vertex_free(struct maat_group *group)
{
if (NULL == group) {
return;
}
if (group->incl_super_group_ids != NULL) {
utarray_free(group->incl_super_group_ids);
group->incl_super_group_ids = NULL;
@@ -194,6 +198,10 @@ struct maat_group_topology *maat_group_topology_new(struct log_handle *logger)
void maat_group_topology_free(struct maat_group_topology *group_topo)
{
if (NULL == group_topo) {
return;
}
struct maat_group *group = NULL, *tmp_group = NULL;
HASH_CLEAR(hh_vertex_id, group_topo->hash_by_vertex_id);//No need group memory clean up.
@@ -770,7 +778,7 @@ void get_candidate_super_group_ids(struct maat_group_topology *group_topo,
{
long long *p = NULL;
//Find super candidates
//Find super candidates
for (p = (long long *)utarray_front(hit_group_ids); p != NULL;
p = (long long *)utarray_next(hit_group_ids, p)) {
struct maat_group *group = group_topology_find_group(group_topo, *p);
@@ -792,6 +800,74 @@ void get_candidate_super_group_ids(struct maat_group_topology *group_topo,
}
}
void verify_group_by_sub_include_groups(struct maat_group *group, UT_array *candidate_group_ids,
UT_array *kept_super_group_ids, UT_array *all_hit_group_ids)
{
size_t remove_idx = 0;
long long *tmp_id = NULL;
// delete groups whose all incl sub not in all_hit_group_ids
if (utarray_len(group->incl_sub_group_ids) != 0) {
int sub_incl_flag = 0;
for (tmp_id = (long long *)utarray_front(group->incl_sub_group_ids); tmp_id != NULL;
tmp_id = (long long *)utarray_next(group->incl_sub_group_ids, tmp_id)) {
if (utarray_find(candidate_group_ids, tmp_id, compare_group_id)) {
sub_incl_flag = 1;
break;
}
}
if (0 == sub_incl_flag) {
tmp_id = utarray_find(all_hit_group_ids, &(group->group_id), compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(all_hit_group_ids, tmp_id);
utarray_erase(all_hit_group_ids, remove_idx, 1);
}
tmp_id = utarray_find(kept_super_group_ids, &(group->group_id), compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(kept_super_group_ids, tmp_id);
utarray_erase(kept_super_group_ids, remove_idx, 1);
}
}
}
}
void verify_group_by_sub_exclude_groups(struct maat_group *group, UT_array *candidate_group_ids,
UT_array *kept_super_group_ids, UT_array *all_hit_group_ids)
{
if (0 == utarray_len(group->excl_sub_group_ids)) {
return;
}
// delete groups whose excl sub in all_hit_group_ids
int sub_excl_flag = 0;
long long *tmp_id = NULL;
for (tmp_id = (long long *)utarray_front(group->excl_sub_group_ids); tmp_id != NULL;
tmp_id = (long long *)utarray_next(group->excl_sub_group_ids, tmp_id)) {
if (utarray_find(candidate_group_ids, tmp_id, compare_group_id)) {
sub_excl_flag = 1;
break;
}
}
if (1 == sub_excl_flag) {
size_t remove_idx = 0;
tmp_id = utarray_find(all_hit_group_ids, &(group->group_id), compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(all_hit_group_ids, tmp_id);
utarray_erase(all_hit_group_ids, remove_idx, 1);
}
tmp_id = utarray_find(kept_super_group_ids, &(group->group_id), compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(kept_super_group_ids, tmp_id);
utarray_erase(kept_super_group_ids, remove_idx, 1);
}
}
}
void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
UT_array *candidate_super_group_ids,
UT_array *all_hit_group_ids,
@@ -802,6 +878,7 @@ void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
utarray_new(candidate_group_ids, &ut_group_id_icd);
/* merge this round of candidate super groups with hit groups from the previous round */
for (p = (long long *)utarray_front(candidate_super_group_ids); p != NULL;
p = (long long *)utarray_next(candidate_super_group_ids, p)) {
utarray_push_back(candidate_group_ids, p);
@@ -832,10 +909,11 @@ void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
}
}
//delete group_id from kept_group_ids
//kept super groups should not store this group
if (1 == sub_excl_flag) {
continue;
}
utarray_push_back(kept_super_group_ids, p);
utarray_sort(kept_super_group_ids, compare_group_id);
@@ -855,7 +933,8 @@ void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
}
/**
* delete groups whose all incl sub is non-exist in all_hit_group_ids
* 1. delete groups whose excl sub in all_hit_group_ids
* 2. delete groups whose all incl sub is non-exist in all_hit_group_ids
*/
for (p = (long long *)utarray_front(candidate_group_ids); p != NULL;
p = (long long *)utarray_next(candidate_group_ids, p)) {
@@ -864,38 +943,15 @@ void verify_candidate_super_group_ids(struct maat_group_topology *group_topo,
continue;
}
if (0 == utarray_len(group->incl_sub_group_ids)) {
continue;
}
int sub_incl_flag = 0;
long long *tmp_id = NULL;
for (tmp_id = (long long *)utarray_front(group->incl_sub_group_ids); tmp_id != NULL;
tmp_id = (long long *)utarray_next(group->incl_sub_group_ids, tmp_id)) {
if (utarray_find(candidate_group_ids, tmp_id, compare_group_id)) {
sub_incl_flag = 1;
break;
}
}
if (0 == sub_incl_flag) {
tmp_id = utarray_find(all_hit_group_ids, p, compare_group_id);
assert(tmp_id != NULL);
size_t remove_idx = utarray_eltidx(all_hit_group_ids, tmp_id);
utarray_erase(all_hit_group_ids, remove_idx, 1);
tmp_id = utarray_find(kept_super_group_ids, p, compare_group_id);
if (tmp_id != NULL) {
remove_idx = utarray_eltidx(kept_super_group_ids, tmp_id);
utarray_erase(kept_super_group_ids, remove_idx, 1);
}
}
verify_group_by_sub_exclude_groups(group, candidate_group_ids,
kept_super_group_ids, all_hit_group_ids);
verify_group_by_sub_include_groups(group, candidate_group_ids,
kept_super_group_ids, all_hit_group_ids);
}
utarray_free(candidate_group_ids);
}
#define MAX_RECURSION_DEPTH 5
void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_group_ids,
UT_array *all_hit_group_ids, size_t depth)
{
@@ -903,6 +959,8 @@ void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_g
UT_array *kept_super_group_ids;
if (depth >= MAX_RECURSION_DEPTH) {
log_error(group_topo->logger, MODULE_GROUP,
"[%s:%d]exceed max recursion depth(5)", __FUNCTION__, __LINE__);
return;
}
@@ -911,7 +969,7 @@ void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_g
/**
candidate super groups means all hit groups' super include group,
no need to consider super exclude groups
don't consider super exclude groups
for example:
hit_groups = {g4, g11}
g4's super include groups = {g7, g8}
@@ -941,9 +999,12 @@ void get_super_group_ids(struct maat_group_topology *group_topo, UT_array *hit_g
x \ / \ \
x \/ \ \
g3 g4 g5 g11
candidates = {g4, g11, g7, g8, g12}
g12's sub_exclude g8 in candidates, so g12 should be dropped
after verify candidates, all hit super groups = {g7, g8},
candidate super groups = {g7, g8, g12}
verify logic:
1. g12's sub_exclude g8 in candidates, so g12 should be dropped
2. g7 & g8, their sub_include in hit groups, so kept them
if their all sub_include not exist in hit groups, they should be dropped
after verify candidates, kept super groups = {g7, g8},
all hit groups = {g4, g11, g7, g8}
*/
verify_candidate_super_group_ids(group_topo, candidate_super_group_ids, all_hit_group_ids,

1
src/maat_interval.c
View File

@@ -423,7 +423,6 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
if (ret < 0) {
if (interval_item != NULL) {
interval_item_free(interval_item);
interval_item = NULL;
}
interval_rt->update_err_cnt++;
return -1;

1
src/maat_ip.c
View File

@@ -535,7 +535,6 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
if (ret < 0) {
if (ip_item != NULL) {
ip_item_free(ip_item);
ip_item = NULL;
}
ip_rt->update_err_cnt++;
return -1;

87
src/maat_ip_plugin.c
View File

@@ -51,7 +51,6 @@ struct ip_plugin_runtime {
void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
size_t read_cnt = 0;
struct ip_plugin_schema *schema = ALLOC(struct ip_plugin_schema, 1);
schema->logger = logger;
@@ -59,13 +58,17 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
schema->table_id = item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table %s has no custom column",
"[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -73,33 +76,54 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->item_id_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no item_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "ip_type");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->ip_type_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no ip_type column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "start_ip");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->start_ip_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no start_ip column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "end_ip");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->end_ip_column = custom_item->valueint;
read_cnt++;
} else {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> schema has no end_ip column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "addr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->addr_format_column = custom_item->valueint;
//TODO: just because test table not add this column
//read_cnt++;
}
}
//TODO: just because test table has no addr_format
// else {
// log_error(logger, MODULE_IP_PLUGIN,
// "[%s:%d] table: <%s> schema has no addr_format column",
// __FUNCTION__, __LINE__, table_name);
// goto error;
// }
// rule_tag is optional
custom_item = cJSON_GetObjectItem(item, "rule_tag");
@@ -108,10 +132,6 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
}
schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 5) {
goto error;
}
return schema;
error:
@@ -128,8 +148,8 @@ void ip_plugin_schema_free(void *ip_plugin_schema)
FREE(ip_plugin_schema);
}
int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line,
struct log_handle *logger)
int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *table_name,
const char *line, struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
@@ -140,8 +160,8 @@ int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table(table_id:%d) has no rule_tag, line:%s",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
@@ -152,12 +172,15 @@ int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table(table_id:%d) has invalid tag format, line:%s",
__FUNCTION__, __LINE__, schema->table_id, line);
"[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
}
@@ -170,7 +193,7 @@ struct ip_rule *
ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
const char *table_name, struct log_handle *logger)
{
int ret = ip_plugin_accept_tag_match(schema, line, logger);
int ret = ip_plugin_accept_tag_match(schema, table_name, line, logger);
if (ret == TAG_MATCH_UNMATCHED) {
return NULL;
}
@@ -185,7 +208,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip plugin table:%s line:%s has no item_id",
"[%s:%d] table: <%s> has no item_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -194,15 +217,15 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip plugin table:%s line:%s has no ip_type",
"[%s:%d] table: <%s> has no ip_type in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_plugin_rule->type = atoi(line + column_offset);
if (ip_plugin_rule->type != IPv4 && ip_plugin_rule->type != IPv6) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s ip_type[%d] invalid",
__FUNCTION__, __LINE__, table_name, line, ip_plugin_rule->type);
"[%s:%d] table: <%s> ip_type[%d] invalid in line:%s",
__FUNCTION__, __LINE__, table_name, ip_plugin_rule->type, line);
goto error;
}
@@ -230,7 +253,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s has no start_ip",
"[%s:%d] table: <%s> has no start_ip in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -239,7 +262,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s has no end_ip",
"[%s:%d] table: <%s> has no end_ip in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -252,7 +275,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
&ip_plugin_rule->ipv4_rule.end_ip);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s ip_format2range(ip4) failed",
"[%s:%d] table: <%s> ip_format2range(ip4) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -264,7 +287,7 @@ ip_plugin_rule_new(const char *line, struct ip_plugin_schema *schema,
ip_plugin_rule->ipv6_rule.end_ip);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table:%s line:%s ip_format2range(ip6) failed",
"[%s:%d] table: <%s> ip_format2range(ip6) failed in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
@@ -359,7 +382,8 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, size_t max_thread_num,
ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger);
if (1 == schema->container_schema.set_flag) {
ex_data_runtime_set_ex_container_schema(ip_plugin_rt->ex_data_rt, &(schema->container_schema));
ex_data_runtime_set_ex_container_schema(ip_plugin_rt->ex_data_rt,
&(schema->container_schema));
}
ip_plugin_rt->n_worker_thread = max_thread_num;
@@ -458,7 +482,8 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
return 0;
}
int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name, long long maat_rt_version)
int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
long long maat_rt_version)
{
if (NULL == ip_plugin_runtime) {
return -1;

15
src/maat_kv.c
View File

@@ -48,10 +48,15 @@ struct maat_kv_pair* maat_kv_pair_new(const char* key, size_t keylen, long long
return kv;
}
void maat_kv_pair_free(struct maat_kv_pair* kv)
void maat_kv_pair_free(struct maat_kv_pair *kv)
{
FREE(kv->key);
kv->key = NULL;
if (NULL == kv) {
return;
}
if (kv->key != NULL) {
FREE(kv->key);
}
FREE(kv);
}
@@ -62,7 +67,7 @@ struct maat_kv_store* maat_kv_store_new(void)
return store;
}
void maat_kv_store_free(struct maat_kv_store* store)
void maat_kv_store_free(struct maat_kv_store *store)
{
if (NULL == store) {
return;
@@ -79,7 +84,7 @@ void maat_kv_store_free(struct maat_kv_store* store)
FREE(store);
}
int maat_kv_register_unNull(struct maat_kv_store* store, const char *key, size_t keylen, long long value)
int maat_kv_register_unNull(struct maat_kv_store *store, const char *key, size_t keylen, long long value)
{
if (keylen > MAAT_KV_MAX_KEY_LEN) {
return -1;

27
src/maat_plugin.c
View File

@@ -87,7 +87,7 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table %s has no table_id column",
"[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -99,7 +99,7 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
custom_item = cJSON_GetObjectItem(item, "key");
if (NULL == custom_item || custom_item->type != cJSON_Number) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s has no key column",
"[%s:%d] table: <%s> schema has no key column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -108,7 +108,7 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
custom_item = cJSON_GetObjectItem(item, "key_type");
if (NULL == custom_item || custom_item->type != cJSON_String) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s has no key_type column",
"[%s:%d] table: <%s> schema has no key_type column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
@@ -119,9 +119,8 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->key_type = PLUGIN_KEY_TYPE_INTEGER;
} else {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table: %s key_type:%s illegal",
__FUNCTION__, __LINE__, table_name,
custom_item->valuestring);
"[%s:%d] table: <%s> schema key_type:%s is illegal, just allow {pointer} or {integer}",
__FUNCTION__, __LINE__, table_name, custom_item->valuestring);
goto error;
}
@@ -274,7 +273,8 @@ void *plugin_runtime_new(void *plugin_schema, size_t max_thread_num,
plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, logger);
if (1 == schema->container_schema.set_flag) {
ex_data_runtime_set_ex_container_schema(plugin_rt->ex_data_rt, &(schema->container_schema));
ex_data_runtime_set_ex_container_schema(plugin_rt->ex_data_rt,
&(schema->container_schema));
}
plugin_rt->ref_garbage_bin = garbage_bin;
@@ -319,7 +319,8 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, table_name,
row, key, key_len);
struct ex_container *ex_container = ex_container_new(ex_data, NULL);
ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container);
ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len,
ex_container);
if (ret < 0) {
return -1;
}
@@ -353,7 +354,7 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *table_name
int ret = get_column_pos(line, schema->rule_tag_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:%s has no rule_tag, line:%s",
"[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
@@ -365,12 +366,15 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *table_name
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] plugin table:%s has invalid tag format, line:%s",
"[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_error(logger, MODULE_PLUGIN,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
}
@@ -427,7 +431,8 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
return 0;
}
int plugin_runtime_commit(void *plugin_runtime, const char *table_name, long long maat_rt_version)
int plugin_runtime_commit(void *plugin_runtime, const char *table_name,
long long maat_rt_version)
{
if (NULL == plugin_runtime) {
return -1;

18
src/maat_redis_monitor.c
View File

@@ -411,16 +411,28 @@ int get_inc_key_list(long long instance_version, long long target_version,
void serial_rule_free(struct serial_rule *s_rule)
{
if (NULL == s_rule) {
return;
}
if (s_rule->table_line != NULL) {
FREE(s_rule->table_line);
}
if (s_rule->n_foreign > 0) {
for (int i = 0; i < s_rule->n_foreign; i++) {
FREE(s_rule->f_keys[i].filename);
FREE(s_rule->f_keys[i].key);
if (s_rule->f_keys[i].filename != NULL) {
FREE(s_rule->f_keys[i].filename);
}
if (s_rule->f_keys[i].key != NULL) {
FREE(s_rule->f_keys[i].key);
}
}
FREE(s_rule->f_keys);
if (s_rule->f_keys != NULL) {
FREE(s_rule->f_keys);
}
}
FREE(s_rule);

2
src/maat_rule.c
View File

@@ -55,7 +55,7 @@ void maat_item_inner_free(void *item_inner)
assert(item->magic_num == ITEM_RULE_MAGIC);
item->magic_num = 0;
free(item);
FREE(item);
}
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)

5
src/maat_stat.c
View File

@@ -185,7 +185,7 @@ void maat_stat_free(struct maat_stat *stat)
stat->fs_handle = NULL;
}
free(stat);
FREE(stat);
}
int maat_stat_init(struct maat_stat *stat, struct table_manager *tbl_mgr,
@@ -287,8 +287,7 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
break;
}
if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_COMPILE ||
table_type == TABLE_TYPE_GROUP2COMPILE || table_type == TABLE_TYPE_GROUP2GROUP) {
if (table_type == TABLE_TYPE_PLUGIN ) {
continue;
}

2
test/CMakeLists.txt
View File

@@ -30,12 +30,14 @@ target_link_libraries(maat_ex_data_gtest maat_frame_static gtest_static)
add_subdirectory(group_exclude)
file(COPY table_info.conf DESTINATION ./)
file(COPY tsg_table_info.conf DESTINATION ./)
file(COPY file_test_tableinfo.conf DESTINATION ./)
file(COPY literal_expr.conf DESTINATION ./)
file(COPY regex_expr.conf DESTINATION ./)
file(COPY maat_json.json DESTINATION ./)
file(COPY maat_json.json DESTINATION ../tools/)
file(COPY ntcrule DESTINATION ./)
file(COPY tsgrule DESTINATION ./)
file(COPY testdata DESTINATION ./)
file(COPY test_streamfiles DESTINATION ./)
file(COPY json_update DESTINATION ./)

136
test/group_exclude/group_exclude_L4.conf Normal file
View File

@@ -0,0 +1,136 @@
{
"add_items": [
{
"group_id": 14,
"super_group_id": 8,
"is_exclude": 0
},
{
"group_id": 14,
"super_group_id": 9,
"is_exclude": 0
},
{
"group_id": 8,
"super_group_id": 5,
"is_exclude": 0
},
{
"group_id": 9,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 16,
"super_group_id": 9,
"is_exclude": 0
},
{
"group_id": 16,
"super_group_id": 10,
"is_exclude": 0
},
{
"group_id": 10,
"super_group_id": 7,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 6,
"is_exclude": 0
},
{
"group_id": 6,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 17,
"super_group_id": 11,
"is_exclude": 0
},
{
"group_id": 17,
"super_group_id": 12,
"is_exclude": 0
},
{
"group_id": 17,
"super_group_id":13,
"is_exclude": 0
},
{
"group_id": 11,
"super_group_id": 6,
"is_exclude": 0
},
{
"group_id": 12,
"super_group_id": 7,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 2,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 4,
"is_exclude": 0
},
{
"group_id": 13,
"super_group_id": 3,
"is_exclude": 0
},
{
"group_id": 15,
"super_group_id": 8,
"is_exclude": 1
},
{
"group_id": 9,
"super_group_id": 5,
"is_exclude": 1
},
{
"group_id": 5,
"super_group_id": 1,
"is_exclude": 1
},
{
"group_id": 10,
"super_group_id": 6,
"is_exclude": 1
},
{
"group_id": 6,
"super_group_id": 2,
"is_exclude": 1
},
{
"group_id": 6,
"super_group_id": 3,
"is_exclude": 1
},
{
"group_id": 18,
"super_group_id":13,
"is_exclude": 1
},
{
"group_id": 13,
"super_group_id": 4,
"is_exclude": 1
}
],
"del_items": [
{
"group_id": 10,
"super_group_id": 6,
"is_exclude": 1
}
]
}

BIN
test/group_exclude/group_exclude_L4.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 74 KiB

76
test/group_exclude/group_exclude_exceed.conf Normal file
View File

@@ -0,0 +1,76 @@
{
"add_items": [
{
"group_id": 6,
"super_group_id": 3,
"is_exclude": 0
},
{
"group_id": 3,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 1,
"is_exclude": 0
},
{
"group_id": 7,
"super_group_id": 3,
"is_exclude": 1
},
{
"group_id": 13,
"super_group_id": 10,
"is_exclude": 0
},
{
"group_id": 10,
"super_group_id": 11,
"is_exclude": 0
},
{
"group_id": 11,
"super_group_id": 8,
"is_exclude": 0
},
{
"group_id": 8,
"super_group_id": 5,
"is_exclude": 0
},
{
"group_id": 5,
"super_group_id": 4,
"is_exclude": 0
},
{
"group_id": 5,
"super_group_id": 12,
"is_exclude": 0
},
{
"group_id": 4,
"super_group_id": 2,
"is_exclude": 0
},
{
"group_id": 4,
"super_group_id": 1,
"is_exclude": 1
},
{
"group_id": 2,
"super_group_id": 12,
"is_exclude": 1
}
],
"del_items": [
{
"group_id": 13,
"super_group_id": 10,
"is_exclude": 0
}
]
}

BIN
test/group_exclude/group_exclude_exceed.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 46 KiB

303
test/group_exclude/group_exclude_gtest.cpp
View File

@@ -11,6 +11,7 @@
#define MAX_ITEM_NUM 64
#define WAIT_FOR_EFFECTIVE_S 2
#define MAX_G2G_SCAN_TIMES (1000 * 1000)
const char *table_info_path = "./table_info.conf";
@@ -138,38 +139,34 @@ protected:
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_schema_new failed.");
assert(0);
}
g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
}
static void TearDownTestCase() {
log_handle_destroy(logger);
maat_garbage_bin_free(garbage_bin);
group2group_runtime_free(g2g_runtime);
group2group_schema_free(g2g_schema);
}
static void *g2g_schema;
static void *g2g_runtime;
static struct log_handle *logger;
static struct maat_garbage_bin *garbage_bin;
};
void *MaatGroupExclude::g2g_schema;
void *MaatGroupExclude::g2g_runtime;
struct log_handle *MaatGroupExclude::logger;
struct maat_garbage_bin *MaatGroupExclude::garbage_bin;
TEST_F(MaatGroupExclude, level_3_basic) {
TEST_F(MaatGroupExclude, level_3_function) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_L3.conf", &rules);
EXPECT_EQ(ret, 0);
@@ -199,7 +196,7 @@ TEST_F(MaatGroupExclude, level_3_basic) {
group2group_runtime_commit(g2g_runtime, table_name, 2);
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 3);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 6);
@@ -213,12 +210,290 @@ TEST_F(MaatGroupExclude, level_3_basic) {
group2group_runtime_commit(g2g_runtime, table_name, 3);
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 4);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 6);
EXPECT_EQ(super_group_ids[3], 7);
group2group_runtime_free(g2g_runtime);
}
TEST_F(MaatGroupExclude, level_3_perf) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_L3.conf", &rules);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < rules.n_add_item; i++) {
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.add_items[i].group_id,
rules.add_items[i].super_group_id, rules.add_items[i].is_exclude, 1);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
}
group2group_runtime_commit(g2g_runtime, table_name, 4);
long long hit_group_ids[2] = {11, 13};
long long super_group_ids[MAX_ITEM_NUM];
size_t super_group_cnt = 0;
struct timespec start, end;
struct log_handle *logger = MaatGroupExclude::logger;
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 2);
EXPECT_EQ(super_group_ids[0], 2);
EXPECT_EQ(super_group_ids[1], 7);
}
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_3_basic hit 2 super_groups scan consume time %lldms",
time_elapse_ms);
//delete group_id = 7, super_group_id = 6, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[0].group_id,
rules.del_items[0].super_group_id, rules.del_items[0].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 5);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 3);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 6);
EXPECT_EQ(super_group_ids[2], 7);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_3_basic hit 3 super_groups scan consume time %lldms",
time_elapse_ms);
//delete group_id = 13, super_group_id = 4, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[1].group_id,
rules.del_items[1].super_group_id, rules.del_items[1].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 6);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 4);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 6);
EXPECT_EQ(super_group_ids[3], 7);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_3_basic hit 4 super_groups scan consume time %lldms",
time_elapse_ms);
group2group_runtime_free(g2g_runtime);
}
TEST_F(MaatGroupExclude, level_4_function) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_L4.conf", &rules);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < rules.n_add_item; i++) {
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.add_items[i].group_id,
rules.add_items[i].super_group_id, rules.add_items[i].is_exclude, 1);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
}
group2group_runtime_commit(g2g_runtime, table_name, 7);
long long hit_group_ids[2] = {14, 16};
long long super_group_ids[MAX_ITEM_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 5);
EXPECT_EQ(super_group_ids[0], 4);
EXPECT_EQ(super_group_ids[1], 7);
EXPECT_EQ(super_group_ids[2], 8);
EXPECT_EQ(super_group_ids[3], 9);
EXPECT_EQ(super_group_ids[4], 10);
//delete group_id = 10, super_group_id = 6, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[0].group_id,
rules.del_items[0].super_group_id, rules.del_items[0].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 8);
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 7);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 6);
EXPECT_EQ(super_group_ids[3], 7);
EXPECT_EQ(super_group_ids[4], 8);
EXPECT_EQ(super_group_ids[5], 9);
EXPECT_EQ(super_group_ids[6], 10);
group2group_runtime_free(g2g_runtime);
}
TEST_F(MaatGroupExclude, level_4_perf) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_L4.conf", &rules);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < rules.n_add_item; i++) {
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.add_items[i].group_id,
rules.add_items[i].super_group_id, rules.add_items[i].is_exclude, 1);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
}
group2group_runtime_commit(g2g_runtime, table_name, 7);
long long hit_group_ids[2] = {14, 16};
long long super_group_ids[MAX_ITEM_NUM];
size_t super_group_cnt = 0;
struct timespec start, end;
struct log_handle *logger = MaatGroupExclude::logger;
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 5);
EXPECT_EQ(super_group_ids[0], 4);
EXPECT_EQ(super_group_ids[1], 7);
EXPECT_EQ(super_group_ids[2], 8);
EXPECT_EQ(super_group_ids[3], 9);
EXPECT_EQ(super_group_ids[4], 10);
}
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_4_basic hit 5 super_groups scan consume time %lldms",
time_elapse_ms);
//delete group_id = 10, super_group_id = 6, is_exclude = 1
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[0].group_id,
rules.del_items[0].super_group_id, rules.del_items[0].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 8);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < MAX_G2G_SCAN_TIMES; i++) {
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 7);
EXPECT_EQ(super_group_ids[0], 1);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 6);
EXPECT_EQ(super_group_ids[3], 7);
EXPECT_EQ(super_group_ids[4], 8);
EXPECT_EQ(super_group_ids[5], 9);
EXPECT_EQ(super_group_ids[6], 10);
}
clock_gettime(CLOCK_MONOTONIC, &end);
time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(logger, MODULE_GROUP_EXCLUDE_GTEST, "level_4_basic hit 7 super_groups scan consume time %lldms",
time_elapse_ms);
group2group_runtime_free(g2g_runtime);
}
TEST_F(MaatGroupExclude, level_exceed_function) {
char table_line[1024];
struct group2group_rule rules;
const char *table_name = "EXCLUDE_GROUP2GROUP";
void *g2g_runtime = group2group_runtime_new(g2g_schema, 1, garbage_bin, logger);
if (NULL == g2g_runtime) {
log_error(logger, MODULE_GROUP_EXCLUDE_GTEST, "group2group_runtime_new failed.");
assert(0);
}
int ret = parse_config_file("group_exclude_exceed.conf", &rules);
EXPECT_EQ(ret, 0);
for (size_t i = 0; i < rules.n_add_item; i++) {
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.add_items[i].group_id,
rules.add_items[i].super_group_id, rules.add_items[i].is_exclude, 1);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
}
group2group_runtime_commit(g2g_runtime, table_name, 8);
long long hit_group_ids[2] = {7, 13};
long long super_group_ids[MAX_ITEM_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 6);
EXPECT_EQ(super_group_ids[0], 4);
EXPECT_EQ(super_group_ids[1], 5);
EXPECT_EQ(super_group_ids[2], 8);
EXPECT_EQ(super_group_ids[3], 10);
EXPECT_EQ(super_group_ids[4], 11);
EXPECT_EQ(super_group_ids[5], 12);
//delete group_id = 13, super_group_id = 10, is_exclude = 0
memset(table_line, 0, sizeof(table_line));
sprintf(table_line, "%lld\t%lld\t%d\t%d", rules.del_items[0].group_id,
rules.del_items[0].super_group_id, rules.del_items[0].is_exclude, 0);
group2group_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
group2group_runtime_commit(g2g_runtime, table_name, 9);
hit_group_ids[0] = 7;
hit_group_ids[1] = 10;
super_group_cnt = group2group_runtime_get_super_groups(g2g_runtime, hit_group_ids, 2,
super_group_ids, MAX_ITEM_NUM);
EXPECT_EQ(super_group_cnt, 5);
EXPECT_EQ(super_group_ids[0], 2);
EXPECT_EQ(super_group_ids[1], 4);
EXPECT_EQ(super_group_ids[2], 5);
EXPECT_EQ(super_group_ids[3], 8);
EXPECT_EQ(super_group_ids[4], 11);
group2group_runtime_free(g2g_runtime);
}
int main(int argc, char ** argv)
@@ -227,4 +502,4 @@ int main(int argc, char ** argv)
::testing::InitGoogleTest(&argc, argv);
ret=RUN_ALL_TESTS();
return ret;
}
}

796
test/maat_framework_gtest.cpp
View File

File diff suppressed because it is too large Load Diff

758
test/maat_framework_perf_gtest.cpp
View File

@@ -294,7 +294,8 @@ class MaatPerfStringScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
@@ -308,7 +309,7 @@ protected:
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
@@ -362,7 +363,7 @@ void *perf_string_scan_thread(void *arg)
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d time_elapse:%lldms hit_times:%d",
"thread_id:%d string_scan time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
@@ -421,7 +422,7 @@ void *perf_ip_scan_thread(void *arg)
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d time_elapse:%lldms hit_times:%d",
"thread_id:%d ip_scan time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
@@ -456,7 +457,7 @@ void *perf_ip_update_thread(void *arg)
return is_all_hit;
}
TEST_F(MaatPerfStringScan, basic) {
TEST_F(MaatPerfStringScan, MultiThread) {
const char *table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = MaatPerfStringScan::_shared_maat_instance;
@@ -498,14 +499,169 @@ TEST_F(MaatPerfStringScan, basic) {
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
//EXPECT_GT(scan_per_second, 800 * 1000);
printf("High match rate on %d-threads speed %lld lookups/s/thread\n", PERF_THREAD_NUM, scan_per_second);
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"StringScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfStreamScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfStreamScan::_shared_maat_instance;
struct log_handle *MaatPerfStreamScan::logger;
void *perf_stream_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const char *table_name = param->table_name;
struct timespec start, end;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567";
long long results[ARRAY_SIZE] = {0};
int ret = 0, hit_times = 0;
size_t n_hit_result = 0;
struct maat_state *state_array[ARRAY_SIZE];
struct maat_stream *sp[ARRAY_SIZE];
int table_id = maat_get_table_id(maat_instance, table_name);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) {
for (int j = 0; j < ARRAY_SIZE; j++) {
state_array[j] = maat_state_new(maat_instance, param->thread_id);
sp[j] = maat_stream_new(maat_instance, table_id, state_array[j]);
ret = maat_stream_scan(sp[j], scan_data, strlen(scan_data), results, ARRAY_SIZE,
&n_hit_result, state_array[j]);
if (ret == MAAT_SCAN_HIT) {
hit_times++;
}
maat_stream_free(sp[j]);
maat_state_free(state_array[j]);
}
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = ((hit_times == param->test_count*ARRAY_SIZE) ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d stream_scan time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void *perf_stream_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10;
char keyword_buf[128];
for (int i = 0; i < CMD_EXPR_NUM; i++) {
random_keyword_generate(keyword_buf, sizeof(keyword_buf));
test_add_expr_command(maat_instance, table_name, keyword_buf);
sleep(1);
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;
return is_all_hit;
}
TEST_F(MaatPerfStreamScan, MultiThread) {
const char *table_name = "HTTP_URL";
struct maat *maat_instance = MaatPerfStreamScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = 100 * 1000;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_stream_scan_thread, thread_params+i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_stream_update_thread, thread_params+i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
//maybe expr_runtime rebuild in stream_scan, so should not expect is_all_hit always 1
//EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"StreamScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfIPScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
@@ -544,7 +700,7 @@ protected:
struct maat *MaatPerfIPScan::_shared_maat_instance;
struct log_handle *MaatPerfIPScan::logger;
TEST_F(MaatPerfIPScan, basic)
TEST_F(MaatPerfIPScan, MultiThread)
{
const char *table_name = "IP_PLUS_CONFIG";
struct maat *maat_instance = MaatPerfIPScan::_shared_maat_instance;
@@ -561,7 +717,7 @@ TEST_F(MaatPerfIPScan, basic)
thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = 1 * 1000 * 1000;
thread_params[i].test_count = 10 * 1000 * 1000;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
@@ -586,8 +742,588 @@ TEST_F(MaatPerfIPScan, basic)
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
//EXPECT_GT(scan_per_second, 800 * 1000);
printf("High match rate on %d-threads speed %lld lookups/s/thread\n", PERF_THREAD_NUM, scan_per_second);
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"IPScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfFQDNPluginScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfFQDNPluginScan::_shared_maat_instance;
struct log_handle *MaatPerfFQDNPluginScan::logger;
struct perf_fqdn_plugin_ud {
long long rule_id;
int catid;
int ref_cnt;
};
void perf_fqdn_plugin_EX_new_cb(const char *table_name, int table_id, const char *key,
const char *table_line, void **ad, long argl, void *argp)
{
int *counter=(int *)argp;
size_t column_offset=0, column_len=0;
struct perf_fqdn_plugin_ud *ud = ALLOC(struct perf_fqdn_plugin_ud, 1);
int ret = maat_helper_read_column(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->rule_id = atoll(table_line+column_offset);
ret = maat_helper_read_column(table_line, 4, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
sscanf(table_line+column_offset, "catid=%d",&ud->catid);
ud->ref_cnt = 1;
*ad = ud;
(*counter)++;
}
void perf_fqdn_plugin_EX_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct perf_fqdn_plugin_ud *u = (struct perf_fqdn_plugin_ud *)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) {
free(u);
*ad = NULL;
}
}
void perf_fqdn_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct perf_fqdn_plugin_ud *u = (struct perf_fqdn_plugin_ud *)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
void* perf_fqdn_plugin_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
struct perf_fqdn_plugin_ud *result[ARRAY_SIZE];
int i=0, j=0, ret=0, hit_times=0;
int table_id = maat_get_table_id(maat_instance, param->table_name);
memset(&result, 0, sizeof(result));
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < param->test_count; i++) {
ret = maat_fqdn_plugin_table_get_ex_data(maat_instance, table_id,
"r3---sn-i3belne6.example2.com",
(void**)result, ARRAY_SIZE);
if (ret == 2) {
hit_times++;
}
for (j = 0; j < ret; j++) {
perf_fqdn_plugin_EX_free_cb(0, (void**)&(result[j]), 0, NULL);
}
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec-start.tv_sec)*1000 + (end.tv_nsec-start.tv_nsec)/1000000;
int* is_all_hit = (int*)malloc(sizeof(int));
*is_all_hit = (hit_times == param->test_count) ? 1 : 0;
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d fqdn_plugin_get_ex_data time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void random_fqdn_generate(char *fqdn_buff, int sz)
{
int min_fqdn_len = 6, max_fqdn_len = 32;
int i=0, j=0;
int len = random() % (max_fqdn_len - min_fqdn_len) + min_fqdn_len;
if (len >= sz) {
len = sz - 1;
}
for (i = 0; i < len-4; i++) {
fqdn_buff[i] = 'a' + random() % ('z' - 'a');
if (j > 5) {
if (random() % 3 == 0) {
fqdn_buff[i] = '.';
j = 0;
}
}
j++;
}
fqdn_buff[i] = '\0';
}
void *perf_fqdn_plugin_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const int CMD_EXPR_NUM = 20;
int i = 0;
struct maat_cmd_line line_rule;
char line_buff[1024], fqdn_buff[256];
for (i = 0; i < CMD_EXPR_NUM; i++) {
line_rule.rule_id = (int)maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1);
line_rule.table_name = param->table_name;
random_fqdn_generate(fqdn_buff, sizeof(fqdn_buff));
snprintf(line_buff, 1024, "%lld\t1\t%s\tcatid=4\t1", line_rule.rule_id, fqdn_buff);
line_rule.table_line = line_buff;
line_rule.expire_after = 0;
maat_cmd_set_line(maat_instance, &line_rule);
sleep(WAIT_FOR_EFFECTIVE_S);
}
int *is_all_hit = (int *)malloc(sizeof(int));
*is_all_hit = 1;
return is_all_hit;
}
TEST_F(MaatPerfFQDNPluginScan, MultiThread) {
struct maat *maat_instance = MaatPerfFQDNPluginScan::_shared_maat_instance;
const char *table_name = "TEST_FQDN_PLUGIN_WITH_EXDATA";
int fqdn_plugin_ex_data_counter = 0;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name,
perf_fqdn_plugin_EX_new_cb,
perf_fqdn_plugin_EX_free_cb,
perf_fqdn_plugin_EX_dup_cb,
0, &fqdn_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = 1 * 1000 * 1000;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_fqdn_plugin_scan_thread, thread_params + i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_fqdn_plugin_update_thread, thread_params + i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"FQDNPluginScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfBoolPluginScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in BoolPluginScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfBoolPluginScan::_shared_maat_instance;
struct log_handle *MaatPerfBoolPluginScan::logger;
struct bool_plugin_ud {
int id;
char *name;
int ref_cnt;
};
void perf_bool_plugin_ex_new_cb(const char *table_name, int table_id, const char *key,
const char *table_line, void **ad, long argl, void *argp)
{
int *counter=(int *)argp;
size_t column_offset=0, column_len=0;
struct bool_plugin_ud *ud = ALLOC(struct bool_plugin_ud, 1);
int ret = get_column_pos(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->id = atoi(table_line + column_offset);
ret = get_column_pos(table_line, 3, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->name = (char *)malloc(column_len+1);
memcpy(ud->name, table_line+column_offset, column_len);
ud->ref_cnt = 1;
*ad = ud;
(*counter)++;
}
void perf_bool_plugin_ex_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct bool_plugin_ud *u = (struct bool_plugin_ud *)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
{
free(u->name);
free(u);
*ad = NULL;
}
}
void perf_bool_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct bool_plugin_ud *u = (struct bool_plugin_ud *)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
void* perf_bool_plugin_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
struct perf_fqdn_plugin_ud *result[ARRAY_SIZE];
int i=0, j=0, ret=0, hit_times=0;
int table_id = maat_get_table_id(maat_instance, param->table_name);
memset(&result, 0, sizeof(result));
struct timespec start, end;
unsigned long long items_4[]={7, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7};
clock_gettime(CLOCK_MONOTONIC, &start);
for (i = 0; i < param->test_count; i++) {
ret = maat_bool_plugin_table_get_ex_data(maat_instance, table_id, items_4,
sizeof(items_4)/sizeof(unsigned long long),
(void**)result, 6);
if (ret == 1) {
hit_times++;
}
for (j = 0; j < ret; j++) {
perf_bool_plugin_ex_free_cb(0, (void**)&(result[j]), 0, NULL);
}
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec-start.tv_sec)*1000 + (end.tv_nsec-start.tv_nsec)/1000000;
int* is_all_hit = (int*)malloc(sizeof(int));
*is_all_hit = (hit_times == param->test_count) ? 1 : 0;
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d bool_plugin_get_ex_data time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void *perf_bool_plugin_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const int CMD_EXPR_NUM = 20;
int i = 0;
struct maat_cmd_line line_rule;
char line_buff[1024];
for (i = 0; i < CMD_EXPR_NUM; i++) {
line_rule.rule_id = (int)maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1);
line_rule.table_name = param->table_name;
snprintf(line_buff, 1024, "%lld\t1&%d\ttunnel2\t1", line_rule.rule_id, i);
line_rule.table_line = line_buff;
line_rule.expire_after = 0;
maat_cmd_set_line(maat_instance, &line_rule);
sleep(WAIT_FOR_EFFECTIVE_S);
}
int *is_all_hit = (int *)malloc(sizeof(int));
*is_all_hit = 1;
return is_all_hit;
}
TEST_F(MaatPerfBoolPluginScan, MultiThread) {
int ex_data_counter = 0;
const char *table_name = "TEST_BOOL_PLUGIN_WITH_EXDATA";
struct maat *maat_instance = MaatPerfBoolPluginScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name,
perf_bool_plugin_ex_new_cb,
perf_bool_plugin_ex_free_cb,
perf_bool_plugin_ex_dup_cb,
0, &ex_data_counter);
ASSERT_TRUE(ret >= 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = 1 * 1000 * 1000;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_bool_plugin_scan_thread, thread_params + i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_bool_plugin_update_thread, thread_params + i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"BoolPluginScan match rate on %d-threads speed %lld lookups/s/thread",
PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfFileScan : public testing::Test
{
protected:
static void SetUpTestCase() {
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
const char *rule_folder = "./tsgrule/full/index";
const char *table_info = "./tsg_table_info.conf";
struct maat_options *opts = maat_options_new();
maat_options_set_iris(opts, rule_folder, rule_folder);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatPerfFileScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfFileScan::_shared_maat_instance;
struct log_handle *MaatPerfFileScan::logger;
struct perf_ip_plugin_ud {
long long rule_id;
int ref_cnt;
};
void perf_ip_plugin_EX_new_cb(const char *table_name, int table_id, const char *key,
const char *table_line, void **ad, long argl, void *argp)
{
int *counter=(int *)argp, ret=0;
size_t column_offset=0, column_len=0;
struct perf_ip_plugin_ud *ud = ALLOC(struct perf_ip_plugin_ud, 1);
ret = maat_helper_read_column(table_line, 1, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->rule_id = atoll(table_line+column_offset);
ret = maat_helper_read_column(table_line, 5, &column_offset, &column_len);
EXPECT_EQ(ret, 0);
ud->ref_cnt = 1;
*ad = ud;
(*counter)++;
}
void perf_ip_plugin_EX_free_cb(int table_id, void **ad, long argl, void *argp)
{
struct perf_ip_plugin_ud *u = (struct perf_ip_plugin_ud*)(*ad);
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) {
free(u);
*ad = NULL;
}
}
void perf_ip_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
{
struct perf_ip_plugin_ud *u = (struct perf_ip_plugin_ud*)(*from);
__sync_add_and_fetch(&(u->ref_cnt), 1);
*to = u;
}
static void *ip_plugin_get_thread(void *arg)
{
const char *table_name = "TSG_IP_LOCATION_BUILT_IN";
int test_times = 1000*1000, hit_times = 0;
int ret = 0, i=0, j=0;
struct maat *maat_instance = (struct maat *)arg;
int table_id = maat_get_table_id(maat_instance, table_name);
struct timespec start, end;
struct ip_addr ipv4;
ipv4.ip_type = IPv4;
inet_pton(AF_INET, "191.70.72.1", &ipv4.ipv4);
clock_gettime(CLOCK_MONOTONIC, &start);
struct perf_ip_plugin_ud *results[ARRAY_SIZE];
for (i = 0; i < test_times; i++) {
ret = maat_ip_plugin_table_get_ex_data(maat_instance, table_id, &ipv4, (void**)results, 4);
if (ret > 0) {
hit_times++;
}
for (j = 0; j < ret; j++) {
perf_ip_plugin_EX_free_cb(table_id, (void**)&(results[j]), 0, NULL);
}
}
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
log_info(maat_instance->logger, MODULE_FRAMEWORK_PERF_GTEST,
"ip_plugin_get_ex_data time_elapse:%lldms hit_times:%d",
time_elapse_ms, hit_times);
int *is_all_hit = (int *)malloc(sizeof(int));
*is_all_hit = (hit_times == test_times) ? 1 : 0;
return is_all_hit;
}
TEST_F(MaatPerfFileScan, IPPlugin) {
struct maat *maat_instance = MaatPerfFileScan::_shared_maat_instance;
const char* table_name = "TSG_IP_LOCATION_BUILT_IN";
int ip_plugin_ex_data_counter = 0;
int* is_all_hit = NULL;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_plugin_table_ex_schema_register(maat_instance, table_name,
perf_ip_plugin_EX_new_cb,
perf_ip_plugin_EX_free_cb,
perf_ip_plugin_EX_dup_cb,
0, &ip_plugin_ex_data_counter);
ASSERT_TRUE(ret>=0);
int i = 0;
pthread_t threads[PERF_THREAD_NUM];
for (i = 0; i < PERF_THREAD_NUM; i++) {
pthread_create(&(threads[i]), NULL, ip_plugin_get_thread, maat_instance);
}
for (i = 0; i < PERF_THREAD_NUM; i++) {
pthread_join(threads[i], (void**)&is_all_hit);
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit=0;
free(is_all_hit);
}
}
int main(int argc, char ** argv)

106
test/maat_json.json
View File

@@ -2836,6 +2836,112 @@
"group_name": "Untitled"
}
]
},
{
"compile_id": 206,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "duplicateRuleFor191",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "expr",
"table_name": "KEYWORDS_TABLE",
"table_content": {
"keywords": "54455354",
"expr_type": "none",
"format": "hexbin",
"match_method": "sub"
}
}
],
"group_name": "Untitled"
}
]
},
{
"compile_id": 207,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"user_region": "duplicateRuleFor192",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "flag",
"table_name": "FLAG_CONFIG",
"table_content": {
"flag": 1,
"flag_mask": 3
}
}
]
}
]
},
{
"compile_id": 208,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "duplicateRuleFor154",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv4",
"addr_format": "range",
"ip1": "10.0.7.100",
"ip2": "10.0.7.106",
"port_format": "range",
"port1": "65530",
"port2": "65535",
"protocol": 6
}
}
],
"not_flag": 0
}
]
},
{
"compile_id": 209,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "duplicateRuleFor179",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "INTERGER_PLUS",
"table_type": "interval_plus",
"table_content": {
"district": "interval.plus",
"low_boundary": 2020,
"up_boundary": 2020
}
}
]
}
]
}
],
"plugin_table": [

15
test/tsg_table_info.conf Normal file
View File

@@ -0,0 +1,15 @@
[
{
"table_id": 42,
"table_name":"TSG_IP_LOCATION_BUILT_IN",
"table_type":"ip_plugin",
"valid_column":18,
"custom": {
"item_id":1,
"ip_type":3,
"start_ip":4,
"end_ip":5,
"addr_format":7
}
}
]

1001
test/tsgrule/TSG_IP_LOCATION_BUILT_IN.head_1k Normal file
View File

File diff suppressed because it is too large Load Diff

2
test/tsgrule/full/index/full_config_index.00000000000000106967 Normal file
View File

@@ -0,0 +1,2 @@
TSG_IP_LOCATION_BUILT_IN 1000 ./tsgrule/TSG_IP_LOCATION_BUILT_IN.head_1k