gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

group_exclude(only one hierarchical group can be referenced)

Browse Source
This commit is contained in:
liuwentan
2023-05-04 17:10:19 +08:00
parent 33015d5aac
commit 8a3683fa30
15 changed files with 1098 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/maat.h
View File

@@ -102,8 +102,11 @@ int maat_options_set_iris(struct maat_options *opts, const char *full_directory,
int maat_options_set_json_file(struct maat_options *opts, const char *json_filename); int maat_options_set_json_file(struct maat_options *opts, const char *json_filename);
/* Indicate whether the JSON file is compressed by gzip */ /**
int maat_options_set_json_file_gzip_flag(struct maat_options *opts, int gzip_flag); * Indicate whether the JSON file is compressed by gzip
* flag: 1(compressed) 0(uncompressed)
* */
int maat_options_set_json_file_gzip_flag(struct maat_options *opts, int flag);
/* Specify the decryption key for the JSON file to be decrypted */ /* Specify the decryption key for the JSON file to be decrypted */
int maat_options_set_json_file_decrypt_key(struct maat_options *opts, const char *decrypt_key); int maat_options_set_json_file_decrypt_key(struct maat_options *opts, const char *decrypt_key);

5
src/inc_internal/maat_group.h
View File

@@ -41,8 +41,9 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *
int group2group_runtime_commit(void *g2g_runtime, const char *table_name, long long maat_rt_version); int group2group_runtime_commit(void *g2g_runtime, const char *table_name, long long maat_rt_version);
int group2group_runtime_get_top_groups(void *g2g_runtime, long long *group_ids, size_t group2group_runtime_get_super_groups(void *g2g_runtime, long long *group_ids,
size_t n_group_ids, long long *top_group_ids); size_t n_group_ids, long long *super_group_ids,
size_t super_group_ids_size);
long long group2group_runtime_rule_count(void *g2g_runtime); long long group2group_runtime_rule_count(void *g2g_runtime);

36
src/json2iris.c
View File

@@ -696,7 +696,7 @@ int write_group2compile_line(int group_id, int compile_id, int group_not_flag,
return 0; return 0;
} }
int write_group2group_line(int group_id, int super_group_id, int write_group2group_line(int group_id, int super_group_id, int is_exclude,
struct iris_description *p_iris) struct iris_description *p_iris)
{ {
char buff[4096] = {0}; char buff[4096] = {0};
@@ -705,8 +705,8 @@ int write_group2group_line(int group_id, int super_group_id,
return -1; return -1;
} }
snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id, snprintf(buff, sizeof(buff), "%d\t%d\t%d\t1\n", group_id,
super_group_id); super_group_id, is_exclude);
table->write_pos += memcat(&(table->buff), table->write_pos, table->write_pos += memcat(&(table->buff), table->write_pos,
&(table->buff_sz), buff, strlen(buff)); &(table->buff_sz), buff, strlen(buff));
table->line_count++; table->line_count++;
@@ -714,14 +714,14 @@ int write_group2group_line(int group_id, int super_group_id,
return 0; return 0;
} }
int write_group_rule(cJSON *group_json, int parent_id, int write_group_rule(cJSON *group_json, const char *parent_name, int parent_id,
int parent_type, int tracking_compile_id, int parent_type, int tracking_compile_id,
int Nth_group, struct iris_description *p_iris, int Nth_group, struct iris_description *p_iris,
struct log_handle *logger) struct log_handle *logger)
{ {
int ret = 0; int ret = 0;
int group_not_flag = 0; int group_not_flag = 0;
int clause_index = 0; int clause_index = 0, is_exclude = 0;
const char *str_parent_type[2] = {"compile", "group"}; const char *str_parent_type[2] = {"compile", "group"};
const char *group_name = NULL; const char *group_name = NULL;
const char *virtual_table = NULL; const char *virtual_table = NULL;
@@ -734,6 +734,13 @@ int write_group_rule(cJSON *group_json, int parent_id,
group_name = item->valuestring; group_name = item->valuestring;
} }
item = cJSON_GetObjectItem(group_json, "is_exclude");
if (NULL == item || item->type != cJSON_Number) {
is_exclude = 0;
} else {
is_exclude = item->valueint;
}
if (parent_type == PARENT_TYPE_COMPILE) { if (parent_type == PARENT_TYPE_COMPILE) {
item = cJSON_GetObjectItem(group_json, "virtual_table"); item = cJSON_GetObjectItem(group_json, "virtual_table");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
@@ -790,8 +797,9 @@ int write_group_rule(cJSON *group_json, int parent_id,
int i = 0; int i = 0;
cJSON_ArrayForEach(item, sub_groups) { cJSON_ArrayForEach(item, sub_groups) {
i++; i++;
ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, ret = write_group_rule(item, group_name, group_info->group_id,
tracking_compile_id, i, p_iris, logger); PARENT_TYPE_GROUP, tracking_compile_id,
i, p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -806,10 +814,14 @@ int write_group_rule(cJSON *group_json, int parent_id,
} }
if (parent_type == PARENT_TYPE_COMPILE) { if (parent_type == PARENT_TYPE_COMPILE) {
// printf("[group2compile] group:[%s:%d] parent:[%s:%d]\n",
// group_info->group_name, group_info->group_id, parent_name, parent_id);
ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag, ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag,
clause_index, virtual_table, p_iris, g2c_table); clause_index, virtual_table, p_iris, g2c_table);
} else { } else {
ret = write_group2group_line(group_info->group_id, parent_id, p_iris); // printf("[group2group] group:[%s:%d] parent:[%s:%d]\n",
// group_info->group_name, group_info->group_id, parent_name, parent_id);
ret = write_group2group_line(group_info->group_id, parent_id, is_exclude, p_iris);
} }
if (ret < 0) { if (ret < 0) {
@@ -1018,8 +1030,8 @@ int write_iris(cJSON *json, struct iris_description *p_iris,
parent_group = group_info_add_unsafe(p_iris, parent_group_name); parent_group = group_info_add_unsafe(p_iris, parent_group_name);
} }
ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, ret = write_group_rule(group_obj, parent_group_name, parent_group->group_id,
0, 0, p_iris, logger); PARENT_TYPE_GROUP, 0, 0, p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1061,8 +1073,8 @@ int write_iris(cJSON *json, struct iris_description *p_iris,
i = 0; i = 0;
cJSON *group_obj = NULL; cJSON *group_obj = NULL;
cJSON_ArrayForEach(group_obj, group_array) { cJSON_ArrayForEach(group_obj, group_array) {
ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, ret = write_group_rule(group_obj, "referenced by compile", compile_id,
compile_id, i, p_iris, logger); PARENT_TYPE_COMPILE, compile_id, i, p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }

152
src/maat_api.c
View File

@@ -201,13 +201,13 @@ int maat_options_set_json_file(struct maat_options *opts, const char *json_filen
return 0; return 0;
} }
int maat_options_set_json_file_gzip_flag(struct maat_options *opts, int gzip_flag) int maat_options_set_json_file_gzip_flag(struct maat_options *opts, int flag)
{ {
if (NULL == opts || (gzip_flag != 0 && gzip_flag != 1)) { if (NULL == opts || (flag != 0 && flag != 1)) {
return -1; return -1;
} }
opts->maat_json_is_gzipped = gzip_flag; opts->maat_json_is_gzipped = flag;
return 0; return 0;
} }
@@ -612,79 +612,102 @@ int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_
return ret; return ret;
} }
int generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, const char *table_name, void plugin_runtime_commit_ex_schema(void *runtime, void *schema, const char *table_name,
int table_id, enum table_type table_type, int valid_column) int valid_column)
{ {
if (NULL == runtime || NULL == schema || valid_column < 0) { struct ex_container_schema *container_schema = plugin_table_get_ex_container_schema(schema);
return -1; struct ex_data_runtime *ex_data_rt = plugin_runtime_get_ex_data_rt(runtime);
}
struct ex_container_schema *container_schema = NULL;
struct ex_data_runtime *ex_data_rt = NULL;
switch (table_type) {
case TABLE_TYPE_PLUGIN:
container_schema = plugin_table_get_ex_container_schema(schema);
ex_data_rt = plugin_runtime_get_ex_data_rt(runtime);
ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema); ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema);
break;
case TABLE_TYPE_IP_PLUGIN:
container_schema = ip_plugin_table_get_ex_container_schema(schema);
ex_data_rt = ip_plugin_runtime_get_ex_data_rt(runtime);
ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema);
break;
case TABLE_TYPE_FQDN_PLUGIN:
container_schema = fqdn_plugin_table_get_ex_container_schema(schema);
ex_data_rt = fqdn_plugin_runtime_get_ex_data_rt(runtime);
ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema);
break;
case TABLE_TYPE_BOOL_PLUGIN:
container_schema = bool_plugin_table_get_ex_container_schema(schema);
ex_data_rt = bool_plugin_runtime_get_ex_data_rt(runtime);
ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema);
break;
default:
break;
}
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt); size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
for (size_t i = 0; i < n_cached_row; i++) { for (size_t i = 0; i < n_cached_row; i++) {
const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i); const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i);
switch (table_type) {
case TABLE_TYPE_PLUGIN:
plugin_runtime_update(runtime, schema, table_name, row, valid_column); plugin_runtime_update(runtime, schema, table_name, row, valid_column);
break;
case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_update(runtime, schema, table_name, row, valid_column);
break;
case TABLE_TYPE_FQDN_PLUGIN:
fqdn_plugin_runtime_update(runtime, schema, table_name, row, valid_column);
break;
case TABLE_TYPE_BOOL_PLUGIN:
bool_plugin_runtime_update(runtime, schema, table_name, row, valid_column);
break;
default:
break;
}
} }
ex_data_runtime_clear_row_cache(ex_data_rt); ex_data_runtime_clear_row_cache(ex_data_rt);
plugin_runtime_commit(runtime, table_name, 0);
}
void ip_plugin_runtime_commit_ex_schema(void *runtime, void *schema, const char *table_name,
int valid_column)
{
struct ex_container_schema *container_schema = ip_plugin_table_get_ex_container_schema(schema);
struct ex_data_runtime *ex_data_rt = ip_plugin_runtime_get_ex_data_rt(runtime);
ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema);
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
for (size_t i = 0; i < n_cached_row; i++) {
const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i);
ip_plugin_runtime_update(runtime, schema, table_name, row, valid_column);
}
ex_data_runtime_clear_row_cache(ex_data_rt);
ip_plugin_runtime_commit(runtime, table_name, 0);
}
void fqdn_plugin_runtime_commit_ex_schema(void *runtime, void *schema, const char *table_name,
int valid_column)
{
struct ex_container_schema *container_schema = fqdn_plugin_table_get_ex_container_schema(schema);
struct ex_data_runtime *ex_data_rt = fqdn_plugin_runtime_get_ex_data_rt(runtime);
ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema);
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
for (size_t i = 0; i < n_cached_row; i++) {
const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i);
fqdn_plugin_runtime_update(runtime, schema, table_name, row, valid_column);
}
ex_data_runtime_clear_row_cache(ex_data_rt);
fqdn_plugin_runtime_commit(runtime, table_name, 0);
}
void bool_plugin_runtime_commit_ex_schema(void *runtime, void *schema, const char *table_name,
int valid_column)
{
struct ex_container_schema *container_schema = bool_plugin_table_get_ex_container_schema(schema);
struct ex_data_runtime *ex_data_rt = bool_plugin_runtime_get_ex_data_rt(runtime);
ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema);
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
for (size_t i = 0; i < n_cached_row; i++) {
const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i);
bool_plugin_runtime_update(runtime, schema, table_name, row, valid_column);
}
ex_data_runtime_clear_row_cache(ex_data_rt);
bool_plugin_runtime_commit(runtime, table_name, 0);
}
int generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, const char *table_name,
enum table_type table_type, int valid_column,
struct log_handle *logger)
{
if (NULL == runtime || NULL == schema || valid_column < 0) {
log_error(logger, MODULE_MAAT_API,
"[%s:%d] input parameter invalid, runtime:%p, schema:%p, valid_column:%d",
__FUNCTION__, __LINE__, runtime, schema, valid_column);
return -1;
}
switch (table_type) { switch (table_type) {
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
plugin_runtime_commit(runtime, table_name, 0); plugin_runtime_commit_ex_schema(runtime, schema, table_name, valid_column);
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_commit(runtime, table_name, 0); ip_plugin_runtime_commit_ex_schema(runtime, schema, table_name, valid_column);
break; break;
case TABLE_TYPE_FQDN_PLUGIN: case TABLE_TYPE_FQDN_PLUGIN:
fqdn_plugin_runtime_commit(runtime, table_name, 0); fqdn_plugin_runtime_commit_ex_schema(runtime, schema, table_name, valid_column);
break; break;
case TABLE_TYPE_BOOL_PLUGIN: case TABLE_TYPE_BOOL_PLUGIN:
bool_plugin_runtime_commit(runtime, table_name, 0); bool_plugin_runtime_commit_ex_schema(runtime, schema, table_name, valid_column);
break; break;
default: default:
break; log_error(logger, MODULE_MAAT_API,
"[%s:%d] table_type:%d invalid", __FUNCTION__, __LINE__, table_type);
return -1;
} }
return 0; return 0;
@@ -714,11 +737,15 @@ int generic_plugin_table_ex_schema_register(struct maat *maat_instance,
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
if (table_type == TABLE_TYPE_INVALID || valid_column < 0) { if (table_type == TABLE_TYPE_INVALID || valid_column < 0) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] table_type:%d or valid_column:%d invalid",
__FUNCTION__, __LINE__, table_type, valid_column);
return -1; return -1;
} }
ret = generic_plugin_runtime_commit_ex_schema(runtime, schema, table_name, table_id, ret = generic_plugin_runtime_commit_ex_schema(runtime, schema, table_name,
table_type, valid_column); table_type, valid_column,
maat_instance->logger);
} }
return ret; return ret;
@@ -740,6 +767,9 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance,
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_instance, table_name);
if (table_id < 0) { if (table_id < 0) {
log_error(maat_instance->logger, MODULE_MAAT_API,
"[%s:%d] [table:%s] is not registered before.",
__FUNCTION__, __LINE__, table_name);
return -1; return -1;
} }
@@ -749,12 +779,10 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance,
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (TABLE_TYPE_COMPILE == table_type) { if (TABLE_TYPE_COMPILE == table_type) {
ret = compile_table_ex_schema_register(maat_instance, table_id, ret = compile_table_ex_schema_register(maat_instance, table_id,
new_func, free_func, dup_func, new_func, free_func, dup_func, argl, argp);
argl, argp);
} else { } else {
ret = generic_plugin_table_ex_schema_register(maat_instance, table_name, table_id, ret = generic_plugin_table_ex_schema_register(maat_instance, table_name, table_id,
new_func, free_func, dup_func, new_func, free_func, dup_func, argl, argp);
argl, argp);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));

43
src/maat_compile.c
View File

@@ -893,7 +893,6 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
has_clause_num = 0; has_clause_num = 0;
for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
struct maat_clause_state *clause_state = iter_compile->clause_states + i; struct maat_clause_state *clause_state = iter_compile->clause_states + i;
clause_state->clause_id = 0;
if (!clause_state->in_use) { if (!clause_state->in_use) {
continue; continue;
} }
@@ -918,7 +917,6 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
if (iter_compile->clause_states[i].not_flag) { if (iter_compile->clause_states[i].not_flag) {
iter_compile->not_clause_cnt++; iter_compile->not_clause_cnt++;
} }
// TODO:mytest need to delete // TODO:mytest need to delete
#if 0 #if 0
struct maat_literal_id *p = NULL; struct maat_literal_id *p = NULL;
@@ -1250,8 +1248,7 @@ int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, struct group2comp
/* compile neither in effective hash nor in updating hash, so new one */ /* compile neither in effective hash nor in updating hash, so new one */
compile = maat_compile_new(compile_id); compile = maat_compile_new(compile_id);
assert(compile != NULL); assert(compile != NULL);
ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, g2c_item->not_flag);
g2c_item->not_flag);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d of compile %d failed", "[%s:%d] add literal_id{group_id:%d, vtable_id:%d} to clause_index: %d of compile %d failed",
@@ -1999,8 +1996,10 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile
struct compile_rule *compile_rules[compile_ids_size]; struct compile_rule *compile_rules[compile_ids_size];
// all hit clause_id -> compile_id // all hit clause_id -> compile_id
size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, is_last_scan, compile_state, size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, is_last_scan,
(void **)compile_rules, compile_ids_size); compile_state,
(void **)compile_rules,
compile_ids_size);
if (bool_match_ret > 0) { if (bool_match_ret > 0) {
qsort(compile_rules, bool_match_ret, sizeof(struct compile_rule *), qsort(compile_rules, bool_match_ret, sizeof(struct compile_rule *),
compare_compile_rule); compare_compile_rule);
@@ -2063,17 +2062,19 @@ int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
return 0; return 0;
} }
long long top_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, hit_group_ids, //hit_group_ids also will be added to super_group_ids
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids,
hit_group_cnt, hit_group_cnt,
top_group_ids); super_group_ids,
if (top_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) { MAX_SCANNER_HIT_GROUP_NUM);
top_group_cnt = MAX_SCANNER_HIT_GROUP_NUM; if (super_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) {
super_group_cnt = MAX_SCANNER_HIT_GROUP_NUM;
} }
for (int j = 0; j < top_group_cnt; j++) { for (int j = 0; j < super_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt, maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
top_group_ids[j], vtable_id); super_group_ids[j], vtable_id);
} }
for (int j = 0; j < hit_group_cnt; j++) { for (int j = 0; j < hit_group_cnt; j++) {
@@ -2097,11 +2098,11 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp
/* /*
NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths
*/ */
long long top_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; long long super_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
memset(top_group_ids, -1, sizeof(top_group_ids)); memset(super_group_ids, -1, sizeof(super_group_ids));
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &(internal_path->group_id), size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, &(internal_path->group_id),
1, top_group_ids); 1, super_group_ids, MAX_SCANNER_HIT_GROUP_NUM);
if (top_group_cnt <= 0) { if (0 == super_group_cnt) {
/* /*
item->group_id has no top group, this group can only be referenced by compile item->group_id has no top group, this group can only be referenced by compile
------------------------------------------------------------------------------ ------------------------------------------------------------------------------
@@ -2115,16 +2116,16 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp
NOTE: Add the hit path as long as the item is hit NOTE: Add the hit path as long as the item is hit
*/ */
top_group_cnt = 1; // add one hit path which top_group_ids[0] = -1 super_group_cnt = 1; // add one hit path which top_group_ids[0] = -1
} }
struct maat_hit_path tmp_path; struct maat_hit_path tmp_path;
for (int j = 0; j < top_group_cnt && hit_path_cnt < array_size; j++) { for (int j = 0; j < super_group_cnt && hit_path_cnt < array_size; j++) {
memset(&tmp_path, 0, sizeof(tmp_path)); memset(&tmp_path, 0, sizeof(tmp_path));
tmp_path.Nth_scan = internal_path->Nth_scan; tmp_path.Nth_scan = internal_path->Nth_scan;
tmp_path.item_id = internal_path->item_id; tmp_path.item_id = internal_path->item_id;
tmp_path.sub_group_id = internal_path->group_id; tmp_path.sub_group_id = internal_path->group_id;
tmp_path.top_group_id = top_group_ids[j]; tmp_path.top_group_id = super_group_ids[j];
tmp_path.vtable_id = internal_path->vtable_id; tmp_path.vtable_id = internal_path->vtable_id;
tmp_path.compile_id = -1; tmp_path.compile_id = -1;

4
src/maat_config_monitor.c
View File

@@ -414,8 +414,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
return -1; return -1;
} }
json_buff=decrypted_buff; json_buff = decrypted_buff;
json_buff_sz=decrypted_buff_sz; json_buff_sz = decrypted_buff_sz;
} }
if (maat_instance->opts.maat_json_is_gzipped) { if (maat_instance->opts.maat_json_is_gzipped) {

390
src/maat_group.c
View File

@@ -15,6 +15,7 @@
#include "maat_group.h" #include "maat_group.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
#include "uthash/utarray.h"
#include "igraph/igraph.h" #include "igraph/igraph.h"
#include "maat_kv.h" #include "maat_kv.h"
@@ -23,11 +24,13 @@
struct group2group_item { struct group2group_item {
long long group_id; long long group_id;
long long super_group_id; long long super_group_id;
int is_exclude;
}; };
struct group2group_schema { struct group2group_schema {
int group_id_column; int group_id_column;
int super_group_id_column; int super_group_id_column;
int is_exclude_column;
int table_id;//ugly int table_id;//ugly
struct table_manager *ref_tbl_mgr; struct table_manager *ref_tbl_mgr;
}; };
@@ -38,16 +41,16 @@ struct maat_group {
int ref_by_super_group_cnt; int ref_by_super_group_cnt;
int ref_by_sub_group_cnt; int ref_by_sub_group_cnt;
size_t top_group_cnt; UT_array *incl_super_group_ids;
long long *top_group_ids; UT_array *excl_super_group_ids;
UT_hash_handle hh_group_id; UT_hash_handle hh_group_id;
UT_hash_handle hh_vertex_id; UT_hash_handle hh_vertex_id;
}; };
struct maat_group_topology { struct maat_group_topology {
struct maat_group *hash_group_by_id; //key: group_id, value: struct maat_group *. struct maat_group *hash_by_group_id; //key: group_id, value: struct maat_group *.
struct maat_group *hash_group_by_vertex; //key: vetex_id, value: struct maat_group *. Multimap (Items with multiple keys). struct maat_group *hash_by_vertex_id; //key: vetex_id, value: struct maat_group *. Multimap (Items with multiple keys).
igraph_t group_graph; igraph_t group_graph;
igraph_integer_t grp_vertex_id_generator; igraph_integer_t grp_vertex_id_generator;
struct log_handle *logger; struct log_handle *logger;
@@ -65,6 +68,21 @@ struct group2group_runtime {
struct log_handle *logger; struct log_handle *logger;
}; };
UT_icd ut_group_id_icd = {sizeof(long long), NULL, NULL, NULL};
static inline int compare_group_id(const void *a, const void *b)
{
long long ret = *(const long long *)a - *(const long long *)b;
if (0 == ret) {
return 0;
} else if(ret < 0) {
return -1;
} else {
return 1;
}
}
void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr, void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger) const char *table_name, struct log_handle *logger)
{ {
@@ -93,7 +111,8 @@ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2g_schema->group_id_column = custom_item->valueint; g2g_schema->group_id_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_GROUP, log_error(logger, MODULE_GROUP,
"[%s:%d] table %s has no group_id column", table_name); "[%s:%d] table %s has no group_id column",
__FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -102,7 +121,18 @@ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr,
g2g_schema->super_group_id_column = custom_item->valueint; g2g_schema->super_group_id_column = custom_item->valueint;
} else { } else {
log_error(logger, MODULE_GROUP, log_error(logger, MODULE_GROUP,
"[%s:%d] table %s has no super_group_id column", table_name); "[%s:%d] table %s has no super_group_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "is_exclude");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2g_schema->is_exclude_column = custom_item->valueint;
} else {
log_error(logger, MODULE_GROUP,
"[%s:%d] table %s has no is_exclude column",
__FUNCTION__, __LINE__, table_name);
goto error; goto error;
} }
@@ -120,7 +150,10 @@ void group2group_schema_free(void *g2g_schema)
void group_vertex_free(struct maat_group *group) void group_vertex_free(struct maat_group *group)
{ {
FREE(group->top_group_ids); utarray_free(group->incl_super_group_ids);
utarray_free(group->excl_super_group_ids);
group->incl_super_group_ids = NULL;
group->excl_super_group_ids = NULL;
FREE(group); FREE(group);
} }
@@ -129,10 +162,10 @@ struct maat_group_topology *maat_group_topology_new(struct log_handle *logger)
struct maat_group_topology *group_topo = ALLOC(struct maat_group_topology, 1); struct maat_group_topology *group_topo = ALLOC(struct maat_group_topology, 1);
UNUSED int ret = 0; UNUSED int ret = 0;
group_topo->hash_group_by_id = NULL; group_topo->hash_by_group_id = NULL;
group_topo->hash_group_by_vertex = NULL; group_topo->hash_by_vertex_id = NULL;
ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED); ret = igraph_empty(&(group_topo->group_graph), 0, IGRAPH_DIRECTED);
assert(ret == IGRAPH_SUCCESS); assert(ret == IGRAPH_SUCCESS);
group_topo->logger = logger; group_topo->logger = logger;
@@ -144,12 +177,12 @@ void maat_group_topology_free(struct maat_group_topology *group_topo)
{ {
struct maat_group *group = NULL, *tmp_group = NULL; struct maat_group *group = NULL, *tmp_group = NULL;
HASH_CLEAR(hh_vertex_id, group_topo->hash_group_by_vertex);//No need group memory clean up. HASH_CLEAR(hh_vertex_id, group_topo->hash_by_vertex_id);//No need group memory clean up.
HASH_ITER(hh_group_id, group_topo->hash_group_by_id, group, tmp_group) { HASH_ITER(hh_group_id, group_topo->hash_by_group_id, group, tmp_group) {
HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group); HASH_DELETE(hh_group_id, group_topo->hash_by_group_id, group);
group_vertex_free(group); group_vertex_free(group);
} }
assert(group_topo->hash_group_by_id == NULL); assert(group_topo->hash_by_group_id == NULL);
igraph_destroy(&group_topo->group_graph); igraph_destroy(&group_topo->group_graph);
FREE(group_topo); FREE(group_topo);
@@ -163,11 +196,18 @@ struct maat_group *maat_group_clone(struct maat_group *group)
group_copy->vertex_id = group->vertex_id; group_copy->vertex_id = group->vertex_id;
group_copy->ref_by_sub_group_cnt = group->ref_by_sub_group_cnt; group_copy->ref_by_sub_group_cnt = group->ref_by_sub_group_cnt;
group_copy->ref_by_super_group_cnt = group->ref_by_super_group_cnt; group_copy->ref_by_super_group_cnt = group->ref_by_super_group_cnt;
group_copy->top_group_cnt = group->top_group_cnt; utarray_new(group_copy->incl_super_group_ids, &ut_group_id_icd);
if (group_copy->top_group_cnt > 0) { utarray_new(group_copy->excl_super_group_ids, &ut_group_id_icd);
group_copy->top_group_ids = ALLOC(long long, group_copy->top_group_cnt);
memcpy(group_copy->top_group_ids, group->top_group_ids, long long *p = NULL;
group_copy->top_group_cnt * sizeof(long long)); for (p = (long long *)utarray_front(group->incl_super_group_ids); p != NULL;
p = (long long *)utarray_next(group->incl_super_group_ids, p)) {
utarray_push_back(group_copy->incl_super_group_ids, p);
}
for (p = (long long *)utarray_front(group->excl_super_group_ids); p != NULL;
p = (long long *)utarray_next(group->excl_super_group_ids, p)) {
utarray_push_back(group_copy->excl_super_group_ids, p);
} }
return group_copy; return group_copy;
@@ -182,12 +222,12 @@ struct maat_group_topology *maat_group_topology_clone(struct maat_group_topology
struct maat_group_topology *group_topo_copy = ALLOC(struct maat_group_topology, 1); struct maat_group_topology *group_topo_copy = ALLOC(struct maat_group_topology, 1);
struct maat_group *group = NULL, *tmp_group = NULL; struct maat_group *group = NULL, *tmp_group = NULL;
HASH_ITER(hh_group_id, group_topo->hash_group_by_id, group, tmp_group) { HASH_ITER(hh_group_id, group_topo->hash_by_group_id, group, tmp_group) {
struct maat_group *group_copy = maat_group_clone(group); struct maat_group *group_copy = maat_group_clone(group);
HASH_ADD(hh_group_id, group_topo_copy->hash_group_by_id, group_id, HASH_ADD(hh_group_id, group_topo_copy->hash_by_group_id, group_id,
sizeof(group_copy->group_id), group_copy); sizeof(group_copy->group_id), group_copy);
HASH_ADD(hh_vertex_id, group_topo_copy->hash_group_by_vertex, vertex_id, HASH_ADD(hh_vertex_id, group_topo_copy->hash_by_vertex_id, vertex_id,
sizeof(group_copy->vertex_id), group_copy); sizeof(group_copy->vertex_id), group_copy);
} }
@@ -263,6 +303,16 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
} }
g2g_item->super_group_id = atoll(line + column_offset); g2g_item->super_group_id = atoll(line + column_offset);
ret = get_column_pos(line, g2g_schema->is_exclude_column,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_GROUP,
"[%s:%d] group2group table:%s line:%s has no is_exclude",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
g2g_item->is_exclude = atoi(line + column_offset);
return g2g_item; return g2g_item;
error: error:
FREE(g2g_item); FREE(g2g_item);
@@ -293,17 +343,19 @@ struct maat_group *group_topology_add_group(struct maat_group_topology *group_to
struct maat_group *group = ALLOC(struct maat_group, 1); struct maat_group *group = ALLOC(struct maat_group, 1);
group->group_id = group_id; group->group_id = group_id;
group->vertex_id = group_topo->grp_vertex_id_generator++; group->vertex_id = group_topo->grp_vertex_id_generator++;
utarray_new(group->incl_super_group_ids, &ut_group_id_icd);
utarray_new(group->excl_super_group_ids, &ut_group_id_icd);
assert(igraph_vcount(&group_topo->group_graph)==group->vertex_id); assert(igraph_vcount(&group_topo->group_graph)==group->vertex_id);
igraph_add_vertices(&group_topo->group_graph, 1, NULL); //Add 1 vertice. igraph_add_vertices(&group_topo->group_graph, 1, NULL); //Add 1 vertice.
HASH_ADD(hh_group_id, group_topo->hash_group_by_id, group_id, sizeof(group->group_id), group); HASH_ADD(hh_group_id, group_topo->hash_by_group_id, group_id, sizeof(group->group_id), group);
HASH_ADD(hh_vertex_id, group_topo->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group); HASH_ADD(hh_vertex_id, group_topo->hash_by_vertex_id, vertex_id, sizeof(group->vertex_id), group);
return group; return group;
} }
void group_topology_remove_group(struct maat_group_topology *group_topo, void group_topology_del_group(struct maat_group_topology *group_topo,
struct maat_group *group) struct maat_group *group)
{ {
if (NULL == group_topo || NULL == group) { if (NULL == group_topo || NULL == group) {
@@ -325,12 +377,12 @@ void group_topology_remove_group(struct maat_group_topology *group_topo,
assert(0); assert(0);
} }
igraph_vector_destroy(&v); igraph_vector_destroy(&v);
assert(group->top_group_ids==NULL); assert(group->incl_super_group_ids==NULL);
assert(group->excl_super_group_ids==NULL);
//We should not call igraph_delete_vertices, because this is function changes the ids of the vertices. //We should not call igraph_delete_vertices, because this is function changes the ids of the vertices.
//igraph_delete_vertices(&hier->group_graph, igraph_vss_1(group->vertex_id));
HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group); HASH_DELETE(hh_group_id, group_topo->hash_by_group_id, group);
HASH_DELETE(hh_vertex_id, group_topo->hash_group_by_vertex, group); HASH_DELETE(hh_vertex_id, group_topo->hash_by_vertex_id, group);
group_vertex_free(group); group_vertex_free(group);
} }
@@ -342,13 +394,68 @@ struct maat_group *group_topology_find_group(struct maat_group_topology *group_t
} }
struct maat_group *group = NULL; struct maat_group *group = NULL;
HASH_FIND(hh_group_id, group_topo->hash_group_by_id, &group_id, sizeof(group_id), group); HASH_FIND(hh_group_id, group_topo->hash_by_group_id, &group_id, sizeof(group_id), group);
return group; return group;
} }
void maat_group_reference_super_group(struct maat_group *group, long long super_group_id,
int is_exclude)
{
if (NULL == group || super_group_id < 0) {
return;
}
if (0 == is_exclude) {
//include superior group
if (!utarray_find(group->incl_super_group_ids, &super_group_id,
compare_group_id)) {
utarray_push_back(group->incl_super_group_ids, &super_group_id);
utarray_sort(group->incl_super_group_ids, compare_group_id);
}
} else {
//exclude superior group
if (!utarray_find(group->excl_super_group_ids, &super_group_id,
compare_group_id)) {
utarray_push_back(group->excl_super_group_ids, &super_group_id);
utarray_sort(group->excl_super_group_ids, compare_group_id);
}
}
}
void maat_group_dereference_super_group(struct maat_group *group, long long super_group_id,
int is_exclude)
{
if (NULL == group || super_group_id < 0) {
return;
}
size_t remove_idx = 0;
if (0 == is_exclude) {
//include superior group
if (!utarray_find(group->incl_super_group_ids, &super_group_id, compare_group_id)) {
return;
}
remove_idx = utarray_eltidx(group->incl_super_group_ids, &super_group_id);
utarray_erase(group->incl_super_group_ids, remove_idx, 1);
utarray_sort(group->incl_super_group_ids, compare_group_id);
} else {
//exclude superior group
if (!utarray_find(group->excl_super_group_ids, &super_group_id, compare_group_id)) {
return;
}
remove_idx = utarray_eltidx(group->excl_super_group_ids, &super_group_id);
utarray_erase(group->excl_super_group_ids, remove_idx, 1);
utarray_sort(group->excl_super_group_ids, compare_group_id);
}
}
int group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_topology_add_group_to_group(struct maat_group_topology *group_topo,
long long group_id, long long super_group_id) long long group_id, long long super_group_id,
int is_exclude)
{ {
if (NULL == group_topo) { if (NULL == group_topo) {
return -1; return -1;
@@ -364,6 +471,8 @@ int group_topology_add_group_to_group(struct maat_group_topology *group_topo,
super_group = group_topology_add_group(group_topo, super_group_id); super_group = group_topology_add_group(group_topo, super_group_id);
} }
maat_group_reference_super_group(group, super_group_id, is_exclude);
igraph_integer_t edge_id; igraph_integer_t edge_id;
int ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, int ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id,
super_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); super_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
@@ -385,8 +494,9 @@ int group_topology_add_group_to_group(struct maat_group_topology *group_topo,
return ret; return ret;
} }
int group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_topology_del_group_from_group(struct maat_group_topology *group_topo,
long long group_id, long long super_group_id) long long group_id, long long super_group_id,
int is_exclude)
{ {
if (NULL == group_topo) { if (NULL == group_topo) {
return -1; return -1;
@@ -409,6 +519,8 @@ int group_topology_remove_group_from_group(struct maat_group_topology *group_top
return -1; return -1;
} }
maat_group_dereference_super_group(group, super_group_id, is_exclude);
igraph_es_t es; igraph_es_t es;
igraph_integer_t edge_num_before = 0, edge_num_after = 0; igraph_integer_t edge_num_before = 0, edge_num_after = 0;
@@ -439,33 +551,12 @@ int group_topology_remove_group_from_group(struct maat_group_topology *group_top
return 0; return 0;
} }
static size_t effective_vertices_count(igraph_vector_t *vids) int group_topology_build_super_groups(struct maat_group_topology *group_topo)
{
size_t i = 0;
int tmp_vid = 0;
for (i = 0; i < (size_t)igraph_vector_size(vids); i++) {
tmp_vid = (int) VECTOR(*vids)[i];
if (tmp_vid < 0) {
break;
}
}
return i;
}
int group_topology_build_top_groups(struct maat_group_topology *group_topo)
{ {
if (NULL == group_topo) { if (NULL == group_topo) {
return -1; return -1;
} }
struct maat_group *group = NULL, *tmp = NULL;
struct maat_group *super_group = NULL;
int tmp_vid = 0;
size_t top_group_cnt = 0;
long long *temp_group_ids = NULL;
igraph_bool_t is_dag; igraph_bool_t is_dag;
igraph_is_dag(&(group_topo->group_graph), &is_dag); igraph_is_dag(&(group_topo->group_graph), &is_dag);
if (!is_dag) { if (!is_dag) {
@@ -474,58 +565,19 @@ int group_topology_build_top_groups(struct maat_group_topology *group_topo)
return -1; return -1;
} }
igraph_integer_t group_graph_vcount = igraph_vcount(&group_topo->group_graph); struct maat_group *group = NULL, *tmp_group = NULL;
igraph_vector_t dfs_vids; HASH_ITER (hh_group_id, group_topo->hash_by_group_id, group, tmp_group) {
igraph_vector_init(&dfs_vids, group_graph_vcount);
HASH_ITER (hh_group_id, group_topo->hash_group_by_id, group, tmp) {
top_group_cnt = 0;
temp_group_ids = NULL;
//Orphan, Not reference by any one, free it. //Orphan, Not reference by any one, free it.
if (0 == group->ref_by_super_group_cnt if (0 == group->ref_by_super_group_cnt
&& 0 == group->ref_by_sub_group_cnt) { && 0 == group->ref_by_sub_group_cnt) {
FREE(group->top_group_ids); FREE(group->incl_super_group_ids);
group_topology_remove_group(group_topo, group); FREE(group->excl_super_group_ids);
group_topology_del_group(group_topo, group);
continue; continue;
} }
//A group is referenced by superior groups.
if (group->ref_by_super_group_cnt > 0) {
igraph_vector_t *vids = &dfs_vids;
igraph_dfs(&group_topo->group_graph, group->vertex_id, IGRAPH_OUT,
0, vids, NULL, NULL, NULL, NULL, NULL, NULL);
temp_group_ids = ALLOC(long long, effective_vertices_count(vids));
for (size_t i = 0; i < (size_t)igraph_vector_size(vids); i++) {
tmp_vid = (int)VECTOR(*vids)[i];
if (tmp_vid < 0) {
break;
} }
HASH_FIND(hh_vertex_id, group_topo->hash_group_by_vertex, &tmp_vid,
sizeof(tmp_vid), super_group);
temp_group_ids[top_group_cnt] = super_group->group_id;
top_group_cnt++;
}
}
FREE(group->top_group_ids);
group->top_group_cnt = top_group_cnt;
if (top_group_cnt > 0) {
group->top_group_ids = ALLOC(long long, group->top_group_cnt);
memcpy(group->top_group_ids, temp_group_ids, sizeof(long long)*group->top_group_cnt);
}
if (temp_group_ids != NULL) {
FREE(temp_group_ids);
}
}
igraph_vector_destroy(&dfs_vids);
return 0; return 0;
} }
@@ -562,8 +614,8 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
if (0 == is_valid) { if (0 == is_valid) {
//delete //delete
ret = group_topology_remove_group_from_group(g2g_rt->updating_group_topo, g2g_item->group_id, ret = group_topology_del_group_from_group(g2g_rt->updating_group_topo, g2g_item->group_id,
g2g_item->super_group_id); g2g_item->super_group_id, g2g_item->is_exclude);
if (0 == ret) { if (0 == ret) {
g2g_rt->rule_num--; g2g_rt->rule_num--;
} else { } else {
@@ -572,7 +624,7 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema,
} else { } else {
//add //add
ret = group_topology_add_group_to_group(g2g_rt->updating_group_topo, g2g_item->group_id, ret = group_topology_add_group_to_group(g2g_rt->updating_group_topo, g2g_item->group_id,
g2g_item->super_group_id); g2g_item->super_group_id, g2g_item->is_exclude);
if (0 == ret) { if (0 == ret) {
g2g_rt->rule_num++; g2g_rt->rule_num++;
} else { } else {
@@ -601,7 +653,7 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name, long l
return 0; return 0;
} }
int ret = group_topology_build_top_groups(g2g_rt->updating_group_topo); int ret = group_topology_build_super_groups(g2g_rt->updating_group_topo);
if (ret < 0) { if (ret < 0) {
log_error(g2g_rt->logger, MODULE_GROUP, log_error(g2g_rt->logger, MODULE_GROUP,
"[%s:%d] table[%s] group2group runtime commit failed", "[%s:%d] table[%s] group2group runtime commit failed",
@@ -619,12 +671,118 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name, long l
g2g_rt->version = maat_rt_version; g2g_rt->version = maat_rt_version;
log_info(g2g_rt->logger, MODULE_GROUP, log_info(g2g_rt->logger, MODULE_GROUP,
"table[%s] commit %zu g2g rules and rebuild top_groups completed, version:%lld", "table[%s] commit %zu g2g rules and rebuild super_groups completed, version:%lld",
table_name, g2g_rt->rule_num, g2g_rt->version); table_name, g2g_rt->rule_num, g2g_rt->version);
return 0; return 0;
} }
#define MAX_RECURSION_DEPTH 5
void get_one_round_hit_group_ids(struct maat_group_topology *group_topo, UT_array *hit_group_ids,
UT_array *all_hit_group_ids, size_t depth)
{
UT_array *incl_super_group_ids;
UT_array *excl_super_group_ids;
UT_array *one_round_hit_group_ids;
if (depth >= MAX_RECURSION_DEPTH) {
return;
}
utarray_new(incl_super_group_ids, &ut_group_id_icd);
utarray_new(excl_super_group_ids, &ut_group_id_icd);
utarray_new(one_round_hit_group_ids, &ut_group_id_icd);
long long *p = NULL;
for (p = (long long *)utarray_front(hit_group_ids); p != NULL;
p = (long long *)utarray_next(hit_group_ids, p)) {
struct maat_group *group = group_topology_find_group(group_topo, *p);
if (NULL == group) {
continue;
}
long long *tmp = NULL;
for (tmp = (long long *)utarray_front(group->incl_super_group_ids); tmp != NULL;
tmp = (long long *)utarray_next(group->incl_super_group_ids, tmp)) {
utarray_push_back(incl_super_group_ids, tmp);
}
for (tmp = (long long *)utarray_front(group->excl_super_group_ids); tmp != NULL;
tmp = (long long *)utarray_next(group->excl_super_group_ids, tmp)) {
utarray_push_back(excl_super_group_ids, tmp);
}
}
for (p = (long long *)utarray_front(incl_super_group_ids); p != NULL;
p = (long long *)utarray_next(incl_super_group_ids, p)) {
if (utarray_find(excl_super_group_ids, p, compare_group_id)) {
continue;
}
utarray_push_back(one_round_hit_group_ids, p);
utarray_push_back(all_hit_group_ids, p);
}
utarray_free(incl_super_group_ids);
utarray_free(excl_super_group_ids);
if (utarray_len(one_round_hit_group_ids) == 0) {
goto next;
}
depth++;
get_one_round_hit_group_ids(group_topo, one_round_hit_group_ids, all_hit_group_ids, depth);
next:
utarray_free(one_round_hit_group_ids);
}
size_t group_topology_get_super_groups(struct maat_group_topology *group_topo,
long long *group_ids, size_t n_group_ids,
long long *super_group_ids,
size_t super_group_ids_size)
{
size_t i = 0, idx = 0, depth = 0;
UT_array *one_round_hit_group_ids;
UT_array *all_hit_group_ids;
utarray_new(one_round_hit_group_ids, &ut_group_id_icd);
utarray_new(all_hit_group_ids, &ut_group_id_icd);
for (i = 0; i < n_group_ids; i++) {
utarray_push_back(one_round_hit_group_ids, &(group_ids[i]));
}
get_one_round_hit_group_ids(group_topo, one_round_hit_group_ids, all_hit_group_ids, depth);
long long *p = NULL;
for (p = (long long *)utarray_front(all_hit_group_ids); p != NULL;
p = (long long *)utarray_next(all_hit_group_ids, p)) {
if (idx >= super_group_ids_size) {
break;
}
super_group_ids[idx++] = *p;
}
utarray_free(one_round_hit_group_ids);
utarray_free(all_hit_group_ids);
return idx;
}
size_t group2group_runtime_get_super_groups(void *g2g_runtime, long long *group_ids,
size_t n_group_ids, long long *super_group_ids,
size_t super_group_ids_size)
{
if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) {
return 0;
}
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
return group_topology_get_super_groups(g2g_rt->group_topo, group_ids, n_group_ids,
super_group_ids, super_group_ids_size);
}
long long group2group_runtime_rule_count(void *g2g_runtime) long long group2group_runtime_rule_count(void *g2g_runtime)
{ {
if (NULL == g2g_runtime) { if (NULL == g2g_runtime) {
@@ -644,27 +802,3 @@ long long group2group_runtime_update_err_count(void *g2g_runtime)
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
return g2g_rt->update_err_cnt; return g2g_rt->update_err_cnt;
} }
int group2group_runtime_get_top_groups(void *g2g_runtime, long long *group_ids,
size_t n_group_ids, long long *top_group_ids)
{
if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) {
return -1;
}
size_t top_group_index = 0;
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
for (size_t i = 0; i < n_group_ids; i++) {
struct maat_group *group = group_topology_find_group(g2g_rt->group_topo, group_ids[i]);
if (!group) {
continue;
}
for (size_t j = 0; j < group->top_group_cnt; j++) {
top_group_ids[top_group_index++] = group->top_group_ids[j];
}
}
return top_group_index;
}

1
src/maat_stat.c
View File

@@ -273,6 +273,7 @@ void maat_fieldstat_table_row_output(struct maat_stat *stat, int perf_on)
break; break;
case TABLE_TYPE_GROUP2GROUP: case TABLE_TYPE_GROUP2GROUP:
g2g_rule_num += group2group_runtime_rule_count(runtime); g2g_rule_num += group2group_runtime_rule_count(runtime);
//TODO exclude group num
break; break;
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS: case TABLE_TYPE_EXPR_PLUS:

3
test/file_test_tableinfo.conf
View File

@@ -18,7 +18,8 @@
"valid_column":3, "valid_column":3,
"custom": { "custom": {
"group_id":1, "group_id":1,
"super_group_id":2 "super_group_id":2,
"is_exclude":3
} }
}, },
{ {

355
test/maat_framework_gtest.cpp
View File

@@ -194,10 +194,10 @@ int group2compile_table_set_line(struct maat *maat_instance, const char *table_n
} }
int group2group_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op, int group2group_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op,
long long group_id, long long superior_group_id, int expire_after) long long group_id, long long superior_group_id, int is_exclude, int expire_after)
{ {
char table_line[128] = {0}; char table_line[128] = {0};
sprintf(table_line, "%lld\t%lld\t%d", group_id, superior_group_id, op); sprintf(table_line, "%lld\t%lld\t%d\t%d", group_id, superior_group_id, is_exclude, op);
struct maat_cmd_line line_rule; struct maat_cmd_line line_rule;
line_rule.rule_id = TO_GROUP2X_KEY(group_id, superior_group_id); line_rule.rule_id = TO_GROUP2X_KEY(group_id, superior_group_id);
@@ -2150,6 +2150,351 @@ TEST_F(NOTLogic, ScanNotIP) {
state = NULL; state = NULL;
} }
class ExcludeLogic : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_GTEST,
"[%s:%d] create maat instance in NOTLogic failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *ExcludeLogic::_shared_maat_instance;
struct log_handle *ExcludeLogic::logger;
TEST_F(ExcludeLogic, ScanExcludeAtFirst) {
const char *string_should_not_hit = "This string ONLY contains must-not-contained-string-of-rule-199.";
const char *string_should_hit = "This string contains must-contained-string-of-rule-199";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
const char *not_hit_table_name = "KEYWORDS_TABLE";
const char *hit_table_name = "HTTP_URL";
struct maat *maat_instance = ExcludeLogic::_shared_maat_instance;
struct maat_state *state = maat_state_new(maat_instance, thread_id);
int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name);
ASSERT_GT(not_hit_table_id, 0);
int ret = maat_scan_string(maat_instance, not_hit_table_id, string_should_not_hit, strlen(string_should_not_hit),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
int hit_table_id = maat_get_table_id(maat_instance, hit_table_name);
ASSERT_GT(hit_table_id, 0);
ret = maat_scan_string(maat_instance, hit_table_id, string_should_hit, strlen(string_should_hit),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 199);
maat_state_free(state);
state = NULL;
}
TEST_F(ExcludeLogic, ScanExcludeAtLast) {
const char *string_should_hit = "This string ONLY contains must-contained-string-of-rule-200.";
const char *string_should_not_hit = "This string contains both must-contained-string-of-rule-200 and must-not-contained-string-of-rule-200.";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
const char *table_name = "HTTP_URL";
struct maat *maat_instance = ExcludeLogic::_shared_maat_instance;
struct maat_state *state = maat_state_new(maat_instance, thread_id);
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
int ret = maat_scan_string(maat_instance, table_id, string_should_hit, strlen(string_should_hit),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 200);
maat_state_reset(state);
ret = maat_scan_string(maat_instance, table_id, string_should_not_hit, strlen(string_should_not_hit),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
maat_state_free(state);
state = NULL;
}
TEST_F(ExcludeLogic, ScanIrrelavantAtLast) {
const char *string_should_hit = "This string ONLY contains must-contained-string-of-rule-200.";
const char *string_irrelevant = "This string contains nothing to hit.";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
const char *hit_table_name = "HTTP_URL";
const char *not_hit_table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = ExcludeLogic::_shared_maat_instance;
struct maat_state *state = maat_state_new(maat_instance, thread_id);
int hit_table_id = maat_get_table_id(maat_instance, hit_table_name);
ASSERT_GT(hit_table_id, 0);
int ret = maat_scan_string(maat_instance, hit_table_id, string_should_hit, strlen(string_should_hit),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 200);
int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name);
ASSERT_GT(hit_table_id, 0);
ret = maat_scan_string(maat_instance, not_hit_table_id, string_irrelevant, strlen(string_irrelevant),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_OK);
maat_state_free(state);
state = NULL;
}
TEST_F(ExcludeLogic, ScanVirtualTable) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_instance = ExcludeLogic::_shared_maat_instance;
struct maat_state *state = maat_state_new(maat_instance, thread_id);
const char *table_name = "VIRTUAL_IP_PLUS_TABLE";
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
uint32_t should_hit_ip;
uint32_t should_not_hit_ip;
inet_pton(AF_INET, "100.64.1.1", &should_hit_ip);
uint16_t port = htons(5210);
int ret = maat_scan_ipv4(maat_instance, table_id, should_hit_ip, port, 6,
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 202);
maat_state_reset(state);
inet_pton(AF_INET, "100.64.1.5", &should_hit_ip);
ret = maat_scan_ipv4(maat_instance, table_id, should_hit_ip, port, 6,
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 202);
maat_state_reset(state);
inet_pton(AF_INET, "100.64.1.6", &should_not_hit_ip);
ret = maat_scan_ipv4(maat_instance, table_id, should_not_hit_ip, port, 6,
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
maat_state_reset(state);
inet_pton(AF_INET, "100.64.1.11", &should_not_hit_ip);
ret = maat_scan_ipv4(maat_instance, table_id, should_not_hit_ip, port, 6,
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
maat_state_reset(state);
maat_state_free(state);
}
TEST_F(ExcludeLogic, ScanWithMultiClause) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat *maat_instance = ExcludeLogic::_shared_maat_instance;
struct maat_state *state = maat_state_new(maat_instance, thread_id);
const char *ip_table_name = "VIRTUAL_IP_PLUS_TABLE";
int ip_table_id = maat_get_table_id(maat_instance, ip_table_name);
ASSERT_GT(ip_table_id, 0);
uint32_t ip_addr;
inet_pton(AF_INET, "192.168.50.43", &ip_addr);
uint16_t port = htons(56168);
int ret = maat_scan_ipv4(maat_instance, ip_table_id, ip_addr, port, 6,
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
inet_pton(AF_INET, "47.92.108.93", &ip_addr);
port = htons(443);
ret = maat_scan_ipv4(maat_instance, ip_table_id, ip_addr, port, 6,
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
const char *expr_table_name = "HTTP_RESPONSE_KEYWORDS";
int expr_table_id = maat_get_table_id(maat_instance, expr_table_name);
ASSERT_GT(expr_table_id, 0);
const char *should_hit_expr = "www.baidu.com";
ret = maat_scan_string(maat_instance, expr_table_id, should_hit_expr, strlen(should_hit_expr),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 203);
maat_state_free(state);
state = NULL;
}
// TEST_F(ExcludeLogic, ScanHitAtLastEmptyExpr) {
// const char *string_should_not_hit = "This string should not hit.";
// const char *string_match_no_region = "This string is matched against a empty table.";
// long long results[ARRAY_SIZE] = {0};
// size_t n_hit_result = 0;
// int thread_id = 0;
// const char *not_hit_table_name = "HTTP_URL";
// const char *hit_table_name = "IP_PLUS_CONFIG";
// const char *empty_table_name = "EMPTY_KEYWORD";
// struct maat *maat_instance = ExcludeLogic::_shared_maat_instance;
// struct maat_state *state = maat_state_new(maat_instance, thread_id);
// int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name);
// ASSERT_GT(not_hit_table_id, 0);
// int ret = maat_scan_string(maat_instance, not_hit_table_id,
// string_should_not_hit, strlen(string_should_not_hit),
// results, ARRAY_SIZE, &n_hit_result, state);
// EXPECT_EQ(ret, MAAT_SCAN_OK);
// uint32_t sip;
// inet_pton(AF_INET, "10.0.8.186", &sip);
// uint16_t port = htons(18611);
// int proto = 6;
// int hit_table_id = maat_get_table_id(maat_instance, hit_table_name);
// ASSERT_GT(hit_table_id, 0);
// ret = maat_scan_ipv4(maat_instance, hit_table_id, sip, port, proto, results,
// ARRAY_SIZE, &n_hit_result, state);
// EXPECT_EQ(ret, MAAT_SCAN_HIT);
// EXPECT_EQ(n_hit_result, 1);
// EXPECT_EQ(results[0], 186);
// int empty_table_id = maat_get_table_id(maat_instance, empty_table_name);
// ASSERT_GT(empty_table_id, 0);
// ret = maat_scan_string(maat_instance, empty_table_id, string_match_no_region,
// strlen(string_match_no_region), results, ARRAY_SIZE,
// &n_hit_result, state);
// EXPECT_EQ(ret, MAAT_SCAN_OK);
// maat_state_free(state);
// state = NULL;
// }
// TEST_F(ExcludeLogic, ScanHitAtLastEmptyInteger) {
// const char *string_should_not_hit = "This string should not hit.";
// long long results[ARRAY_SIZE] = {0};
// size_t n_hit_result = 0;
// int thread_id = 0;
// const char *not_hit_table_name = "HTTP_URL";
// const char *hit_table_name = "IP_PLUS_CONFIG";
// const char *empty_table_name = "EMPTY_INTERGER";
// struct maat *maat_instance = ExcludeLogic::_shared_maat_instance;
// struct maat_state *state = maat_state_new(maat_instance, thread_id);
// int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name);
// ASSERT_GT(not_hit_table_id, 0);
// int ret = maat_scan_string(maat_instance, not_hit_table_id, string_should_not_hit,
// strlen(string_should_not_hit), results, ARRAY_SIZE,
// &n_hit_result, state);
// EXPECT_EQ(ret, MAAT_SCAN_OK);
// uint32_t sip;
// inet_pton(AF_INET, "10.0.8.187", &sip);
// uint16_t port = htons(18611);
// int proto = 6;
// int hit_table_id = maat_get_table_id(maat_instance, hit_table_name);
// ASSERT_GT(hit_table_id, 0);
// ret = maat_scan_ipv4(maat_instance, hit_table_id, sip, port, proto,
// results, ARRAY_SIZE, &n_hit_result, state);
// EXPECT_EQ(ret, MAAT_SCAN_HIT);
// EXPECT_EQ(n_hit_result, 1);
// EXPECT_EQ(results[0], 187);
// int empty_table_id = maat_get_table_id(maat_instance, empty_table_name);
// ASSERT_GT(empty_table_id, 0);
// ret = maat_scan_integer(maat_instance, empty_table_id, 2015,
// results, ARRAY_SIZE, &n_hit_result, state);
// EXPECT_EQ(ret, MAAT_SCAN_OK);
// maat_state_free(state);
// state = NULL;
// }
TEST_F(ExcludeLogic, ScanNotIP) {
const char *string_should_hit = "This string ONLY contains must-contained-string-of-rule-201.";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
const char *hit_table_name = "HTTP_URL";
const char *not_hit_table_name = "IP_CONFIG";
struct maat *maat_instance = ExcludeLogic::_shared_maat_instance;
struct maat_state *state = maat_state_new(maat_instance, thread_id);
int hit_table_id = maat_get_table_id(maat_instance, hit_table_name);
ASSERT_GT(hit_table_id, 0);
int ret = maat_scan_string(maat_instance, hit_table_id, string_should_hit,
strlen(string_should_hit), results, ARRAY_SIZE,
&n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 201);
uint32_t sip;
inet_pton(AF_INET, "10.0.6.205", &sip);
uint16_t port = htons(50001);
int proto = 6;
int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name);
ASSERT_GT(not_hit_table_id, 0);
ret = maat_scan_ipv4(maat_instance, not_hit_table_id, sip, port, proto,
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
maat_state_free(state);
state = NULL;
}
void maat_read_entry_start_cb(int update_type, void *u_para) void maat_read_entry_start_cb(int update_type, void *u_para)
{ {
@@ -3964,7 +4309,7 @@ TEST_F(MaatCmdTest, SubGroup) {
//group2 -> group1 -> compile1 //group2 -> group1 -> compile1
long long group2_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); long long group2_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
ret = group2group_table_set_line(maat_instance, g2g_table_name, MAAT_OP_ADD, ret = group2group_table_set_line(maat_instance, g2g_table_name, MAAT_OP_ADD,
group2_id, group1_id, 0); group2_id, group1_id, 0, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
/* item1 -> group2 -> group1 -> compile1 /* item1 -> group2 -> group1 -> compile1
@@ -4033,7 +4378,7 @@ TEST_F(MaatCmdTest, SubGroup) {
*/ */
long long group3_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); long long group3_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
ret = group2group_table_set_line(maat_instance, g2g_table_name, MAAT_OP_ADD, group3_id, ret = group2group_table_set_line(maat_instance, g2g_table_name, MAAT_OP_ADD, group3_id,
group1_id, 0); group1_id, 0, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long item2_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); long long item2_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1);
@@ -4999,7 +5344,7 @@ TEST_F(MaatCmdTest, HitPath) {
*/ */
long long group2_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); long long group2_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
ret = group2group_table_set_line(maat_instance, g2g_table_name, MAAT_OP_ADD, group2_id, ret = group2group_table_set_line(maat_instance, g2g_table_name, MAAT_OP_ADD, group2_id,
group21_id, 0); group21_id, 0, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
/* item1 -> group1 -> compile1 /* item1 -> group1 -> compile1

14
test/maat_framework_perf_gtest.cpp
View File

@@ -341,11 +341,11 @@ void *perf_string_scan_thread(void *arg)
struct timespec start, end; struct timespec start, end;
const char *scan_data = "String TEST should hit"; const char *scan_data = "String TEST should hit";
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
int hit_times = 0;
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = maat_state_new(maat_instance, param->thread_id); struct maat_state *state = maat_state_new(maat_instance, param->thread_id);
int table_id = maat_get_table_id(maat_instance, table_name); int table_id = maat_get_table_id(maat_instance, table_name);
int hit_times = 0;
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) { for (int i = 0; i < param->test_count; i++) {
@@ -400,24 +400,18 @@ void *perf_ip_scan_thread(void *arg)
int ret = inet_pton(AF_INET, ip_str, &ip_addr); int ret = inet_pton(AF_INET, ip_str, &ip_addr);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int table_id = maat_get_table_id(maat_instance, table_name);
int hit_times = 0; int hit_times = 0;
int not_hit_times = 0;
clock_gettime(CLOCK_MONOTONIC, &start);
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
printf("param->thread_id:%d\n", param->thread_id);
struct maat_state *state = maat_state_new(maat_instance, param->thread_id); struct maat_state *state = maat_state_new(maat_instance, param->thread_id);
int table_id = maat_get_table_id(maat_instance, table_name);
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) { for (int i = 0; i < param->test_count; i++) {
int ret = maat_scan_ipv4(maat_instance, table_id, ip_addr, port, 6, int ret = maat_scan_ipv4(maat_instance, table_id, ip_addr, port, 6,
results, ARRAY_SIZE, &n_hit_result, state); results, ARRAY_SIZE, &n_hit_result, state);
if (ret == MAAT_SCAN_HIT) { if (ret == MAAT_SCAN_HIT) {
hit_times++; hit_times++;
} else {
not_hit_times++;
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d no hit_times:%d ", param->thread_id, not_hit_times);
} }
maat_state_reset(state); maat_state_reset(state);
} }

311
test/maat_json.json
View File

@@ -2381,6 +2381,317 @@
] ]
} }
] ]
},
{
"compile_id": 199,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "ExcludeLogic.ScanNotAtLast",
"is_valid": "yes",
"groups": [
{
"group_name": "ExcludeLogicGroup199",
"sub_groups":[
{
"group_name": "ExcludeLogicGroup199_1",
"is_exclude": 0,
"clause_index": 0,
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-contained-string-of-rule-199",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"group_name": "ExcludeLogicGroup199_2",
"is_exclude": 1,
"clause_index": 0,
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "expr",
"table_content": {
"keywords": "must-not-contained-string-of-rule-199",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
]
},
{
"compile_id": 200,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "ExcludeLogic.OneRegion",
"is_valid": "yes",
"groups": [
{
"group_name": "ExcludeLogicGroup200",
"sub_groups":[
{
"group_name": "ExcludeLogicGroup200_1",
"is_exclude": 0,
"clause_index": 0,
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-contained-string-of-rule-200",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"group_name": "ExcludeLogicGroup200_2",
"is_exclude": 1,
"clause_index": 0,
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-not-contained-string-of-rule-200",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
}
]
},
{
"compile_id": 201,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "ExcludeLogic.ScanNotIP",
"is_valid": "yes",
"groups": [
{
"group_name": "ExcludeLogicGroup201",
"sub_groups":[
{
"group_name": "ExcludeLogicGroup201_1",
"is_exclude": 0,
"clause_index": 0,
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "expr",
"table_content": {
"keywords": "must-contained-string-of-rule-201",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"group_name": "123_IP_group",
"is_exclude": 1,
"clause_index": 0
}
]
}
]
},
{
"compile_id": 202,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "null",
"is_valid": "yes",
"groups": [
{
"group_name": "ExcludeLogicGroup202",
"virtual_table": "VIRTUAL_IP_PLUS_TABLE",
"clause_index": 0,
"sub_groups":[
{
"group_name": "ExcludeLogicGroup202_1",
"is_exclude": 0,
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv4",
"addr_format": "range",
"ip1": "100.64.1.0",
"ip2": "100.64.1.20",
"port_format": "range",
"port1": "5210",
"port2": "5211",
"protocol": 6
}
}
]
},
{
"group_name": "ExcludeLogicGroup202_2",
"is_exclude": 1,
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv4",
"addr_format": "range",
"ip1": "100.64.1.6",
"ip2": "100.64.1.10",
"port_format": "range",
"port1": "5210",
"port2": "5211",
"protocol": 6
}
}
]
},
{
"group_name": "ExcludeLogicGroup202_3",
"is_exclude": 1,
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv4",
"addr_format": "range",
"ip1": "100.64.1.11",
"ip2": "100.64.1.20",
"port_format": "range",
"port1": "5210",
"port2": "5211",
"protocol": 6
}
}
]
}
]
}
]
},
{
"compile_id": 203,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "null",
"is_valid": "yes",
"groups": [
{
"group_name": "ExcludeLogicGroup203_1",
"virtual_table": "VIRTUAL_IP_PLUS_SOURCE",
"clause_index": 0,
"regions": [
{
"table_name": "IP_PLUS_CONFIG",
"table_type": "ip_plus",
"table_content": {
"addr_type": "ipv4",
"addr_format": "range",
"ip1": "192.168.50.43",
"ip2": "192.168.50.43",
"port_format": "range",
"port1": "56168",
"port2": "56168",
"protocol": -1,
"direction": "double"
}
}
]
},
{
"group_name": "ExcludeLogicGroup203_2",
"virtual_table": "VIRTUAL_IP_PLUS_DESTINATION",
"clause_index": 1,
"regions": [
{
"table_name": "IP_PLUS_CONFIG",
"table_type": "ip_plus",
"table_content": {
"addr_type": "ipv4",
"addr_format": "range",
"ip1": "47.92.108.93",
"ip2": "47.92.108.93",
"port_format": "range",
"port1": "443",
"port2": "443",
"protocol": -1,
"direction": "double"
}
}
]
},
{
"group_name": "ExcludeLogicGroup203_3",
"virtual_table": "HTTP_RESPONSE_KEYWORDS",
"clause_index": 2,
"sub_groups": [
{
"group_name": "ExcludeLogicGroup203_3_1",
"is_exclude": 0,
"regions": [
{
"table_type":"expr",
"table_name":"KEYWORDS_TABLE",
"table_content":{
"format":"uncase plain",
"match_method":"suffix",
"keywords":".com",
"expr_type":"none"
}
}
]
},
{
"group_name": "ExcludeLogicGroup203_3_2",
"is_exclude": 1,
"regions": [
{
"table_type":"expr",
"table_name":"KEYWORDS_TABLE",
"table_content":{
"format":"uncase plain",
"match_method":"complete",
"keywords":"jianshu.com",
"expr_type":"none"
}
}
]
}
]
}
]
} }
], ],
"plugin_table": [ "plugin_table": [

5
test/table_info.conf
View File

@@ -55,10 +55,11 @@
"table_id":4, "table_id":4,
"table_name":"GROUP2GROUP", "table_name":"GROUP2GROUP",
"table_type":"group2group", "table_type":"group2group",
"valid_column":3, "valid_column":4,
"custom": { "custom": {
"group_id":1, "group_id":1,
"super_group_id":2 "super_group_id":2,
"is_exclude":3
} }
}, },
{ {