diff --git a/scanner/expr_matcher/adapter_hs/adapter_hs.cpp b/scanner/expr_matcher/adapter_hs/adapter_hs.cpp index fc0a2b5..0140b7a 100644 --- a/scanner/expr_matcher/adapter_hs/adapter_hs.cpp +++ b/scanner/expr_matcher/adapter_hs/adapter_hs.cpp @@ -739,7 +739,7 @@ int hs_build_lit_db(void **hs_lit_db, void *compile_data, struct log_handle *log NULL, (hs_database_t **)hs_lit_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { - log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s", + log_fatal(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s", __FUNCTION__, __LINE__, compile_err->message); } @@ -766,7 +766,7 @@ int hs_build_regex_db(void **hs_regex_db, void *compile_data, struct log_handle &compile_err); if (err != HS_SUCCESS) { if (compile_err) { - log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s", + log_fatal(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s", __FUNCTION__, __LINE__, compile_err->message); } hs_free_compile_error(compile_err); diff --git a/src/inc_internal/maat_limits.h b/src/inc_internal/maat_limits.h index 75b9b81..c8685fb 100644 --- a/src/inc_internal/maat_limits.h +++ b/src/inc_internal/maat_limits.h @@ -21,6 +21,7 @@ extern "C" #define MAX_NAME_STR_LEN 64 #define MAX_IP_STR_LEN 64 #define MAX_INSTANCE_NAME_LEN 15 +#define MAX_GROUP_IDS_STR_LEN 256 #ifdef __cplusplus } diff --git a/src/json2iris.c b/src/json2iris.c index 56758cf..a439f3d 100644 --- a/src/json2iris.c +++ b/src/json2iris.c @@ -660,7 +660,8 @@ static int write_region_rule(cJSON *region_json, int compile_id, int group_id, return ret; } -static int write_group2compile_line(int group_id, int compile_id, int group_not_flag, +static int write_group2compile_line(int *group_ids, size_t n_group_id, + int compile_id, int group_not_flag, int clause_index, const char *vtable, struct iris_description *p_iris, struct iris_table *g2c_table) @@ -677,8 +678,22 @@ static int write_group2compile_line(int group_id, int compile_id, int group_not_ table = p_iris->group2compile_table; } - snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, - group_not_flag, vtable, clause_index); + if (n_group_id > 1) { + char tmp_str[64] = {0}; + char group_id_str[2048] = {0}; + + for (size_t i = 0; i < n_group_id; i++) { + snprintf(tmp_str, sizeof(tmp_str), "%d,", group_ids[i]); + strcat(group_id_str, tmp_str); + } + group_id_str[strlen(group_id_str) - 1] = '\0'; + snprintf(buff, sizeof(buff), "%s\t%d\t1\t%d\t%s\t%d\n", group_id_str, compile_id, + group_not_flag, vtable, clause_index); + } else { + snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_ids[0], compile_id, + group_not_flag, vtable, clause_index); + } + table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); table->line_count++; @@ -714,15 +729,32 @@ static int write_group_rule(cJSON *group_json, int parent_id, int clause_index = 0, is_exclude = 0; const char *str_parent_type[2] = {"compile", "group"}; const char *group_name = NULL; + char group_name_array[32][MAX_NAME_STR_LEN] = {{0},}; + size_t group_name_cnt = 0; long long group_id = -1; const char *virtual_table = NULL; struct iris_table *g2c_table = NULL; cJSON *item = cJSON_GetObjectItem(group_json, "group_name"); - if (NULL == item || item->type != cJSON_String) { + if (NULL == item) { group_name = untitled_group_name; - } else { + } else if (item->type == cJSON_String) { group_name = item->valuestring; + } else if (item->type == cJSON_Array) { + group_name_cnt = cJSON_GetArraySize(item); + assert(group_name_cnt <= 32); + for (size_t i = 0; i < group_name_cnt; i++) { + cJSON *tmp_json = cJSON_GetArrayItem(item, i); + if (NULL == tmp_json || tmp_json->type != cJSON_String) { + log_fatal(logger, MODULE_JSON2IRIS, + "[%s:%d] group_name of compile:%d format error", + __FUNCTION__, __LINE__, parent_id); + return -1; + } + memset(group_name_array[i], 0, sizeof(group_name_array[i])); + memcpy(group_name_array[i], tmp_json->valuestring, + strlen(tmp_json->valuestring)); + } } item = cJSON_GetObjectItem(group_json, "group_id"); @@ -768,66 +800,89 @@ static int write_group_rule(cJSON *group_json, int parent_id, group_not_flag = 0; } - struct group_info *group_info = group_info_read(p_iris->group_name_map, group_name); - //exist group name, regions and sub groups will be ommit. - if (NULL == group_info) { - if (0 == strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name))) { - group_id = untitled_group_id; - } - - if (-1 == group_id) { - log_fatal(logger, MODULE_JSON2IRIS, - "[%s:%d] group_name:<%s> has no group_id", - __FUNCTION__, __LINE__, group_name); - return -1; - } - - group_info = group_info_add_unsafe(p_iris, group_name, group_id); - cJSON *region_json = cJSON_GetObjectItem(group_json, "regions"); - if (region_json != NULL) { - cJSON *region_rule = NULL; - cJSON_ArrayForEach(region_rule, region_json) { - ret = write_region_rule(region_rule, tracking_compile_id, - group_info->group_id, p_iris, logger); - if (ret < 0) { - log_fatal(logger, MODULE_JSON2IRIS, - "[%s:%d] compile rule %d write region error", - __FUNCTION__, __LINE__, tracking_compile_id); - return -1; - } - } - } - - cJSON *sub_groups = cJSON_GetObjectItem(group_json, "sub_groups"); - if (sub_groups != NULL) { - //recursively - int i = 0; - cJSON_ArrayForEach(item, sub_groups) { - i++; - ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, - tracking_compile_id, i, p_iris, logger); - if (ret < 0) { - return -1; - } + if (group_name_cnt > 0) { + int group_ids[group_name_cnt]; + for (size_t i = 0; i < group_name_cnt; i++) { + struct group_info *group_info = group_info_read(p_iris->group_name_map, + group_name_array[i]); + if (NULL == group_info) { + log_fatal(logger, MODULE_JSON2IRIS, "[%s:%d] group_name:%s has no group_id", + __FUNCTION__, __LINE__, group_name_array[i]); + return -1; } + group_ids[i] = group_info->group_id; } + assert(parent_type == PARENT_TYPE_COMPILE); + ret = write_group2compile_line(group_ids, group_name_cnt, parent_id, + group_not_flag, clause_index, + virtual_table, p_iris, g2c_table); - if (NULL == region_json && NULL == sub_groups) { - log_info(logger, MODULE_JSON2IRIS, - "[%s:%d] A group of compile rule %d has neither regions, " - "sub groups, nor refered another existed group", - __FUNCTION__, __LINE__, tracking_compile_id); - } - } + } else { + struct group_info *group_info = group_info_read(p_iris->group_name_map, group_name); + // exist group name, regions and sub groups will be ommit. + if (NULL == group_info) { + if (0 == strncasecmp(group_name, untitled_group_name, + strlen(untitled_group_name))) { + group_id = untitled_group_id; + } - if (parent_type == PARENT_TYPE_COMPILE) { - ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag, - clause_index, virtual_table, p_iris, g2c_table); - } else { - ret = write_group2group_line(group_info->group_id, parent_id, is_exclude, p_iris); - } + if (-1 == group_id) { + log_fatal(logger, MODULE_JSON2IRIS, + "[%s:%d] group_name:<%s> has no group_id", __FUNCTION__, + __LINE__, group_name); + return -1; + } - if (ret < 0) { + group_info = group_info_add_unsafe(p_iris, group_name, group_id); + cJSON *region_json = cJSON_GetObjectItem(group_json, "regions"); + if (region_json != NULL) { + cJSON *region_rule = NULL; + cJSON_ArrayForEach(region_rule, region_json) { + ret = write_region_rule(region_rule, tracking_compile_id, + group_info->group_id, p_iris, logger); + if (ret < 0) { + log_fatal(logger, MODULE_JSON2IRIS, + "[%s:%d] compile rule %d write region error", + __FUNCTION__, __LINE__, tracking_compile_id); + return -1; + } + } + } + + cJSON *sub_groups = cJSON_GetObjectItem(group_json, "sub_groups"); + if (sub_groups != NULL) { + // recursively + int i = 0; + cJSON_ArrayForEach(item, sub_groups) { + i++; + ret = write_group_rule(item, group_info->group_id, + PARENT_TYPE_GROUP, tracking_compile_id, + i, p_iris, logger); + if (ret < 0) { + return -1; + } + } + } + + if (NULL == region_json && NULL == sub_groups) { + log_info(logger, MODULE_JSON2IRIS, + "[%s:%d] A group of compile rule %d has neither regions, " + "sub groups, nor refered another existed group", + __FUNCTION__, __LINE__, tracking_compile_id); + } + } + + if (parent_type == PARENT_TYPE_COMPILE) { + ret = write_group2compile_line(&(group_info->group_id), 1, parent_id, + group_not_flag, clause_index, + virtual_table, p_iris, g2c_table); + } else { + ret = write_group2group_line(group_info->group_id, parent_id, + is_exclude, p_iris); + } + } + + if (ret < 0) { log_fatal(logger, MODULE_JSON2IRIS, "[%s:%d] %s rule %d write group error", __FUNCTION__, __LINE__, str_parent_type[parent_type], parent_id); diff --git a/src/maat_compile.c b/src/maat_compile.c index 7d7fec2..44e1f8f 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -28,8 +28,9 @@ #define MODULE_COMPILE module_name_str("maat.compile") -#define MAX_SUPER_GROUP_CNT 128 -#define MAX_NOT_CLAUSE_NUM 8 +#define MAX_GROUP_CNT 128 +#define MAX_SUPER_GROUP_CNT 128 +#define MAX_NOT_CLAUSE_NUM 8 enum clause_not_flag { CLAUSE_NOT_FLAG_UNSET = 0, @@ -65,21 +66,21 @@ struct compile_item { }; struct group2compile_item { - long long group_id; + UT_array *group_ids; long long compile_id; int not_flag; int vtable_id; int clause_index; }; -struct literal_id { +struct clause_query_key { long long group_id; - int vtable_id; + int vtable_id; int not_flag; }; -struct literal_clause { - struct literal_id key; +struct clause_id_kv { + struct clause_query_key key; UT_array *clause_ids; UT_hash_handle hh; }; @@ -104,15 +105,15 @@ struct compile_runtime { struct rcu_hash_table *cfg_hash; // struct rcu_hash_table *tbl_cfg_hash; // struct maat_runtime *ref_maat_rt; - time_t version; - struct literal_clause *literal2clause_hash; //store clause_ids(not_flag == 0) - struct literal_clause *literal2not_clause_hash; //store NOT_clause_ids(not_flag == 1) - - long long rule_num; - long long update_err_cnt; + struct clause_id_kv *clause_id_kv_hash; //store clause_ids(not_flag == 0) + struct clause_id_kv *not_clause_id_kv_hash; //store NOT_clause_ids(not_flag == 1) struct bool_expr_match *expr_match_buff; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + time_t version; + + long long rule_num; + long long update_err_cnt; }; struct group2compile_runtime { @@ -123,9 +124,15 @@ struct group2compile_runtime { struct table_clause *tbl_not_clause_hash; //each virtual table's not clause number <= MAX_NOT_CLAUSE_NUM }; -struct maat_clause { +struct clause_literal { + long long group_ids[MAX_GROUP_CNT]; + int group_cnt; + int vtable_id; +}; + +struct compile_clause { long long clause_id; - UT_array *literal_ids; + UT_array *literals; char not_flag; // 1 byte char in_use; // 1 byte char pad[6]; // for 8 bytes alignment @@ -144,7 +151,7 @@ struct maat_compile { long long compile_id; char table_name[MAX_NAME_STR_LEN]; void *user_data; // compile_item - struct maat_clause clauses[MAX_ITEMS_PER_BOOL_EXPR]; + struct compile_clause clauses[MAX_ITEMS_PER_BOOL_EXPR]; }; struct internal_hit_path { @@ -169,14 +176,15 @@ struct compile_state { UT_array *all_hit_clauses; UT_array *this_scan_hit_clauses; UT_array *this_scan_hit_not_clauses; + UT_array *exclude_not_clauses; UT_array *direct_hit_groups; UT_array *indirect_hit_groups; UT_array *hit_compile_table_ids; - struct table_group *hit_not_groups; + struct table_group *hit_not_tbl_groups; }; -UT_icd ut_literal_id_icd = {sizeof(struct literal_id), NULL, NULL, NULL}; UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL}; +UT_icd ut_clause_literal_icd = {sizeof(struct clause_literal), NULL, NULL, NULL}; UT_icd ut_compile_group_id_icd = {sizeof(long long), NULL, NULL, NULL}; UT_icd ut_maat_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL}; UT_icd ut_hit_path_icd = {sizeof(struct internal_hit_path), NULL, NULL, NULL}; @@ -190,7 +198,7 @@ static struct maat_compile *maat_compile_new(long long compile_id) compile->compile_id = compile_id; for(int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - utarray_new(compile->clauses[i].literal_ids, &ut_literal_id_icd); + utarray_new(compile->clauses[i].literals, &ut_clause_literal_icd); compile->clauses[i].in_use = 0; compile->clauses[i].clause_id = 0; } @@ -320,7 +328,7 @@ static void compile_item_free(struct compile_item *item) static void maat_compile_free(struct maat_compile *compile) { - struct maat_clause *clause = NULL; + struct compile_clause *clause = NULL; if (compile->user_data != NULL) { compile_item_free(compile->user_data); @@ -330,9 +338,9 @@ static void maat_compile_free(struct maat_compile *compile) for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { clause = compile->clauses + i; - if (clause->literal_ids != NULL) { - utarray_free(clause->literal_ids); - clause->literal_ids = NULL; + if (clause->literals != NULL) { + utarray_free(clause->literals); + clause->literals = NULL; } clause->in_use = 0; @@ -535,33 +543,33 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num, compile_rt->version = time(NULL); compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL, 0); compile_rt->tbl_cfg_hash = rcu_hash_new(rcu_compile_table_cfg_free, NULL, 0); - compile_rt->literal2clause_hash = NULL; - compile_rt->literal2not_clause_hash = NULL; + compile_rt->clause_id_kv_hash = NULL; + compile_rt->not_clause_id_kv_hash = NULL; compile_rt->logger = logger; compile_rt->ref_garbage_bin = garbage_bin; return compile_rt; } -static void literal2clause_hash_free(struct literal_clause *hash) +static void clause_id_kv_hash_free(struct clause_id_kv *hash) { - struct literal_clause *l2c_val = NULL, *tmp_l2c_val = NULL; + struct clause_id_kv *clause_id_kv = NULL, *tmp_clause_id_kv = NULL; - HASH_ITER(hh, hash, l2c_val, tmp_l2c_val) { - HASH_DEL(hash, l2c_val); - if (l2c_val->clause_ids != NULL) { - utarray_free(l2c_val->clause_ids); - l2c_val->clause_ids = NULL; + HASH_ITER(hh, hash, clause_id_kv, tmp_clause_id_kv) { + HASH_DEL(hash, clause_id_kv); + if (clause_id_kv->clause_ids != NULL) { + utarray_free(clause_id_kv->clause_ids); + clause_id_kv->clause_ids = NULL; } - FREE(l2c_val); + FREE(clause_id_kv); } assert(hash == NULL); } -static void garbage_literal2clause_hash_free(void *l2c_hash, void *arg) +static void garbage_clause_id_kv_hash_free(void *hash, void *arg) { - literal2clause_hash_free((struct literal_clause *)l2c_hash); + clause_id_kv_hash_free((struct clause_id_kv *)hash); } void compile_runtime_free(void *compile_runtime) @@ -587,14 +595,14 @@ void compile_runtime_free(void *compile_runtime) compile_rt->tbl_cfg_hash = NULL; } - if (compile_rt->literal2clause_hash != NULL) { - literal2clause_hash_free(compile_rt->literal2clause_hash); - compile_rt->literal2clause_hash = NULL; + if (compile_rt->clause_id_kv_hash != NULL) { + clause_id_kv_hash_free(compile_rt->clause_id_kv_hash); + compile_rt->clause_id_kv_hash = NULL; } - if (compile_rt->literal2not_clause_hash != NULL) { - literal2clause_hash_free(compile_rt->literal2not_clause_hash); - compile_rt->literal2not_clause_hash = NULL; + if (compile_rt->not_clause_id_kv_hash != NULL) { + clause_id_kv_hash_free(compile_rt->not_clause_id_kv_hash); + compile_rt->not_clause_id_kv_hash = NULL; } if (compile_rt->expr_match_buff != NULL) { @@ -671,6 +679,47 @@ static int is_valid_table_name(const char *str) return 1; } +static int group_ids_str2longlong(const char *group_ids_str, UT_array *group_ids) +{ + int counter = 0; + char *str = NULL; + char *saveptr = NULL; + char *subtoken = NULL; + const char *seps = ","; + char *dup_line = maat_strdup(group_ids_str); + + for (str = dup_line; ; str = NULL) { + subtoken = strtok_r(str, seps, &saveptr); + if (subtoken == NULL) + break; + long long group_id = atoll(subtoken); + utarray_push_back(group_ids, &group_id); + counter++; + } + + FREE(dup_line); + + if (0 == counter) { + return -1; + } + + return 0; +} + +static void group2compile_item_free(struct group2compile_item *g2c_item) +{ + if (NULL == g2c_item) { + return; + } + + if (g2c_item->group_ids != NULL) { + utarray_free(g2c_item->group_ids); + g2c_item->group_ids = NULL; + } + + FREE(g2c_item); +} + static struct group2compile_item * group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema, const char *table_name, struct log_handle *logger) @@ -679,6 +728,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema size_t column_len = 0; char vtable_name[MAX_NAME_STR_LEN + 1] = {0}; struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1); + utarray_new(g2c_item->group_ids, &ut_compile_group_id_icd); int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len); @@ -688,7 +738,24 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema __FUNCTION__, __LINE__, table_name, line); goto error; } - g2c_item->group_id = atoll(line + column_offset); + + char group_ids_str[MAX_GROUP_IDS_STR_LEN] = {0}; + memcpy(group_ids_str, line + column_offset, MIN(MAX_GROUP_IDS_STR_LEN, column_len)); + + ret = group_ids_str2longlong(group_ids_str, g2c_item->group_ids); + if (ret < 0) { + log_fatal(logger, MODULE_COMPILE, + "[%s:%d] g2c table:<%s> group_ids str2longlong failed in line:%s", + __FUNCTION__, __LINE__, table_name, line); + goto error; + } + + if (utarray_len(g2c_item->group_ids) > MAX_GROUP_CNT) { + log_fatal(logger, MODULE_COMPILE, + "[%s:%d] g2c table:<%s> group_ids exceed maximum:%d in line:%s", + __FUNCTION__, __LINE__, table_name, MAX_GROUP_CNT, line); + goto error; + } ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset, &column_len); @@ -768,87 +835,111 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema } return g2c_item; + error: - FREE(g2c_item); + group2compile_item_free(g2c_item); return NULL; } -static void group2compile_item_free(struct group2compile_item *g2c_item) +static inline int compare_group_id(const void *a, const void *b) { - if (NULL == g2c_item) { - return; - } + long long ret = *(const long long *)a - *(const long long *)b; - FREE(g2c_item); + if (0 == ret) { + return 0; + } else if(ret < 0) { + return -1; + } else { + return 1; + } } -static int compare_literal_id(const void *pa, const void *pb) +void compile_clause_add_literal(struct compile_clause *clause, + struct group2compile_item *g2c_item) { - struct literal_id *la = (struct literal_id *)pa; - struct literal_id *lb = (struct literal_id *)pb; + struct clause_literal tmp_literal; + tmp_literal.vtable_id = g2c_item->vtable_id; + tmp_literal.group_cnt = utarray_len(g2c_item->group_ids); + utarray_sort(g2c_item->group_ids, compare_group_id); - long long ret = la->vtable_id - lb->vtable_id; - if (0 == ret) { - ret = la->group_id - lb->group_id; - if (0 == ret) { - ret = la->not_flag - lb->not_flag; + for (size_t i = 0; i < utarray_len(g2c_item->group_ids); i++) { + tmp_literal.group_ids[i] = *(long long *)utarray_eltptr(g2c_item->group_ids, i); + } + utarray_push_back(clause->literals, &tmp_literal); +} + +void compile_clause_remove_literal(struct compile_clause *clause, + struct group2compile_item *g2c_item) +{ + struct clause_literal *tmp_literal = NULL; + + for (size_t i = 0; i < utarray_len(clause->literals); i++) { + tmp_literal = (struct clause_literal *)utarray_eltptr(clause->literals, i); + if (tmp_literal->vtable_id == g2c_item->vtable_id) { + size_t remove_idx = utarray_eltidx(clause->literals, tmp_literal); + utarray_erase(clause->literals, remove_idx, 1); + break; } - } + } +} - return ret; +int compile_clause_find_literal(struct compile_clause *clause, + struct group2compile_item *g2c_item) +{ + int found = 0; + struct clause_literal *tmp_literal = NULL; + + for (size_t i = 0; i < utarray_len(clause->literals); i++) { + tmp_literal = (struct clause_literal *)utarray_eltptr(clause->literals, i); + if (tmp_literal->vtable_id == g2c_item->vtable_id) { + found = 1; + } + } + + return found; } static int maat_compile_clause_add_literal(struct maat_compile *compile, - struct literal_id *literal_id, - int clause_index, int clause_not_flag) + struct group2compile_item *g2c_item) { - struct maat_clause *clause = compile->clauses + clause_index; + struct compile_clause *clause = compile->clauses + g2c_item->clause_index; - clause->not_flag = clause_not_flag; - if (!clause->in_use) { - clause->in_use = 1; - compile->actual_clause_num++; - } + clause->not_flag = g2c_item->not_flag; - struct literal_id *tmp = NULL; - tmp = (struct literal_id *)utarray_find(clause->literal_ids, - literal_id, compare_literal_id); - if (tmp) { - assert(tmp->group_id == literal_id->group_id); - assert(tmp->vtable_id == literal_id->vtable_id); - assert(tmp->not_flag == literal_id->not_flag); - return -1; - } else { - utarray_push_back(clause->literal_ids, literal_id); - utarray_sort(clause->literal_ids, compare_literal_id); - } + if (0 == clause->in_use) { + clause->in_use = 1; + compile->actual_clause_num++; + } - return 0; + if (compile_clause_find_literal(clause, g2c_item) > 0) { + //found + return -1; + } + + compile_clause_add_literal(clause, g2c_item); + + return 0; } static int maat_compile_clause_remove_literal(struct maat_compile *compile, - struct literal_id *literal_id, - int clause_index) + struct group2compile_item *g2c_item) { - struct maat_clause *clause = compile->clauses + clause_index; - struct literal_id *tmp = NULL; - tmp = (struct literal_id *)utarray_find(clause->literal_ids, - literal_id, compare_literal_id); - if (tmp) { - assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); - } else { - return -1; - } + struct compile_clause *clause = compile->clauses + g2c_item->clause_index; - size_t remove_idx = utarray_eltidx(clause->literal_ids, tmp); - utarray_erase(clause->literal_ids, remove_idx, 1); - if (0 == utarray_len(clause->literal_ids)) { - clause->in_use = 0; - compile->actual_clause_num--; - } + if (0 == compile_clause_find_literal(clause, g2c_item)) { + //not found + return -1; + } - return 0; + compile_clause_remove_literal(clause, g2c_item); + + if (0 == utarray_len(clause->literals)) { + clause->in_use = 0; + compile->actual_clause_num--; + } + + return 0; } static struct bool_matcher * @@ -861,7 +952,7 @@ maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, size_t *compil size_t i = 0, j = 0, idx = 0; int has_clause_num = 0; - // STEP 1, update clause_id of each compile and literal + // STEP 1, update clause_id of each compile void **data_array = NULL; struct maat_compile *iter_compile = NULL; size_t rule_cnt = rcu_updating_hash_list(compile_rt->cfg_hash, &data_array); @@ -871,7 +962,7 @@ maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, size_t *compil iter_compile = (struct maat_compile *)data_array[idx]; has_clause_num = 0; for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - struct maat_clause *clause = iter_compile->clauses + i; + struct compile_clause *clause = iter_compile->clauses + i; if (!clause->in_use) { continue; } @@ -894,11 +985,14 @@ maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, size_t *compil if (iter_compile->clauses[i].in_use) { // TODO:mytest need to delete #if 0 - struct literal_id *p = NULL; - for(p = (struct literal_id *)utarray_front(iter_compile->clauses[i].literal_ids); p!=NULL; - p = (struct literal_id *)utarray_next(iter_compile->clauses[i].literal_ids, p)) { - printf(" compile_rt:%p compile_id:%lld, clause_id:%llu, literal{%lld: %d, %d}\n", - compile_rt, iter_compile->compile_id, iter_compile->clauses[i].clause_id, p->group_id, p->vtable_id, p->not_flag); + struct clause_literal *tmp_cl = NULL; + for(tmp_cl = (struct clause_literal *)utarray_front(iter_compile->clauses[i].literals); tmp_cl !=NULL; + tmp_cl = (struct clause_literal *)utarray_next(iter_compile->clauses[i].literals, tmp_cl)) { + for (size_t it = 0; it < tmp_cl->group_cnt; it++) { + printf(" compile_rt:%p compile_id:%lld, clause_id:%llu, clause_query_key{%lld: %d, %d}\n", + compile_rt, iter_compile->compile_id, iter_compile->clauses[i].clause_id, tmp_cl->group_ids[it], + tmp_cl->vtable_id, iter_compile->clauses[i].not_flag); + } } #endif bool_expr_array[expr_cnt].items[j].item_id = iter_compile->clauses[i].clause_id; @@ -985,32 +1079,30 @@ static inline int compare_compile_id(const void *a, const void *b) } /** - * @brief build hash for clause or not_clause + * @brief build hash for clause or not_clause * * @param compile_rt: compile runtime handle * @param not_flag: specify whether to build clause or NOT_clause hash for compile runtime * 0 -> clause hash * 1 -> NOT_clause hash * - * @retval generated literal clause hash + * @retval generated clause_id_kv_hash */ -static struct literal_clause * -maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt, int not_flag) +static struct clause_id_kv * +build_clause_id_kv_hash(struct compile_runtime *compile_rt, int not_flag) { if (NULL == compile_rt) { return NULL; } void **data_array = NULL; - struct literal_id *tmp_literal_id = NULL; - struct literal_clause *l2c_value = NULL; - struct literal_clause *literal2clause_hash = NULL; + struct clause_id_kv *clause_id_kv_hash = NULL; size_t compile_cnt = rcu_updating_hash_list(compile_rt->cfg_hash, &data_array); for (size_t idx = 0; idx < compile_cnt; idx++) { struct maat_compile *compile = (struct maat_compile *)data_array[idx]; for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - struct maat_clause *clause = compile->clauses + i; + struct compile_clause *clause = compile->clauses + i; if (!clause->in_use) { continue; } @@ -1025,32 +1117,38 @@ maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt, int n } } - for (size_t j = 0; j < utarray_len(clause->literal_ids); j++) { - tmp_literal_id = (struct literal_id *)utarray_eltptr(clause->literal_ids, j); - HASH_FIND(hh, literal2clause_hash, tmp_literal_id, sizeof(struct literal_id), l2c_value); - if (NULL == l2c_value) { - l2c_value = ALLOC(struct literal_clause, 1); - l2c_value->key = *tmp_literal_id; - utarray_new(l2c_value->clause_ids, &ut_clause_id_icd); - HASH_ADD(hh, literal2clause_hash, key, sizeof(l2c_value->key), l2c_value); - } + struct clause_literal *tmp_cl = NULL; + for (size_t j = 0; j < utarray_len(clause->literals); j++) { + tmp_cl = (struct clause_literal *)utarray_eltptr(clause->literals, j); + for (size_t k = 0; k < tmp_cl->group_cnt; k++) { + struct clause_query_key key = {tmp_cl->group_ids[k], tmp_cl->vtable_id, clause->not_flag}; + struct clause_id_kv *clause_id_kv = NULL; + + HASH_FIND(hh, clause_id_kv_hash, &key, sizeof(struct clause_query_key), clause_id_kv); + if (NULL == clause_id_kv) { + clause_id_kv = ALLOC(struct clause_id_kv, 1); + clause_id_kv->key = key; + utarray_new(clause_id_kv->clause_ids, &ut_clause_id_icd); + HASH_ADD_KEYPTR(hh, clause_id_kv_hash, &clause_id_kv->key, sizeof(clause_id_kv->key), clause_id_kv); + } - if (utarray_find(l2c_value->clause_ids, &(clause->clause_id), compare_clause_id)) { - continue; + if (utarray_find(clause_id_kv->clause_ids, &(clause->clause_id), compare_clause_id)) { + continue; + } + utarray_push_back(clause_id_kv->clause_ids, &(clause->clause_id)); + utarray_sort(clause_id_kv->clause_ids, compare_clause_id); } - utarray_push_back(l2c_value->clause_ids, &(clause->clause_id)); - utarray_sort(l2c_value->clause_ids, compare_clause_id); } } } FREE(data_array); - return literal2clause_hash; + return clause_id_kv_hash; } static int maat_compile_has_clause(struct maat_compile *compile, long long clause_id) { - struct maat_clause *clause = NULL; + struct compile_clause *clause = NULL; for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { clause = compile->clauses + i; @@ -1203,15 +1301,15 @@ maat_compile_clone(struct maat_compile *compile, int deep_copy) new_compile->user_data = compile_item_clone((struct compile_item *)compile->user_data); } - struct literal_id *literal_id = NULL; - for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + struct clause_literal *tmp_literal = NULL; + for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { new_compile->clauses[i].clause_id = compile->clauses[i].clause_id; new_compile->clauses[i].in_use = compile->clauses[i].in_use; new_compile->clauses[i].not_flag = compile->clauses[i].not_flag; - utarray_new(new_compile->clauses[i].literal_ids, &ut_literal_id_icd); - for (int j = 0; j < utarray_len(compile->clauses[i].literal_ids); j++) { - literal_id = (struct literal_id *)utarray_eltptr(compile->clauses[i].literal_ids, j); - utarray_push_back(new_compile->clauses[i].literal_ids, literal_id); + utarray_new(new_compile->clauses[i].literals, &ut_clause_literal_icd); + for (size_t j = 0; j < utarray_len(compile->clauses[i].literals); j++) { + tmp_literal = (struct clause_literal *)utarray_eltptr(compile->clauses[i].literals, j); + utarray_push_back(new_compile->clauses[i].literals, tmp_literal); } } @@ -1219,39 +1317,33 @@ maat_compile_clone(struct maat_compile *compile, int deep_copy) } static int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, - struct group2compile_item *g2c_item, - struct log_handle *logger) + struct group2compile_item *g2c_item, + struct log_handle *logger) { int ret = -1; long long compile_id = g2c_item->compile_id; struct maat_compile *compile = NULL; - struct literal_id literal_id = {g2c_item->group_id, g2c_item->vtable_id, - g2c_item->not_flag}; int updating_flag = rcu_hash_is_updating(hash_tbl); if (1 == updating_flag) { compile = rcu_updating_hash_find(hash_tbl, (char *)&compile_id, sizeof(long long)); if (compile != NULL) { /* compile found in updating hash(added by compile runtime), it can be modified directly */ - ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, - g2c_item->not_flag); + ret = maat_compile_clause_add_literal(compile, g2c_item); if (ret < 0) { log_fatal(logger, MODULE_COMPILE, - "[%s:%d] add literal_id{group_id:%lld, vtable_id:%d} to clause_index: %d" - " of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id, - g2c_item->vtable_id, g2c_item->clause_index, compile_id); + "[%s:%d] add clause(index:%d) to compile %lld failed", + __FUNCTION__, __LINE__, g2c_item->clause_index, compile_id); } } else { /* compile neither in effective hash nor in updating hash, so new one */ compile = maat_compile_new(compile_id); assert(compile != NULL); - ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, - g2c_item->not_flag); + ret = maat_compile_clause_add_literal(compile, g2c_item); if (ret < 0) { log_fatal(logger, MODULE_COMPILE, - "[%s:%d] add literal_id{group_id:%lld, vtable_id:%d} to clause_index: %d" - " of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id, - g2c_item->vtable_id, g2c_item->clause_index, compile_id); + "[%s:%d] add clause(index:%d) to compile %lld failed", + __FUNCTION__, __LINE__, g2c_item->clause_index, compile_id); } rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), compile); } @@ -1273,26 +1365,22 @@ static int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, /* delete compile from rcu hash */ rcu_hash_del(hash_tbl, (char *)&compile_id, sizeof(long long)); - ret = maat_compile_clause_add_literal(copy_compile, &literal_id, g2c_item->clause_index, - g2c_item->not_flag); + ret = maat_compile_clause_add_literal(copy_compile, g2c_item); if (ret < 0) { log_fatal(logger, MODULE_COMPILE, - "[%s:%d] add literal_id{group_id:%lld, vtable_id:%d} to clause_index: %d" - " of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id, - g2c_item->vtable_id, g2c_item->clause_index, compile_id); + "[%s:%d] add clause(index:%d) to compile %lld failed", + __FUNCTION__, __LINE__, g2c_item->clause_index, compile_id); } rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), copy_compile); } else { compile = maat_compile_new(compile_id); assert(compile != NULL); - ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, - g2c_item->not_flag); + ret = maat_compile_clause_add_literal(compile, g2c_item); if (ret < 0) { log_fatal(logger, MODULE_COMPILE, - "[%s:%d] add literal_id{group_id:%lld, vtable_id:%d} to clause_index: %d" - " of compile %d failed", __FUNCTION__, __LINE__, g2c_item->group_id, - g2c_item->vtable_id, g2c_item->clause_index, compile_id); + "[%s:%d] add clause(index:%d) to compile %lld failed", + __FUNCTION__, __LINE__, g2c_item->clause_index, compile_id); } rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), compile); } @@ -1302,14 +1390,12 @@ static int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl, } static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl, - struct group2compile_item *g2c_item, - struct log_handle *logger) + struct group2compile_item *g2c_item, + struct log_handle *logger) { int ret = -1; long long compile_id = g2c_item->compile_id; struct maat_compile *compile = NULL; - struct literal_id literal_id = {g2c_item->group_id, g2c_item->vtable_id, - g2c_item->not_flag}; int updating_flag = rcu_hash_is_updating(hash_tbl); if (1 == updating_flag) { @@ -1317,19 +1403,17 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl, sizeof(long long)); if (NULL == compile) { log_fatal(logger, MODULE_COMPILE, - "[%s:%d] Remove group_id:%lld from compile_id:%lld failed, compile" - " is not existed.", __FUNCTION__, __LINE__, g2c_item->group_id, - compile_id); + "[%s:%d] Remove clause(index:%d) from compile %lld failed," + "compile not existed.", __FUNCTION__, __LINE__, + g2c_item->clause_index, compile_id); return -1; } else { /* compile found in updating hash, it can be modified directly */ - ret = maat_compile_clause_remove_literal(compile, &literal_id, - g2c_item->clause_index); + ret = maat_compile_clause_remove_literal(compile, g2c_item); if (ret < 0) { log_fatal(logger, MODULE_COMPILE, - "[%s:%d] Remove group_id:%lld vtable_id %d from clause %d of " - "compile_id:%lld failed, literal is not in compile.", __FUNCTION__, - __LINE__, g2c_item->group_id, g2c_item->vtable_id, + "[%s:%d] Remove clause(index:%d) from compile %lld failed," + "clause not exist.", __FUNCTION__, __LINE__, g2c_item->clause_index, compile_id); } @@ -1356,14 +1440,12 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl, /* delete compile from rcu hash */ rcu_hash_del(hash_tbl, (char *)&compile_id, sizeof(long long)); - ret = maat_compile_clause_remove_literal(copy_compile, &literal_id, - g2c_item->clause_index); + ret = maat_compile_clause_remove_literal(copy_compile, g2c_item); if (ret < 0) { log_fatal(logger, MODULE_COMPILE, - "[%s:%d] Remove group_id:%lld vtable_id %d from clause %d of compile_id:" - "%lld failed, literal is not in compile.", __FUNCTION__, __LINE__, - g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, - compile_id); + "[%s:%d] Remove clause(index:%d) from compile %lld failed," + "clause not exist.", __FUNCTION__, __LINE__, + g2c_item->clause_index, compile_id); } if (0 == copy_compile->actual_clause_num && NULL == copy_compile->user_data) { @@ -1373,9 +1455,9 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl, } } else { log_fatal(logger, MODULE_COMPILE, - "[%s:%d] Remove group_id:%lld from compile_id:%lld failed, " + "[%s:%d] Remove clause(index:%d) from compile_id %lld failed," "compile is not existed.", __FUNCTION__, __LINE__, - g2c_item->group_id, compile_id); + g2c_item->clause_index, compile_id); return -1; } } @@ -1391,15 +1473,16 @@ struct compile_state *compile_state_new(void) utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->this_scan_hit_not_clauses, &ut_clause_id_icd); + utarray_new(compile_state->exclude_not_clauses, &ut_clause_id_icd); utarray_new(compile_state->direct_hit_groups, &ut_maat_hit_group_icd); utarray_new(compile_state->indirect_hit_groups, &ut_maat_hit_group_icd); utarray_new(compile_state->hit_compile_table_ids, &ut_hit_compile_table_id_icd); - compile_state->hit_not_groups = NULL; + compile_state->hit_not_tbl_groups = NULL; return compile_state; } -static long long compile_state_hit_not_groups_free(struct compile_state *compile_state) +static long long compile_state_hit_not_tbl_groups_free(struct compile_state *compile_state) { if (NULL == compile_state) { return 0; @@ -1407,9 +1490,9 @@ static long long compile_state_hit_not_groups_free(struct compile_state *compile long long free_bytes = 0; struct table_group *tbl_group = NULL, *tmp_tbl_group = NULL; - HASH_ITER(hh, compile_state->hit_not_groups, tbl_group, tmp_tbl_group) { + HASH_ITER(hh, compile_state->hit_not_tbl_groups, tbl_group, tmp_tbl_group) { free_bytes += (sizeof(tbl_group) + utarray_len(tbl_group->group_ids) * sizeof(long long)); - HASH_DEL(compile_state->hit_not_groups, tbl_group); + HASH_DEL(compile_state->hit_not_tbl_groups, tbl_group); if (tbl_group->group_ids != NULL) { utarray_free(tbl_group->group_ids); tbl_group->group_ids = NULL; @@ -1434,12 +1517,13 @@ void compile_state_reset(struct compile_state *compile_state) utarray_clear(compile_state->all_hit_clauses); utarray_clear(compile_state->this_scan_hit_clauses); utarray_clear(compile_state->this_scan_hit_not_clauses); + utarray_clear(compile_state->exclude_not_clauses); utarray_clear(compile_state->direct_hit_groups); utarray_clear(compile_state->indirect_hit_groups); utarray_clear(compile_state->hit_compile_table_ids); struct table_group *tbl_group = NULL, *tmp_tbl_group = NULL; - HASH_ITER(hh, compile_state->hit_not_groups, tbl_group, tmp_tbl_group) { + HASH_ITER(hh, compile_state->hit_not_tbl_groups, tbl_group, tmp_tbl_group) { utarray_clear(tbl_group->group_ids); } } @@ -1477,6 +1561,12 @@ void compile_state_free(struct compile_state *compile_state, compile_state->this_scan_hit_not_clauses = NULL; } + if (compile_state->exclude_not_clauses != NULL) { + free_bytes += utarray_size(compile_state->exclude_not_clauses) * sizeof(long long); + utarray_free(compile_state->exclude_not_clauses); + compile_state->exclude_not_clauses = NULL; + } + if (compile_state->direct_hit_groups != NULL) { free_bytes += utarray_size(compile_state->direct_hit_groups) * sizeof(struct maat_hit_group); utarray_free(compile_state->direct_hit_groups); @@ -1495,7 +1585,7 @@ void compile_state_free(struct compile_state *compile_state, compile_state->hit_compile_table_ids = NULL; } - free_bytes += compile_state_hit_not_groups_free(compile_state); + free_bytes += compile_state_hit_not_tbl_groups_free(compile_state); FREE(compile_state); @@ -1522,25 +1612,33 @@ static void compile_state_add_internal_hit_path(struct compile_state *compile_st utarray_push_back(compile_state->internal_hit_paths, &new_path); } -static int maat_compile_has_literal(struct maat_compile *compile, - struct literal_id *literal_id) +static int maat_compile_has_clause_query_key(struct maat_compile *compile, + struct clause_query_key *key) { - int i = 0; - struct literal_id *tmp = NULL; - - for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - struct maat_clause *clause = compile->clauses+i; + for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + struct compile_clause *clause = compile->clauses+i; if(!clause->in_use) { continue; } - tmp = (struct literal_id*)utarray_find(clause->literal_ids, - literal_id, compare_literal_id); - if (tmp) { - assert(tmp->group_id == literal_id->group_id && - tmp->vtable_id == literal_id->vtable_id); - return 1; - } + struct clause_literal *tmp_cl = NULL; + for (size_t j = 0; j < utarray_len(clause->literals); j++) { + tmp_cl = (struct clause_literal *)utarray_eltptr(clause->literals, j); + if (tmp_cl->vtable_id != key->vtable_id) { + continue; + } + + if (clause->not_flag != key->not_flag) { + continue; + } + + long long *tmp_group_id = bsearch(&(key->group_id), tmp_cl->group_ids, + tmp_cl->group_cnt, sizeof(long long), + compare_group_id); + if (tmp_group_id != NULL) { + return 1; + } + } } return 0; @@ -1566,7 +1664,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr /* assign hit_path_array[].compile_id */ size_t new_hit_path_cnt = 0; struct maat_compile *compile = NULL; - struct literal_id literal_id = {0, 0}; + struct clause_query_key key = {0, 0, 0}; struct bool_expr_match *expr_match = compile_rt->expr_match_buff + (thread_id * MAX_HIT_COMPILE_NUM); assert(thread_id >= 0); @@ -1589,14 +1687,14 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr for (size_t j = 0; j < n_hit_path && (n_hit_path + new_hit_path_cnt) < array_size; j++) { if (hit_path_array[j].top_group_id < 0) { - literal_id.group_id = hit_path_array[j].sub_group_id; + key.group_id = hit_path_array[j].sub_group_id; } else { - literal_id.group_id = hit_path_array[j].top_group_id; + key.group_id = hit_path_array[j].top_group_id; } - literal_id.vtable_id = hit_path_array[j].vtable_id; - literal_id.not_flag = hit_path_array[j].NOT_flag; - if (maat_compile_has_literal(compile, &literal_id)) { + key.vtable_id = hit_path_array[j].vtable_id; + key.not_flag = hit_path_array[j].NOT_flag; + if (maat_compile_has_clause_query_key(compile, &key)) { if (hit_path_array[j].top_group_id < 0) { hit_path_array[j].top_group_id = hit_path_array[j].sub_group_id; } @@ -1604,7 +1702,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr if (hit_path_array[j].compile_id < 0) { hit_path_array[j].compile_id = compile->compile_id; } else { - // means same literal_id hit more than one compile_id + // means same clause_query_id hit more than one compile_id struct maat_hit_path tmp_path = hit_path_array[j]; tmp_path.compile_id = compile->compile_id; if(!maat_compile_is_hit_path_existed(hit_path_array, n_hit_path + new_hit_path_cnt, &tmp_path)) { @@ -1680,6 +1778,19 @@ static void compile_state_add_hit_clauses(struct compile_state *compile_state, } } +static void compile_state_add_exclude_not_clauses(struct compile_state *compile_state, + UT_array *clause_id_array) +{ + for (size_t i = 0; i < utarray_len(clause_id_array); i++) { + long long *clause_id = (long long *)utarray_eltptr(clause_id_array, i); + if (utarray_find(compile_state->exclude_not_clauses, clause_id, compare_clause_id)) { + continue; + } + utarray_push_back(compile_state->exclude_not_clauses, clause_id); + utarray_sort(compile_state->exclude_not_clauses, compare_clause_id); + } +} + static void compile_state_add_hit_not_clauses(struct compile_state *compile_state, UT_array *clause_id_array) { @@ -1692,6 +1803,11 @@ static void compile_state_add_hit_not_clauses(struct compile_state *compile_stat if (utarray_find(compile_state->all_hit_clauses, clause_id, compare_clause_id)) { continue; } + + if (utarray_find(compile_state->exclude_not_clauses, clause_id, compare_clause_id)) { + continue; + } + utarray_push_back(compile_state->this_scan_hit_not_clauses, clause_id); } @@ -1715,29 +1831,19 @@ static void compile_state_update_hit_clauses(struct compile_state *compile_state return; } - - struct literal_id literal_id = {group_id, vtable_id, 0}; - struct literal_clause *l2c_val = NULL; + struct clause_query_key key = {group_id, vtable_id, 0}; + struct clause_id_kv *clause_id_kv = NULL; - HASH_FIND(hh, compile_rt->literal2clause_hash, &literal_id, sizeof(literal_id), l2c_val); - if (!l2c_val) { - return; + HASH_FIND(hh, compile_rt->clause_id_kv_hash, &key, sizeof(key), clause_id_kv); + if (clause_id_kv != NULL) { + compile_state_add_hit_clauses(compile_state, clause_id_kv->clause_ids); } - compile_state_add_hit_clauses(compile_state, l2c_val->clause_ids); -} - -static inline int compare_group_id(const void *a, const void *b) -{ - long long ret = *(const long long *)a - *(const long long *)b; - - if (0 == ret) { - return 0; - } else if(ret < 0) { - return -1; - } else { - return 1; - } + key.not_flag = 1; + HASH_FIND(hh, compile_rt->not_clause_id_kv_hash, &key, sizeof(key), clause_id_kv); + if (clause_id_kv != NULL) { + compile_state_add_exclude_not_clauses(compile_state, clause_id_kv->clause_ids); + } } static void compile_state_cache_hit_not_groups(struct compile_state *compile_state, @@ -1754,7 +1860,7 @@ static void compile_state_cache_hit_not_groups(struct compile_state *compile_sta } struct table_group *tbl_group = NULL; - HASH_FIND(hh, compile_state->hit_not_groups, &vtable_id, sizeof(int), tbl_group); + HASH_FIND(hh, compile_state->hit_not_tbl_groups, &vtable_id, sizeof(int), tbl_group); if (tbl_group != NULL) { for (size_t i = 0; i < n_hit_group_id; i++) { long long *group_id = (long long *)utarray_find(tbl_group->group_ids, @@ -1768,13 +1874,13 @@ static void compile_state_cache_hit_not_groups(struct compile_state *compile_sta } } - struct literal_clause *l2c_val = NULL, *tmp_l2c_val = NULL; - HASH_ITER(hh, compile_rt->literal2not_clause_hash, l2c_val, tmp_l2c_val) { - if (l2c_val->key.vtable_id != vtable_id) { + struct clause_id_kv *clause_id_kv = NULL, *tmp_clause_id_kv = NULL; + HASH_ITER(hh, compile_rt->not_clause_id_kv_hash, clause_id_kv, tmp_clause_id_kv) { + if (clause_id_kv->key.vtable_id != vtable_id) { continue; } - long long *tmp_group_id = bsearch(&(l2c_val->key.group_id), hit_group_ids, + long long *tmp_group_id = bsearch(&(clause_id_kv->key.group_id), hit_group_ids, n_hit_group_id, sizeof(long long), compare_group_id); if (tmp_group_id != NULL) { continue; @@ -1784,11 +1890,11 @@ static void compile_state_cache_hit_not_groups(struct compile_state *compile_sta tbl_group = ALLOC(struct table_group, 1); tbl_group->vtable_id = vtable_id; utarray_new(tbl_group->group_ids, &ut_compile_group_id_icd); - HASH_ADD_INT(compile_state->hit_not_groups, vtable_id, tbl_group); + HASH_ADD_INT(compile_state->hit_not_tbl_groups, vtable_id, tbl_group); } - if (!utarray_find(tbl_group->group_ids, &(l2c_val->key.group_id), compare_group_id)) { - utarray_push_back(tbl_group->group_ids, &(l2c_val->key.group_id)); + if (!utarray_find(tbl_group->group_ids, &(clause_id_kv->key.group_id), compare_group_id)) { + utarray_push_back(tbl_group->group_ids, &(clause_id_kv->key.group_id)); utarray_sort(tbl_group->group_ids, compare_group_id); } } @@ -2174,16 +2280,16 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name, maat_rt_version, time_elapse_ms); } - struct literal_clause *old_literal2clause = NULL; - struct literal_clause *new_literal2clause = NULL; - struct literal_clause *old_literal2not_clause = NULL; - struct literal_clause *new_literal2not_clause = NULL; + struct clause_id_kv *old_clause_id_kv_hash = NULL; + struct clause_id_kv *new_clause_id_kv_hash = NULL; + struct clause_id_kv *old_not_clause_id_kv_hash = NULL; + struct clause_id_kv *new_not_clause_id_kv_hash = NULL; - new_literal2clause = maat_compile_build_literal2clause_hash(compile_rt, 0); - new_literal2not_clause = maat_compile_build_literal2clause_hash(compile_rt, 1); + new_clause_id_kv_hash = build_clause_id_kv_hash(compile_rt, 0); + new_not_clause_id_kv_hash = build_clause_id_kv_hash(compile_rt, 1); - old_literal2clause = compile_rt->literal2clause_hash; - old_literal2not_clause = compile_rt->literal2not_clause_hash; + old_clause_id_kv_hash = compile_rt->clause_id_kv_hash; + old_not_clause_id_kv_hash = compile_rt->not_clause_id_kv_hash; old_bool_matcher = compile_rt->bm; /* @@ -2202,17 +2308,17 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name, */ compile_rt->bm = new_bool_matcher; - compile_rt->literal2clause_hash = new_literal2clause; - compile_rt->literal2not_clause_hash = new_literal2not_clause; + compile_rt->clause_id_kv_hash = new_clause_id_kv_hash; + compile_rt->not_clause_id_kv_hash = new_not_clause_id_kv_hash; rcu_hash_commit(compile_rt->cfg_hash); rcu_hash_commit(compile_rt->tbl_cfg_hash); maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL, garbage_bool_matcher_free); - maat_garbage_bagging(compile_rt->ref_garbage_bin, old_literal2clause, NULL, - garbage_literal2clause_hash_free); - maat_garbage_bagging(compile_rt->ref_garbage_bin, old_literal2not_clause, NULL, - garbage_literal2clause_hash_free); + maat_garbage_bagging(compile_rt->ref_garbage_bin, old_clause_id_kv_hash, NULL, + garbage_clause_id_kv_hash_free); + maat_garbage_bagging(compile_rt->ref_garbage_bin, old_not_clause_id_kv_hash, NULL, + garbage_clause_id_kv_hash_free); compile_rt->rule_num = rcu_hash_count(compile_rt->cfg_hash); @@ -2369,23 +2475,22 @@ void compile_state_not_logic_update(struct compile_state *compile_state, utarray_clear(compile_state->this_scan_hit_not_clauses); struct table_group *tbl_group = NULL; - HASH_FIND(hh, compile_state->hit_not_groups, &vtable_id, sizeof(int), tbl_group); + HASH_FIND(hh, compile_state->hit_not_tbl_groups, &vtable_id, sizeof(int), tbl_group); if (NULL == tbl_group) { return; } - struct literal_clause *l2c_val = NULL; + struct clause_id_kv *clause_id_kv = NULL; for (size_t i = 0; i < utarray_len(tbl_group->group_ids); i++) { long long *group_id = utarray_eltptr(tbl_group->group_ids, i); - struct literal_id literal_id = {*group_id, vtable_id, 1}; + struct clause_query_key key = {*group_id, vtable_id, 1}; - HASH_FIND(hh, compile_rt->literal2not_clause_hash, &literal_id, - sizeof(struct literal_id), l2c_val); - if (NULL == l2c_val) { + HASH_FIND(hh, compile_rt->not_clause_id_kv_hash, &key, sizeof(key), clause_id_kv); + if (NULL == clause_id_kv) { continue; } - compile_state_add_hit_not_clauses(compile_state, l2c_val->clause_ids); + compile_state_add_hit_not_clauses(compile_state, clause_id_kv->clause_ids); if (1 == maat_inst->opts.hit_path_on) { compile_state_add_internal_hit_path(compile_state, -1, *group_id, vtable_id, 1, Nth_scan); diff --git a/src/maat_rule.c b/src/maat_rule.c index 16c3eab..e513bea 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -181,7 +181,7 @@ static long long maat_runtime_rule_num(struct maat_runtime *maat_rt) if (rule_cnt != 0) { log_info(maat_rt->logger, MODULE_MAAT_RULE, - "table:<%s> rule count:%lld", + "table:<%s> rule_count:%lld", table_manager_get_table_name(maat_rt->ref_tbl_mgr, i), rule_cnt); } } diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 54b5cdb..2a838af 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -3404,7 +3404,7 @@ TEST_F(MaatGroupScan, basic) { int table_id = maat_get_table_id(maat_inst, table_name); ASSERT_GE(table_id, 0); - long long group_id = 158; + long long group_id = 247; int ret = maat_scan_group(maat_inst, table_id, &group_id, 1, results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); @@ -3432,7 +3432,7 @@ TEST_F(MaatGroupScan, SetScanCompileTable) { int ret = maat_state_set_scan_compile_table(state, compile_table_id); EXPECT_EQ(ret, 0); - long long group_id = 159; + long long group_id = 248; ret = maat_scan_group(maat_inst, table_id, &group_id, 1, results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); @@ -3545,15 +3545,12 @@ TEST_F(NOTLogic, ScanNotAtLast) { int hit_table_id = maat_get_table_id(maat_inst, hit_table_name); ASSERT_GT(hit_table_id, 0); + // scan string_should_hit(HTTP_URL_FILTER) & string_should_not_hit(HTTP_RESPONSE_KEYWORDS) => not hit compile int ret = maat_scan_string(maat_inst, hit_table_id, string_should_hit, strlen(string_should_hit), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_not_logic(maat_inst, hit_table_id, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - int not_hit_table_id = maat_get_table_id(maat_inst, not_hit_table_name); ASSERT_GT(not_hit_table_id, 0); @@ -3562,10 +3559,6 @@ TEST_F(NOTLogic, ScanNotAtLast) { &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_not_logic(maat_inst, not_hit_table_id, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - ret = maat_scan_string(maat_inst, not_hit_table_id, string_contain_nothing, strlen(string_contain_nothing), results, ARRAY_SIZE, &n_hit_result, state); @@ -3573,7 +3566,24 @@ TEST_F(NOTLogic, ScanNotAtLast) { ret = maat_scan_not_logic(maat_inst, not_hit_table_id, results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + //scan string_should_hit(HTTP_URL_FILTER) & nothing(HTTP_RESPONSE_KEYWORDS) => hit compile144 + ret = maat_scan_string(maat_inst, hit_table_id, string_should_hit, + strlen(string_should_hit), results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_string(maat_inst, not_hit_table_id, string_contain_nothing, + strlen(string_contain_nothing), results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_not_logic(maat_inst, not_hit_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 144); @@ -3749,6 +3759,7 @@ TEST_F(NOTLogic, ScanNotIP) { int hit_table_id = maat_get_table_id(maat_inst, hit_table_name); ASSERT_GT(hit_table_id, 0); + // scan string_should_hit(HTTP_URL) & hit ip(VIRTUAL_IP_CONFIG) => not hit compile int ret = maat_scan_string(maat_inst, hit_table_id, string_should_hit, strlen(string_should_hit), results, ARRAY_SIZE, &n_hit_result, state); @@ -3774,6 +3785,14 @@ TEST_F(NOTLogic, ScanNotIP) { &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_OK); + maat_state_reset(state); + + // scan string_should_hit(HTTP_URL) & not hit ip(VIRTUAL_IP_CONFIG) => hit compile145 + ret = maat_scan_string(maat_inst, hit_table_id, string_should_hit, + strlen(string_should_hit), results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + inet_pton(AF_INET, "10.1.0.0", &sip); ret = maat_scan_ipv4(maat_inst, not_hit_table_id, sip, port, proto, results, ARRAY_SIZE, &n_hit_result, state); @@ -3805,15 +3824,12 @@ TEST_F(NOTLogic, ScanNotWithDistrict) { int url_table_id = maat_get_table_id(maat_inst, url_table_name); ASSERT_GT(url_table_id, 0); + // scan string1(HTTP_URL) & string2(HTTP_REQUEST_HEADER) => not hit compile int ret = maat_scan_string(maat_inst, url_table_id, string1, strlen(string1), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_not_logic(maat_inst, url_table_id, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - int virtual_table_id = maat_get_table_id(maat_inst, virtual_table_name); ASSERT_GT(virtual_table_id, 0); @@ -3829,6 +3845,18 @@ TEST_F(NOTLogic, ScanNotWithDistrict) { &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_OK); + maat_state_reset(state); + + // scan string1(HTTP_URL) & string3(HTTP_REQUEST_HEADER) => hit compile221 + ret = maat_scan_string(maat_inst, url_table_id, string1, + strlen(string1), results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_state_set_scan_district(state, virtual_table_id, district_str1, + strlen(district_str1)); + ASSERT_EQ(ret, 0); + ret = maat_scan_string(maat_inst, virtual_table_id, string3, strlen(string3), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_OK); @@ -3860,6 +3888,7 @@ TEST_F(NOTLogic, NotUrlAndNotIp) { int url_table_id = maat_get_table_id(maat_inst, url_table_name); ASSERT_GT(url_table_id, 0); + //scan string_should_half_hit(HTTP_URL_FILTER) & hit ip(VIRTUAL_IP_CONFIG) => not hit compile int ret = maat_scan_string(maat_inst, url_table_id, string_should_half_hit, strlen(string_should_half_hit), results, ARRAY_SIZE, &n_hit_result, state); @@ -3870,7 +3899,7 @@ TEST_F(NOTLogic, NotUrlAndNotIp) { EXPECT_EQ(ret, MAAT_SCAN_OK); uint32_t sip; - inet_pton(AF_INET, "10.1.0.0", &sip); + inet_pton(AF_INET, "10.0.6.201", &sip); uint16_t port = htons(50001); int proto = 6; @@ -3879,12 +3908,15 @@ TEST_F(NOTLogic, NotUrlAndNotIp) { ret = maat_scan_ipv4(maat_inst, ip_table_id, sip, port, proto, results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); ret = maat_scan_not_logic(maat_inst, ip_table_id, results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_OK); + maat_state_reset(state); + + // scan string_should_half_hit(HTTP_RESPONSE_KEYWORDS) & not hit ip(VIRTUAL_IP_CONFIG) => not hit compile int http_table_id = maat_get_table_id(maat_inst, http_table_name); ASSERT_GT(http_table_id, 0); @@ -3897,6 +3929,23 @@ TEST_F(NOTLogic, NotUrlAndNotIp) { &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_OK); + inet_pton(AF_INET, "10.1.0.0", &sip); + ret = maat_scan_ipv4(maat_inst, ip_table_id, sip, port, proto, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_not_logic(maat_inst, ip_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + // scan scan string_should_half_hit(HTTP_URL_FILTER) & not hit ip(VIRTUAL_IP_CONFIG) => hit compile146 + ret = maat_scan_string(maat_inst, url_table_id, string_should_half_hit, + strlen(string_should_half_hit), results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + ret = maat_scan_string(maat_inst, http_table_id, string_nothing, strlen(string_nothing), results, ARRAY_SIZE, &n_hit_result, state); @@ -3904,6 +3953,15 @@ TEST_F(NOTLogic, NotUrlAndNotIp) { ret = maat_scan_not_logic(maat_inst, http_table_id, results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + inet_pton(AF_INET, "10.1.0.0", &sip); + ret = maat_scan_ipv4(maat_inst, ip_table_id, sip, port, proto, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_not_logic(maat_inst, ip_table_id, results, ARRAY_SIZE, + &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 146); @@ -3930,6 +3988,7 @@ TEST_F(NOTLogic, NotPhysicalTable) { int vtable_id = maat_get_table_id(maat_inst, vtable_name); ASSERT_GT(vtable_id, 0); + // scan hit string1(KEYWORDS_TABLE) & hit string2(HTTP_RESPONSE_KEYWORDS) => not hit compile int ret = maat_scan_string(maat_inst, phy_table_id, string1, strlen(string1), results, ARRAY_SIZE, &n_hit_result, state); @@ -3943,17 +4002,20 @@ TEST_F(NOTLogic, NotPhysicalTable) { results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_not_logic(maat_inst, vtable_id, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); + maat_state_reset(state); + //scan not hit string1(KEYWORDS_TABLE) & hit string2(HTTP_RESPONSE_KEYWORDS) => hit compile224 ret = maat_scan_string(maat_inst, phy_table_id, string3, strlen(string3), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_OK); ret = maat_scan_not_logic(maat_inst, phy_table_id, results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, vtable_id, string2, strlen(string2), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 224); @@ -4191,53 +4253,459 @@ TEST_F(NOTLogic, NotClauseAndExcludeGroup2) { state = NULL; } -TEST_F(NOTLogic, SameClauseHasMultiNotGroups) { - const char *not_string1 = "This string ONLY contains not_logic_compile_222_1"; - const char *not_string2 = "This string ONLY contains not_logic_compile_222_2"; - const char *string3 = "This string contain logic_compile_222_3"; +TEST_F(NOTLogic, SingleNotClause) { + const char *string_nothing = "nothing string"; + const char *string_should_hit = "string has not_logic_keywords_222"; + const char *table_name = "HTTP_NOT_LOGIC_1"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - const char *table_name = "HTTP_URL_FILTER"; struct maat *maat_inst = NOTLogic::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); int table_id = maat_get_table_id(maat_inst, table_name); ASSERT_GT(table_id, 0); - int ret = maat_scan_string(maat_inst, table_id, not_string1, strlen(not_string1), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - ret = maat_scan_string(maat_inst, table_id, string3, strlen(string3), - results, ARRAY_SIZE, &n_hit_result, state); + //string_should_hit(HTTP_NOT_LOGIC_1) => not hit compile + int ret = maat_scan_string(maat_inst, table_id, string_should_hit, + strlen(string_should_hit), results, ARRAY_SIZE, + &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 222); + EXPECT_EQ(ret, MAAT_SCAN_OK); maat_state_reset(state); - ret = maat_scan_string(maat_inst, table_id, not_string1, strlen(not_string1), + //string nothing(HTTP_NOT_LOGIC_1) => hit compile222 + ret = maat_scan_string(maat_inst, table_id, string_nothing, strlen(string_nothing), results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - ret = maat_scan_string(maat_inst, table_id, not_string2, strlen(not_string2), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - ret = maat_scan_string(maat_inst, table_id, string3, strlen(string3), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + EXPECT_EQ(ret, MAAT_SCAN_OK); ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 222); + EXPECT_EQ(results[0], 222); + + maat_state_free(state); + state = NULL; +} + +TEST_F(NOTLogic, MultiNotClauses) { + const char *string_nothing = "nothing string"; + const char *string1 = "string has not_logic_compile_223_1"; + const char *string2 = "string has not_logic_compile_223_1"; + const char *string3 = "string has not_logic_compile_223_1"; + const char *table_name = "HTTP_NOT_LOGIC"; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + int thread_id = 0; + struct maat *maat_inst = NOTLogic::_shared_maat_inst; + struct maat_state *state = maat_state_new(maat_inst, thread_id); + + int table_id = maat_get_table_id(maat_inst, table_name); + ASSERT_GT(table_id, 0); + + // compile223 = !string1 & !string2 & !string3 + //Case1: scan string1 & !string2 & !string3 + int ret = maat_scan_string(maat_inst, table_id, string1, strlen(string1), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_string(maat_inst, table_id, string_nothing, strlen(string_nothing), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + //Case2: scan !string1 & string2 & !string3 + ret = maat_scan_string(maat_inst, table_id, string_nothing, strlen(string_nothing), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, table_id, string2, strlen(string2), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + //Case3: scan !string1 & !string2 & string3 + ret = maat_scan_string(maat_inst, table_id, string_nothing, strlen(string_nothing), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, table_id, string3, strlen(string3), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + //Case4: scan !string1 & !string2 & !string3 + ret = maat_scan_string(maat_inst, table_id, string_nothing, strlen(string_nothing), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 223); + + maat_state_free(state); + state = NULL; +} + +TEST_F(NOTLogic, MultiGroupsInOneNotClause) { + const char *src_asn1 = "AS1234"; + const char *src_asn2 = "AS6789"; + const char *src_asn3 = "AS9001"; + const char *src_asn_nothing = "nothing string"; + const char *dst_asn = "AS2345"; + const char *src_asn_table_name = "ASN_NOT_LOGIC"; + const char *dst_asn_table_name = "DESTINATION_IP_ASN"; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + int thread_id = 0; + struct maat *maat_inst = NOTLogic::_shared_maat_inst; + struct maat_state *state = maat_state_new(maat_inst, thread_id); + + //-------------------------------------- + // Source ASN1 & Dest ASN => not hit compile + //-------------------------------------- + int src_table_id = maat_get_table_id(maat_inst, src_asn_table_name); + ASSERT_GT(src_table_id, 0); + + int ret = maat_scan_string(maat_inst, src_table_id, src_asn1, strlen(src_asn1), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + int dst_table_id = maat_get_table_id(maat_inst, dst_asn_table_name); + ASSERT_GT(dst_table_id, 0); + + ret = maat_scan_string(maat_inst, dst_table_id, dst_asn, strlen(dst_asn), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + maat_state_reset(state); + + //-------------------------------------- + // Source ASN2 & Dest ASN => not hit compile + //-------------------------------------- + ret = maat_scan_string(maat_inst, src_table_id, src_asn2, strlen(src_asn2), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, dst_table_id, dst_asn, strlen(dst_asn), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + maat_state_reset(state); + + //-------------------------------------- + // Source ASN3 & Dest ASN => not hit compile + //-------------------------------------- + ret = maat_scan_string(maat_inst, src_table_id, src_asn3, strlen(src_asn3), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, dst_table_id, dst_asn, strlen(dst_asn), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + maat_state_reset(state); + + // Source nothing & Dest ASN => hit compile177 + ret = maat_scan_string(maat_inst, src_table_id, src_asn_nothing, + strlen(src_asn_nothing),results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, dst_table_id, dst_asn, strlen(dst_asn), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 177); + + maat_state_free(state); + state = NULL; +} + +TEST_F(NOTLogic, MultiLiteralsInOneNotClause) { + const char *src_asn1 = "AS1234"; + const char *src_asn2 = "AS6789"; + const char *src_nothing = "nothing"; + const char *my_county = "Greece.Sparta"; + const char *ip_table_name = "IP_PLUS_CONFIG"; + const char *src_asn_table_name = "SOURCE_IP_ASN"; + const char *ip_geo_table_name = "SOURCE_IP_GEO"; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + int thread_id = 0; + struct maat *maat_inst = NOTLogic::_shared_maat_inst; + struct maat_state *state = maat_state_new(maat_inst, thread_id); + + int src_table_id = maat_get_table_id(maat_inst, src_asn_table_name); + ASSERT_GT(src_table_id, 0); + + int ip_geo_table_id = maat_get_table_id(maat_inst, ip_geo_table_name); + ASSERT_GT(ip_geo_table_id, 0); + + int ip_table_id = maat_get_table_id(maat_inst, ip_table_name); + ASSERT_GT(ip_table_id, 0); + + //------------------------------------------- + // Source ASN1 & IP Geo + //------------------------------------------- + int ret = maat_scan_string(maat_inst, src_table_id, src_asn1, strlen(src_asn1), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_string(maat_inst, ip_geo_table_id, my_county, strlen(my_county), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + //------------------------------------------- + // Source nothing & IP Geo + //------------------------------------------- + ret = maat_scan_string(maat_inst, src_table_id, src_nothing, strlen(src_nothing), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, ip_geo_table_id, my_county, strlen(my_county), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 181); + + maat_state_reset(state); + + //------------------------------------------- + // Source ASN2 & IP Geo + //------------------------------------------- + ret = maat_scan_string(maat_inst, src_table_id, src_asn2, strlen(src_asn2), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_string(maat_inst, ip_geo_table_id, my_county, strlen(my_county), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + //-------------------------------------- + // hit IP & IP Geo + //-------------------------------------- + uint32_t ip_addr; + inet_pton(AF_INET, "192.168.40.88", &ip_addr); + uint16_t port = htons(8888); + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_string(maat_inst, ip_geo_table_id, my_county, strlen(my_county), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, ip_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + //-------------------------------------- + // not hit IP & IP Geo + //-------------------------------------- + inet_pton(AF_INET, "192.168.40.89", &ip_addr); + + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, ip_geo_table_id, my_county, strlen(my_county), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, ip_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 181); + + maat_state_free(state); + state = NULL; +} + +TEST_F(NOTLogic, SameVtableInMultiClause) { + const char *src_asn1 = "AS1234"; + const char *src_asn2 = "AS9002"; + const char *src_asn3 = "AS9003"; + const char *my_county = "Greece.Sparta"; + const char *ip_table_name = "IP_PLUS_CONFIG"; + const char *dst_asn_table_name = "DESTINATION_IP_ASN"; + const char *ip_geo_table_name = "SOURCE_IP_GEO"; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + int thread_id = 0; + struct maat *maat_inst = NOTLogic::_shared_maat_inst; + struct maat_state *state = maat_state_new(maat_inst, thread_id); + + int dst_table_id = maat_get_table_id(maat_inst, dst_asn_table_name); + ASSERT_GT(dst_table_id, 0); + + int ip_geo_table_id = maat_get_table_id(maat_inst, ip_geo_table_name); + ASSERT_GT(ip_geo_table_id, 0); + + int ip_table_id = maat_get_table_id(maat_inst, ip_table_name); + ASSERT_GT(ip_table_id, 0); + + uint32_t ip_addr; + inet_pton(AF_INET, "192.168.40.88", &ip_addr); + uint16_t port = htons(8888); + //------------------------------------------- + // Dest ASN1 & Dest ASN3 & IP Config + //------------------------------------------- + int ret = maat_scan_string(maat_inst, dst_table_id, src_asn1, strlen(src_asn1), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_string(maat_inst, dst_table_id, src_asn3, strlen(src_asn3), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, dst_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + //------------------------------------------- + // Dest ASN2 & Dest ASN3 & IP Config + //------------------------------------------- + ret = maat_scan_string(maat_inst, dst_table_id, src_asn2, strlen(src_asn2), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_string(maat_inst, dst_table_id, src_asn3, strlen(src_asn3), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, dst_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + maat_state_reset(state); + + //------------------------------------------- + // Dest IP Geo & Dest ASN3 & IP Config + //------------------------------------------- + ret = maat_scan_string(maat_inst, ip_geo_table_id, my_county, strlen(my_county), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, ip_geo_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, dst_table_id, src_asn3, strlen(src_asn3), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, dst_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + maat_state_reset(state); + + //------------------------------------------- + // Dest ASN3 & IP Geo + //------------------------------------------- + ret = maat_scan_string(maat_inst, dst_table_id, src_asn3, strlen(src_asn3), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, dst_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 185); + + maat_state_reset(state); + + //-------------------------------------- + // IP Config & IP Geo + //-------------------------------------- + ret = maat_scan_string(maat_inst, dst_table_id, src_asn3, strlen(src_asn3), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + inet_pton(AF_INET, "192.168.40.89", &ip_addr); + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_not_logic(maat_inst, dst_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); maat_state_free(state); state = NULL; @@ -6353,13 +6821,13 @@ TEST_F(HierarchyTest, OneGroupInTwoVirtual) { state = NULL; } -TEST_F(HierarchyTest, TwoVirtualInOneClause) { - const char *src_asn = "AS1234", *dst_asn = "AS2345"; - const char *my_county = "Greece.Sparta"; +TEST_F(HierarchyTest, MultiGroupsInOneClause) { + const char *src_asn1 = "AS1234"; + const char *src_asn2 = "AS6789"; + const char *src_asn3 = "AS9001"; + const char *dst_asn = "AS2345"; const char *src_asn_table_name = "SOURCE_IP_ASN"; const char *dst_asn_table_name = "DESTINATION_IP_ASN"; - const char *ip_table_name = "IP_CONFIG"; - const char *src_ip_geo_table_name = "SOURCE_IP_GEO"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; @@ -6367,93 +6835,155 @@ TEST_F(HierarchyTest, TwoVirtualInOneClause) { struct maat_state *state = maat_state_new(maat_inst, thread_id); //-------------------------------------- - // Source ASN & Dest ASN + // Source ASN1 & Dest ASN //-------------------------------------- - int table_id = maat_get_table_id(maat_inst, src_asn_table_name); - ASSERT_GT(table_id, 0); + int src_table_id = maat_get_table_id(maat_inst, src_asn_table_name); + ASSERT_GT(src_table_id, 0); - int ret = maat_scan_string(maat_inst, table_id, src_asn, strlen(src_asn), + int ret = maat_scan_string(maat_inst, src_table_id, src_asn1, strlen(src_asn1), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_OK); - table_id = maat_get_table_id(maat_inst, dst_asn_table_name); - ASSERT_GT(table_id, 0); + int dst_table_id = maat_get_table_id(maat_inst, dst_asn_table_name); + ASSERT_GT(dst_table_id, 0); - ret = maat_scan_string(maat_inst, table_id, dst_asn, strlen(dst_asn), + ret = maat_scan_string(maat_inst, dst_table_id, dst_asn, strlen(dst_asn), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 178); - ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, + ret = maat_scan_not_logic(maat_inst, dst_table_id, results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_OK); maat_state_reset(state); //-------------------------------------- - // Source IP & Dest ASN + // Source ASN2 & Dest ASN //-------------------------------------- - table_id = maat_get_table_id(maat_inst, ip_table_name); - ASSERT_GT(table_id, 0); + ret = maat_scan_string(maat_inst, src_table_id, src_asn2, strlen(src_asn2), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, dst_table_id, dst_asn, strlen(dst_asn), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 178); + + ret = maat_scan_not_logic(maat_inst, dst_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_reset(state); + + //-------------------------------------- + // Source ASN3 & Dest ASN + //-------------------------------------- + ret = maat_scan_string(maat_inst, src_table_id, src_asn3, strlen(src_asn3), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_not_logic(maat_inst, src_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + ret = maat_scan_string(maat_inst, dst_table_id, dst_asn, strlen(dst_asn), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 178); + + ret = maat_scan_not_logic(maat_inst, dst_table_id, results, ARRAY_SIZE, + &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_OK); + + maat_state_free(state); + state = NULL; +} + +TEST_F(HierarchyTest, MultiLiteralsInOneClause) { + const char *src_asn1 = "AS1234"; + const char *src_asn2 = "AS6789"; + const char *my_county = "Greece.Sparta"; + const char *ip_table_name = "IP_CONFIG"; + const char *src_asn_table_name = "SOURCE_IP_ASN"; + const char *ip_geo_table_name = "SOURCE_IP_GEO"; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + int thread_id = 0; + struct maat *maat_inst = HierarchyTest::_shared_maat_inst; + struct maat_state *state = maat_state_new(maat_inst, thread_id); + + int src_table_id = maat_get_table_id(maat_inst, src_asn_table_name); + ASSERT_GT(src_table_id, 0); + + int ip_geo_table_id = maat_get_table_id(maat_inst, ip_geo_table_name); + ASSERT_GT(ip_geo_table_id, 0); + + int ip_table_id = maat_get_table_id(maat_inst, ip_table_name); + ASSERT_GT(ip_table_id, 0); + + //-------------------------------------- + // Source ASN1 & IP + //-------------------------------------- + + int ret = maat_scan_string(maat_inst, src_table_id, src_asn1, strlen(src_asn1), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); uint32_t ip_addr; inet_pton(AF_INET, "192.168.40.88", &ip_addr); uint16_t port = htons(8888); - ret = maat_scan_ipv4(maat_inst, table_id, ip_addr, port, 6, + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - - table_id = maat_get_table_id(maat_inst, dst_asn_table_name); - ASSERT_GT(table_id, 0); - - ret = maat_scan_string(maat_inst, table_id, dst_asn, strlen(dst_asn), - results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 178); - - ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); + EXPECT_EQ(results[0], 180); maat_state_reset(state); //-------------------------------------- - // Source Geo & Dest ASN + // IP Geo & IP //-------------------------------------- - table_id = maat_get_table_id(maat_inst, src_ip_geo_table_name); - ASSERT_GT(table_id, 0); - - ret = maat_scan_string(maat_inst, table_id, my_county, strlen(my_county), + ret = maat_scan_string(maat_inst, ip_geo_table_id, my_county, strlen(my_county), results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - - table_id = maat_get_table_id(maat_inst, dst_asn_table_name); - ASSERT_GT(table_id, 0); - - ret = maat_scan_string(maat_inst, table_id, dst_asn, strlen(dst_asn), - results, ARRAY_SIZE, &n_hit_result, state); + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 178); + EXPECT_EQ(results[0], 180); - ret = maat_scan_not_logic(maat_inst, table_id, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_OK); + maat_state_reset(state); + + //-------------------------------------- + // (Source ASN2 | IP Geo) & IP + //-------------------------------------- + ret = maat_scan_string(maat_inst, src_table_id, src_asn2, strlen(src_asn2), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_string(maat_inst, ip_geo_table_id, my_county, strlen(my_county), + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + ret = maat_scan_ipv4(maat_inst, ip_table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 180); maat_state_free(state); state = NULL; diff --git a/test/maat_json.json b/test/maat_json.json index ee64e62..b46c944 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -36,34 +36,62 @@ ] }, { - "group_name": "financial-department-ip", + "group_name": "ASN6789", "group_id": 3, "regions": [ { - "table_name": "IP_CONFIG", - "table_type": "ip_plus", + "table_name": "AS_NUMBER", + "table_type": "expr", "table_content": { - "addr_type": "ipv4", - "addr_format": "mask", - "ip1": "192.168.40.88", - "ip2": "255.255.255.255", - "port_format": "range", - "port1": "0", - "port2": "65535", - "protocol": 6 + "keywords": "AS6789", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" } } ] }, { - "group_name": "Country-Sparta-IP", + "group_name": "ASN9001", "group_id": 4, "regions": [ { - "table_name": "GeoLocation", + "table_name": "AS_NUMBER", "table_type": "expr", "table_content": { - "keywords": "Greece.Sparta", + "keywords": "AS9001", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "ASN9002", + "group_id": 5, + "regions": [ + { + "table_name": "AS_NUMBER", + "table_type": "expr", + "table_content": { + "keywords": "AS9002", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "ASN9003", + "group_id": 6, + "regions": [ + { + "table_name": "AS_NUMBER", + "table_type": "expr", + "table_content": { + "keywords": "AS9003", "expr_type": "none", "match_method": "exact", "format": "uncase plain" @@ -73,7 +101,7 @@ }, { "group_name": "IPv4-composition-source-only", - "group_id": 5, + "group_id": 7, "regions": [ { "table_type": "ip_plus", @@ -93,7 +121,7 @@ }, { "group_name": "FQDN_OBJ1", - "group_id": 6, + "group_id": 8, "regions": [ { "table_name": "KEYWORDS_TABLE", @@ -109,7 +137,7 @@ }, { "group_name": "FQDN_CAT1", - "group_id": 7, + "group_id": 9, "regions": [ { "table_name": "INTERGER_PLUS", @@ -124,7 +152,7 @@ }, { "group_name": "IPv4-composition-NOT-client-ip", - "group_id": 8, + "group_id": 10, "regions": [ { "table_type": "ip_plus", @@ -144,7 +172,7 @@ }, { "group_name": "IPv4-composition-NOT-server-ip", - "group_id": 9, + "group_id": 11, "regions": [ { "table_type": "ip_plus", @@ -161,6 +189,82 @@ } } ] + }, + { + "group_name": "financial-department-ip", + "group_id": 12, + "regions": [ + { + "table_name": "IP_CONFIG", + "table_type": "ip_plus", + "table_content": { + "addr_type": "ipv4", + "addr_format": "mask", + "ip1": "192.168.40.88", + "ip2": "255.255.255.255", + "port_format": "range", + "port1": "0", + "port2": "65535", + "protocol": 6 + } + } + ] + }, + { + "group_name": "security-department-ip", + "group_id": 13, + "regions": [ + { + "table_name": "IP_PLUS_CONFIG", + "table_type": "ip_plus", + "table_content": { + "addr_type": "ipv4", + "addr_format": "mask", + "ip1": "192.168.40.88", + "ip2": "255.255.255.255", + "port_format": "range", + "port1": "0", + "port2": "65535", + "protocol": 6 + } + } + ] + }, + { + "group_name": "develop-department-ip", + "group_id": 14, + "regions": [ + { + "table_name": "IP_PLUS_CONFIG", + "table_type": "ip_plus", + "table_content": { + "addr_type": "ipv4", + "addr_format": "mask", + "ip1": "192.168.40.88", + "ip2": "255.255.255.255", + "port_format": "range", + "port1": "0", + "port2": "65535", + "protocol": 6 + } + } + ] + }, + { + "group_name": "Country-Sparta-IP", + "group_id": 15, + "regions": [ + { + "table_name": "GeoLocation", + "table_type": "expr", + "table_content": { + "keywords": "Greece.Sparta", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] } ], "rules": [ @@ -176,7 +280,7 @@ { "virtual_table": "IP_CONFIG", "group_name": "123_IP_group", - "group_id": 10, + "group_id": 100, "regions": [ { "table_name": "IP_CONFIG", @@ -211,7 +315,7 @@ { "virtual_table": "HTTP_URL", "group_name": "123_url_group", - "group_id": 11, + "group_id": 101, "regions": [ { "table_name": "HTTP_URL", @@ -243,7 +347,7 @@ { "virtual_table": "CONTENT_SIZE", "group_name": "124_interval_group", - "group_id": 12, + "group_id": 102, "regions": [ { "table_name": "CONTENT_SIZE", @@ -269,7 +373,7 @@ { "virtual_table":"HTTP_URL", "group_name": "125_url_group", - "group_id": 13, + "group_id": 103, "regions": [ { "table_name": "HTTP_URL", @@ -297,7 +401,7 @@ { "virtual_table": "HTTP_URL", "group_name": "126_url_group", - "group_id": 14, + "group_id": 105, "regions": [ { "table_name": "HTTP_URL", @@ -314,7 +418,7 @@ { "virtual_table": "CONTENT_SIZE", "group_name": "126_interval_group", - "group_id": 15, + "group_id": 106, "regions": [ { "table_name": "CONTENT_SIZE", @@ -340,7 +444,7 @@ { "virtual_table": "HTTP_SIGNATURE", "group_name": "128_expr_plus_group", - "group_id": 16, + "group_id": 107, "regions": [ { "table_name": "HTTP_SIGNATURE", @@ -369,7 +473,7 @@ { "virtual_table": "HTTP_URL", "group_name": "129_url_group", - "group_id": 17, + "group_id": 108, "regions": [ { "table_name": "HTTP_URL", @@ -397,7 +501,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "130_keywords_group", - "group_id": 18, + "group_id": 109, "regions": [ { "table_name": "KEYWORDS_TABLE", @@ -425,7 +529,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "131_keywords_group", - "group_id": 19, + "group_id": 110, "regions": [ { "table_name": "KEYWORDS_TABLE", @@ -453,7 +557,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "TakeMeHome", - "group_id": 20, + "group_id": 111, "regions": [ { "table_name": "KEYWORDS_TABLE", @@ -481,7 +585,7 @@ { "virtual_table": "HTTP_HOST", "group_name": "133_host_group", - "group_id": 21, + "group_id": 112, "regions": [ { "table_name": "HTTP_HOST", @@ -509,7 +613,7 @@ { "virtual_table": "HTTP_URL", "group_name": "134_url_group", - "group_id": 22, + "group_id": 113, "regions": [ { "table_name": "HTTP_URL", @@ -537,7 +641,7 @@ { "virtual_table": "IMAGE_FP", "group_name": "136_expr_group", - "group_id": 23, + "group_id": 114, "regions": [ { "table_name": "IMAGE_FP", @@ -565,7 +669,7 @@ { "virtual_table": "IMAGE_FP", "group_name": "137_expr_group", - "group_id": 24, + "group_id": 115, "regions": [ { "table_name": "IMAGE_FP", @@ -595,7 +699,7 @@ { "virtual_table": "HTTP_URL", "group_name": "138_url_group", - "group_id": 25, + "group_id": 116, "regions": [ { "table_name": "HTTP_URL", @@ -625,7 +729,7 @@ { "virtual_table": "HTTP_URL", "group_name": "139_url_group", - "group_id": 26, + "group_id": 117, "regions": [ { "table_name": "HTTP_URL", @@ -653,7 +757,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "140_keywords_group", - "group_id": 27, + "group_id": 118, "regions": [ { "table_name": "KEYWORDS_TABLE", @@ -683,7 +787,7 @@ "g2c_table_name": "GROUP2COMPILE_ALIAS", "virtual_table": "HTTP_URL", "group_name": "141_url_group", - "group_id": 28, + "group_id": 119, "regions": [ { "table_name": "HTTP_URL", @@ -711,7 +815,7 @@ { "virtual_table": "HTTP_URL", "group_name": "142_url_group", - "group_id": 29, + "group_id": 120, "regions": [ { "table_name": "HTTP_URL", @@ -739,7 +843,7 @@ { "virtual_table": "HTTP_URL_FILTER", "group_name": "143_url_group1", - "group_id": 30, + "group_id": 121, "not_flag": 0, "regions": [ { @@ -757,7 +861,7 @@ { "virtual_table": "HTTP_URL_FILTER", "group_name": "143_url_group2", - "group_id": 31, + "group_id": 122, "not_flag": 1, "regions": [ { @@ -786,7 +890,7 @@ { "virtual_table": "HTTP_URL_FILTER", "group_name": "144_url_group", - "group_id": 32, + "group_id": 123, "not_flag": 0, "regions": [ { @@ -804,7 +908,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS", "group_name": "144_keywords_group", - "group_id": 33, + "group_id": 124, "not_flag": 1, "regions": [ { @@ -833,7 +937,7 @@ { "virtual_table": "HTTP_URL", "group_name": "145_url_group", - "group_id": 34, + "group_id": 125, "not_flag": 0, "regions": [ { @@ -867,7 +971,7 @@ { "virtual_table": "HTTP_URL_FILTER", "group_name": "146_url_group", - "group_id": 35, + "group_id": 126, "not_flag": 0, "clause_index": 0, "regions": [ @@ -886,7 +990,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS", "group_name": "146_keywords_group", - "group_id": 36, + "group_id": 127, "not_flag": 1, "clause_index": 1, "regions": [ @@ -922,7 +1026,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS_1", "group_name": "147_keywords_group1", - "group_id": 37, + "group_id": 128, "not_flag": 1, "clause_index": 0, "regions": [ @@ -941,7 +1045,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS_2", "group_name": "147_keywords_group2", - "group_id": 38, + "group_id": 129, "not_flag": 1, "clause_index": 1, "regions": [ @@ -960,7 +1064,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS_3", "group_name": "147_keywords_group3", - "group_id": 39, + "group_id": 130, "not_flag": 1, "clause_index": 2, "regions": [ @@ -979,7 +1083,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS_4", "group_name": "147_keywords_group4", - "group_id": 40, + "group_id": 131, "not_flag": 1, "clause_index": 3, "regions": [ @@ -998,7 +1102,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS_5", "group_name": "147_keywords_group5", - "group_id": 41, + "group_id": 132, "not_flag": 1, "clause_index": 4, "regions": [ @@ -1017,7 +1121,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS_6", "group_name": "147_keywords_group6", - "group_id": 42, + "group_id": 133, "not_flag": 1, "clause_index": 5, "regions": [ @@ -1036,7 +1140,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS_7", "group_name": "147_keywords_group7", - "group_id": 43, + "group_id": 134, "not_flag": 1, "clause_index": 6, "regions": [ @@ -1055,7 +1159,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS_8", "group_name": "147_keywords_group8", - "group_id": 44, + "group_id": 135, "not_flag": 1, "clause_index": 7, "regions": [ @@ -1085,7 +1189,7 @@ { "virtual_table": "HTTP_URL", "group_name": "148_url_group", - "group_id": 45, + "group_id": 136, "regions": [ { "table_name": "HTTP_URL", @@ -1113,7 +1217,7 @@ { "virtual_table": "APP_PAYLOAD", "group_name": "149_app_group", - "group_id": 46, + "group_id": 137, "regions": [ { "table_name": "APP_PAYLOAD", @@ -1142,7 +1246,7 @@ { "virtual_table": "TROJAN_PAYLOAD", "group_name": "billgates_regist1", - "group_id": 47, + "group_id": 138, "regions": [ { "table_type": "expr", @@ -1159,7 +1263,7 @@ { "virtual_table": "TROJAN_PAYLOAD", "group_name": "billgates_regist2", - "group_id": 48, + "group_id": 139, "regions": [ { "table_type": "expr", @@ -1187,7 +1291,7 @@ { "virtual_table": "MAIL_ADDR", "group_name": "151_expr_group", - "group_id": 49, + "group_id": 140, "regions": [ { "table_type": "expr", @@ -1215,7 +1319,7 @@ { "virtual_table": "MAIL_ADDR", "group_name": "152_mail_addr", - "group_id": 50, + "group_id": 141, "regions": [ { "table_type": "expr", @@ -1242,7 +1346,7 @@ { "virtual_table": "CONTENT_SIZE", "group_name": "interval_group_refered", - "group_id": 51, + "group_id": 142, "sub_groups": [ { "group_name": "126_interval_group" @@ -1263,7 +1367,7 @@ { "virtual_table": "MAIL_ADDR", "group_name": "153_expr_group", - "group_id": 52, + "group_id": 143, "not_flag": 0, "regions": [ { @@ -1287,7 +1391,7 @@ { "virtual_table": "IP_CONFIG", "group_name": "IP_group_refered", - "group_id": 53, + "group_id": 144, "sub_groups": [ { "group_name": "123_IP_group" @@ -1308,7 +1412,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "154_IP_group", - "group_id": 54, + "group_id": 145, "not_flag": 0, "regions": [ { @@ -1341,7 +1445,8 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "155_IP_group", - "group_id": 55, + "group_id": 146, + "not_flag": 0, "regions": [ { "table_type": "ip_plus", @@ -1357,8 +1462,7 @@ "protocol": 6 } } - ], - "not_flag": 0 + ] } ] }, @@ -1374,7 +1478,7 @@ { "virtual_table": "HTTP_SIGNATURE", "group_name": "156_expr_group", - "group_id": 56, + "group_id": 147, "regions": [ { "table_name": "HTTP_SIGNATURE", @@ -1403,7 +1507,7 @@ { "virtual_table": "TROJAN_PAYLOAD", "group_name": "157_expr_group", - "group_id": 57, + "group_id": 148, "regions": [ { "table_type": "expr", @@ -1431,7 +1535,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "158_IP_group", - "group_id": 58, + "group_id": 149, "regions": [ { "table_type": "ip_plus", @@ -1463,7 +1567,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "159_IP_group", - "group_id": 59, + "group_id": 150, "regions": [ { "table_type": "ip_plus", @@ -1500,7 +1604,7 @@ { "virtual_table": "HTTP_URL", "group_name": "160_url_group", - "group_id": 60, + "group_id": 151, "not_flag": 0, "regions": [ { @@ -1529,7 +1633,7 @@ { "virtual_table": "HTTP_SIGNATURE", "group_name": "vt_grp_http_sig1", - "group_id": 61, + "group_id": 152, "not_flag": 0, "regions": [ { @@ -1548,7 +1652,7 @@ { "virtual_table": "HTTP_SIGNATURE", "group_name": "vt_grp_http_sig2", - "group_id": 62, + "group_id": 153, "not_flag": 0, "regions": [ { @@ -1631,7 +1735,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "164_keywords_group", - "group_id": 63, + "group_id": 154, "regions": [ { "table_name": "KEYWORDS_TABLE", @@ -1660,7 +1764,7 @@ { "virtual_table": "HTTP_URL", "group_name": "165_url_group", - "group_id": 64, + "group_id": 155, "regions": [ { "table_name": "HTTP_URL", @@ -1677,7 +1781,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "165_IP_group", - "group_id": 65, + "group_id": 156, "not_flag": 0, "regions": [ { @@ -1711,7 +1815,7 @@ { "virtual_table": "HTTP_URL", "group_name": "166_url_group", - "group_id": 66, + "group_id": 157, "regions": [ { "table_name": "HTTP_URL", @@ -1740,7 +1844,7 @@ { "virtual_table": "HTTP_URL", "group_name": "167_url_group", - "group_id": 67, + "group_id": 158, "regions": [ { "table_name": "HTTP_URL", @@ -1769,7 +1873,7 @@ { "virtual_table": "HTTP_URL", "group_name": "168_url_group", - "group_id": 68, + "group_id": 159, "regions": [ { "table_name": "HTTP_URL", @@ -1797,7 +1901,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "169_IP_group", - "group_id": 69, + "group_id": 160, "not_flag" : 0, "regions": [ { @@ -1830,7 +1934,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "ipv4_virtual.source", - "group_id": 70, + "group_id": 161, "not_flag": 0, "regions": [ { @@ -1863,7 +1967,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "ipv4_virtual.destination", - "group_id": 71, + "group_id": 162, "not_flag": 0, "regions": [ { @@ -1917,7 +2021,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "ipv4_composition.source", - "group_id": 72, + "group_id": 163, "not_flag": 0, "regions": [ { @@ -1950,7 +2054,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "ipv4_composition.destination", - "group_id": 73, + "group_id": 164, "not_flag": 0, "regions": [ { @@ -1983,7 +2087,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "ipv4_composition.session", - "group_id": 74, + "group_id": 165, "not_flag": 0, "regions": [ { @@ -2004,30 +2108,41 @@ } ] }, + { + "compile_id": 177, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "NOTLogic.MultiGroupsInOneNotClause", + "is_valid": "yes", + "groups": [ + { + "virtual_table": "ASN_NOT_LOGIC", + "group_name": ["ASN1234", "ASN6789", "ASN9001"], + "not_flag": 1, + "clause_index": 0 + }, + { + "virtual_table": "DESTINATION_IP_ASN", + "group_name": "ASN2345", + "not_flag": 0, + "clause_index": 1 + } + ] + }, { "compile_id": 178, "service": 1, "action": 1, "do_blacklist": 1, "do_log": 1, - "user_region": "Hierarchy.TwoVirtualInOneClause", + "user_region": "Hierarchy.MultiGroupInOneClause", "is_valid": "yes", "groups": [ { "virtual_table": "SOURCE_IP_ASN", - "group_name": "ASN1234", - "not_flag": 0, - "clause_index": 0 - }, - { - "virtual_table": "IP_CONFIG", - "group_name": "financial-department-ip", - "not_flag": 0, - "clause_index": 0 - }, - { - "virtual_table": "SOURCE_IP_GEO", - "group_name": "Country-Sparta-IP", + "group_name": ["ASN1234", "ASN6789", "ASN9001"], "not_flag": 0, "clause_index": 0 }, @@ -2051,7 +2166,7 @@ { "virtual_table": "INTERGER_PLUS", "group_name": "179_interval_group", - "group_id": 75, + "group_id": 166, "regions": [ { "table_name": "INTERGER_PLUS", @@ -2066,6 +2181,64 @@ } ] }, + { + "compile_id": 180, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "Hierarchy.MultiGroupInOneClause", + "is_valid": "yes", + "groups": [ + { + "virtual_table": "SOURCE_IP_ASN", + "group_name": ["ASN1234", "ASN6789", "ASN9001"], + "not_flag": 0, + "clause_index": 0 + }, + { + "virtual_table": "SOURCE_IP_GEO", + "group_name": "Country-Sparta-IP", + "not_flag": 0, + "clause_index": 0 + }, + { + "virtual_table": "IP_CONFIG", + "group_name": "financial-department-ip", + "not_flag": 0, + "clause_index": 1 + } + ] + }, + { + "compile_id": 181, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "NOTLogic.MultiLiteralsInOneNotClause", + "is_valid": "yes", + "groups": [ + { + "virtual_table": "SOURCE_IP_ASN", + "group_name": ["ASN1234", "ASN6789", "ASN9001"], + "not_flag": 1, + "clause_index": 0 + }, + { + "virtual_table": "IP_PLUS_CONFIG", + "group_name": "develop-department-ip", + "not_flag": 1, + "clause_index": 0 + }, + { + "virtual_table": "SOURCE_IP_GEO", + "group_name": "Country-Sparta-IP", + "not_flag": 0, + "clause_index": 1 + } + ] + }, { "compile_id": 182, "service": 1, @@ -2078,7 +2251,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "182_keywords_group", - "group_id": 76, + "group_id": 167, "regions": [ { "table_name": "KEYWORDS_TABLE", @@ -2106,7 +2279,7 @@ { "virtual_table": "CORNER_CASE_TABLE", "group_name": "183_expr_group", - "group_id": 77, + "group_id": 168, "regions": [ { "table_name": "CORNER_CASE_TABLE", @@ -2135,7 +2308,7 @@ { "virtual_table": "IP_CONFIG", "group_name": "184_IP_group", - "group_id": 78, + "group_id": 169, "regions": [ { "table_name": "IP_CONFIG", @@ -2155,6 +2328,47 @@ } ] }, + { + "compile_id": 185, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "NOTLogic.SameVtableInMultiClause", + "is_valid": "yes", + "groups": [ + { + "virtual_table": "DESTINATION_IP_ASN", + "group_name": ["ASN1234", "ASN6789", "ASN9001"], + "not_flag": 1, + "clause_index": 0 + }, + { + "virtual_table": "SOURCE_IP_GEO", + "group_name": "Country-Sparta-IP", + "not_flag": 1, + "clause_index": 0 + }, + { + "virtual_table": "DESTINATION_IP_ASN", + "group_name": "ASN9002", + "not_flag": 1, + "clause_index": 1 + }, + { + "virtual_table": "DESTINATION_IP_ASN", + "group_name": "ASN9003", + "not_flag": 0, + "clause_index": 2 + }, + { + "virtual_table": "IP_PLUS_CONFIG", + "group_name": "security-department-ip", + "not_flag": 0, + "clause_index": 3 + } + ] + }, { "compile_id": 186, "service": 1, @@ -2167,7 +2381,7 @@ { "virtual_table": "HTTP_URL_FILTER", "group_name": "186_expr_group", - "group_id": 79, + "group_id": 170, "not_flag": 1, "regions": [ { @@ -2185,7 +2399,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "186_IP_group", - "group_id": 80, + "group_id": 171, "not_flag": 0, "regions": [ { @@ -2218,7 +2432,7 @@ { "virtual_table": "HTTP_URL_FILTER", "group_name": "187_url_group", - "group_id": 81, + "group_id": 172, "not_flag": 1, "regions": [ { @@ -2236,7 +2450,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "187_IP_group", - "group_id": 82, + "group_id": 173, "not_flag": 0, "regions": [ { @@ -2269,7 +2483,7 @@ { "virtual_table": "HTTP_URL_FILTER", "group_name": "188_url_group", - "group_id": 83, + "group_id": 174, "not_flag": 1, "regions": [ { @@ -2287,7 +2501,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "188_IP_group", - "group_id": 84, + "group_id": 175, "not_flag": 0, "regions": [ { @@ -2320,18 +2534,18 @@ { "virtual_table": "APP_PAYLOAD", "group_name": "189_app_group", - "group_id": 85, + "group_id": 176, "regions": [ { "table_name": "APP_PAYLOAD", + "table_type": "expr_plus", "table_content": { "format": "hexbin", "match_method": "sub", "district": "tcp.payload.c2s_first_data", "keywords": "ab00", "expr_type": "none" - }, - "table_type": "expr_plus" + } } ] } @@ -2349,7 +2563,7 @@ { "virtual_table": "HTTP_SIGNATURE", "group_name": "190_expr_group", - "group_id": 86, + "group_id": 177, "regions": [ { "table_name": "HTTP_SIGNATURE", @@ -2378,7 +2592,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "191_keywords_group", - "group_id": 87, + "group_id": 178, "regions": [ { "table_type": "expr", @@ -2406,7 +2620,7 @@ { "virtual_table": "FLAG_CONFIG", "group_name": "192_flag_group", - "group_id": 88, + "group_id": 179, "regions": [ { "table_type": "flag", @@ -2432,7 +2646,7 @@ { "virtual_table": "FLAG_CONFIG", "group_name": "193_flag_group", - "group_id": 89, + "group_id": 180, "regions": [ { "table_type": "flag", @@ -2447,7 +2661,7 @@ { "virtual_table": "HTTP_URL", "group_name": "193_url_group", - "group_id": 90, + "group_id": 181, "regions": [ { "table_name": "HTTP_URL", @@ -2475,7 +2689,7 @@ { "virtual_table": "FLAG_CONFIG", "group_name": "194_flag_group", - "group_id": 91, + "group_id": 182, "regions": [ { "table_type": "flag", @@ -2501,7 +2715,7 @@ { "virtual_table": "HTTP_SIGNATURE", "group_name": "195_signature_group", - "group_id": 92, + "group_id": 183, "regions": [ { "table_name": "HTTP_SIGNATURE", @@ -2519,7 +2733,7 @@ { "virtual_table": "HTTP_URL", "group_name": "195_url_group", - "group_id": 93, + "group_id": 184, "regions": [ { "table_name": "HTTP_URL", @@ -2547,7 +2761,7 @@ { "virtual_table": "FLAG_PLUS_CONFIG", "group_name": "196_flag_group", - "group_id": 94, + "group_id": 185, "regions": [ { "table_type": "flag_plus", @@ -2574,7 +2788,7 @@ { "virtual_table": "HTTP_URL", "group_name": "197_url_group", - "group_id": 95, + "group_id": 186, "regions": [ { "table_name": "HTTP_URL", @@ -2604,7 +2818,7 @@ "g2c_table_name": "GROUP2COMPILE_FIREWALL", "virtual_table": "HTTP_URL", "group_name": "198_url_group", - "group_id": 96, + "group_id": 187, "regions": [ { "table_name": "HTTP_URL", @@ -2632,11 +2846,11 @@ { "virtual_table": "HTTP_URL", "group_name": "ExcludeLogicGroup199", - "group_id": 97, + "group_id": 188, "sub_groups":[ { "group_name": "ExcludeLogicGroup199_1", - "group_id": 98, + "group_id": 189, "is_exclude": 0, "clause_index": 0, "regions": [ @@ -2654,7 +2868,7 @@ }, { "group_name": "ExcludeLogicGroup199_2", - "group_id": 99, + "group_id": 190, "is_exclude": 1, "clause_index": 0, "regions": [ @@ -2686,12 +2900,12 @@ { "virtual_table": "HTTP_URL", "group_name": "ExcludeLogicGroup200", - "group_id": 100, + "group_id": 191, "sub_groups":[ { "virtual_table": "HTTP_URL", "group_name": "ExcludeLogicGroup200_1", - "group_id": 101, + "group_id": 192, "is_exclude": 0, "clause_index": 0, "regions": [ @@ -2710,7 +2924,7 @@ { "virtual_table": "HTTP_URL", "group_name": "ExcludeLogicGroup200_2", - "group_id": 102, + "group_id": 193, "is_exclude": 1, "clause_index": 0, "regions": [ @@ -2742,12 +2956,12 @@ { "virtual_table": "VIRTUAL_IP_PLUS_TABLE", "group_name": "ExcludeLogicGroup202", - "group_id": 103, + "group_id": 194, "clause_index": 0, "sub_groups":[ { "group_name": "ExcludeLogicGroup202_1", - "group_id": 104, + "group_id": 195, "is_exclude": 0, "regions": [ { @@ -2768,7 +2982,7 @@ }, { "group_name": "ExcludeLogicGroup202_2", - "group_id": 105, + "group_id": 196, "is_exclude": 1, "regions": [ { @@ -2789,7 +3003,7 @@ }, { "group_name": "ExcludeLogicGroup202_3", - "group_id": 106, + "group_id": 197, "is_exclude": 1, "regions": [ { @@ -2824,7 +3038,7 @@ { "virtual_table": "VIRTUAL_IP_PLUS_SOURCE", "group_name": "ExcludeLogicGroup203_1", - "group_id": 107, + "group_id": 198, "clause_index": 0, "regions": [ { @@ -2848,7 +3062,7 @@ { "virtual_table": "VIRTUAL_IP_PLUS_DESTINATION", "group_name": "ExcludeLogicGroup203_2", - "group_id": 108, + "group_id": 199, "clause_index": 1, "regions": [ { @@ -2871,12 +3085,12 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS", "group_name": "ExcludeLogicGroup203_3", - "group_id": 109, + "group_id": 200, "clause_index": 2, "sub_groups": [ { "group_name": "ExcludeLogicGroup203_3_1", - "group_id": 110, + "group_id": 201, "is_exclude": 0, "regions": [ { @@ -2893,7 +3107,7 @@ }, { "group_name": "ExcludeLogicGroup203_3_2", - "group_id": 111, + "group_id": 202, "is_exclude": 1, "regions": [ { @@ -2924,7 +3138,7 @@ { "virtual_table": "VIRTUAL_IP_PLUS_SOURCE", "group_name": "ExcludeLogicGroup204_1", - "group_id": 112, + "group_id": 203, "clause_index": 0, "regions": [ { @@ -2948,7 +3162,7 @@ { "virtual_table": "VIRTUAL_IP_PLUS_DESTINATION", "group_name": "ExcludeLogicGroup204_2", - "group_id": 113, + "group_id":204, "clause_index": 1, "regions": [ { @@ -2971,17 +3185,17 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS", "group_name": "ExcludeLogicGroup204_3", - "group_id": 114, + "group_id": 205, "clause_index": 2, "sub_groups": [ { "group_name": "ExcludeLogicGroup204_3_1", - "group_id": 115, + "group_id": 206, "is_exclude": 0, "sub_groups" : [ { "group_name": "ExcludeLogicGroup204_3_1_1", - "group_id": 116, + "group_id": 207, "is_exclude": 0, "regions": [ { @@ -2998,7 +3212,7 @@ }, { "group_name": "ExcludeLogicGroup204_3_1_2", - "group_id": 117, + "group_id": 208, "is_exclude": 1, "regions": [ { @@ -3017,7 +3231,7 @@ }, { "group_name": "ExcludeLogicGroup204_3_2", - "group_id": 118, + "group_id": 209, "is_exclude": 1, "regions": [ { @@ -3048,7 +3262,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "205_keywords_group", - "group_id": 119, + "group_id": 210, "regions": [ { "table_type": "expr", @@ -3076,7 +3290,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "206_keywords_group", - "group_id": 120, + "group_id": 211, "regions": [ { "table_type": "expr", @@ -3104,7 +3318,7 @@ { "virtual_table": "FLAG_CONFIG", "group_name": "207_flag_group", - "group_id": 121, + "group_id": 212, "regions": [ { "table_type": "flag", @@ -3130,7 +3344,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "208_IP_group", - "group_id": 122, + "group_id": 213, "not_flag": 0, "regions": [ { @@ -3163,7 +3377,7 @@ { "virtual_table": "INTERGER_PLUS", "group_name": "209_interval_group", - "group_id": 123, + "group_id": 214, "regions": [ { "table_name": "INTERGER_PLUS", @@ -3190,7 +3404,7 @@ { "virtual_table": "IP_PLUS_CONFIG", "group_name": "210_IP_group", - "group_id": 124, + "group_id": 215, "regions": [ { "table_type": "ip_plus", @@ -3222,7 +3436,7 @@ { "virtual_table": "IP_PERF_CONFIG", "group_name": "211_IP_group", - "group_id": 125, + "group_id": 216, "not_flag": 0, "regions": [ { @@ -3255,7 +3469,7 @@ { "virtual_table": "INTEGER_PERF_CONFIG", "group_name": "212_interval_group", - "group_id": 126, + "group_id": 217, "regions": [ { "table_name": "INTEGER_PERF_CONFIG", @@ -3281,7 +3495,7 @@ { "virtual_table": "EXPR_LITERAL_PERF_CONFIG", "group_name": "213_expr_group", - "group_id": 127, + "group_id": 218, "regions": [ { "table_name": "EXPR_LITERAL_PERF_CONFIG", @@ -3309,7 +3523,7 @@ { "virtual_table": "FLAG_PERF_CONFIG", "group_name": "214_flag_group", - "group_id": 128, + "group_id": 219, "regions": [ { "table_type": "flag", @@ -3335,7 +3549,7 @@ { "virtual_table": "EXPR_REGEX_PERF_CONFIG", "group_name": "215_expr_group", - "group_id": 129, + "group_id": 220, "regions": [ { "table_name": "EXPR_REGEX_PERF_CONFIG", @@ -3369,7 +3583,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS", "group_name": "NOTClauseAndExcludeGroup216", - "group_id": 130, + "group_id": 221, "not_flag": 1, "clause_index": 1, "regions": [ @@ -3399,13 +3613,13 @@ { "virtual_table": "HTTP_URL_FILTER", "group_name": "NOTClauseAndExcludeGroup217_1", - "group_id": 131, + "group_id": 222, "not_flag": 1, "clause_index": 0, "sub_groups": [ { "group_name": "ExcludeLogicGroup217_1_1", - "group_id": 132, + "group_id": 223, "is_exclude": 0, "regions": [ { @@ -3422,7 +3636,7 @@ }, { "group_name": "ExcludeLogicGroup217_1_2", - "group_id": 133, + "group_id": 224, "is_exclude": 1, "regions": [ { @@ -3442,7 +3656,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS", "group_name": "NOTClauseAndExcludeGroup217_2", - "group_id": 134, + "group_id": 225, "not_flag": 0, "clause_index": 1, "regions": [ @@ -3472,7 +3686,7 @@ { "virtual_table": "CONTENT_SIZE", "group_name": "218_interval_group", - "group_id": 135, + "group_id": 226, "regions": [ { "table_name": "CONTENT_SIZE", @@ -3498,7 +3712,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup219_1", - "group_id": 136, + "group_id": 227, "not_flag": 0, "clause_index": 0, "regions": [ @@ -3517,7 +3731,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup219_2", - "group_id": 137, + "group_id": 228, "not_flag": 1, "clause_index": 1, "regions": [ @@ -3536,7 +3750,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup219_3", - "group_id": 138, + "group_id": 229, "not_flag": 1, "clause_index": 2, "regions": [ @@ -3555,7 +3769,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup219_4", - "group_id": 139, + "group_id": 230, "not_flag": 1, "clause_index": 3, "regions": [ @@ -3574,7 +3788,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup219_5", - "group_id": 140, + "group_id": 231, "not_flag": 1, "clause_index": 4, "regions": [ @@ -3593,7 +3807,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup219_6", - "group_id": 141, + "group_id": 232, "not_flag": 1, "clause_index": 5, "regions": [ @@ -3612,7 +3826,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup219_7", - "group_id": 142, + "group_id": 233, "not_flag": 1, "clause_index": 6, "regions": [ @@ -3631,7 +3845,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup219_8", - "group_id": 143, + "group_id": 234, "not_flag": 1, "clause_index": 7, "regions": [ @@ -3661,7 +3875,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup220_1", - "group_id": 144, + "group_id": 235, "not_flag": 0, "clause_index": 0, "regions": [ @@ -3680,7 +3894,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup220_2", - "group_id": 145, + "group_id": 236, "not_flag": 1, "clause_index": 1, "regions": [ @@ -3699,7 +3913,7 @@ { "virtual_table": "HTTP_DUMMY", "group_name": "NOTClauseAndExcludeGroup220_3", - "group_id": 146, + "group_id": 237, "not_flag": 1, "clause_index": 2, "regions": [ @@ -3729,7 +3943,7 @@ { "virtual_table": "HTTP_REQUEST_HEADER", "group_name": "NOTLogicGroup_221_1", - "group_id": 147, + "group_id": 238, "not_flag": 1, "regions": [ { @@ -3748,7 +3962,7 @@ { "virtual_table": "HTTP_URL", "group_name": "NOTLogicGroup_221_2", - "group_id": 148, + "group_id": 239, "not_flag": 0, "regions": [ { @@ -3771,59 +3985,21 @@ "action": 0, "do_blacklist": 0, "do_log": 0, - "user_region": "NOTLogic.SameClauseHasMultiNotGroups", + "user_region": "NOTLogic.SingleNotClause", "is_valid": "yes", "groups": [ { - "virtual_table": "HTTP_URL_FILTER", - "group_name": "NOTLogicGroup_222_1", - "group_id": 149, - "clause_index": 0, + "virtual_table": "HTTP_NOT_LOGIC_1", + "group_name": "NOTLogicGroup_222", + "group_id": 240, "not_flag": 1, - "regions": [ - { - "table_name": "HTTP_URL", - "table_type": "expr", - "table_content": { - "keywords": "not_logic_compile_222_1", - "expr_type": "none", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - }, - { - "virtual_table": "HTTP_URL_FILTER", - "group_name": "NOTLogicGroup_222_2", - "group_id": 150, "clause_index": 0, - "not_flag": 1, "regions": [ { - "table_name": "HTTP_URL", + "table_name": "KEYWORDS_TABLE", "table_type": "expr", "table_content": { - "keywords": "not_logic_compile_222_2", - "expr_type": "none", - "match_method": "sub", - "format": "uncase plain" - } - } - ] - }, - { - "virtual_table": "HTTP_URL_FILTER", - "group_name": "NOTLogicGroup_222_3", - "group_id": 151, - "clause_index": 1, - "not_flag": 0, - "regions": [ - { - "table_name": "HTTP_URL", - "table_type": "expr", - "table_content": { - "keywords": "logic_compile_222_3", + "keywords": "not_logic_keywords_222", "expr_type": "none", "match_method": "sub", "format": "uncase plain" @@ -3845,7 +4021,7 @@ { "virtual_table": "HTTP_NOT_LOGIC", "group_name": "NOTLogicGroup_223_1", - "group_id": 152, + "group_id": 241, "not_flag": 1, "clause_index": 0, "regions": [ @@ -3864,7 +4040,7 @@ { "virtual_table": "HTTP_NOT_LOGIC", "group_name": "NOTLogicGroup_223_2", - "group_id": 153, + "group_id": 242, "not_flag": 1, "clause_index": 1, "regions": [ @@ -3883,7 +4059,7 @@ { "virtual_table": "HTTP_NOT_LOGIC", "group_name": "NOTLogicGroup_223_1", - "group_id": 154, + "group_id": 243, "not_flag": 1, "clause_index": 2, "regions": [ @@ -3913,7 +4089,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "NOTLogicGroup_224_1", - "group_id": 155, + "group_id": 244, "not_flag": 1, "clause_index": 0, "regions": [ @@ -3932,7 +4108,7 @@ { "virtual_table": "HTTP_RESPONSE_KEYWORDS", "group_name": "NOTLogicGroup_224_2", - "group_id": 156, + "group_id": 245, "not_flag": 0, "clause_index": 1, "regions": [ @@ -3962,7 +4138,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "EscapeGroup_225_1", - "group_id": 157, + "group_id": 246, "not_flag": 0, "clause_index": 0, "regions": [ @@ -3992,7 +4168,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "226_url_group", - "group_id": 158 + "group_id":247 } ] }, @@ -4009,7 +4185,7 @@ { "virtual_table": "KEYWORDS_TABLE", "group_name": "227_url_group", - "group_id": 159, + "group_id": 248, "g2c_table_name": "GROUP2COMPILE_FIREWALL" } ] diff --git a/test/table_info.conf b/test/table_info.conf index 07d655d..dfd9deb 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -712,5 +712,17 @@ "table_name":"HTTP_NOT_LOGIC", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" + }, + { + "table_id":64, + "table_name":"HTTP_NOT_LOGIC_1", + "table_type":"virtual", + "physical_table": "KEYWORDS_TABLE" + }, + { + "table_id":65, + "table_name":"ASN_NOT_LOGIC", + "table_type":"virtual", + "physical_table":"AS_NUMBER" } ] \ No newline at end of file