gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
This repository has been archived on 2025-09-14. You can view files and clone it, but cannot push or open issues or pull requests.
Files
e65239abe7c579f56c31391224380fe8ea82196a
tango-maat/src/maat_compile.c

2675 lines
96 KiB
C
Raw Blame History

/*
**********************************************************************************************
* File: maat_compile.c
* Description:
* Authors: Liu wentan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#include <assert.h>
#include <pthread.h>
#include <linux/limits.h>
#include "maat_utils.h"
#include "log/log.h"
#include "maat.h"
#include "uthash/utarray.h"
#include "uthash/uthash.h"
#include "bool_matcher.h"
#include "igraph/igraph.h"
#include "maat_compile.h"
#include "maat_garbage_collection.h"
#include "maat_group.h"
#include "maat_ex_data.h"
#include "maat_table.h"
#include "alignment.h"
#define MODULE_COMPILE module_name_str("maat.compile")
#define MAX_GROUP_CNT 128
#define MAX_SUPER_GROUP_CNT 128
#define MAX_NOT_CLAUSE_NUM 8
enum clause_not_flag {
CLAUSE_NOT_FLAG_UNSET = 0,
CLAUSE_NOT_FLAG_SET
};
struct compile_schema {
int compile_id_column;
int rule_tag_column;
int declared_clause_num_column;
int table_id;
struct table_manager *ref_tbl_mgr;
struct log_handle *logger;
};
struct group2compile_schema {
int group_id_column;
int compile_id_column;
int not_flag_column;
int vtable_name_column;
int clause_index_column;
int asso_compile_table_id; //asso is abbreviation for associated
int table_id;
struct table_manager *ref_tbl_mgr;
};
struct compile_item {
int declared_clause_num;
long long compile_id;
char *table_line;
size_t table_line_len;
};
struct group2compile_item {
UT_array *group_ids;
long long compile_id;
int not_flag;
int vtable_id;
int clause_index;
};
struct clause_query_key {
long long group_id;
int vtable_id;
int not_flag;
};
struct clause_id_kv {
struct clause_query_key key;
UT_array *clause_ids;
UT_hash_handle hh;
};
struct table_clause {
int vtable_id;
int actual_clause_num;
UT_array *clause_ids;
UT_array *group_ids;
UT_hash_handle hh;
};
struct table_group {
int vtable_id;
UT_array *group_ids;
UT_hash_handle hh;
};
/* compile_runtime and group2compile_runtime share compile_hash_map */
struct compile_runtime {
struct bool_matcher *bm;
struct rcu_hash_table *cfg_hash; // <compile_id, struct maat_compile>
struct maat_runtime *ref_maat_rt;
struct clause_id_kv *clause_id_kv_hash; //store clause_ids(not_flag == 0)
struct clause_id_kv *not_clause_id_kv_hash; //store NOT_clause_ids(not_flag == 1)
struct bool_expr_match *expr_match_buff;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
time_t version;
long long rule_num;
long long update_err_cnt;
};
struct group2compile_runtime {
long long not_clause_cnt;
long long rule_num;
long long update_err_cnt;
struct compile_runtime *ref_compile_rt;
struct table_clause *tbl_not_clause_hash; //each virtual table's not clause number <= MAX_NOT_CLAUSE_NUM
};
struct clause_literal {
long long group_ids[MAX_GROUP_CNT];
int group_cnt;
int vtable_id;
};
struct compile_clause {
long long clause_id;
UT_array *literals; //struct clause_literal
char not_flag; // 1 byte
char in_use; // 1 byte
char pad[6]; // for 8 bytes alignment
};
struct compile_sort_para {
int declared_clause_num;
long long compile_id;
};
#define MAAT_COMPILE_MAGIC 0x4a5b6c7d
struct maat_compile {
uint32_t magic_num;
int actual_clause_num;
int declared_clause_num;
int table_id;
long long compile_id;
void *user_data; // compile_item
struct compile_clause clauses[MAX_ITEMS_PER_BOOL_EXPR];
};
struct internal_hit_path {
long long item_id;
long long group_id;
int Nth_scan;
int vtable_id;
int NOT_flag; // 1 means NOT clause
};
struct compile2table_id {
long long compile_id;
int table_id;
};
struct compile_state {
int Nth_scan;
int this_scan_not_logic;
time_t compile_rt_version;
UT_array *internal_hit_paths;
UT_array *all_hit_clauses;
UT_array *this_scan_hit_clauses;
UT_array *this_scan_hit_not_clauses;
UT_array *exclude_not_clauses;
UT_array *direct_hit_groups;
UT_array *indirect_hit_groups;
UT_array *last_hit_group_ids;
UT_array *hit_compile_table_ids;
struct table_group *hit_not_tbl_groups;
};
UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL};
UT_icd ut_clause_literal_icd = {sizeof(struct clause_literal), NULL, NULL, NULL};
UT_icd ut_compile_group_id_icd = {sizeof(long long), NULL, NULL, NULL};
UT_icd ut_maat_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL};
UT_icd ut_hit_path_icd = {sizeof(struct internal_hit_path), NULL, NULL, NULL};
UT_icd ut_hit_compile_table_id_icd = {sizeof(struct compile2table_id), NULL, NULL, NULL};
static struct maat_compile *maat_compile_new(long long compile_id)
{
struct maat_compile *compile = ALLOC(struct maat_compile, 1);
compile->magic_num = MAAT_COMPILE_MAGIC;
compile->compile_id = compile_id;
for(int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
utarray_new(compile->clauses[i].literals, &ut_clause_literal_icd);
compile->clauses[i].in_use = 0;
compile->clauses[i].clause_id = 0;
}
return compile;
}
static int maat_compile_set(struct maat_compile *compile, int table_id,
int declared_clause_num, void *user_data)
{
if (NULL == compile) {
return -1;
}
compile->table_id = table_id;
compile->declared_clause_num = declared_clause_num;
compile->user_data = user_data;
return 0;
}
static int compile_accept_tag_match(struct compile_schema *schema, const char *line,
const char *table_name, struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr);
if (schema->rule_tag_column > 0 && n_tag > 0) {
int ret = get_column_pos(line, schema->rule_tag_column,
&column_offset, &column_len);
if (ret < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has no rule_tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (column_len > 2) {
char *tag_str = ALLOC(char, column_len + 1);
memcpy(tag_str, (line + column_offset), column_len);
ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str);
FREE(tag_str);
if (TAG_MATCH_ERR == ret) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has invalid tag format in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_ERR;
}
if (TAG_MATCH_UNMATCHED == ret) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has unmatched tag in line:%s",
__FUNCTION__, __LINE__, table_name, line);
return TAG_MATCH_UNMATCHED;
}
}
}
return TAG_MATCH_MATCHED;
}
static struct compile_item *
compile_item_new(const char *table_line, struct compile_schema *schema,
const char *table_name, struct log_handle *logger)
{
int ret = compile_accept_tag_match(schema, table_line, table_name, logger);
if (ret == TAG_MATCH_UNMATCHED) {
return NULL;
}
size_t column_offset = 0;
size_t column_len = 0;
struct compile_item *compile_item = ALLOC(struct compile_item, 1);
ret = get_column_pos(table_line, schema->compile_id_column,
&column_offset, &column_len);
if (ret < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has no compile_id in line:%s",
__FUNCTION__, __LINE__, table_name, table_line);
goto error;
}
compile_item->compile_id = atoll(table_line + column_offset);
ret = get_column_pos(table_line, schema->declared_clause_num_column,
&column_offset, &column_len);
if (ret < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> has no clause_num in line:%s",
__FUNCTION__, __LINE__, table_name, table_line);
goto error;
}
compile_item->declared_clause_num = atoi(table_line + column_offset);
if (compile_item->declared_clause_num < 0 ||
compile_item->declared_clause_num > MAX_NOT_CLAUSE_NUM) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> clause_num:%d exceed maximum:%d in line:%s",
__FUNCTION__, __LINE__, table_name, compile_item->declared_clause_num,
MAX_NOT_CLAUSE_NUM, table_line);
goto error;
}
compile_item->table_line_len = strlen(table_line);
compile_item->table_line = ALLOC(char, compile_item->table_line_len + 1);
memcpy(compile_item->table_line, table_line, compile_item->table_line_len);
return compile_item;
error:
FREE(compile_item);
return NULL;
}
static void compile_item_free(struct compile_item *item)
{
item->declared_clause_num = 0;
if (item->table_line != NULL) {
FREE(item->table_line);
}
FREE(item);
}
static void maat_compile_free(struct maat_compile *compile)
{
struct compile_clause *clause = NULL;
if (compile->user_data != NULL) {
compile_item_free(compile->user_data);
compile->user_data = NULL;
}
for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
clause = compile->clauses + i;
if (clause->literals != NULL) {
utarray_free(clause->literals);
clause->literals = NULL;
}
clause->in_use = 0;
clause->clause_id = 0;
}
compile->magic_num = 0;
FREE(compile);
}
static void rcu_compile_cfg_free(void *user_ctx, void *data)
{
struct maat_compile *compile = (struct maat_compile *)data;
maat_compile_free(compile);
}
void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name,
struct log_handle *logger)
{
struct compile_schema *schema = ALLOC(struct compile_schema, 1);
schema->logger = logger;
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
schema->table_id = item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "compile_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->compile_id_column = custom_item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no compile_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "tags");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->rule_tag_column = custom_item->valueint;
}
custom_item = cJSON_GetObjectItem(item, "clause_num");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->declared_clause_num_column = custom_item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no clause_num column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
schema->ref_tbl_mgr = tbl_mgr;
return schema;
error:
FREE(schema);
return NULL;
}
void compile_schema_free(void *compile_schema)
{
FREE(compile_schema);
}
void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
struct group2compile_schema *g2c_schema = ALLOC(struct group2compile_schema, 1);
cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
g2c_schema->table_id = item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
item = cJSON_GetObjectItem(json, "associated_compile_table_id");
if (item != NULL && item->type == cJSON_Number) {
g2c_schema->asso_compile_table_id = item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no associated_compile_table_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no custom column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2c_schema->group_id_column = custom_item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no group_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "compile_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2c_schema->compile_id_column = custom_item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no compile_id column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "not_flag");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2c_schema->not_flag_column = custom_item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no not_flag column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "virtual_table_name");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2c_schema->vtable_name_column = custom_item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no virtual_table_name column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
custom_item = cJSON_GetObjectItem(item, "clause_index");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2c_schema->clause_index_column = custom_item->valueint;
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> schema has no clause_index column",
__FUNCTION__, __LINE__, table_name);
goto error;
}
g2c_schema->ref_tbl_mgr = tbl_mgr;
return g2c_schema;
error:
FREE(g2c_schema);
return NULL;
}
void group2compile_schema_free(void *g2c_schema)
{
FREE(g2c_schema);
}
int group2compile_associated_compile_table_id(void *g2c_schema)
{
struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema;
return schema->asso_compile_table_id;
}
#define COMPILE_GC_TIMEOUT_S 5
void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == compile_schema) {
return NULL;
}
struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1);
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match,
max_thread_num * MAX_HIT_COMPILE_NUM);
compile_rt->version = time(NULL);
compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL, COMPILE_GC_TIMEOUT_S);
compile_rt->clause_id_kv_hash = NULL;
compile_rt->not_clause_id_kv_hash = NULL;
compile_rt->logger = logger;
compile_rt->ref_garbage_bin = garbage_bin;
return compile_rt;
}
static void clause_id_kv_hash_free(struct clause_id_kv *hash)
{
struct clause_id_kv *clause_id_kv = NULL, *tmp_clause_id_kv = NULL;
HASH_ITER(hh, hash, clause_id_kv, tmp_clause_id_kv) {
HASH_DEL(hash, clause_id_kv);
if (clause_id_kv->clause_ids != NULL) {
utarray_free(clause_id_kv->clause_ids);
clause_id_kv->clause_ids = NULL;
}
FREE(clause_id_kv);
}
assert(hash == NULL);
}
static void garbage_clause_id_kv_hash_free(void *hash, void *arg)
{
clause_id_kv_hash_free((struct clause_id_kv *)hash);
}
void compile_runtime_free(void *compile_runtime)
{
if (NULL == compile_runtime) {
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
if (compile_rt->bm != NULL) {
bool_matcher_free(compile_rt->bm);
compile_rt->bm = NULL;
}
if (compile_rt->cfg_hash != NULL) {
rcu_hash_free(compile_rt->cfg_hash);
compile_rt->cfg_hash = NULL;
}
if (compile_rt->clause_id_kv_hash != NULL) {
clause_id_kv_hash_free(compile_rt->clause_id_kv_hash);
compile_rt->clause_id_kv_hash = NULL;
}
if (compile_rt->not_clause_id_kv_hash != NULL) {
clause_id_kv_hash_free(compile_rt->not_clause_id_kv_hash);
compile_rt->not_clause_id_kv_hash = NULL;
}
if (compile_rt->expr_match_buff != NULL) {
FREE(compile_rt->expr_match_buff);
}
FREE(compile_rt);
}
void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt)
{
if (NULL == compile_runtime) {
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
compile_rt->ref_maat_rt = maat_rt;
}
void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == g2c_schema) {
return NULL;
}
struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1);
g2c_rt->tbl_not_clause_hash = NULL;
return g2c_rt;
}
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime)
{
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
g2c_rt->ref_compile_rt = (struct compile_runtime *)compile_runtime;
}
void group2compile_runtime_free(void *g2c_runtime)
{
if (NULL == g2c_runtime) {
return;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
if (g2c_rt->tbl_not_clause_hash != NULL) {
struct table_clause *not_clause = NULL, *tmp_not_clause = NULL;
HASH_ITER(hh, g2c_rt->tbl_not_clause_hash, not_clause, tmp_not_clause) {
HASH_DEL(g2c_rt->tbl_not_clause_hash, not_clause);
FREE(not_clause);
}
}
assert(g2c_rt->tbl_not_clause_hash == NULL);
FREE(g2c_runtime);
}
static int is_valid_table_name(const char *str)
{
size_t integer_cnt=0;
for (size_t i = 0; i < strlen(str); i++) {
if (str[i] >= '0' && str[i] <= '9') {
integer_cnt++;
}
}
if (strlen(str) == 0 || integer_cnt == strlen(str) ||
0 == strcasecmp(str, "null")) {
return 0;
}
return 1;
}
static int group_ids_str2longlong(const char *group_ids_str, UT_array *group_ids)
{
int counter = 0;
char *str = NULL;
char *saveptr = NULL;
char *subtoken = NULL;
const char *seps = ",";
char *dup_line = maat_strdup(group_ids_str);
for (str = dup_line; ; str = NULL) {
subtoken = strtok_r(str, seps, &saveptr);
if (subtoken == NULL)
break;
long long group_id = atoll(subtoken);
utarray_push_back(group_ids, &group_id);
counter++;
}
FREE(dup_line);
if (0 == counter) {
return -1;
}
return 0;
}
static void group2compile_item_free(struct group2compile_item *g2c_item)
{
if (NULL == g2c_item) {
return;
}
if (g2c_item->group_ids != NULL) {
utarray_free(g2c_item->group_ids);
g2c_item->group_ids = NULL;
}
FREE(g2c_item);
}
static struct group2compile_item *
group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema,
const char *table_name, struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
char vtable_name[MAX_NAME_STR_LEN + 1] = {0};
struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1);
utarray_new(g2c_item->group_ids, &ut_compile_group_id_icd);
int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> has no group_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
char group_ids_str[MAX_GROUP_IDS_STR_LEN] = {0};
memcpy(group_ids_str, line + column_offset, MIN(MAX_GROUP_IDS_STR_LEN, column_len));
ret = group_ids_str2longlong(group_ids_str, g2c_item->group_ids);
if (ret < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> group_ids str2longlong failed in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
if (utarray_len(g2c_item->group_ids) > MAX_GROUP_CNT) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> group_ids exceed maximum:%d in line:%s",
__FUNCTION__, __LINE__, table_name, MAX_GROUP_CNT, line);
goto error;
}
ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset,
&column_len);
if (ret < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> has no compile_id in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
g2c_item->compile_id = atoll(line + column_offset);
ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset,
&column_len);
if (ret < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> has no NOT_flag in line:%s ",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
g2c_item->not_flag = atoi(line + column_offset);
if (g2c_item->not_flag != CLAUSE_NOT_FLAG_SET &&
g2c_item->not_flag != CLAUSE_NOT_FLAG_UNSET) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> NOT_flag:%d is illegal in line:%s ",
__FUNCTION__, __LINE__, table_name, g2c_item->not_flag, line);
goto error;
}
ret = get_column_pos(line, g2c_schema->vtable_name_column, &column_offset,
&column_len);
if (ret < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> has no virtual_table_name in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
if (column_len > MAX_NAME_STR_LEN) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> virtual_table_name length exceed "
"maxium:%d in line:%s", __FUNCTION__, __LINE__, table_name,
MAX_NAME_STR_LEN, line);
goto error;
}
memset(vtable_name, 0, sizeof(vtable_name));
memcpy(vtable_name, (line + column_offset), column_len);
if (is_valid_table_name(vtable_name)) {
g2c_item->vtable_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr,
vtable_name);
if (g2c_item->vtable_id < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> has unknown virtual table:%s in line:%s",
__FUNCTION__, __LINE__, table_name, vtable_name, line);
goto error;
}
}
ret = get_column_pos(line, g2c_schema->clause_index_column, &column_offset,
&column_len);
if (ret < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> has no clause_index in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
g2c_item->clause_index = atoi(line + column_offset);
if (g2c_item->clause_index < 0 || g2c_item->clause_index >= MAX_NOT_CLAUSE_NUM) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> clause_index:%d exceed maximum:%d in line:%s",
__FUNCTION__, __LINE__, table_name, g2c_item->clause_index,
MAX_NOT_CLAUSE_NUM, line);
goto error;
}
return g2c_item;
error:
group2compile_item_free(g2c_item);
return NULL;
}
static inline int compare_group_id(const void *a, const void *b)
{
long long ret = *(const long long *)a - *(const long long *)b;
if (0 == ret) {
return 0;
} else if(ret < 0) {
return -1;
} else {
return 1;
}
}
static void compile_clause_add_literal(struct compile_clause *clause,
struct group2compile_item *g2c_item)
{
struct clause_literal tmp_literal;
tmp_literal.vtable_id = g2c_item->vtable_id;
tmp_literal.group_cnt = utarray_len(g2c_item->group_ids);
utarray_sort(g2c_item->group_ids, compare_group_id);
for (size_t i = 0; i < utarray_len(g2c_item->group_ids); i++) {
tmp_literal.group_ids[i] = *(long long *)utarray_eltptr(g2c_item->group_ids, i);
}
utarray_push_back(clause->literals, &tmp_literal);
}
void compile_clause_remove_literal(struct compile_clause *clause,
struct group2compile_item *g2c_item)
{
struct clause_literal *tmp_literal = NULL;
for (size_t i = 0; i < utarray_len(clause->literals); i++) {
tmp_literal = (struct clause_literal *)utarray_eltptr(clause->literals, i);
if (tmp_literal->vtable_id == g2c_item->vtable_id) {
size_t remove_idx = utarray_eltidx(clause->literals, tmp_literal);
utarray_erase(clause->literals, remove_idx, 1);
break;
}
}
}
static int maat_compile_clause_find_literal(struct maat_compile *compile,
struct group2compile_item *g2c_item)
{
struct compile_clause *clause = compile->clauses + g2c_item->clause_index;
struct clause_literal *tmp_literal = NULL;
int found = 0;
for (size_t i = 0; i < utarray_len(clause->literals); i++) {
tmp_literal = (struct clause_literal *)utarray_eltptr(clause->literals, i);
if (tmp_literal->vtable_id == g2c_item->vtable_id) {
found = 1;
}
}
return found;
}
static void maat_compile_clause_add_literal(struct maat_compile *compile,
struct group2compile_item *g2c_item)
{
struct compile_clause *clause = compile->clauses + g2c_item->clause_index;
clause->not_flag = g2c_item->not_flag;
if (0 == clause->in_use) {
clause->in_use = 1;
compile->actual_clause_num++;
}
compile_clause_add_literal(clause, g2c_item);
}
static void maat_compile_clause_remove_literal(struct maat_compile *compile,
struct group2compile_item *g2c_item)
{
struct compile_clause *clause = compile->clauses + g2c_item->clause_index;
compile_clause_remove_literal(clause, g2c_item);
if (0 == utarray_len(clause->literals)) {
clause->in_use = 0;
compile->actual_clause_num--;
}
}
static struct bool_matcher *
maat_compile_bool_matcher_new(struct compile_runtime *compile_rt, size_t *compile_cnt)
{
if (NULL == compile_rt) {
return NULL;
}
size_t i = 0, j = 0, idx = 0;
int has_clause_num = 0;
// STEP 1, update clause_id of each compile
void **data_array = NULL;
struct maat_compile *iter_compile = NULL;
size_t rule_cnt = rcu_updating_hash_list(compile_rt->cfg_hash, &data_array);
*compile_cnt = rule_cnt;
for (idx = 0; idx < rule_cnt; idx++) {
iter_compile = (struct maat_compile *)data_array[idx];
has_clause_num = 0;
for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
struct compile_clause *clause = iter_compile->clauses + i;
if (!clause->in_use) {
continue;
}
has_clause_num++;
if (0 == clause->clause_id) {
clause->clause_id = maat_runtime_get_sequence(compile_rt->ref_maat_rt, "clause_id");
}
}
assert(has_clause_num == iter_compile->actual_clause_num);
}
// STEP 2, serial compile clause states to a bool expression array
size_t expr_cnt = 0;
struct bool_expr *bool_expr_array = ALLOC(struct bool_expr, rule_cnt);
for (idx = 0; idx < rule_cnt; idx++) {
iter_compile = (struct maat_compile *)data_array[idx];
for (i = 0, j = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
if (iter_compile->clauses[i].in_use) {
// TODO:mytest need to delete
#if 0
struct clause_literal *tmp_cl = NULL;
for(tmp_cl = (struct clause_literal *)utarray_front(iter_compile->clauses[i].literals); tmp_cl !=NULL;
tmp_cl = (struct clause_literal *)utarray_next(iter_compile->clauses[i].literals, tmp_cl)) {
for (size_t it = 0; it < tmp_cl->group_cnt; it++) {
printf("<before bool_matcher_new> compile_rt:%p compile_id:%lld, clause_id:%llu, clause_query_key{%lld: %d, %d}\n",
compile_rt, iter_compile->compile_id, iter_compile->clauses[i].clause_id, tmp_cl->group_ids[it],
tmp_cl->vtable_id, iter_compile->clauses[i].not_flag);
}
}
#endif
bool_expr_array[expr_cnt].items[j].item_id = iter_compile->clauses[i].clause_id;
bool_expr_array[expr_cnt].items[j].not_flag = 0;
j++;
}
}
// some compile may have zero groups, e.g. default policy.
if (j == (size_t)iter_compile->declared_clause_num && j > 0) {
bool_expr_array[expr_cnt].expr_id = iter_compile->compile_id;
bool_expr_array[expr_cnt].user_tag = iter_compile;
bool_expr_array[expr_cnt].item_num = j;
expr_cnt++;
}
}
FREE(data_array);
// STEP 3, build bool matcher
size_t mem_size = 0;
if (0 == expr_cnt) {
log_fatal(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] No bool expression to build bool matcher.",
__FUNCTION__, __LINE__);
FREE(bool_expr_array);
return NULL;
}
struct bool_matcher *bm = bool_matcher_new(bool_expr_array, expr_cnt, &mem_size);
if (bm != NULL) {
log_info(compile_rt->logger, MODULE_COMPILE,
"Build bool matcher of %zu expressions with %zu bytes memory.",
expr_cnt, mem_size);
} else {
log_fatal(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] Build bool matcher failed!", __FUNCTION__, __LINE__);
}
FREE(bool_expr_array);
return bm;
}
static inline int compare_clause_id(const void *a, const void *b)
{
long long ret = *(const long long *)a - *(const long long *)b;
if (0 == ret) {
return 0;
} else if(ret < 0) {
return -1;
} else {
return 1;
}
}
static inline int compare_hit_group(const void *pa, const void *pb)
{
struct maat_hit_group *la=(struct maat_hit_group *)pa;
struct maat_hit_group *lb=(struct maat_hit_group *)pb;
long long ret = la->item_id - lb->item_id;
if (0 == ret) {
ret = la->group_id - lb->group_id;
if (0 == ret) {
ret = la->vtable_id - lb->vtable_id;
}
}
return ret;
}
static inline int compare_compile_id(const void *a, const void *b)
{
long long ret = *(const long long *)a - *(const long long *)b;
if (0 == ret) {
return 0;
} else if (ret < 0) {
return -1;
} else {
return 1;
}
}
/**
* @brief build <clause_query_key, clause_id_array> hash for clause or not_clause
*
* @param compile_rt: compile runtime handle
* @param not_flag: specify whether to build clause or NOT_clause hash for compile runtime
* 0 -> clause hash
* 1 -> NOT_clause hash
*
* @retval generated clause_id_kv_hash
*/
static struct clause_id_kv *
build_clause_id_kv_hash(struct compile_runtime *compile_rt, int not_flag)
{
if (NULL == compile_rt) {
return NULL;
}
void **data_array = NULL;
struct clause_id_kv *clause_id_kv_hash = NULL;
size_t compile_cnt = rcu_updating_hash_list(compile_rt->cfg_hash, &data_array);
for (size_t idx = 0; idx < compile_cnt; idx++) {
struct maat_compile *compile = (struct maat_compile *)data_array[idx];
for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
struct compile_clause *clause = compile->clauses + i;
if (!clause->in_use) {
continue;
}
if (0 == not_flag) {
if (CLAUSE_NOT_FLAG_SET == clause->not_flag) {
continue;
}
} else {
if (CLAUSE_NOT_FLAG_UNSET == clause->not_flag) {
continue;
}
}
struct clause_literal *tmp_cl = NULL;
for (size_t j = 0; j < utarray_len(clause->literals); j++) {
tmp_cl = (struct clause_literal *)utarray_eltptr(clause->literals, j);
for (size_t k = 0; k < tmp_cl->group_cnt; k++) {
struct clause_query_key key = {tmp_cl->group_ids[k], tmp_cl->vtable_id, clause->not_flag};
struct clause_id_kv *clause_id_kv = NULL;
HASH_FIND(hh, clause_id_kv_hash, &key, sizeof(struct clause_query_key), clause_id_kv);
if (NULL == clause_id_kv) {
clause_id_kv = ALLOC(struct clause_id_kv, 1);
clause_id_kv->key = key;
utarray_new(clause_id_kv->clause_ids, &ut_clause_id_icd);
HASH_ADD_KEYPTR(hh, clause_id_kv_hash, &clause_id_kv->key, sizeof(clause_id_kv->key), clause_id_kv);
}
if (utarray_find(clause_id_kv->clause_ids, &(clause->clause_id), compare_clause_id)) {
continue;
}
utarray_push_back(clause_id_kv->clause_ids, &(clause->clause_id));
utarray_sort(clause_id_kv->clause_ids, compare_clause_id);
}
}
}
}
FREE(data_array);
return clause_id_kv_hash;
}
static int maat_compile_has_clause(struct maat_compile *compile, long long clause_id)
{
struct compile_clause *clause = NULL;
for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
clause = compile->clauses + i;
if (!clause->in_use) {
continue;
}
if (clause->clause_id == clause_id) {
return 1;
}
}
return 0;
}
static size_t compile_state_if_new_hit_compile(struct compile_state *compile_state,
struct maat_compile *compile)
{
size_t i = 0;
size_t r_in_c_cnt = 0;
int ret = 0;
long long new_hit_clause_id = 0;
if (0 == compile_state->this_scan_not_logic) {
for (i = 0; i < utarray_len(compile_state->this_scan_hit_clauses); i++) {
new_hit_clause_id = *(long long*)utarray_eltptr(compile_state->this_scan_hit_clauses, i);
ret = maat_compile_has_clause(compile, new_hit_clause_id);
if (ret) {
r_in_c_cnt++;
}
}
} else {
for (i = 0; i < utarray_len(compile_state->this_scan_hit_not_clauses); i++) {
new_hit_clause_id = *(long long*)utarray_eltptr(compile_state->this_scan_hit_not_clauses, i);
ret = maat_compile_has_clause(compile, new_hit_clause_id);
if (ret) {
r_in_c_cnt++;
}
}
}
return r_in_c_cnt;
}
static void compile_state_update_hit_compile_table_id(struct compile_state *compile_state,
long long compile_id, int table_id)
{
if (!utarray_find(compile_state->hit_compile_table_ids, &compile_id, compare_compile_id)) {
struct compile2table_id compile_table_id = {compile_id, table_id};
utarray_push_back(compile_state->hit_compile_table_ids, &compile_table_id);
utarray_sort(compile_state->hit_compile_table_ids, compare_compile_id);
}
}
static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt,
struct compile_state *compile_state,
int thread_id, void **user_data_array,
size_t ud_array_size)
{
size_t ud_result_cnt = 0;
struct maat_compile *compile = NULL;
struct bool_expr_match *expr_match = compile_rt->expr_match_buff +
(thread_id * MAX_HIT_COMPILE_NUM);
assert(thread_id >= 0);
if (0 == compile_state->compile_rt_version) {
compile_state->compile_rt_version = compile_rt->version;
}
if (NULL == compile_rt->bm || 0 == utarray_len(compile_state->all_hit_clauses)
|| compile_state->compile_rt_version != compile_rt->version) {
return 0;
}
//TODO:mytest need to delete
#if 0
unsigned long long *p = utarray_eltptr(compile_state->all_hit_clauses, 0);
for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clauses); p != NULL;
p = (unsigned long long *)utarray_next(compile_state->all_hit_clauses, p)) {
printf("before bool_matcher_match compile_rt:%p compile_state clause_id:%llu\n", compile_rt, *p);
}
#endif
int bool_match_ret = bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses),
expr_match, MAX_HIT_COMPILE_NUM);
for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) {
compile = (struct maat_compile *)expr_match[i].user_tag;
assert(compile->magic_num == MAAT_COMPILE_MAGIC);
assert((unsigned long long)compile->compile_id == expr_match[i].expr_id);
if (0 == compile->actual_clause_num) {
continue;
}
size_t n_new_hit_compile = compile_state_if_new_hit_compile(compile_state, compile);
if (compile->user_data != NULL && n_new_hit_compile > 0) {
user_data_array[ud_result_cnt] = compile->user_data;
ud_result_cnt++;
compile_state_update_hit_compile_table_id(compile_state, compile->compile_id,
compile->table_id);
}
}
return ud_result_cnt;
}
static struct compile_item *compile_item_clone(struct compile_item *item)
{
struct compile_item *new_item = ALLOC(struct compile_item, 1);
new_item->compile_id = item->compile_id;
new_item->declared_clause_num = item->declared_clause_num;
new_item->table_line_len = item->table_line_len;
new_item->table_line = ALLOC(char, new_item->table_line_len + 1);
memcpy(new_item->table_line, item->table_line, new_item->table_line_len);
return new_item;
}
static struct maat_compile *
maat_compile_clone(struct maat_compile *compile, int deep_copy)
{
struct maat_compile *new_compile = ALLOC(struct maat_compile, 1);
new_compile->magic_num = compile->magic_num;
new_compile->compile_id = compile->compile_id;
new_compile->actual_clause_num = compile->actual_clause_num;
new_compile->declared_clause_num = compile->declared_clause_num;
if (1 == deep_copy && compile->user_data != NULL) {
new_compile->user_data = compile_item_clone((struct compile_item *)compile->user_data);
}
struct clause_literal *tmp_literal = NULL;
for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
new_compile->clauses[i].clause_id = compile->clauses[i].clause_id;
new_compile->clauses[i].in_use = compile->clauses[i].in_use;
new_compile->clauses[i].not_flag = compile->clauses[i].not_flag;
utarray_new(new_compile->clauses[i].literals, &ut_clause_literal_icd);
for (size_t j = 0; j < utarray_len(compile->clauses[i].literals); j++) {
tmp_literal = (struct clause_literal *)utarray_eltptr(compile->clauses[i].literals, j);
utarray_push_back(new_compile->clauses[i].literals, tmp_literal);
}
}
return new_compile;
}
static int maat_add_group_to_compile(struct rcu_hash_table *hash_tbl,
struct group2compile_item *g2c_item,
struct log_handle *logger)
{
int ret = 0;
long long compile_id = g2c_item->compile_id;
struct maat_compile *compile = NULL;
int updating_flag = rcu_hash_is_updating(hash_tbl);
if (1 == updating_flag) {
compile = rcu_updating_hash_find(hash_tbl, (char *)&compile_id, sizeof(long long));
if (compile != NULL) {
ret = maat_compile_clause_find_literal(compile, g2c_item);
if (ret > 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]compile:%lld clause(index:%d) already has vtable_id:%d's "
"literal, can't add again", __FUNCTION__, __LINE__, compile->compile_id,
g2c_item->clause_index, g2c_item->vtable_id);
return -1;
}
/* compile found in updating hash(added by compile runtime), it can
* be modified directly */
maat_compile_clause_add_literal(compile, g2c_item);
} else {
/* compile neither in effective hash nor in updating hash, so new one */
compile = maat_compile_new(compile_id);
assert(compile != NULL);
maat_compile_clause_add_literal(compile, g2c_item);
rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), compile);
}
} else {
compile = rcu_hash_find(hash_tbl, (char *)&compile_id, sizeof(long long));
if (compile != NULL) {
/*******************************************************************
compile found in effective hash(added by compile runtime), which means
1. rcu_hash_add(hash_tbl, compile) ==> finished
2. rcu_hash_commit(hash_tbl) ==> finished
can only be deleted but not modified
before delete it, we need to make a copy for further use
*********************************************************************/
ret = maat_compile_clause_find_literal(compile, g2c_item);
if (ret > 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]compile:%lld clause(index:%d) already has vtable_id:%d's "
"literal, can't add again", __FUNCTION__, __LINE__, compile->compile_id,
g2c_item->clause_index, g2c_item->vtable_id);
return -1;
}
struct maat_compile *copy_compile = maat_compile_clone(compile, 1);
assert(copy_compile != NULL);
/* delete compile from rcu hash */
rcu_hash_del(hash_tbl, (char *)&compile_id, sizeof(long long));
maat_compile_clause_add_literal(copy_compile, g2c_item);
rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), copy_compile);
} else {
compile = maat_compile_new(compile_id);
assert(compile != NULL);
maat_compile_clause_add_literal(compile, g2c_item);
rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), compile);
}
}
return 0;
}
static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
struct group2compile_item *g2c_item,
struct log_handle *logger)
{
int ret = 0;
long long compile_id = g2c_item->compile_id;
struct maat_compile *compile = NULL;
int updating_flag = rcu_hash_is_updating(hash_tbl);
if (1 == updating_flag) {
compile = rcu_updating_hash_find(hash_tbl, (char *)&compile_id,
sizeof(long long));
if (NULL == compile) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] Remove clause(index:%d) from compile %lld failed,"
"compile not existed.", __FUNCTION__, __LINE__,
g2c_item->clause_index, compile_id);
return -1;
} else {
ret = maat_compile_clause_find_literal(compile, g2c_item);
if (0 == ret) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]compile:%lld clause(index:%d) has no vtable_id:%d's "
"literal, can't be removed", __FUNCTION__, __LINE__,
compile->compile_id, g2c_item->clause_index, g2c_item->vtable_id);
return -1;
}
/* compile found in updating hash, it can be modified directly */
maat_compile_clause_remove_literal(compile, g2c_item);
if (0 == compile->actual_clause_num && NULL == compile->user_data) {
rcu_hash_del(hash_tbl, (char *)&compile_id, sizeof(long long));
}
}
} else {
//find in effetive hash
compile = rcu_hash_find(hash_tbl, (char *)&compile_id, sizeof(long long));
if (compile != NULL) {
ret = maat_compile_clause_find_literal(compile, g2c_item);
if (0 == ret) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]compile:%lld clause(index:%d) has no vtable_id:%d's "
"literal, can't be removed", __FUNCTION__, __LINE__,
compile->compile_id, g2c_item->clause_index, g2c_item->vtable_id);
return -1;
}
/*******************************************************************
compile found in effective hash, which means
1. rcu_hash_add(hash_tbl, compile) ==> finished
2. rcu_hash_commit(hash_tbl) ==> finished
can only be deleted but not modified
before delete it, we need to make a copy for further use
*********************************************************************/
struct maat_compile *copy_compile = maat_compile_clone(compile, 1);
assert(copy_compile != NULL);
/* delete compile from rcu hash */
rcu_hash_del(hash_tbl, (char *)&compile_id, sizeof(long long));
maat_compile_clause_remove_literal(copy_compile, g2c_item);
if (0 == copy_compile->actual_clause_num && NULL == copy_compile->user_data) {
maat_compile_free(copy_compile);
} else {
rcu_hash_add(hash_tbl, (char *)&compile_id, sizeof(long long), copy_compile);
}
} else {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d] Remove clause(index:%d) from compile_id %lld failed,"
"compile is not existed.", __FUNCTION__, __LINE__,
g2c_item->clause_index, compile_id);
return -1;
}
}
return 0;
}
struct compile_state *compile_state_new(void)
{
struct compile_state *compile_state = ALLOC(struct compile_state, 1);
utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd);
utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->this_scan_hit_not_clauses, &ut_clause_id_icd);
utarray_new(compile_state->exclude_not_clauses, &ut_clause_id_icd);
utarray_new(compile_state->direct_hit_groups, &ut_maat_hit_group_icd);
utarray_new(compile_state->indirect_hit_groups, &ut_maat_hit_group_icd);
utarray_new(compile_state->last_hit_group_ids, &ut_compile_group_id_icd);
utarray_new(compile_state->hit_compile_table_ids, &ut_hit_compile_table_id_icd);
compile_state->hit_not_tbl_groups = NULL;
return compile_state;
}
static long long compile_state_hit_not_tbl_groups_free(struct compile_state *compile_state)
{
if (NULL == compile_state) {
return 0;
}
long long free_bytes = 0;
struct table_group *tbl_group = NULL, *tmp_tbl_group = NULL;
HASH_ITER(hh, compile_state->hit_not_tbl_groups, tbl_group, tmp_tbl_group) {
free_bytes += (sizeof(tbl_group) + utarray_len(tbl_group->group_ids) * sizeof(long long));
HASH_DEL(compile_state->hit_not_tbl_groups, tbl_group);
if (tbl_group->group_ids != NULL) {
utarray_free(tbl_group->group_ids);
tbl_group->group_ids = NULL;
}
FREE(tbl_group);
}
return free_bytes;
}
void compile_state_reset(struct compile_state *compile_state)
{
if (NULL == compile_state) {
return;
}
compile_state->this_scan_not_logic = 0;
compile_state->Nth_scan = 0;
compile_state->compile_rt_version = 0;
utarray_clear(compile_state->internal_hit_paths);
utarray_clear(compile_state->all_hit_clauses);
utarray_clear(compile_state->this_scan_hit_clauses);
utarray_clear(compile_state->this_scan_hit_not_clauses);
utarray_clear(compile_state->exclude_not_clauses);
utarray_clear(compile_state->direct_hit_groups);
utarray_clear(compile_state->indirect_hit_groups);
utarray_clear(compile_state->last_hit_group_ids);
utarray_clear(compile_state->hit_compile_table_ids);
struct table_group *tbl_group = NULL, *tmp_tbl_group = NULL;
HASH_ITER(hh, compile_state->hit_not_tbl_groups, tbl_group, tmp_tbl_group) {
utarray_clear(tbl_group->group_ids);
}
}
void compile_state_free(struct compile_state *compile_state,
struct maat *maat_inst, int thread_id)
{
if (NULL == compile_state) {
return;
}
long long free_bytes = 0;
if (compile_state->internal_hit_paths != NULL) {
free_bytes += utarray_size(compile_state->internal_hit_paths) *
sizeof(struct internal_hit_path);
utarray_free(compile_state->internal_hit_paths);
compile_state->internal_hit_paths = NULL;
}
if (compile_state->all_hit_clauses != NULL) {
free_bytes += utarray_size(compile_state->all_hit_clauses) * sizeof(long long);
utarray_free(compile_state->all_hit_clauses);
compile_state->all_hit_clauses = NULL;
}
if (compile_state->this_scan_hit_clauses != NULL) {
free_bytes += utarray_size(compile_state->this_scan_hit_clauses) * sizeof(long long);
utarray_free(compile_state->this_scan_hit_clauses);
compile_state->this_scan_hit_clauses = NULL;
}
if (compile_state->this_scan_hit_not_clauses != NULL) {
free_bytes += utarray_size(compile_state->this_scan_hit_not_clauses) * sizeof(long long);
utarray_free(compile_state->this_scan_hit_not_clauses);
compile_state->this_scan_hit_not_clauses = NULL;
}
if (compile_state->exclude_not_clauses != NULL) {
free_bytes += utarray_size(compile_state->exclude_not_clauses) * sizeof(long long);
utarray_free(compile_state->exclude_not_clauses);
compile_state->exclude_not_clauses = NULL;
}
if (compile_state->direct_hit_groups != NULL) {
free_bytes += utarray_size(compile_state->direct_hit_groups) * sizeof(struct maat_hit_group);
utarray_free(compile_state->direct_hit_groups);
compile_state->direct_hit_groups = NULL;
}
if (compile_state->indirect_hit_groups != NULL) {
free_bytes += utarray_size(compile_state->indirect_hit_groups) * sizeof(struct maat_hit_group);
utarray_free(compile_state->indirect_hit_groups);
compile_state->indirect_hit_groups = NULL;
}
if (compile_state->last_hit_group_ids != NULL) {
free_bytes += utarray_size(compile_state->last_hit_group_ids) * sizeof(long long);
utarray_free(compile_state->last_hit_group_ids);
compile_state->last_hit_group_ids = NULL;
}
if (compile_state->hit_compile_table_ids != NULL) {
free_bytes += utarray_size(compile_state->hit_compile_table_ids) * sizeof(struct compile2table_id);
utarray_free(compile_state->hit_compile_table_ids);
compile_state->hit_compile_table_ids = NULL;
}
free_bytes += compile_state_hit_not_tbl_groups_free(compile_state);
FREE(compile_state);
free_bytes += sizeof(struct compile_state);
alignment_int64_array_add(maat_inst->stat->maat_state_free_bytes,
thread_id, free_bytes);
}
static void compile_state_add_internal_hit_path(struct compile_state *compile_state,
long long item_id, long long group_id,
int vtable_id, int NOT_flag, int Nth_scan)
{
if (NULL == compile_state) {
return;
}
struct internal_hit_path new_path;
new_path.item_id = item_id;
new_path.Nth_scan = Nth_scan;
new_path.group_id = group_id;
new_path.vtable_id = vtable_id;
new_path.NOT_flag = NOT_flag;
utarray_push_back(compile_state->internal_hit_paths, &new_path);
}
static int maat_compile_has_clause_query_key(struct maat_compile *compile,
struct clause_query_key *key)
{
for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
struct compile_clause *clause = compile->clauses+i;
if(!clause->in_use) {
continue;
}
struct clause_literal *tmp_cl = NULL;
for (size_t j = 0; j < utarray_len(clause->literals); j++) {
tmp_cl = (struct clause_literal *)utarray_eltptr(clause->literals, j);
if (tmp_cl->vtable_id != key->vtable_id) {
continue;
}
if (clause->not_flag != key->not_flag) {
continue;
}
long long *tmp_group_id = bsearch(&(key->group_id), tmp_cl->group_ids,
tmp_cl->group_cnt, sizeof(long long),
compare_group_id);
if (tmp_group_id != NULL) {
return 1;
}
}
}
return 0;
}
static size_t maat_compile_get_hit_clause_index(struct maat_compile *compile,
int vtable_id, long long hit_group_id,
int *clause_idx_array, size_t array_size)
{
size_t hit_clause_cnt = 0;
struct compile_clause *tmp_clause = NULL;
for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
tmp_clause = &compile->clauses[i];
if (!tmp_clause->in_use) {
continue;
}
struct clause_literal *tmp_cl = NULL;
for (size_t j = 0; j < utarray_len(tmp_clause->literals); j++) {
tmp_cl = (struct clause_literal *)utarray_eltptr(tmp_clause->literals, j);
if (tmp_cl->vtable_id != vtable_id) {
continue;
}
long long *tmp_group_id = bsearch(&hit_group_id, tmp_cl->group_ids,
tmp_cl->group_cnt, sizeof(long long),
compare_group_id);
if (tmp_group_id != NULL) {
clause_idx_array[hit_clause_cnt++] = i;
break;
}
}
}
return hit_clause_cnt;
}
static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths,
size_t n_path, const struct maat_hit_path *find)
{
for (size_t i = 0; i < n_path; i++) {
if (0 == memcmp(hit_paths + i, find, sizeof(*find))) {
return 1;
}
}
return 0;
}
void populate_hit_path_with_compile(struct maat_hit_path *hit_path_array,
size_t array_idx, size_t n_hit_path,
size_t *n_new_hit_path, struct maat_compile *compile)
{
size_t i = 0;
size_t idx = array_idx;
size_t n_clause_index = 0;
size_t new_hit_path_cnt = *n_new_hit_path;
int clause_index_array[MAX_ITEMS_PER_BOOL_EXPR] = {0};
if (hit_path_array[idx].top_group_id < 0) {
hit_path_array[idx].top_group_id = hit_path_array[idx].sub_group_id;
}
struct maat_hit_path tmp_path;
if (hit_path_array[idx].compile_id < 0) {
hit_path_array[idx].compile_id = compile->compile_id;
// find out which clause in compile hit
n_clause_index = maat_compile_get_hit_clause_index(compile, hit_path_array[idx].vtable_id,
hit_path_array[idx].top_group_id, clause_index_array,
MAX_ITEMS_PER_BOOL_EXPR);
hit_path_array[idx].clause_index = clause_index_array[0];
if (n_clause_index > 1) {
for (i = 1; i < n_clause_index; i++) {
tmp_path = hit_path_array[idx];
tmp_path.clause_index = clause_index_array[i];
hit_path_array[n_hit_path + new_hit_path_cnt] = tmp_path;
new_hit_path_cnt++;
}
}
} else {
// means same clause_query_id hit more than one compile_id
tmp_path = hit_path_array[idx];
tmp_path.compile_id = compile->compile_id;
if (!maat_compile_is_hit_path_existed(hit_path_array, n_hit_path + new_hit_path_cnt, &tmp_path)) {
hit_path_array[n_hit_path + new_hit_path_cnt] = tmp_path;
new_hit_path_cnt++;
n_clause_index = maat_compile_get_hit_clause_index(compile, tmp_path.vtable_id, tmp_path.top_group_id,
clause_index_array, MAX_ITEMS_PER_BOOL_EXPR);
hit_path_array[n_hit_path + new_hit_path_cnt - 1].clause_index = clause_index_array[0];
if (n_clause_index > 1) {
for (i = 1; i < n_clause_index; i++) {
tmp_path = hit_path_array[n_hit_path + new_hit_path_cnt - 1];
tmp_path.clause_index = clause_index_array[i];
hit_path_array[n_hit_path + new_hit_path_cnt] = tmp_path;
new_hit_path_cnt++;
}
}
}
}
*n_new_hit_path = new_hit_path_cnt;
}
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thread_id,
struct compile_state *compile_state,
struct maat_hit_path *hit_path_array,
size_t array_size, size_t n_hit_path)
{
/* assign hit_path_array[].compile_id */
size_t n_new_hit_path = 0;
struct maat_compile *compile = NULL;
struct clause_query_key key = {0, 0, 0};
struct bool_expr_match *expr_match = compile_rt->expr_match_buff +
(thread_id * MAX_HIT_COMPILE_NUM);
assert(thread_id >= 0);
if (compile_state->compile_rt_version != compile_rt->version) {
return 0;
}
int bool_match_ret = bool_matcher_match(compile_rt->bm,
(unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
utarray_len(compile_state->all_hit_clauses), expr_match, MAX_HIT_COMPILE_NUM);
for (int idx = 0; idx < bool_match_ret; idx++) {
compile = (struct maat_compile *)expr_match[idx].user_tag;
assert(compile->magic_num == MAAT_COMPILE_MAGIC);
assert((unsigned long long)compile->compile_id == expr_match[idx].expr_id);
if (0 == compile->actual_clause_num || NULL == compile->user_data) {
continue;
}
for (size_t j = 0; j < n_hit_path && (n_hit_path + n_new_hit_path) < array_size; j++) {
if (hit_path_array[j].top_group_id < 0) {
key.group_id = hit_path_array[j].sub_group_id;
} else {
key.group_id = hit_path_array[j].top_group_id;
}
key.vtable_id = hit_path_array[j].vtable_id;
key.not_flag = hit_path_array[j].NOT_flag;
if (maat_compile_has_clause_query_key(compile, &key)) {
populate_hit_path_with_compile(hit_path_array, j, n_hit_path, &n_new_hit_path, compile);
}
}
}
return (n_hit_path + n_new_hit_path);
}
static void compile_state_add_direct_hit_groups(struct compile_state *compile_state,
struct maat_item *hit_items,
size_t n_hit_items, int vtable_id)
{
if (NULL == compile_state || NULL == hit_items) {
return;
}
struct maat_hit_group hit_group;
for (size_t i = 0; i < n_hit_items; i++) {
hit_group.item_id = hit_items[i].item_id;
hit_group.group_id = hit_items[i].group_id;
hit_group.vtable_id = vtable_id;
utarray_push_back(compile_state->direct_hit_groups, &hit_group);
}
}
static void compile_state_add_indirect_hit_groups(struct compile_state *compile_state,
long long *group_ids,
size_t n_group_ids, int vtable_id)
{
if (NULL == compile_state || NULL == group_ids) {
return;
}
struct maat_hit_group hit_group;
for (size_t i = 0; i < n_group_ids; i++) {
hit_group.item_id = 0;
hit_group.group_id = group_ids[i];
hit_group.vtable_id = vtable_id;
utarray_push_back(compile_state->indirect_hit_groups, &hit_group);
}
}
static void compile_state_add_hit_clauses(struct compile_state *compile_state,
UT_array *clause_id_array)
{
size_t i = 0;
long long *clause_id = NULL;
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
for (i = 0; i < utarray_len(clause_id_array); i++) {
clause_id = (long long *)utarray_eltptr(clause_id_array, i);
if (utarray_find(compile_state->all_hit_clauses, clause_id, compare_clause_id)) {
continue;
}
utarray_push_back(compile_state->this_scan_hit_clauses, clause_id);
}
if ((utarray_len(compile_state->this_scan_hit_clauses) - new_clause_idx) > 0) {
utarray_reserve(compile_state->all_hit_clauses,
utarray_len(compile_state->this_scan_hit_clauses) - new_clause_idx);
for (i = new_clause_idx; i < utarray_len(compile_state->this_scan_hit_clauses); i++) {
clause_id = (long long *)utarray_eltptr(compile_state->this_scan_hit_clauses, i);
utarray_push_back(compile_state->all_hit_clauses, clause_id);
}
utarray_sort(compile_state->all_hit_clauses, compare_clause_id);
}
}
static void compile_state_add_exclude_not_clauses(struct compile_state *compile_state,
UT_array *clause_id_array)
{
for (size_t i = 0; i < utarray_len(clause_id_array); i++) {
long long *clause_id = (long long *)utarray_eltptr(clause_id_array, i);
if (utarray_find(compile_state->exclude_not_clauses, clause_id, compare_clause_id)) {
continue;
}
utarray_push_back(compile_state->exclude_not_clauses, clause_id);
}
utarray_sort(compile_state->exclude_not_clauses, compare_clause_id);
}
static void compile_state_add_hit_not_clauses(struct compile_state *compile_state,
UT_array *clause_id_array)
{
size_t i = 0;
long long *clause_id = NULL;
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_not_clauses);
for (i = 0; i < utarray_len(clause_id_array); i++) {
clause_id = (long long *)utarray_eltptr(clause_id_array, i);
if (utarray_find(compile_state->all_hit_clauses, clause_id, compare_clause_id)) {
continue;
}
if (utarray_find(compile_state->exclude_not_clauses, clause_id, compare_clause_id)) {
continue;
}
utarray_push_back(compile_state->this_scan_hit_not_clauses, clause_id);
}
if ((utarray_len(compile_state->this_scan_hit_not_clauses) - new_clause_idx) > 0) {
utarray_reserve(compile_state->all_hit_clauses,
utarray_len(compile_state->this_scan_hit_not_clauses) - new_clause_idx);
for (i = new_clause_idx; i < utarray_len(compile_state->this_scan_hit_not_clauses); i++) {
clause_id = (long long *)utarray_eltptr(compile_state->this_scan_hit_not_clauses, i);
utarray_push_back(compile_state->all_hit_clauses, clause_id);
}
utarray_sort(compile_state->all_hit_clauses, compare_clause_id);
}
}
static void compile_state_update_hit_clauses(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
long long group_id, int vtable_id)
{
if (NULL == compile_state || NULL == compile_rt) {
return;
}
struct clause_query_key key = {group_id, vtable_id, 0};
struct clause_id_kv *clause_id_kv = NULL;
HASH_FIND(hh, compile_rt->clause_id_kv_hash, &key, sizeof(key), clause_id_kv);
if (clause_id_kv != NULL) {
compile_state_add_hit_clauses(compile_state, clause_id_kv->clause_ids);
}
key.not_flag = 1;
HASH_FIND(hh, compile_rt->not_clause_id_kv_hash, &key, sizeof(key), clause_id_kv);
if (clause_id_kv != NULL) {
compile_state_add_exclude_not_clauses(compile_state, clause_id_kv->clause_ids);
}
}
static void compile_state_cache_hit_not_groups(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
long long *hit_group_ids,
size_t n_hit_group_id, int vtable_id)
{
if (NULL == compile_state || NULL == compile_rt) {
return;
}
if (n_hit_group_id != 0) {
qsort(hit_group_ids, n_hit_group_id, sizeof(long long *), compare_group_id);
}
struct table_group *tbl_group = NULL;
HASH_FIND(hh, compile_state->hit_not_tbl_groups, &vtable_id, sizeof(int), tbl_group);
if (tbl_group != NULL) {
for (size_t i = 0; i < n_hit_group_id; i++) {
long long *group_id = (long long *)utarray_find(tbl_group->group_ids,
&hit_group_ids[i],
compare_group_id);
if (NULL == group_id) {
continue;
}
size_t remove_idx = utarray_eltidx(tbl_group->group_ids, group_id);
utarray_erase(tbl_group->group_ids, remove_idx, 1);
}
}
struct clause_id_kv *clause_id_kv = NULL, *tmp_clause_id_kv = NULL;
HASH_ITER(hh, compile_rt->not_clause_id_kv_hash, clause_id_kv, tmp_clause_id_kv) {
if (clause_id_kv->key.vtable_id != vtable_id) {
continue;
}
long long *tmp_group_id = bsearch(&(clause_id_kv->key.group_id), hit_group_ids,
n_hit_group_id, sizeof(long long), compare_group_id);
if (tmp_group_id != NULL) {
continue;
}
if (NULL == tbl_group) {
tbl_group = ALLOC(struct table_group, 1);
tbl_group->vtable_id = vtable_id;
utarray_new(tbl_group->group_ids, &ut_compile_group_id_icd);
HASH_ADD_INT(compile_state->hit_not_tbl_groups, vtable_id, tbl_group);
}
if (!utarray_find(tbl_group->group_ids, &(clause_id_kv->key.group_id), compare_group_id)) {
utarray_push_back(tbl_group->group_ids, &(clause_id_kv->key.group_id));
}
}
if (tbl_group != NULL) {
utarray_sort(tbl_group->group_ids, compare_group_id);
}
}
int compile_state_get_compile_table_id(struct compile_state *compile_state,
long long compile_id)
{
struct compile2table_id *tmp = NULL;
tmp = utarray_find(compile_state->hit_compile_table_ids, &compile_id,
compare_compile_id);
if (NULL == tmp) {
return -1;
}
return tmp->table_id;
}
static int compile_runtime_add_compile(struct compile_runtime *compile_rt,
struct compile_schema *schema,
long long compile_id, const char *table_name,
const char *line, struct log_handle *logger)
{
struct maat_compile *compile = NULL;
struct compile_item *compile_item = compile_item_new(line, schema, table_name,
compile_rt->logger);
if (NULL == compile_item) {
return -1;
}
int table_id = table_manager_get_table_id(schema->ref_tbl_mgr, table_name);
if (table_id < 0) {
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]table_name:%s has invalid table_id:%d, drop line:%s", __FUNCTION__,
__LINE__, table_name, table_id, line);
return -1;
}
int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash);
if (1 == updating_flag) {
compile = rcu_updating_hash_find(compile_rt->cfg_hash, (char *)&compile_id,
sizeof(long long));
if (compile != NULL) {
/****************************************************************
compile found in updating hash(added by group2compile runtime), which means
1. rcu_hash_add(htable, compile) ==> finished
2. rcu_hash_commit(htable) ==> undo
because it's in updating hash, we can modify it directly
******************************************************************/
/* compile has group2compile_table info, so set compile_table info */
maat_compile_set(compile, table_id, compile_item->declared_clause_num, compile_item);
} else {
// compile neither in effective hash nor in updating hash
compile = maat_compile_new(compile_item->compile_id);
assert(compile != NULL);
maat_compile_set(compile, table_id, compile_item->declared_clause_num, compile_item);
rcu_hash_add(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long), compile);
}
} else {
compile = rcu_hash_find(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
if (compile != NULL) {
/********************************************************************************
compile found in effective hash(added by group2compile runtime), which means
1. rcu_hash_add(htable, compile) ==> finished
2. rcu_hash_commit(htable) ==> finished
can only be deleted but not modified
before delete it, we need to make a copy for further use
***********************************************************************************/
struct maat_compile *copy_compile = maat_compile_clone(compile, 0);
assert(copy_compile != NULL);
/* delete compile from rcu hash */
rcu_hash_del(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
/* copy_compile has group2compile_table info, so set compile_table info */
maat_compile_set(copy_compile, table_id, compile_item->declared_clause_num, compile_item);
/* add copy_compile to rcu hash */
rcu_hash_add(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long), copy_compile);
} else {
compile = maat_compile_new(compile_item->compile_id);
assert(compile != NULL);
maat_compile_set(compile, table_id, compile_item->declared_clause_num, compile_item);
rcu_hash_add(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long), compile);
}
}
return 0;
}
static void garbage_compile_item_free(void *data, void *arg)
{
if (NULL == data) {
return;
}
struct compile_item *compile_item = (struct compile_item *)data;
compile_item_free(compile_item);
}
static void compile_runtime_del_compile(struct compile_runtime *compile_rt,
long long compile_id)
{
struct maat_compile *compile = NULL;
int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash);
if (1 == updating_flag) {
// find in updating hash
compile = rcu_updating_hash_find(compile_rt->cfg_hash, (char *)&compile_id,
sizeof(long long));
if (compile != NULL) {
/****************************************************************
compile found in updating hash, which means
1. rcu_hash_del(htable, compile) ==> finished
2. rcu_hash_commit(htable) ==> undo
because it's in updating hash, we can modify it directly
******************************************************************/
if (compile->user_data != NULL) {
maat_garbage_bagging(compile_rt->ref_garbage_bin, compile->user_data, NULL,
garbage_compile_item_free);
compile->user_data = NULL;
}
if (0 == compile->actual_clause_num) {
rcu_hash_del(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
}
}
} else {
// find in effective hash
compile = rcu_hash_find(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
if (compile != NULL) {
/*******************************************************************
compile found in effective hash, which means
1. rcu_hash_add(htable, compile) ==> finished
2. rcu_hash_commit(htable) ==> finished
can only be deleted but not modified
before delete it, we need to make a copy for further use
*********************************************************************/
struct maat_compile *copy_compile = maat_compile_clone(compile, 0);
assert(copy_compile != NULL);
/* delete compile from rcu hash */
rcu_hash_del(compile_rt->cfg_hash, (char *)&compile_id, sizeof(long long));
if (0 == copy_compile->actual_clause_num) {
maat_compile_free(copy_compile);
} else {
rcu_hash_add(compile_rt->cfg_hash, (char *)&compile_id,
sizeof(long long), copy_compile);
}
}
}
}
int compile_runtime_update(void *compile_runtime, void *compile_schema,
const char *table_name, const char *line,
int valid_column)
{
if (NULL == compile_runtime || NULL == compile_schema || NULL == line) {
return -1;
}
struct compile_schema *schema = (struct compile_schema *)compile_schema;
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_fatal(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] compile table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
compile_rt->update_err_cnt++;
return -1;
}
long long compile_id = get_column_value(line, schema->compile_id_column);
if (compile_id < 0) {
log_fatal(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] compile table:<%s> has no compile_id(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
schema->compile_id_column, line);
compile_rt->update_err_cnt++;
return -1;
}
if (0 == is_valid) {
// delete
compile_runtime_del_compile(compile_rt, compile_id);
} else {
// add
int ret = compile_runtime_add_compile(compile_rt, schema, compile_id,
table_name, line, compile_rt->logger);
if (ret < 0) {
compile_rt->update_err_cnt++;
}
}
return 0;
}
static int validate_table_not_clause(struct group2compile_runtime *g2c_rt,
struct table_manager *tbl_mgr, int table_id,
int is_valid, struct log_handle *logger)
{
struct table_clause *not_clause = NULL;
HASH_FIND_INT(g2c_rt->tbl_not_clause_hash, &table_id, not_clause);
if (0 == is_valid) {
//delete
if (NULL == not_clause || 0 == not_clause->actual_clause_num) {
return 0;
} else {
not_clause->actual_clause_num--;
}
} else {
//add
if (NULL == not_clause) {
not_clause = ALLOC(struct table_clause, 1);
not_clause->vtable_id = table_id;
not_clause->actual_clause_num++;
HASH_ADD_INT(g2c_rt->tbl_not_clause_hash, vtable_id, not_clause);
} else {
if (not_clause->actual_clause_num >= MAX_NOT_CLAUSE_NUM) {
const char *table_name = table_manager_get_table_name(tbl_mgr, table_id);
log_fatal(logger, MODULE_COMPILE,
"[%s:%d]table:<%s> NOT clause num exceed maximum:%d",
__FUNCTION__, __LINE__, table_name, MAX_NOT_CLAUSE_NUM);
return -1;
}
not_clause->actual_clause_num++;
}
}
return 0;
}
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
const char *table_name, const char *line,
int valid_column)
{
if (NULL == g2c_runtime || NULL == g2c_schema || NULL == line) {
return -1;
}
struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema;
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
struct compile_runtime *compile_rt = g2c_rt->ref_compile_rt;
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
log_fatal(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> has no is_valid(column seq:%d)"
" in table_line:%s", __FUNCTION__, __LINE__, table_name,
valid_column, line);
g2c_rt->update_err_cnt++;
return -1;
}
int ret = -1;
struct group2compile_item *g2c_item = group2compile_item_new(line, schema, table_name,
compile_rt->logger);
if (NULL == g2c_item) {
g2c_rt->update_err_cnt++;
return -1;
}
if (1 == g2c_item->not_flag) {
ret = validate_table_not_clause(g2c_rt, schema->ref_tbl_mgr,
g2c_item->vtable_id, is_valid,
compile_rt->logger);
if (ret < 0) {
log_fatal(compile_rt->logger, MODULE_COMPILE,
"[%s:%d]validate NOT clause failed, abandon config:%s",
__FUNCTION__, __LINE__, line);
goto next;
}
}
if (0 == is_valid) {
//delete
ret = maat_remove_group_from_compile(compile_rt->cfg_hash, g2c_item,
compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
g2c_rt->not_clause_cnt--;
}
g2c_rt->rule_num--;
} else {
g2c_rt->update_err_cnt++;
}
} else {
//add
ret = maat_add_group_to_compile(compile_rt->cfg_hash, g2c_item,
compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
g2c_rt->not_clause_cnt++;
}
g2c_rt->rule_num++;
} else {
g2c_rt->update_err_cnt++;
}
}
next:
group2compile_item_free(g2c_item);
return ret;
}
long long group2compile_runtime_not_clause_count(void *g2c_runtime)
{
if (NULL == g2c_runtime) {
return 0;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
return g2c_rt->not_clause_cnt;
}
long long group2compile_runtime_rule_count(void *g2c_runtime)
{
if (NULL == g2c_runtime) {
return 0;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
return g2c_rt->rule_num;
}
long long group2compile_runtime_update_err_count(void *g2c_runtime)
{
if (NULL == g2c_runtime) {
return 0;
}
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
return g2c_rt->update_err_cnt;
}
int compile_runtime_commit(void *compile_runtime, const char *table_name,
long long maat_rt_version)
{
if (NULL == compile_runtime) {
return -1;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
int updating_flag = rcu_hash_is_updating(compile_rt->cfg_hash);
if (0 == updating_flag) {
return 0;
}
int ret = 0;
size_t compile_cnt = 0;
struct bool_matcher *old_bool_matcher = NULL;
struct bool_matcher *new_bool_matcher = NULL;
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_bool_matcher = maat_compile_bool_matcher_new(compile_rt, &compile_cnt);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_bool_matcher) {
log_fatal(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] table[%s] rebuild compile bool_matcher failed, compile"
" rules count:%zu", __FUNCTION__, __LINE__, table_name, compile_cnt);
ret = -1;
} else {
log_info(compile_rt->logger, MODULE_COMPILE,
"table[%s] commit %zu compile rules and rebuild compile bool_matcher"
" completed, version:%lld, consume:%lldms", table_name, compile_cnt,
maat_rt_version, time_elapse_ms);
}
struct clause_id_kv *old_clause_id_kv_hash = NULL;
struct clause_id_kv *new_clause_id_kv_hash = NULL;
struct clause_id_kv *old_not_clause_id_kv_hash = NULL;
struct clause_id_kv *new_not_clause_id_kv_hash = NULL;
new_clause_id_kv_hash = build_clause_id_kv_hash(compile_rt, 0);
new_not_clause_id_kv_hash = build_clause_id_kv_hash(compile_rt, 1);
old_clause_id_kv_hash = compile_rt->clause_id_kv_hash;
old_not_clause_id_kv_hash = compile_rt->not_clause_id_kv_hash;
old_bool_matcher = compile_rt->bm;
/*
rule_monitor_loop thread
STEP_1. compile_rt->bm = new_bool_matcher
STEP_2: rcu_hash_commit(new_compile)
scan thread
Assume_1. If scan thread is using bool_matcher_match(compile_rt->bm)
before STEP_1 or after STEP_2, it's ok.
Assume_2. If scan thread is using bool_matcher_match(compile_rt->bm)
between STEP_1 and STEP_2.
P1: If new compile is hit and returned, then caller can get this compile's
ex_data by using maat_plugin_table_get_ex_data(hit_compile_id) because
STEP_2 is fast enough.
*/
compile_rt->bm = new_bool_matcher;
compile_rt->clause_id_kv_hash = new_clause_id_kv_hash;
compile_rt->not_clause_id_kv_hash = new_not_clause_id_kv_hash;
rcu_hash_commit(compile_rt->cfg_hash); //after commit, old cfg still available within COMPILE_GC_TIMEOUT_S.
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL,
garbage_bool_matcher_free);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_clause_id_kv_hash, NULL,
garbage_clause_id_kv_hash_free);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_not_clause_id_kv_hash, NULL,
garbage_clause_id_kv_hash_free);
compile_rt->rule_num = rcu_hash_count(compile_rt->cfg_hash);
return ret;
}
long long compile_runtime_rule_count(void *compile_runtime)
{
if (NULL == compile_runtime) {
return 0;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
return compile_rt->rule_num;
}
long long compile_runtime_update_err_count(void *compile_runtime)
{
if (NULL == compile_runtime) {
return 0;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
return compile_rt->update_err_cnt;
}
static int compile_sort_para_compare(const struct compile_sort_para *a,
const struct compile_sort_para *b)
{
//If compile rule's execute sequences are not specified or equal.
if (a->declared_clause_num != b->declared_clause_num) {
return (a->declared_clause_num - b->declared_clause_num);
} else {
return (b->compile_id - a->compile_id);
}
}
static void compile_sort_para_set(struct compile_sort_para *para,
const struct compile_item *item)
{
para->compile_id = item->compile_id;
para->declared_clause_num = item->declared_clause_num;
}
static int compare_compile_item(const void *a, const void *b)
{
const struct compile_item *ra = *(const struct compile_item **)a;
const struct compile_item *rb = *(const struct compile_item **)b;
struct compile_sort_para sa, sb;
compile_sort_para_set(&sa, ra);
compile_sort_para_set(&sb, rb);
return compile_sort_para_compare(&sa, &sb);
}
int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile_ids,
size_t compile_ids_size, struct maat_state *state)
{
struct compile_state *compile_state = state->compile_state;
struct compile_item *compile_items[compile_ids_size];
// all hit clause_id -> compile_id
size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, compile_state,
state->thread_id,
(void **)compile_items,
compile_ids_size);
if (bool_match_ret > 0) {
qsort(compile_items, bool_match_ret, sizeof(struct compile_item *),
compare_compile_item);
}
for (size_t i = 0; i < bool_match_ret; i++) {
compile_ids[i] = compile_items[i]->compile_id;
}
return MIN(bool_match_ret, compile_ids_size);
}
int compile_state_update(struct compile_state *compile_state, struct maat *maat_inst,
int vtable_id, int custom_compile_tbl_id, int Nth_scan,
struct maat_item *hit_items, size_t n_hit_item)
{
size_t i = 0, j = 0;
size_t hit_cnt = n_hit_item;
long long hit_group_ids[MAX_HIT_GROUP_NUM];
utarray_clear(compile_state->this_scan_hit_clauses);
utarray_clear(compile_state->last_hit_group_ids);
compile_state->this_scan_not_logic = 0;
compile_state->Nth_scan = Nth_scan;
for (i = 0; i < hit_cnt; i++) {
hit_group_ids[i] = hit_items[i].group_id;
utarray_push_back(compile_state->last_hit_group_ids, &hit_items[i].group_id);
}
int g2g_table_id = table_manager_get_group2group_table_id(maat_inst->tbl_mgr);
void *g2g_rt = table_manager_get_runtime(maat_inst->tbl_mgr, g2g_table_id);
long long super_group_ids[MAX_HIT_GROUP_NUM];
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, hit_group_ids,
hit_cnt, super_group_ids,
MAX_HIT_GROUP_NUM);
for (i = 0; i < super_group_cnt; i++) {
utarray_push_back(compile_state->last_hit_group_ids, &super_group_ids[i]);
}
if (1 == maat_inst->opts.hit_path_on && hit_cnt > 0) {
for (i = 0; i < hit_cnt; i++) {
compile_state_add_internal_hit_path(compile_state, hit_items[i].item_id,
hit_items[i].group_id, vtable_id, 0, Nth_scan);
}
}
if (1 == maat_inst->opts.hit_group_on) {
compile_state_add_direct_hit_groups(compile_state, hit_items, hit_cnt, vtable_id);
compile_state_add_indirect_hit_groups(compile_state, super_group_ids,
super_group_cnt, vtable_id);
}
/* update hit clause */
int compile_table_id = table_manager_get_default_compile_table_id(maat_inst->tbl_mgr);
if (custom_compile_tbl_id > 0) {
compile_table_id = custom_compile_tbl_id;
}
struct compile_runtime *compile_rt = table_manager_get_runtime(maat_inst->tbl_mgr,
compile_table_id);
if (NULL == compile_rt) {
return 0;
}
for (j = 0; j < super_group_cnt && hit_cnt < MAX_HIT_GROUP_NUM; j++) {
hit_group_ids[hit_cnt++] = super_group_ids[j];
}
for (i = 0; i < hit_cnt; i++) {
compile_state_update_hit_clauses(compile_state, compile_rt,
hit_group_ids[i], vtable_id);
}
compile_state_cache_hit_not_groups(compile_state, compile_rt, hit_group_ids,
hit_cnt, vtable_id);
return hit_cnt;
}
void compile_state_not_logic_update(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
struct maat *maat_inst, int vtable_id,
int Nth_scan)
{
if (NULL == compile_state || NULL == maat_inst) {
return;
}
compile_state->this_scan_not_logic = 1;
compile_state->Nth_scan = Nth_scan;
utarray_clear(compile_state->this_scan_hit_not_clauses);
struct table_group *tbl_group = NULL;
HASH_FIND(hh, compile_state->hit_not_tbl_groups, &vtable_id, sizeof(int), tbl_group);
if (NULL == tbl_group) {
return;
}
struct clause_id_kv *clause_id_kv = NULL;
for (size_t i = 0; i < utarray_len(tbl_group->group_ids); i++) {
long long *group_id = utarray_eltptr(tbl_group->group_ids, i);
struct clause_query_key key = {*group_id, vtable_id, 1};
HASH_FIND(hh, compile_rt->not_clause_id_kv_hash, &key, sizeof(key), clause_id_kv);
if (NULL == clause_id_kv) {
continue;
}
compile_state_add_hit_not_clauses(compile_state, clause_id_kv->clause_ids);
if (1 == maat_inst->opts.hit_path_on) {
compile_state_add_internal_hit_path(compile_state, -1, *group_id,
vtable_id, 1, Nth_scan);
}
}
}
size_t compile_state_get_indirect_hit_groups(struct compile_state *compile_state,
struct maat_hit_group *group_array,
size_t array_size)
{
size_t i = 0;
struct maat_hit_group *hit_group = NULL;
for (i = 0; i < utarray_len(compile_state->indirect_hit_groups) && i < array_size; i++) {
hit_group = (struct maat_hit_group *)utarray_eltptr(compile_state->indirect_hit_groups, i);
group_array[i].item_id = hit_group->item_id;
group_array[i].group_id = hit_group->group_id;
group_array[i].vtable_id = hit_group->vtable_id;
}
utarray_clear(compile_state->indirect_hit_groups);
return i;
}
size_t compile_state_get_indirect_hit_group_cnt(struct compile_state *compile_state)
{
return utarray_len(compile_state->indirect_hit_groups);
}
size_t compile_state_get_last_hit_group_id(struct compile_state *compile_state,
long long *group_id_array,
size_t array_size)
{
size_t i = 0;
for (i = 0; i < utarray_len(compile_state->last_hit_group_ids) && i < array_size; i++) {
group_id_array[i] = *(long long *)utarray_eltptr(compile_state->last_hit_group_ids, i);
}
return i;
}
size_t compile_state_get_last_hit_group_id_cnt(struct compile_state *compile_state)
{
return utarray_len(compile_state->last_hit_group_ids);
}
size_t compile_state_get_direct_hit_groups(struct compile_state *compile_state,
struct maat_hit_group *group_array,
size_t array_size)
{
UT_array *direct_hit_group = compile_state->direct_hit_groups;
size_t i = 0;
struct maat_hit_group *group = NULL;
for (i = 0; i < utarray_len(direct_hit_group) && i < array_size; i++) {
group = (struct maat_hit_group *)utarray_eltptr(direct_hit_group, i);
group_array[i].item_id = group->item_id;
group_array[i].group_id = group->group_id;
group_array[i].vtable_id = group->vtable_id;
}
utarray_clear(compile_state->direct_hit_groups);
return i;
}
size_t compile_state_get_direct_hit_group_cnt(struct compile_state *compile_state)
{
return utarray_len(compile_state->direct_hit_groups);
}
size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state,
struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt,
struct maat_hit_path *hit_path_array,
size_t array_size)
{
size_t hit_path_cnt = 0;
struct internal_hit_path *internal_path = NULL;
for (int i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) {
internal_path = (struct internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i);
/*
NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths
*/
long long super_group_ids[MAX_HIT_GROUP_NUM];
UT_array *valid_super_group_ids;
utarray_new(valid_super_group_ids, &ut_compile_group_id_icd);
size_t super_group_cnt = group2group_runtime_get_super_groups(g2g_rt, &(internal_path->group_id), 1,
super_group_ids, MAX_HIT_GROUP_NUM);
for (size_t idx = 0; idx < super_group_cnt; idx++) {
utarray_push_back(valid_super_group_ids, &super_group_ids[idx]);
}
/*
internal_path->group_id can be referenced directly by compile,
so add it to hit_path which super_group_ids is -1
------------------------------------------------------------------------------
NOTE: Add the hit path as long as the item is hit
*/
long long super_group_id = -1;
utarray_push_back(valid_super_group_ids, &super_group_id);
long long *p = NULL;
struct maat_hit_path tmp_path;
for (p = utarray_front(valid_super_group_ids); p != NULL && hit_path_cnt < array_size;
p = utarray_next(valid_super_group_ids, p)) {
memset(&tmp_path, 0, sizeof(tmp_path));
tmp_path.Nth_scan = internal_path->Nth_scan;
tmp_path.item_id = internal_path->item_id;
tmp_path.sub_group_id = internal_path->group_id;
tmp_path.top_group_id = *p;
tmp_path.vtable_id = internal_path->vtable_id;
tmp_path.NOT_flag = internal_path->NOT_flag;
tmp_path.clause_index = -1;
tmp_path.compile_id = -1;
/* check if internal_path is duplicated from hit_path_array[]
* element */
if (hit_path_cnt > 0) {
if (maat_compile_is_hit_path_existed(hit_path_array, hit_path_cnt, &tmp_path)) {
continue;
}
}
hit_path_array[hit_path_cnt] = tmp_path;
hit_path_cnt++;
}
utarray_free(valid_super_group_ids);
}
return hit_path_cnt;
}
Reference in New Issue View Git Blame Copy Permalink