gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rollback to v4.0.31

Browse Source
This commit is contained in:
liuwentan
2023-08-09 19:22:09 +08:00
parent d29eef0423
commit fb0cb5405d
40 changed files with 1907 additions and 5448 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/CMakeLists.txt
View File

@@ -23,15 +23,13 @@ include_directories(/opt/MESA/include/MESA/)
include_directories(${PROJECT_SOURCE_DIR}/include/)
include_directories(${PROJECT_SOURCE_DIR}/deps/)
include_directories(${PROJECT_SOURCE_DIR}/scanner)
include_directories(${PROJECT_SOURCE_DIR}/scanner/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/fqdn_engine)
include_directories(${PROJECT_SOURCE_DIR}/scanner/bool_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/ip_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/flag_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/interval_matcher)
include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_hs)
include_directories(${PROJECT_SOURCE_DIR}/scanner/expr_matcher/adapter_rs)
# Static Library Output
add_library(maat_frame_static STATIC ${MAAT_SRC} ${LIB_SOURCE_FILES})

19
src/inc_internal/maat_compile.h
View File

@@ -37,6 +37,12 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void group2compile_schema_free(void *g2c_schema);
int group2compile_associated_compile_table_id(void *g2c_schema);
int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp);
/* compile runtime API */
void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
@@ -64,6 +70,14 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr
struct maat_hit_path *hit_path_array,
size_t array_size, size_t n_internal_hit_path);
void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema,
long long compile_id);
void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema);
void compile_runtime_garbage_collect_routine(void *compile_runtime);
/* group2compile runtime API */
void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin,
@@ -105,13 +119,10 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
struct maat_hit_group *hit_group_array,
size_t array_size);
int maat_compile_state_get_compile_table_id(struct maat_compile_state *compile_state,
long long compile_id);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);
#ifdef __cplusplus
}
#endif
#endif
#endif

11
src/inc_internal/maat_expr.h
View File

@@ -52,13 +52,14 @@ long long expr_runtime_get_version(void *expr_runtime);
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int vtable_id, struct maat_state *state);
struct expr_matcher_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
struct adapter_hs_stream *expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct expr_matcher_stream *s_handle,
const char *data, size_t data_len, int vtable_id, struct maat_state *state);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,
struct expr_matcher_stream *stream);
struct adapter_hs_stream *s_handle);
int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district,
size_t district_len, long long *district_id);
@@ -83,4 +84,4 @@ long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt);
}
#endif
#endif
#endif

7
src/inc_internal/maat_kv.h
View File

@@ -26,15 +26,12 @@ void maat_kv_store_free(struct maat_kv_store *store);
int maat_kv_register(struct maat_kv_store *store, const char *key, long long value);
int maat_kv_read(struct maat_kv_store *store, const char *key, long long *value_array, size_t n_array);
int maat_kv_read(struct maat_kv_store *store, const char *key, long long *value);
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz,
long long *value_array, size_t n_array);
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, long long *value);
int maat_kv_write(struct maat_kv_store *store, const char *key, long long value);
int maat_kv_append(struct maat_kv_store *store, const char *key, long long value);
struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store);
#ifdef __cplusplus

3
src/inc_internal/maat_rule.h
View File

@@ -128,7 +128,6 @@ struct maat_options {
int rule_effect_interval_ms;
int rule_update_checking_interval_ms;
enum maat_expr_engine expr_engine;
enum data_source input_mode;
union {
struct source_iris_ctx iris_ctx;
@@ -226,4 +225,4 @@ void garbage_maat_kv_store_free(void *kv_store, void *arg);
}
#endif
#endif
#endif

24
src/inc_internal/maat_table.h
View File

@@ -19,7 +19,6 @@ extern "C"
#include <stddef.h>
#include <cJSON/cJSON.h>
#include "maat.h"
#include "log/log.h"
#include "maat_garbage_collection.h"
@@ -48,8 +47,7 @@ struct table_manager;
struct table_manager *
table_manager_create(const char *table_info_path, const char *accept_tags,
enum maat_expr_engine expr_engine, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_thread_num,
struct maat_garbage_bin *garbage_bin);
@@ -59,23 +57,7 @@ void table_manager_destroy(struct table_manager *tbl_mgr);
size_t table_manager_table_size(struct table_manager *tbl_mgr);
size_t table_manager_table_count(struct table_manager *tbl_mgr);
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_name);
/**
* @brief get table_name's all conjunction parents' table_id
*
* for example: "table_id": 1,
* "table_name":"COMPILE_CONJ"
* "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"]
*
* "table_id": 2,
* "table_name": "COMPILE_PLUGIN",
* "db_tables": ["COMPILE_DEFAULT"]
*
* so COMPILE_DEFAULT has two conjunction parents whose table_id is 1 and 2.
*/
int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name,
long long *table_ids_array, size_t n_table_ids_array);
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name);
const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id);
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id);
@@ -85,8 +67,6 @@ int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr);
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr);
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr);
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags);

4
src/json2iris.c
View File

@@ -377,7 +377,7 @@ static int direct_write_rule(cJSON *json, struct maat_kv_store *str2int,
if (1 == cmd[i].str2int_flag) {
long long int_value = 0;
char *p = item->valuestring;
ret = maat_kv_read(str2int, p, &int_value, 1);
ret = maat_kv_read(str2int, p, &int_value);
if (ret < 0) {
log_error(logger, MODULE_JSON2IRIS,
"[%s:%d] %s's value %s is not valid format",
@@ -636,7 +636,7 @@ static int write_region_rule(cJSON *region_json, int compile_id, int group_id,
const char *table_type_str = item->valuestring;
long long table_type_int;
int ret = maat_kv_read(p_iris->str2int_map, table_type_str, &(table_type_int), 1);
int ret = maat_kv_read(p_iris->str2int_map, table_type_str, &table_type_int);
if (ret != 1) {
log_error(logger, MODULE_JSON2IRIS,
"[%s:%d] compile rule %d table name %s's table_type %s invalid",

106
src/maat_api.c
View File

@@ -51,7 +51,7 @@ enum district_flag {
struct maat_stream {
struct maat *ref_maat_inst;
struct expr_matcher_stream *handle; //each physical table open one stream
struct adapter_hs_stream *handle; //each physical table open one stream
long long last_full_version;
long long expr_rt_version;
struct log_handle *logger;
@@ -70,7 +70,6 @@ struct maat_options* maat_options_new(void)
options->rule_update_checking_interval_ms = 1 * 1000;
options->gc_timeout_ms = 10 * 1000;
options->input_mode = DATA_SOURCE_NONE;
options->expr_engine = MAAT_EXPR_ENGINE_HS;
options->log_level = 0;
return options;
@@ -255,19 +254,6 @@ int maat_options_set_stat_file(struct maat_options *opts, const char *stat_filen
return 0;
}
int maat_options_set_expr_engine(struct maat_options *opts,
enum maat_expr_engine expr_engine)
{
if (NULL == opts ||
(expr_engine != MAAT_EXPR_ENGINE_HS && expr_engine != MAAT_EXPR_ENGINE_RS)) {
return -1;
}
opts->expr_engine = expr_engine;
return 0;
}
int maat_options_set_logger(struct maat_options *opts, const char *log_path,
enum log_level level)
{
@@ -371,8 +357,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
pthread_mutex_init(&(maat_inst->background_update_mutex), NULL);
maat_inst->tbl_mgr = table_manager_create(table_info_path, maat_inst->opts.accept_tags,
maat_inst->opts.expr_engine, maat_inst->garbage_bin,
maat_inst->logger);
maat_inst->garbage_bin, maat_inst->logger);
if (NULL == maat_inst->tbl_mgr) {
goto failed;
}
@@ -425,7 +410,12 @@ int maat_helper_verify_regex_expression(const char *regex_expr)
return 0;
}
return expr_matcher_verify_regex_expression(regex_expr, NULL);
int ret = adapter_hs_verify_regex_expression(regex_expr, NULL);
if (ret < 0) {
return 0;
} else {
return 1;
}
}
int maat_get_table_id(struct maat *maat_inst, const char *table_name)
@@ -522,6 +512,32 @@ int maat_table_callback_register(struct maat *maat_inst, int table_id,
return 0;
}
static int compile_table_ex_schema_register(struct maat *maat_inst, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp)
{
void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id);
assert(schema != NULL);
int ret = compile_table_set_ex_data_schema((struct compile_schema *)schema, table_id,
new_func, free_func, dup_func,
argl, argp);
if (ret < 0) {
return -1;
}
if (maat_inst->maat_rt != NULL) {
void *runtime = table_manager_get_runtime(maat_inst->tbl_mgr, table_id);
assert(runtime != NULL);
compile_runtime_ex_data_iterate((struct compile_runtime *)runtime,
(struct compile_schema *)schema);
}
return 0;
}
static int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
@@ -755,14 +771,16 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_inst,
int ret = -1;
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr,
table_id);
if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_IP_PLUGIN ||
table_type == TABLE_TYPE_BOOL_PLUGIN || table_type == TABLE_TYPE_FQDN_PLUGIN ) {
if (TABLE_TYPE_COMPILE == table_type) {
ret = compile_table_ex_schema_register(maat_inst, table_id, new_func,
free_func, dup_func, argl, argp);
} else {
ret = generic_plugin_table_ex_schema_register(maat_inst, table_name,
table_id, new_func, free_func,
dup_func, argl, argp);
}
pthread_mutex_unlock(&(maat_inst->background_update_mutex));
pthread_mutex_unlock(&(maat_inst->background_update_mutex));
return ret;
}
@@ -793,7 +811,9 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_inst, int table_id,
void *ret = NULL;
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr,
table_id);
if (TABLE_TYPE_PLUGIN == table_type) {
if (TABLE_TYPE_COMPILE == table_type) {
ret = compile_runtime_get_ex_data(runtime, schema, *(long long *)key);
} else if (TABLE_TYPE_PLUGIN == table_type) {
ret = plugin_runtime_get_ex_data(runtime, schema, key, key_len);
}
@@ -1148,7 +1168,9 @@ int maat_scan_flag(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = flag_scan(maat_inst->tbl_mgr, state->thread_id, flag,
phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1156,8 +1178,6 @@ int maat_scan_flag(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1237,7 +1257,9 @@ int maat_scan_integer(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = interval_scan(maat_inst->tbl_mgr, state->thread_id, integer,
phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1245,8 +1267,6 @@ int maat_scan_integer(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1326,7 +1346,9 @@ int maat_scan_ipv4(struct maat *maat_inst, int table_id, uint32_t ip_addr,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = ipv4_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr,
port, protocol, phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1334,8 +1356,6 @@ int maat_scan_ipv4(struct maat *maat_inst, int table_id, uint32_t ip_addr,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1416,7 +1436,9 @@ int maat_scan_ipv6(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = ipv6_scan(maat_inst->tbl_mgr, state->thread_id, ip_addr,
port, protocol, phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1424,8 +1446,6 @@ int maat_scan_ipv6(struct maat *maat_inst, int table_id,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1505,7 +1525,9 @@ int maat_scan_string(struct maat *maat_inst, int table_id, const char *data,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
alignment_int64_array_add(maat_inst->stat->thread_call_cnt, state->thread_id, 1);
int hit_group_cnt = string_scan(maat_inst->tbl_mgr, state->thread_id, data,
data_len, phy_table_id, vtable_id, state);
if (hit_group_cnt < 0) {
@@ -1513,8 +1535,6 @@ int maat_scan_string(struct maat *maat_inst, int table_id, const char *data,
return MAAT_SCAN_ERR;
}
maat_runtime_ref_inc(maat_rt, state->thread_id);
size_t sum_hit_compile_cnt = 0;
if (hit_group_cnt > 0 || scan_status_should_compile_NOT(state)) {
sum_hit_compile_cnt = group_to_compile(maat_inst, results, n_result, state);
@@ -1589,8 +1609,8 @@ struct maat_stream *maat_stream_new(struct maat *maat_inst, int table_id,
stream->expr_rt_version = expr_runtime_get_version(expr_rt);
maat_runtime_ref_inc(maat_inst->maat_rt, state->thread_id);
struct expr_matcher_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt,
state->thread_id);
struct adapter_hs_stream *handle = expr_runtime_stream_open((struct expr_runtime *)expr_rt,
state->thread_id);
if (NULL == handle) {
goto error;
}
@@ -1869,22 +1889,6 @@ int maat_state_set_scan_compile_table(struct maat_state *state, int compile_tabl
return 0;
}
int maat_state_get_compile_table_ids(struct maat_state *state, long long *compile_ids,
size_t n_compile_ids, int *compile_table_ids)
{
if (NULL == state || NULL == compile_ids || 0 == n_compile_ids ||
NULL == compile_table_ids) {
return -1;
}
for (size_t i = 0; i < n_compile_ids; i++) {
compile_table_ids[i] = maat_compile_state_get_compile_table_id(state->compile_state,
compile_ids[i]);
}
return n_compile_ids;
}
int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *paths,
size_t n_path)
{
@@ -1956,4 +1960,4 @@ int maat_state_get_hit_groups(struct maat_state *state, enum maat_list_type type
int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group)
{
return 0;
}
}

281
src/maat_compile.c
View File

@@ -27,12 +27,16 @@
#include "alignment.h"
#define MODULE_COMPILE module_name_str("maat.compile")
#define DEFAULT_GC_TIMEOUT_S 10
struct compile_schema {
int compile_id_column;
int rule_tag_column;
int declared_clause_num_column;
int set_flag;
int gc_timeout_s;
int table_id; //ugly
struct ex_data_schema ex_schema;
struct table_manager *ref_tbl_mgr;
struct log_handle *logger;
};
@@ -53,6 +57,8 @@ struct compile_item {
long long compile_id;
char *table_line;
size_t table_line_len;
struct compile_schema *ref_schema;
void **ex_data;
char table_name[MAX_NAME_STR_LEN];
};
@@ -79,8 +85,7 @@ struct literal_clause {
/* compile_runtime and group2compile_runtime share compile_hash_map */
struct compile_runtime {
struct bool_matcher *bm;
struct rcu_hash_table *cfg_hash; // <compile_id, struct maat_compile>
struct rcu_hash_table *tbl_cfg_hash; // <compile_id, table_id>
struct rcu_hash_table *cfg_hash; // <compile_id, struct maat_compile>
struct maat_runtime *ref_maat_rt;
time_t version;
struct literal_clause *literal2clause_hash;
@@ -132,15 +137,9 @@ struct maat_internal_hit_path {
int vtable_id;
};
struct maat_compile_table {
long long compile_id;
int table_id;
};
struct maat_compile_state {
uint8_t this_scan_hit_item_flag;
uint8_t not_clause_hit_flag;
uint8_t inc_hit_path_flag;
int Nth_scan;
time_t compile_rt_version;
@@ -148,14 +147,12 @@ struct maat_compile_state {
UT_array *internal_inc_hit_paths;
UT_array *all_hit_clauses;
UT_array *this_scan_hit_clauses;
UT_array *hit_compile_tables;
};
UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL};
UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL};
UT_icd ut_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL};
UT_icd ut_hit_path_icd = {sizeof(struct maat_internal_hit_path), NULL, NULL, NULL};
UT_icd ut_hit_compile_table_icd = {sizeof(struct maat_compile_table)};
static struct maat_compile *maat_compile_new(long long compile_id)
{
@@ -188,6 +185,23 @@ static int maat_compile_set(struct maat_compile *compile, const char *table_name
return 0;
}
static void *rule_ex_data_new(const char *table_name, int table_id,
const char *table_line,
struct ex_data_schema *ex_schema)
{
void *ex_data = NULL;
ex_schema->new_func(table_name, table_id, NULL, table_line, &ex_data,
ex_schema->argl, ex_schema->argp);
return ex_data;
}
static void rule_ex_data_free(int table_id, void **ex_data,
const struct ex_data_schema *ex_schema)
{
ex_schema->free_func(table_id, ex_data, ex_schema->argl, ex_schema->argp);
}
static int compile_accept_tag_match(struct compile_schema *schema, const char *line,
const char *table_name, struct log_handle *logger)
{
@@ -262,11 +276,18 @@ compile_item_new(const char *table_line, struct compile_schema *schema,
}
compile_item->declared_clause_num = atoi(table_line + column_offset);
compile_item->ref_schema = schema;
compile_item->ex_data = ALLOC(void *, 1);
memcpy(compile_item->table_name, table_name, sizeof(compile_item->table_name));
compile_item->table_line_len = strlen(table_line) + 1;
compile_item->table_line = ALLOC(char, compile_item->table_line_len);
memcpy(compile_item->table_line, table_line, compile_item->table_line_len);
if (1 == schema->set_flag) {
*(compile_item->ex_data) = rule_ex_data_new(table_name, schema->table_id,
compile_item->table_line,
&(schema->ex_schema));
}
return compile_item;
error:
FREE(compile_item);
@@ -275,6 +296,17 @@ error:
static void compile_item_free(struct compile_item *item)
{
struct compile_schema *schema = item->ref_schema;
if (1 == schema->set_flag) {
rule_ex_data_free(schema->table_id, item->ex_data, &(schema->ex_schema));
*item->ex_data = NULL;
}
if (item->ex_data != NULL) {
FREE(item->ex_data);
}
item->declared_clause_num = -1;
if (item->table_line != NULL) {
@@ -315,9 +347,70 @@ static void rcu_compile_cfg_free(void *user_ctx, void *data)
maat_compile_free(compile);
}
static void rcu_compile_table_cfg_free(void *user_ctx, void *data)
int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int table_id,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
maat_ex_dup_func_t *dup_func,
long argl, void *argp)
{
FREE(data);
if (1 == compile_schema->set_flag) {
log_error(compile_schema->logger, MODULE_COMPILE,
"[%s:%d] compile table(table_id:%d)ex schema has been set already,"
" can't set again", __FUNCTION__, __LINE__, table_id);
return -1;
}
compile_schema->ex_schema.new_func = new_func;
compile_schema->ex_schema.free_func = free_func;
compile_schema->ex_schema.dup_func = dup_func;
compile_schema->ex_schema.argl = argl;
compile_schema->ex_schema.argp = argp;
compile_schema->set_flag = 1;
return 0;
}
static void *compile_runtime_get_user_data(struct compile_runtime *compile_rt,
long long compile_id)
{
struct maat_compile *compile = rcu_hash_find(compile_rt->cfg_hash,
(char *)&compile_id,
sizeof(long long));
void *ret = NULL;
if (compile != NULL) {
ret = compile->user_data;
}
return ret;
}
static void rule_ex_data_new_cb(void *user_data, void *param,
const char *table_name, int table_id)
{
struct ex_data_schema *ex_schema = (struct ex_data_schema *)param;
struct compile_item *compile = (struct compile_item *)user_data;
void *ad = rule_ex_data_new(table_name, table_id, compile->table_line, ex_schema);
*compile->ex_data = ad;
}
static void compile_runtime_user_data_iterate(struct compile_runtime *compile_rt,
void (*callback)(void *user_data, void *param,
const char *table_name, int table_id),
void *param, int table_id)
{
/* I'm in background_update_mutex, config update can't happen, so no need to lock cfg_hash */
void **data_array = NULL;
size_t data_cnt = rcu_hash_list(compile_rt->cfg_hash, &data_array);
for (size_t i = 0; i < data_cnt; i++) {
struct maat_compile *compile = (struct maat_compile *)data_array[i];
if (compile->user_data) {
callback(compile->user_data, param, compile->table_name, table_id);
}
}
FREE(data_array);
}
void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
@@ -371,6 +464,12 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
goto error;
}
//gc_timeout_s is optional
custom_item = cJSON_GetObjectItem(item, "gc_timeout_s");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->gc_timeout_s = custom_item->valueint;
}
schema->ref_tbl_mgr = tbl_mgr;
return schema;
error:
@@ -494,13 +593,14 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num,
return NULL;
}
struct compile_schema *schema = (struct compile_schema *)compile_schema;
struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1);
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match,
max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM);
compile_rt->version = time(NULL);
compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL, 0);
compile_rt->tbl_cfg_hash = rcu_hash_new(rcu_compile_table_cfg_free, NULL, 0);
compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL,
schema->gc_timeout_s + DEFAULT_GC_TIMEOUT_S);
compile_rt->literal2clause_hash = NULL;
compile_rt->logger = logger;
compile_rt->ref_garbage_bin = garbage_bin;
@@ -547,11 +647,6 @@ void compile_runtime_free(void *compile_runtime)
compile_rt->cfg_hash = NULL;
}
if (compile_rt->tbl_cfg_hash != NULL) {
rcu_hash_free(compile_rt->tbl_cfg_hash);
compile_rt->tbl_cfg_hash = NULL;
}
if (compile_rt->literal2clause_hash != NULL) {
literal2clause_hash_free(compile_rt->literal2clause_hash);
compile_rt->literal2clause_hash = NULL;
@@ -892,30 +987,14 @@ static inline int compare_hit_group(const void *pa, const void *pb)
struct maat_hit_group *la=(struct maat_hit_group *)pa;
struct maat_hit_group *lb=(struct maat_hit_group *)pb;
long long ret = la->item_id - lb->item_id;
long long ret = la->group_id - lb->group_id;
if (ret == 0) {
ret = la->group_id - lb->group_id;
if (ret == 0) {
ret = la->vtable_id - lb->vtable_id;
}
ret = la->vtable_id - lb->vtable_id;
}
return ret;
}
static inline int compare_compile_id(const void *a, const void *b)
{
long long ret = *(const long long *)a - *(const long long *)b;
if (0 == ret) {
return 0;
} else if (ret < 0) {
return -1;
} else {
return 1;
}
}
static struct literal_clause *
maat_compile_build_literal2clause_hash(struct compile_runtime *compile_rt)
{
@@ -996,29 +1075,6 @@ static size_t compile_state_if_new_hit_compile(struct maat_compile_state *compil
return r_in_c_cnt;
}
static void compile_state_update_hit_compile_table_id(struct maat_compile_state *compile_state,
long long compile_id, int table_id)
{
if (!utarray_find(compile_state->hit_compile_tables, &compile_id, compare_compile_id)) {
struct maat_compile_table compile_table = {compile_id, table_id};
utarray_push_back(compile_state->hit_compile_tables, &compile_table);
utarray_sort(compile_state->hit_compile_tables, compare_compile_id);
}
}
static int compile_runtime_get_compile_table_id(struct compile_runtime *compile_rt,
long long compile_id)
{
if (NULL == compile_rt || compile_id < 0) {
return -1;
}
int *table_id = rcu_hash_find(compile_rt->tbl_cfg_hash, (char *)&compile_id,
sizeof(long long));
return *table_id;
}
static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt,
int is_last_scan, int thread_id,
struct maat_compile_state *compile_state,
@@ -1071,11 +1127,6 @@ static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt
hit a compile that refer a NOT-logic group in previous scan */
user_data_array[ud_result_cnt] = compile->user_data;
ud_result_cnt++;
int table_id = compile_runtime_get_compile_table_id(compile_rt, compile->compile_id);
if (table_id >= 0) {
compile_state_update_hit_compile_table_id(compile_state, compile->compile_id, table_id);
}
}
}
}
@@ -1091,11 +1142,18 @@ static struct compile_item *compile_item_clone(struct compile_item *item)
new_item->compile_id = item->compile_id;
new_item->declared_clause_num = item->declared_clause_num;
new_item->ref_schema = item->ref_schema;
new_item->ex_data = ALLOC(void *, 1);
memcpy(new_item->table_name, item->table_name, sizeof(new_item->table_name));
new_item->table_line_len = item->table_line_len;
new_item->table_line = ALLOC(char, new_item->table_line_len);
memcpy(new_item->table_line, item->table_line, new_item->table_line_len);
if (1 == item->ref_schema->set_flag) {
*(new_item->ex_data) = rule_ex_data_new(item->table_name, item->ref_schema->table_id,
item->table_line, &(item->ref_schema->ex_schema));
}
return new_item;
}
@@ -1231,7 +1289,7 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
if (NULL == compile) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group_id:%lld from compile_id:%lld failed, compile"
" is not existed.", __FUNCTION__, __LINE__, g2c_item->group_id,
" is not exisited.", __FUNCTION__, __LINE__, g2c_item->group_id,
compile_id);
return -1;
} else {
@@ -1287,7 +1345,7 @@ static int maat_remove_group_from_compile(struct rcu_hash_table *hash_tbl,
} else {
log_error(logger, MODULE_COMPILE,
"[%s:%d] Remove group_id:%lld from compile_id:%lld failed, "
"compile is not existed.", __FUNCTION__, __LINE__,
"compile is not exisited.", __FUNCTION__, __LINE__,
g2c_item->group_id, compile_id);
return -1;
}
@@ -1304,7 +1362,6 @@ struct maat_compile_state *maat_compile_state_new(void)
utarray_new(compile_state->internal_inc_hit_paths, &ut_hit_path_icd);
utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd);
utarray_new(compile_state->hit_compile_tables, &ut_hit_compile_table_icd);
return compile_state;
}
@@ -1319,14 +1376,11 @@ void maat_compile_state_reset(struct maat_compile_state *compile_state)
compile_state->compile_rt_version = 0;
compile_state->this_scan_hit_item_flag = 0;
compile_state->not_clause_hit_flag = 0;
compile_state->inc_hit_path_flag = 0;
utarray_clear(compile_state->internal_hit_paths);
utarray_clear(compile_state->internal_inc_hit_paths);
utarray_clear(compile_state->all_hit_clauses);
utarray_clear(compile_state->this_scan_hit_clauses);
utarray_clear(compile_state->hit_compile_tables);
}
void maat_compile_state_free(struct maat_compile_state *compile_state,
@@ -1361,12 +1415,6 @@ void maat_compile_state_free(struct maat_compile_state *compile_state,
compile_state->this_scan_hit_clauses = NULL;
}
if (compile_state->hit_compile_tables != NULL) {
free_bytes += utarray_len(compile_state->hit_compile_tables) * sizeof(struct maat_compile_table);
utarray_free(compile_state->hit_compile_tables);
compile_state->hit_compile_tables = NULL;
}
FREE(compile_state);
free_bytes += sizeof(struct maat_compile_state);
@@ -1493,14 +1541,10 @@ static void maat_compile_state_update_hit_path(struct maat_compile_state *compil
if (compile_state->Nth_scan != Nth_scan) {
assert(compile_state->this_scan_hit_item_flag == 0);
compile_state->Nth_scan = Nth_scan;
utarray_clear(compile_state->internal_inc_hit_paths);
utarray_clear(compile_state->this_scan_hit_clauses);
}
if (1 == compile_state->inc_hit_path_flag) {
compile_state->inc_hit_path_flag = 0;
utarray_clear(compile_state->internal_inc_hit_paths);
}
maat_compile_hit_path_add(compile_state->internal_inc_hit_paths, item_id, group_id,
vtable_id, Nth_scan, Nth_item_result);
@@ -1550,25 +1594,46 @@ static void maat_compile_state_update_hit_clause(struct maat_compile_state *comp
}
}
int maat_compile_state_get_compile_table_id(struct maat_compile_state *compile_state,
long long compile_id)
{
struct maat_compile_table *compile_table = NULL;
compile_table = utarray_find(compile_state->hit_compile_tables, &compile_id,
compare_compile_id);
if (NULL == compile_table) {
return -1;
}
return compile_table->table_id;
}
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state)
{
return compile_state->not_clause_hit_flag;
}
void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt,
struct compile_schema *compile_schema)
{
if (NULL == compile_rt || NULL == compile_schema ||
(0 == compile_schema->set_flag)) {
return;
}
compile_runtime_user_data_iterate(compile_rt, rule_ex_data_new_cb,
&(compile_schema->ex_schema),
compile_schema->table_id);
}
void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt,
struct compile_schema *schema,
long long compile_id)
{
if (NULL == compile_rt || NULL == schema || compile_id < 0
|| (0 == schema->set_flag)) {
return NULL;
}
struct compile_item *item = NULL;
item = (struct compile_item *)compile_runtime_get_user_data(compile_rt,
compile_id);
if (NULL == item) {
return NULL;
}
void *ex_data = NULL;
schema->ex_schema.dup_func(schema->table_id, &ex_data, item->ex_data,
schema->ex_schema.argl, schema->ex_schema.argp);
return ex_data;
}
static int compile_runtime_add_compile(struct compile_runtime *compile_rt,
struct compile_schema *schema,
long long compile_id, const char *table_name,
@@ -1737,7 +1802,6 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
if (0 == is_valid) {
// delete
compile_runtime_del_compile(compile_rt, compile_id);
rcu_hash_del(compile_rt->tbl_cfg_hash, (char *)&compile_id, sizeof(long long));
} else {
// add
int ret = compile_runtime_add_compile(compile_rt, schema, compile_id,
@@ -1745,14 +1809,6 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
if (ret < 0) {
compile_rt->update_err_cnt++;
}
int *table_id = ALLOC(int, 1);
*table_id = table_manager_get_table_id(schema->ref_tbl_mgr, table_name);
ret = rcu_hash_add(compile_rt->tbl_cfg_hash, (char *)&compile_id,
sizeof(long long), table_id);
if (ret < 0) {
FREE(table_id);
}
}
return 0;
@@ -1905,7 +1961,6 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
compile_rt->bm = new_bool_matcher;
compile_rt->literal2clause_hash = new_literal2clause;
rcu_hash_commit(compile_rt->cfg_hash);
rcu_hash_commit(compile_rt->tbl_cfg_hash);
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL,
garbage_bool_matcher_free);
@@ -2070,7 +2125,6 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
tmp_hit_path = compile_state->internal_hit_paths;
} else if (type == MAAT_LIST_TYPE_INC) {
tmp_hit_path = compile_state->internal_inc_hit_paths;
compile_state->inc_hit_path_flag = 1;
}
for (i = 0; i < utarray_len(tmp_hit_path); i++) {
@@ -2085,7 +2139,6 @@ size_t maat_compile_state_get_hit_groups(struct maat_compile_state *compile_stat
for (size_t idx = 0; idx < super_group_cnt; idx++) {
struct maat_hit_group hit_group;
hit_group.item_id = internal_path->item_id;
hit_group.group_id = super_group_ids[idx];
hit_group.vtable_id = internal_path->vtable_id;
if (utarray_find(all_hit_groups, &hit_group, compare_hit_group)) {
@@ -2167,4 +2220,16 @@ size_t maat_compile_state_get_internal_hit_paths(struct maat_compile_state *comp
}
return hit_path_cnt;
}
void compile_runtime_garbage_collect_routine(void *compile_runtime)
{
if (NULL == compile_runtime) {
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
if (compile_rt->cfg_hash != NULL) {
rcu_hash_garbage_collect_routine(compile_rt->cfg_hash);
}
}

133
src/maat_expr.c
View File

@@ -60,7 +60,7 @@ struct expr_item {
long long group_id;
char keywords[MAX_KEYWORDS_STR];
enum expr_type expr_type;
enum expr_match_mode match_mode;
enum hs_match_mode match_mode;
int is_hexbin;
int is_case_sensitive;
void *user_data;
@@ -68,7 +68,7 @@ struct expr_item {
};
struct expr_runtime {
struct expr_matcher *matcher;
struct adapter_hs *hs;
struct rcu_hash_table *item_hash; // <item_id, struct expr_item>
long long version; //expr_rt version
@@ -79,7 +79,6 @@ struct expr_runtime {
struct log_handle *logger;
struct maat_garbage_bin *ref_garbage_bin;
enum maat_expr_engine expr_engine;
int district_num;
struct maat_kv_store *district_map;
struct maat_kv_store *tmp_district_map;
@@ -115,22 +114,22 @@ static enum expr_type int_to_expr_type(int expr_type)
return type;
}
static enum expr_match_mode int_to_match_mode(int match_method)
static enum hs_match_mode int_to_match_mode(int match_method)
{
enum expr_match_mode mode = EXPR_MATCH_MODE_INVALID;
enum hs_match_mode mode = HS_MATCH_MODE_INVALID;
switch (match_method) {
case 0:
mode = EXPR_MATCH_MODE_SUB;
mode = HS_MATCH_MODE_SUB;
break;
case 1:
mode = EXPR_MATCH_MODE_SUFFIX;
mode = HS_MATCH_MODE_SUFFIX;
break;
case 2:
mode = EXPR_MATCH_MODE_PREFIX;
mode = HS_MATCH_MODE_PREFIX;
break;
case 3:
mode = EXPR_MATCH_MODE_EXACTLY;
mode = HS_MATCH_MODE_EXACTLY;
break;
default:
break;
@@ -144,13 +143,13 @@ static int expr_runtime_get_district_id(struct expr_runtime *expr_rt,
{
long long district_id = DISTRICT_ANY;
int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id, 1);
int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id);
if (map_ret < 0) {
if (NULL == expr_rt->tmp_district_map) {
expr_rt->tmp_district_map = maat_kv_store_duplicate(expr_rt->district_map);
}
map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id, 1);
map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id);
if (map_ret < 0) {
district_id = expr_rt->district_num;
maat_kv_register(expr_rt->tmp_district_map, district, district_id);
@@ -168,7 +167,7 @@ int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *dis
return -1;
}
return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id, 1);
return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id);
}
static struct expr_item *
@@ -235,8 +234,8 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name,
__FUNCTION__, __LINE__, table_name, line);
goto error;
} else if (expr_item->expr_type == EXPR_TYPE_REGEX) {
ret = expr_matcher_verify_regex_expression(expr_item->keywords, expr_rt->logger);
if (0 == ret) {
ret = adapter_hs_verify_regex_expression(expr_item->keywords, expr_rt->logger);
if (ret < 0) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> regex expression(item_id:%lld):%s illegal,"
" will be dropped", __FUNCTION__, __LINE__, table_name,
@@ -278,7 +277,7 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name,
match_method_type = atoi(line + column_offset);
expr_item->match_mode = int_to_match_mode(match_method_type);
if (expr_item->match_mode == EXPR_MATCH_MODE_INVALID) {
if (expr_item->match_mode == HS_MATCH_MODE_INVALID) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] expr table:<%s> has invalid match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line);
@@ -473,14 +472,12 @@ void *expr_runtime_new(void *expr_schema, size_t max_thread_num,
return NULL;
}
struct expr_schema *schema = (struct expr_schema *)expr_schema;
struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1);
expr_rt->item_hash = rcu_hash_new(expr_item_free_cb, NULL, 0);
expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger;
expr_rt->expr_engine = table_manager_get_expr_engine(schema->ref_tbl_mgr);
expr_rt->district_map = maat_kv_store_new();
expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
@@ -498,9 +495,9 @@ void expr_runtime_free(void *expr_runtime)
}
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
if (expr_rt->matcher != NULL) {
expr_matcher_free(expr_rt->matcher);
expr_rt->matcher = NULL;
if (expr_rt->hs != NULL) {
adapter_hs_free(expr_rt->hs);
expr_rt->hs = NULL;
}
if (expr_rt->item_hash != NULL) {
@@ -561,18 +558,18 @@ static int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
return 0;
}
static enum expr_pattern_type expr_type2pattern_type(enum expr_type expr_type)
static enum hs_pattern_type expr_type2pattern_type(enum expr_type expr_type)
{
enum expr_pattern_type pattern_type = EXPR_PATTERN_TYPE_STR;
enum hs_pattern_type pattern_type;
switch (expr_type) {
case EXPR_TYPE_STRING:
case EXPR_TYPE_AND:
case EXPR_TYPE_OFFSET:
pattern_type = EXPR_PATTERN_TYPE_STR;
pattern_type = HS_PATTERN_TYPE_STR;
break;
case EXPR_TYPE_REGEX:
pattern_type = EXPR_PATTERN_TYPE_REG;
pattern_type = HS_PATTERN_TYPE_REG;
break;
default:
break;
@@ -689,12 +686,12 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
}
sub_expr_cnt = i;
break;
case EXPR_TYPE_STRING: //AND/OFFSET/STRING type expression use \b to represent blank(' ')
case EXPR_TYPE_STRING:
sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords;
sub_key_array[0] = str_unescape(sub_key_array[0]);
break;
case EXPR_TYPE_REGEX: //only regex type expression use \s to represent blank(' ')
case EXPR_TYPE_REGEX:
sub_expr_cnt = 1;
sub_key_array[0] = expr_item->keywords;
break;
@@ -713,15 +710,15 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
if (TRUE == expr_item->is_case_sensitive) {
// insensitive
expr_rule->patterns[i].case_sensitive = EXPR_CASE_SENSITIVE;
expr_rule->patterns[i].case_sensitive = HS_CASE_SENSITIVE;
} else {
expr_rule->patterns[i].case_sensitive = EXPR_CASE_INSENSITIVE;
expr_rule->patterns[i].case_sensitive = HS_CASE_INSENSITIVE;
}
expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type);
expr_rule->patterns[i].pattern_type = expr_type2pattern_type(expr_item->expr_type);
if (TRUE == expr_item->is_hexbin &&
expr_rule->patterns[i].type != EXPR_PATTERN_TYPE_REG) {
expr_rule->patterns[i].pattern_type != HS_PATTERN_TYPE_REG) {
region_str_len = strlen(sub_key_array[i]) * 8 + 1;
region_string = ALLOC(char, region_str_len);
region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]),
@@ -741,13 +738,13 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
}
expr_rule->patterns[i].match_mode = expr_item->match_mode;
if (expr_rule->patterns[i].match_mode == EXPR_MATCH_MODE_SUB) {
if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) {
expr_rule->patterns[i].start_offset = key_left_offset[i];
expr_rule->patterns[i].end_offset = key_right_offset[i];
}
}
expr_rule->expr_id = expr_item->item_id;
expr_rule->tag = expr_item->user_data;
expr_rule->user_tag = expr_item->user_data;
expr_rule->n_patterns = sub_expr_cnt;
return 0;
@@ -813,10 +810,10 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
return 0;
}
static void garbage_expr_matcher_free(void *expr_matcher, void *arg)
static void garbage_adapter_hs_free(void *adapter_hs, void *arg)
{
struct expr_matcher *matcher = (struct expr_matcher *)expr_matcher;
expr_matcher_free(matcher);
struct adapter_hs *hs = (struct adapter_hs *)adapter_hs;
adapter_hs_free(hs);
}
int expr_runtime_commit(void *expr_runtime, const char *table_name,
@@ -867,42 +864,38 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
}
}
struct expr_matcher *new_matcher = NULL;
struct expr_matcher *old_matcher = NULL;
struct adapter_hs *new_adapter_hs = NULL;
struct adapter_hs *old_adapter_hs = NULL;
if (rule_cnt > 0) {
enum expr_engine_type engine_type = EXPR_ENGINE_TYPE_HS;
if (expr_rt->expr_engine == MAAT_EXPR_ENGINE_RS) {
engine_type = EXPR_ENGINE_TYPE_RS;
}
new_matcher = expr_matcher_new(rules, real_rule_cnt, engine_type,
expr_rt->n_worker_thread, expr_rt->logger);
if (NULL == new_matcher) {
new_adapter_hs = adapter_hs_new(rules, real_rule_cnt, expr_rt->n_worker_thread,
expr_rt->logger);
if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild expr_matcher failed when update"
"[%s:%d] table[%s] rebuild adapter_hs engine failed when update"
" %zu expr rules", __FUNCTION__, __LINE__, table_name, real_rule_cnt);
ret = -1;
} else {
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, maat_rt_version);
}
}
old_matcher = expr_rt->matcher;
expr_rt->matcher = new_matcher;
old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs;
rcu_hash_commit(expr_rt->item_hash);
if (old_matcher != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_matcher, NULL, garbage_expr_matcher_free);
if (old_adapter_hs != NULL) {
maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, NULL,
garbage_adapter_hs_free);
}
expr_rt->rule_num = real_rule_cnt;
expr_rt->regex_rule_num = real_regex_rule_cnt;
expr_rt->version = maat_rt_version;
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, expr_rt->version);
if (rules != NULL) {
for (i = 0; i < rule_cnt; i++) {
expr_rule_reset(&rules[i]);
@@ -956,14 +949,15 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
return 0;
}
if (NULL == expr_rt->matcher) {
if (NULL == expr_rt->hs) {
return 0;
}
size_t n_hit_item = 0;
struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = expr_matcher_match(expr_rt->matcher, thread_id, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM,
&n_hit_item);
if (ret < 0) {
return -1;
}
@@ -1006,15 +1000,14 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
return real_hit_item_cnt;
}
struct expr_matcher_stream *
struct adapter_hs_stream *
expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
{
if (NULL == expr_rt || thread_id < 0) {
return NULL;
}
struct expr_matcher_stream *stream = expr_matcher_stream_open(expr_rt->matcher,
thread_id);
struct adapter_hs_stream *stream = adapter_hs_stream_open(expr_rt->hs, thread_id);
if (NULL == stream) {
return NULL;
}
@@ -1023,7 +1016,7 @@ expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
}
int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
struct expr_matcher_stream *s_handle,
struct adapter_hs_stream *s_handle,
const char *data, size_t data_len,
int vtable_id, struct maat_state *state)
{
@@ -1033,10 +1026,10 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
}
size_t n_hit_item = 0;
struct expr_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
int ret = expr_matcher_stream_match(s_handle, data, data_len, hit_results,
MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results,
MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
if (ret < 0) {
return -1;
}
@@ -1074,13 +1067,13 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt,
}
void expr_runtime_stream_close(struct expr_runtime *expr_rt, int thread_id,
struct expr_matcher_stream *stream)
struct adapter_hs_stream *s_handle)
{
if (NULL == expr_rt || thread_id < 0 || NULL == stream) {
if (NULL == expr_rt || thread_id < 0 || NULL == s_handle) {
return;
}
expr_matcher_stream_close(stream);
adapter_hs_stream_close(s_handle);
}
void expr_runtime_hit_inc(struct expr_runtime *expr_rt, int thread_id)
@@ -1173,4 +1166,4 @@ long long expr_runtime_scan_bytes(struct expr_runtime *expr_rt)
alignment_int64_array_reset(expr_rt->scan_bytes, expr_rt->n_worker_thread);
return sum;
}
}

6
src/maat_flag.c
View File

@@ -269,13 +269,13 @@ static int flag_runtime_get_district_id(struct flag_runtime *flag_rt,
{
long long district_id = DISTRICT_ANY;
int map_ret = maat_kv_read(flag_rt->district_map, district, &district_id, 1);
int map_ret = maat_kv_read(flag_rt->district_map, district, &district_id);
if (map_ret < 0) {
if (NULL == flag_rt->tmp_district_map) {
flag_rt->tmp_district_map = maat_kv_store_duplicate(flag_rt->district_map);
}
map_ret = maat_kv_read(flag_rt->tmp_district_map, district, &district_id, 1);
map_ret = maat_kv_read(flag_rt->tmp_district_map, district, &district_id);
if (map_ret < 0) {
district_id = flag_rt->district_num;
maat_kv_register(flag_rt->tmp_district_map, district, district_id);
@@ -293,7 +293,7 @@ int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *dis
return -1;
}
return maat_kv_read_unNull(flag_rt->district_map, district, district_len, district_id, 1);
return maat_kv_read_unNull(flag_rt->district_map, district, district_len, district_id);
}
static struct flag_item *

6
src/maat_interval.c
View File

@@ -242,13 +242,13 @@ static int interval_runtime_get_district_id(struct interval_runtime *interval_rt
{
long long district_id = DISTRICT_ANY;
int map_ret = maat_kv_read(interval_rt->district_map, district, &district_id, 1);
int map_ret = maat_kv_read(interval_rt->district_map, district, &district_id);
if (map_ret < 0) {
if (NULL == interval_rt->tmp_district_map) {
interval_rt->tmp_district_map = maat_kv_store_duplicate(interval_rt->district_map);
}
map_ret = maat_kv_read(interval_rt->tmp_district_map, district, &district_id, 1);
map_ret = maat_kv_read(interval_rt->tmp_district_map, district, &district_id);
if (map_ret < 0) {
district_id = interval_rt->district_num;
maat_kv_register(interval_rt->tmp_district_map, district, district_id);
@@ -268,7 +268,7 @@ int interval_runtime_set_scan_district(struct interval_runtime *interval_rt,
}
return maat_kv_read_unNull(interval_rt->district_map, district,
district_len, district_id, 1);
district_len, district_id);
}
static struct interval_item *

101
src/maat_kv.c
View File

@@ -17,16 +17,11 @@
#define MAAT_KV_MAX_KEY_LEN 512
#define MAX_ENTITY_ID_CNT 4
struct kv_entity {
long long id[MAX_ENTITY_ID_CNT];
int id_cnt; //0~4
};
struct maat_kv_pair
{
char* key; //must be lower case.
size_t key_len;
struct kv_entity *val;
long long val;
UT_hash_handle hh;
};
@@ -43,32 +38,15 @@ static void strlowercase(const char* src, size_t src_len, char* dst, size_t dst_
}
static struct maat_kv_pair *
maat_kv_pair_new(const char* key, size_t keylen, long long value)
maat_kv_pair_new(const char* key, size_t key_len, long long value)
{
struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1);
kv->key = ALLOC(char, keylen);
kv->key_len = keylen;
kv->val = ALLOC(struct kv_entity, 1);
strlowercase(key, keylen, kv->key, kv->key_len);
kv->val->id[0] = value;
kv->val->id_cnt = 1;
return kv;
}
static struct maat_kv_pair *
maat_kv_pair_clone(const char *key, size_t key_len, struct kv_entity *entity)
{
struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1);
kv->key = ALLOC(char, key_len);
kv->key_len = key_len;
kv->val = ALLOC(struct kv_entity, 1);
kv->val = value;
strlowercase(key, key_len, kv->key, kv->key_len);
*(kv->val) = *entity;
return kv;
}
@@ -83,10 +61,6 @@ static void maat_kv_pair_free(struct maat_kv_pair *kv)
FREE(kv->key);
}
if (kv->val != NULL) {
FREE(kv->val);
}
FREE(kv);
}
@@ -114,23 +88,23 @@ void maat_kv_store_free(struct maat_kv_store *store)
}
static int maat_kv_register_unNull(struct maat_kv_store *store, const char *key,
size_t keylen, long long value)
size_t key_len, long long value)
{
if (keylen > MAAT_KV_MAX_KEY_LEN) {
if (key_len > MAAT_KV_MAX_KEY_LEN) {
return -1;
}
struct maat_kv_pair *kv = NULL;
struct maat_kv_pair *tmp_kv = NULL;
kv = maat_kv_pair_new(key, keylen, value);
HASH_FIND(hh, store->hash, kv->key, keylen, tmp_kv);
kv = maat_kv_pair_new(key, key_len, value);
HASH_FIND(hh, store->hash, kv->key, key_len, tmp_kv);
if (tmp_kv) {
maat_kv_pair_free(kv);
return -1;
}
HASH_ADD_KEYPTR(hh, store->hash, kv->key, keylen, kv);
HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv);
return 1;
}
@@ -142,34 +116,31 @@ int maat_kv_register(struct maat_kv_store* store, const char *key, long long val
return ret;
}
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t keylen,
long long *value_array, size_t n_array)
int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_len,
long long *value)
{
struct maat_kv_pair *kv = NULL;
char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0};
if (keylen > MAAT_KV_MAX_KEY_LEN) {
if (key_len > MAAT_KV_MAX_KEY_LEN) {
return -1;
}
strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase));
HASH_FIND(hh, store->hash, key_lowercase, keylen, kv);
strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase));
HASH_FIND(hh, store->hash, key_lowercase, key_len, kv);
int i = 0;
if (kv) {
for (i = 0; i < kv->val->id_cnt && i < (int)n_array; i++) {
value_array[i] = kv->val->id[i];
}
return i;
*value = kv->val;
return 1;
} else {
return -1;
}
}
int maat_kv_read(struct maat_kv_store *store, const char * key,
long long *value_array, size_t n_array)
int maat_kv_read(struct maat_kv_store *store, const char * key, long long *value)
{
return maat_kv_read_unNull(store, key, strlen(key), value_array, n_array);
return maat_kv_read_unNull(store, key, strlen(key), value);
}
int maat_kv_write_unNull(struct maat_kv_store* store, const char* key,
@@ -185,8 +156,7 @@ int maat_kv_write_unNull(struct maat_kv_store* store, const char* key,
strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase));
HASH_FIND(hh, store->hash, key_lowercase, key_len, kv);
if (kv) {
kv->val->id[0] = value;
kv->val->id_cnt = 1;
kv->val = value;
} else {
kv = maat_kv_pair_new(key, key_len, value);
HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv);
@@ -200,46 +170,13 @@ int maat_kv_write(struct maat_kv_store *store, const char *key, long long value)
return maat_kv_write_unNull(store, key, strlen(key), value);
}
int maat_kv_append_unNull(struct maat_kv_store* store, const char* key,
size_t key_len, long long value)
{
struct maat_kv_pair *kv = NULL;
char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0};
if (key_len > MAAT_KV_MAX_KEY_LEN) {
return -1;
}
strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase));
HASH_FIND(hh, store->hash, key_lowercase, key_len, kv);
if (kv) {
size_t id_cnt = kv->val->id_cnt;
if (id_cnt >= MAX_ENTITY_ID_CNT) {
return -1;
}
kv->val->id[id_cnt] = value;
kv->val->id_cnt++;
} else {
kv = maat_kv_pair_new(key, key_len, value);
HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv);
}
return 1;
}
int maat_kv_append(struct maat_kv_store *store, const char *key, long long value)
{
return maat_kv_append_unNull(store, key, strlen(key), value);
}
struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map)
{
struct maat_kv_store *target = maat_kv_store_new();
struct maat_kv_pair *kv = NULL, *tmp_kv = NULL, *copy_kv = NULL;
HASH_ITER (hh, origin_map->hash, kv, tmp_kv) {
copy_kv = maat_kv_pair_clone(kv->key, kv->key_len, kv->val);
copy_kv = maat_kv_pair_new(kv->key, kv->key_len, kv->val);
HASH_ADD_KEYPTR(hh, target->hash, copy_kv->key, copy_kv->key_len, copy_kv);
}

54
src/maat_rule.c
View File

@@ -127,7 +127,8 @@ static int maat_update_cb(const char *table_name, const char *line, void *u_para
return 0;
}
struct maat *maat_inst =(struct maat *)u_param;
struct maat *maat_inst =(struct maat *)u_param;
struct maat_runtime* maat_rt = NULL;
int table_id = table_manager_get_table_id(maat_inst->tbl_mgr, table_name);
if (table_id < 0) {
log_error(maat_inst->logger, MODULE_MAAT_RULE,
@@ -136,36 +137,23 @@ static int maat_update_cb(const char *table_name, const char *line, void *u_para
return -1;
}
int update_type = MAAT_UPDATE_TYPE_INC;
if (maat_inst->creating_maat_rt != NULL) { // Full update
void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id);
if (NULL == schema) {
log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] update warning, table name %s doesn't have table schema",
__FUNCTION__, __LINE__, table_name);
return -1;
}
int update_type = MAAT_UPDATE_TYPE_INC;
if (maat_inst->creating_maat_rt != NULL) { //Full update
maat_rt = maat_inst->creating_maat_rt;
update_type = MAAT_UPDATE_TYPE_FULL;
}
// find conjunction id for table_id
long long conj_parent_table_ids[4];
int conj_parent_table_cnt = table_manager_get_conj_parent_table_ids(maat_inst->tbl_mgr, table_name,
conj_parent_table_ids, 4);
if (conj_parent_table_cnt > 0) {
for (int i = 0; i < conj_parent_table_cnt; i++) {
int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name,
(int)conj_parent_table_ids[i], line, update_type);
if (ret < 0) {
log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] table<%s> update runtime error for rule:%s",
__FUNCTION__, __LINE__, table_name, line);
continue;
}
}
} else {
int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name,
table_id, line, update_type);
if (ret < 0) {
log_error(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] table<%s> update runtime error for rules:%s",
__FUNCTION__, __LINE__, table_name, line);
return -1;
}
}
} else {
maat_rt = maat_inst->maat_rt;
}
table_manager_update_runtime(maat_rt->ref_tbl_mgr, table_name, table_id, line, update_type);
return 0;
}
@@ -213,6 +201,10 @@ static void maat_plugin_table_garbage_collect_routine(struct table_manager *tbl_
table_type = table_manager_get_table_type(tbl_mgr, i);
switch (table_type) {
case TABLE_TYPE_COMPILE:
runtime = table_manager_get_runtime(tbl_mgr, i);
compile_runtime_garbage_collect_routine(runtime);
break;
case TABLE_TYPE_PLUGIN:
runtime = table_manager_get_runtime(tbl_mgr, i);
ex_data_rt = plugin_runtime_get_ex_data_rt(runtime);
@@ -353,7 +345,7 @@ long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *ke
}
long long sequence = 1;
int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence, 1);
int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence);
if (map_ret < 0) {
maat_kv_register(maat_rt->sequence_map, key, sequence);
} else {

318
src/maat_table.c
View File

@@ -47,11 +47,9 @@ struct table_manager {
struct rule_tag *accept_tags;
size_t n_accept_tag;
enum maat_expr_engine expr_engine;
int default_compile_table_id;
int g2g_table_id;
struct maat_kv_store *tablename2id_map;
struct maat_kv_store *conj_tablename2id_map;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
@@ -518,7 +516,7 @@ maat_table_new(cJSON *json, struct maat_kv_store *reserved_word_map,
}
ret = maat_kv_read(reserved_word_map, item->valuestring,
(long long *)&(ptable->table_type), 1);
(long long *)&(ptable->table_type));
if (ret < 0) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table:%s table_type %s is illegal",
@@ -580,159 +578,90 @@ static void maat_table_free(struct maat_table *maat_tbl)
FREE(maat_tbl);
}
int register_single_tablename2id(struct maat_kv_store *tablename2id_map, const char *table_name,
int table_id, struct log_handle *logger)
static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map,
struct log_handle *logger)
{
if (strlen(table_name) >= NAME_MAX) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table:<%s> name length exceed maxium:%d",
__FUNCTION__, __LINE__, table_name, NAME_MAX);
return -1;
}
long long tmp_table_id = -1;
int ret = maat_kv_read(tablename2id_map, table_name, &tmp_table_id, 1);
if (ret > 0 && tmp_table_id != table_id) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table:<%s>(table_id:%lld) has already been registered"
", can't register again",
__FUNCTION__, __LINE__, table_name, tmp_table_id);
return -1;
}
maat_kv_register(tablename2id_map, table_name, table_id);
return 0;
}
static int register_conjunction_tablename2id(cJSON *root, struct maat_kv_store *conj_tablename2id_map,
struct log_handle *logger)
{
int json_array_size = cJSON_GetArraySize(root);
for (int i = 0; i < json_array_size; i++) {
cJSON *json = cJSON_GetArrayItem(root, i);
cJSON *item = cJSON_GetObjectItem(json, "table_id");
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "db_tables");
if (NULL == item || item->type != cJSON_Array) {
continue;
}
int n_table_name = cJSON_GetArraySize(item);
for (int j = 0; j < n_table_name; j++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, j);
if (NULL == tmp_item || tmp_item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) db_tables element format invalid"
", should be string", __FUNCTION__, __LINE__, table_id);
return -1;
}
int ret = maat_kv_append(conj_tablename2id_map, tmp_item->valuestring, table_id);
if (ret < 0) {
return -1;
}
}
}
return 0;
}
static int register_tablename2id(cJSON *root, struct maat_kv_store *tablename2id_map,
const char *table_info_path, struct log_handle *logger)
{
int i = 0;
int json_array_size = cJSON_GetArraySize(root);
//pre register tablename2id
for (i = 0; i < json_array_size; i++) {
cJSON *json = cJSON_GetArrayItem(root, i);
if (NULL == json || json->type != cJSON_Object) {
log_error(logger, MODULE_TABLE,
"[%s:%d] %s has invalid json object",
__FUNCTION__, __LINE__, table_info_path);
return -1;
}
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
log_error(logger, MODULE_TABLE,
"[%s:%d] %s has invalid json object, happened in table_id column ",
__FUNCTION__, __LINE__, table_info_path);
return -1;
}
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "table_name");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] %s has invalid json object, happened in table_name column",
__FUNCTION__, __LINE__, table_info_path);
return -1;
}
int ret = register_single_tablename2id(tablename2id_map, item->valuestring,
table_id, logger);
if (ret < 0) {
return -1;
}
}
//register db_tables's table_name
for (i = 0; i < json_array_size; i++) {
cJSON *json = cJSON_GetArrayItem(root, i);
cJSON *item = cJSON_GetObjectItem(json, "table_id");
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "db_tables");
if (NULL == item || item->type != cJSON_Array) {
continue;
}
int n_table_name = cJSON_GetArraySize(item);
for (int j = 0; j < n_table_name; j++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, j);
if (NULL == tmp_item || tmp_item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) db_tables element format invalid"
", should be string", __FUNCTION__, __LINE__, table_id);
return -1;
}
long long tmp_table_id = -1;
int ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &tmp_table_id, 1);
if (ret > 0) {
continue;
}
ret = register_single_tablename2id(tablename2id_map, tmp_item->valuestring,
table_id, logger);
if (ret < 0) {
return -1;
}
}
}
return 0;
}
int maat_default_compile_table_id(cJSON *json, struct log_handle *logger)
{
cJSON *item = cJSON_GetObjectItem(json, "default_compile_table");
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
return -1;
}
int table_id = item->valueint;
item = cJSON_GetObjectItem(json, "table_id");
//item is cJSON_Number which has been checked in maat_table_new
return item->valueint;
item = cJSON_GetObjectItem(json, "db_tables");
if (item != NULL && item->type != cJSON_Array) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) has db_tables, but format is invalid"
", should be array", __FUNCTION__, __LINE__, table_id);
return -1;
}
int ret = 0;
long long tmp_table_id = 0;
if (item != NULL) {
int n_table_name = cJSON_GetArraySize(item);
for (int i = 0; i < n_table_name; i++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, i);
if (NULL == tmp_item || tmp_item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) db_tables element format invalid"
", should be string", __FUNCTION__, __LINE__, table_id);
return -1;
}
if (strlen(tmp_item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) db_tables string %s length "
"exceed maxium:%d", __FUNCTION__, __LINE__, table_id,
tmp_item->valuestring, NAME_MAX);
return -1;
}
ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &tmp_table_id);
if (ret > 0 && tmp_table_id != table_id) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table:<%s>(table_id:%lld) has already been registered"
", can't register again", __FUNCTION__, __LINE__,
tmp_item->valuestring, tmp_table_id);
return -1;
}
maat_kv_register(tablename2id_map, tmp_item->valuestring, table_id);
}
}
item = cJSON_GetObjectItem(json, "table_name");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) has no table_name",
__FUNCTION__, __LINE__, table_id);
return -1;
}
if (strlen(item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table(table_id:%d) table_name %s length too long",
__FUNCTION__, __LINE__, table_id, item->valuestring);
return -1;
}
ret = maat_kv_read(tablename2id_map, item->valuestring, &tmp_table_id);
if (ret > 0 && tmp_table_id != table_id) {
log_error(logger, MODULE_TABLE,
"[%s:%d] table:<%s>(table_id:%lld) has already been registered,"
" can't register again", __FUNCTION__, __LINE__, item->valuestring,
tmp_table_id);
return -1;
}
maat_kv_register(tablename2id_map, item->valuestring, table_id);
return 0;
}
struct table_manager *
table_manager_create(const char *table_info_path, const char *accept_tags,
enum maat_expr_engine expr_engine, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
struct maat_garbage_bin *garbage_bin, struct log_handle *logger)
{
if (NULL == table_info_path) {
return NULL;
@@ -773,28 +702,23 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger);
tbl_mgr->logger = logger;
tbl_mgr->tablename2id_map = maat_kv_store_new();
tbl_mgr->conj_tablename2id_map = maat_kv_store_new();
tbl_mgr->expr_engine = expr_engine;
tbl_mgr->ref_garbage_bin = garbage_bin;
ret = register_tablename2id(root, tbl_mgr->tablename2id_map, table_info_path, logger);
if (ret < 0) {
log_error(logger, MODULE_TABLE,
"[%s:%d] register_tablename2id failed.", __FUNCTION__, __LINE__);
FREE(json_buff);
cJSON_Delete(root);
table_manager_destroy(tbl_mgr);
return NULL;
}
for (int i = 0; i < json_array_size; i++) {
json = cJSON_GetArrayItem(root, i);
ret = register_conjunction_tablename2id(root, tbl_mgr->conj_tablename2id_map, logger);
if (ret < 0) {
log_error(logger, MODULE_TABLE,
"[%s:%d] register_conjunction_tablename2id failed.", __FUNCTION__, __LINE__);
FREE(json_buff);
cJSON_Delete(root);
table_manager_destroy(tbl_mgr);
return NULL;
if (json != NULL && json->type == cJSON_Object) {
ret = register_tablename2id(json, tbl_mgr->tablename2id_map, logger);
if (ret < 0) {
log_error(logger, MODULE_TABLE,
"[%s:%d] register_tablename2id failed",
__FUNCTION__, __LINE__);
FREE(json_buff);
cJSON_Delete(root);
table_manager_destroy(tbl_mgr);
return NULL;
}
}
}
int default_compile_table_id = -1;
@@ -812,27 +736,22 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
goto next;
}
long long parent_table_ids[4];
int parent_table_cnt = table_manager_get_conj_parent_table_ids(tbl_mgr, maat_tbl->table_name,
parent_table_ids, 4);
if (parent_table_cnt <= 0) {
// if table has conjuncion parent, which can share schema and
// runtime with parent.
maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type,
tbl_mgr, logger);
if (NULL == maat_tbl->schema) {
log_error(logger, MODULE_TABLE,
"[%s:%d] Maat table schema new failed, table_name:%s",
__FUNCTION__, __LINE__, maat_tbl->table_name);
ret = -1;
goto next;
}
maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name,
maat_tbl->table_type, tbl_mgr, logger);
if (NULL == maat_tbl->schema) {
log_error(logger, MODULE_TABLE,
"[%s:%d] Maat table schema new failed, table_name:%s",
__FUNCTION__, __LINE__, maat_tbl->table_name);
ret = -1;
goto next;
}
if (maat_tbl->table_type == TABLE_TYPE_COMPILE) {
if (default_compile_table_id < 0) {
default_compile_table_id = maat_default_compile_table_id(json, logger);
}
default_compile_table_id = maat_tbl->table_id;
} else if (maat_tbl->table_id < default_compile_table_id) {
default_compile_table_id = maat_tbl->table_id;
}
}
if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) {
@@ -997,11 +916,6 @@ void table_manager_destroy(struct table_manager *tbl_mgr)
tbl_mgr->tablename2id_map = NULL;
}
if (tbl_mgr->conj_tablename2id_map != NULL) {
maat_kv_store_free(tbl_mgr->conj_tablename2id_map);
tbl_mgr->conj_tablename2id_map = NULL;
}
FREE(tbl_mgr);
}
@@ -1019,14 +933,14 @@ size_t table_manager_table_count(struct table_manager *tbl_mgr)
return tbl_mgr->n_table;
}
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_name)
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
{
if (NULL == tbl_mgr || NULL == table_name) {
if (NULL == tbl_mgr || NULL == name) {
return -1;
}
long long table_id = -1;
int ret = maat_kv_read(tbl_mgr->tablename2id_map, table_name, &table_id, 1);
int ret = maat_kv_read(tbl_mgr->tablename2id_map, name, &table_id);
if (ret < 0) {
return -1;
}
@@ -1034,17 +948,6 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_
return (int)table_id;
}
int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name,
long long *table_ids_array, size_t n_table_ids_array)
{
if (NULL == tbl_mgr || NULL == table_name) {
return -1;
}
return maat_kv_read(tbl_mgr->conj_tablename2id_map, table_name,
table_ids_array, n_table_ids_array);
}
const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id)
{
if (NULL == tbl_mgr || table_id < 0) {
@@ -1115,15 +1018,6 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->valid_column;
}
enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr)
{
if (NULL == tbl_mgr) {
return MAAT_EXPR_ENGINE_HS;
}
return tbl_mgr->expr_engine;
}
size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr)
{
if (NULL == tbl_mgr) {