From c65ab26536b10f45ab83330d99dffda282685bb2 Mon Sep 17 00:00:00 2001 From: liuwentan Date: Thu, 1 Jun 2023 14:47:20 +0800 Subject: [PATCH] [optimize]delayed allocation for maat_compile_state --- src/inc_internal/maat_virtual.h | 2 +- src/maat_api.c | 23 +---- src/maat_compile.c | 4 + src/maat_table.c | 160 ++++++++++++++++++-------------- src/maat_virtual.c | 16 +++- 5 files changed, 116 insertions(+), 89 deletions(-) diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h index 4b6479a..191cc9d 100644 --- a/src/inc_internal/maat_virtual.h +++ b/src/inc_internal/maat_virtual.h @@ -38,7 +38,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, void virtual_schema_free(void *virtual_schema); -int virtual_table_get_physical_table_id(void *virtual_schema); +int vtable_get_physical_table_id(struct table_manager *tbl_mgr, int vtable_id); #ifdef __cplusplus } diff --git a/src/maat_api.c b/src/maat_api.c index 7698891..6939c4a 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -869,7 +869,7 @@ int maat_bool_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, static inline int scan_status_should_compile_NOT(struct maat_state *state) { - if (state && (LAST_SCAN_SET == state->is_last_scan) && state->compile_state + if (state && (LAST_SCAN_SET == state->is_last_scan) && state->compile_state && maat_compile_state_has_NOT_clause(state->compile_state)) { return 1; } @@ -885,20 +885,6 @@ size_t hit_group_to_compile(void *compile_runtime, long long *compile_ids, return n_hit_compile; } -static int vtable_get_physical_table_id(struct table_manager *tbl_mgr, int table_id) -{ - enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); - - if (table_type != TABLE_TYPE_VIRTUAL) { - return -1; - } - - // find physical table id - void *virtual_schema = table_manager_get_schema(tbl_mgr, table_id); - assert(virtual_schema != NULL); - return virtual_table_get_physical_table_id(virtual_schema); -} - int flag_scan(struct table_manager *tbl_mgr, int thread_id, long long flag, int physical_table_id, int vtable_id, struct maat_state *state) { @@ -1683,7 +1669,7 @@ struct maat_state *maat_state_new(struct maat *maat_instance, int thread_id) state->district_id = DISTRICT_ANY; state->thread_id = thread_id; - state->compile_state = maat_compile_state_new(thread_id); + //state->compile_state no need to alloc memory at this point, but alloc it after hitting items alignment_int64_array_add(maat_instance->stat->maat_state_cnt, thread_id, 1); return state; @@ -1701,8 +1687,9 @@ void maat_state_reset(struct maat_state *state) state->district_id = DISTRICT_ANY; state->scan_cnt = 0; - assert(state->compile_state != NULL); - maat_compile_state_reset(state->compile_state); + if (state->compile_state != NULL) { + maat_compile_state_reset(state->compile_state); + } } void maat_state_free(struct maat_state *state) diff --git a/src/maat_compile.c b/src/maat_compile.c index a58f56e..8ea93b4 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -2147,6 +2147,10 @@ void maat_compile_state_update(int vtable_id, struct maat_item *hit_items, hit_cnt = MAX_SCANNER_HIT_GROUP_NUM; } + if (NULL == state->compile_state) { + state->compile_state = maat_compile_state_new(state->thread_id); + } + for (i = 0; i < hit_cnt; i++) { maat_compile_state_update_hit_path(state->compile_state, hit_items[i].item_id, hit_items[i].group_id, vtable_id, state->scan_cnt, i); diff --git a/src/maat_table.c b/src/maat_table.c index 2924b02..3d25370 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -477,70 +477,6 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map) maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); } -static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map, - struct log_handle *logger) -{ - cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (NULL == item || item->type != cJSON_Number) { - return -1; - } - int table_id = item->valueint; - - item = cJSON_GetObjectItem(json, "db_tables"); - if (item != NULL && item->type != cJSON_Array) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) has db_tables, but format is invalid, should be array", - __FUNCTION__, __LINE__, table_id); - return -1; - } - - if (item != NULL) { - int n_table_name = cJSON_GetArraySize(item); - - for (int i = 0; i < n_table_name; i++) { - cJSON *tmp_item = cJSON_GetArrayItem(item, i); - if (NULL == tmp_item || tmp_item->type != cJSON_String) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) db_tables element format invalid, should be string", - __FUNCTION__, __LINE__, table_id); - return -1; - } - - if (strlen(tmp_item->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) db_tables element string %s length too long", - __FUNCTION__, __LINE__, table_id, tmp_item->valuestring); - return -1; - } - - maat_kv_register(tablename2id_map, tmp_item->valuestring, table_id); - log_info(logger, MODULE_TABLE, "table_name[%s] -> table_id:[%d]", - tmp_item->valuestring, table_id); - } - } - - item = cJSON_GetObjectItem(json, "table_name"); - if (NULL == item || item->type != cJSON_String) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) has no table_name", - __FUNCTION__, __LINE__, table_id); - return -1; - } - - if (strlen(item->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) table_name %s length too long", - __FUNCTION__, __LINE__, table_id, item->valuestring); - return -1; - } - - maat_kv_register(tablename2id_map, item->valuestring, table_id); - log_info(logger, MODULE_TABLE, "table_name[%s] -> table_id:[%d]", - item->valuestring, table_id); - - return 0; -} - struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_word_map, struct log_handle *logger) { @@ -639,6 +575,81 @@ void maat_table_free(struct maat_table *maat_tbl) FREE(maat_tbl); } +static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map, + struct log_handle *logger) +{ + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + return -1; + } + int table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "db_tables"); + if (item != NULL && item->type != cJSON_Array) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) has db_tables, but format is invalid" + ", should be array", __FUNCTION__, __LINE__, table_id); + return -1; + } + + long long tmp_table_id = 0; + if (item != NULL) { + int n_table_name = cJSON_GetArraySize(item); + + for (int i = 0; i < n_table_name; i++) { + cJSON *tmp_item = cJSON_GetArrayItem(item, i); + if (NULL == tmp_item || tmp_item->type != cJSON_String) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) db_tables element format invalid" + ", should be string", __FUNCTION__, __LINE__, table_id); + return -1; + } + + if (strlen(tmp_item->valuestring) >= NAME_MAX) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) db_tables string %s length exceed maxium:%d", + __FUNCTION__, __LINE__, table_id, tmp_item->valuestring, NAME_MAX); + return -1; + } + + if (maat_kv_read(tablename2id_map, tmp_item->valuestring, &tmp_table_id) > 0) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table:<%s>(table_id:%lld) has already been registered, can't register again", + __FUNCTION__, __LINE__, tmp_item->valuestring, tmp_table_id); + return -1; + } + + maat_kv_register(tablename2id_map, tmp_item->valuestring, table_id); + } + } + + item = cJSON_GetObjectItem(json, "table_name"); + if (NULL == item || item->type != cJSON_String) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) has no table_name", + __FUNCTION__, __LINE__, table_id); + return -1; + } + + if (strlen(item->valuestring) >= NAME_MAX) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) table_name %s length too long", + __FUNCTION__, __LINE__, table_id, item->valuestring); + return -1; + } + + if (maat_kv_read(tablename2id_map, item->valuestring, &tmp_table_id) > 0) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table:<%s>(table_id:%lld) has already been registered, can't register again", + __FUNCTION__, __LINE__, item->valuestring, tmp_table_id); + return -1; + } + + maat_kv_register(tablename2id_map, item->valuestring, table_id); + + return 0; +} + struct table_manager * table_manager_create(const char *table_info_path, const char *accept_tags, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) @@ -667,7 +678,7 @@ table_manager_create(const char *table_info_path, const char *accept_tags, FREE(json_buff); return NULL; } - + int json_array_size = cJSON_GetArraySize(root); if (json_array_size <= 0) { log_error(logger, MODULE_TABLE, @@ -677,7 +688,7 @@ table_manager_create(const char *table_info_path, const char *accept_tags, cJSON_Delete(root); return NULL; } - + struct table_manager *tbl_mgr = ALLOC(struct table_manager, 1); tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger); tbl_mgr->logger = logger; @@ -691,7 +702,8 @@ table_manager_create(const char *table_info_path, const char *accept_tags, ret = register_tablename2id(json, tbl_mgr->tablename2id_map, logger); if (ret < 0) { log_error(logger, MODULE_TABLE, - "[%s:%d] register_tablename2id failed", __FUNCTION__, __LINE__); + "[%s:%d] register_tablename2id failed", + __FUNCTION__, __LINE__); FREE(json_buff); cJSON_Delete(root); table_manager_destroy(tbl_mgr); @@ -724,8 +736,6 @@ table_manager_create(const char *table_info_path, const char *accept_tags, ret = -1; goto next; } - log_info(logger, MODULE_TABLE, "successfully register table[%s]->table_id:%d", - maat_tbl->table_name, maat_tbl->table_id); if (maat_tbl->table_type == TABLE_TYPE_COMPILE) { if (default_compile_table_id < 0) { @@ -741,6 +751,18 @@ table_manager_create(const char *table_info_path, const char *accept_tags, tbl_mgr->tbl[maat_tbl->table_id] = maat_tbl; tbl_mgr->n_table++; + + if (maat_tbl->table_type == TABLE_TYPE_VIRTUAL) { + int phy_table_id = vtable_get_physical_table_id(tbl_mgr, maat_tbl->table_id); + log_info(logger, MODULE_TABLE, + "successfully register virtual table[%s]->vtable_id:%d," + " physical_table_id:%d", maat_tbl->table_name, maat_tbl->table_id, + phy_table_id); + } else { + log_info(logger, MODULE_TABLE, + "successfully register physical table[%s]->table_id:%d", + maat_tbl->table_name, maat_tbl->table_id); + } } } diff --git a/src/maat_virtual.c b/src/maat_virtual.c index a4ce24f..865c8d4 100644 --- a/src/maat_virtual.c +++ b/src/maat_virtual.c @@ -67,7 +67,7 @@ void virtual_schema_free(void *virtual_schema) FREE(virtual_schema); } -int virtual_table_get_physical_table_id(void *virtual_schema) +static int virtual_table_get_physical_table_id(void *virtual_schema) { if (NULL == virtual_schema) { return 0; @@ -76,4 +76,18 @@ int virtual_table_get_physical_table_id(void *virtual_schema) struct virtual_schema *schema = (struct virtual_schema *)virtual_schema; return schema->physical_table_id; +} + +int vtable_get_physical_table_id(struct table_manager *tbl_mgr, int table_id) +{ + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + + if (table_type != TABLE_TYPE_VIRTUAL) { + return -1; + } + + // find physical table id + void *virtual_schema = table_manager_get_schema(tbl_mgr, table_id); + assert(virtual_schema != NULL); + return virtual_table_get_physical_table_id(virtual_schema); } \ No newline at end of file