From 5e907a171f3aa362802cc743e4ea0cb3a993fb36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=96=87=E5=9D=9B?= Date: Mon, 30 Oct 2023 08:00:49 +0000 Subject: [PATCH] [FEATURE]Compile table must register plugin table to get compile ex_data --- docs/table_schema.md | 91 +++++--- include/maat.h | 15 +- readme.md | 34 ++- src/inc_internal/maat_compile.h | 24 +- src/inc_internal/maat_kv.h | 8 +- src/inc_internal/maat_table.h | 18 +- src/json2iris.c | 4 +- src/maat_api.c | 76 +++---- src/maat_compile.c | 339 +++++++++++++---------------- src/maat_expr.c | 7 +- src/maat_flag.c | 7 +- src/maat_interval.c | 8 +- src/maat_ipport_plugin.c | 1 - src/maat_kv.c | 104 +++++++-- src/maat_rule.c | 54 +++-- src/maat_table.c | 277 +++++++++++++++-------- test/file_test_tableinfo.conf | 43 +++- test/json_update/corrupted.json | 2 +- test/json_update/new.json | 4 +- test/json_update/old.json | 4 +- test/maat_framework_gtest.cpp | 186 +++++++++------- test/maat_framework_perf_gtest.cpp | 16 +- test/maat_json.json | 100 +-------- test/table_info.conf | 207 ++++++++++++------ 24 files changed, 920 insertions(+), 709 deletions(-) diff --git a/docs/table_schema.md b/docs/table_schema.md index 180f926..e4e8f25 100644 --- a/docs/table_schema.md +++ b/docs/table_schema.md @@ -11,10 +11,12 @@ The types of physical tables are as follows: - [ip_plugin table](#6-ip_plugin-table) - [fqdn_plugin table](#7-fqdn_plugin-table) - [bool_plugin table](#8-bool_plugin-table) +- [virtual table](#9-virtual-table) +- [conjunction table](#10-conjunction-table) -Different physical tables can be combined into one table, see [conjunction table](#112-12-conjunction-table) +Different physical tables can be combined into one table, see [conjunction table](#10-conjunction-table) -A virtual table can only reference one physical table or conjuntion table, see [virtual table](#111-11-virtual-table) +A virtual table can only reference one physical table or conjuntion table, see [virtual table](#9-virtual-table) ## 1. Item table @@ -166,9 +168,33 @@ table schema stored in table_info.conf }, { "table_id":1, + "table_name":"COMPILE_ALIAS", + "table_type":"compile", + "valid_column":8, + "custom": { + "compile_id":1, + "tags":6, + "clause_num":9 + } + }, + { + "table_id":2, + "table_name":"COMPILE_CONJUNCTION", + "db_tables":["COMPILE", "COMPILE_ALIAS"], + "default_compile_table":2, //key:indicate this is the default compile table, value:can be anything(not care) + "table_type":"compile", + "valid_column":8, + "custom": { + "compile_id":1, + "tags":6, + "clause_num":9 + } + }, + { + "table_id":3, "table_name":"GROUP2COMPILE", "table_type":"group2compile", - "associated_compile_table_id":0, + "associated_compile_table_id":2, //associate compile conjunction table "valid_column":3, "custom": { "group_id":1, @@ -179,7 +205,7 @@ table schema stored in table_info.conf } }, { - "table_id":2, + "table_id":4, "table_name":"GROUP2GROUP", "table_type":"group2group", "valid_column":4, @@ -190,7 +216,7 @@ table schema stored in table_info.conf } }, { - "table_id":3, + "table_id":5, "table_name":"HTTP_URL", "table_type":"expr", "valid_column":7, @@ -677,11 +703,26 @@ Plugin table supports three types of keys(pointer, integer and ip_addr) for ex_d (3) register callback ```c +void plugin_EX_new_cb(const char *table_name, int table_id, const char *key, + const char *table_line, void **ad, long argl, void *argp); +void plugin_EX_free_cb(int table_id, void **ad, long argl, void *argp); + +void plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *argp); + +const char *table_name = "TEST_PLUGIN_POINTER_KEY_TYPE"; +int table_id = maat_get_table_id(maat_inst, table_name); + +int plugin_ex_data_counter = 0; +maat_plugin_table_ex_schema_register(maat_inst, table_name, + plugin_EX_new_cb, + plugin_EX_free_cb, + plugin_EX_dup_cb, + 0, &plugin_ex_data_counter); ``` (4) get ex_data -``` +```c const char *key1 = "HeBei"; const char *table_name = "TEST_PLUGIN_POINTER_KEY_TYPE"; @@ -694,7 +735,7 @@ maat_plugin_table_get_ex_data(maat_instance, table_id, key1, strlen(key1)); support integers of different lengths, such as int(4 bytes), long long(8 bytes). (1) schema -``` +```json { "table_id":1, "table_name":"TEST_PLUGIN_INT_KEY_TYPE", @@ -723,7 +764,7 @@ support integers of different lengths, such as int(4 bytes), long long(8 bytes). ``` (2) plugin table rules -``` +```json { "table_name": "TEST_PLUGIN_INT_KEY_TYPE", "table_content": [ @@ -745,21 +786,18 @@ support integers of different lengths, such as int(4 bytes), long long(8 bytes). } ``` -(3) get ex_data -``` +(3) register callback +same as above + +(4) get ex_data +```c //int -int key1 = 101; +int key1 = 101; //key length must be 4 bytes which specified in table_info.conf const char *table_name = "TEST_PLUGIN_INT_KEY_TYPE"; int table_id = maat_get_table_id(maat_instance, table_name); maat_plugin_table_get_ex_data(maat_instance, table_id, key1, sizeof(key1)); -//long long -long long key2 = 11111111; -const char *table_name = "TEST_PLUGIN_LONG_KEY_TYPE"; - -table_id = maat_get_table_id(maat_instance, table_name); -maat_plugin_table_get_ex_data(maat_instance, table_id, key2, sizeof(key2)); ``` **ip_addr key** @@ -767,7 +805,7 @@ maat_plugin_table_get_ex_data(maat_instance, table_id, key2, sizeof(key2)); support ip address(ipv4 or ipv6) as key. (1) schema -``` +```json { "table_id":1, "table_name":"TEST_PLUGIN_IP_KEY_TYPE", @@ -783,7 +821,7 @@ support ip address(ipv4 or ipv6) as key. The addr_type column indicates whether the key is a v4 or v6 address. (2) plugin table rules -``` +```json { "table_name": "TEST_PLUGIN_IP_KEY_TYPE", "table_content": [ @@ -795,8 +833,11 @@ The addr_type column indicates whether the key is a v4 or v6 address. } ``` -(3) get ex_data -``` +(3) register callback +same as above + +(4) get ex_data +```c uint32_t ipv4_addr; inet_pton(AF_INET, "100.64.1.1", &ipv4_addr); const char *table_name = "TEST_PLUGIN_IP_KEY_TYPE"; @@ -834,7 +875,7 @@ For example: 布尔表达式规则为“&”分隔的数字，例如“1&2&1000”。 -### 1.11 virtual table +### 9. virtual table 虚拟一个配置表，其内容为特定物理域配置表的视图。实践中，通常采用网络流量的属性作为虚拟表名，如HTTP_HOST、SSL_SNI等。一个虚拟表可以建立在多个不同类型的物理表之上，但不允许建立在其它虚拟表上。 @@ -845,7 +886,7 @@ For example: | **keyword_group_1** | compile_1 | 1 | 0 | 0 | REQUEST_BODY | | **keyword_group_1** | compile_1 | 1 | 0 | 0 | RESPONSE_BODY | -### 1.12 conjunction table +### 10. conjunction table 表名不同，但table id相同的表。旨在数据库表文件和MAAT API之间提供一个虚拟层，通过API调用一次扫描，即可扫描多张同类配置表。 @@ -858,7 +899,7 @@ For example: 支持所有类型表的连接，包括各类域配置、回调类配置。配置分组和配置编译的连接没有意义。 -## 2. Foreign Files +### 11. Foreign Files 回调类配置中，特定字段可以指向一个外部内容，目前支持指向Redis中的一个key。 @@ -876,7 +917,7 @@ For example: ​ 内容外键的声明方法，参见本文档-配置表描述文件一节。 -## 3. Tags +### 12. Tags 通过将Maat接受标签与配置标签的匹配，实现有选择的配置加载。其中配置标签是一个标签数组的集合，记为”tag_sets”，Maat接受标签是标签数组，记为”tags”。 diff --git a/include/maat.h b/include/maat.h index 061b503..d2bd748 100644 --- a/include/maat.h +++ b/include/maat.h @@ -61,11 +61,6 @@ enum maat_expr_engine { MAAT_EXPR_ENGINE_RS //rulescan }; -enum maat_list_type { - MAAT_LIST_TYPE_FULL = 1, - MAAT_LIST_TYPE_INC -}; - struct ip_addr { int ip_type; //4:IPV4, 6:IPV6 union { @@ -302,6 +297,9 @@ int maat_state_set_scan_compile_table(struct maat_state *state, int compile_tabl int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *path_array, size_t array_size); + +int maat_state_get_compile_table_ids(struct maat_state *state, long long *compile_ids, + size_t n_compile_ids, int *compile_table_ids); /** * @brief get the total number of scans after maat_state_new @@ -313,9 +311,10 @@ size_t maat_state_get_scan_count(struct maat_state *state); * * NOTE: hit groups may be duplicated */ -int maat_state_get_direct_hit_groups(struct maat_state *state, enum maat_list_type type, +int maat_state_get_direct_hit_groups(struct maat_state *state, struct maat_hit_group *group_array, size_t array_size); +size_t maat_state_get_direct_hit_group_cnt(struct maat_state *state); /** * @brief indirect group means superior group @@ -325,9 +324,7 @@ int maat_state_get_direct_hit_groups(struct maat_state *state, enum maat_list_ty int maat_state_get_indirect_hit_groups(struct maat_state *state, struct maat_hit_group *group_array, size_t array_size); - -/* return hit object compile_id */ -int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group); +size_t maat_state_get_indirect_hit_group_cnt(struct maat_state *state); #ifdef __cplusplus } diff --git a/readme.md b/readme.md index 3356040..d3a22a4 100644 --- a/readme.md +++ b/readme.md @@ -38,7 +38,7 @@ A complete use case consists of three parts ### 1. table schema Table schema is stored in a json file(such as table_info.conf), which is loaded when maat instance is created. -```shell +```json [ { "table_id":0, @@ -53,9 +53,33 @@ Table schema is stored in a json file(such as table_info.conf), which is loaded }, { "table_id":1, + "table_name":"COMPILE_ALIAS", + "table_type":"compile", + "valid_column":8, + "custom": { + "compile_id":1, + "tags":6, + "clause_num":9 + } + }, + { + "table_id":2, + "table_name":"COMPILE_CONJUNCTION", + "db_tables":["COMPILE", "COMPILE_ALIAS"], + "default_compile_table":2, //key:indicate this is the default compile table, value:can be anything(not care) + "table_type":"compile", + "valid_column":8, + "custom": { + "compile_id":1, + "tags":6, + "clause_num":9 + } + }, + { + "table_id":3, "table_name":"GROUP2COMPILE", "table_type":"group2compile", - "associated_compile_table_id":0, + "associated_compile_table_id":2, //associate compile conjunction table "valid_column":3, "custom": { "group_id":1, @@ -66,7 +90,7 @@ Table schema is stored in a json file(such as table_info.conf), which is loaded } }, { - "table_id":2, + "table_id":4, "table_name":"GROUP2GROUP", "table_type":"group2group", "valid_column":4, @@ -77,7 +101,7 @@ Table schema is stored in a json file(such as table_info.conf), which is loaded } }, { - "table_id":3, + "table_id":5, "table_name":"HTTP_URL", "table_type":"expr", "valid_column":7, @@ -95,7 +119,7 @@ Table schema is stored in a json file(such as table_info.conf), which is loaded ### 2. rule Rules are stored in a json file(such as maat_json.json), which is loaded when maat instance is created. -```shell +```json { "compile_table": "COMPILE", "group2compile_table": "GROUP2COMPILE", diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index a591d6b..042dfe5 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -37,12 +37,6 @@ void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, void group2compile_schema_free(void *g2c_schema); int group2compile_associated_compile_table_id(void *g2c_schema); -int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int table_id, - maat_ex_new_func_t *new_func, - maat_ex_free_func_t *free_func, - maat_ex_dup_func_t *dup_func, - long argl, void *argp); - /* compile runtime API */ void *compile_runtime_new(void *compile_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, @@ -70,14 +64,6 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr struct maat_hit_path *hit_path_array, size_t array_size, size_t n_hit_path); -void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt, - struct compile_schema *compile_schema, - long long compile_id); -void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt, - struct compile_schema *compile_schema); - -void compile_runtime_garbage_collect_routine(void *compile_runtime); - /* group2compile runtime API */ void *group2compile_runtime_new(void *g2c_schema, size_t max_thread_num, struct maat_garbage_bin *garbage_bin, @@ -103,7 +89,7 @@ struct compile_state *compile_state_new(void); void compile_state_reset(struct compile_state *compile_state); -void compile_state_free(struct compile_state *compile_state, +void compile_state_free(struct compile_state *compile_state, struct maat *maat_instance, int thread_id); int compile_state_update(int vtable_id, struct maat_item *hit_items, @@ -116,15 +102,19 @@ size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state, size_t array_size); size_t compile_state_get_direct_hit_groups(struct compile_state *compile_state, - enum maat_list_type type, struct maat_hit_group *group_array, size_t array_size); +size_t compile_state_get_direct_hit_group_cnt(struct compile_state *compile_state); + size_t compile_state_get_indirect_hit_groups(struct compile_state *compile_state, struct maat_hit_group *group_array, size_t array_size); -int compile_state_has_NOT_clause(struct compile_state *compile_state); +size_t compile_state_get_indirect_hit_group_cnt(struct compile_state *compile_state); + +int compile_state_get_compile_table_id(struct compile_state *compile_state, + long long compile_id); #ifdef __cplusplus } diff --git a/src/inc_internal/maat_kv.h b/src/inc_internal/maat_kv.h index 896ec1d..59040b8 100644 --- a/src/inc_internal/maat_kv.h +++ b/src/inc_internal/maat_kv.h @@ -26,12 +26,16 @@ void maat_kv_store_free(struct maat_kv_store *store); int maat_kv_register(struct maat_kv_store *store, const char *key, long long value); -int maat_kv_read(struct maat_kv_store *store, const char *key, long long *value); +int maat_kv_read(struct maat_kv_store *store, const char *key, + long long *value_array, size_t n_array); -int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, long long *value); +int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, + long long *value_array, size_t n_array); int maat_kv_write(struct maat_kv_store *store, const char *key, long long value); +int maat_kv_append(struct maat_kv_store *store, const char *key, long long value); + struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store); #ifdef __cplusplus diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index 768df6f..6c38a70 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -60,7 +60,23 @@ void table_manager_destroy(struct table_manager *tbl_mgr); size_t table_manager_table_size(struct table_manager *tbl_mgr); size_t table_manager_table_count(struct table_manager *tbl_mgr); -int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name); +int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_name); +/** + * @brief get table_name's all conjunction parents' table_id + * + * for example: "table_id": 1, + * "table_name":"COMPILE_CONJ" + * "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"] + * + * "table_id": 2, + * "table_name": "COMPILE_PLUGIN", + * "db_tables": ["COMPILE_DEFAULT"] + * + * so COMPILE_DEFAULT has two conjunction parents whose table_id is 1 and 2. +*/ +int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name, + long long *table_ids_array, size_t n_table_ids_array); + const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id); enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id); diff --git a/src/json2iris.c b/src/json2iris.c index 7e97470..2b03ddb 100644 --- a/src/json2iris.c +++ b/src/json2iris.c @@ -379,7 +379,7 @@ static int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, if (1 == cmd[i].str2int_flag) { long long int_value = 0; char *p = item->valuestring; - ret = maat_kv_read(str2int, p, &int_value); + ret = maat_kv_read(str2int, p, &int_value, 1); if (ret < 0) { log_error(logger, MODULE_JSON2IRIS, "[%s:%d] %s's value %s is not valid format", @@ -638,7 +638,7 @@ static int write_region_rule(cJSON *region_json, int compile_id, int group_id, const char *table_type_str = item->valuestring; long long table_type_int; - int ret = maat_kv_read(p_iris->str2int_map, table_type_str, &table_type_int); + int ret = maat_kv_read(p_iris->str2int_map, table_type_str, &table_type_int, 1); if (ret != 1) { log_error(logger, MODULE_JSON2IRIS, "[%s:%d] compile rule %d table name %s's table_type %s invalid", diff --git a/src/maat_api.c b/src/maat_api.c index 491fa5d..8460e19 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -569,32 +569,6 @@ int maat_table_callback_register(struct maat *maat_inst, int table_id, return 0; } -static int compile_table_ex_schema_register(struct maat *maat_inst, int table_id, - maat_ex_new_func_t *new_func, - maat_ex_free_func_t *free_func, - maat_ex_dup_func_t *dup_func, - long argl, void *argp) -{ - void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id); - assert(schema != NULL); - - int ret = compile_table_set_ex_data_schema((struct compile_schema *)schema, table_id, - new_func, free_func, dup_func, - argl, argp); - if (ret < 0) { - return -1; - } - - if (maat_inst->maat_rt != NULL) { - void *runtime = table_manager_get_runtime(maat_inst->tbl_mgr, table_id); - assert(runtime != NULL); - compile_runtime_ex_data_iterate((struct compile_runtime *)runtime, - (struct compile_schema *)schema); - } - - return 0; -} - static int generic_plugin_table_set_ex_schema(struct table_manager *tbl_mgr, int table_id, maat_ex_new_func_t *new_func, maat_ex_free_func_t *free_func, @@ -857,10 +831,9 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_inst, int ret = -1; enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id); - if (TABLE_TYPE_COMPILE == table_type) { - ret = compile_table_ex_schema_register(maat_inst, table_id, new_func, - free_func, dup_func, argl, argp); - } else { + if (table_type == TABLE_TYPE_PLUGIN || table_type == TABLE_TYPE_IP_PLUGIN || + table_type == TABLE_TYPE_IPPORT_PLUGIN || table_type == TABLE_TYPE_FQDN_PLUGIN || + table_type == TABLE_TYPE_BOOL_PLUGIN) { ret = generic_plugin_table_ex_schema_register(maat_inst, table_name, table_id, new_func, free_func, dup_func, argl, argp); @@ -902,9 +875,7 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_inst, int table_id, void *ret = NULL; enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id); - if (TABLE_TYPE_COMPILE == table_type) { - ret = compile_runtime_get_ex_data(runtime, schema, *(long long *)key); - } else if (TABLE_TYPE_PLUGIN == table_type) { + if (TABLE_TYPE_PLUGIN == table_type) { ret = plugin_runtime_get_ex_data(runtime, schema, key, key_len); } @@ -1950,6 +1921,22 @@ int maat_state_set_scan_compile_table(struct maat_state *state, int compile_tabl return 0; } +int maat_state_get_compile_table_ids(struct maat_state *state, long long *compile_ids, + size_t n_compile_ids, int *compile_table_ids) +{ + if (NULL == state || NULL == compile_ids || 0 == n_compile_ids || + NULL == compile_table_ids) { + return -1; + } + + for (size_t i = 0; i < n_compile_ids; i++) { + compile_table_ids[i] = compile_state_get_compile_table_id(state->compile_state, + compile_ids[i]); + } + + return n_compile_ids; +} + int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *path_array, size_t array_size) { @@ -1999,7 +1986,7 @@ size_t maat_state_get_scan_count(struct maat_state *state) return state->scan_cnt; } -int maat_state_get_direct_hit_groups(struct maat_state *state, enum maat_list_type type, +int maat_state_get_direct_hit_groups(struct maat_state *state, struct maat_hit_group *group_array, size_t array_size) { @@ -2011,10 +1998,19 @@ int maat_state_get_direct_hit_groups(struct maat_state *state, enum maat_list_ty return 0; } - return compile_state_get_direct_hit_groups(state->compile_state, type, + return compile_state_get_direct_hit_groups(state->compile_state, group_array, array_size); } +size_t maat_state_get_direct_hit_group_cnt(struct maat_state *state) +{ + if (NULL == state || NULL == state->compile_state) { + return 0; + } + + return compile_state_get_direct_hit_group_cnt(state->compile_state); +} + int maat_state_get_indirect_hit_groups(struct maat_state *state, struct maat_hit_group *group_array, size_t array_size) @@ -2031,7 +2027,11 @@ int maat_state_get_indirect_hit_groups(struct maat_state *state, group_array, array_size); } -int maat_hit_group_compile_id(struct maat *instance, struct maat_hit_group *group) +size_t maat_state_get_indirect_hit_group_cnt(struct maat_state *state) { - return 0; -} + if (NULL == state || NULL == state->compile_state) { + return 0; + } + + return compile_state_get_indirect_hit_group_cnt(state->compile_state); +} \ No newline at end of file diff --git a/src/maat_compile.c b/src/maat_compile.c index 8cde926..35140b9 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -28,7 +28,6 @@ #define MODULE_COMPILE module_name_str("maat.compile") -#define DEFAULT_GC_TIMEOUT_S 10 #define MAX_SUPER_GROUP_CNT 128 #define MAX_NOT_CLAUSE_NUM 8 #define VTABLE_MAX_NOT_GROUP_NUM 8 @@ -42,10 +41,7 @@ struct compile_schema { int compile_id_column; int rule_tag_column; int declared_clause_num_column; - int set_flag; - int gc_timeout_s; int table_id; //ugly - struct ex_data_schema ex_schema; struct table_manager *ref_tbl_mgr; struct log_handle *logger; }; @@ -66,8 +62,6 @@ struct compile_item { long long compile_id; char *table_line; size_t table_line_len; - struct compile_schema *ref_schema; - void **ex_data; char table_name[MAX_NAME_STR_LEN]; }; @@ -100,7 +94,8 @@ struct table_not_clause { /* compile_runtime and group2compile_runtime share compile_hash_map */ struct compile_runtime { struct bool_matcher *bm; - struct rcu_hash_table *cfg_hash; // + struct rcu_hash_table *cfg_hash; // + struct rcu_hash_table *tbl_cfg_hash; // struct maat_runtime *ref_maat_rt; time_t version; struct literal_clause *literal2clause_hash; //store clause_ids(not_flag == 0) @@ -153,6 +148,11 @@ struct internal_hit_path { int NOT_flag; // 1 means NOT clause }; +struct compile2table_id { + long long compile_id; + int table_id; +}; + struct compile_state { int Nth_scan; time_t compile_rt_version; @@ -162,12 +162,14 @@ struct compile_state { UT_array *this_scan_hit_clauses; UT_array *direct_hit_groups; UT_array *indirect_hit_groups; + UT_array *hit_compile_table_ids; }; UT_icd ut_literal_id_icd = {sizeof(struct literal_id), NULL, NULL, NULL}; UT_icd ut_clause_id_icd = {sizeof(long long), NULL, NULL, NULL}; UT_icd ut_maat_hit_group_icd = {sizeof(struct maat_hit_group), NULL, NULL, NULL}; UT_icd ut_hit_path_icd = {sizeof(struct internal_hit_path), NULL, NULL, NULL}; +UT_icd ut_hit_compile_table_id_icd = {sizeof(struct compile2table_id), NULL, NULL, NULL}; static struct maat_compile *maat_compile_new(long long compile_id) { @@ -200,23 +202,6 @@ static int maat_compile_set(struct maat_compile *compile, const char *table_name return 0; } -static void *rule_ex_data_new(const char *table_name, int table_id, - const char *table_line, - struct ex_data_schema *ex_schema) -{ - void *ex_data = NULL; - - ex_schema->new_func(table_name, table_id, NULL, table_line, &ex_data, - ex_schema->argl, ex_schema->argp); - return ex_data; -} - -static void rule_ex_data_free(int table_id, void **ex_data, - const struct ex_data_schema *ex_schema) -{ - ex_schema->free_func(table_id, ex_data, ex_schema->argl, ex_schema->argp); -} - static int compile_accept_tag_match(struct compile_schema *schema, const char *line, const char *table_name, struct log_handle *logger) { @@ -300,18 +285,11 @@ compile_item_new(const char *table_line, struct compile_schema *schema, goto error; } - compile_item->ref_schema = schema; - compile_item->ex_data = ALLOC(void *, 1); memcpy(compile_item->table_name, table_name, sizeof(compile_item->table_name)); compile_item->table_line_len = strlen(table_line); compile_item->table_line = ALLOC(char, compile_item->table_line_len + 1); memcpy(compile_item->table_line, table_line, compile_item->table_line_len); - if (1 == schema->set_flag) { - *(compile_item->ex_data) = rule_ex_data_new(table_name, schema->table_id, - compile_item->table_line, - &(schema->ex_schema)); - } return compile_item; error: FREE(compile_item); @@ -320,17 +298,6 @@ error: static void compile_item_free(struct compile_item *item) { - struct compile_schema *schema = item->ref_schema; - - if (1 == schema->set_flag) { - rule_ex_data_free(schema->table_id, item->ex_data, &(schema->ex_schema)); - *item->ex_data = NULL; - } - - if (item->ex_data != NULL) { - FREE(item->ex_data); - } - item->declared_clause_num = 0; if (item->table_line != NULL) { @@ -371,70 +338,9 @@ static void rcu_compile_cfg_free(void *user_ctx, void *data) maat_compile_free(compile); } -int compile_table_set_ex_data_schema(struct compile_schema *compile_schema, int table_id, - maat_ex_new_func_t *new_func, - maat_ex_free_func_t *free_func, - maat_ex_dup_func_t *dup_func, - long argl, void *argp) +static void rcu_compile_table_cfg_free(void *user_ctx, void *data) { - if (1 == compile_schema->set_flag) { - log_error(compile_schema->logger, MODULE_COMPILE, - "[%s:%d] compile table(table_id:%d)ex schema has been set already," - " can't set again", __FUNCTION__, __LINE__, table_id); - return -1; - } - - compile_schema->ex_schema.new_func = new_func; - compile_schema->ex_schema.free_func = free_func; - compile_schema->ex_schema.dup_func = dup_func; - compile_schema->ex_schema.argl = argl; - compile_schema->ex_schema.argp = argp; - compile_schema->set_flag = 1; - - return 0; -} - -static void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, - long long compile_id) -{ - struct maat_compile *compile = rcu_hash_find(compile_rt->cfg_hash, - (char *)&compile_id, - sizeof(long long)); - void *ret = NULL; - if (compile != NULL) { - ret = compile->user_data; - } - - return ret; -} - -static void rule_ex_data_new_cb(void *user_data, void *param, - const char *table_name, int table_id) -{ - struct ex_data_schema *ex_schema = (struct ex_data_schema *)param; - struct compile_item *compile = (struct compile_item *)user_data; - - void *ad = rule_ex_data_new(table_name, table_id, compile->table_line, ex_schema); - *compile->ex_data = ad; -} - -static void compile_runtime_user_data_iterate(struct compile_runtime *compile_rt, - void (*callback)(void *user_data, void *param, - const char *table_name, int table_id), - void *param, int table_id) -{ - /* I'm in background_update_mutex, config update can't happen, so no need to lock cfg_hash */ - void **data_array = NULL; - size_t data_cnt = rcu_hash_list(compile_rt->cfg_hash, &data_array); - - for (size_t i = 0; i < data_cnt; i++) { - struct maat_compile *compile = (struct maat_compile *)data_array[i]; - if (compile->user_data) { - callback(compile->user_data, param, compile->table_name, table_id); - } - } - - FREE(data_array); + FREE(data); } void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -488,12 +394,6 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, goto error; } - //gc_timeout_s is optional - custom_item = cJSON_GetObjectItem(item, "gc_timeout_s"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - schema->gc_timeout_s = custom_item->valueint; - } - schema->ref_tbl_mgr = tbl_mgr; return schema; error: @@ -617,14 +517,13 @@ void *compile_runtime_new(void *compile_schema, size_t max_thread_num, return NULL; } - struct compile_schema *schema = (struct compile_schema *)compile_schema; struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1); compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM); compile_rt->version = time(NULL); - compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL, - schema->gc_timeout_s + DEFAULT_GC_TIMEOUT_S); + compile_rt->cfg_hash = rcu_hash_new(rcu_compile_cfg_free, NULL, 0); + compile_rt->tbl_cfg_hash = rcu_hash_new(rcu_compile_table_cfg_free, NULL, 0); compile_rt->literal2clause_hash = NULL; compile_rt->literal2not_clause_hash = NULL; compile_rt->logger = logger; @@ -672,6 +571,11 @@ void compile_runtime_free(void *compile_runtime) compile_rt->cfg_hash = NULL; } + if (compile_rt->tbl_cfg_hash != NULL) { + rcu_hash_free(compile_rt->tbl_cfg_hash); + compile_rt->tbl_cfg_hash = NULL; + } + if (compile_rt->literal2clause_hash != NULL) { literal2clause_hash_free(compile_rt->literal2clause_hash); compile_rt->literal2clause_hash = NULL; @@ -1040,6 +944,35 @@ static inline int compare_clause_id(const void *a, const void *b) } } +static inline int compare_hit_group(const void *pa, const void *pb) +{ + struct maat_hit_group *la=(struct maat_hit_group *)pa; + struct maat_hit_group *lb=(struct maat_hit_group *)pb; + + long long ret = la->item_id - lb->item_id; + if (0 == ret) { + ret = la->group_id - lb->group_id; + if (0 == ret) { + ret = la->vtable_id - lb->vtable_id; + } + } + + return ret; +} + +static inline int compare_compile_id(const void *a, const void *b) +{ + long long ret = *(const long long *)a - *(const long long *)b; + + if (0 == ret) { + return 0; + } else if (ret < 0) { + return -1; + } else { + return 1; + } +} + /** * @brief build hash for clause or not_clause * @@ -1140,10 +1073,33 @@ static size_t compile_state_if_new_hit_compile(struct compile_state *compile_sta return r_in_c_cnt; } -size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, - struct compile_state *compile_state, - int thread_id, void **user_data_array, - size_t ud_array_size) +static void compile_state_update_hit_compile_table_id(struct compile_state *compile_state, + long long compile_id, int table_id) +{ + if (!utarray_find(compile_state->hit_compile_table_ids, &compile_id, compare_compile_id)) { + struct compile2table_id compile_table_id = {compile_id, table_id}; + utarray_push_back(compile_state->hit_compile_table_ids, &compile_table_id); + utarray_sort(compile_state->hit_compile_table_ids, compare_compile_id); + } +} + +static int compile_runtime_get_compile_table_id(struct compile_runtime *compile_rt, + long long compile_id) +{ + if (NULL == compile_rt || compile_id < 0) { + return -1; + } + + int *table_id = rcu_hash_find(compile_rt->tbl_cfg_hash, (char *)&compile_id, + sizeof(long long)); + + return *table_id; +} + +static size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, + struct compile_state *compile_state, + int thread_id, void **user_data_array, + size_t ud_array_size) { size_t ud_result_cnt = 0; struct maat_compile *compile = NULL; @@ -1186,6 +1142,11 @@ size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, if (compile->user_data != NULL && n_new_hit_compile > 0) { user_data_array[ud_result_cnt] = compile->user_data; ud_result_cnt++; + + int table_id = compile_runtime_get_compile_table_id(compile_rt, compile->compile_id); + if (table_id >= 0) { + compile_state_update_hit_compile_table_id(compile_state, compile->compile_id, table_id); + } } } @@ -1198,18 +1159,11 @@ static struct compile_item *compile_item_clone(struct compile_item *item) new_item->compile_id = item->compile_id; new_item->declared_clause_num = item->declared_clause_num; - new_item->ref_schema = item->ref_schema; - new_item->ex_data = ALLOC(void *, 1); memcpy(new_item->table_name, item->table_name, sizeof(new_item->table_name)); new_item->table_line_len = item->table_line_len; new_item->table_line = ALLOC(char, new_item->table_line_len + 1); memcpy(new_item->table_line, item->table_line, new_item->table_line_len); - if (1 == item->ref_schema->set_flag) { - *(new_item->ex_data) = rule_ex_data_new(item->table_name, item->ref_schema->table_id, - item->table_line, &(item->ref_schema->ex_schema)); - } - return new_item; } @@ -1420,6 +1374,7 @@ struct compile_state *compile_state_new(void) utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd); utarray_new(compile_state->direct_hit_groups, &ut_maat_hit_group_icd); utarray_new(compile_state->indirect_hit_groups, &ut_maat_hit_group_icd); + utarray_new(compile_state->hit_compile_table_ids, &ut_hit_compile_table_id_icd); return compile_state; } @@ -1438,6 +1393,7 @@ void compile_state_reset(struct compile_state *compile_state) utarray_clear(compile_state->this_scan_hit_clauses); utarray_clear(compile_state->direct_hit_groups); utarray_clear(compile_state->indirect_hit_groups); + utarray_clear(compile_state->hit_compile_table_ids); } void compile_state_free(struct compile_state *compile_state, @@ -1479,6 +1435,12 @@ void compile_state_free(struct compile_state *compile_state, compile_state->indirect_hit_groups = NULL; } + if (compile_state->hit_compile_table_ids != NULL) { + free_bytes += utarray_size(compile_state->hit_compile_table_ids) * sizeof(struct compile2table_id); + utarray_free(compile_state->hit_compile_table_ids); + compile_state->hit_compile_table_ids = NULL; + } + FREE(compile_state); free_bytes += sizeof(struct compile_state); @@ -1486,9 +1448,9 @@ void compile_state_free(struct compile_state *compile_state, thread_id, free_bytes); } -static void maat_compile_hit_path_add(UT_array *hit_paths, long long item_id, - long long group_id, int vtable_id, int NOT_flag, - int Nth_scan) +static void add_internal_hit_path(UT_array *hit_paths, long long item_id, + long long group_id, int vtable_id, int NOT_flag, + int Nth_scan) { if (NULL == hit_paths || utarray_len(hit_paths) >= MAX_HIT_PATH_NUM) { return; @@ -1601,9 +1563,9 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, int thr return (n_hit_path + new_hit_path_cnt); } -static void compile_state_update_direct_hit_groups(UT_array *hit_group_array, - struct maat_item *hit_items, - size_t n_hit_items, int vtable_id) +static void update_direct_hit_groups(UT_array *hit_group_array, + struct maat_item *hit_items, + size_t n_hit_items, int vtable_id) { if (NULL == hit_group_array) { return; @@ -1618,9 +1580,9 @@ static void compile_state_update_direct_hit_groups(UT_array *hit_group_array, } } -static void compile_state_update_indirect_hit_groups(UT_array *hit_group_array, - long long *group_ids, - size_t n_group_ids, int vtable_id) +static void update_indirect_hit_groups(UT_array *hit_group_array, + long long *group_ids, + size_t n_group_ids, int vtable_id) { if (NULL == hit_group_array) { return; @@ -1696,10 +1658,10 @@ static inline int compare_group_id(const void *a, const void *b) } static size_t compile_state_update_hit_not_clauses(struct compile_state *compile_state, - struct compile_runtime *compile_rt, - long long *group_ids, size_t n_group_ids, - int vtable_id, long long *NOT_group_ids_array, - size_t NOT_group_ids_array_size) + struct compile_runtime *compile_rt, + long long *group_ids, size_t n_group_ids, + int vtable_id, long long *NOT_group_ids_array, + size_t NOT_group_ids_array_size) { if (NULL == compile_state || NULL == compile_rt) { return 0; @@ -1734,39 +1696,18 @@ static size_t compile_state_update_hit_not_clauses(struct compile_state *compile return hit_NOT_group_cnt; } -void compile_runtime_ex_data_iterate(struct compile_runtime *compile_rt, - struct compile_schema *compile_schema) +int compile_state_get_compile_table_id(struct compile_state *compile_state, + long long compile_id) { - if (NULL == compile_rt || NULL == compile_schema || - (0 == compile_schema->set_flag)) { - return; - } + struct compile2table_id *tmp = NULL; - compile_runtime_user_data_iterate(compile_rt, rule_ex_data_new_cb, - &(compile_schema->ex_schema), - compile_schema->table_id); -} - -void *compile_runtime_get_ex_data(struct compile_runtime *compile_rt, - struct compile_schema *schema, - long long compile_id) -{ - if (NULL == compile_rt || NULL == schema || compile_id < 0 - || (0 == schema->set_flag)) { - return NULL; - } - - struct compile_item *item = NULL; - item = (struct compile_item *)compile_runtime_get_user_data(compile_rt, - compile_id); - if (NULL == item) { - return NULL; + tmp = utarray_find(compile_state->hit_compile_table_ids, &compile_id, + compare_compile_id); + if (NULL == tmp) { + return -1; } - - void *ex_data = NULL; - schema->ex_schema.dup_func(schema->table_id, &ex_data, item->ex_data, - schema->ex_schema.argl, schema->ex_schema.argp); - return ex_data; + + return tmp->table_id; } static int compile_runtime_add_compile(struct compile_runtime *compile_rt, @@ -1836,7 +1777,7 @@ static int compile_runtime_add_compile(struct compile_runtime *compile_rt, return 0; } -void garbage_compile_item_free(void *data, void *arg) +static void garbage_compile_item_free(void *data, void *arg) { if (NULL == data) { return; @@ -1937,6 +1878,7 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, if (0 == is_valid) { // delete compile_runtime_del_compile(compile_rt, compile_id); + rcu_hash_del(compile_rt->tbl_cfg_hash, (char *)&compile_id, sizeof(long long)); } else { // add int ret = compile_runtime_add_compile(compile_rt, schema, compile_id, @@ -1944,6 +1886,14 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, if (ret < 0) { compile_rt->update_err_cnt++; } + + int *table_id = ALLOC(int, 1); + *table_id = table_manager_get_table_id(schema->ref_tbl_mgr, table_name); + ret = rcu_hash_add(compile_rt->tbl_cfg_hash, (char *)&compile_id, + sizeof(long long), table_id); + if (ret < 0) { + FREE(table_id); + } } return 0; @@ -2157,6 +2107,7 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name, compile_rt->literal2clause_hash = new_literal2clause; compile_rt->literal2not_clause_hash = new_literal2not_clause; rcu_hash_commit(compile_rt->cfg_hash); + rcu_hash_commit(compile_rt->tbl_cfg_hash); maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, NULL, garbage_bool_matcher_free); @@ -2274,16 +2225,15 @@ int compile_state_update(int vtable_id, struct maat_item *hit_items, MAX_SCANNER_HIT_GROUP_NUM); if (1 == maat_inst->opts.hit_path_on && hit_cnt > 0) { for (i = 0; i < hit_cnt; i++) { - maat_compile_hit_path_add(compile_state->internal_hit_paths, hit_items[i].item_id, - hit_items[i].group_id, vtable_id, 0, state->scan_cnt); + add_internal_hit_path(compile_state->internal_hit_paths, hit_items[i].item_id, + hit_items[i].group_id, vtable_id, 0, state->scan_cnt); } } if (1 == maat_inst->opts.hit_group_on) { - compile_state_update_direct_hit_groups(compile_state->direct_hit_groups, - hit_items, hit_cnt, vtable_id); - compile_state_update_indirect_hit_groups(compile_state->indirect_hit_groups, - super_group_ids, super_group_cnt, vtable_id); + update_direct_hit_groups(compile_state->direct_hit_groups, hit_items, hit_cnt, vtable_id); + update_indirect_hit_groups(compile_state->indirect_hit_groups, + super_group_ids, super_group_cnt, vtable_id); } /* update hit clause */ @@ -2314,8 +2264,8 @@ int compile_state_update(int vtable_id, struct maat_item *hit_items, if (1 == maat_inst->opts.hit_path_on && hit_not_cnt > 0) { for (i = 0; i < hit_not_cnt; i++) { - maat_compile_hit_path_add(compile_state->internal_hit_paths, -1, hit_NOT_group_ids[i], - vtable_id, 1, state->scan_cnt); + add_internal_hit_path(compile_state->internal_hit_paths, -1, hit_NOT_group_ids[i], + vtable_id, 1, state->scan_cnt); } } @@ -2344,10 +2294,18 @@ size_t compile_state_get_indirect_hit_groups(struct compile_state *compile_state return i; } +size_t compile_state_get_indirect_hit_group_cnt(struct compile_state *compile_state) +{ + if (NULL == compile_state) { + return 0; + } + + return utarray_len(compile_state->indirect_hit_groups); +} + size_t compile_state_get_direct_hit_groups(struct compile_state *compile_state, - enum maat_list_type type, - struct maat_hit_group *group_array, - size_t array_size) + struct maat_hit_group *group_array, + size_t array_size) { if (NULL == compile_state) { return 0; @@ -2369,6 +2327,15 @@ size_t compile_state_get_direct_hit_groups(struct compile_state *compile_state, return i; } +size_t compile_state_get_direct_hit_group_cnt(struct compile_state *compile_state) +{ + if (NULL == compile_state) { + return 0; + } + + return utarray_len(compile_state->direct_hit_groups); +} + UT_icd ut_compile_group_id_icd = {sizeof(long long), NULL, NULL, NULL}; size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state, struct compile_runtime *compile_rt, @@ -2431,16 +2398,4 @@ size_t compile_state_get_internal_hit_paths(struct compile_state *compile_state, } return hit_path_cnt; -} - -void compile_runtime_garbage_collect_routine(void *compile_runtime) -{ - if (NULL == compile_runtime) { - return; - } - - struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; - if (compile_rt->cfg_hash != NULL) { - rcu_hash_garbage_collect_routine(compile_rt->cfg_hash); - } } \ No newline at end of file diff --git a/src/maat_expr.c b/src/maat_expr.c index 414d1b0..5be2ecb 100644 --- a/src/maat_expr.c +++ b/src/maat_expr.c @@ -144,13 +144,13 @@ static int expr_runtime_get_district_id(struct expr_runtime *expr_rt, { long long district_id = DISTRICT_ANY; - int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id); + int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id, 1); if (map_ret < 0) { if (NULL == expr_rt->tmp_district_map) { expr_rt->tmp_district_map = maat_kv_store_duplicate(expr_rt->district_map); } - map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id); + map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id, 1); if (map_ret < 0) { district_id = expr_rt->district_num; maat_kv_register(expr_rt->tmp_district_map, district, district_id); @@ -168,7 +168,8 @@ int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *dis return -1; } - return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id); + return maat_kv_read_unNull(expr_rt->district_map, district, district_len, + district_id, 1); } static struct expr_item * diff --git a/src/maat_flag.c b/src/maat_flag.c index f2046cd..65b48d9 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -269,13 +269,13 @@ static int flag_runtime_get_district_id(struct flag_runtime *flag_rt, { long long district_id = DISTRICT_ANY; - int map_ret = maat_kv_read(flag_rt->district_map, district, &district_id); + int map_ret = maat_kv_read(flag_rt->district_map, district, &district_id, 1); if (map_ret < 0) { if (NULL == flag_rt->tmp_district_map) { flag_rt->tmp_district_map = maat_kv_store_duplicate(flag_rt->district_map); } - map_ret = maat_kv_read(flag_rt->tmp_district_map, district, &district_id); + map_ret = maat_kv_read(flag_rt->tmp_district_map, district, &district_id, 1); if (map_ret < 0) { district_id = flag_rt->district_num; maat_kv_register(flag_rt->tmp_district_map, district, district_id); @@ -293,7 +293,8 @@ int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *dis return -1; } - return maat_kv_read_unNull(flag_rt->district_map, district, district_len, district_id); + return maat_kv_read_unNull(flag_rt->district_map, district, district_len, + district_id, 1); } static struct flag_item * diff --git a/src/maat_interval.c b/src/maat_interval.c index 6e19516..189ef39 100644 --- a/src/maat_interval.c +++ b/src/maat_interval.c @@ -242,13 +242,13 @@ static int interval_runtime_get_district_id(struct interval_runtime *interval_rt { long long district_id = DISTRICT_ANY; - int map_ret = maat_kv_read(interval_rt->district_map, district, &district_id); + int map_ret = maat_kv_read(interval_rt->district_map, district, &district_id, 1); if (map_ret < 0) { if (NULL == interval_rt->tmp_district_map) { interval_rt->tmp_district_map = maat_kv_store_duplicate(interval_rt->district_map); } - map_ret = maat_kv_read(interval_rt->tmp_district_map, district, &district_id); + map_ret = maat_kv_read(interval_rt->tmp_district_map, district, &district_id, 1); if (map_ret < 0) { district_id = interval_rt->district_num; maat_kv_register(interval_rt->tmp_district_map, district, district_id); @@ -267,8 +267,8 @@ int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, return -1; } - return maat_kv_read_unNull(interval_rt->district_map, district, - district_len, district_id); + return maat_kv_read_unNull(interval_rt->district_map, district, district_len, + district_id, 1); } static struct interval_item * diff --git a/src/maat_ipport_plugin.c b/src/maat_ipport_plugin.c index afaeada..4ab324e 100644 --- a/src/maat_ipport_plugin.c +++ b/src/maat_ipport_plugin.c @@ -58,7 +58,6 @@ struct ipport_item { struct ipport_plugin_runtime { struct ipport_matcher *matcher; - struct rcu_hash_table *item_hash; // struct ex_data_runtime *ex_data_rt; size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; diff --git a/src/maat_kv.c b/src/maat_kv.c index 4850434..df9938d 100644 --- a/src/maat_kv.c +++ b/src/maat_kv.c @@ -17,11 +17,16 @@ #define MAAT_KV_MAX_KEY_LEN 512 #define MAX_ENTITY_ID_CNT 4 +struct kv_entity { + long long id[MAX_ENTITY_ID_CNT]; + int id_cnt; //0~4 +}; + struct maat_kv_pair { char* key; //must be lower case. size_t key_len; - long long val; + struct kv_entity *val; UT_hash_handle hh; }; @@ -38,15 +43,32 @@ static void strlowercase(const char* src, size_t src_len, char* dst, size_t dst_ } static struct maat_kv_pair * -maat_kv_pair_new(const char* key, size_t key_len, long long value) +maat_kv_pair_new(const char* key, size_t keylen, long long value) { struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1); + kv->key = ALLOC(char, keylen); + kv->key_len = keylen; + kv->val = ALLOC(struct kv_entity, 1); + + strlowercase(key, keylen, kv->key, kv->key_len); + kv->val->id[0] = value; + kv->val->id_cnt = 1; + + return kv; +} + +static struct maat_kv_pair * +maat_kv_pair_clone(const char *key, size_t key_len, struct kv_entity *entity) +{ + struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1); + kv->key = ALLOC(char, key_len); kv->key_len = key_len; - kv->val = value; + kv->val = ALLOC(struct kv_entity, 1); strlowercase(key, key_len, kv->key, kv->key_len); + *(kv->val) = *entity; return kv; } @@ -61,6 +83,10 @@ static void maat_kv_pair_free(struct maat_kv_pair *kv) FREE(kv->key); } + if (kv->val != NULL) { + FREE(kv->val); + } + FREE(kv); } @@ -88,23 +114,23 @@ void maat_kv_store_free(struct maat_kv_store *store) } static int maat_kv_register_unNull(struct maat_kv_store *store, const char *key, - size_t key_len, long long value) + size_t keylen, long long value) { - if (key_len > MAAT_KV_MAX_KEY_LEN) { + if (keylen > MAAT_KV_MAX_KEY_LEN) { return -1; } struct maat_kv_pair *kv = NULL; struct maat_kv_pair *tmp_kv = NULL; - kv = maat_kv_pair_new(key, key_len, value); - HASH_FIND(hh, store->hash, kv->key, key_len, tmp_kv); + kv = maat_kv_pair_new(key, keylen, value); + HASH_FIND(hh, store->hash, kv->key, keylen, tmp_kv); if (tmp_kv) { maat_kv_pair_free(kv); return -1; } - HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv); + HASH_ADD_KEYPTR(hh, store->hash, kv->key, keylen, kv); return 1; } @@ -116,30 +142,34 @@ int maat_kv_register(struct maat_kv_store* store, const char *key, long long val return ret; } -int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_len, - long long *value) +int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t keylen, + long long *value_array, size_t n_array) { struct maat_kv_pair *kv = NULL; char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0}; - if (key_len > MAAT_KV_MAX_KEY_LEN) { + if (keylen > MAAT_KV_MAX_KEY_LEN) { return -1; } - strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase)); - HASH_FIND(hh, store->hash, key_lowercase, key_len, kv); + strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase)); + HASH_FIND(hh, store->hash, key_lowercase, keylen, kv); + int i = 0; if (kv) { - *value = kv->val; - return 1; + for (i = 0; i < kv->val->id_cnt && i < (int)n_array; i++) { + value_array[i] = kv->val->id[i]; + } + return i; } else { return -1; } } -int maat_kv_read(struct maat_kv_store *store, const char * key, long long *value) +int maat_kv_read(struct maat_kv_store *store, const char * key, + long long *value_array, size_t n_array) { - return maat_kv_read_unNull(store, key, strlen(key), value); + return maat_kv_read_unNull(store, key, strlen(key), value_array, n_array); } int maat_kv_write_unNull(struct maat_kv_store* store, const char* key, @@ -155,7 +185,8 @@ int maat_kv_write_unNull(struct maat_kv_store* store, const char* key, strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase)); HASH_FIND(hh, store->hash, key_lowercase, key_len, kv); if (kv) { - kv->val = value; + kv->val->id[0] = value; + kv->val->id_cnt = 1; } else { kv = maat_kv_pair_new(key, key_len, value); HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv); @@ -169,15 +200,48 @@ int maat_kv_write(struct maat_kv_store *store, const char *key, long long value) return maat_kv_write_unNull(store, key, strlen(key), value); } +int maat_kv_append_unNull(struct maat_kv_store* store, const char* key, + size_t key_len, long long value) +{ + struct maat_kv_pair *kv = NULL; + char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0}; + + if (key_len > MAAT_KV_MAX_KEY_LEN) { + return -1; + } + + strlowercase(key, key_len, key_lowercase, sizeof(key_lowercase)); + HASH_FIND(hh, store->hash, key_lowercase, key_len, kv); + if (kv) { + size_t id_cnt = kv->val->id_cnt; + if (id_cnt >= MAX_ENTITY_ID_CNT) { + return -1; + } + + kv->val->id[id_cnt] = value; + kv->val->id_cnt++; + } else { + kv = maat_kv_pair_new(key, key_len, value); + HASH_ADD_KEYPTR(hh, store->hash, kv->key, key_len, kv); + } + + return 1; +} + +int maat_kv_append(struct maat_kv_store *store, const char *key, long long value) +{ + return maat_kv_append_unNull(store, key, strlen(key), value); +} + struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map) { struct maat_kv_store *target = maat_kv_store_new(); struct maat_kv_pair *kv = NULL, *tmp_kv = NULL, *copy_kv = NULL; HASH_ITER (hh, origin_map->hash, kv, tmp_kv) { - copy_kv = maat_kv_pair_new(kv->key, kv->key_len, kv->val); + copy_kv = maat_kv_pair_clone(kv->key, kv->key_len, kv->val); HASH_ADD_KEYPTR(hh, target->hash, copy_kv->key, copy_kv->key_len, copy_kv); } return target; -} +} \ No newline at end of file diff --git a/src/maat_rule.c b/src/maat_rule.c index 67f04ca..50678b1 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -128,8 +128,7 @@ static int maat_update_cb(const char *table_name, const char *line, void *u_para return 0; } - struct maat *maat_inst =(struct maat *)u_param; - struct maat_runtime* maat_rt = NULL; + struct maat *maat_inst =(struct maat *)u_param; int table_id = table_manager_get_table_id(maat_inst->tbl_mgr, table_name); if (table_id < 0) { log_error(maat_inst->logger, MODULE_MAAT_RULE, @@ -138,23 +137,36 @@ static int maat_update_cb(const char *table_name, const char *line, void *u_para return -1; } - void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id); - if (NULL == schema) { - log_error(maat_inst->logger, MODULE_MAAT_RULE, - "[%s:%d] update warning, table name %s doesn't have table schema", - __FUNCTION__, __LINE__, table_name); - return -1; - } - - int update_type = MAAT_UPDATE_TYPE_INC; - if (maat_inst->creating_maat_rt != NULL) { //Full update - maat_rt = maat_inst->creating_maat_rt; + int update_type = MAAT_UPDATE_TYPE_INC; + if (maat_inst->creating_maat_rt != NULL) { // Full update update_type = MAAT_UPDATE_TYPE_FULL; - } else { - maat_rt = maat_inst->maat_rt; - } - - table_manager_update_runtime(maat_rt->ref_tbl_mgr, table_name, table_id, line, update_type); + } + + // find conjunction id for table_id + long long conj_parent_table_ids[4]; + int conj_parent_table_cnt = table_manager_get_conj_parent_table_ids(maat_inst->tbl_mgr, table_name, + conj_parent_table_ids, 4); + if (conj_parent_table_cnt > 0) { + for (int i = 0; i < conj_parent_table_cnt; i++) { + int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name, + (int)conj_parent_table_ids[i], line, update_type); + if (ret < 0) { + log_error(maat_inst->logger, MODULE_MAAT_RULE, + "[%s:%d] table<%s> update runtime error for rule:%s", + __FUNCTION__, __LINE__, table_name, line); + continue; + } + } + } else { + int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name, + table_id, line, update_type); + if (ret < 0) { + log_error(maat_inst->logger, MODULE_MAAT_RULE, + "[%s:%d] table<%s> update runtime error for rules:%s", + __FUNCTION__, __LINE__, table_name, line); + return -1; + } + } return 0; } @@ -202,10 +214,6 @@ static void maat_plugin_table_garbage_collect_routine(struct table_manager *tbl_ table_type = table_manager_get_table_type(tbl_mgr, i); switch (table_type) { - case TABLE_TYPE_COMPILE: - runtime = table_manager_get_runtime(tbl_mgr, i); - compile_runtime_garbage_collect_routine(runtime); - break; case TABLE_TYPE_PLUGIN: runtime = table_manager_get_runtime(tbl_mgr, i); ex_data_rt = plugin_runtime_get_ex_data_rt(runtime); @@ -350,7 +358,7 @@ long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *ke } long long sequence = 1; - int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence); + int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence, 1); if (map_ret < 0) { maat_kv_register(maat_rt->sequence_map, key, sequence); } else { diff --git a/src/maat_table.c b/src/maat_table.c index a8bf0df..0b3b305 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -51,7 +51,8 @@ struct table_manager { enum maat_expr_engine expr_engine; int default_compile_table_id; int g2g_table_id; - struct maat_kv_store *tablename2id_map; + struct maat_kv_store *tbl_name2id_map; + struct maat_kv_store *conj_tbl_name2id_map; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -516,7 +517,7 @@ maat_table_new(cJSON *json, struct maat_kv_store *reserved_word_map, ptable->table_id = item->valueint; item = cJSON_GetObjectItem(json, "table_name"); - // already validate in register_tablename2id + // already validate in register_tbl_name2id if (item->type == cJSON_Array) { cJSON *tmp_item = cJSON_GetArrayItem(item, 0); memcpy(ptable->table_name, tmp_item->valuestring, @@ -535,7 +536,7 @@ maat_table_new(cJSON *json, struct maat_kv_store *reserved_word_map, } ret = maat_kv_read(reserved_word_map, item->valuestring, - (long long *)&(ptable->table_type)); + (long long *)&(ptable->table_type), 1); if (ret < 0) { log_error(logger, MODULE_TABLE, "[%s:%d] table:%s table_type %s is illegal", @@ -597,30 +598,49 @@ static void maat_table_free(struct maat_table *maat_tbl) FREE(maat_tbl); } -static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map, - struct log_handle *logger) +static int register_single_tbl_name2id(struct maat_kv_store *tbl_name2id_map, + const char *table_name, int table_id, + struct log_handle *logger) { - cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (NULL == item || item->type != cJSON_Number) { + if (strlen(table_name) >= NAME_MAX) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table:<%s> name length exceed maxium:%d", + __FUNCTION__, __LINE__, table_name, NAME_MAX); return -1; } - int table_id = item->valueint; - item = cJSON_GetObjectItem(json, "db_tables"); - if (item != NULL && item->type != cJSON_Array) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) has db_tables, but format is invalid" - ", should be array", __FUNCTION__, __LINE__, table_id); + long long tmp_table_id = -1; + int ret = maat_kv_read(tbl_name2id_map, table_name, &tmp_table_id, 1); + if (ret > 0 && tmp_table_id != table_id) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table:<%s>(table_id:%lld) has already been registered" + ", can't register again", + __FUNCTION__, __LINE__, table_name, tmp_table_id); return -1; } - - int ret = 0; - long long tmp_table_id = 0; - if (item != NULL) { + + maat_kv_register(tbl_name2id_map, table_name, table_id); + return 0; +} + +static int register_conjunction_tbl_name2id(struct maat_kv_store *conj_tbl_name2id_map, + cJSON *root, struct log_handle *logger) +{ + int json_array_size = cJSON_GetArraySize(root); + + for (int i = 0; i < json_array_size; i++) { + cJSON *json = cJSON_GetArrayItem(root, i); + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + int table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "db_tables"); + if (NULL == item || item->type != cJSON_Array) { + continue; + } + int n_table_name = cJSON_GetArraySize(item); - - for (int i = 0; i < n_table_name; i++) { - cJSON *tmp_item = cJSON_GetArrayItem(item, i); + for (int j = 0; j < n_table_name; j++) { + cJSON *tmp_item = cJSON_GetArrayItem(item, j); if (NULL == tmp_item || tmp_item->type != cJSON_String) { log_error(logger, MODULE_TABLE, "[%s:%d] table(table_id:%d) db_tables element format invalid" @@ -628,56 +648,106 @@ static int register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id return -1; } - if (strlen(tmp_item->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) db_tables string %s length " - "exceed maxium:%d", __FUNCTION__, __LINE__, table_id, - tmp_item->valuestring, NAME_MAX); + int ret = maat_kv_append(conj_tbl_name2id_map, tmp_item->valuestring, table_id); + if (ret < 0) { return -1; } - - ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &tmp_table_id); - if (ret > 0 && tmp_table_id != table_id) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table:<%s>(table_id:%lld) has already been registered" - ", can't register again", __FUNCTION__, __LINE__, - tmp_item->valuestring, tmp_table_id); - return -1; - } - - maat_kv_register(tablename2id_map, tmp_item->valuestring, table_id); } } - item = cJSON_GetObjectItem(json, "table_name"); - if (NULL == item || item->type != cJSON_String) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) has no table_name", - __FUNCTION__, __LINE__, table_id); - return -1; + return 0; +} + +static int register_tbl_name2id(struct maat_kv_store *tbl_name2id_map, cJSON *root, + const char *table_info_path, struct log_handle *logger) +{ + int i = 0; + int json_array_size = cJSON_GetArraySize(root); + + //pre register tablename2id + for (i = 0; i < json_array_size; i++) { + cJSON *json = cJSON_GetArrayItem(root, i); + if (NULL == json || json->type != cJSON_Object) { + log_error(logger, MODULE_TABLE, + "[%s:%d] %s has invalid json object", + __FUNCTION__, __LINE__, table_info_path); + return -1; + } + + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + log_error(logger, MODULE_TABLE, + "[%s:%d] %s has invalid json object, happened in table_id column ", + __FUNCTION__, __LINE__, table_info_path); + return -1; + } + int table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "table_name"); + if (NULL == item || item->type != cJSON_String) { + log_error(logger, MODULE_TABLE, + "[%s:%d] %s has invalid json object, happened in table_name column", + __FUNCTION__, __LINE__, table_info_path); + return -1; + } + + int ret = register_single_tbl_name2id(tbl_name2id_map, item->valuestring, + table_id, logger); + if (ret < 0) { + return -1; + } } - if (strlen(item->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table(table_id:%d) table_name %s length too long", - __FUNCTION__, __LINE__, table_id, item->valuestring); - return -1; - } + //register db_tables's table_name + for (i = 0; i < json_array_size; i++) { + cJSON *json = cJSON_GetArrayItem(root, i); + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + int table_id = item->valueint; - ret = maat_kv_read(tablename2id_map, item->valuestring, &tmp_table_id); - if (ret > 0 && tmp_table_id != table_id) { - log_error(logger, MODULE_TABLE, - "[%s:%d] table:<%s>(table_id:%lld) has already been registered," - " can't register again", __FUNCTION__, __LINE__, item->valuestring, - tmp_table_id); - return -1; - } + item = cJSON_GetObjectItem(json, "db_tables"); + if (NULL == item || item->type != cJSON_Array) { + continue; + } - maat_kv_register(tablename2id_map, item->valuestring, table_id); + int n_table_name = cJSON_GetArraySize(item); + for (int j = 0; j < n_table_name; j++) { + cJSON *tmp_item = cJSON_GetArrayItem(item, j); + if (NULL == tmp_item || tmp_item->type != cJSON_String) { + log_error(logger, MODULE_TABLE, + "[%s:%d] table(table_id:%d) db_tables element format invalid" + ", should be string", __FUNCTION__, __LINE__, table_id); + return -1; + } + + long long tmp_table_id = -1; + int ret = maat_kv_read(tbl_name2id_map, tmp_item->valuestring, &tmp_table_id, 1); + if (ret > 0) { + continue; + } + + ret = register_single_tbl_name2id(tbl_name2id_map, tmp_item->valuestring, + table_id, logger); + if (ret < 0) { + return -1; + } + } + } return 0; } +int maat_default_compile_table_id(cJSON *json, struct log_handle *logger) +{ + cJSON *item = cJSON_GetObjectItem(json, "default_compile_table"); + if (NULL == item || item->type != cJSON_Number) { + return -1; + } + + item = cJSON_GetObjectItem(json, "table_id"); + //item is cJSON_Number which has been checked in maat_table_new + return item->valueint; +} + struct table_manager * table_manager_create(const char *table_info_path, const char *accept_tags, enum maat_expr_engine expr_engine, struct maat_garbage_bin *garbage_bin, @@ -721,25 +791,29 @@ table_manager_create(const char *table_info_path, const char *accept_tags, struct table_manager *tbl_mgr = ALLOC(struct table_manager, 1); tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger); tbl_mgr->logger = logger; - tbl_mgr->tablename2id_map = maat_kv_store_new(); + tbl_mgr->tbl_name2id_map = maat_kv_store_new(); + tbl_mgr->conj_tbl_name2id_map = maat_kv_store_new(); tbl_mgr->expr_engine = expr_engine; tbl_mgr->ref_garbage_bin = garbage_bin; - for (int i = 0; i < json_array_size; i++) { - json = cJSON_GetArrayItem(root, i); + ret = register_tbl_name2id(tbl_mgr->tbl_name2id_map, root, table_info_path, logger); + if (ret < 0) { + log_error(logger, MODULE_TABLE, + "[%s:%d] register_tbl_name2id failed.", __FUNCTION__, __LINE__); + FREE(json_buff); + cJSON_Delete(root); + table_manager_destroy(tbl_mgr); + return NULL; + } - if (json != NULL && json->type == cJSON_Object) { - ret = register_tablename2id(json, tbl_mgr->tablename2id_map, logger); - if (ret < 0) { - log_error(logger, MODULE_TABLE, - "[%s:%d] register_tablename2id failed", - __FUNCTION__, __LINE__); - FREE(json_buff); - cJSON_Delete(root); - table_manager_destroy(tbl_mgr); - return NULL; - } - } + ret = register_conjunction_tbl_name2id(tbl_mgr->conj_tbl_name2id_map, root, logger); + if (ret < 0) { + log_error(logger, MODULE_TABLE, + "[%s:%d] register_conjunction_tbl_name2id failed.", __FUNCTION__, __LINE__); + FREE(json_buff); + cJSON_Delete(root); + table_manager_destroy(tbl_mgr); + return NULL; } int default_compile_table_id = -1; @@ -757,22 +831,27 @@ table_manager_create(const char *table_info_path, const char *accept_tags, goto next; } - maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, - maat_tbl->table_type, tbl_mgr, logger); - if (NULL == maat_tbl->schema) { - log_error(logger, MODULE_TABLE, - "[%s:%d] Maat table schema new failed, table_name:%s", - __FUNCTION__, __LINE__, maat_tbl->table_name); - ret = -1; - goto next; + long long parent_table_ids[4]; + int parent_table_cnt = table_manager_get_conj_parent_table_ids(tbl_mgr, maat_tbl->table_name, + parent_table_ids, 4); + if (parent_table_cnt <= 0) { + // if table has conjuncion parent, which can share schema and + // runtime with parent. + maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type, + tbl_mgr, logger); + if (NULL == maat_tbl->schema) { + log_error(logger, MODULE_TABLE, + "[%s:%d] Maat table schema new failed, table_name:%s", + __FUNCTION__, __LINE__, maat_tbl->table_name); + ret = -1; + goto next; + } } - + if (maat_tbl->table_type == TABLE_TYPE_COMPILE) { if (default_compile_table_id < 0) { - default_compile_table_id = maat_tbl->table_id; - } else if (maat_tbl->table_id < default_compile_table_id) { - default_compile_table_id = maat_tbl->table_id; - } + default_compile_table_id = maat_default_compile_table_id(json, logger); + } } if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) { @@ -932,9 +1011,14 @@ void table_manager_destroy(struct table_manager *tbl_mgr) FREE(tbl_mgr->accept_tags); } - if (tbl_mgr->tablename2id_map != NULL) { - maat_kv_store_free(tbl_mgr->tablename2id_map); - tbl_mgr->tablename2id_map = NULL; + if (tbl_mgr->tbl_name2id_map != NULL) { + maat_kv_store_free(tbl_mgr->tbl_name2id_map); + tbl_mgr->tbl_name2id_map = NULL; + } + + if (tbl_mgr->conj_tbl_name2id_map != NULL) { + maat_kv_store_free(tbl_mgr->conj_tbl_name2id_map); + tbl_mgr->conj_tbl_name2id_map = NULL; } FREE(tbl_mgr); @@ -954,14 +1038,14 @@ size_t table_manager_table_count(struct table_manager *tbl_mgr) return tbl_mgr->n_table; } -int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) +int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_name) { - if (NULL == tbl_mgr || NULL == name) { + if (NULL == tbl_mgr || NULL == table_name) { return -1; } long long table_id = -1; - int ret = maat_kv_read(tbl_mgr->tablename2id_map, name, &table_id); + int ret = maat_kv_read(tbl_mgr->tbl_name2id_map, table_name, &table_id, 1); if (ret < 0) { return -1; } @@ -969,6 +1053,17 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) return (int)table_id; } +int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name, + long long *table_ids_array, size_t n_table_ids_array) +{ + if (NULL == tbl_mgr || NULL == table_name) { + return -1; + } + + return maat_kv_read(tbl_mgr->conj_tbl_name2id_map, table_name, + table_ids_array, n_table_ids_array); +} + const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int table_id) { if (NULL == tbl_mgr || table_id < 0) { diff --git a/test/file_test_tableinfo.conf b/test/file_test_tableinfo.conf index d666035..541a8fa 100644 --- a/test/file_test_tableinfo.conf +++ b/test/file_test_tableinfo.conf @@ -1,8 +1,7 @@ [ { "table_id":0, - "table_name": "FILE_COMPILE", - "db_tables": ["NTC_COMPILE", "WHITE_LIST_COMPILE"], + "table_name": "NTC_COMPILE", "table_type":"compile", "valid_column":8, "custom": { @@ -13,6 +12,30 @@ }, { "table_id":1, + "table_name": "WHITE_LIST_COMPILE", + "table_type":"compile", + "valid_column":8, + "custom": { + "compile_id":1, + "tags":6, + "clause_num":9 + } + }, + { + "table_id":2, + "table_name": "FILE_COMPILE", + "db_tables": ["NTC_COMPILE", "WHITE_LIST_COMPILE"], + "default_compile_table":2, + "table_type":"compile", + "valid_column":8, + "custom": { + "compile_id":1, + "tags":6, + "clause_num":9 + } + }, + { + "table_id":3, "table_name":"NTC_GROUP2GROUP", "table_type":"group2group", "valid_column":3, @@ -23,10 +46,10 @@ } }, { - "table_id":2, + "table_id":4, "table_name":"NTC_GROUP2COMPILE", "table_type":"group2compile", - "associated_compile_table_id":0, + "associated_compile_table_id":2, "valid_column":3, "custom": { "group_id":1, @@ -37,7 +60,7 @@ } }, { - "table_id":3, + "table_id":5, "table_name":"NTC_UNIVERSAL_IP", "table_type":"ip_plus", "valid_column":11, @@ -55,7 +78,7 @@ } }, { - "table_id":4, + "table_id":6, "table_name":"NTC_UNIVERSAL_PROTO_TYPE", "table_type":"intval", "valid_column":5, @@ -67,7 +90,7 @@ } }, { - "table_id":5, + "table_id":7, "table_name":"WHITE_LIST_IP", "table_type":"ip_plus", "valid_column":11, @@ -85,7 +108,7 @@ } }, { - "table_id":7, + "table_id":8, "table_name":"FILE_HTTP_URL", "db_tables": ["NTC_HTTP_URL", "WHITE_LIST_DOMAIN"], "table_type":"expr", @@ -100,7 +123,7 @@ } }, { - "table_id":8, + "table_id":9, "table_name":"FILE_HTTP_HDR_REGION", "db_tables":["NTC_HTTP_REQ_HDR", "NTC_HTTP_RES_HDR"], "table_type":"expr_plus", @@ -116,7 +139,7 @@ } }, { - "table_id":9, + "table_id":10, "table_name":"FILE_HTTP_BODY_REGION", "db_tables":["NTC_HTTP_REQ_BODY", "NTC_HTTP_RES_BODY"], "table_type":"expr", diff --git a/test/json_update/corrupted.json b/test/json_update/corrupted.json index 5508ec0..fb491db 100644 --- a/test/json_update/corrupted.json +++ b/test/json_update/corrupted.json @@ -1,5 +1,5 @@ { - "compile_table": "COMPILE", + "compile_table": "COMPILE_DEFAULT", "group_table": "GROUP", "rules": [ { diff --git a/test/json_update/new.json b/test/json_update/new.json index b1d7650..07e3156 100644 --- a/test/json_update/new.json +++ b/test/json_update/new.json @@ -1,6 +1,6 @@ { - "compile_table": "COMPILE", - "group2compile_table": "GROUP2COMPILE", + "compile_table": "COMPILE_DEFAULT", + "group2compile_table": "GROUP2COMPILE_DEFAULT", "group2group_table": "GROUP2GROUP", "rules": [ { diff --git a/test/json_update/old.json b/test/json_update/old.json index 399ae55..278272b 100644 --- a/test/json_update/old.json +++ b/test/json_update/old.json @@ -1,6 +1,6 @@ { - "compile_table": "COMPILE", - "group2compile_table": "GROUP2COMPILE", + "compile_table": "COMPILE_DEFAULT", + "group2compile_table": "GROUP2COMPILE_DEFAULT", "group2group_table": "GROUP2GROUP", "rules": [ { diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 8d292c1..effc07e 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -311,12 +311,12 @@ int test_add_expr_command(struct maat *maat_inst, const char *expr_table, memset(huge_serv_def, 's', sizeof(huge_serv_def) - 1); huge_serv_def[sizeof(huge_serv_def) - 1] = '\0'; - int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, + int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, huge_serv_def, 1, timeout); EXPECT_EQ(ret, 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, compile_id, 0, expr_table, 1, timeout); EXPECT_EQ(ret, 1); @@ -330,7 +330,7 @@ int test_add_expr_command(struct maat *maat_inst, const char *expr_table, int del_command(struct maat *maat_inst, int compile_id) { - return compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_DEL, + return compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_DEL, compile_id, "null", 1, 0); } @@ -1378,8 +1378,8 @@ TEST_F(MaatHsStringScan, dynamic_config) { maat_state_reset(state); - const char *compile_table_name = "COMPILE"; - const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; /* compile table add line */ long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); @@ -2130,8 +2130,8 @@ TEST_F(MaatRsStringScan, dynamic_config) { EXPECT_EQ(n_hit_result, 0); maat_state_reset(state); - const char *compile_table_name = "COMPILE"; - const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; /* compile table add line */ long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); @@ -2711,8 +2711,8 @@ TEST_F(MaatIPScan, dynamic_config) { EXPECT_EQ(n_hit_result, 0); maat_state_reset(state); - const char *compile_table_name = "COMPILE"; - const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; /* compile table add line */ long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); @@ -4776,7 +4776,7 @@ void compile_ex_param_dup(int table_id, void **to, void **from, long argl, void TEST_F(CompileTable, CompileRuleUpdate) { struct maat *maat_inst = CompileTable::_shared_maat_inst; - const char *compile_table_name = "COMPILE"; + const char *compile_table_name = "COMPILE_DEFAULT"; long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); int ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_ADD, compile_id, "null", 1, 0); @@ -4977,21 +4977,26 @@ TEST_F(Policy, CompileRuleTags) { } TEST_F(Policy, CompileEXData) { + + const char *url = "firewall should hit"; + const char *table_name = "HTTP_URL"; + const char *plugin_table_name = "COMPILE_FIREWALL_PLUGIN"; + const char *conj_compile_table_name = "COMPILE_FIREWALL_CONJUNCTION"; + const char *phy_compile_table_name = "COMPILE_FIREWALL_DEFAULT"; + const char *expect_name = "I have a name"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; - const char *url = "firewall should hit"; - const char *table_name = "HTTP_URL"; - const char *compile_table_name = "COMPILE_FIREWALL"; - const char *expect_name = "I have a name"; struct maat *maat_inst = Policy::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); int table_id = maat_get_table_id(maat_inst, table_name); - int compile_table_id = maat_get_table_id(maat_inst, compile_table_name); + int plugin_table_id = maat_get_table_id(maat_inst, plugin_table_name); + int conj_compile_table_id = maat_get_table_id(maat_inst, conj_compile_table_name); + int phy_compile_table_id = maat_get_table_id(maat_inst, phy_compile_table_name); int ex_data_counter = 0; - int ret = maat_plugin_table_ex_schema_register(maat_inst, compile_table_name, + int ret = maat_plugin_table_ex_schema_register(maat_inst, plugin_table_name, compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, @@ -4999,7 +5004,7 @@ TEST_F(Policy, CompileEXData) { ASSERT_TRUE(ret == 0); EXPECT_EQ(ex_data_counter, 1); - ret = maat_state_set_scan_compile_table(state, compile_table_id); + ret = maat_state_set_scan_compile_table(state, conj_compile_table_id); EXPECT_EQ(ret, 0); ret = maat_scan_string(maat_inst, table_id, url, strlen(url), @@ -5008,7 +5013,12 @@ TEST_F(Policy, CompileEXData) { EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 198); - void *ex_data = maat_plugin_table_get_ex_data(maat_inst, compile_table_id, + int compile_table_ids[ARRAY_SIZE]; + ret = maat_state_get_compile_table_ids(state, results, 1, compile_table_ids); + EXPECT_EQ(ret, 1); + EXPECT_EQ(compile_table_ids[0], phy_compile_table_id); + + void *ex_data = maat_plugin_table_get_ex_data(maat_inst, plugin_table_id, (char *)&results[0], sizeof(long long)); ASSERT_TRUE(ex_data!=NULL); struct rule_ex_param *param = (struct rule_ex_param *)ex_data; @@ -5049,6 +5059,14 @@ TEST_F(Policy, SubGroup) { EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(results[0], 153); + const char *compile_table_name = "COMPILE_DEFAULT"; + int phy_compile_table_id = maat_get_table_id(maat_inst, compile_table_name); + + int compile_table_ids[ARRAY_SIZE]; + ret = maat_state_get_compile_table_ids(state, results, 1, compile_table_ids); + EXPECT_EQ(ret, 1); + EXPECT_EQ(compile_table_ids[0], phy_compile_table_id); + maat_state_free(state); } @@ -5572,8 +5590,8 @@ TEST_F(MaatCmdTest, SetIP) { size_t n_hit_result = 0; int thread_id = 0; const char *ip_table_name = "IP_CONFIG"; - const char *compile_table_name = "COMPILE"; - const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; struct maat_state *state = maat_state_new(maat_inst, thread_id); @@ -5627,6 +5645,7 @@ TEST_F(MaatCmdTest, SetExpr) { const char *keywords1 = "Hiredis"; const char *keywords2 = "C Client"; + const char *compile_table_name = "COMPILE_DEFAULT"; char escape_buff1[256], escape_buff2[256]; char keywords[512]; @@ -5656,10 +5675,10 @@ TEST_F(MaatCmdTest, SetExpr) { EXPECT_TRUE(results[0] == compile_id || results[0] == (compile_id - 1)); maat_state_reset(state); - ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_DEL, compile_id-1, + ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_DEL, compile_id-1, "null", 1, 0); EXPECT_EQ(ret, 1); - ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_DEL, compile_id, + ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_DEL, compile_id, "null", 1, 0); EXPECT_EQ(ret, 1); sleep(WAIT_FOR_EFFECTIVE_S); @@ -5684,9 +5703,9 @@ TEST_F(MaatCmdTest, SetExpr8) { const char *scan_data8 = "string1, string2, string3, string4, string5, string6, string7, string8"; const char *scan_data7 = "string1, string2, string3, string4, string5, string6, string7"; - const char *compile_table_name = "COMPILE"; + const char *compile_table_name = "COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; const char *table_name = "KEYWORDS_TABLE"; - const char *g2c_table_name = "GROUP2COMPILE"; const char *keywords8 = "string1&string2&string3&string4&string5&string6&string7&string8"; const char *keywords7 = "string1&string2&string3&string4&string5&string6&string7"; @@ -5755,6 +5774,8 @@ TEST_F(MaatCmdTest, SameFilterRefByOneCompile) { const char *vtable_name = "HTTP_URL_FILTER"; const char *scan_data = "http://filtermenot.com"; const char *keywords = "menot.com"; + const char *compile_table_name = "COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; int thread_id = 0; @@ -5765,17 +5786,17 @@ TEST_F(MaatCmdTest, SameFilterRefByOneCompile) { ASSERT_GT(table_id, 0); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, + int ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_ADD, compile_id, "null", 2, 0); // compile has two clause EXPECT_EQ(ret, 1); //clause1 & clause2 has same filter => {vtable_id, group_id} long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, + ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_ADD, group_id, compile_id, 0, vtable_name, 1, 0); EXPECT_EQ(ret, 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, + ret = group2compile_table_set_line(maat_inst, g2c_table_name, MAAT_OP_ADD, group_id, compile_id, 0, vtable_name, 2, 0); EXPECT_EQ(ret, 1); @@ -5882,9 +5903,9 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder) { TEST_F(MaatCmdTest, SubGroup) { const char *table_name = "HTTP_URL"; - const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; const char *g2g_table_name = "GROUP2GROUP"; - const char *compile_table_name = "COMPILE"; const char *scan_data1 = "www.v2ex.com/t/573028#程序员的核心竞争力是什么"; const char *keyword1 = "程序员&核心竞争力"; const char *scan_data2 = "https://ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc" @@ -6038,8 +6059,8 @@ TEST_F(MaatCmdTest, SubGroup) { TEST_F(MaatCmdTest, RefGroup) { const char *table_name = "HTTP_URL"; - const char* g2c_table_name = "GROUP2COMPILE"; - const char* compile_table_name = "COMPILE"; + const char* compile_table_name = "COMPILE_DEFAULT"; + const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; const char* scan_data1 = "m.facebook.com/help/2297503110373101?helpref=hc_nav&refid=69"; const char* keyword1 = "something-should-not-hit"; const char* keyword2 = "facebook.com/help/2297503110373101"; @@ -6116,8 +6137,8 @@ TEST_F(MaatCmdTest, RefGroup) { } TEST_F(MaatCmdTest, VirtualTable) { - const char* g2c_table_name = "GROUP2COMPILE"; - const char* compile_table_name = "COMPILE"; + const char* compile_table_name = "COMPILE_DEFAULT"; + const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; const char* table_name="HTTP_SIGNATURE"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -6512,11 +6533,12 @@ void plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *arg } TEST_F(MaatCmdTest, CompileEXData) { - const char *compile_table_name = "COMPILE_FIREWALL"; + const char *plugin_table_name = "COMPILE_FIREWALL_PLUGIN"; + const char *compile_table_name = "COMPILE_FIREWALL_DEFAULT"; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; int *ex_data_counter = MaatCmdTest::_ex_data_counter; - int compile_table_id = maat_get_table_id(maat_inst, compile_table_name); - EXPECT_GT(compile_table_id, 0); + int plugin_table_id = maat_get_table_id(maat_inst, plugin_table_name); + EXPECT_GT(plugin_table_id, 0); long long compile1_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); int ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_ADD, @@ -6529,7 +6551,7 @@ TEST_F(MaatCmdTest, CompileEXData) { sleep(WAIT_FOR_EFFECTIVE_S); *ex_data_counter = 0; - ret = maat_plugin_table_ex_schema_register(maat_inst, compile_table_name, + ret = maat_plugin_table_ex_schema_register(maat_inst, plugin_table_name, compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, @@ -6537,14 +6559,14 @@ TEST_F(MaatCmdTest, CompileEXData) { ASSERT_TRUE(ret == 0); EXPECT_EQ(*ex_data_counter, 2); - void *ex_data = maat_plugin_table_get_ex_data(maat_inst, compile_table_id, + void *ex_data = maat_plugin_table_get_ex_data(maat_inst, plugin_table_id, (char *)&compile1_id, sizeof(long long)); ASSERT_TRUE(ex_data != NULL); struct rule_ex_param *param = (struct rule_ex_param *)ex_data; EXPECT_EQ(param->id, 1111); - ex_data = maat_plugin_table_get_ex_data(maat_inst, compile_table_id, + ex_data = maat_plugin_table_get_ex_data(maat_inst, plugin_table_id, (char *)&compile2_id, sizeof(long long)); ASSERT_TRUE(ex_data != NULL); @@ -6553,10 +6575,10 @@ TEST_F(MaatCmdTest, CompileEXData) { ret = compile_table_set_line(maat_inst, compile_table_name, MAAT_OP_DEL, compile2_id, "test:compile2,2222", 1, 0); - sleep(WAIT_FOR_EFFECTIVE_S * 5); + sleep(WAIT_FOR_EFFECTIVE_S); EXPECT_EQ(param->id, 2222); sleep(2); - //exceed gc_timeout_s(11s), the data pointed by param has been freed + //excced gc_timeout_s(3s), the data pointed by param has been freed } TEST_F(MaatCmdTest, PluginEXData) { @@ -6896,8 +6918,8 @@ TEST_F(MaatCmdTest, UpdateBoolPlugin) { #define COMPILE_ID_NUMS 1000 TEST_F(MaatCmdTest, GroupInMassCompiles) { - const char* g2c_table_name = "GROUP2COMPILE"; - const char* compile_table_name = "COMPILE"; + const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* compile_table_name = "COMPILE_DEFAULT"; const char* table_url = "HTTP_URL"; const char* table_appid = "APP_ID"; int thread_id = 0; @@ -7001,8 +7023,8 @@ TEST_F(MaatCmdTest, GroupInMassCompiles) { } TEST_F(MaatCmdTest, HitGroup) { - const char *compile_table_name = "COMPILE"; - const char *g2c_table_name = "GROUP2COMPILE"; + const char *compile_table_name = "COMPILE_DEFAULT"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; const char *g2g_table_name = "GROUP2GROUP"; const char *http_sig_table_name = "HTTP_SIGNATURE"; const char *ip_table_name = "IP_CONFIG"; @@ -7127,15 +7149,16 @@ TEST_F(MaatCmdTest, HitGroup) { struct maat_hit_group hit_groups[128]; memset(hit_groups, 0, sizeof(hit_groups)); - int n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, - hit_groups, 128); + size_t n_hit_group = maat_state_get_direct_hit_group_cnt(state); + maat_state_get_direct_hit_groups(state, hit_groups, n_hit_group); EXPECT_EQ(n_hit_group, 1); EXPECT_EQ(hit_groups[0].item_id, item1_id); EXPECT_EQ(hit_groups[0].group_id, group1_id); EXPECT_EQ(hit_groups[0].vtable_id, http_req_table_id); memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_indirect_hit_groups(state, hit_groups, 128); + n_hit_group = maat_state_get_indirect_hit_group_cnt(state); + maat_state_get_indirect_hit_groups(state, hit_groups, n_hit_group); EXPECT_EQ(n_hit_group, 1); EXPECT_EQ(hit_groups[0].item_id, 0); EXPECT_EQ(hit_groups[0].group_id, group11_id); @@ -7160,8 +7183,8 @@ TEST_F(MaatCmdTest, HitGroup) { EXPECT_EQ(scan_count, 2); memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, - hit_groups, 128); + n_hit_group = maat_state_get_direct_hit_group_cnt(state); + maat_state_get_direct_hit_groups(state, hit_groups, n_hit_group); EXPECT_EQ(n_hit_group, 1); EXPECT_EQ(hit_groups[0].item_id, item2_id); @@ -7169,7 +7192,8 @@ TEST_F(MaatCmdTest, HitGroup) { EXPECT_EQ(hit_groups[0].vtable_id, http_res_table_id); memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_indirect_hit_groups(state, hit_groups, 128); + n_hit_group = maat_state_get_indirect_hit_group_cnt(state); + maat_state_get_indirect_hit_groups(state, hit_groups, n_hit_group); EXPECT_EQ(n_hit_group, 1); EXPECT_EQ(hit_groups[0].item_id, 0); EXPECT_EQ(hit_groups[0].group_id, group21_id); @@ -7201,8 +7225,8 @@ TEST_F(MaatCmdTest, HitGroup) { EXPECT_EQ(scan_count, 4); memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, - hit_groups, 128); + n_hit_group = maat_state_get_direct_hit_group_cnt(state); + maat_state_get_direct_hit_groups(state, hit_groups, n_hit_group); EXPECT_EQ(n_hit_group, 2); EXPECT_EQ(hit_groups[0].item_id, item4_id); @@ -7221,8 +7245,8 @@ TEST_F(MaatCmdTest, HitGroup) { EXPECT_EQ(scan_count, 5); memset(hit_groups, 0, sizeof(hit_groups)); - n_hit_group = maat_state_get_direct_hit_groups(state, MAAT_LIST_TYPE_INC, - hit_groups, 128); + n_hit_group = maat_state_get_direct_hit_group_cnt(state); + maat_state_get_direct_hit_groups(state, hit_groups, n_hit_group); EXPECT_EQ(n_hit_group, 1); EXPECT_EQ(hit_groups[0].item_id, item5_id); @@ -7236,8 +7260,8 @@ TEST_F(MaatCmdTest, HitGroup) { TEST_F(MaatCmdTest, HitPath) { const char *g2g_table_name = "GROUP2GROUP"; - const char *g2c_table_name = "GROUP2COMPILE"; - const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE_DEFAULT"; const char *http_sig_table_name = "HTTP_SIGNATURE"; const char *ip_table_name = "IP_CONFIG"; const char *keywords_table_name = "KEYWORDS_TABLE"; @@ -7503,8 +7527,8 @@ that the edges be all directed in the same direction."; TEST_F(MaatCmdTest, HitPathHasNotGroup) { const char *g2g_table_name = "GROUP2GROUP"; - const char *g2c_table_name = "GROUP2COMPILE"; - const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE_DEFAULT"; const char *http_sig_table_name = "HTTP_SIGNATURE"; const char *ip_table_name = "IP_CONFIG"; const char *keywords_table_name = "KEYWORDS_TABLE"; @@ -7777,8 +7801,8 @@ TEST_F(MaatCmdTest, SameSuperGroupRefByMultiCompile) { char temp[1024]={0}; int thread_id = 0; const char *g2g_table_name = "GROUP2GROUP"; - const char *g2c_table_name = "GROUP2COMPILE"; - const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE_DEFAULT"; const char *http_sig_table_name = "HTTP_SIGNATURE"; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -7871,8 +7895,8 @@ TEST_F(MaatCmdTest, SameSuperGroupRefByMultiCompile) { } TEST_F(MaatCmdTest, SameScanStatusWhenClauseUpdate_TSG6419) { - const char *g2c_table_name = "GROUP2COMPILE"; - const char* compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* compile_table_name = "COMPILE_DEFAULT"; const char* ip_table_name = "IP_PLUS_CONFIG"; const char *app_id_table_name = "APP_ID"; int thread_id = 0; @@ -7967,8 +7991,8 @@ TEST_F(MaatCmdTest, SameScanStatusWhenClauseUpdate_TSG6419) { } TEST_F(MaatCmdTest, GroupEdit) { - const char *g2c_table_name = "GROUP2COMPILE"; - const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE_DEFAULT"; const char *ip_table_name = "IP_PLUS_CONFIG"; const char *app_id_table_name = "APP_ID"; int thread_id = 0; @@ -8090,8 +8114,8 @@ TEST_F(MaatCmdTest, GroupEdit) { } TEST_F(MaatCmdTest, CompileDelete_TSG6548) { - const char* g2c_table_name = "GROUP2COMPILE"; - const char* compile_table_name = "COMPILE"; + const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* compile_table_name = "COMPILE_DEFAULT"; const char* ip_table_name = "IP_PLUS_CONFIG"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -8163,8 +8187,8 @@ TEST_F(MaatCmdTest, CompileDelete_TSG6548) { } TEST_F(MaatCmdTest, UpdateDeadLockDetection) { - const char* g2c_table_name = "GROUP2COMPILE"; - const char* compile_table_name = "COMPILE"; + const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* compile_table_name = "COMPILE_DEFAULT"; const char* table_http_url = "HTTP_URL"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -8239,8 +8263,8 @@ TEST_F(MaatCmdTest, UpdateDeadLockDetection) { } TEST_F(MaatCmdTest, StreamScanWhenExprTableIncUpdate) { - const char* g2c_table_name = "GROUP2COMPILE"; - const char* compile_table_name = "COMPILE"; + const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* compile_table_name = "COMPILE_DEFAULT"; const char* scan_table_name = "KEYWORDS_TABLE"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -8302,8 +8326,8 @@ TEST_F(MaatCmdTest, StreamScanWhenExprTableIncUpdate) { } TEST_F(MaatCmdTest, StreamScanSegfaultWhenVersionRollBack_TSG6324) { - const char* g2c_table_name = "GROUP2COMPILE"; - const char* compile_table_name = "COMPILE"; + const char* g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char* compile_table_name = "COMPILE_DEFAULT"; const char* scan_table_name = "KEYWORDS_TABLE"; int thread_id = 0; struct maat *maat_inst = MaatCmdTest::_shared_maat_inst; @@ -8361,8 +8385,8 @@ TEST_F(MaatCmdTest, StreamScanSegfaultWhenVersionRollBack_TSG6324) { } TEST_F(MaatCmdTest, IPAndStreamScanWhenIncUpdate) { - const char *g2c_table_name = "GROUP2COMPILE"; - const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE_DEFAULT"; const char *expr_table_name = "KEYWORDS_TABLE"; const char *ip_table_name = "IP_PLUS_CONFIG"; int thread_id = 0; @@ -8454,8 +8478,8 @@ TEST_F(MaatCmdTest, IPAndStreamScanWhenIncUpdate) { } TEST_F(MaatCmdTest, IPAndStreamScanWhenFullUpdate) { - const char *g2c_table_name = "GROUP2COMPILE"; - const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE_DEFAULT"; const char *ip_table_name = "IP_PLUS_CONFIG"; const char *expr_table_name = "KEYWORDS_TABLE"; int thread_id = 0; @@ -8546,8 +8570,8 @@ TEST_F(MaatCmdTest, IPAndStreamScanWhenFullUpdate) { } TEST_F(MaatCmdTest, IPAndStringScanWhenIncUpdate) { - const char *g2c_table_name = "GROUP2COMPILE"; - const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE_DEFAULT"; const char *expr_table_name = "HTTP_URL"; const char *ip_table_name = "IP_PLUS_CONFIG"; const char *keywords = "IP&stringinc"; @@ -8638,8 +8662,8 @@ TEST_F(MaatCmdTest, IPAndStringScanWhenIncUpdate) { } TEST_F(MaatCmdTest, IPAndStringScanWhenFullupdate) { - const char *g2c_table_name = "GROUP2COMPILE"; - const char *compile_table_name = "COMPILE"; + const char *g2c_table_name = "GROUP2COMPILE_DEFAULT"; + const char *compile_table_name = "COMPILE_DEFAULT"; const char *ip_table_name = "IP_PLUS_CONFIG"; const char *expr_table_name = "HTTP_URL"; const char *keywords = "IP&string"; diff --git a/test/maat_framework_perf_gtest.cpp b/test/maat_framework_perf_gtest.cpp index 440247d..02ff0a0 100644 --- a/test/maat_framework_perf_gtest.cpp +++ b/test/maat_framework_perf_gtest.cpp @@ -306,11 +306,11 @@ static void test_add_expr_command(struct maat *maat_inst, const char *table_name const char *keywords) { long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0); + int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); EXPECT_EQ(ret, 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, compile_id, 0, table_name, 1, 0); EXPECT_EQ(ret, 1); @@ -324,11 +324,11 @@ static void test_add_ip_command(struct maat *maat_inst, const char *table_name, const char *ip, uint16_t port) { long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0); + int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); EXPECT_EQ(ret, 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, compile_id, 0, table_name, 1, 0); EXPECT_EQ(ret, 1); @@ -342,11 +342,11 @@ static void test_add_integer_command(struct maat *maat_inst, const char *table_n int low_bound, int up_bound) { long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0); + int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); EXPECT_EQ(ret, 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, compile_id, 0, table_name, 1, 0); EXPECT_EQ(ret, 1); @@ -360,11 +360,11 @@ static void test_add_flag_command(struct maat *maat_inst, const char *table_name long long flag, long long flag_mask) { long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); - int ret = compile_table_set_line(maat_inst, "COMPILE", MAAT_OP_ADD, compile_id, "null", 1, 0); + int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, compile_id, "null", 1, 0); EXPECT_EQ(ret, 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); - ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE", MAAT_OP_ADD, group_id, + ret = group2compile_table_set_line(maat_inst, "GROUP2COMPILE_DEFAULT", MAAT_OP_ADD, group_id, compile_id, 0, table_name, 1, 0); EXPECT_EQ(ret, 1); diff --git a/test/maat_json.json b/test/maat_json.json index a0f1d82..19184a8 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -1872,27 +1872,6 @@ } ] }, - { - "compile_id": 175, - "service": 0, - "action": 0, - "do_blacklist": 0, - "do_log": 0, - "user_region": "ipv4_composition.match", - "is_valid": "yes", - "groups": [ - { - "group_name": "ipv4_composition.source", - "virtual_table": "COMPOSITION_IP_SOURCE", - "not_flag": 0 - }, - { - "group_name": "ipv4_composition.destination", - "virtual_table": "COMPOSITION_IP_DESTINATION", - "not_flag": 0 - } - ] - }, { "compile_id": 176, "service": 0, @@ -1925,23 +1904,6 @@ } ] }, - { - "compile_id": 177, - "service": 0, - "action": 0, - "do_blacklist": 0, - "do_log": 0, - "user_region": "ipv4_composition.session.match", - "is_valid": "yes", - "groups": [ - { - "group_name": "ipv4_composition.session", - "virtual_table": "COMPOSITION_IP_SESSION", - "not_flag": 0, - "clause_index": 1 - } - ] - }, { "compile_id": 178, "service": 1, @@ -2003,45 +1965,6 @@ } ] }, - { - "compile_id": 180, - "service": 0, - "action": 0, - "do_blacklist": 0, - "do_log": 0, - "user_region": "Hierarchy_VirtualWithTwoPhysical", - "is_valid": "yes", - "groups": [ - { - "group_name": "FQDN_OBJ1", - "virtual_table": "VIRTUAL_SSL_SNI", - "not_flag": 0, - "clause_index": 0 - }, - { - "group_name": "FQDN_CAT1", - "virtual_table": "VIRTUAL_SSL_SNI", - "not_flag": 0, - "clause_index": 0 - } - ] - }, - { - "compile_id": 181, - "service": 0, - "action": 0, - "do_blacklist": 0, - "do_log": 0, - "user_region": "ipv4_composition.match", - "is_valid": "yes", - "groups": [ - { - "group_name": "IPv4-composition-source-only", - "virtual_table": "COMPOSITION_IP_SOURCE", - "not_flag": 0 - } - ] - }, { "compile_id": 182, "service": 1, @@ -2127,27 +2050,6 @@ } ] }, - { - "compile_id": 185, - "service": 0, - "action": 0, - "do_blacklist": 0, - "do_log": 0, - "user_region": "ipv4_composition.NOT_match", - "is_valid": "yes", - "groups": [ - { - "group_name": "IPv4-composition-NOT-client-ip", - "virtual_table": "COMPOSITION_IP_SOURCE", - "not_flag": 0 - }, - { - "group_name": "IPv4-composition-NOT-server-ip", - "virtual_table": "COMPOSITION_IP_DESTINATION", - "not_flag": 1 - } - ] - }, { "compile_id": 186, "service": 1, @@ -2560,7 +2462,7 @@ "do_blacklist": 1, "do_log": 1, "user_region": "Something:I\\bhave\\ba\\bname,7799", - "compile_table_name": "COMPILE_FIREWALL", + "compile_table_name": "COMPILE_FIREWALL_DEFAULT", "is_valid": "yes", "groups": [ { diff --git a/test/table_info.conf b/test/table_info.conf index e461e00..f376982 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -1,12 +1,10 @@ [ { "table_id":0, - "table_name":"COMPILE", - "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"], + "table_name":"COMPILE_DEFAULT", "table_type":"compile", "valid_column":8, "custom": { - "gc_timeout_s": 3, "compile_id":1, "tags":6, "clause_num":9 @@ -14,22 +12,20 @@ }, { "table_id":1, - "table_name":"GROUP2COMPILE", - "db_tables":["GROUP2COMPILE_DEFAULT", "GROUP2COMPILE_ALIAS"], - "table_type":"group2compile", - "associated_compile_table_id":0, - "valid_column":3, + "table_name":"COMPILE_ALIAS", + "table_type":"compile", + "valid_column":8, "custom": { - "group_id":1, - "compile_id":2, - "not_flag":4, - "virtual_table_name":5, - "clause_index":6 + "compile_id":1, + "tags":6, + "clause_num":9 } - }, + }, { "table_id":2, - "table_name":"COMPILE_FIREWALL", + "table_name":"COMPILE_CONJUNCTION", + "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"], + "default_compile_table":2, "table_type":"compile", "valid_column":8, "custom": { @@ -40,7 +36,8 @@ }, { "table_id":3, - "table_name":"GROUP2COMPILE_FIREWALL", + "table_name":"GROUP2COMPILE", + "db_tables":["GROUP2COMPILE_DEFAULT", "GROUP2COMPILE_ALIAS"], "table_type":"group2compile", "associated_compile_table_id":2, "valid_column":3, @@ -54,6 +51,43 @@ }, { "table_id":4, + "table_name":"COMPILE_FIREWALL_DEFAULT", + "table_type":"compile", + "valid_column":8, + "custom": { + "compile_id":1, + "tags":6, + "clause_num":9 + } + }, + { + "table_id":5, + "table_name":"COMPILE_FIREWALL_CONJUNCTION", + "db_tables":["COMPILE_FIREWALL_DEFAULT"], + "table_type":"compile", + "valid_column":8, + "custom": { + "compile_id":1, + "tags":6, + "clause_num":9 + } + }, + { + "table_id":6, + "table_name":"GROUP2COMPILE_FIREWALL", + "table_type":"group2compile", + "associated_compile_table_id":5, + "valid_column":3, + "custom": { + "group_id":1, + "compile_id":2, + "not_flag":4, + "virtual_table_name":5, + "clause_index":6 + } + }, + { + "table_id":7, "table_name":"GROUP2GROUP", "table_type":"group2group", "valid_column":4, @@ -64,7 +98,33 @@ } }, { - "table_id":5, + "table_id":8, + "table_name":"COMPILE_PLUGIN", + "db_tables":["COMPILE_DEFAULT", "COMPILE_ALIAS"], + "table_type":"plugin", + "valid_column":8, + "custom": { + "gc_timeout_s":3, + "key_type":"integer", + "key_len":8, + "key":1 + } + }, + { + "table_id":9, + "table_name":"COMPILE_FIREWALL_PLUGIN", + "db_tables":["COMPILE_FIREWALL_DEFAULT"], + "table_type":"plugin", + "valid_column":8, + "custom": { + "gc_timeout_s":3, + "key_type":"integer", + "key_len":8, + "key":1 + } + }, + { + "table_id":10, "table_name":"HTTP_REGION", "db_tables":["HTTP_URL", "HTTP_HOST"], "table_type":"expr", @@ -79,7 +139,7 @@ } }, { - "table_id":6, + "table_id":11, "table_name":"KEYWORDS_TABLE", "table_type":"expr", "valid_column":7, @@ -93,7 +153,7 @@ } }, { - "table_id":7, + "table_id":12, "table_name":"IP_CONFIG", "table_type":"ip_plus", "valid_column":11, @@ -111,7 +171,7 @@ } }, { - "table_id":8, + "table_id":13, "table_name":"CONTENT_SIZE", "table_type":"intval", "valid_column":5, @@ -123,18 +183,19 @@ } }, { - "table_id":9, + "table_id":14, "table_name":"QD_ENTRY_INFO", "table_type":"plugin", "valid_column":4, "custom": { + "gc_timeout_s":3, "key_type":"integer", "key_len":8, "key":1 } }, { - "table_id":10, + "table_id":15, "table_name":"HTTP_SIGNATURE", "table_type":"expr_plus", "valid_column":8, @@ -149,7 +210,7 @@ } }, { - "table_id":11, + "table_id":16, "table_name":"IMAGE_FP", "table_type":"expr", "valid_column":7, @@ -163,11 +224,12 @@ } }, { - "table_id":12, + "table_id":17, "table_name":"TEST_EFFECTIVE_RANGE_TABLE", "table_type":"plugin", "valid_column":4, "custom": { + "gc_timeout_s":3, "key_type":"integer", "key_len":8, "key":1, @@ -175,11 +237,12 @@ } }, { - "table_id":13, + "table_id":18, "table_name":"TEST_FOREIGN_KEY", "table_type":"plugin", "valid_column":4, "custom": { + "gc_timeout_s":3, "key_type":"pointer", "key":2, "tag":3, @@ -187,7 +250,7 @@ } }, { - "table_id":14, + "table_id":19, "table_name":"TEST_PLUGIN_EXDATA_TABLE", "table_type":"plugin", "valid_column":4, @@ -199,18 +262,19 @@ } }, { - "table_id":15, + "table_id":20, "table_name":"IR_INTERCEPT_IP", "table_type":"plugin", "valid_column":14, "custom": { + "gc_timeout_s":3, "key_type":"pointer", "key":2, "tag":18 } }, { - "table_id":16, + "table_id":21, "table_name":"APP_PAYLOAD", "table_type":"expr_plus", "valid_column":8, @@ -225,7 +289,7 @@ } }, { - "table_id":17, + "table_id":22, "table_name":"TROJAN_PAYLOAD", "table_type":"expr", "valid_column":7, @@ -240,7 +304,7 @@ } }, { - "table_id":18, + "table_id":23, "table_name":"MAIL_ADDR", "table_type":"expr", "valid_column":7, @@ -254,7 +318,7 @@ } }, { - "table_id":19, + "table_id":24, "table_name":"IP_PLUS_CONFIG", "table_type":"ip_plus", "valid_column":11, @@ -272,32 +336,32 @@ } }, { - "table_id":20, + "table_id":25, "table_name":"HTTP_RESPONSE_KEYWORDS", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":21, + "table_id":26, "table_name":"HTTP_REQUEST_HEADER", "table_type":"virtual", "physical_table": "HTTP_SIGNATURE" }, { - "table_id":22, + "table_id":27, "table_name":"HTTP_RESPONSE_HEADER", "table_type":"virtual", "physical_table": "HTTP_SIGNATURE" }, { - "table_id":23, + "table_id":28, "table_name":"VIRTUAL_IP_PLUS_TABLE", "db_tables":["VIRTUAL_IP_PLUS_SOURCE", "VIRTUAL_IP_PLUS_DESTINATION"], "table_type":"virtual", "physical_table": "IP_PLUS_CONFIG" }, { - "table_id":24, + "table_id":29, "table_name":"TEST_IP_PLUGIN_WITH_EXDATA", "table_type":"ip_plugin", "valid_column":6, @@ -310,7 +374,7 @@ } }, { - "table_id":25, + "table_id":30, "table_name":"AS_NUMBER", "table_type":"expr", "valid_column":7, @@ -324,19 +388,19 @@ } }, { - "table_id":26, + "table_id":31, "table_name":"SOURCE_IP_ASN", "table_type":"virtual", "physical_table":"AS_NUMBER" }, { - "table_id":27, + "table_id":32, "table_name":"DESTINATION_IP_ASN", "table_type":"virtual", "physical_table":"AS_NUMBER" }, { - "table_id":28, + "table_id":33, "table_name":"GeoLocation", "table_type":"expr", "valid_column":7, @@ -350,13 +414,13 @@ } }, { - "table_id":29, + "table_id":34, "table_name":"SOURCE_IP_GEO", "table_type":"virtual", "physical_table":"GeoLocation" }, { - "table_id":30, + "table_id":35, "table_name":"INTERGER_PLUS", "table_type":"intval_plus", "valid_column":6, @@ -369,7 +433,7 @@ } }, { - "table_id":31, + "table_id":36, "table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA", "table_type":"fqdn_plugin", "valid_column":5, @@ -381,7 +445,7 @@ } }, { - "table_id":32, + "table_id":37, "table_name":"APP_ID", "table_type":"intval", "valid_column":5, @@ -393,7 +457,7 @@ } }, { - "table_id":33, + "table_id":38, "table_name":"EMPTY_KEYWORD", "table_type":"expr", "valid_column":7, @@ -407,7 +471,7 @@ } }, { - "table_id":34, + "table_id":39, "table_name":"EMPTY_INTERGER", "table_type":"intval", "valid_column":5, @@ -419,7 +483,7 @@ } }, { - "table_id":35, + "table_id":40, "table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA", "table_type":"bool_plugin", "valid_column":4, @@ -430,7 +494,7 @@ } }, { - "table_id":36, + "table_id":41, "table_name":"FLAG_CONFIG", "table_type":"flag", "valid_column":5, @@ -442,7 +506,7 @@ } }, { - "table_id":37, + "table_id":42, "table_name":"FLAG_PLUS_CONFIG", "table_type":"flag_plus", "valid_column":6, @@ -455,11 +519,12 @@ } }, { - "table_id":38, + "table_id":43, "table_name":"TEST_PLUGIN_LONG_KEY_TYPE_TABLE", "table_type":"plugin", "valid_column":4, "custom": { + "gc_timeout_s":3, "key_type":"integer", "key_len":8, "key":2, @@ -467,11 +532,12 @@ } }, { - "table_id":39, + "table_id":44, "table_name":"TEST_PLUGIN_INT_KEY_TYPE_TABLE", "table_type":"plugin", "valid_column":4, "custom": { + "gc_timeout_s":3, "key_type":"integer", "key_len":4, "key":2, @@ -479,24 +545,25 @@ } }, { - "table_id":40, + "table_id":45, "table_name":"TEST_PLUGIN_IP_KEY_TYPE_TABLE", "table_type":"plugin", "valid_column":4, "custom": { + "gc_timeout_s":3, "key_type":"ip_addr", "addr_type":1, "key":2 } }, { - "table_id":41, + "table_id":46, "table_name":"HTTP_URL_FILTER", "table_type":"virtual", "physical_table": "HTTP_URL" }, { - "table_id":42, + "table_id":47, "table_name":"IP_PERF_CONFIG", "table_type":"ip_plus", "valid_column":11, @@ -514,7 +581,7 @@ } }, { - "table_id":43, + "table_id":48, "table_name":"INTEGER_PERF_CONFIG", "table_type":"intval", "valid_column":5, @@ -526,7 +593,7 @@ } }, { - "table_id":44, + "table_id":49, "table_name":"EXPR_LITERAL_PERF_CONFIG", "table_type":"expr", "valid_column":7, @@ -540,7 +607,7 @@ } }, { - "table_id":45, + "table_id":50, "table_name":"EXPR_REGEX_PERF_CONFIG", "table_type":"expr", "valid_column":7, @@ -554,7 +621,7 @@ } }, { - "table_id":46, + "table_id":51, "table_name":"FLAG_PERF_CONFIG", "table_type":"flag", "valid_column":5, @@ -566,7 +633,7 @@ } }, { - "table_id":47, + "table_id":52, "table_name":"TEST_IPPORT_PLUGIN_WITH_EXDATA", "table_type":"ipport_plugin", "valid_column":6, @@ -580,67 +647,67 @@ } }, { - "table_id":48, + "table_id":53, "table_name":"VIRTUAL_IP_CONFIG", "table_type":"virtual", "physical_table": "IP_CONFIG" }, { - "table_id":49, + "table_id":54, "table_name":"HTTP_RESPONSE_KEYWORDS_1", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":50, + "table_id":55, "table_name":"HTTP_RESPONSE_KEYWORDS_2", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":51, + "table_id":56, "table_name":"HTTP_RESPONSE_KEYWORDS_3", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":52, + "table_id":57, "table_name":"HTTP_RESPONSE_KEYWORDS_4", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":53, + "table_id":58, "table_name":"HTTP_RESPONSE_KEYWORDS_5", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":54, + "table_id":59, "table_name":"HTTP_RESPONSE_KEYWORDS_6", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":55, + "table_id":60, "table_name":"HTTP_RESPONSE_KEYWORDS_7", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":56, + "table_id":61, "table_name":"HTTP_RESPONSE_KEYWORDS_8", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":57, + "table_id":62, "table_name":"HTTP_DUMMY", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE" }, { - "table_id":58, + "table_id":63, "table_name":"HTTP_NOT_LOGIC", "table_type":"virtual", "physical_table": "KEYWORDS_TABLE"