diff --git a/include/maat/maat.h b/include/maat/maat.h index d42bc7f..83def00 100644 --- a/include/maat/maat.h +++ b/include/maat/maat.h @@ -32,34 +32,45 @@ enum ip_type { IP_TYPE_V6 }; +struct maat_hit_path { + int Nth_scan; + int item_id; + int sub_group_id; + int top_group_id; + int virtual_table_id; // 0 is not a virtual table. + int compile_id; +}; + +struct maat_matched { + int virtual_table_id; + int group_id; +}; + enum maat_scan_opt { MAAT_SET_SCAN_DISTRICT = 1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default. MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination. - MAAT_GET_SCAN_HIT_PATH //VALUE is struct Maat_hit_path_t*, an array of struct Maat_hit_path_t, SIZE= sizeof(struct Maat_hit_path_t)*N, + MAAT_GET_SCAN_HIT_PATH, //VALUE is struct maat_hit_path*, an array of struct maat_hit_path, SIZE= sizeof(struct maat_hit_path)*N, //Maat_get_scan_status returns actual got number. + MAAT_GET_SCAN_MATCHED //VALUE is struct maat_matched*, an array of struct maat_matched, SIZE= sizeof(struct maat_matched)*N, }; /* network order */ -struct ipv4_4tuple { +struct ipv4_2tuple { uint32_t sip; uint32_t dip; - uint16_t sport; - uint16_t dport; }; -struct ipv6_4tuple { +struct ipv6_2tuple { uint8_t sip[16]; uint8_t dip[16]; - uint16_t sport; - uint16_t dport; }; -struct addr_4tuple { +struct addr_2tuple { enum ip_type type; union { - struct ipv4_4tuple ipv4; - struct ipv6_4tuple ipv6; + struct ipv4_2tuple ipv4; + struct ipv6_2tuple ipv6; }; }; @@ -83,18 +94,15 @@ typedef void maat_rule_ex_dup_func_t(int idx, void *to, void *from, long argl, v /* maat_instance options API */ struct maat_options; struct maat_options* maat_options_new(void); -int maat_options_set_worker_thread_number(struct maat_options *opts, size_t nr_worker_threads); +int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread); int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms); int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms); int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms); int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len); int maat_options_set_deferred_load_on(struct maat_options *opts); -int maat_options_set_iris_full_index_dir(struct maat_options *opts, const char *full_idx_dir); -int maat_options_set_iris_inc_index_dir(struct maat_options *opts, const char *inc_idx_dir); +int maat_options_set_iris(struct maat_options *opts, const char *full_directory, const char *increment_directory); int maat_options_set_json_file(struct maat_options *opts, const char *json_filename); -int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip); -int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port); -int maat_options_set_redis_db(struct maat_options *opts, int db_index); +int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, uint16_t redis_port, int redis_db); int maat_options_set_logger(struct maat_options *opts, void *logger); /* maat_instance API */ @@ -111,12 +119,13 @@ int maat_table_callback_register(struct maat *instance, int table_id, void *u_para); /* maat plugin table API */ int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id, - maat_plugin_ex_new_func_t *new_func, - maat_plugin_ex_free_func_t *free_func, - maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp); - -void *maat_plugin_table_dup_ex_data(struct maat *instance, int table_id, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp); +/* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register, + caller is responsible to free the data. */ +void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id, const char *key, size_t key_len); /* maat scan API */ struct maat_state; @@ -125,7 +134,7 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id, struct maat_state **state); int maat_scan_ip(struct maat *instance, int table_id, int thread_id, - struct addr_4tuple *addr, int results[], size_t *n_result, + struct addr_2tuple *addr, int results[], size_t *n_result, struct maat_state **state); int maat_scan_string(struct maat *instance, int table_id, int thread_id, @@ -140,12 +149,16 @@ int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* dat void maat_scan_stream_close(struct maat_stream **stream); +/* maat state API */ int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, const void *value, int size); //return >=0 if success, return -1 when failed; int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, void *value, int size); -void maat_state_reset(struct maat_state **state); +void maat_state_free(struct maat_state **state); + +/* return matched compile_id */ +int maat_matched_compile_id(struct maat *instance, struct maat_matched *matched); #ifdef __cpluscplus } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 186b72e..4aa78c9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,15 +29,15 @@ set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z) # Shared Library Output -#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) -#set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX) -#set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe) -#set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) -#set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}) -#set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION}) +add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) +set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe) +set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) +set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}) +set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION}) -#target_link_libraries(maat_frame_shared hyperscan_static) +target_link_libraries(maat_frame_shared hyperscan_static) # install set(CMAKE_INSTALL_PREFIX /opt/MESA/) -install(FILES ${PROJECT_SOURCE_DIR}/inc/Maat_rule.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) \ No newline at end of file +#install(FILES ${PROJECT_SOURCE_DIR}/include/maat DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) \ No newline at end of file diff --git a/src/inc_internal/maat_common.h b/src/inc_internal/maat_common.h index 62c997f..17646c2 100644 --- a/src/inc_internal/maat_common.h +++ b/src/inc_internal/maat_common.h @@ -23,6 +23,7 @@ extern "C" struct maat_options { char instance_name[NAME_MAX]; + char compile_tablename[NAME_MAX]; size_t nr_worker_threads; int rule_effect_interval_ms; int rule_update_checking_interval_ms; diff --git a/src/inc_internal/maat_group.h b/src/inc_internal/maat_group.h new file mode 100644 index 0000000..511b4f9 --- /dev/null +++ b/src/inc_internal/maat_group.h @@ -0,0 +1,66 @@ +/* +********************************************************************************************** +* File: maat_group.h +* Description: +* Authors: Liu wentan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_GROUP_H_ +#define _MAAT_GROUP_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +struct maat_item { + int item_id; + int group_id; + + struct maat_group *ref_parent_group; + UT_hash_handle hh; + + void *user_data; +}; + +struct maat_group { + igraph_integer_t vertex_id; + int group_id; + int ref_by_compile_cnt; + int ref_by_superior_group_cnt; + int ref_by_subordinate_group_cnt; + int ref_by_item_cnt; + + size_t top_group_cnt; + int *top_group_ids; + UT_hash_handle hh_group_id; + UT_hash_handle hh_vertex_id; +}; + +struct maat_group_topology; + +struct maat_group_topology *maat_group_topology_new(struct log_handle *logger); +void maat_group_topology_free(struct maat_group_topology *group_topo); + +struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id); +void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group); + +/** + * @retval if not found, return NULL +*/ +struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id); + +int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id); +int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id); + +/* build top groups */ +int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_hierarchy.h b/src/inc_internal/maat_hierarchy.h index 99b0980..44f4121 100644 --- a/src/inc_internal/maat_hierarchy.h +++ b/src/inc_internal/maat_hierarchy.h @@ -23,46 +23,43 @@ struct maat_hierarchy; struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger); void maat_hierarchy_free(struct maat_hierarchy *hier); int maat_hierarchy_rebuild(struct maat_hierarchy *hier); -size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - struct maat_hit_path_t *hit_paths, size_t n_path); -void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)); -void maat_hierarchy_set_region_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)); +size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, + struct maat_hit_path *hit_paths, size_t n_path); /* maat hierarchy compile mid API */ struct maat_hierarchy_compile_mid; struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id); void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid); void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result); + int item_id, int virtual_table_id, int Nth_scan, int Nth_item_result); int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid); /* maat hierarchy compile API */ int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, int declared_clause_num, void* user_data); int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id); +void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)); void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id); void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id); void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param); -int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - int is_last_compile, void **user_data_array, size_t ud_array_sz); -void *maat_hierarchy_region_dettach_user_data(struct maat_hierarchy *hier, int region_id); - -/* maat hierarchy region2group API */ -int maat_hierarchy_add_region_to_group(struct maat_hierarchy *hier, int group_id, int region_id, int table_id, void* user_data); -int maat_hierarchy_remove_region_from_group(struct maat_hierarchy *hier, int group_id, int region_id); - -/* maat hierarchy group2group API */ -int maat_hierarchy_add_group_to_group(struct maat_hierarchy *hier, int group_id, int superior_group_id); -int maat_hierarchy_remove_group_from_group(struct maat_hierarchy *hier, int group_id, int superior_group_id); - /* maat hierarchy group2compile API */ int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag, int clause_index, int compile_id); int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag, int clause_index, int compile_id); +/* maat hierarchy item API */ +int maat_hierarchy_item_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, + int is_last_compile, void **user_data_array, size_t ud_array_sz); +void *maat_hierarchy_item_dettach_user_data(struct maat_hierarchy *hier, int item_id); +void maat_hierarchy_set_item_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)); + +/* maat hierarchy item2group API */ +int maat_hierarchy_add_item_to_group(struct maat_hierarchy *hier, int group_id, int item_id, void* user_data); +int maat_hierarchy_remove_item_from_group(struct maat_hierarchy *hier, int group_id, int item_id); + #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 422214f..b4e22e6 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -53,34 +53,46 @@ struct maat_rule { char service_defined[MAX_SERVICE_DEFINE_LEN]; }; -#define REGION_RULE_MAGIC 0x4d3c2b1a -struct maat_region_inner -{ +#define ITEM_RULE_MAGIC 0x4d3c2b1a +struct maat_item_inner { long long magic_num; - int region_id; + int item_id; int group_id; int district_id; - int table_id; int expr_id_cnt; int expr_id_lb; //low boundary int expr_id_ub; //up boundary }; #define COMPILE_RULE_MAGIC 0x1a2b3c4d -struct maat_compile_rule -{ +struct maat_compile_rule { long long magic_num; + int compile_id; struct maat_rule_head head;// fix len of Maat_rule_t char *service_defined; int is_valid; int declared_clause_num; double evaluation_order; struct table_schema *ref_table; - void *ex_data; - int compile_id; + void **ex_data; pthread_rwlock_t rwlock; }; +struct group2compile_rule { + int group_id; + int compile_id; + int is_valid; + int not_flag; + int virtual_table_id; + int clause_index; +}; + +struct group2group_rule { + int group_id; + int superior_group_id; + int is_valid; +}; + struct maat_runtime { /* maat_runtime can be created and destroy dynamic, so need version info */ long long version; @@ -94,10 +106,8 @@ struct maat_runtime { size_t max_thread_num; uint32_t rule_num; - struct maat_hierarchy *hier; struct maat_garbage_bin *ref_garbage_bin; - - struct scan_result *region_result_buff; + struct scan_result *item_result_buff; struct log_handle *logger; }; @@ -158,6 +168,11 @@ struct expected_reply { redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE]; }; +struct rule_tag { + char *tag_name; + char *tag_val; +}; + struct maat { char instance_name[NAME_MAX]; @@ -173,6 +188,9 @@ struct maat { struct source_redis_ctx mr_ctx; }; + struct rule_tag *accept_tags; + int n_accept_tag; + struct log_handle *logger; int deferred_load; @@ -192,10 +210,10 @@ struct maat { struct maat_garbage_bin *garbage_bin; - char compile_tn[NAME_MAX]; - char group_tn[NAME_MAX]; - char group2compile_tn[NAME_MAX]; - char group2group_tn[NAME_MAX]; + char compile_tablename[NAME_MAX]; + //char group_tn[NAME_MAX]; + //char group2compile_tn[NAME_MAX]; + //char group2group_tn[NAME_MAX]; char decrypt_key[NAME_MAX]; char decrypt_algo[NAME_MAX]; @@ -263,6 +281,8 @@ void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule); void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id, const char *table_name, const char *line, long long timeout); +void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, const char *srv_def, int srv_def_len); + #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_table_runtime.h b/src/inc_internal/maat_table_runtime.h index 8101d0b..fa59ae2 100644 --- a/src/inc_internal/maat_table_runtime.h +++ b/src/inc_internal/maat_table_runtime.h @@ -20,9 +20,8 @@ extern "C" #include "maat_table_schema.h" #include "maat_garbage_collection.h" -struct table_rt_2tuple { +struct ip_addr { enum ip_type ip_type; - uint16_t port; union { uint32_t ipv4; uint32_t ipv6[4]; @@ -36,7 +35,7 @@ struct table_runtime_manager; /* table runtime manager API */ struct table_runtime_manager * table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num, - struct maat_garbage_bin *bin); + struct maat_garbage_bin *bin, struct log_handle *logger); void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr); @@ -57,7 +56,7 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t */ int table_runtime_updating_flag(struct table_runtime *table_rt); -void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, struct log_handle *logger); +void table_runtime_commit(struct table_runtime *table_rt, long long version, size_t nr_worker_thread, struct log_handle *logger); /* table runtime scan API */ int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len, @@ -67,7 +66,7 @@ void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id); int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result); void table_runtime_stream_close(struct table_runtime *table_rt); -int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, struct scan_result *results, size_t *n_result, size_t n_result_array); +int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct ip_addr *data, struct scan_result *results, size_t *n_result, size_t n_result_array); /* table runtime cached row API */ size_t table_runtime_cached_row_count(struct table_runtime *table_rt); @@ -78,7 +77,7 @@ const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt); void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, - int nr_worker_num, struct log_handle *logger); + int nr_worker_thread, long long version, struct log_handle *logger); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_table_schema.h b/src/inc_internal/maat_table_schema.h index f1839ab..0a4c8d9 100644 --- a/src/inc_internal/maat_table_schema.h +++ b/src/inc_internal/maat_table_schema.h @@ -17,15 +17,18 @@ extern "C" #endif #include +#include #include "maat/maat.h" #include "adapter_hs.h" -#define MAX_TABLE_NUM 256 -#define MAX_DISTRICT_STR 128 -#define MAX_IP_STR 128 -#define MAX_KEYWORDS_STR 1024 +#define MAX_TABLE_NUM 256 +#define MAX_DISTRICT_STR 128 +#define MAX_IP_STR 128 +#define MAX_KEYWORDS_STR 1024 #define MAX_FOREIGN_CLMN_NUM 8 +#define MAX_TABLE_LINE_SIZE (1024 * 16) +#define MAX_COMPILE_EX_DATA_NUM 2 enum component_table_type { COMPONENT_TABLE_TYPE_NONE = -1, @@ -85,6 +88,34 @@ enum match_method { MATCH_METHOD_MAX }; +struct compile_item { + int compile_id; + int service_id; + int action; + int do_blacklist; + int do_log; + char user_region[MAX_TABLE_LINE_SIZE]; + int is_valid; + int clause_num; + int evaluation_order; +}; + +struct group2compile_item { + int group_id; + int compile_id; + int is_valid; + int not_flag; + int virtual_table_id; + int clause_index; + int associated_compile_table_id; +}; + +struct group2group_item { + int group_id; + int superior_group_id; + int is_valid; +}; + struct expr_item { int item_id; int group_id; @@ -139,13 +170,14 @@ struct ip_plugin_item { char end_ip[MAX_IP_STR]; int is_valid; int rule_tag; - int have_exdata; - void *ex_data; }; struct table_item { enum table_type table_type; union { + struct compile_item compile_item; + struct group2compile_item g2c_item; + struct group2group_item g2g_item; struct expr_item expr_item; struct ip_plus_item ip_plus_item; struct plugin_item plugin_item; @@ -214,14 +246,19 @@ enum table_type table_schema_get_table_type(struct table_schema *table_schema); int table_schema_get_table_id(struct table_schema *table_schema); +/* get group2compile table's associated compile table id */ +int table_schema_get_associated_table_id(struct table_schema *table_schema); + enum scan_type table_schema_get_scan_type(struct table_schema *table_schema); struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema, - struct log_handle *logger); + struct rule_tag *accept_tags, int n_accept_tag, struct log_handle *logger); +void table_item_free(struct table_item *table_item); int table_schema_get_valid_flag_column(struct table_schema *table_schema); void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name); +const char *table_schema_get_updating_name(struct table_schema *table_schema); /* expr table schema API */ enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema); diff --git a/src/maat_api.cpp b/src/maat_api.cpp index 10bd428..28e3b77 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -33,8 +33,7 @@ #define DISTRICT_ANY -1 #define DISTRICT_UNKNOWN -2 -struct maat_state -{ +struct maat_state { struct maat *maat_instance; int16_t thread_id; unsigned char is_set_district; @@ -44,6 +43,26 @@ struct maat_state struct maat_hierarchy_compile_mid *compile_mid; }; +struct scan_item_hit_wrapper { + int Nth_scan; + struct maat_item_inner* wrapped_items[MAX_SCANNER_HIT_NUM]; + size_t n_wrapped_item; + + int *virtual_table_ids; + int virtual_table_id; + int is_last_item; +}; + +inline int scan_state_should_compile_NOT(struct maat_state *mid) +{ + if (mid && mid->is_last_scan==1 && mid->compile_mid && + maat_hierarchy_compile_mid_has_NOT_clause(mid->compile_mid)) { + return 1; + } else { + return 0; + } +} + struct maat_options* maat_options_new(void) { struct maat_options *options = ALLOC(struct maat_options, 1); @@ -59,9 +78,9 @@ struct maat_options* maat_options_new(void) return options; } -int maat_options_set_worker_thread_number(struct maat_options *opts, size_t n_worker_threads) +int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread) { - opts->nr_worker_threads = n_worker_threads; + opts->nr_worker_threads = n_thread; return 0; } @@ -101,25 +120,14 @@ int maat_options_set_deferred_load_on(struct maat_options *opts) return 0; } -int maat_options_set_iris_full_index_dir(struct maat_options *opts, const char *full_idx_dir) +int maat_options_set_iris(struct maat_options *opts, const char *full_directory, const char *increment_directory) { - if (strlen(full_idx_dir) >= NAME_MAX) { + if (strlen(full_directory) >= NAME_MAX || strlen(increment_directory) >= NAME_MAX) { return -1; } - memcpy(opts->iris_ctx.full_idx_dir, full_idx_dir, strlen(full_idx_dir)); - opts->input_mode = DATA_SOURCE_IRIS_FILE; - - return 0; -} - -int maat_options_set_iris_inc_index_dir(struct maat_options *opts, const char *inc_idx_dir) -{ - if (strlen(inc_idx_dir) >= NAME_MAX) { - return -1; - } - - memcpy(opts->iris_ctx.inc_idx_dir, inc_idx_dir, strlen(inc_idx_dir)); + memcpy(opts->iris_ctx.full_idx_dir, full_directory, strlen(full_directory)); + memcpy(opts->iris_ctx.inc_idx_dir, increment_directory, strlen(increment_directory)); opts->input_mode = DATA_SOURCE_IRIS_FILE; return 0; @@ -133,24 +141,12 @@ int maat_options_set_json_file(struct maat_options *opts, const char *json_filen return 0; } -int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip) +int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, uint16_t redis_port, int redis_db) { memcpy(opts->redis_ctx.redis_ip, redis_ip, strlen(redis_ip)); - opts->input_mode = DATA_SOURCE_REDIS; - - return 0; -} - -int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port) -{ opts->redis_ctx.redis_port = redis_port; - - return 0; -} - -int maat_options_set_redis_db(struct maat_options *opts, int db_index) -{ - opts->redis_ctx.redis_db = db_index; + opts->redis_ctx.redis_db = redis_db; + opts->input_mode = DATA_SOURCE_REDIS; return 0; } @@ -254,6 +250,9 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) goto failed; } + if (opts->compile_tablename != NULL) { + memcpy(maat_instance->compile_tablename, opts->compile_tablename, strlen(opts->compile_tablename)); + } maat_instance->input_mode = opts->input_mode; switch (maat_instance->input_mode) { case DATA_SOURCE_REDIS: @@ -334,16 +333,6 @@ inline void maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id) alignment_int64_array_add(maat_rt->ref_cnt, thread_id, -1); } -inline int scan_state_should_compile_NOT(struct maat_state *mid) -{ - if (mid && (1 == mid->is_last_scan) && mid->compile_mid && - maat_hierarchy_compile_mid_has_NOT_clause(mid->compile_mid)) { - return 1; - } else { - return 0; - } -} - int maat_table_callback_register(struct maat *maat_instance, int table_id, maat_start_callback_t *start, maat_update_callback_t *update, @@ -410,15 +399,14 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i if (maat_instance->maat_rt != NULL) { table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id); table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->nr_worker_thread, - maat_instance->logger); + maat_instance->maat_rt->version, maat_instance->logger); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); return 0; } -void *maat_plugin_table_dup_ex_data(struct maat *maat_instance, int table_id, - const char *key, size_t key_len) +void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, const char *key, size_t key_len) { struct maat_runtime *maat_rt = maat_instance->maat_rt; if (NULL == maat_rt) { @@ -460,7 +448,7 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id) return outer_state; } -struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instance, int thread_id, int is_hit_region) +struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instance, int thread_id, int is_hit_item) { struct maat_state *mid = *state; @@ -478,7 +466,7 @@ struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instanc alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1); } - if (is_hit_region == 1) { + if (is_hit_item == 1) { if (NULL == mid->compile_mid) { mid->compile_mid = maat_hierarchy_compile_mid_new(maat_instance->maat_rt->hier, thread_id); alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1); @@ -487,6 +475,140 @@ struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instanc return mid; } +void scan_item_hit_wrapper_build(struct scan_item_hit_wrapper* wrapper, struct scan_result* results, size_t n_result, + int district_id, int is_last_item, int virtual_table_id, int Nth_scan) +{ + size_t i=0; + struct maat_item_inner *item = NULL; + wrapper->n_wrapped_item = 0; + wrapper->virtual_table_id = 0; + wrapper->virtual_table_ids = NULL; + + for (i = 0; i < n_result; i++) { + item = (struct maat_item_inner *)(results[i].tag); + if (item->district_id == district_id || district_id == DISTRICT_ANY) { + wrapper->wrapped_items[wrapper->n_wrapped_item] = item; + wrapper->n_wrapped_item++; + } + } + wrapper->is_last_item = is_last_item; + wrapper->virtual_table_id = virtual_table_id; + wrapper->Nth_scan = Nth_scan; + wrapper->virtual_table_ids = NULL; +} + +struct compile_sort_para { + double evaluation_order; + int declared_clause_num; + int compile_id; + void *user; +}; + +static void compile_sort_para_set(struct compile_sort_para *para, + const struct maat_compile_rule *compile_relation, void *user) +{ + para->compile_id=compile_relation->compile_id; + para->evaluation_order=compile_relation->evaluation_order; + para->declared_clause_num=compile_relation->declared_clause_num; + para->user = user; +} + +static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b) +{ + //If both of compile rule's evaluation order are specified, compile rule with small evaluation order is priority. + if(a->evaluation_order!=0 && b->evaluation_order!=0) + { + if(a->evaluation_order - b->evaluation_order <0) + { + return -1; + } + else if(a->evaluation_order - b->evaluation_order >0) + { + return 1; + } + } + //If one of compile rule's evaluation order is zero, compile rule with big evaluation order is priority. + else if(a->evaluation_order + b->evaluation_order!= 0) + { + return (a->evaluation_order - b->evaluation_order >0) ? -1 : 1; + } + //If compile rule's execute sequences are not specified or equal. + if(a->declared_clause_num!=b->declared_clause_num) + { + return (a->declared_clause_num-b->declared_clause_num); + } + else + { + return (b->compile_id-a->compile_id); + } + +} + +static int compare_compile_rule(const void *a, const void *b) +{ + const struct maat_compile_rule *ra=*(const struct Maat_compile_rule **)a; + const struct maat_compile_rule *rb=*(const struct Maat_compile_rule **)b; + + struct compile_sort_para sa, sb; + compile_sort_para_set(&sa, ra, NULL); + compile_sort_para_set(&sb, rb, NULL); + + return compile_sort_para_compare(&sa, &sb); +} + +int item_compile(struct maat *maat_instance, struct maat_hierarchy_compile_mid *compile_mid, + const struct scan_item_hit_wrapper *item_hit_wrapper, int *result, + int size, int thread_id) +{ + int is_last_item = item_hit_wrapper->is_last_item; + size_t item_hit_num = item_hit_wrapper->n_wrapped_item; + + int scan_ret=0; + int i=0; + + struct maat_compile_rule *compile_rule_array[size]; + struct maat_compile_rule *compile_rule = NULL; + int virtual_table_id = 0; + + struct maat_item_inner *item = NULL; + + for (i = 0; (size_t)i < item_hit_num;i++) { + item = item_hit_wrapper->wrapped_items[i]; + assert(item->magic_num == ITEM_RULE_MAGIC); + if (item_hit_wrapper->virtual_table_ids) { + virtual_table_id = item_hit_wrapper->virtual_table_ids[i]; + } else { + virtual_table_id = item_hit_wrapper->virtual_table_id; + } + maat_hierarchy_compile_mid_update(feather->scanner->hier, compile_mid, item->item_id, virtual_table_id, item_hit_wrapper->Nth_scan, i); + } + + scan_ret = maat_hierarchy_item_compile(feather->scanner->hier, compile_mid, is_last_item, (void **)compile_rule_array, size); + //Maat_hierarchy is rwlock protected, it always returns non-NULL compile_rule. + if (scan_ret > 1) { + qsort(compile_rule_array, scan_ret, sizeof(struct maat_compile_rule *), + compare_compile_rule); + } + for (i = 0; i < scan_ret && i < size; i++) { + compile_rule = compile_rule_array[i]; + assert(compile_rule->magic_num == COMPILE_RULE_MAGIC); + result[i] = compile_rule->compile_id; + /* + fill_maat_rule(&(result[i]), &(compile_rule->head), compile_rule->service_defined, + compile_rule->head.serv_def_len);*/ + } + + if (scan_ret > 0) { + alignment_int64_array_add(feather->hit_cnt, thread_id, 1); + } + + if (item_hit_num == 0 && scan_ret > 0) { + alignment_int64_array_add(feather->not_grp_hit_cnt, thread_id, 1); + } + + return MIN(scan_ret, size); +} + int maat_scan_integer(struct maat *instance, int table_id, int thread_id, unsigned int intval, int results[], size_t *n_result, struct maat_state **state) @@ -494,19 +616,17 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id, return 0; } -static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4tuple *addr, enum component_table_type child_type) +static int ip_scan_data_set(struct ip_addr *scan_data, struct addr_2tuple *addr, enum component_table_type child_type) { switch (addr->type) { case IP_TYPE_V4: - table_rt_addr->ip_type = IP_TYPE_V4; + scan_data->ip_type = IP_TYPE_V4; switch (child_type) { case COMPONENT_TABLE_TYPE_SIP: - table_rt_addr->ipv4 = ntohl(addr->ipv4.sip); - table_rt_addr->port = ntohs(addr->ipv4.sport); + scan_data->ipv4 = ntohl(addr->ipv4.sip); break; case COMPONENT_TABLE_TYPE_DIP: - table_rt_addr->ipv4 = ntohl(addr->ipv4.dip); - table_rt_addr->port = ntohs(addr->ipv4.dport); + scan_data->ipv4 = ntohl(addr->ipv4.dip); break; default: assert(0); @@ -514,17 +634,15 @@ static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4 } break; case IP_TYPE_V6: - table_rt_addr->ip_type = IP_TYPE_V6; + scan_data->ip_type = IP_TYPE_V6; switch (child_type) { case COMPONENT_TABLE_TYPE_SIP: - memcpy(table_rt_addr->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip)); - ipv6_ntoh(table_rt_addr->ipv6); - table_rt_addr->port = ntohs(addr->ipv6.sport); + memcpy(scan_data->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip)); + ipv6_ntoh(scan_data->ipv6); break; case COMPONENT_TABLE_TYPE_DIP: - memcpy(table_rt_addr->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip)); - ipv6_ntoh(table_rt_addr->ipv6); - table_rt_addr->port = ntohs(addr->ipv6.dport); + memcpy(scan_data->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip)); + ipv6_ntoh(scan_data->ipv6); break; default: assert(0); @@ -539,12 +657,12 @@ static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4 return 0; } -static int ip_composition_scan(int thread_id, struct addr_4tuple *addr, +static int ip_composition_scan(int thread_id, struct addr_2tuple *addr, int parent_table_id, enum component_table_type child_type, int *virtual_table_id, struct table_schema_manager *table_schema_mgr, struct table_runtime_manager *table_rt_mgr, - struct scan_result *region_results, size_t n_result_array) + struct scan_result *item_results, size_t n_result_array) { int child_table_id = 0; @@ -576,12 +694,12 @@ static int ip_composition_scan(int thread_id, struct addr_4tuple *addr, return 0; } - struct table_rt_2tuple scan_data; - memset(&scan_data, 0, sizeof(struct table_rt_2tuple)); + struct ip_addr scan_data; + memset(&scan_data, 0, sizeof(struct ip_addr)); ip_scan_data_set(&scan_data, addr, child_type); size_t hit_cnt = 0; - int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, region_results, &hit_cnt, n_result_array); + int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, item_results, &hit_cnt, n_result_array); if (ret < 0) { return -1; } @@ -589,7 +707,7 @@ static int ip_composition_scan(int thread_id, struct addr_4tuple *addr, return hit_cnt; } -int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_4tuple *addr, +int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_2tuple *addr, int results[], size_t *n_result, struct maat_state **state) { if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || @@ -615,49 +733,70 @@ int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct return -1; } - int region_ret = 0; + int item_ret = 0; int virtual_table_id = 0; - struct scan_result *region_result = maat_rt->region_result_buff + thread_id * MAX_SCANNER_HIT_NUM; - int region_hit_cnt = 0; - int region_result_virtual_table_ids[MAX_SCANNER_HIT_NUM]; + struct scan_result *item_result = maat_rt->item_result_buff + thread_id * MAX_SCANNER_HIT_NUM; + int item_hit_cnt = 0; + int item_result_virtual_table_ids[MAX_SCANNER_HIT_NUM]; alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); maat_runtime_ref_inc(maat_rt, thread_id); if (table_type == TABLE_TYPE_COMPOSITION) { - region_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_SIP, &virtual_table_id, + /* + item_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_SIP, &virtual_table_id, maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, - region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt); - region_hit_cnt += region_ret; - /*enum component_table_type childs[3] = {COMPONENT_TABLE_TYPE_SIP, COMPONENT_TABLE_TYPE_DIP, COMPONENT_TABLE_TYPE_SESSION}; + item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt); + item_hit_cnt += item_ret; */ + enum component_table_type childs[3] = {COMPONENT_TABLE_TYPE_SIP, COMPONENT_TABLE_TYPE_DIP, COMPONENT_TABLE_TYPE_SESSION}; for (int i = 0; i < 3; i++) { - region_ret = ip_composition_scan(thread_id, addr, table_id, childs[i], &virtual_table_id, + item_ret = ip_composition_scan(thread_id, addr, table_id, childs[i], &virtual_table_id, maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, - region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt); - if (region_ret < 0) { + item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt); + if (item_ret < 0) { maat_instance->scan_err_cnt++; } else { - for (int j = 0; j < region_ret; j++) { - region_result_virtual_table_ids[region_hit_cnt++] = virtual_table_id; + for (int j = 0; j < item_ret; j++) { + item_result_virtual_table_ids[item_hit_cnt++] = virtual_table_id; } } - }*/ + } } else { - region_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_NONE, &virtual_table_id, + item_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_NONE, &virtual_table_id, maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, - region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt); - if (region_ret < 0) { + item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt); + if (item_ret < 0) { maat_instance->scan_err_cnt++; } else { - region_hit_cnt += region_ret; + item_hit_cnt += item_ret; } } - *n_result = region_ret; - for (int i = 0; i < region_ret; i++) { - results[i] = region_result[i].rule_id; - } + int compile_ret = 0; + struct scan_item_hit_wrapper item_hit_wrapper; + if (item_hit_cnt > 0 || scan_state_should_compile_NOT(mid)) { + mid = grab_mid(state, maat_instance, thread_id, 1); + scan_item_hit_wrapper_build(&item_hit_wrapper, item_result, item_hit_cnt, -1, + mid->is_last_scan, virtual_table_id, mid->scan_cnt); + if (table_type == TABLE_TYPE_COMPOSITION) { + item_hit_wrapper.virtual_table_ids = item_result_virtual_table_ids; + } - return 0; + compile_ret = item_compile(maat_instance, mid->compile_mid, + &item_hit_wrapper, + results, item_hit_cnt, + thread_id); + assert(mid->is_last_scan < 2); + if (mid->is_last_scan == 1) { + mid->is_last_scan = 2; + } + } + maat_runtime_ref_dec(maat_rt, thread_id); + + if (compile_ret == 0 && item_hit_cnt > 0) { + return -2; + } + *n_result = compile_ret; + return compile_ret; } int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, @@ -705,7 +844,12 @@ int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_sca } -void maat_state_reset(struct maat_state **state) +void maat_state_free(struct maat_state **state) +{ + +} + +int maat_matched_compile_id(struct maat *instance, struct maat_matched *matched) { } \ No newline at end of file diff --git a/src/maat_group.cpp b/src/maat_group.cpp new file mode 100644 index 0000000..1f33255 --- /dev/null +++ b/src/maat_group.cpp @@ -0,0 +1,313 @@ +/* +********************************************************************************************** +* File: maat_group.cpp +* Description: +* Authors: Liu wentan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include +#include + +#include "maat_group.h" +#include "utils.h" +#include "maat_utils.h" +#include "uthash/uthash.h" +#include "igraph/igraph.h" +#include "log/log.h" + +#define MODULE_GROUP module_name_str("maat.group") + +struct maat_group_topology { + struct maat_group *hash_group_by_id; //key: group_id, value: struct maat_group *. + struct maat_group *hash_group_by_vertex; //key: vetex_id, value: struct maat_group *. Multimap (Items with multiple keys). + igraph_t group_graph; + igraph_integer_t group_graph_vcount; + igraph_vector_t dfs_vids; + igraph_integer_t grp_vertex_id_generator; + + pthread_rwlock_t rwlock; + struct log_handle *logger; +}; + +void group_vertex_free(struct maat_group *group) +{ + free(group->top_group_ids); + free(group); +} + +struct maat_group_topology *maat_group_topology_new(struct log_handle *logger) +{ + struct maat_group_topology *group_topo = ALLOC(struct maat_group_topology, 1); + UNUSED int ret = 0; + + group_topo->hash_group_by_id = NULL; + group_topo->hash_group_by_vertex = NULL; + + ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED); + assert(ret == IGRAPH_SUCCESS); + + ret = pthread_rwlock_init(&group_topo->rwlock, NULL); + assert(ret == 0); + + group_topo->logger = logger; + + return group_topo; +} + +void maat_group_topology_free(struct maat_group_topology *group_topo) +{ + struct maat_group *group = NULL, *tmp_group = NULL; + + HASH_CLEAR(hh_vertex_id, group_topo->hash_group_by_vertex);//No need group memory clean up. + HASH_ITER(hh_group_id, group_topo->hash_group_by_id, group, tmp_group) { + HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group); + group_vertex_free(group); + } + assert(group_topo->hash_group_by_id == NULL); + + igraph_destroy(&group_topo->group_graph); + + pthread_rwlock_unlock(&group_topo->rwlock); + pthread_rwlock_destroy(&group_topo->rwlock); +} + +size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) { + long int i; + int printed = 0; + + for (i = 0; i < igraph_vector_size(v); i++) { + printed += snprintf(buff + printed, sz - printed, " %li", (long int) VECTOR(*v)[i]); + } + + return printed; +} + +struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id) +{ + struct maat_group *group = ALLOC(struct maat_group, 1); + + group->group_id = group_id; + group->vertex_id = group_topo->grp_vertex_id_generator++; + + assert(igraph_vcount(&group_topo->group_graph)==group->vertex_id); + igraph_add_vertices(&group_topo->group_graph, 1, NULL); //Add 1 vertice. + + HASH_ADD(hh_group_id, group_topo->hash_group_by_id, group_id, sizeof(group->group_id), group); + HASH_ADD(hh_vertex_id, group_topo->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group); + + return group; +} + +void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group) +{ + igraph_vector_t v; + char buff[4096] = {0}; + + assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0); + igraph_vector_init(&v, 8); + igraph_neighbors(&group_topo->group_graph, &v, group->vertex_id, IGRAPH_ALL); + if (igraph_vector_size(&v) > 0) { + print_igraph_vector(&v, buff, sizeof(buff)); + log_error(group_topo->logger, MODULE_GROUP, "Del group %d exception, still reached by %s.", + group->vertex_id, buff); + assert(0); + } + igraph_vector_destroy(&v); + assert(group->top_group_ids==NULL); + //We should not call igraph_delete_vertices, because this is function changes the ids of the vertices. + //igraph_delete_vertices(&hier->group_graph, igraph_vss_1(group->vertex_id)); + + + HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group); + HASH_DELETE(hh_vertex_id, group_topo->hash_group_by_vertex, group); + + group_vertex_free(group); +} + +struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id) +{ + struct maat_group *group = NULL; + + HASH_FIND(hh_group_id, group_topo->hash_group_by_id, &group_id, sizeof(group_id), group); + + return group; +} + +int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id) +{ + int ret = 0; + igraph_integer_t edge_id; + struct maat_group *group = NULL, *superior_group = NULL; + + group = maat_group_topology_find_group(group_topo, group_id); + if (NULL == group) { + group = maat_group_topology_add_group(group_topo, group_id); + } + + superior_group = maat_group_topology_find_group(group_topo, superior_group_id); + if (NULL == superior_group) { + superior_group = maat_group_topology_add_group(group_topo, superior_group_id); + } + + ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); + + //No duplicated edges between two groups. + if (edge_id > 0) { + log_error(group_topo->logger, MODULE_GROUP, + "Add group %d to group %d failed, relation already exisited.", + group->group_id, superior_group->group_id); + ret = -1; + } else { + igraph_add_edge(&group_topo->group_graph, group->vertex_id, superior_group->vertex_id); + group->ref_by_superior_group_cnt++; + superior_group->ref_by_subordinate_group_cnt++; + ret = 0; + } + + return ret; +} + +int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id) +{ + int ret = 0; + struct maat_group *group = NULL, *superior_group = NULL; + + //No hash write operation, LOCK protection is unnecessary. + group = maat_group_topology_find_group(group_topo, group_id); + if (NULL == group) { + log_error(group_topo->logger, MODULE_GROUP, + "Del group %d from group %d failed, group %d not exisited.", + group_id, superior_group_id, group_id); + return -1; + } + + superior_group = maat_group_topology_find_group(group_topo, superior_group_id); + if (NULL == superior_group) { + log_error(group_topo->logger, MODULE_GROUP, + "Del group %d from group %d failed, superior group %d not exisited.", + group_id, superior_group_id, superior_group_id); + return -1; + } + + igraph_es_t es; + igraph_integer_t edge_num_before = 0, edge_num_after = 0; + + edge_num_before = igraph_ecount(&group_topo->group_graph); + // The edges between the given pairs of vertices will be included in the edge selection. + //The vertex pairs must be given as the arguments of the function call, the third argument + //is the first vertex of the first edge, the fourth argument is the second vertex of the + //first edge, the fifth is the first vertex of the second edge and so on. The last element + //of the argument list must be -1 to denote the end of the argument list. + //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small + ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); + assert(ret==IGRAPH_SUCCESS); + // ignore no such edge to abort(). + igraph_set_error_handler(igraph_error_handler_ignore); + ret = igraph_delete_edges(&group_topo->group_graph, es); + edge_num_after = igraph_ecount(&group_topo->group_graph); + igraph_es_destroy(&es); + + if (ret != IGRAPH_SUCCESS || edge_num_before - edge_num_after != 1) { + assert(0); + return -1; + } + + group->ref_by_superior_group_cnt--; + superior_group->ref_by_subordinate_group_cnt--; + + return 0; +} + +static size_t effective_vertices_count(igraph_vector_t *vids) +{ + size_t i = 0; + int tmp_vid = 0; + + for (i = 0; i < (size_t)igraph_vector_size(vids); i++) { + tmp_vid = (int) VECTOR(*vids)[i]; + if (tmp_vid < 0) { + break; + } + } + + return i; +} + +int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo) +{ + struct maat_group *group = NULL, *tmp = NULL; + struct maat_group *superior_group = NULL; + int tmp_vid=0; + size_t i=0, top_group_cnt=0; + int* temp_group_ids=NULL; + + igraph_bool_t is_dag; + igraph_is_dag(&(group_topo->group_graph), &is_dag); + if (!is_dag) { + log_error(group_topo->logger, MODULE_GROUP, "Sub group cycle detected!"); + return -1; + } + + group_topo->group_graph_vcount = igraph_vcount(&group_topo->group_graph); + igraph_vector_init(&(group_topo->dfs_vids), group_topo->group_graph_vcount); + + HASH_ITER (hh_group_id, group_topo->hash_group_by_id, group, tmp) { + top_group_cnt = 0; + temp_group_ids = NULL; + //Orphan, Not reference by any one, free it. + if (0 == group->ref_by_compile_cnt + && 0 == group->ref_by_superior_group_cnt + && 0 == group->ref_by_subordinate_group_cnt + && 0 == group->ref_by_item_cnt) { + + FREE(group->top_group_ids); + maat_group_topology_remove_group(group_topo, group); + continue; + } + + //A group is need to build top groups when it has items and referenced by superior groups or compiles. + if (group->ref_by_item_cnt > 0 && + (group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0)) { + if (0 == group->ref_by_superior_group_cnt) { + //fast path, group is only referenced by compile rules. + top_group_cnt = 1; + temp_group_ids = ALLOC(int, top_group_cnt); + temp_group_ids[0] = group->group_id; + } else { + igraph_vector_t *vids = &(group_topo->dfs_vids); + igraph_dfs(&group_topo->group_graph, group->vertex_id, IGRAPH_OUT, + 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); + + temp_group_ids = ALLOC(int, effective_vertices_count(vids)); + + for (size_t i = 0; i < (size_t)igraph_vector_size(vids); i++) { + tmp_vid = (int) VECTOR(*vids)[i]; + if (tmp_vid < 0) { + break; + } + + HASH_FIND(hh_vertex_id, group_topo->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group); + + //including itself + if (superior_group->ref_by_compile_cnt > 0) { + temp_group_ids[top_group_cnt] = superior_group->group_id; + top_group_cnt++; + } + } + } + } + + free(group->top_group_ids); + group->top_group_cnt = top_group_cnt; + group->top_group_ids = ALLOC(int, group->top_group_cnt); + memcpy(group->top_group_ids, temp_group_ids, sizeof(int)*group->top_group_cnt); + + FREE(temp_group_ids); + } + igraph_vector_destroy(&group_topo->dfs_vids); + + return 0; +} \ No newline at end of file diff --git a/src/maat_hierarchy.cpp b/src/maat_hierarchy.cpp index e547624..e43ddfb 100644 --- a/src/maat_hierarchy.cpp +++ b/src/maat_hierarchy.cpp @@ -20,6 +20,8 @@ #include "igraph/igraph.h" #include "maat_hierarchy.h" #include "maat_garbage_collection.h" +#include "maat_group.h" +#include "maat/maat.h" #define MODULE_HIERARCHY module_name_str("maat.hierarchy") @@ -41,31 +43,6 @@ struct maat_hierarchy_compile { struct maat_hierarchy_clause_state clause_states[MAX_ITEMS_PER_BOOL_EXPR]; }; -struct maat_hierarchy_group { - igraph_integer_t vertex_id; - int group_id; - int ref_by_compile_cnt; - int ref_by_superior_group_cnt; - int ref_by_subordinate_group_cnt; - int ref_by_region_cnt; - - size_t top_group_cnt; - int *top_group_ids; - UT_hash_handle hh_group_id; - UT_hash_handle hh_vertex_id; -}; - -struct maat_hierarchy_region { - int region_id; - int group_id; - int table_id; - - struct maat_hierarchy_group *ref_parent_group; - UT_hash_handle hh; - - void *user_data; -}; - struct maat_hierarchy_literal_id { int group_id; int vt_id; @@ -78,19 +55,19 @@ struct maat_hierarchy_clause { UT_hash_handle hh; }; -struct region2clause_key { - int region_id; +struct item2clause_key { + int item_id; int vt_id; }; -struct group2region { +struct group2item { int group_id; - UT_array *region_ids; + UT_array *item_ids; UT_hash_handle hh; //index to }; -struct region2clause_value { - struct region2clause_key key; +struct item2clause_value { + struct item2clause_key key; UT_array *clause_ids; int group_id; UT_hash_handle hh; //index to @@ -98,55 +75,42 @@ struct region2clause_value { struct maat_hierarchy_internal_hit_path { int Nth_scan; - int Nth_hit_region; - int region_id; + int Nth_hit_item; + int item_id; int virtual_table_id; }; -struct maat_hierarchy -{ +struct maat_hierarchy { pthread_rwlock_t rwlock; - pthread_mutex_t mutex; time_t version; //After full update, clause id may indicate a different clause. Comparing hier->version and mid->hier_ver can prevent false positive match. int changed_flag; - struct maat_hierarchy_compile *hash_compile_by_id; //key: compile_id, value: struct Maat_hierarchy_compile*. + struct maat_hierarchy_compile *hash_compile_by_id; //key: compile_id, value: struct maat_hierarchy_compile*. void (* compile_user_data_free)(void *compile_ud); - struct maat_hierarchy_group *hash_group_by_id; //key: group_id, value: struct Maat_hierarchy_group*. - struct maat_hierarchy_group *hash_group_by_vertex; //key:vetex_id, value: struct Maat_hierarchy_group*. Multimap (Items with multiple keys). + struct maat_hierarchy_item *hash_item_by_id; //key: item_id, value: struct maat_hierarchy_item*. - struct maat_hierarchy_region *hash_region_by_id; //key: region_id, value: struct Maat_hierarchy_region*. - - struct maat_hierarchy_clause *hash_dedup_clause_by_literals; //key: literal combination, value: struct Maat_hierarchy_clause*. For generating unique clause_id. - unsigned long long clause_id_generator; //Increasing number. + struct maat_hierarchy_clause *hash_dedup_clause_by_literals; //key: literal combination, value: struct maat_hierarchy_clause*. For generating unique clause_id. + unsigned long long clause_id_generator; //Increasing number. - void (* region_user_data_free)(void *region_ud); - - - igraph_t group_graph; - igraph_integer_t group_graph_vcount; - igraph_vector_t dfs_vids; - - igraph_integer_t grp_vertex_id_generator; + void (* item_user_data_free)(void *item_ud); /*Following members are accessed from scan threads.*/ - struct region2clause_value *hash_region2clause; //key: region_id+virtual_table_id, value: struct region2clause_value. + struct item2clause_value *hash_item2clause; //key: item_id+virtual_table_id, value: struct item2clause_value. struct bool_matcher *bm; - + struct maat_group_topology *ref_group_topo; int thread_num; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; struct bool_expr_match *expr_match_buff; }; -struct maat_hierarchy_compile_mid -{ +struct maat_hierarchy_compile_mid { int thread_id; int Nth_scan; time_t hier_ver; - size_t this_scan_region_hit_cnt; + size_t this_scan_item_hit_cnt; int not_clause_hitted_flag; int is_no_count_scan; size_t hit_path_cnt; @@ -164,28 +128,23 @@ struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bi hier->thread_num = thread_num; hier->version = time(NULL); - hier->hash_group_by_id = NULL; - hier->hash_group_by_vertex = NULL; hier->hash_compile_by_id = NULL; - hier->hash_region2clause = NULL; - hier->hash_region_by_id = NULL; + hier->hash_item2clause = NULL; + hier->hash_item_by_id = NULL; hier->hash_dedup_clause_by_literals = NULL; hier->clause_id_generator = 0; hier->ref_garbage_bin = bin; hier->expr_match_buff = ALLOC(struct bool_expr_match, thread_num * MAX_SCANNER_HIT_NUM); - pthread_mutex_init(&hier->mutex, NULL); ret = pthread_rwlock_init(&hier->rwlock, NULL); assert(ret == 0); - ret = igraph_empty(&hier->group_graph, 0, IGRAPH_DIRECTED); - assert(ret == IGRAPH_SUCCESS); return hier; } UT_icd ut_literal_id_icd = {sizeof(struct maat_hierarchy_literal_id), NULL, NULL, NULL}; UT_icd ut_clause_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; -UT_icd ut_region_id_icd = {sizeof(int), NULL, NULL, NULL}; +UT_icd ut_item_id_icd = {sizeof(int), NULL, NULL, NULL}; UT_icd ut_hit_path_icd = {sizeof(struct maat_hierarchy_internal_hit_path), NULL, NULL, NULL}; #define MAAT_HIER_COMPILE_MAGIC 0x4a5b6c7d @@ -220,44 +179,43 @@ static void maat_hierarchy_compile_free(struct maat_hierarchy_compile *compile) free(compile); } -void maat_hierarchy_free_region2clause_hash(struct region2clause_value* hash) +void maat_hierarchy_free_item2clause_hash(struct item2clause_value* hash) { - struct region2clause_value *r2c_val = NULL, *tmp_r2c_val = NULL; - HASH_ITER(hh, hash, r2c_val, tmp_r2c_val) { - HASH_DEL(hash, r2c_val); - utarray_free(r2c_val->clause_ids); - free(r2c_val); + struct item2clause_value *i2c_val = NULL, *tmp_i2c_val = NULL; + HASH_ITER(hh, hash, i2c_val, tmp_i2c_val) { + HASH_DEL(hash, i2c_val); + utarray_free(i2c_val->clause_ids); + free(i2c_val); } assert(hash == NULL); } -static struct maat_hierarchy_region * -maat_hierarchy_region_new(struct maat_hierarchy *hier, int region_id, int group_id, int table_id, - struct maat_hierarchy_group *parent_group, void *user_data) +static struct maat_hierarchy_item * +maat_hierarchy_item_new(struct maat_hierarchy *hier, int item_id, int group_id, + struct maat_group *parent_group, void *user_data) { - struct maat_hierarchy_region *region = NULL; - region = ALLOC(struct maat_hierarchy_region, 1); - region->group_id = group_id; - region->region_id = region_id; - region->table_id = table_id; - region->ref_parent_group = parent_group; - region->user_data = user_data; - HASH_ADD_INT(hier->hash_region_by_id, region_id, region); - parent_group->ref_by_region_cnt++; + struct maat_hierarchy_item *item = NULL; + item = ALLOC(struct maat_hierarchy_item, 1); + item->group_id = group_id; + item->item_id = item_id; + item->ref_parent_group = parent_group; + item->user_data = user_data; + HASH_ADD_INT(hier->hash_item_by_id, item_id, item); + parent_group->ref_by_item_cnt++; - return region; + return item; } -static void maat_hierarchy_region_free(struct maat_hierarchy *hier, struct maat_hierarchy_region *region) +static void maat_hierarchy_item_free(struct maat_hierarchy *hier, struct maat_hierarchy_item *item) { - HASH_DELETE(hh, hier->hash_region_by_id, region); - region->ref_parent_group->ref_by_region_cnt--; + HASH_DELETE(hh, hier->hash_item_by_id, item); + item->ref_parent_group->ref_by_item_cnt--; - if (hier->region_user_data_free && region->user_data) { - hier->region_user_data_free(region->user_data); - region->user_data = NULL; + if (hier->item_user_data_free && item->user_data) { + hier->item_user_data_free(item->user_data); + item->user_data = NULL; } - free(region); + free(item); } static const struct maat_hierarchy_clause * @@ -291,18 +249,11 @@ static void maat_hierarchy_clause_free(struct maat_hierarchy *hier, struct maat_ free(clause); } -static void group_vertex_free(struct maat_hierarchy_group *group) -{ - free(group->top_group_ids); - free(group); -} - void maat_hierarchy_free(struct maat_hierarchy *hier) { struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; - struct maat_hierarchy_group *group = NULL, *tmp_group = NULL; - struct region2clause_value *r2c_val = NULL, *tmp_r2c_val = NULL; - struct maat_hierarchy_region *region = NULL, *tmp_region = NULL; + struct item2clause_value *i2c_val = NULL, *tmp_i2c_val = NULL; + struct maat_hierarchy_item *item = NULL, *tmp_item = NULL; struct maat_hierarchy_clause *clause = NULL, *tmp_clause = NULL; pthread_rwlock_wrlock(&hier->rwlock); @@ -321,31 +272,22 @@ void maat_hierarchy_free(struct maat_hierarchy *hier) } assert(hier->hash_compile_by_id == NULL); - HASH_ITER(hh, hier->hash_region2clause, r2c_val, tmp_r2c_val) { - HASH_DEL(hier->hash_region2clause, r2c_val); - utarray_free(r2c_val->clause_ids); - free(r2c_val); + HASH_ITER(hh, hier->hash_item2clause, i2c_val, tmp_i2c_val) { + HASH_DEL(hier->hash_item2clause, i2c_val); + utarray_free(i2c_val->clause_ids); + free(i2c_val); } - maat_hierarchy_free_region2clause_hash(hier->hash_region2clause); + maat_hierarchy_free_item2clause_hash(hier->hash_item2clause); - HASH_ITER(hh, hier->hash_region_by_id, region, tmp_region) { - maat_hierarchy_region_free(hier, region); + HASH_ITER(hh, hier->hash_item_by_id, item, tmp_item) { + maat_hierarchy_item_free(hier, item); } HASH_ITER(hh, hier->hash_dedup_clause_by_literals, clause, tmp_clause) { maat_hierarchy_clause_free(hier, clause); } - //Free group as the last. - HASH_CLEAR(hh_vertex_id, hier->hash_group_by_vertex);//No need group memory clean up. - HASH_ITER(hh_group_id, hier->hash_group_by_id, group, tmp_group) { - HASH_DELETE(hh_group_id, hier->hash_group_by_id, group); - group_vertex_free(group); - } - assert(hier->hash_group_by_id == NULL); - - igraph_destroy(&hier->group_graph); bool_matcher_free(hier->bm); hier->bm = NULL; @@ -367,138 +309,6 @@ size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) return printed; } -struct maat_hierarchy_group *maat_hierarchy_group_new(struct maat_hierarchy *hier, int group_id) -{ - struct maat_hierarchy_group *group = NULL; - - group = ALLOC(struct maat_hierarchy_group, 1); - group->group_id = group_id; - group->vertex_id = hier->grp_vertex_id_generator++; - assert(igraph_vcount(&hier->group_graph) == group->vertex_id); - igraph_add_vertices(&hier->group_graph, 1, NULL); //Add 1 vertice. - - HASH_ADD(hh_group_id, hier->hash_group_by_id, group_id, sizeof(group->group_id), group); - HASH_ADD(hh_vertex_id, hier->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group); - - return group; -} - -static void maat_hierarchy_group_free(struct maat_hierarchy *hier, struct maat_hierarchy_group *group) -{ - igraph_vector_t v; - char buff[4096] = {0}; - - assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0); - igraph_vector_init(&v, 8); - igraph_neighbors(&hier->group_graph, &v, group->vertex_id, IGRAPH_ALL); - - if (igraph_vector_size(&v) > 0) { - print_igraph_vector(&v, buff, sizeof(buff)); - log_error(hier->logger, MODULE_HIERARCHY, "Del group %d exception, still reached by %s.", - group->vertex_id, buff); - assert(0); - } - igraph_vector_destroy(&v); - assert(group->top_group_ids==NULL); - //We should not call igraph_delete_vertices, because this is function changes the ids of the vertices. - //igraph_delete_vertices(&hier->group_graph, igraph_vss_1(group->vertex_id)); - - HASH_DELETE(hh_group_id, hier->hash_group_by_id, group); - HASH_DELETE(hh_vertex_id, hier->hash_group_by_vertex, group); - - group_vertex_free(group); -} - -static size_t effective_vertices_count(igraph_vector_t *vids) -{ - size_t i = 0; - int tmp_vid = 0; - for (i = 0; i < (size_t)igraph_vector_size(vids); i++) { - tmp_vid = (int) VECTOR(*vids)[i]; - if (tmp_vid < 0) { - break; - } - } - - return i; -} - -static int maat_hierarchy_build_top_groups(struct maat_hierarchy *hier) -{ - struct maat_hierarchy_group *group = NULL, *tmp = NULL; - struct maat_hierarchy_group *superior_group = NULL; - int tmp_vid=0; - size_t i=0, top_group_cnt=0; - int* temp_group_ids=NULL; - - igraph_bool_t is_dag; - igraph_is_dag(&(hier->group_graph), &is_dag); - if (!is_dag) { - log_error(hier->logger, MODULE_HIERARCHY, "Sub group cycle detected!"); - return -1; - } - - hier->group_graph_vcount = igraph_vcount(&hier->group_graph); - igraph_vector_init(&(hier->dfs_vids), hier->group_graph_vcount); - - HASH_ITER (hh_group_id, hier->hash_group_by_id, group, tmp) { - top_group_cnt = 0; - temp_group_ids = NULL; - //Orphan, Not reference by any one, free it. - if (0 == group->ref_by_compile_cnt - && 0 == group->ref_by_superior_group_cnt - && 0 == group->ref_by_subordinate_group_cnt - && 0 == group->ref_by_region_cnt) { - - FREE(group->top_group_ids); - maat_hierarchy_group_free(hier, group); - continue; - } - - //A group is need to build top groups when it has regions and referenced by superior groups or compiles. - if (group->ref_by_region_cnt > 0 && - (group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0)) { - if (0 == group->ref_by_superior_group_cnt) { - //fast path, group is only referenced by compile rules. - top_group_cnt = 1; - temp_group_ids = ALLOC(int, top_group_cnt); - temp_group_ids[0] = group->group_id; - } else { - igraph_vector_t *vids = &(hier->dfs_vids); - igraph_dfs(&hier->group_graph, group->vertex_id, IGRAPH_OUT, - 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); - - temp_group_ids = ALLOC(int, effective_vertices_count(vids)); - - for (size_t i = 0; i < (size_t)igraph_vector_size(vids); i++) { - tmp_vid = (int) VECTOR(*vids)[i]; - if (tmp_vid < 0) { - break; - } - - HASH_FIND(hh_vertex_id, hier->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group); - - //including itself - if (superior_group->ref_by_compile_cnt > 0) { - temp_group_ids[top_group_cnt] = superior_group->group_id; - top_group_cnt++; - } - } - } - } - - free(group->top_group_ids); - group->top_group_cnt = top_group_cnt; - group->top_group_ids = ALLOC(int, group->top_group_cnt); - memcpy(group->top_group_ids, temp_group_ids, sizeof(int)*group->top_group_cnt); - - FREE(temp_group_ids); - } - igraph_vector_destroy(&hier->dfs_vids); - - return 0; -} - static struct bool_matcher *maat_hierarchy_build_bool_matcher(struct maat_hierarchy *hier) { struct bool_matcher *bm = NULL; @@ -595,46 +405,46 @@ static inline int compare_clause_id(const void *a, const void *b) } } -static inline int compare_region_id(const void *a, const void *b) +static inline int compare_item_id(const void *a, const void *b) { return (*(int*)a - *(int*)b); } -struct region2clause_value *maat_hierarchy_build_region2clause_hash(struct maat_hierarchy *hier) +struct item2clause_value *maat_hierarchy_build_item2clause_hash(struct maat_hierarchy *hier) { size_t i = 0, j = 0, k = 0; struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; struct maat_hierarchy_literal_id *literal_id = NULL; struct maat_hierarchy_clause_state *clause_state = NULL; - struct maat_hierarchy_region *region = NULL, *tmp_region = NULL; - struct maat_hierarchy_group *group = NULL; - struct group2region *g2r_hash = NULL, *g2r = NULL, *g2r_tmp = NULL; - struct region2clause_value *region2clause_hash = NULL, *r2c_val = NULL; - struct region2clause_key r2c_key; + struct maat_hierarchy_item *item = NULL, *tmp_item = NULL; + struct maat_group *group = NULL; + struct group2item *g2i_hash = NULL, *g2i = NULL, *g2i_tmp = NULL; + struct item2clause_value *item2clause_hash = NULL, *i2c_val = NULL; + struct item2clause_key i2c_key; - //Build a temporary hash that maps group to its regions. - HASH_ITER(hh, hier->hash_region_by_id, region, tmp_region) { - group = region->ref_parent_group; + //Build a temporary hash that maps group to its items. + HASH_ITER(hh, hier->hash_item_by_id, item, tmp_item) { + group = item->ref_parent_group; for (i = 0; i < group->top_group_cnt; i++) { - HASH_FIND_INT(g2r_hash, group->top_group_ids+i, g2r); - if (!g2r) { - g2r = ALLOC(struct group2region, 1); - utarray_new(g2r->region_ids, &ut_region_id_icd); - utarray_reserve(g2r->region_ids, group->ref_by_region_cnt); - g2r->group_id = group->top_group_ids[i]; - HASH_ADD_INT(g2r_hash, group_id, g2r); + HASH_FIND_INT(g2i_hash, group->top_group_ids+i, g2i); + if (!g2i) { + g2i = ALLOC(struct group2item, 1); + utarray_new(g2i->item_ids, &ut_item_id_icd); + utarray_reserve(g2i->item_ids, group->ref_by_item_cnt); + g2i->group_id = group->top_group_ids[i]; + HASH_ADD_INT(g2i_hash, group_id, g2i); } - //One region belongs to one group, one group may have many regions. So duplicate region check is unnecessary. - //if(utarray_find(g2r->region_ids, &(region->region_id), compare_region_id)) assert(0); + //One item belongs to one group, one group may have many items. So duplicate item check is unnecessary. + //if(utarray_find(g2i->item_ids, &(item->item_id), compare_item_id)) assert(0); - utarray_push_back(g2r->region_ids, &(region->region_id)); - //utarray_sort(g2r->region_ids, compare_region_id); + utarray_push_back(g2i->item_ids, &(item->item_id)); + //utarray_sort(g2i->item_ids, compare_item_id); } } - //Build short cut hash that maps region_id+vt_id to clause_ids. + //Build short cut hash that maps item_id+vt_id to clause_ids. HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { clause_state = compile->clause_states + i; @@ -648,91 +458,206 @@ struct region2clause_value *maat_hierarchy_build_region2clause_hash(struct maat_ if (!group) { continue; } - HASH_FIND_INT(g2r_hash, &(group->group_id), g2r); + HASH_FIND_INT(g2i_hash, &(group->group_id), g2i); - //group declared by compile, but has no subordinate or region. - if (!g2r) { + //group declared by compile, but has no subordinate or item. + if (!g2i) { continue; } - for (k = 0; k < utarray_len(g2r->region_ids); k++) { - r2c_key.region_id = *((int*)utarray_eltptr(g2r->region_ids, k)); - r2c_key.vt_id = literal_id->vt_id; - HASH_FIND(hh, region2clause_hash, &r2c_key, sizeof(r2c_key), r2c_val); - if (!r2c_val) { - r2c_val = ALLOC(struct region2clause_value, 1); - r2c_val->key = r2c_key; - r2c_val->group_id = g2r->group_id; - utarray_new(r2c_val->clause_ids, &ut_clause_id_icd); - HASH_ADD(hh, region2clause_hash, key, sizeof(r2c_val->key), r2c_val); + for (k = 0; k < utarray_len(g2i->item_ids); k++) { + i2c_key.item_id = *((int*)utarray_eltptr(g2i->item_ids, k)); + i2c_key.vt_id = literal_id->vt_id; + HASH_FIND(hh, item2clause_hash, &i2c_key, sizeof(i2c_key), i2c_val); + if (!i2c_val) { + i2c_val = ALLOC(struct item2clause_value, 1); + i2c_val->key = i2c_key; + i2c_val->group_id = g2i->group_id; + utarray_new(i2c_val->clause_ids, &ut_clause_id_icd); + HASH_ADD(hh, item2clause_hash, key, sizeof(i2c_val->key), i2c_val); } - if (utarray_find(r2c_val->clause_ids, &(clause_state->clause_id), compare_clause_id)) { + if (utarray_find(i2c_val->clause_ids, &(clause_state->clause_id), compare_clause_id)) { continue; } - utarray_push_back(r2c_val->clause_ids, &(clause_state->clause_id)); - utarray_sort(r2c_val->clause_ids, compare_clause_id); + utarray_push_back(i2c_val->clause_ids, &(clause_state->clause_id)); + utarray_sort(i2c_val->clause_ids, compare_clause_id); } } } } int tmp1 = 0, tmp2 = 0; - HASH_ITER(hh, g2r_hash, g2r, g2r_tmp) { - HASH_DEL(g2r_hash, g2r); + HASH_ITER(hh, g2i_hash, g2i, g2i_tmp) { + HASH_DEL(g2i_hash, g2i); //Sanity Check - utarray_sort(g2r->region_ids, compare_region_id); - for (i = 1; i < utarray_len(g2r->region_ids); i++) { - tmp1 = *((int*)utarray_eltptr(g2r->region_ids, i-1)); - tmp2 = *((int*)utarray_eltptr(g2r->region_ids, i)); + utarray_sort(g2i->item_ids, compare_item_id); + for (i = 1; i < utarray_len(g2i->item_ids); i++) { + tmp1 = *((int*)utarray_eltptr(g2i->item_ids, i-1)); + tmp2 = *((int*)utarray_eltptr(g2i->item_ids, i)); assert(tmp1!=tmp2); } - utarray_free(g2r->region_ids); - g2r->region_ids = NULL; - free(g2r); + utarray_free(g2i->item_ids); + g2i->item_ids = NULL; + free(g2i); } - log_info(hier->logger, MODULE_HIERARCHY, "Build region2clause hash with %llu element.", - HASH_COUNT(region2clause_hash)); - return region2clause_hash; + log_info(hier->logger, MODULE_HIERARCHY, "Build item2clause hash with %llu element.", + HASH_COUNT(item2clause_hash)); + return item2clause_hash; } int maat_hierarchy_rebuild(struct maat_hierarchy *hier) { int ret=0; struct bool_matcher *new_bm = NULL, *old_bm = NULL; - struct region2clause_value *new_region2clause_hash = NULL, *old_region2clause_hash = NULL; + struct item2clause_value *new_item2clause_hash = NULL, *old_item2clause_hash = NULL; //Read hier from update thread is OK. if (!hier->changed_flag) { return ret; } - ret = maat_hierarchy_build_top_groups(hier); + ret = maat_group_topology_build_top_groups(hier->ref_group_topo); new_bm = maat_hierarchy_build_bool_matcher(hier); - new_region2clause_hash = maat_hierarchy_build_region2clause_hash(hier); + new_item2clause_hash = maat_hierarchy_build_item2clause_hash(hier); pthread_rwlock_wrlock(&hier->rwlock); old_bm = hier->bm; - old_region2clause_hash = hier->hash_region2clause; + old_item2clause_hash = hier->hash_item2clause; hier->bm = new_bm; - hier->hash_region2clause = new_region2clause_hash; + hier->hash_item2clause = new_item2clause_hash; hier->changed_flag = 0; pthread_rwlock_unlock(&hier->rwlock); maat_garbage_bagging(hier->ref_garbage_bin, old_bm, (void (*)(void*))bool_matcher_free); - maat_garbage_bagging(hier->ref_garbage_bin, old_region2clause_hash, - (void (*)(void*))maat_hierarchy_free_region2clause_hash); + maat_garbage_bagging(hier->ref_garbage_bin, old_item2clause_hash, + (void (*)(void*))maat_hierarchy_free_item2clause_hash); return ret; } -size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - struct maat_hit_path_t *hit_paths, size_t n_path) +static int maat_hierarchy_compile_has_literal(struct maat_hierarchy_compile *compile, struct maat_hierarchy_literal_id *literal_id) { + if (NULL == compile || NULL == literal_id) { + return 0; + } + for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + struct maat_hierarchy_clause_state *clause_state = compile->clause_states + i; + if (!clause_state->in_use) { + continue; + } + + struct maat_hierarchy_literal_id *tmp = (struct maat_hierarchy_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); + if (tmp) { + assert(tmp->group_id == literal_id->group_id && tmp->vt_id == literal_id->vt_id); + return 1; + } + } + + return 0; +} + +static int maat_hierarchy_is_hit_path_existed(const struct maat_hit_path *hit_paths, size_t n_path, const struct maat_hit_path *find) +{ + if (NULL == hit_paths || NULL == find) { + return 0; + } + + for (size_t i = 0; i < n_path; i++) { + if (0 == memcmp(hit_paths + i, find, sizeof(*find))) { + return 1; + } + } + + return 0; +} + +size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, + struct maat_hit_path *hit_paths, size_t n_path) +{ + struct maat_hierarchy_internal_hit_path *p = NULL; + struct maat_hierarchy_item *item = NULL; + struct maat_group *group = NULL; + struct maat_hierarchy_compile *compile = NULL; + struct maat_hierarchy_literal_id literal_id = {0, 0}; + size_t n_made_by_item = 0, n_made_by_compile = 0; + size_t i = 0, j = 0, bool_match_ret = 0; + struct bool_expr_match *expr_match = hier->expr_match_buff + mid->thread_id * MAX_SCANNER_HIT_NUM; + struct maat_hit_path tmp_path; + + if (hier->version != mid->hier_ver) { + return 0; + } + pthread_rwlock_rdlock(&hier->rwlock); + + for (i = 0; i < utarray_len(mid->internal_hit_paths); i++) { + p = (struct maat_hierarchy_internal_hit_path *)utarray_eltptr(mid->internal_hit_paths, i); + + HASH_FIND_INT(hier->hash_item_by_id, &(p->item_id), item); + if (!item) { + continue; + } + + group = item->ref_parent_group; + if (group->top_group_cnt == 0 && n_made_by_item < n_path) { + hit_paths[n_made_by_item].Nth_scan = p->Nth_scan; + hit_paths[n_made_by_item].item_id = p->item_id; + hit_paths[n_made_by_item].sub_group_id = group->group_id; + hit_paths[n_made_by_item].top_group_id = -1; + hit_paths[n_made_by_item].virtual_table_id = p->virtual_table_id; + hit_paths[n_made_by_item].compile_id = -1; + n_made_by_item++; + } else { + for (j = 0; j < group->top_group_cnt && n_made_by_item < n_path; j++, n_made_by_item++) { + hit_paths[n_made_by_item].Nth_scan = p->Nth_scan; + hit_paths[n_made_by_item].item_id = p->item_id; + hit_paths[n_made_by_item].sub_group_id = group->group_id; + hit_paths[n_made_by_item].top_group_id = group->top_group_ids[j]; + hit_paths[n_made_by_item].virtual_table_id = p->virtual_table_id; + hit_paths[n_made_by_item].compile_id = -1; + } + } + } + + bool_match_ret = bool_matcher_match(hier->bm, (unsigned long long *)utarray_eltptr(mid->all_hit_clause_array, 0), + utarray_len(mid->all_hit_clause_array), + expr_match, MAX_SCANNER_HIT_NUM); + for (i = 0; i < bool_match_ret; i++) { + compile = (struct maat_hierarchy_compile *)expr_match[i].user_tag; + assert(compile->magic == MAAT_HIER_COMPILE_MAGIC); + assert((unsigned long long)compile->compile_id == expr_match[i].expr_id); + if (compile->actual_clause_num == 0 || !compile->user_data) { + continue; + } + + for (j = 0; j < n_made_by_item && n_made_by_item + n_made_by_compile < n_path; j++) { + if (hit_paths[j].top_group_id < 0) { + continue; + } + + literal_id.group_id = hit_paths[j].top_group_id; + literal_id.vt_id = hit_paths[j].virtual_table_id; + if (maat_hierarchy_compile_has_literal(compile, &literal_id)) { + if (hit_paths[j].compile_id < 0) { + hit_paths[j].compile_id = compile->compile_id; + } else { + tmp_path = hit_paths[j]; + tmp_path.compile_id=compile->compile_id; + if (maat_hierarchy_is_hit_path_existed(hit_paths, n_made_by_item + n_made_by_compile, &tmp_path)) { + hit_paths[n_made_by_item + n_made_by_compile] = tmp_path; + n_made_by_compile++; + } + } + } + } + } + + pthread_rwlock_unlock(&hier->rwlock); + return (n_made_by_item + n_made_by_compile); } void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)) @@ -740,9 +665,9 @@ void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, hier->compile_user_data_free = func; } -void maat_hierarchy_set_region_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)) +void maat_hierarchy_set_item_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)) { - hier->region_user_data_free = func; + hier->item_user_data_free = func; } struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id) @@ -766,13 +691,13 @@ void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid) free(mid); } -static int maat_hierarchy_hit_path_add(UT_array *hit_paths, int region_id, int virtual_table_id, - int Nth_scan, int Nth_region_result) +static int maat_hierarchy_hit_path_add(UT_array *hit_paths, int item_id, int virtual_table_id, + int Nth_scan, int Nth_item_result) { struct maat_hierarchy_internal_hit_path new_path; - new_path.region_id = region_id; - new_path.Nth_hit_region = Nth_region_result; + new_path.item_id = item_id; + new_path.Nth_hit_item = Nth_item_result; new_path.Nth_scan = Nth_scan; new_path.virtual_table_id = virtual_table_id; @@ -782,30 +707,30 @@ static int maat_hierarchy_hit_path_add(UT_array *hit_paths, int region_id, int v void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - int region_id, int virtual_table_id, - int Nth_scan, int Nth_region_result) + int item_id, int virtual_table_id, + int Nth_scan, int Nth_item_result) { if (mid->Nth_scan != Nth_scan) { - assert(mid->this_scan_region_hit_cnt == 0); + assert(mid->this_scan_item_hit_cnt == 0); mid->Nth_scan = Nth_scan; utarray_clear(mid->this_scan_hit_clause_ids); } - int ret = maat_hierarchy_hit_path_add(mid->internal_hit_paths, region_id, virtual_table_id, - Nth_scan, Nth_region_result); + int ret = maat_hierarchy_hit_path_add(mid->internal_hit_paths, item_id, virtual_table_id, + Nth_scan, Nth_item_result); if (!ret) { return; } mid->hit_path_cnt++; - mid->this_scan_region_hit_cnt++; + mid->this_scan_item_hit_cnt++; - struct region2clause_value* r2c_val = NULL; - struct region2clause_key r2c_key; - r2c_key.region_id = region_id; - r2c_key.vt_id = virtual_table_id; + struct item2clause_value* i2c_val = NULL; + struct item2clause_key i2c_key; + i2c_key.item_id = item_id; + i2c_key.vt_id = virtual_table_id; - HASH_FIND(hh, hier->hash_region2clause, &r2c_key, sizeof(r2c_key), r2c_val); - if (!r2c_val) { + HASH_FIND(hh, hier->hash_item2clause, &i2c_key, sizeof(i2c_key), i2c_val); + if (!i2c_val) { return; } @@ -968,7 +893,7 @@ static size_t maat_hierarchy_compile_mid_if_new_hit_compile(struct maat_hierarch return r_in_c_cnt; } -int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, +int maat_hierarchy_item_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, int is_last_compile, void **user_data_array, size_t ud_array_sz) { int bool_match_ret = 0, i = 0; @@ -977,10 +902,10 @@ int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hiera size_t r_in_c_cnt = 0; size_t ud_result_cnt = 0; - size_t this_scan_region_hits = mid->this_scan_region_hit_cnt; + size_t this_scan_item_hits = mid->this_scan_item_hit_cnt; if (!hier->bm || 0 == utarray_len(mid->all_hit_clause_array) || hier->version != mid->hier_ver) { - mid->this_scan_region_hit_cnt = 0; + mid->this_scan_item_hit_cnt = 0; return 0; } @@ -1000,7 +925,7 @@ int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hiera } else if(compile->user_data) { //For compile may be dettached by Maat_hierarchy_compile_dettach_user_data, only return non-NULL userdata. if (r_in_c_cnt > 0 || //compile hitted becasue of new reigon - this_scan_region_hits == 0) //or hit a compile that refer a NOT-logic group in previous scan. + this_scan_item_hits == 0) //or hit a compile that refer a NOT-logic group in previous scan. { user_data_array[ud_result_cnt]=compile->user_data; ud_result_cnt++; @@ -1008,22 +933,22 @@ int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hiera } } - mid->this_scan_region_hit_cnt = 0; + mid->this_scan_item_hit_cnt = 0; return ud_result_cnt; } -void *maat_hierarchy_region_dettach_user_data(struct maat_hierarchy *hier, int region_id) +void *maat_hierarchy_item_dettach_user_data(struct maat_hierarchy *hier, int item_id) { - struct maat_hierarchy_region *region = NULL; + struct maat_hierarchy_item *item = NULL; void *ret = NULL; pthread_rwlock_wrlock(&hier->rwlock); hier->changed_flag = 1; - HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); - if (region) { - ret = region->user_data; - region->user_data = NULL; + HASH_FIND_INT(hier->hash_item_by_id, &item_id, item); + if (item) { + ret = item->user_data; + item->user_data = NULL; } pthread_rwlock_unlock(&hier->rwlock); @@ -1096,16 +1021,12 @@ int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_i int not_flag, int clause_index, int compile_id) { int ret = 0; - struct maat_hierarchy_group *group = NULL; + struct maat_group *group = NULL; struct maat_hierarchy_literal_id literal_id = {group_id, vt_id}; struct maat_hierarchy_compile *compile = NULL; pthread_rwlock_wrlock(&hier->rwlock); hier->changed_flag = 1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if (!group) { - group = maat_hierarchy_group_new(hier, group_id); - } HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile); if (!compile) { @@ -1120,7 +1041,6 @@ int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_i ret = -1; } else { ret = 0; - group->ref_by_compile_cnt++; } pthread_rwlock_unlock(&hier->rwlock); @@ -1130,20 +1050,12 @@ int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_i int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag, int clause_index, int compile_id) { - struct maat_hierarchy_group *group = NULL; struct maat_hierarchy_literal_id literal_id = {group_id, vt_id}; struct maat_hierarchy_compile *compile = NULL; int ret = 0; pthread_rwlock_wrlock(&hier->rwlock); hier->changed_flag = 1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if (!group) { - log_error(hier->logger, MODULE_HIERARCHY, - "Remove group %d from compile %d failed, group is not exisited.", - group_id, compile_id); - goto error_out; - } HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile); if (!compile) { @@ -1173,115 +1085,29 @@ error_out: return -1; } -int maat_hierarchy_add_group_to_group(struct maat_hierarchy *hier, int group_id, int superior_group_id) +int maat_hierarchy_add_item_to_group(struct maat_hierarchy *hier, int group_id, int item_id, void *user_data) { - igraph_integer_t edge_id; - struct maat_hierarchy_group *group = NULL, *superior_group = NULL; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag = 1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if (!group) { - group = maat_hierarchy_group_new(hier, group_id); - } - - HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group); - if (!superior_group) { - superior_group = maat_hierarchy_group_new(hier, superior_group_id); - } - - int ret = igraph_get_eid(&hier->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); - //No duplicated edges between two groups. - if (edge_id > 0) { - log_error(hier->logger, MODULE_HIERARCHY, - "Add group %d to group %d failed, relation already exisited.", - group->group_id, superior_group->group_id); - ret = -1; - } else { - igraph_add_edge(&hier->group_graph, group->vertex_id, superior_group->vertex_id); - group->ref_by_superior_group_cnt++; - superior_group->ref_by_subordinate_group_cnt++; - ret = 0; - } - pthread_rwlock_unlock(&hier->rwlock); - - return ret; -} - -int maat_hierarchy_remove_group_from_group(struct maat_hierarchy *hier, int group_id, int superior_group_id) -{ - struct maat_hierarchy_group *group = NULL, *superior_group = NULL; - - //No hash write operation, LOCK protection is unnecessary. - hier->changed_flag = 1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if (NULL == group) { - log_error(hier->logger, MODULE_HIERARCHY, - "Del group %d from group %d failed, group %d not exisited.", - group_id, superior_group_id, group_id); - return -1; - } - - HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group); - if (NULL == superior_group) { - log_error(hier->logger, MODULE_HIERARCHY, - "Del group %d from group %d failed, superior group %d not exisited.", - group_id, superior_group_id, superior_group_id); - return -1; - } - - igraph_es_t es; - igraph_integer_t edge_num_before = 0, edge_num_after = 0; - - edge_num_before = igraph_ecount(&hier->group_graph); - // The edges between the given pairs of vertices will be included in the edge selection. - //The vertex pairs must be given as the arguments of the function call, the third argument - //is the first vertex of the first edge, the fourth argument is the second vertex of the - //first edge, the fifth is the first vertex of the second edge and so on. The last element - //of the argument list must be -1 to denote the end of the argument list. - //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small - int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); - assert(ret == IGRAPH_SUCCESS); - // ignore no such edge to abort(). - igraph_set_error_handler(igraph_error_handler_ignore); - ret = igraph_delete_edges(&hier->group_graph, es); - edge_num_after = igraph_ecount(&hier->group_graph); - igraph_es_destroy(&es); - - if (ret != IGRAPH_SUCCESS || edge_num_before - edge_num_after != 1) { - assert(0); - return -1; - } - - group->ref_by_superior_group_cnt--; - superior_group->ref_by_subordinate_group_cnt--; - - return 0; -} - -int maat_hierarchy_add_region_to_group(struct maat_hierarchy *hier, int group_id, int region_id, - int table_id, void *user_data) -{ - //A region rule belongs to ONE group only. - struct maat_hierarchy_group *group = NULL; - struct maat_hierarchy_region *region = NULL; + //A item rule belongs to ONE group only. + struct maat_group *group = NULL; + struct maat_hierarchy_item *item = NULL; int ret = 0; pthread_rwlock_wrlock(&hier->rwlock); hier->changed_flag = 1; + /* HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if (!group) { group = maat_hierarchy_group_new(hier, group_id); - } + }*/ - HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); - if (region) { + HASH_FIND_INT(hier->hash_item_by_id, &item_id, item); + if (item) { log_error(hier->logger, MODULE_HIERARCHY, - "Add region %d to group %d failed, region already in group %d.", - region_id, group_id, region->ref_parent_group->group_id); + "Add item %d to group %d failed, item already in group %d.", + item_id, group_id, item->ref_parent_group->group_id); ret = -1; } else { - region = maat_hierarchy_region_new(hier, region_id, group_id, table_id, group, user_data); + item = maat_hierarchy_item_new(hier, item_id, group_id, group, user_data); ret = 0; } pthread_rwlock_unlock(&hier->rwlock); @@ -1289,31 +1115,31 @@ int maat_hierarchy_add_region_to_group(struct maat_hierarchy *hier, int group_id return ret; } -int maat_hierarchy_remove_region_from_group(struct maat_hierarchy *hier, int group_id, int region_id) +int maat_hierarchy_remove_item_from_group(struct maat_hierarchy *hier, int group_id, int item_id) { - struct maat_hierarchy_group *group = NULL; - struct maat_hierarchy_region *region = NULL; + struct maat_group *group = NULL; + struct maat_hierarchy_item *item = NULL; pthread_rwlock_wrlock(&hier->rwlock); hier->changed_flag = 1; - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); + HASH_FIND(hh_group_id, hier->ref_group_topo->hash_group_by_id, &group_id, sizeof(group_id), group); if (!group) { log_error(hier->logger, MODULE_HIERARCHY, - "Remove region %d from group %d failed, group is not existed.", - region_id, group_id); + "Remove item %d from group %d failed, group is not existed.", + item_id, group_id); goto error_out; } - HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); - if (!region) { + HASH_FIND_INT(hier->hash_item_by_id, &item_id, item); + if (!item) { log_error(hier->logger, MODULE_HIERARCHY, - "Remove region %d from group %d failed, region is not exisited.", - region_id, group_id); + "Remove item %d from group %d failed, item is not exisited.", + item_id, group_id); goto error_out; } - assert(region->group_id == group->group_id); - maat_hierarchy_region_free(hier, region); + assert(item->group_id == group->group_id); + maat_hierarchy_item_free(hier, item); pthread_rwlock_unlock(&hier->rwlock); return 0; diff --git a/src/maat_rule.cpp b/src/maat_rule.cpp index 785f9d4..5d56f64 100644 --- a/src/maat_rule.cpp +++ b/src/maat_rule.cpp @@ -30,50 +30,6 @@ #define MODULE_MAAT_RULE module_name_str("maat.rule") -void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, - const char *srv_def, int srv_def_len) -{ - memcpy(rule, rule_head, sizeof(struct maat_rule_head)); - memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN)); -} - -void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, void *ex_data, - const struct compile_ex_data_schema *ex_schema) -{ - struct maat_rule rule; - - memset(&rule, 0, sizeof(rule)); - fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); - ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, ex_schema->argl, ex_schema->argp); -} - -void destroy_compile_rule(struct maat_compile_rule *compile_rule) -{ - size_t n_compile_ex_schema = table_schema_compile_rule_ex_data_schema_count(compile_rule->ref_table); - assert(compile_rule->magic_num == COMPILE_RULE_MAGIC); - - for (size_t i = 0; i < n_compile_ex_schema; i++) { - struct compile_ex_data_schema *compile_ex_schema = table_schema_get_compile_rule_ex_data_schema(compile_rule->ref_table, i); - rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, compile_ex_schema); - void *ex_data = compile_rule->ex_data + 1; - ex_data = NULL; - } - free(compile_rule->ex_data); - - compile_rule->is_valid = 0; - compile_rule->declared_clause_num = -1; - FREE(compile_rule->service_defined); - FREE(compile_rule); -} - -void maat_region_inner_free(struct maat_region_inner *region) -{ - assert(region->magic_num == REGION_RULE_MAGIC); - assert(region->expr_id_cnt == 0 ||region->expr_id_cnt == (region->expr_id_ub - region->expr_id_lb + 1)); - region->magic_num = 0; - FREE(region); -} - struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance) { struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); @@ -81,17 +37,14 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in maat_rt->version = version; maat_rt->table_rt_mgr = table_runtime_manager_create(maat_instance->table_schema_mgr, maat_instance->nr_worker_thread, - maat_instance->garbage_bin); + maat_instance->garbage_bin, + maat_instance->logger); maat_rt->max_table_num = table_schema_manager_get_size(maat_instance->table_schema_mgr); maat_rt->max_thread_num = maat_instance->nr_worker_thread; - maat_rt->hier = maat_hierarchy_new(maat_instance->nr_worker_thread, maat_instance->garbage_bin, - maat_instance->logger); - maat_hierarchy_set_compile_user_data_free_func(maat_rt->hier, (void (*)(void*))destroy_compile_rule); - maat_hierarchy_set_region_user_data_free_func(maat_rt->hier, (void (*)(void*))maat_region_inner_free); maat_rt->logger = maat_instance->logger; maat_rt->ref_garbage_bin = maat_instance->garbage_bin; - maat_rt->region_result_buff = ALLOC(struct scan_result, MAX_SCANNER_HIT_NUM * maat_instance->nr_worker_thread); + maat_rt->item_result_buff = ALLOC(struct scan_result, MAX_SCANNER_HIT_NUM * maat_instance->nr_worker_thread); maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->nr_worker_thread); return maat_rt; @@ -105,7 +58,7 @@ void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger continue; } - table_runtime_commit(table_rt, maat_rt->max_thread_num, logger); + table_runtime_commit(table_rt, maat_rt->version, maat_rt->max_thread_num, logger); } maat_rt->last_update_time = time(NULL); @@ -180,11 +133,12 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param) maat_rt = maat_instance->maat_rt; } - struct table_item *table_item = table_schema_line_to_item(line, table_schema, maat_instance->logger); + struct table_item *table_item = table_schema_line_to_item(line, table_schema, maat_instance->accept_tags, + maat_instance->n_accept_tag, maat_instance->logger); if (table_item != NULL) { struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); table_runtime_update(table_rt, table_schema, line, table_item, maat_instance->logger); - FREE(table_item); + table_item_free(table_item); } return 0; diff --git a/src/maat_table_runtime.cpp b/src/maat_table_runtime.cpp index 0dcc032..b8baece 100644 --- a/src/maat_table_runtime.cpp +++ b/src/maat_table_runtime.cpp @@ -16,7 +16,9 @@ #include "utils.h" #include "log/log.h" +#include "cJSON/cJSON.h" #include "maat_utils.h" +#include "maat_table_schema.h" #include "maat_table_runtime.h" #include "uthash/uthash.h" #include "maat_ex_data.h" @@ -24,6 +26,10 @@ #include "rcu_hash.h" #include "IPMatcher.h" #include "alignment.h" +#include "maat_hierarchy.h" +#include "maat_rule.h" +#include "igraph/igraph.h" +#include "maat_group.h" #define MODULE_TABLE_RUNTIME module_name_str("maat.table_runtime") @@ -40,6 +46,20 @@ struct ex_container_ctx { struct ex_data_schema *ex_schema; }; +struct compile_runtime { + struct maat_hierarchy *hier; +}; + +struct group2compile_runtime { + long long not_flag_group; + struct compile_runtime *ref_compile_rt; + struct group2group_runtime *ref_g2g_rt; +}; + +struct group2group_runtime { + struct maat_group_topology *group_topo; +}; + struct expr_runtime { enum hs_scan_mode scan_mode; struct adapter_hs *hs; @@ -62,8 +82,10 @@ struct table_runtime { uint32_t updating_rule_num; enum table_type table_type; union { + struct compile_runtime compile_rt; + struct group2compile_runtime g2c_rt; + struct group2group_runtime g2g_rt; struct expr_runtime expr_rt; - //struct ip_plugin_runtime ip_rt; struct ip_plugin_runtime ip_plus_rt; struct plugin_runtime plugin_rt; struct ip_plugin_runtime ip_plugin_rt; @@ -82,6 +104,7 @@ struct table_runtime_manager { struct table_runtime **table_rt; size_t n_table_rt; + struct maat_item *hash_item_by_id; struct maat_garbage_bin *garbage_bin; }; @@ -116,34 +139,108 @@ void expr_rule_free(and_expr_t *expr_rule) for (size_t i = 0; i < expr_rule->n_patterns; i++) { FREE(expr_rule->patterns[i].pat); } + + FREE(expr_rule); } void expr_ex_data_free(void *user_ctx, void *data) { and_expr_t *expr_rule = (and_expr_t *)data; expr_rule_free(expr_rule); - FREE(data); } -struct table_runtime *table_runtime_new(struct table_schema *table_schema, int max_thread_num, struct maat_garbage_bin* bin) +void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, + const char *srv_def, int srv_def_len) { + memcpy(rule, rule_head, sizeof(struct maat_rule_head)); + memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN)); +} + +void *rule_ex_data_new(const struct maat_rule_head *rule_head, const char *srv_def, const struct compile_ex_data_schema *ex_schema) +{ + void *ex_data = NULL; + struct maat_rule rule; + + fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); + ex_schema->new_func(ex_schema->idx, &rule, srv_def, &ex_data, ex_schema->argl, ex_schema->argp); + + return ex_data; +} + +void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, void *ex_data, + const struct compile_ex_data_schema *ex_schema) +{ + struct maat_rule rule; + + memset(&rule, 0, sizeof(rule)); + fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); + ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, ex_schema->argl, ex_schema->argp); +} + +void destroy_compile_rule(struct maat_compile_rule *compile_rule) +{ + size_t n_compile_ex_schema = table_schema_compile_rule_ex_data_schema_count(compile_rule->ref_table); + assert(compile_rule->magic_num == COMPILE_RULE_MAGIC); + + for (size_t i = 0; i < n_compile_ex_schema; i++) { + struct compile_ex_data_schema *compile_ex_schema = table_schema_get_compile_rule_ex_data_schema(compile_rule->ref_table, i); + rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, compile_ex_schema); + void *ex_data = compile_rule->ex_data + 1; + ex_data = NULL; + } + free(compile_rule->ex_data); + + compile_rule->is_valid = 0; + compile_rule->declared_clause_num = -1; + FREE(compile_rule->service_defined); + FREE(compile_rule); +} + +void maat_item_inner_free(struct maat_item_inner *item) +{ + assert(item->magic_num == ITEM_RULE_MAGIC); + assert(item->expr_id_cnt == 0 || item->expr_id_cnt == (item->expr_id_ub - item->expr_id_lb + 1)); + item->magic_num = 0; + FREE(item); +} + +struct table_runtime *table_runtime_new(struct table_schema *table_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + struct ex_container_ctx *ex_container_ctx = NULL; int table_id = table_schema_get_table_id(table_schema); struct table_runtime *table_rt = ALLOC(struct table_runtime, 1); - table_rt->ref_garbage_bin = bin; + table_rt->ref_garbage_bin = garbage_bin; table_rt->table_type = table_schema_get_table_type(table_schema); switch (table_rt->table_type) { + case TABLE_TYPE_COMPILE: + table_rt->compile_rt.hier = maat_hierarchy_new(max_thread_num, garbage_bin, logger); + maat_hierarchy_set_compile_user_data_free_func(table_rt->compile_rt.hier, (void (*)(void*))destroy_compile_rule); + maat_hierarchy_set_item_user_data_free_func(table_rt->compile_rt.hier, (void (*)(void*))maat_item_inner_free); + break; + case TABLE_TYPE_GROUP2COMPILE: + break; + case TABLE_TYPE_GROUP2GROUP: + table_rt->g2g_rt.group_topo = maat_group_topology_new(logger); + break; case TABLE_TYPE_EXPR: table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free); table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema); break; case TABLE_TYPE_IP_PLUS: table_rt->ip_plus_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); + ex_container_ctx = ALLOC(struct ex_container_ctx, 1); + ex_container_ctx->custom_data_free = free; + //ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx); break; case TABLE_TYPE_PLUGIN: table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); break; case TABLE_TYPE_IP_PLUGIN: table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); + ex_container_ctx = ALLOC(struct ex_container_ctx, 1); + ex_container_ctx->custom_data_free = free; + //ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx); break; default: break; @@ -156,13 +253,21 @@ struct table_runtime *table_runtime_new(struct table_schema *table_schema, int m return table_rt; } -void table_runtime_free(struct table_runtime * table_rt) +void table_runtime_free(struct table_runtime *table_rt) { if (NULL == table_rt) { return; } switch (table_rt->table_type) { + case TABLE_TYPE_COMPILE: + maat_hierarchy_free(table_rt->compile_rt.hier); + break; + case TABLE_TYPE_GROUP2COMPILE: + break; + case TABLE_TYPE_GROUP2GROUP: + maat_group_topology_free(table_rt->group2group_rt.group_topo); + break; case TABLE_TYPE_EXPR: adapter_hs_destroy(table_rt->expr_rt.hs); rcu_hash_free(table_rt->expr_rt.htable); @@ -183,7 +288,7 @@ void table_runtime_free(struct table_runtime * table_rt) struct table_runtime_manager * table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num, - struct maat_garbage_bin* garbage_bin) + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == table_schema_mgr) { return NULL; @@ -193,12 +298,33 @@ table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int table_rt_mgr->n_table_rt = table_schema_manager_get_size(table_schema_mgr); table_rt_mgr->table_rt = ALLOC(struct table_runtime *, table_rt_mgr->n_table_rt); - for (size_t i = 0; i < table_rt_mgr->n_table_rt; i++) { + size_t i = 0; + for (i = 0; i < table_rt_mgr->n_table_rt; i++) { struct table_schema *table_schema = table_schema_get(table_schema_mgr, i); if (NULL == table_schema) { continue; } - table_rt_mgr->table_rt[i] = table_runtime_new(table_schema, max_thread_num, garbage_bin); + table_rt_mgr->table_rt[i] = table_runtime_new(table_schema, max_thread_num, garbage_bin, logger); + } + + /* group2compile table_rt depends on associated compile table_rt, + must make sure associated compile table_rt already exist */ + for (i = 0; i < table_rt_mgr->n_table_rt; i++) { + struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, i); + if (NULL == table_rt) { + continue; + } + + enum table_type table_type = table_runtime_get_type(table_rt); + if (table_type != TABLE_TYPE_GROUP2COMPILE) { + continue; + } + + struct table_schema *table_schema = table_schema_get(table_schema_mgr, i); + int associated_compile_table_id = table_schema_get_associated_table_id(table_schema); + struct table_runtime *compile_table_rt = table_runtime_get(table_rt_mgr, associated_compile_table_id); + table_rt->group2compile_rt.ref_compile_rt = &compile_table_rt->compile_rt; + assert(table_rt->group2compile_rt.ref_compile_rt != NULL); } return table_rt_mgr; @@ -294,7 +420,7 @@ enum IP_TYPE ip_type_transform(enum ip_type type) } } -int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, +int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct ip_addr *scan_data, struct scan_result *results, size_t *n_result, size_t n_result_array) { if (NULL == table_rt) { @@ -306,11 +432,11 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct struct scan_result scan_results[n_result_array] = {0}; struct ip_data ip; - ip.type = ip_type_transform(data->ip_type); + ip.type = ip_type_transform(scan_data->ip_type); if (ip.type == IPv4) { - ip.ipv4 = data->ipv4; + ip.ipv4 = scan_data->ipv4; } else { - memcpy(ip.ipv6, data->ipv6, sizeof(data->ipv6)); + memcpy(ip.ipv6, scan_data->ipv6, sizeof(scan_data->ipv6)); } n_hit_result = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, n_result_array); @@ -319,12 +445,7 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct } int index = 0; - /* filter rule_id by port */ for (i = 0; i < n_hit_result; i++) { - struct port_range *port_range = (struct port_range *)scan_results[i].tag; - if (data->port < port_range->min_port || data->port > port_range->max_port) { - continue; - } results[index++].rule_id = scan_results[i].rule_id; } *n_result = index; @@ -394,7 +515,7 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl } if (i >= MAAT_MAX_EXPR_ITEM_NUM) { - log_error(logger, MODULE_TABLE_RUNTIME, "item_id:%d too many patterns", + log_error(logger, MODULE_TABLE_RUNTIME, "expr item_id:%d too many patterns", expr_item->item_id); return NULL; } @@ -429,16 +550,128 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl return expr_rule; } +struct maat_item_inner* maat_item_inner_new(int group_id, int item_id, int district_id) +{ + struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1); + item->magic_num = ITEM_RULE_MAGIC; + item->item_id = item_id; + item->group_id = group_id; + item->district_id = district_id; + + return item; +} + +void maat_item_inner_free(struct maat_item_inner *item) +{ + assert(item->magic_num == ITEM_RULE_MAGIC); + assert(item->expr_id_cnt == 0 || item->expr_id_cnt == item->expr_id_ub - item->expr_id_lb + 1); + item->magic_num = 0; + + free(item); +} + +int compile_runtime_update_row(struct compile_runtime *compile_rt, const char *table_name, int compile_id, + struct maat_compile_rule *compile_rule, struct maat_garbage_bin *garbage_bin, + int is_valid, struct log_handle *logger) +{ + int ret = -1; + struct maat_hierarchy *maat_hier = compile_rt->hier; + + if (0 == is_valid) { + // delete + struct maat_compile_rule *p_compile = (struct maat_compile_rule *)maat_hierarchy_compile_dettach_user_data(compile_rt->hier, compile_id); + if (p_compile != NULL) { + ret = maat_hierarchy_compile_remove(compile_rt->hier, compile_id); + assert(ret == 0); + maat_garbage_bagging(garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule); + } + } else { + // add + ret = maat_hierarchy_compile_add(compile_rt->hier, compile_id, compile_rule->declared_clause_num, compile_rule); + if (ret != 0) { + log_info(logger, MODULE_TABLE_RUNTIME, "duplicate config of compile table %s compile_id %d", + table_name, compile_id); + destroy_compile_rule(compile_rule); + } + } + + return ret; +} + +int group2compile_runtime_update_row(struct group2compile_runtime *g2c_rt, + int compile_id, struct group2compile_item *g2c_item, + int is_valid, struct log_handle *logger) +{ + int ret = -1; + struct compile_runtime *compile_rt = g2c_rt->ref_compile_rt; + struct group2group_runtime *g2g_rt = g2c_rt->ref_g2g_rt; + struct maat_group *group = NULL; + + if (0 == is_valid) { + /*TODO: by luis*/ + group = maat_group_topology_find_group(g2g_rt->group_topo, g2c_item->group_id); + if (NULL == group) { + log_error(logger, MODULE_TABLE_RUNTIME, + "Remove group %d from compile %d failed, group is not exisited.", + g2c_item->group_id, compile_id); + return ret; + } + + ret = maat_hierarchy_remove_group_from_compile(compile_rt->hier, g2c_item->group_id, g2c_item->virtual_table_id, + g2c_item->not_flag, g2c_item->clause_index, g2c_item->compile_id); + if (0 == ret) { + group->ref_by_compile_cnt--; + if (g2c_item->not_flag) { + g2c_rt->not_flag_group--; + } + } + } else { + group = maat_group_topology_find_group(g2g_rt->group_topo, g2c_item->group_id); + if (NULL == group) { + group = maat_group_topology_add_group(g2g_rt->group_topo, g2c_item->group_id); + } + + ret = maat_hierarchy_add_group_to_compile(compile_rt->hier, g2c_item->group_id, g2c_item->virtual_table_id, + g2c_item->not_flag, g2c_item->clause_index, g2c_item->compile_id); + if (0 == ret) { + group->ref_by_compile_cnt++; + if (g2c_item->not_flag) { + g2c_rt->not_flag_group++; + } + } + } + + return ret; +} + +int group2group_runtime_update_row(struct group2group_runtime *g2g_rt, int group_id, + struct group2group_item *g2g_item, + int is_valid, struct log_handle *logger) +{ + int ret = -1; + + if (0 == is_valid) { + //delete + ret = maat_group_topology_add_group_to_group(g2g_rt->group_topo, group_id, g2g_item->superior_group_id); + } else { + //add + ret = maat_group_topology_remove_group_from_group(g2g_rt->group_topo, group_id, g2g_item->superior_group_id); + } + + return ret; +} + void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len, and_expr_t *expr_rule, int is_valid, struct log_handle *logger) { void *data = NULL; - if (is_valid == 0) { + if (0 == is_valid) { //delete data = rcu_hash_find(expr_rt->htable, key, key_len); if (NULL == data) { - log_error(logger, MODULE_TABLE_RUNTIME, "the key:%s not exist, so can't be deleted", key); + log_error(logger, MODULE_TABLE_RUNTIME, + "the key of expr rule not exist, can't be deleted, expr_id:%d", expr_rule->expr_id); return; } rcu_hash_del(expr_rt->htable, key, key_len); @@ -446,36 +679,28 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key //add data = rcu_hash_find(expr_rt->htable, key, key_len); if (data != NULL) { - log_error(logger, MODULE_TABLE_RUNTIME, "the key:%s already exist, so can't be added", key); + log_error(logger, MODULE_TABLE_RUNTIME, + "the key of expr rule already exist, can't be added, expr_id:%d", expr_rule->expr_id); return; } - and_expr_t *data = ALLOC(and_expr_t, 1); - memcpy(data, expr_rule, sizeof(and_expr_t)); - for (size_t i = 0; i < expr_rule->n_patterns; i++) { - data->patterns[i].pat = ALLOC(char, expr_rule->patterns[i].pat_len); - memcpy(data->patterns[i].pat, expr_rule->patterns[i].pat, expr_rule->patterns[i].pat_len); - } - rcu_hash_add(expr_rt->htable, key, key_len, (void *)data); + rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule); } } -void ip_plus_runtime_update_row(struct ip_plugin_runtime *ip_plus_rt, struct table_schema *table_schema, - const char *row, char *key, size_t key_len, struct ip_plus_item *ip_plus_item, - int is_valid, struct log_handle *logger) +void ip_plus_runtime_update_row(struct ip_plugin_runtime *ip_plus_rt, char *key, size_t key_len, + struct ip_plus_item *ip_plus_item, int is_valid, + struct log_handle *logger) { - struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt; - ex_container->ex_data = NULL; - ex_container->custom_data = ip_plus_item; - struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1); - ctx->custom_data_free = free; - ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx); if (0 == is_valid) { // delete ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger); } else { // add + struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); + ex_container->ex_data = NULL; + ex_container->custom_data = ip_plus_item; ex_data_runtime_add_ex_container(ip_plus_rt->ex_data_rt, key, key_len, ex_container); } } @@ -516,13 +741,7 @@ void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct const char *row, char *key, size_t key_len, struct ip_plugin_item *ip_plugin_item, int is_valid, struct log_handle *logger) { - struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt; - ex_container->ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); - ex_container->custom_data = ip_plugin_item; - struct ex_container_ctx *ctx = ex_data_runtime_get_ex_container_ctx(ex_data_rt); - ctx->custom_data_free = free; - int set_flag = table_schema_ex_data_schema_flag(table_schema); if (1 == set_flag) { @@ -531,6 +750,9 @@ void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger); } else { //add + struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); + ex_container->ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); + ex_container->custom_data = ip_plugin_item; ex_data_runtime_add_ex_container(ip_plugin_rt->ex_data_rt, key, key_len, ex_container); } } else { @@ -538,50 +760,226 @@ void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct } } +struct maat_compile_rule *compile_item_to_compile_rule(struct compile_item *compile_item, struct table_schema *table_schema, struct log_handle *logger) +{ + struct maat_rule_head rule_head; + rule_head.config_id = compile_item->compile_id; + rule_head.service_id = compile_item->service_id; + rule_head.action = compile_item->action; + rule_head.do_blacklist = compile_item->do_blacklist; + rule_head.do_log = compile_item->do_log; + + struct maat_compile_rule *p = ALLOC(struct maat_compile_rule, 1); + + p->magic_num = COMPILE_RULE_MAGIC; + p->head = rule_head; + p->declared_clause_num = compile_item->clause_num; + p->ex_data = ALLOC(void *, MAX_COMPILE_EX_DATA_NUM); + + p->ref_table = table_schema; + p->head.serv_def_len = strlen(compile_item->user_region); + p->service_defined = ALLOC(char, p->head.serv_def_len); + memcpy(p->service_defined, compile_item->user_region, p->head.serv_def_len); + p->evaluation_order = compile_item->evaluation_order; + + size_t n_rule_ex_schema = table_schema_compile_rule_ex_data_schema_count(table_schema); + for (size_t i = 0; i < n_rule_ex_schema; i++) { + struct compile_ex_data_schema *ex_schema = table_schema_get_compile_rule_ex_data_schema(table_schema, i); + p->ex_data[i] = rule_ex_data_new(&p->head, p->service_defined, ex_schema); + } + p->is_valid = 1; + p->compile_id = compile_item->compile_id; + pthread_rwlock_init(&p->rwlock, NULL); + + return p; +} + +int compile_runtime_update(struct compile_runtime *compile_rt, struct compile_item *compile_item, + struct table_schema *table_schema, const char *table_name, + struct maat_garbage_bin *garbage_bin, int is_valid, struct log_handle *logger) +{ + int ret = -1; + + if (NULL == compile_rt || NULL == compile_item) { + return ret; + } + + struct maat_compile_rule *compile_rule = NULL; + if (1 == is_valid) { + compile_rule = compile_item_to_compile_rule(compile_item, table_schema, logger); + if (NULL == compile_rule) { + log_error(logger, MODULE_TABLE_RUNTIME, + "transform compile table:%s item to compile_rule failed, compile_id:%d", + table_name, compile_item->compile_id); + return ret; + } + } + + ret = compile_runtime_update_row(compile_rt, table_name, compile_item->compile_id, compile_rule, garbage_bin, is_valid, logger); + return ret; +} + +int group2compile_runtime_update(struct group2compile_runtime *g2c_rt, struct group2compile_item *g2c_item, + const char *table_name, int is_valid, struct log_handle *logger) +{ + int ret = -1; + + if (NULL == g2c_rt || NULL == g2c_item) { + return ret; + } + + ret = group2compile_runtime_update_row(g2c_rt, g2c_item->compile_id, g2c_item, is_valid, logger); + return ret; +} + +int group2group_runtime_update(struct group2group_runtime *g2g_rt, struct group2group_item *g2g_item, + const char *table_name, int is_valid, struct log_handle *logger) +{ + int ret = -1; + + if (NULL == g2g_rt || NULL == g2g_item) { + return ret; + } + + ret = group2group_runtime_update_row(g2g_rt, g2g_item->group_id, g2g_item, is_valid, logger); + return ret; +} + +int expr_runtime_update(struct expr_runtime *expr_rt, struct expr_item *expr_item, + const char *table_name, int is_valid, struct log_handle *logger) +{ + if (NULL == expr_rt || NULL == expr_item) { + return -1; + } + + /* add item to group */ + struct maat_item_inner *u_para = NULL; + u_para = maat_item_inner_new(expr_item->group_id, expr_item->item_id, -1); + int ret = maat_hierarchy_add_item_to_group(scanner->hier, expr_item->group_id, expr_item->item_id, u_para); + if (ret != 0) { + maat_item_inner_free(u_para); + u_para = NULL; + return -1; + } + + and_expr_t *expr_rule = NULL; + if (1 == is_valid) { + expr_rule = expr_item_to_expr_rule(expr_item, logger); + if (NULL == expr_rule) { + log_error(logger, MODULE_TABLE_RUNTIME, "transform expr table:%s item to expr_rule failed, item_id:%d", + table_name, expr_item->item_id); + return -1; + } + } + + char *key = (char *)&(expr_item->item_id); + expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, is_valid, logger); + + return 0; +} + +int ip_plus_runtime_update(struct ip_plugin_runtime *ip_plus_rt, struct ip_plus_item *ip_plus_item, + const char *table_name, int is_valid, struct log_handle *logger) +{ + if (NULL == ip_plus_rt || NULL == ip_plus_item) { + return -1; + } + + struct ip_plus_item *item = NULL; + if (1 == is_valid) { + item = ALLOC(struct ip_plus_item, 1); + memcpy(item, ip_plus_item, sizeof(struct ip_plus_item)); + } + + char *key = (char *)&(ip_plus_item->item_id); + ip_plus_runtime_update_row(ip_plus_rt, key, sizeof(int), ip_plus_item, is_valid, logger); + + return 0; +} + +int plugin_runtime_update(struct plugin_runtime *plugin_rt, struct plugin_item *plugin_item, + struct table_schema *table_schema, const char *row, int is_valid, + struct log_handle *logger) +{ + if (NULL == plugin_rt || NULL == plugin_item) { + return -1; + } + + char *key = plugin_item->key; + size_t key_len = plugin_item->key_len; + + plugin_runtime_update_row(plugin_rt, table_schema, row, key, key_len, is_valid, logger); + + return 0; +} + +int ip_plugin_runtime_update(struct ip_plugin_runtime *ip_plugin_rt, + struct ip_plugin_item *ip_plugin_item, + struct table_schema *table_schema, const char *row, + int is_valid, struct log_handle *logger) +{ + if (NULL == ip_plugin_rt || NULL == ip_plugin_item) { + return -1; + } + + struct ip_plugin_item *item = NULL; + if (1 == is_valid) { + item = ALLOC(struct ip_plugin_item, 1); + memcpy(item, ip_plugin_item, sizeof(struct ip_plugin_item)); + } + + char *key = (char *)&(ip_plugin_item->item_id); + ip_plugin_runtime_update_row(ip_plugin_rt, table_schema, row, key, sizeof(int), + item, is_valid, logger); + + return 0; +} + void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, const char *row, struct table_item *table_item, struct log_handle *logger) { + int ret = -1; int is_valid = -1; - char *key = NULL; - size_t key_len = 0; - and_expr_t *expr_rule = NULL; - struct ip_plus_item *ip_plus_item = NULL; - struct ip_plugin_item *ip_plugin_item = NULL; + const char *table_name = table_schema_get_updating_name(table_schema); + int table_id = table_schema_get_table_id(table_schema); switch (table_rt->table_type) { + case TABLE_TYPE_COMPILE: + is_valid = table_item->compile_item.is_valid; + ret = compile_runtime_update(&(table_rt->compile_rt), &(table_item->compile_item), table_schema, table_name, table_rt->ref_garbage_bin, is_valid, logger); + break; + case TABLE_TYPE_GROUP2COMPILE: + is_valid = table_item->g2c_item.is_valid; + ret = group2compile_runtime_update(&(table_rt->g2c_rt), &(table_item->g2c_item), table_name, is_valid, logger); + break; + case TABLE_TYPE_GROUP2GROUP: + is_valid = table_item->g2g_item.is_valid; + ret = group2group_runtime_update(&(table_rt->g2g_rt), &(table_item->g2g_item), table_name, is_valid, logger); + break; case TABLE_TYPE_EXPR: is_valid = table_item->expr_item.is_valid; - expr_rule = expr_item_to_expr_rule(&table_item->expr_item, logger); - key = (char *)&(table_item->expr_item.item_id); - expr_runtime_update_row(&(table_rt->expr_rt), key, sizeof(int), expr_rule, is_valid, logger); - expr_rule_free(expr_rule); + ret = expr_runtime_update(&(table_rt->expr_rt), &(table_item->expr_item), table_name, is_valid, logger); break; case TABLE_TYPE_IP_PLUS: is_valid = table_item->ip_plus_item.is_valid; - ip_plus_item = ALLOC(struct ip_plus_item, 1); - memcpy(ip_plus_item, &(table_item->ip_plus_item), sizeof(table_item->ip_plus_item)); - key = (char *)&(table_item->ip_plus_item.item_id); - ip_plus_runtime_update_row(&(table_rt->ip_plus_rt), table_schema, row, key, sizeof(int), - ip_plus_item, is_valid, logger); + ret = ip_plus_runtime_update(&(table_rt->ip_plus_rt), &(table_item->ip_plus_item), table_name, is_valid, logger); break; case TABLE_TYPE_PLUGIN: is_valid = table_item->plugin_item.is_valid; - key = table_item->plugin_item.key; - key_len = table_item->plugin_item.key_len; - plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, key, key_len, is_valid, logger); + ret = plugin_runtime_update(&(table_rt->plugin_rt), &(table_item->plugin_item), table_schema, row, is_valid, logger); break; case TABLE_TYPE_IP_PLUGIN: is_valid = table_item->ip_plugin_item.is_valid; - ip_plugin_item = ALLOC(struct ip_plugin_item, 1); - memcpy(ip_plugin_item, &(table_item->ip_plugin_item), sizeof(table_item->ip_plugin_item)); - key = (char *)&(table_item->ip_plugin_item.item_id); - ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int), - ip_plugin_item, is_valid, logger); + ret = ip_plugin_runtime_update(&(table_rt->ip_plugin_rt), &(table_item->ip_plugin_item), table_schema, row, is_valid, logger); break; default: break; } + if (ret < 0) { + return; + } + if (is_valid == 0) { table_rt->rule_num--; } else { @@ -589,6 +987,22 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t } } +int compile_runtime_commit(struct table_runtime *table_rt, long long version, struct log_handle *logger) +{ + int ret = -1; + + ret = maat_hierarchy_rebuild(table_rt->compile_rt.hier); + if (ret != 0) { + log_error(logger, MODULE_TABLE_RUNTIME, "version %lld hierarchy rebuild failed.", + version); + return -1; + } else { + log_info(logger, MODULE_TABLE_RUNTIME, "version %lld hierarchy rebuild success.", + version); + return 0; + } +} + int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, struct log_handle *logger) { @@ -763,10 +1177,15 @@ int table_runtime_updating_flag(struct table_runtime *table_rt) return updating_flag; } -void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, +void table_runtime_commit(struct table_runtime *table_rt, long long version, size_t nr_worker_thread, struct log_handle *logger) { switch (table_rt->table_type) { + case TABLE_TYPE_COMPILE: + case TABLE_TYPE_GROUP2COMPILE: + case TABLE_TYPE_GROUP2GROUP: + compile_runtime_commit(table_rt, version, logger); + break; case TABLE_TYPE_EXPR: expr_runtime_commit(table_rt, nr_worker_thread, logger); break; @@ -840,7 +1259,7 @@ struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table } void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, - int nr_worker_thread, struct log_handle *logger) + int nr_worker_thread, long long version, struct log_handle *logger) { struct ex_data_schema *ex_data_schema = table_schema_get_ex_data_schema(table_schema); struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt); @@ -867,5 +1286,5 @@ void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct } ex_data_runtime_clear_row_cache(ex_data_rt); - table_runtime_commit(table_rt, nr_worker_thread, logger); + table_runtime_commit(table_rt, version, nr_worker_thread, logger); } diff --git a/src/maat_table_schema.cpp b/src/maat_table_schema.cpp index 6f41103..196e5d4 100644 --- a/src/maat_table_schema.cpp +++ b/src/maat_table_schema.cpp @@ -25,12 +25,9 @@ #define MODULE_TABLE_SCHEMA module_name_str("maat.table_schema") -#define MAX_TABLE_LINE_SIZE (1024 * 16) #define MAX_FOREIGN_CLMN_NUM 8 #define MAX_CONJUNCTION_TABLE_NUM 8 -#define MAX_COMPILE_EX_DATA_NUM 2 - enum user_region_encode { USER_REGION_ENCODE_NONE=0, USER_REGION_ENCODE_ESCAPE, @@ -38,11 +35,37 @@ enum user_region_encode { }; struct compile_table_schema { + int compile_id_column; + int service_id_column; + int action_column; + int do_blacklist_column; + int do_log_column; + int tags_column; + int user_region_column; + int is_valid_column; + int clause_num_column; + int evaluation_order_column; enum user_region_encode user_region_encoding; size_t n_ex_schema; struct compile_ex_data_schema ex_schema[MAX_COMPILE_EX_DATA_NUM]; }; +struct group2compile_table_schema { + int group_id_column; + int compile_id_column; + int is_valid_column; + int not_flag_column; + int virtual_table_name_column; + int clause_index_column; + char associated_compile_table_id; +}; + +struct group2group_table_schema { + int group_id_column; + int superior_group_id_column; + int is_valid_column; +}; + struct expr_table_schema { int item_id_column; int group_id_column; @@ -117,6 +140,8 @@ struct table_schema { enum table_type table_type; union { struct compile_table_schema compile; + struct group2compile_table_schema group2compile; + struct group2group_table_schema group2group; struct expr_table_schema expr; struct ip_plus_table_schema ip_plus; struct plugin_table_schema plugin; @@ -124,12 +149,16 @@ struct table_schema { struct virtual_table_schema virtual_table; struct composition_table_schema composition; }; + + uint64_t update_err_cnt; + uint64_t unmatched_tag_cnt; }; struct table_schema_manager { struct table_schema *schema_table[MAX_TABLE_NUM]; size_t n_schema_table; size_t n_active_plugin_table; + char default_compile_table[NAME_MAX]; struct maat_kv_store *tablename2id_map; struct log_handle *logger; }; @@ -147,7 +176,7 @@ void table_schema_free(struct table_schema *ptable) FREE(ptable); } -int read_expr_table_schema(cJSON *root, struct table_schema *ptable, +int read_compile_table_schema(cJSON *root, struct table_schema *ptable, struct maat_kv_store* reserved_word_map, struct log_handle *logger) { @@ -160,6 +189,239 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable, read_cnt++; } + json = cJSON_GetObjectItem(root, "table_name"); + if (json != NULL && json->type == cJSON_String) { + if (strlen(json->valuestring) >= NAME_MAX) { + log_error(logger, MODULE_TABLE_SCHEMA, + "compile table name %s length too long", json->valuestring); + return -1; + } + memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "custom"); + if (json == NULL || json->type != cJSON_Object) { + log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column", + ptable->table_name[0]); + return -1; + } + + cJSON *item = cJSON_GetObjectItem(json, "compile_id"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.compile_id_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "service_id"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.service_id_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "action"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.action_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "do_blacklist"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.do_blacklist_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "do_log"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.do_log_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "tags"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.tags_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "user_region"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.user_region_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "is_valid"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.is_valid_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "clause_num"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.clause_num_column = item->valueint; + read_cnt++; + } + item = cJSON_GetObjectItem(json, "evaluation_order"); + if (item != NULL && item->type == cJSON_Number) { + ptable->compile.evaluation_order_column = item->valueint; + read_cnt++; + } + + if (read_cnt < 12) { + return -1; + } + + return 0; +} + +int read_group2compile_table_schema(cJSON *root, struct table_schema *ptable, + struct maat_kv_store* reserved_word_map, + struct log_handle *logger) +{ + int read_cnt = 0; + cJSON *json = NULL; + + json = cJSON_GetObjectItem(root, "table_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->table_id = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "table_name"); + if (json != NULL && json->type == cJSON_String) { + if (strlen(json->valuestring) >= NAME_MAX) { + log_error(logger, MODULE_TABLE_SCHEMA, + "group2compile table name %s length too long", json->valuestring); + return -1; + } + memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "associated_compile_table_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->group2compile.associated_compile_table_id = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "custom"); + if (json == NULL || json->type != cJSON_Object) { + log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column", + ptable->table_name[0]); + return -1; + } + + cJSON *item = cJSON_GetObjectItem(json, "group_id"); + if (item != NULL && item->type == cJSON_Number) { + ptable->group2compile.group_id_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "compile_id"); + if (item != NULL && item->type == cJSON_Number) { + ptable->group2compile.compile_id_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "is_valid"); + if (item != NULL && item->type == cJSON_Number) { + ptable->group2compile.is_valid_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "not_flag"); + if (item != NULL && item->type == cJSON_Number) { + ptable->group2compile.not_flag_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "virtual_table_name"); + if (item != NULL && item->type == cJSON_Number) { + ptable->group2compile.virtual_table_name_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "clause_index"); + if (item != NULL && item->type == cJSON_Number) { + ptable->group2compile.clause_index_column = item->valueint; + read_cnt++; + } + + if (read_cnt < 8) { + return -1; + } + + return 0; +} + +int read_group2group_table_schema(cJSON *root, struct table_schema *ptable, + struct maat_kv_store* reserved_word_map, + struct log_handle *logger) +{ + int read_cnt = 0; + cJSON *json = NULL; + + json = cJSON_GetObjectItem(root, "table_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->table_id = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "table_name"); + if (json != NULL && json->type == cJSON_String) { + if (strlen(json->valuestring) >= NAME_MAX) { + log_error(logger, MODULE_TABLE_SCHEMA, + "group2group table name %s length too long", json->valuestring); + return -1; + } + memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "custom"); + if (json == NULL || json->type != cJSON_Object) { + log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column", + ptable->table_name[0]); + return -1; + } + + cJSON *item = cJSON_GetObjectItem(json, "group_id"); + if (item != NULL && item->type == cJSON_Number) { + ptable->group2group.group_id_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "superior_group_id"); + if (item != NULL && item->type == cJSON_Number) { + ptable->group2group.superior_group_id_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "is_valid"); + if (item != NULL && item->type == cJSON_Number) { + ptable->group2group.is_valid_column = item->valueint; + read_cnt++; + } + + if (read_cnt < 5) { + return -1; + } + + return 0; +} + +int read_expr_table_schema(cJSON *root, struct table_schema *ptable, + struct maat_kv_store* reserved_word_map, + struct log_handle *logger) +{ + int read_cnt = 0; + cJSON *json = NULL; + + json = cJSON_GetObjectItem(root, "table_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->table_id = json->valueint; + read_cnt++; + } + json = cJSON_GetObjectItem(root, "table_name"); if (json != NULL && json->type == cJSON_String) { if (strlen(json->valuestring) >= NAME_MAX) { @@ -704,49 +966,44 @@ int table_schema_populate(cJSON *json, struct table_schema **table_array, return -1; } } - + switch (ptable->table_type) { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - ret = read_expr_table_schema(json, ptable, reserved_word_map, logger); - if (ret < 0) { - return -1; - } - break; - case TABLE_TYPE_IP_PLUS: - ret = read_ip_plus_table_schema(json, ptable, logger); - if (ret < 0) { - return -1; - } - break; - case TABLE_TYPE_PLUGIN: - ret = read_plugin_table_schema(json, ptable, logger); - if (ret < 0) { - return -1; - } - break; - case TABLE_TYPE_IP_PLUGIN: - ret = read_ip_plugin_table_schema(json, ptable, logger); - if (ret < 0) { - return -1; - } - break; - case TABLE_TYPE_VIRTUAL: - ret = read_virtual_table_schema(json, table_array, ptable, tablename2id_map, logger); - if (ret < 0) { - return -1; - } - break; - case TABLE_TYPE_COMPOSITION: - ret = read_composition_table_schema(json, ptable, tablename2id_map, logger); - if (ret < 0) { - return -1; - } - break; - default: - break; + case TABLE_TYPE_COMPILE: + ret = read_compile_table_schema(json, ptable, reserved_word_map, logger); + break; + case TABLE_TYPE_GROUP2COMPILE: + ret = read_group2compile_table_schema(json, ptable, reserved_word_map, logger); + break; + case TABLE_TYPE_GROUP2GROUP: + ret = read_group2group_table_schema(json, ptable, reserved_word_map, logger); + break; + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + ret = read_expr_table_schema(json, ptable, reserved_word_map, logger); + break; + case TABLE_TYPE_IP_PLUS: + ret = read_ip_plus_table_schema(json, ptable, logger); + break; + case TABLE_TYPE_PLUGIN: + ret = read_plugin_table_schema(json, ptable, logger); + break; + case TABLE_TYPE_IP_PLUGIN: + ret = read_ip_plugin_table_schema(json, ptable, logger); + break; + case TABLE_TYPE_VIRTUAL: + ret = read_virtual_table_schema(json, table_array, ptable, tablename2id_map, logger); + break; + case TABLE_TYPE_COMPOSITION: + ret = read_composition_table_schema(json, ptable, tablename2id_map, logger); + break; + default: + break; } - + + if (ret < 0) { + return -1; + } + return 0; } @@ -783,6 +1040,9 @@ table_schema_manager_create(const char *table_info_path, struct log_handle *logg struct maat_kv_store* reserved_word_map = maat_kv_store_new(); /* register table type reserved word */ + maat_kv_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE); + maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); + maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); @@ -1092,6 +1352,15 @@ int table_schema_get_table_id(struct table_schema *table_schema) return table_schema->table_id; } +int table_schema_get_associated_table_id(struct table_schema *table_schema) +{ + if (NULL == table_schema || table_schema->table_type != TABLE_TYPE_GROUP2COMPILE) { + return -1; + } + + return table_schema->group2compile.associated_compile_table_id; +} + enum scan_type table_schema_get_scan_type(struct table_schema *table_schema) { enum scan_type ret = SCAN_TYPE_INVALID; @@ -1146,6 +1415,15 @@ int table_schema_get_valid_flag_column(struct table_schema *table_schema) } switch (table_schema->table_type) { + case TABLE_TYPE_COMPILE: + valid_flag_column = table_schema->compile.is_valid_column; + break; + case TABLE_TYPE_GROUP2COMPILE: + valid_flag_column = table_schema->group2compile.is_valid_column; + break; + case TABLE_TYPE_GROUP2GROUP: + valid_flag_column = table_schema->group2group.is_valid_column; + break; case TABLE_TYPE_EXPR: valid_flag_column = table_schema->expr.is_valid_column; break; @@ -1155,13 +1433,6 @@ int table_schema_get_valid_flag_column(struct table_schema *table_schema) case TABLE_TYPE_IP_PLUGIN: valid_flag_column = table_schema->ip_plugin.is_valid_column; break; - /* - case TABLE_TYPE_FQDN_PLUGIN: - valid_flag_column = table_schema->fqdn_plugin.valid_flag_column; - break; - case TABLE_TYPE_BOOL_PLUGIN: - valid_flag_column = table_schema->bool_plugin.valid_flag_column; - break;*/ default: valid_flag_column = -1; break; @@ -1185,6 +1456,326 @@ void table_schema_set_updating_name(struct table_schema *table_schema, const cha assert(i <= table_schema->conj_cnt); } +const char *table_schema_get_updating_name(struct table_schema *table_schema) +{ + return table_schema->table_name[table_schema->updating_name]; +} + +static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, int n_accept_tag) +{ + cJSON *tab_name_obj = cJSON_GetObjectItem(tag_obj, "tag"); + if (NULL == tab_name_obj || tab_name_obj->type != cJSON_String) { + goto error; + } + + const char *tag_name = tab_name_obj->valuestring; + cJSON *tag_vals_array = cJSON_GetObjectItem(tag_obj, "value"); + if (NULL == tag_vals_array || tag_vals_array->type != cJSON_Array) { + goto error; + } + + int name_matched = 0; + int n_val = cJSON_GetArraySize(tag_vals_array); + + for (int i = 0; i < n_accept_tag; i++) { + if (0 != strcmp(accept_tags[i].tag_name, tag_name)) { + continue; + } + name_matched++; + + for (int j = 0; j < n_val; j++) { + cJSON *tag_val_obj = cJSON_GetArrayItem(tag_vals_array, j); + if (NULL == tag_val_obj || tag_val_obj->type != cJSON_String) { + goto error; + } + + const char *tag_val = tag_val_obj->valuestring; + // compare a/b/c with a/b/c/d is a miss. + if (strlen(accept_tags[i].tag_val) < strlen(tag_val)) { + continue; + } + + // compare a1a2/b1/c1 with a1a2/b/ is a miss. + //make sure the overlap is ended with a '/' + size_t compare_len = + if (0 == strncmp(accept_tags[i].tag_val, tag_val, strlen(tag_val)) && + (strlen(accept_tags[i].tag_val) == strlen(tag_val) || accept_tags[i].tag_val[strlen(tag_val)] == '/')) { + return 1; + } + } + } + + //no matched name is considered as a + if (name_matched > 0) { + return 0; + } else { + return 1; + } + +error: + return -1; +} + +//@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}] +static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_tags, int n_accept_tag) +{ + int matched = 0; + + int n_tag = cJSON_GetArraySize(tag_set); + for (int i = 0; i < n_tag; i++) { + cJSON *tag_obj = cJSON_GetArrayItem(tag_set, i); + if (NULL == tag_obj || tag_obj->type != cJSON_Object) { + goto error; + } + + int ret = compare_each_tag(tag_obj, accept_tags, n_accept_tag); + if (ret < 0) { + return -1; + } + + if(1 == ret) { + matched++; + } + } + + if (matched == n_tag) { + return 1; + } else { + return 0; + } + +error: + return -1; +} + +//@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]} +//@return 1 on match, 0 on not match, -1 on error. +static int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag) +{ + int ret = -1; + int n_set = 0; + cJSON *tag_set = NULL; + cJSON *tag_set_array = NULL; + + cJSON *root = cJSON_Parse(value); + if (NULL == root) { + goto next; + } + + tag_set_array = cJSON_GetObjectItem(root, "tag_sets"); + if (NULL == tag_set_array || tag_set_array->type != cJSON_Array) { + goto next; + } + + n_set = cJSON_GetArraySize(tag_set_array); + for (int i = 0; i < n_set; i++) { + tag_set = cJSON_GetArrayItem(tag_set_array, i); + if (NULL == tag_set || tag_set->type != cJSON_Array) { + goto next; + } + + ret = compare_each_tag_set(tag_set, accept_tags, n_tag); + //match or error occurs. + if (ret != 0) { + break; + } + } + +error: + cJSON_Delete(root); + return ret; +} + +int populate_compile_table_item(const char *line, struct table_schema *table_schema, struct table_item *table_item, + const char *table_name, const struct rule_tag *accept_tags, int n_accept_tag, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + struct compile_table_schema *compile_schema = &(table_schema->compile); + struct compile_item *compile_item = &(table_item->compile_item); + + int ret = get_column_pos(line, compile_schema->compile_id_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + compile_item->compile_id = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->service_id_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + compile_item->service_id = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->action_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + compile_item->action = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->do_blacklist_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + compile_item->do_blacklist = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->do_log_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + compile_item->do_log = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->tags_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + + char tag_str[MAX_TABLE_LINE_SIZE] = {0}; + memcpy(tag_str, (line + column_offset), column_len); + + if (n_accept_tag > 0 && strlen(tag_str) > 2) { + str_unescape(tag_str); + ret = compare_accept_tag(tag_str, accept_tags, n_accept_tag); + if (ret < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error, invalid tag format for compile table[%s]:compile_id[%d]", + table_name, compile_item->compile_id); + return -1; + } + + if (0 == ret) { + table_schema->unmatched_tag_cnt++; + return -1; + } + } + + ret = get_column_pos(line, compile_schema->user_region_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + + if (column_len > MAX_TABLE_LINE_SIZE) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: compile table[%s]:compile_id[%d] user_region length too long", + table_name, compile_item->compile_id); + return -1; + } + + memcpy(compile_item->user_region, (line + column_offset), column_len); + + switch (compile_schema->user_region_encoding) { + case USER_REGION_ENCODE_ESCAPE: + str_unescape(compile_item->user_region); + break; + default: + break; + } + + ret = get_column_pos(line, compile_schema->is_valid_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + compile_item->is_valid = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->clause_num_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + compile_item->clause_num = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->evaluation_order_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + compile_item->evaluation_order = atoi(line + column_offset); + + return 0; +} + +int populate_group2compile_table_item(const char *line, struct group2compile_table_schema *group2compile_schema, + struct table_item *table_item, const char *table_name, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + struct group2compile_item *group2compile_item = &(table_item->group2compile_item); + + int ret = get_column_pos(line, group2compile_schema->group_id_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + group2compile_item->group_id = atoi(line + column_offset); + + ret = get_column_pos(line, group2compile_schema->compile_id_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + group2compile_item->compile_id = atoi(line + column_offset); + + ret = get_column_pos(line, group2compile_schema->is_valid_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + group2compile_item->is_valid = atoi(line + column_offset); + + ret = get_column_pos(line, group2compile_schema->not_flag_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + group2compile_item->not_flag = atoi(line + column_offset); + + ret = get_column_pos(line, group2compile_schema->virtual_table_name_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + + if (column_len > NAME_MAX) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: group2compile table[%s]:group_id[%d] virtual table name length too long", + table_name, group2compile_item->group_id); + return -1; + } + memcpy(group2compile_item->virtual_table_name, (line + column_offset), column_len); + + ret = get_column_pos(line, group2compile_schema->clause_index_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + + group2compile_item->clause_index = atoi(line + column_offset); + + return 0; +} + +int populate_group2group_table_item(const char *line, struct group2group_table_schema *group2group_schema, + struct table_item *table_item, const char *table_name, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + struct group2group_item *group2group_item = &(table_item->group2group_item); + + int ret = get_column_pos(line, group2group_schema->group_id_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + group2group_item->group_id = atoi(line + column_offset); + + ret = get_column_pos(line, group2group_schema->superior_group_id_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + group2group_item->superior_group_id = atoi(line + column_offset); + + ret = get_column_pos(line, group2group_schema->is_valid_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + group2group_item->is_valid = atoi(line + column_offset); + + return 0; +} + int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, struct table_item *table_item, const char *table_name, struct log_handle *logger) @@ -1540,68 +2131,87 @@ int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schem return 0; } -struct table_item * -table_schema_line_to_item(const char *line, struct table_schema *table_schema, struct log_handle *logger) +void table_item_free(struct table_item *table_item) +{ + if (NULL == table_item) { + return; + } + + switch (table_item->table_type) { + case TABLE_TYPE_COMPILE: + case TABLE_TYPE_GROUP2COMPILE: + case TABLE_TYPE_GROUP2GROUP: + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + case TABLE_TYPE_IP_PLUS: + case TABLE_TYPE_PLUGIN: + break; + case TABLE_TYPE_IP_PLUGIN: + /* TODO: free ex_data */ + break; + default: + break; + } + + FREE(table_item); +} + +struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema, + struct rule_tag *accept_tags, int n_accept_tag, struct log_handle *logger) { if (NULL == line || NULL == table_schema) { return NULL; } int ret = -1; + const char *table_name = table_schema_get_updating_name(table_schema); struct table_item *table_item = ALLOC(struct table_item, 1); switch (table_schema->table_type) { + case TABLE_TYPE_COMPILE: + table_item->table_type = table_schema->table_type; + ret = populate_compile_table_item(line, table_schema, table_item, table_name, accept_tags, n_accept_tag, logger); + break; + case TABLE_TYPE_GROUP2COMPILE: + table_item->table_type = table_schema->table_type; + ret = populate_group2compile_table_item(line, &table_schema->group2compile, table_item, table_name, logger); + break; + case TABLE_TYPE_GROUP2GROUP: + table_item->table_type = table_schema->table_type; + ret = populate_group2group_table_item(line, &table_schema->group2group, table_item, table_name, logger); + break; case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: table_item->table_type = table_schema->table_type; - ret = populate_expr_table_item(line, &table_schema->expr, table_item, - table_schema->table_name[table_schema->updating_name], logger); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "abandon config: invalid format of expr table %s:%s", - table_schema->table_name[table_schema->updating_name], line); - goto error; - } + ret = populate_expr_table_item(line, &table_schema->expr, table_item, table_name, logger); break; case TABLE_TYPE_IP_PLUS: table_item->table_type = table_schema->table_type; - ret = populate_ip_plus_table_item(line, &table_schema->ip_plus, table_item, - table_schema->table_name[table_schema->updating_name], logger); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "abandon config: invalid format of ip_plus table %s:%s", - table_schema->table_name[table_schema->updating_name], line); - goto error; - } + ret = populate_ip_plus_table_item(line, &table_schema->ip_plus, table_item, table_name, logger); break; case TABLE_TYPE_PLUGIN: table_item->table_type = TABLE_TYPE_PLUGIN; - ret = populate_plugin_table_item(line, &table_schema->plugin, table_item, - table_schema->table_name[table_schema->updating_name], logger); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "abandon config: invalid format of plugin table %s:%s", - table_schema->table_name[table_schema->updating_name], line); - goto error; - } + ret = populate_plugin_table_item(line, &table_schema->plugin, table_item, table_name, logger); break; case TABLE_TYPE_IP_PLUGIN: table_item->table_type = TABLE_TYPE_IP_PLUGIN; - ret = populate_ip_plugin_table_item(line, &table_schema->ip_plugin, table_item, - table_schema->table_name[table_schema->updating_name], logger); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "abandon config: invalid format of ip_plugin table %s:%s", - table_schema->table_name[table_schema->updating_name], line); - goto error; - } + ret = populate_ip_plugin_table_item(line, &table_schema->ip_plugin, table_item, table_name, logger); break; default: break; } + + if (ret < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "abandon config: invalid format of table %s:%s, transform line to item failed. ", + table_name, line); + table_schema->update_err_cnt++; + goto error; + } return table_item; + error: - FREE(table_item); + table_item_free(table_item); return NULL; } diff --git a/src/rcu_hash.cpp b/src/rcu_hash.cpp index a907071..15439a2 100644 --- a/src/rcu_hash.cpp +++ b/src/rcu_hash.cpp @@ -132,6 +132,10 @@ void rcu_hash_free(struct rcu_hash_table *htable) } rcu_hash_garbage_queue_free(&(htable->garbage_q)); + if (htable->user_ctx != NULL) { + FREE(htable->user_ctx); + } + pthread_mutex_destroy(&htable->update_mutex); FREE(htable); diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 942e979..6105e9b 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -365,12 +365,6 @@ int main(int argc, char ** argv) { int ret=0; ::testing::InitGoogleTest(&argc, argv); -/* - struct maat_options *opts = maat_options_new(); - maat_options_set_json_file(opts, json_path); - - g_maat_instance = maat_new(opts, table_info_path); - EXPECT_NE(g_maat_instance, nullptr);*/ char json_iris_path[128] = {0}; char redis_ip[64] = "127.0.0.1"; @@ -426,8 +420,7 @@ int main(int argc, char ** argv) redisFree(c); struct maat_options *opts = maat_options_new(); - maat_options_set_redis_ip(opts, redis_ip); - maat_options_set_redis_port(opts, redis_port); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, logger); g_maat_instance = maat_new(opts, table_info_path); diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 29d0e05..d37a7db 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -67,8 +67,7 @@ TEST(iris_mode, maat_scan_string) { snprintf(tmp_iris_inc_idx_path, sizeof(tmp_iris_inc_idx_path), "%s/index", json_iris_path); struct maat_options *opts = maat_options_new(); - maat_options_set_iris_full_index_dir(opts, tmp_iris_full_idx_path); - maat_options_set_iris_inc_index_dir(opts, tmp_iris_inc_idx_path); + maat_options_set_iris(opts, tmp_iris_full_idx_path, tmp_iris_inc_idx_path); struct maat *maat_instance = maat_new(opts, table_info_path); EXPECT_NE(maat_instance, nullptr); @@ -187,8 +186,7 @@ TEST(redis_mode, maat_scan_string) { redisFree(c); struct maat_options *opts = maat_options_new(); - maat_options_set_redis_ip(opts, redis_ip); - maat_options_set_redis_port(opts, redis_port); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, logger); struct maat *maat_instance = maat_new(opts, table_info_path); diff --git a/test/table_info.conf b/test/table_info.conf index d613789..0697fc9 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -90,5 +90,77 @@ "dest": "COMPOSITION_IP_DEST", "session": "COMPOSITION_IP_SESSION" } + }, + { + "table_id":9, + "table_name":"COMPILE_1", + "table_type":"compile", + "custom": { + "compile_id":1, + "service_id":2, + "action":3, + "do_blacklist":4, + "do_log":5, + "tags":6, + "user_region":7, + "is_valid":8, + "clause_num":9, + "evaluation_order":10 + } + }, + { + "table_id":10, + "table_name":"GROUP2COMPILE_1", + "table_type":"group2compile", + "associated_compile_table_id":9, + "custom": { + "group_id":1, + "compile_id":2, + "is_valid":3, + "not_flag":4, + "virtual_table_name":5, + "clause_index":6 + } + }, + { + "table_id":11, + "table_name":"COMPILE_2", + "table_type":"compile", + "custom": { + "compile_id":1, + "service_id":2, + "action":3, + "do_blacklist":4, + "do_log":5, + "tags":6, + "user_region":7, + "is_valid":8, + "clause_num":9, + "evaluation_order":10 + } + }, + { + "table_id":12, + "table_name":"GROUP2COMPILE_2", + "table_type":"group2compile", + "associated_compile_table_id":11, + "custom": { + "group_id":1, + "compile_id":2, + "is_valid":3, + "not_flag":4, + "virtual_table_name":5, + "clause_index":6 + } + }, + { + "table_id":13, + "table_name":"GROUP2GROUP", + "table_type":"group2group", + "custom": { + "group_id":1, + "superior_group_id":2, + "is_valid":3 + } } ] \ No newline at end of file