diff --git a/include/maat/maat.h b/include/maat/maat.h index 9b4894d..d42bc7f 100644 --- a/include/maat/maat.h +++ b/include/maat/maat.h @@ -27,15 +27,19 @@ extern "C" /* maat instance handle */ struct maat; -struct maat_rule { - int rule_id; -}; - enum ip_type { IP_TYPE_V4, IP_TYPE_V6 }; +enum maat_scan_opt +{ + MAAT_SET_SCAN_DISTRICT = 1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default. + MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination. + MAAT_GET_SCAN_HIT_PATH //VALUE is struct Maat_hit_path_t*, an array of struct Maat_hit_path_t, SIZE= sizeof(struct Maat_hit_path_t)*N, + //Maat_get_scan_status returns actual got number. +}; + /* network order */ struct ipv4_4tuple { uint32_t sip; @@ -70,6 +74,12 @@ typedef void maat_plugin_ex_new_func_t(int table_id, const char *key, const char typedef void maat_plugin_ex_free_func_t(int table_id, void **ad, long argl, void *argp); typedef void maat_plugin_ex_dup_func_t(int table_id, void **to, void **from, long argl, void *argp); +typedef void maat_rule_ex_new_func_t(int idx, const struct maat_rule *rule, const char *srv_def_large, + void *ex_data, long argl, void *argp); +typedef void maat_rule_ex_free_func_t(int idx, const struct maat_rule *rule, const char *srv_def_large, + void *ex_data, long argl, void *argp); +typedef void maat_rule_ex_dup_func_t(int idx, void *to, void *from, long argl, void *argp); + /* maat_instance options API */ struct maat_options; struct maat_options* maat_options_new(void); @@ -112,25 +122,30 @@ void *maat_plugin_table_dup_ex_data(struct maat *instance, int table_id, struct maat_state; int maat_scan_integer(struct maat *instance, int table_id, int thread_id, unsigned int intval, int results[], size_t *n_result, - struct maat_state *state); + struct maat_state **state); int maat_scan_ip(struct maat *instance, int table_id, int thread_id, struct addr_4tuple *addr, int results[], size_t *n_result, - struct maat_state *state); + struct maat_state **state); int maat_scan_string(struct maat *instance, int table_id, int thread_id, const char *data, size_t data_len, int results[], size_t *n_result, - struct maat_state *state); + struct maat_state **state); struct maat_stream; struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id); int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* data, int data_len, - int results[], size_t *n_result, struct maat_state *state); + int results[], size_t *n_result, struct maat_state **state); void maat_scan_stream_close(struct maat_stream **stream); -void maat_state_reset(struct maat_state *state); +int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, const void *value, int size); + +//return >=0 if success, return -1 when failed; +int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, void *value, int size); + +void maat_state_reset(struct maat_state **state); #ifdef __cpluscplus } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index db6ad50..186b72e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,7 +10,7 @@ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp - maat_command.cpp maat_redis_monitor.cpp) + maat_command.cpp maat_redis_monitor.cpp maat_hierarchy.cpp) set(LIB_SOURCE_FILES ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c) @@ -26,7 +26,7 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe) set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) -target_link_libraries(maat_frame_static adapter-static hiredis-static pthread crypto z) +target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z) # Shared Library Output #add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) diff --git a/src/inc_internal/alignment.h b/src/inc_internal/alignment.h new file mode 100644 index 0000000..61ad4fb --- /dev/null +++ b/src/inc_internal/alignment.h @@ -0,0 +1,79 @@ +/* +********************************************************************************************** +* File: alignment.h +* Description: alignment statistics +* Authors: Zheng Chao +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _ALIGNMENT_H_ +#define _ALIGNMENT_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include +#include + +#define CPU_CACHE_ALIGMENT 64 + +inline long long *alignment_int64_array_alloc(int size) +{ + return (long long *)calloc(CPU_CACHE_ALIGMENT, size); +} + +inline long long alignment_int64_array_sum(long long *array, int size) +{ + int offset = 0; + long long sum = 0; + + for (int i = 0; i < size; i++) { + offset = (CPU_CACHE_ALIGMENT / sizeof(long long)) * i; + sum += array[offset]; + } + + return sum; +} + +inline long long alignment_int64_array_add(long long *array, int offset, long long op_val) +{ + int idx = (CPU_CACHE_ALIGMENT / sizeof(long long)) * offset; + array[idx] += op_val; + + return array[idx]; +} + +inline long long alignment_int64_array_cnt(long long *array, int size) +{ + int offset = 0; + int cnt = 0; + + for (int i = 0; i < size; i++) { + offset = (CPU_CACHE_ALIGMENT / sizeof(long long)) * i; + if (array[offset] > 0) { + cnt++; + } + } + + return cnt; +} + +inline void alignment_int64_array_reset(long long *array, int size) +{ + memset(array, 0, CPU_CACHE_ALIGMENT * size); +} + +inline void alignment_int64_array_free(long long *array) +{ + free(array); +} + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index ff73ef4..f383b83 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -18,9 +18,8 @@ extern "C" #include "hiredis/hiredis.h" -int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, - const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, - char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, +int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx, + char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, struct log_handle *logger); #ifdef __cpluscplus diff --git a/src/inc_internal/maat_command.h b/src/inc_internal/maat_command.h index 66a0636..8bbfa41 100644 --- a/src/inc_internal/maat_command.h +++ b/src/inc_internal/maat_command.h @@ -30,6 +30,13 @@ struct maat_cmd_line int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout. }; +/** + * @brief write one line to redis + * + * @retval + * success: number of successfully updated rule. + * failed: -1 +*/ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule); #ifdef __cpluscplus diff --git a/src/inc_internal/maat_hierarchy.h b/src/inc_internal/maat_hierarchy.h new file mode 100644 index 0000000..99b0980 --- /dev/null +++ b/src/inc_internal/maat_hierarchy.h @@ -0,0 +1,70 @@ +/* +********************************************************************************************** +* File: maat_hierarchy.h +* Description: +* Authors: Zheng Chao +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_HIERARCHY_H_ +#define _MAAT_HIERARCHY_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include "maat_garbage_collection.h" + +/* maat hierarchy API */ +struct maat_hierarchy; +struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger); +void maat_hierarchy_free(struct maat_hierarchy *hier); +int maat_hierarchy_rebuild(struct maat_hierarchy *hier); +size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, + struct maat_hit_path_t *hit_paths, size_t n_path); + +void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)); +void maat_hierarchy_set_region_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)); + +/* maat hierarchy compile mid API */ +struct maat_hierarchy_compile_mid; +struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id); +void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid); +void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, + int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result); +int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid); + +/* maat hierarchy compile API */ +int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, int declared_clause_num, void* user_data); +int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id); + +void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id); +void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id); +void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param); + +int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, + int is_last_compile, void **user_data_array, size_t ud_array_sz); +void *maat_hierarchy_region_dettach_user_data(struct maat_hierarchy *hier, int region_id); + +/* maat hierarchy region2group API */ +int maat_hierarchy_add_region_to_group(struct maat_hierarchy *hier, int group_id, int region_id, int table_id, void* user_data); +int maat_hierarchy_remove_region_from_group(struct maat_hierarchy *hier, int group_id, int region_id); + +/* maat hierarchy group2group API */ +int maat_hierarchy_add_group_to_group(struct maat_hierarchy *hier, int group_id, int superior_group_id); +int maat_hierarchy_remove_group_from_group(struct maat_hierarchy *hier, int group_id, int superior_group_id); + +/* maat hierarchy group2compile API */ +int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag, + int clause_index, int compile_id); +int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag, + int clause_index, int compile_id); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 0c62dba..422214f 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -29,6 +29,57 @@ extern "C" #include "uthash/uthash.h" #include "maat_table_schema.h" #include "maat_command.h" +#include "IPMatcher.h" + +struct maat_rule_head { + int config_id; + int service_id; + char do_log; + char do_blacklist; + char action; + char resevered; + int serv_def_len; +}; + +#define MAX_SERVICE_DEFINE_LEN 128 +struct maat_rule { + int config_id; + int service_id; + uint8_t do_log; + uint8_t do_blacklist; + uint8_t action; + uint8_t reserved; + int serv_def_len; + char service_defined[MAX_SERVICE_DEFINE_LEN]; +}; + +#define REGION_RULE_MAGIC 0x4d3c2b1a +struct maat_region_inner +{ + long long magic_num; + int region_id; + int group_id; + int district_id; + int table_id; + int expr_id_cnt; + int expr_id_lb; //low boundary + int expr_id_ub; //up boundary +}; + +#define COMPILE_RULE_MAGIC 0x1a2b3c4d +struct maat_compile_rule +{ + long long magic_num; + struct maat_rule_head head;// fix len of Maat_rule_t + char *service_defined; + int is_valid; + int declared_clause_num; + double evaluation_order; + struct table_schema *ref_table; + void *ex_data; + int compile_id; + pthread_rwlock_t rwlock; +}; struct maat_runtime { /* maat_runtime can be created and destroy dynamic, so need version info */ @@ -36,11 +87,18 @@ struct maat_runtime { time_t last_update_time; + long long *ref_cnt; struct table_runtime_manager *table_rt_mgr; size_t max_table_num; size_t max_thread_num; uint32_t rule_num; + + struct maat_hierarchy *hier; + struct maat_garbage_bin *ref_garbage_bin; + + struct scan_result *region_result_buff; + struct log_handle *logger; }; enum data_source { @@ -148,6 +206,12 @@ struct maat { /* statistics */ long long line_cmd_acc_num; + + long long *outer_mid_cnt; + long long *compile_mid_cnt; + long long *thread_call_cnt; + + long long scan_err_cnt; }; void maat_start_cb(long long new_version, int update_type, void *u_para); diff --git a/src/inc_internal/maat_table_runtime.h b/src/inc_internal/maat_table_runtime.h index 56a67a4..8101d0b 100644 --- a/src/inc_internal/maat_table_runtime.h +++ b/src/inc_internal/maat_table_runtime.h @@ -57,7 +57,7 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t */ int table_runtime_updating_flag(struct table_runtime *table_rt); -void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread); +void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, struct log_handle *logger); /* table runtime scan API */ int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len, @@ -67,7 +67,7 @@ void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id); int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result); void table_runtime_stream_close(struct table_runtime *table_rt); -int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, int results[], size_t *n_result); +int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, struct scan_result *results, size_t *n_result, size_t n_result_array); /* table runtime cached row API */ size_t table_runtime_cached_row_count(struct table_runtime *table_rt); @@ -78,7 +78,7 @@ const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt); void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, - struct log_handle *logger); + int nr_worker_num, struct log_handle *logger); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_table_schema.h b/src/inc_internal/maat_table_schema.h index a27e494..f1839ab 100644 --- a/src/inc_internal/maat_table_schema.h +++ b/src/inc_internal/maat_table_schema.h @@ -27,17 +27,18 @@ extern "C" #define MAX_KEYWORDS_STR 1024 #define MAX_FOREIGN_CLMN_NUM 8 -enum table_composition_type { - COMPOSITION_TYPE_SIP = 0, - COMPOSITION_TYPE_DIP, - COMPOSITION_TYPE_SESSION, - COMPOSITION_TYPE_MAX +enum component_table_type { + COMPONENT_TABLE_TYPE_NONE = -1, + COMPONENT_TABLE_TYPE_SIP = 0, + COMPONENT_TABLE_TYPE_DIP, + COMPONENT_TABLE_TYPE_SESSION, + COMPONENT_TABLE_TYPE_MAX }; enum table_type { + TABLE_TYPE_INVALID = -1, TABLE_TYPE_EXPR = 0, TABLE_TYPE_EXPR_PLUS, - TABLE_TYPE_IP, TABLE_TYPE_IP_PLUS, TABLE_TYPE_INTERVAL, TABLE_TYPE_INTERVAL_PLUS, @@ -53,8 +54,7 @@ enum table_type { TABLE_TYPE_COMPILE, TABLE_TYPE_GROUP, TABLE_TYPE_GROUP2GROUP, - TABLE_TYPE_GROUP2COMPILE, - TABLE_TYPE_MAX + TABLE_TYPE_GROUP2COMPILE }; enum expr_type { @@ -155,10 +155,10 @@ struct table_item { struct plugin_table_callback_schema { - maat_start_callback_t *start; - maat_update_callback_t *update; - maat_finish_callback_t *finish; - void* u_para; + maat_start_callback_t *start; + maat_update_callback_t *update; + maat_finish_callback_t *finish; + void *u_para; }; struct ex_data_schema @@ -172,6 +172,16 @@ struct ex_data_schema int set_flag; }; +struct compile_ex_data_schema { + maat_rule_ex_new_func_t *new_func; + maat_rule_ex_free_func_t *free_func; + maat_rule_ex_dup_func_t *dup_func; + long argl; + void *argp; + int idx; + int table_id; +}; + struct table_schema; struct table_schema_manager; @@ -185,7 +195,7 @@ int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_ * @brief get composition table's child table(specified by type) id */ int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id, - enum table_composition_type type); + enum component_table_type type); enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id); @@ -216,23 +226,33 @@ void table_schema_set_updating_name(struct table_schema *table_schema, const cha /* expr table schema API */ enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema); -/* plugin table schema API */ -int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema, - maat_plugin_ex_new_func_t *new_func, - maat_plugin_ex_free_func_t *free_func, - maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp, - struct log_handle *logger); +/* table schema ex API */ +int table_schema_set_ex_data_schema(struct table_schema *table_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger); -struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schema *table_schema); +struct ex_data_schema *table_schema_get_ex_data_schema(struct table_schema *table_schema); + +/* table schema compile rule ex API */ +int table_schema_set_compile_rule_ex_data_schema(struct table_schema *table_schema, + maat_rule_ex_new_func_t *new_func, + maat_rule_ex_free_func_t *free_func, + maat_rule_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger); +struct compile_ex_data_schema *table_schema_get_compile_rule_ex_data_schema(struct table_schema *table_schema, size_t idx); +size_t table_schema_compile_rule_ex_data_schema_count(struct table_schema *table_schema); /** * @brief if plugin table schema's ex data schema set * * @retval 1(already Set) 0(Not set yet) */ -int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema); +int table_schema_ex_data_schema_flag(struct table_schema *table_schema); -int plugin_table_schema_add_callback(struct table_schema_manager *table_schema_mgr, int table_id, +int table_schema_add_callback(struct table_schema_manager *table_schema_mgr, int table_id, maat_start_callback_t *start, maat_update_callback_t *update, maat_finish_callback_t *finish, @@ -240,11 +260,11 @@ int plugin_table_schema_add_callback(struct table_schema_manager *table_schema_m /** * @brief the number of callback function stored in plugin table schema */ -size_t plugin_table_schema_callback_count(struct table_schema *table_schema); +size_t table_schema_callback_count(struct table_schema *table_schema); -void plugin_table_schema_all_cb_update(struct table_schema *table_schema, const char *row); +void table_schema_all_cb_update(struct table_schema *table_schema, const char *row); -int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns); +int table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_utils.h b/src/inc_internal/maat_utils.h index 4f99bc8..9542d35 100644 --- a/src/inc_internal/maat_utils.h +++ b/src/inc_internal/maat_utils.h @@ -31,6 +31,8 @@ extern "C" #define MIN(a, b) (((a) < (b)) ? (a) : (b)) #endif +#define MAX_SCANNER_HIT_NUM 4096 + enum maat_ip_format { IP_FORMAT_RANGE, IP_FORMAT_MASK, diff --git a/src/json2iris.cpp b/src/json2iris.cpp index 475bf74..890bdd1 100644 --- a/src/json2iris.cpp +++ b/src/json2iris.cpp @@ -130,7 +130,6 @@ int set_iris_descriptor(const char *json_file, cJSON *json, const char *encrypt_ maat_kv_register(iris_cfg->str2int_map, "yes", 1); maat_kv_register(iris_cfg->str2int_map, "no", 0); - maat_kv_register(iris_cfg->str2int_map, "ip", TABLE_TYPE_IP); maat_kv_register(iris_cfg->str2int_map, "ip_plus", TABLE_TYPE_IP_PLUS); maat_kv_register(iris_cfg->str2int_map, "string", TABLE_TYPE_EXPR); maat_kv_register(iris_cfg->str2int_map, "expr", TABLE_TYPE_EXPR); @@ -433,94 +432,6 @@ int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_ip_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, - struct log_handle *logger) -{ - struct translate_command json_cmd[MAX_COLUMN_NUM]; - int cmd_cnt = 0; - memset(json_cmd, 0, sizeof(json_cmd)); - - json_cmd[cmd_cnt].json_string = "region_id"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "group_id"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "addr_type"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].str2int_flag = 1; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "src_ip"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_string = "0.0.0.0"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "mask_src_ip"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_string = "255.255.255.255"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "src_port"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_string = "0"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "mask_src_port"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_string = "65535"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "dst_ip"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_string = "0.0.0.0"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "mask_dst_ip"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_string = "255.255.255.255"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "dst_port"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_string = "0"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "mask_dst_port"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_string = "65535"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "protocol"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_int = 0; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "direction"; - json_cmd[cmd_cnt].json_type = cJSON_String; - json_cmd[cmd_cnt].str2int_flag = 1; - json_cmd[cmd_cnt].empty_allowed = 1; - json_cmd[cmd_cnt].default_string = "double"; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "is_valid"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); -} - int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, struct log_handle *logger) { @@ -779,9 +690,6 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i case TABLE_TYPE_EXPR_PLUS: ret = write_expr_line(table_content, p_iris, table_info, logger); break; - case TABLE_TYPE_IP: - ret = write_ip_line(table_content, p_iris, table_info, logger); - break; case TABLE_TYPE_IP_PLUS: ret = write_ip_plus_line(table_content, p_iris, table_info, logger); break; @@ -1163,13 +1071,15 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle * return 0; } -int json2iris(const char *json_buff, const char *json_filename, const char *compile_tn, - const char *group2compile_tn, const char *group2group_tn, redisContext *redis_write_ctx, +int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx, char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, struct log_handle *logger) { int ret = -1; cJSON *tmp_obj = NULL; + const char *compile_tbl_name = NULL; + const char *group2compile_tbl_name = NULL; + const char *group2group_tbl_name = NULL; struct iris_description iris_cfg; memset(&iris_cfg, 0, sizeof(iris_cfg)); @@ -1182,21 +1092,21 @@ int json2iris(const char *json_buff, const char *json_filename, const char *comp tmp_obj = cJSON_GetObjectItem(json, "compile_table"); if (tmp_obj) { - compile_tn = tmp_obj->valuestring; + compile_tbl_name = tmp_obj->valuestring; } tmp_obj = cJSON_GetObjectItem(json, "group2compile_table"); if (tmp_obj) { - group2compile_tn = tmp_obj->valuestring; + group2compile_tbl_name = tmp_obj->valuestring; } tmp_obj = cJSON_GetObjectItem(json, "group2group_table"); if (tmp_obj) { - group2group_tn = tmp_obj->valuestring; + group2group_tbl_name = tmp_obj->valuestring; } ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, - compile_tn, group2compile_tn, group2group_tn, + compile_tbl_name, group2compile_tbl_name, group2group_tbl_name, redis_write_ctx, &iris_cfg); if (ret < 0) { goto error_out; diff --git a/src/maat_api.cpp b/src/maat_api.cpp index 54c4e74..10bd428 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -25,9 +25,25 @@ #include "maat_table_runtime.h" #include "maat_config_monitor.h" #include "maat_redis_monitor.h" +#include "maat_hierarchy.h" +#include "alignment.h" #define MODULE_MAAT_API module_name_str("maat.api") +#define DISTRICT_ANY -1 +#define DISTRICT_UNKNOWN -2 + +struct maat_state +{ + struct maat *maat_instance; + int16_t thread_id; + unsigned char is_set_district; + unsigned char is_last_scan; + int district_id; //-1: Any District; -2: Unkonwn District; + int scan_cnt; + struct maat_hierarchy_compile_mid *compile_mid; +}; + struct maat_options* maat_options_new(void) { struct maat_options *options = ALLOC(struct maat_options, 1); @@ -267,6 +283,11 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) + (maat_instance->gc_timeout_ms / 1000); maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s); + + maat_instance->outer_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); + maat_instance->compile_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); + maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); + pthread_mutex_init(&(maat_instance->background_update_mutex), NULL); if (0 == maat_instance->deferred_load) { @@ -303,6 +324,26 @@ int maat_table_get_id(struct maat *maat_instance, const char *table_name) return table_id; } +inline void maat_runtime_ref_inc(struct maat_runtime *maat_rt, int thread_id) +{ + alignment_int64_array_add(maat_rt->ref_cnt, thread_id, 1); +} + +inline void maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id) +{ + alignment_int64_array_add(maat_rt->ref_cnt, thread_id, -1); +} + +inline int scan_state_should_compile_NOT(struct maat_state *mid) +{ + if (mid && (1 == mid->is_last_scan) && mid->compile_mid && + maat_hierarchy_compile_mid_has_NOT_clause(mid->compile_mid)) { + return 1; + } else { + return 0; + } +} + int maat_table_callback_register(struct maat *maat_instance, int table_id, maat_start_callback_t *start, maat_update_callback_t *update, @@ -312,7 +353,7 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id, int ret = -1; pthread_mutex_lock(&(maat_instance->background_update_mutex)); - ret = plugin_table_schema_add_callback(maat_instance->table_schema_mgr, table_id, + ret = table_schema_add_callback(maat_instance->table_schema_mgr, table_id, start, update, finish, u_para, maat_instance->logger); if (ret < 0) { pthread_mutex_unlock(&(maat_instance->background_update_mutex)); @@ -358,7 +399,7 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i { struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); pthread_mutex_lock(&(maat_instance->background_update_mutex)); - int ret = plugin_table_schema_set_ex_data_schema(table_schema, new_func, free_func, dup_func, + int ret = table_schema_set_ex_data_schema(table_schema, new_func, free_func, dup_func, argl, argp, maat_instance->logger); if (ret < 0) { pthread_mutex_unlock(&(maat_instance->background_update_mutex)); @@ -368,7 +409,8 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i struct table_runtime *table_rt = NULL; if (maat_instance->maat_rt != NULL) { table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id); - table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->logger); + table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->nr_worker_thread, + maat_instance->logger); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); @@ -401,24 +443,68 @@ void *maat_plugin_table_dup_ex_data(struct maat *maat_instance, int table_id, return ex_data_runtime_dup_ex_data(ex_data_rt, key, key_len); } +static void scan_count_inc(struct maat_state *mid) +{ + mid->scan_cnt++; +} + +struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id) +{ + struct maat_state *outer_state = NULL; + + outer_state = ALLOC(struct maat_state, 1); + outer_state->maat_instance = maat_instance; + outer_state->district_id = DISTRICT_ANY; + outer_state->thread_id = (signed short)thread_id; + + return outer_state; +} + +struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instance, int thread_id, int is_hit_region) +{ + struct maat_state *mid = *state; + + if (NULL == mid) { + mid = make_outer_state(maat_instance, thread_id); + *state = mid; + //Maat_set_scan_status calls grap_mid() with thread_num=-1. + if (mid->thread_id >= 0) { + alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1); + } + } + + if (mid->thread_id < 0 && thread_id >= 0) { + mid->thread_id = thread_id; + alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1); + } + + if (is_hit_region == 1) { + if (NULL == mid->compile_mid) { + mid->compile_mid = maat_hierarchy_compile_mid_new(maat_instance->maat_rt->hier, thread_id); + alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1); + } + } + return mid; +} + int maat_scan_integer(struct maat *instance, int table_id, int thread_id, - unsigned int intval, int results[], size_t n_result, - struct maat_state *state) + unsigned int intval, int results[], size_t *n_result, + struct maat_state **state) { return 0; } -static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4tuple *addr, enum table_composition_type child_type) +static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4tuple *addr, enum component_table_type child_type) { switch (addr->type) { case IP_TYPE_V4: table_rt_addr->ip_type = IP_TYPE_V4; switch (child_type) { - case COMPOSITION_TYPE_SIP: + case COMPONENT_TABLE_TYPE_SIP: table_rt_addr->ipv4 = ntohl(addr->ipv4.sip); table_rt_addr->port = ntohs(addr->ipv4.sport); break; - case COMPOSITION_TYPE_DIP: + case COMPONENT_TABLE_TYPE_DIP: table_rt_addr->ipv4 = ntohl(addr->ipv4.dip); table_rt_addr->port = ntohs(addr->ipv4.dport); break; @@ -430,12 +516,12 @@ static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4 case IP_TYPE_V6: table_rt_addr->ip_type = IP_TYPE_V6; switch (child_type) { - case COMPOSITION_TYPE_SIP: + case COMPONENT_TABLE_TYPE_SIP: memcpy(table_rt_addr->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip)); ipv6_ntoh(table_rt_addr->ipv6); table_rt_addr->port = ntohs(addr->ipv6.sport); break; - case COMPOSITION_TYPE_DIP: + case COMPONENT_TABLE_TYPE_DIP: memcpy(table_rt_addr->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip)); ipv6_ntoh(table_rt_addr->ipv6); table_rt_addr->port = ntohs(addr->ipv6.dport); @@ -454,16 +540,17 @@ static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4 } static int ip_composition_scan(int thread_id, struct addr_4tuple *addr, - int parent_table_id, enum table_composition_type child_type, + int parent_table_id, enum component_table_type child_type, int *virtual_table_id, struct table_schema_manager *table_schema_mgr, - struct table_runtime_manager *table_rt_mgr, int results[], size_t *n_result) + struct table_runtime_manager *table_rt_mgr, + struct scan_result *region_results, size_t n_result_array) { int child_table_id = 0; - if (child_type == COMPOSITION_TYPE_MAX) { + if (child_type == COMPONENT_TABLE_TYPE_NONE) { child_table_id = parent_table_id; - child_type = COMPOSITION_TYPE_SIP; + child_type = COMPONENT_TABLE_TYPE_SESSION; } else { child_table_id = table_schema_manager_get_child_table_id(table_schema_mgr, parent_table_id, child_type); } @@ -493,51 +580,93 @@ static int ip_composition_scan(int thread_id, struct addr_4tuple *addr, memset(&scan_data, 0, sizeof(struct table_rt_2tuple)); ip_scan_data_set(&scan_data, addr, child_type); - - return table_runtime_scan_ip(table_rt, thread_id, &scan_data, results, n_result); -} - -int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_4tuple *addr, - int results[], size_t *n_result, struct maat_state *state) -{ - if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || - (thread_id < 0) || (NULL == addr) || (NULL == results) || (NULL == n_result)) { + size_t hit_cnt = 0; + int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, region_results, &hit_cnt, n_result_array); + if (ret < 0) { return -1; } - struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr; + return hit_cnt; +} + +int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_4tuple *addr, + int results[], size_t *n_result, struct maat_state **state) +{ + if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || + (thread_id < 0) || (NULL == addr) || (NULL == results) || (NULL == n_result) || + (NULL == state)) { + return -1; + } + + struct maat_state *mid = NULL; + mid = grab_mid(state, maat_instance, thread_id, 0); + scan_count_inc(mid); + + struct maat_runtime *maat_rt = maat_instance->maat_rt; + if (NULL == maat_rt) { + return 0; + } + + struct table_runtime_manager *table_rt_mgr = maat_rt->table_rt_mgr; struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); enum table_type table_type = table_runtime_get_type(table_rt); + if (table_type == TABLE_TYPE_INVALID) { + maat_instance->scan_err_cnt++; + return -1; + } int region_ret = 0; int virtual_table_id = 0; - + struct scan_result *region_result = maat_rt->region_result_buff + thread_id * MAX_SCANNER_HIT_NUM; + int region_hit_cnt = 0; + int region_result_virtual_table_ids[MAX_SCANNER_HIT_NUM]; + alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); + maat_runtime_ref_inc(maat_rt, thread_id); + if (table_type == TABLE_TYPE_COMPOSITION) { - /* - enum table_composition_type childs[3] = {COMPOSITION_TYPE_SIP, COMPOSITION_TYPE_DIP, COMPOSITION_TYPE_SESSION}; + region_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_SIP, &virtual_table_id, + maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, + region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt); + region_hit_cnt += region_ret; + /*enum component_table_type childs[3] = {COMPONENT_TABLE_TYPE_SIP, COMPONENT_TABLE_TYPE_DIP, COMPONENT_TABLE_TYPE_SESSION}; for (int i = 0; i < 3; i++) { - region_ret = ip_composition_scan(table_rt, thread_id, ip, table_id, childs[i], maat_instance->table_schema_mgr, - maat_instance->maat_rt->table_rt_mgr, results, n_result); + region_ret = ip_composition_scan(thread_id, addr, table_id, childs[i], &virtual_table_id, + maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, + region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt); + if (region_ret < 0) { + maat_instance->scan_err_cnt++; + } else { + for (int j = 0; j < region_ret; j++) { + region_result_virtual_table_ids[region_hit_cnt++] = virtual_table_id; + } + } }*/ - region_ret = ip_composition_scan(thread_id, addr, table_id, COMPOSITION_TYPE_SIP, &virtual_table_id, - maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, - results, n_result); } else { - region_ret = ip_composition_scan(thread_id, addr, table_id, COMPOSITION_TYPE_MAX, &virtual_table_id, + region_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_NONE, &virtual_table_id, maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, - results, n_result); + region_result + region_hit_cnt, MAX_SCANNER_HIT_NUM - region_hit_cnt); + if (region_ret < 0) { + maat_instance->scan_err_cnt++; + } else { + region_hit_cnt += region_ret; + } } - return region_ret; + *n_result = region_ret; + for (int i = 0; i < region_ret; i++) { + results[i] = region_result[i].rule_id; + } + + return 0; } int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, const char *data, size_t data_len, int results[], size_t *n_result, - struct maat_state *state) + struct maat_state **state) { if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || (thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) || - (NULL == n_result)) { + (NULL == n_result) || (NULL == state)) { return -1; } @@ -552,8 +681,8 @@ struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, i return NULL; } -int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* data, int data_len, - int results[], size_t n_result, struct maat_state *state) +int maat_scan_stream(struct maat_stream **stream, int thread_id, const char *data, int data_len, + int results[], size_t *n_result, struct maat_state **state) { return 0; } @@ -563,7 +692,20 @@ void maat_scan_stream_close(struct maat_stream **stream) } -void maat_state_reset(struct maat_state *state) +int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, + const void *value, int size) +{ + +} + +//return >=0 if success, return -1 when failed; +int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, + void *value, int size) +{ + +} + +void maat_state_reset(struct maat_state **state) { } \ No newline at end of file diff --git a/src/maat_command.cpp b/src/maat_command.cpp index 6670e85..f8d914f 100644 --- a/src/maat_command.cpp +++ b/src/maat_command.cpp @@ -162,9 +162,6 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, case TABLE_TYPE_EXPR: column_seq = 7; break; - case TABLE_TYPE_IP: - column_seq = 14; - break; case TABLE_TYPE_IP_PLUS: column_seq = 18; break; @@ -332,8 +329,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li } enum table_type table_type = table_schema_get_table_type(table_schema); - int is_valid = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_flag_column); - + int valid_offset = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_flag_column); + int is_valid = atoi(line_rule->table_line+valid_offset); if (line_rule->expire_after > 0) { absolute_expire_time = server_time + line_rule->expire_after; } diff --git a/src/maat_config_monitor.cpp b/src/maat_config_monitor.cpp index ce3bd1a..0a44363 100644 --- a/src/maat_config_monitor.cpp +++ b/src/maat_config_monitor.cpp @@ -409,8 +409,7 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, } } - ret = json2iris((const char*)json_buff, json_filename, maat_instance->compile_tn, - maat_instance->group2compile_tn, maat_instance->group2group_tn, NULL, + ret = json2iris((const char*)json_buff, json_filename, NULL, maat_instance->json_ctx.iris_file, sizeof(maat_instance->json_ctx.iris_file), strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL, diff --git a/src/maat_hierarchy.cpp b/src/maat_hierarchy.cpp new file mode 100644 index 0000000..e547624 --- /dev/null +++ b/src/maat_hierarchy.cpp @@ -0,0 +1,1323 @@ +/* +********************************************************************************************** +* File: maat_hierarchy.cpp +* Description: +* Authors: Zheng Chao +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include +#include + +#include "utils.h" +#include "maat_utils.h" +#include "log/log.h" +#include "uthash/utarray.h" +#include "uthash/uthash.h" +#include "bool_matcher.h" +#include "igraph/igraph.h" +#include "maat_hierarchy.h" +#include "maat_garbage_collection.h" + +#define MODULE_HIERARCHY module_name_str("maat.hierarchy") + +struct maat_hierarchy_clause_state { + unsigned long long clause_id; + char not_flag; + char in_use; + UT_array *literal_ids; +}; + +struct maat_hierarchy_compile { + unsigned int magic; + int compile_id; + int actual_clause_num; + int declared_clause_num; + int not_clause_cnt; + void *user_data; + UT_hash_handle hh; + struct maat_hierarchy_clause_state clause_states[MAX_ITEMS_PER_BOOL_EXPR]; +}; + +struct maat_hierarchy_group { + igraph_integer_t vertex_id; + int group_id; + int ref_by_compile_cnt; + int ref_by_superior_group_cnt; + int ref_by_subordinate_group_cnt; + int ref_by_region_cnt; + + size_t top_group_cnt; + int *top_group_ids; + UT_hash_handle hh_group_id; + UT_hash_handle hh_vertex_id; +}; + +struct maat_hierarchy_region { + int region_id; + int group_id; + int table_id; + + struct maat_hierarchy_group *ref_parent_group; + UT_hash_handle hh; + + void *user_data; +}; + +struct maat_hierarchy_literal_id { + int group_id; + int vt_id; +}; + +struct maat_hierarchy_clause { + long long clause_id; + size_t n_literal_id; + struct maat_hierarchy_literal_id *literal_ids; + UT_hash_handle hh; +}; + +struct region2clause_key { + int region_id; + int vt_id; +}; + +struct group2region { + int group_id; + UT_array *region_ids; + UT_hash_handle hh; //index to +}; + +struct region2clause_value { + struct region2clause_key key; + UT_array *clause_ids; + int group_id; + UT_hash_handle hh; //index to +}; + +struct maat_hierarchy_internal_hit_path { + int Nth_scan; + int Nth_hit_region; + int region_id; + int virtual_table_id; +}; + +struct maat_hierarchy +{ + pthread_rwlock_t rwlock; + pthread_mutex_t mutex; + time_t version; //After full update, clause id may indicate a different clause. Comparing hier->version and mid->hier_ver can prevent false positive match. + int changed_flag; + + struct maat_hierarchy_compile *hash_compile_by_id; //key: compile_id, value: struct Maat_hierarchy_compile*. + void (* compile_user_data_free)(void *compile_ud); + + struct maat_hierarchy_group *hash_group_by_id; //key: group_id, value: struct Maat_hierarchy_group*. + struct maat_hierarchy_group *hash_group_by_vertex; //key:vetex_id, value: struct Maat_hierarchy_group*. Multimap (Items with multiple keys). + + struct maat_hierarchy_region *hash_region_by_id; //key: region_id, value: struct Maat_hierarchy_region*. + + struct maat_hierarchy_clause *hash_dedup_clause_by_literals; //key: literal combination, value: struct Maat_hierarchy_clause*. For generating unique clause_id. + unsigned long long clause_id_generator; //Increasing number. + + void (* region_user_data_free)(void *region_ud); + + + igraph_t group_graph; + igraph_integer_t group_graph_vcount; + igraph_vector_t dfs_vids; + + igraph_integer_t grp_vertex_id_generator; + + /*Following members are accessed from scan threads.*/ + struct region2clause_value *hash_region2clause; //key: region_id+virtual_table_id, value: struct region2clause_value. + struct bool_matcher *bm; + + + int thread_num; + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; + struct bool_expr_match *expr_match_buff; +}; + +struct maat_hierarchy_compile_mid +{ + int thread_id; + int Nth_scan; + time_t hier_ver; + size_t this_scan_region_hit_cnt; + int not_clause_hitted_flag; + int is_no_count_scan; + size_t hit_path_cnt; + + UT_array *internal_hit_paths; + UT_array *all_hit_clause_array; + UT_array *this_scan_hit_clause_ids; +}; + +struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger) +{ + struct maat_hierarchy *hier = ALLOC(struct maat_hierarchy, 1); + UNUSED int ret = 0; + hier->logger = logger; + hier->thread_num = thread_num; + hier->version = time(NULL); + + hier->hash_group_by_id = NULL; + hier->hash_group_by_vertex = NULL; + hier->hash_compile_by_id = NULL; + hier->hash_region2clause = NULL; + hier->hash_region_by_id = NULL; + hier->hash_dedup_clause_by_literals = NULL; + hier->clause_id_generator = 0; + hier->ref_garbage_bin = bin; + hier->expr_match_buff = ALLOC(struct bool_expr_match, thread_num * MAX_SCANNER_HIT_NUM); + + pthread_mutex_init(&hier->mutex, NULL); + ret = pthread_rwlock_init(&hier->rwlock, NULL); + assert(ret == 0); + ret = igraph_empty(&hier->group_graph, 0, IGRAPH_DIRECTED); + assert(ret == IGRAPH_SUCCESS); + + return hier; +} + +UT_icd ut_literal_id_icd = {sizeof(struct maat_hierarchy_literal_id), NULL, NULL, NULL}; +UT_icd ut_clause_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; +UT_icd ut_region_id_icd = {sizeof(int), NULL, NULL, NULL}; +UT_icd ut_hit_path_icd = {sizeof(struct maat_hierarchy_internal_hit_path), NULL, NULL, NULL}; + +#define MAAT_HIER_COMPILE_MAGIC 0x4a5b6c7d +static struct maat_hierarchy_compile *maat_hierarchy_compile_new(struct maat_hierarchy *hier, int compile_id) +{ + struct maat_hierarchy_compile *compile = NULL; + compile = ALLOC(struct maat_hierarchy_compile, 1); + compile->magic = MAAT_HIER_COMPILE_MAGIC; + compile->compile_id = compile_id; + HASH_ADD_INT(hier->hash_compile_by_id, compile_id, compile); + + for(int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + utarray_new(compile->clause_states[i].literal_ids, &ut_literal_id_icd); + compile->clause_states[i].in_use=0; + } + return compile; +} + +static void maat_hierarchy_compile_free(struct maat_hierarchy_compile *compile) +{ + struct maat_hierarchy_clause_state *clause_state = NULL; + //user_data must be freed before calling this function. + assert(compile->user_data == NULL); + + for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states + i; + utarray_free(clause_state->literal_ids); + clause_state->literal_ids = NULL; + clause_state->in_use = 0; + } + compile->magic = 0; + free(compile); +} + +void maat_hierarchy_free_region2clause_hash(struct region2clause_value* hash) +{ + struct region2clause_value *r2c_val = NULL, *tmp_r2c_val = NULL; + HASH_ITER(hh, hash, r2c_val, tmp_r2c_val) { + HASH_DEL(hash, r2c_val); + utarray_free(r2c_val->clause_ids); + free(r2c_val); + } + assert(hash == NULL); +} + +static struct maat_hierarchy_region * +maat_hierarchy_region_new(struct maat_hierarchy *hier, int region_id, int group_id, int table_id, + struct maat_hierarchy_group *parent_group, void *user_data) +{ + struct maat_hierarchy_region *region = NULL; + region = ALLOC(struct maat_hierarchy_region, 1); + region->group_id = group_id; + region->region_id = region_id; + region->table_id = table_id; + region->ref_parent_group = parent_group; + region->user_data = user_data; + HASH_ADD_INT(hier->hash_region_by_id, region_id, region); + parent_group->ref_by_region_cnt++; + + return region; +} + +static void maat_hierarchy_region_free(struct maat_hierarchy *hier, struct maat_hierarchy_region *region) +{ + HASH_DELETE(hh, hier->hash_region_by_id, region); + region->ref_parent_group->ref_by_region_cnt--; + + if (hier->region_user_data_free && region->user_data) { + hier->region_user_data_free(region->user_data); + region->user_data = NULL; + } + free(region); +} + +static const struct maat_hierarchy_clause * +maat_hierarchy_clause_fetch(struct maat_hierarchy *hier, struct maat_hierarchy_literal_id *literal_ids, + size_t n_literal_id) +{ + struct maat_hierarchy_clause *clause = NULL; + + HASH_FIND(hh, hier->hash_dedup_clause_by_literals, literal_ids, + n_literal_id * sizeof(struct maat_hierarchy_literal_id), clause); + if (!clause) { + clause = ALLOC(struct maat_hierarchy_clause, 1); + clause->clause_id = hier->clause_id_generator; + clause->n_literal_id = n_literal_id; + clause->literal_ids = ALLOC(struct maat_hierarchy_literal_id, n_literal_id); + memcpy(clause->literal_ids, literal_ids, n_literal_id * sizeof(struct maat_hierarchy_literal_id)); + + hier->clause_id_generator++; + HASH_ADD_KEYPTR(hh, hier->hash_dedup_clause_by_literals, clause->literal_ids, + n_literal_id * sizeof(struct maat_hierarchy_literal_id), clause); + } + + return clause; +} + +static void maat_hierarchy_clause_free(struct maat_hierarchy *hier, struct maat_hierarchy_clause *clause) +{ + HASH_DELETE(hh, hier->hash_dedup_clause_by_literals, clause); + free(clause->literal_ids); + clause->n_literal_id = 0; + free(clause); +} + +static void group_vertex_free(struct maat_hierarchy_group *group) +{ + free(group->top_group_ids); + free(group); +} + +void maat_hierarchy_free(struct maat_hierarchy *hier) +{ + struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; + struct maat_hierarchy_group *group = NULL, *tmp_group = NULL; + struct region2clause_value *r2c_val = NULL, *tmp_r2c_val = NULL; + struct maat_hierarchy_region *region = NULL, *tmp_region = NULL; + struct maat_hierarchy_clause *clause = NULL, *tmp_clause = NULL; + pthread_rwlock_wrlock(&hier->rwlock); + + //Reference: https://troydhanson.github.io/uthash/userguide.html#_what_can_it_do + //Some have asked how uthash cleans up its internal memory. + //The answer is simple: when you delete the final item from a hash table, + //uthash releases all the internal memory associated with that hash table, + //and sets its pointer to NULL. + HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { + if (hier->compile_user_data_free && compile->user_data) { + hier->compile_user_data_free(compile->user_data); + compile->user_data = NULL; + } + HASH_DEL(hier->hash_compile_by_id, compile); + maat_hierarchy_compile_free(compile); + } + assert(hier->hash_compile_by_id == NULL); + + HASH_ITER(hh, hier->hash_region2clause, r2c_val, tmp_r2c_val) { + HASH_DEL(hier->hash_region2clause, r2c_val); + utarray_free(r2c_val->clause_ids); + free(r2c_val); + } + + maat_hierarchy_free_region2clause_hash(hier->hash_region2clause); + + HASH_ITER(hh, hier->hash_region_by_id, region, tmp_region) { + maat_hierarchy_region_free(hier, region); + } + + HASH_ITER(hh, hier->hash_dedup_clause_by_literals, clause, tmp_clause) { + maat_hierarchy_clause_free(hier, clause); + } + + //Free group as the last. + HASH_CLEAR(hh_vertex_id, hier->hash_group_by_vertex);//No need group memory clean up. + HASH_ITER(hh_group_id, hier->hash_group_by_id, group, tmp_group) { + HASH_DELETE(hh_group_id, hier->hash_group_by_id, group); + group_vertex_free(group); + } + assert(hier->hash_group_by_id == NULL); + + igraph_destroy(&hier->group_graph); + bool_matcher_free(hier->bm); + hier->bm = NULL; + + pthread_rwlock_unlock(&hier->rwlock); + pthread_rwlock_destroy(&hier->rwlock); + + FREE(hier->expr_match_buff); + FREE(hier); +} + +size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) +{ + int printed = 0; + + for (long int i = 0; i < igraph_vector_size(v); i++) { + printed += snprintf(buff + printed, sz - printed, " %li", (long int)VECTOR(*v)[i]); + } + + return printed; +} + +struct maat_hierarchy_group *maat_hierarchy_group_new(struct maat_hierarchy *hier, int group_id) +{ + struct maat_hierarchy_group *group = NULL; + + group = ALLOC(struct maat_hierarchy_group, 1); + group->group_id = group_id; + group->vertex_id = hier->grp_vertex_id_generator++; + assert(igraph_vcount(&hier->group_graph) == group->vertex_id); + igraph_add_vertices(&hier->group_graph, 1, NULL); //Add 1 vertice. + + HASH_ADD(hh_group_id, hier->hash_group_by_id, group_id, sizeof(group->group_id), group); + HASH_ADD(hh_vertex_id, hier->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group); + + return group; +} + +static void maat_hierarchy_group_free(struct maat_hierarchy *hier, struct maat_hierarchy_group *group) +{ + igraph_vector_t v; + char buff[4096] = {0}; + + assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0); + igraph_vector_init(&v, 8); + igraph_neighbors(&hier->group_graph, &v, group->vertex_id, IGRAPH_ALL); + + if (igraph_vector_size(&v) > 0) { + print_igraph_vector(&v, buff, sizeof(buff)); + log_error(hier->logger, MODULE_HIERARCHY, "Del group %d exception, still reached by %s.", + group->vertex_id, buff); + assert(0); + } + igraph_vector_destroy(&v); + assert(group->top_group_ids==NULL); + //We should not call igraph_delete_vertices, because this is function changes the ids of the vertices. + //igraph_delete_vertices(&hier->group_graph, igraph_vss_1(group->vertex_id)); + + HASH_DELETE(hh_group_id, hier->hash_group_by_id, group); + HASH_DELETE(hh_vertex_id, hier->hash_group_by_vertex, group); + + group_vertex_free(group); +} + +static size_t effective_vertices_count(igraph_vector_t *vids) +{ + size_t i = 0; + int tmp_vid = 0; + for (i = 0; i < (size_t)igraph_vector_size(vids); i++) { + tmp_vid = (int) VECTOR(*vids)[i]; + if (tmp_vid < 0) { + break; + } + } + + return i; +} + +static int maat_hierarchy_build_top_groups(struct maat_hierarchy *hier) +{ + struct maat_hierarchy_group *group = NULL, *tmp = NULL; + struct maat_hierarchy_group *superior_group = NULL; + int tmp_vid=0; + size_t i=0, top_group_cnt=0; + int* temp_group_ids=NULL; + + igraph_bool_t is_dag; + igraph_is_dag(&(hier->group_graph), &is_dag); + if (!is_dag) { + log_error(hier->logger, MODULE_HIERARCHY, "Sub group cycle detected!"); + return -1; + } + + hier->group_graph_vcount = igraph_vcount(&hier->group_graph); + igraph_vector_init(&(hier->dfs_vids), hier->group_graph_vcount); + + HASH_ITER (hh_group_id, hier->hash_group_by_id, group, tmp) { + top_group_cnt = 0; + temp_group_ids = NULL; + //Orphan, Not reference by any one, free it. + if (0 == group->ref_by_compile_cnt + && 0 == group->ref_by_superior_group_cnt + && 0 == group->ref_by_subordinate_group_cnt + && 0 == group->ref_by_region_cnt) { + + FREE(group->top_group_ids); + maat_hierarchy_group_free(hier, group); + continue; + } + + //A group is need to build top groups when it has regions and referenced by superior groups or compiles. + if (group->ref_by_region_cnt > 0 && + (group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0)) { + if (0 == group->ref_by_superior_group_cnt) { + //fast path, group is only referenced by compile rules. + top_group_cnt = 1; + temp_group_ids = ALLOC(int, top_group_cnt); + temp_group_ids[0] = group->group_id; + } else { + igraph_vector_t *vids = &(hier->dfs_vids); + igraph_dfs(&hier->group_graph, group->vertex_id, IGRAPH_OUT, + 0, vids, NULL, NULL, NULL, NULL, NULL, NULL); + + temp_group_ids = ALLOC(int, effective_vertices_count(vids)); + + for (size_t i = 0; i < (size_t)igraph_vector_size(vids); i++) { + tmp_vid = (int) VECTOR(*vids)[i]; + if (tmp_vid < 0) { + break; + } + + HASH_FIND(hh_vertex_id, hier->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group); + + //including itself + if (superior_group->ref_by_compile_cnt > 0) { + temp_group_ids[top_group_cnt] = superior_group->group_id; + top_group_cnt++; + } + } + } + } + + free(group->top_group_ids); + group->top_group_cnt = top_group_cnt; + group->top_group_ids = ALLOC(int, group->top_group_cnt); + memcpy(group->top_group_ids, temp_group_ids, sizeof(int)*group->top_group_cnt); + + FREE(temp_group_ids); + } + igraph_vector_destroy(&hier->dfs_vids); + + return 0; +} + +static struct bool_matcher *maat_hierarchy_build_bool_matcher(struct maat_hierarchy *hier) +{ + struct bool_matcher *bm = NULL; + size_t compile_num = 0, expr_cnt = 0; + struct bool_expr *bool_expr_array = NULL; + struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; + + struct maat_hierarchy_clause_state *clause_state = NULL; + const struct maat_hierarchy_clause *clause = NULL; + size_t i = 0, j = 0; + int has_clause_num = 0; + compile_num = HASH_COUNT(hier->hash_compile_by_id); + + if (0 == compile_num) { + log_error(hier->logger, MODULE_HIERARCHY, "No compile to build."); + return NULL; + } + + //STEP 1, update clause_id of each compile and literal + struct maat_hierarchy_literal_id *literal_ids = NULL; + size_t n_literal_id = 0; + HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { + has_clause_num = 0; + for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states + i; + clause_state->clause_id = 0; + if (!clause_state->in_use) { + continue; + } + + has_clause_num++; + literal_ids = (struct maat_hierarchy_literal_id *)utarray_eltptr(clause_state->literal_ids, 0); + n_literal_id = utarray_len(clause_state->literal_ids); + clause = maat_hierarchy_clause_fetch(hier, literal_ids, n_literal_id); + clause_state->clause_id = clause->clause_id; + } + assert(has_clause_num == compile->actual_clause_num); + } + + //STEP 2, serial compile clause states to a bool expression array. + compile_num = HASH_COUNT(hier->hash_compile_by_id); + bool_expr_array = ALLOC(struct bool_expr, compile_num); + HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { + for (i = 0, j = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + if (compile->clause_states[i].in_use) { + if (compile->clause_states[i].not_flag) { + compile->not_clause_cnt++; + } + bool_expr_array[expr_cnt].items[j].item_id = compile->clause_states[i].clause_id; + bool_expr_array[expr_cnt].items[j].not_flag = compile->clause_states[i].not_flag; + j++; + } + } + //some compile may have zero groups, e.g. default policy. + if (j == (size_t)compile->declared_clause_num && j > 0) { + bool_expr_array[expr_cnt].expr_id = compile->compile_id; + bool_expr_array[expr_cnt].user_tag = compile; + bool_expr_array[expr_cnt].item_num = j; + expr_cnt++; + } + } + + //STEP 3, build the bool matcher. + size_t mem_size = 0; + if (0 == expr_cnt) { + log_error(hier->logger, MODULE_HIERARCHY, "No bool expression to build."); + goto error_out; + } + + bm = bool_matcher_new(bool_expr_array, expr_cnt, &mem_size); + if (bm != NULL) { + log_info(hier->logger, MODULE_HIERARCHY, + "Build bool matcher of %zu expressions with %zu bytes memory.", expr_cnt, mem_size); + } else { + log_error(hier->logger, MODULE_HIERARCHY, "Build bool matcher failed!"); + } + +error_out: + FREE(bool_expr_array); + + return bm; +} + +static inline int compare_clause_id(const void *a, const void *b) +{ + long long ret = *(const unsigned long long *)a - *(const unsigned long long *)b; + + if (0 == ret) { + return 0; + } else if(ret < 0) { + return -1; + } else { + return 1; + } +} + +static inline int compare_region_id(const void *a, const void *b) +{ + return (*(int*)a - *(int*)b); +} + +struct region2clause_value *maat_hierarchy_build_region2clause_hash(struct maat_hierarchy *hier) +{ + size_t i = 0, j = 0, k = 0; + struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; + struct maat_hierarchy_literal_id *literal_id = NULL; + struct maat_hierarchy_clause_state *clause_state = NULL; + + struct maat_hierarchy_region *region = NULL, *tmp_region = NULL; + struct maat_hierarchy_group *group = NULL; + struct group2region *g2r_hash = NULL, *g2r = NULL, *g2r_tmp = NULL; + struct region2clause_value *region2clause_hash = NULL, *r2c_val = NULL; + struct region2clause_key r2c_key; + + //Build a temporary hash that maps group to its regions. + HASH_ITER(hh, hier->hash_region_by_id, region, tmp_region) { + group = region->ref_parent_group; + for (i = 0; i < group->top_group_cnt; i++) { + HASH_FIND_INT(g2r_hash, group->top_group_ids+i, g2r); + if (!g2r) { + g2r = ALLOC(struct group2region, 1); + utarray_new(g2r->region_ids, &ut_region_id_icd); + utarray_reserve(g2r->region_ids, group->ref_by_region_cnt); + g2r->group_id = group->top_group_ids[i]; + HASH_ADD_INT(g2r_hash, group_id, g2r); + } + + //One region belongs to one group, one group may have many regions. So duplicate region check is unnecessary. + //if(utarray_find(g2r->region_ids, &(region->region_id), compare_region_id)) assert(0); + + utarray_push_back(g2r->region_ids, &(region->region_id)); + //utarray_sort(g2r->region_ids, compare_region_id); + } + } + + //Build short cut hash that maps region_id+vt_id to clause_ids. + HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { + for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states + i; + if (!clause_state->in_use) { + continue; + } + + for (j = 0; j < utarray_len(clause_state->literal_ids); j++) { + literal_id = (struct maat_hierarchy_literal_id *)utarray_eltptr(clause_state->literal_ids, j); + HASH_FIND(hh_group_id, hier->hash_group_by_id, &(literal_id->group_id), sizeof(literal_id->group_id), group); + if (!group) { + continue; + } + HASH_FIND_INT(g2r_hash, &(group->group_id), g2r); + + //group declared by compile, but has no subordinate or region. + if (!g2r) { + continue; + } + + for (k = 0; k < utarray_len(g2r->region_ids); k++) { + r2c_key.region_id = *((int*)utarray_eltptr(g2r->region_ids, k)); + r2c_key.vt_id = literal_id->vt_id; + HASH_FIND(hh, region2clause_hash, &r2c_key, sizeof(r2c_key), r2c_val); + if (!r2c_val) { + r2c_val = ALLOC(struct region2clause_value, 1); + r2c_val->key = r2c_key; + r2c_val->group_id = g2r->group_id; + utarray_new(r2c_val->clause_ids, &ut_clause_id_icd); + HASH_ADD(hh, region2clause_hash, key, sizeof(r2c_val->key), r2c_val); + } + + if (utarray_find(r2c_val->clause_ids, &(clause_state->clause_id), compare_clause_id)) { + continue; + } + utarray_push_back(r2c_val->clause_ids, &(clause_state->clause_id)); + utarray_sort(r2c_val->clause_ids, compare_clause_id); + } + } + } + } + + int tmp1 = 0, tmp2 = 0; + HASH_ITER(hh, g2r_hash, g2r, g2r_tmp) { + HASH_DEL(g2r_hash, g2r); + //Sanity Check + utarray_sort(g2r->region_ids, compare_region_id); + for (i = 1; i < utarray_len(g2r->region_ids); i++) { + tmp1 = *((int*)utarray_eltptr(g2r->region_ids, i-1)); + tmp2 = *((int*)utarray_eltptr(g2r->region_ids, i)); + assert(tmp1!=tmp2); + } + utarray_free(g2r->region_ids); + g2r->region_ids = NULL; + free(g2r); + } + + log_info(hier->logger, MODULE_HIERARCHY, "Build region2clause hash with %llu element.", + HASH_COUNT(region2clause_hash)); + return region2clause_hash; +} + +int maat_hierarchy_rebuild(struct maat_hierarchy *hier) +{ + int ret=0; + struct bool_matcher *new_bm = NULL, *old_bm = NULL; + struct region2clause_value *new_region2clause_hash = NULL, *old_region2clause_hash = NULL; + + //Read hier from update thread is OK. + if (!hier->changed_flag) { + return ret; + } + + ret = maat_hierarchy_build_top_groups(hier); + new_bm = maat_hierarchy_build_bool_matcher(hier); + new_region2clause_hash = maat_hierarchy_build_region2clause_hash(hier); + + pthread_rwlock_wrlock(&hier->rwlock); + + old_bm = hier->bm; + old_region2clause_hash = hier->hash_region2clause; + + hier->bm = new_bm; + hier->hash_region2clause = new_region2clause_hash; + hier->changed_flag = 0; + pthread_rwlock_unlock(&hier->rwlock); + + maat_garbage_bagging(hier->ref_garbage_bin, old_bm, (void (*)(void*))bool_matcher_free); + maat_garbage_bagging(hier->ref_garbage_bin, old_region2clause_hash, + (void (*)(void*))maat_hierarchy_free_region2clause_hash); + + return ret; +} + +size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, + struct maat_hit_path_t *hit_paths, size_t n_path) +{ + +} + +void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)) +{ + hier->compile_user_data_free = func; +} + +void maat_hierarchy_set_region_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)) +{ + hier->region_user_data_free = func; +} + +struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id) +{ + struct maat_hierarchy_compile_mid *mid = ALLOC(struct maat_hierarchy_compile_mid, 1); + + mid->thread_id = thread_id; + mid->hier_ver = hier->version; + utarray_new(mid->internal_hit_paths, &ut_hit_path_icd); + utarray_new(mid->all_hit_clause_array, &ut_clause_id_icd); + utarray_new(mid->this_scan_hit_clause_ids, &ut_clause_id_icd); + + return mid; +} + +void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid) +{ + utarray_free(mid->internal_hit_paths); + utarray_free(mid->all_hit_clause_array); + utarray_free(mid->this_scan_hit_clause_ids); + free(mid); +} + +static int maat_hierarchy_hit_path_add(UT_array *hit_paths, int region_id, int virtual_table_id, + int Nth_scan, int Nth_region_result) +{ + struct maat_hierarchy_internal_hit_path new_path; + + new_path.region_id = region_id; + new_path.Nth_hit_region = Nth_region_result; + new_path.Nth_scan = Nth_scan; + new_path.virtual_table_id = virtual_table_id; + + utarray_push_back(hit_paths, &new_path); + return 1; +} + +void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, + struct maat_hierarchy_compile_mid *mid, + int region_id, int virtual_table_id, + int Nth_scan, int Nth_region_result) +{ + if (mid->Nth_scan != Nth_scan) { + assert(mid->this_scan_region_hit_cnt == 0); + mid->Nth_scan = Nth_scan; + utarray_clear(mid->this_scan_hit_clause_ids); + } + + int ret = maat_hierarchy_hit_path_add(mid->internal_hit_paths, region_id, virtual_table_id, + Nth_scan, Nth_region_result); + if (!ret) { + return; + } + mid->hit_path_cnt++; + mid->this_scan_region_hit_cnt++; + + struct region2clause_value* r2c_val = NULL; + struct region2clause_key r2c_key; + r2c_key.region_id = region_id; + r2c_key.vt_id = virtual_table_id; + + HASH_FIND(hh, hier->hash_region2clause, &r2c_key, sizeof(r2c_key), r2c_val); + if (!r2c_val) { + return; + } + + size_t i = 0; + unsigned long long *clause_id = 0; + size_t new_clause_idx = utarray_len(mid->this_scan_hit_clause_ids); + for (size_t i = 0; i < utarray_len(r2c_val->clause_ids); i++) { + clause_id = (unsigned long long *)utarray_eltptr(r2c_val->clause_ids, i); + if (utarray_find(mid->all_hit_clause_array, clause_id, compare_clause_id)) { + continue; + } + utarray_push_back(mid->this_scan_hit_clause_ids, clause_id); + } + + if (utarray_len(mid->this_scan_hit_clause_ids) - new_clause_idx) { + utarray_reserve(mid->all_hit_clause_array, utarray_len(mid->this_scan_hit_clause_ids) - new_clause_idx); + for (i = new_clause_idx; i < utarray_len(mid->this_scan_hit_clause_ids); i++) { + clause_id = (unsigned long long *)utarray_eltptr(mid->this_scan_hit_clause_ids, i); + utarray_push_back(mid->all_hit_clause_array, clause_id); + } + utarray_sort(mid->all_hit_clause_array, compare_clause_id); + } +} + +int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid) +{ + return mid->not_clause_hitted_flag; +} + +int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, + int declared_clause_num, void *user_data) +{ + int ret = 0; + struct maat_hierarchy_compile *compile = NULL; + + pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag = 1; + HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); + if (!compile) { + assert(declared_clause_num >= 0); + compile = maat_hierarchy_compile_new(hier, compile_id); + compile->declared_clause_num = declared_clause_num; + compile->user_data = user_data; + } else { + if (compile->user_data != NULL) { + log_error(hier->logger, MODULE_HIERARCHY, + "Add compile %d failed, compile is already exisited.", compile_id); + ret = -1; + } else { + compile->declared_clause_num = declared_clause_num; + compile->user_data = user_data; + } + } + pthread_rwlock_unlock(&hier->rwlock); + + return ret; +} + +int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id) +{ + int ret = 0; + struct maat_hierarchy_compile *compile = NULL; + + pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag = 1; + HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); + if (compile) { + if (hier->compile_user_data_free && compile->user_data) { + hier->compile_user_data_free(compile->user_data); + compile->user_data = NULL; + } + + if (0 == compile->actual_clause_num) { + HASH_DEL(hier->hash_compile_by_id, compile); + maat_garbage_bagging(hier->ref_garbage_bin, compile, (void (*)(void*))maat_hierarchy_compile_free); + } + ret = 0; + } else { + log_error(hier->logger, MODULE_HIERARCHY, + "Remove compile %d failed, compile is not exisited.", compile_id); + ret = -1; + } + pthread_rwlock_unlock(&hier->rwlock); + + return ret; +} + +static void *maat_hier_compile_get_user_data(struct maat_hierarchy* hier, int compile_id, int is_dettach) +{ + struct maat_hierarchy_compile *compile = NULL; + void *ret = NULL; + + pthread_rwlock_rdlock(&hier->rwlock); + HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); + if (compile) { + ret = compile->user_data; + if (is_dettach) { + compile->user_data = NULL; + } + } + pthread_rwlock_unlock(&hier->rwlock); + + return ret; +} + +void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id) +{ + return maat_hier_compile_get_user_data(hier, compile_id, 1); +} + +void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id) +{ + return maat_hier_compile_get_user_data(hier, compile_id, 0); +} + +void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param) +{ + struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; + + pthread_rwlock_rdlock(&hier->rwlock); + HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { + if (compile->user_data) { + callback(compile->user_data, param); + } + } + pthread_rwlock_unlock(&hier->rwlock); +} + +static int maat_hierarchy_compile_has_clause(struct maat_hierarchy_compile* compile, unsigned long long clause_id) +{ + struct maat_hierarchy_clause_state *clause_state = NULL; + + for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states + i; + if (!clause_state->in_use) { + continue; + } + + if (clause_state->clause_id == clause_id) { + return 1; + } + } + + return 0; +} + +static size_t maat_hierarchy_compile_mid_if_new_hit_compile(struct maat_hierarchy_compile_mid *mid, + struct maat_hierarchy_compile *compile) +{ + size_t r_in_c_cnt = 0; + + unsigned long long new_hit_clause_id = 0; + for(size_t i = 0; ithis_scan_hit_clause_ids); i++) { + new_hit_clause_id = *(unsigned long long*)utarray_eltptr(mid->this_scan_hit_clause_ids, i); + int ret = maat_hierarchy_compile_has_clause(compile, new_hit_clause_id); + if (ret) { + r_in_c_cnt++; + } + } + return r_in_c_cnt; +} + +int maat_hierarchy_region_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, + int is_last_compile, void **user_data_array, size_t ud_array_sz) +{ + int bool_match_ret = 0, i = 0; + struct maat_hierarchy_compile *compile = NULL; + struct bool_expr_match *expr_match = hier->expr_match_buff + (mid->thread_id * MAX_SCANNER_HIT_NUM); + + size_t r_in_c_cnt = 0; + size_t ud_result_cnt = 0; + size_t this_scan_region_hits = mid->this_scan_region_hit_cnt; + + if (!hier->bm || 0 == utarray_len(mid->all_hit_clause_array) || hier->version != mid->hier_ver) { + mid->this_scan_region_hit_cnt = 0; + return 0; + } + + bool_match_ret = bool_matcher_match(hier->bm, (unsigned long long *)utarray_eltptr(mid->all_hit_clause_array, 0), + utarray_len(mid->all_hit_clause_array), expr_match, MAX_SCANNER_HIT_NUM); + for (i = 0; i < bool_match_ret && ud_result_cnt < ud_array_sz; i++) { + compile = (struct maat_hierarchy_compile *)expr_match[i].user_tag; + assert(compile->magic == MAAT_HIER_COMPILE_MAGIC); + assert((unsigned long long)compile->compile_id == expr_match[i].expr_id); + if (0 == compile->actual_clause_num) { + continue; + } + + r_in_c_cnt = maat_hierarchy_compile_mid_if_new_hit_compile(mid, compile); + if (compile->not_clause_cnt > 0 && !is_last_compile) { + mid->not_clause_hitted_flag = 1; + } else if(compile->user_data) { + //For compile may be dettached by Maat_hierarchy_compile_dettach_user_data, only return non-NULL userdata. + if (r_in_c_cnt > 0 || //compile hitted becasue of new reigon + this_scan_region_hits == 0) //or hit a compile that refer a NOT-logic group in previous scan. + { + user_data_array[ud_result_cnt]=compile->user_data; + ud_result_cnt++; + } + } + } + + mid->this_scan_region_hit_cnt = 0; + + return ud_result_cnt; +} + +void *maat_hierarchy_region_dettach_user_data(struct maat_hierarchy *hier, int region_id) +{ + struct maat_hierarchy_region *region = NULL; + void *ret = NULL; + + pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag = 1; + HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); + if (region) { + ret = region->user_data; + region->user_data = NULL; + } + pthread_rwlock_unlock(&hier->rwlock); + + return ret; +} + +int compare_literal_id(const void *pa, const void *pb) +{ + struct maat_hierarchy_literal_id *la = (struct maat_hierarchy_literal_id *)pa; + struct maat_hierarchy_literal_id *lb = (struct maat_hierarchy_literal_id *)pb; + + int ret = la->vt_id - lb->vt_id; + if (ret == 0) { + ret = la->group_id - lb->group_id; + } + + return ret; +} + +static int maat_hierarchy_compile_add_literal(struct maat_hierarchy_compile *compile, + struct maat_hierarchy_literal_id *literal_id, + int not_flag, int clause_index) +{ + struct maat_hierarchy_literal_id *tmp = NULL; + struct maat_hierarchy_clause_state *clause_state = compile->clause_states + clause_index; + clause_state->not_flag = not_flag; + + if (!clause_state->in_use) { + clause_state->in_use = 1; + compile->actual_clause_num++; + } + + tmp = (struct maat_hierarchy_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); + if (tmp) { + assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); + return -1; + } else { + utarray_push_back(clause_state->literal_ids, literal_id); + utarray_sort(clause_state->literal_ids, compare_literal_id); + } + + return 0; +} + +static int maat_hierarchy_compile_remove_literal(struct maat_hierarchy_compile *compile, + struct maat_hierarchy_literal_id *literal_id, + int clause_index) +{ + struct maat_hierarchy_literal_id *tmp = NULL; + struct maat_hierarchy_clause_state *clause_state = compile->clause_states + clause_index; + + tmp = (struct maat_hierarchy_literal_id *)utarray_find(clause_state->literal_ids, literal_id , compare_literal_id); + if (tmp) { + assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); + } else { + return -1; + } + + size_t remove_idx = utarray_eltidx(clause_state->literal_ids, tmp); + utarray_erase(clause_state->literal_ids, remove_idx, 1); + if (0 == utarray_len(clause_state->literal_ids)) { + clause_state->in_use = 0; + compile->actual_clause_num--; + } + + return 0; +} + +int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, + int not_flag, int clause_index, int compile_id) +{ + int ret = 0; + struct maat_hierarchy_group *group = NULL; + struct maat_hierarchy_literal_id literal_id = {group_id, vt_id}; + struct maat_hierarchy_compile *compile = NULL; + + pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag = 1; + HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); + if (!group) { + group = maat_hierarchy_group_new(hier, group_id); + } + + HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile); + if (!compile) { + compile = maat_hierarchy_compile_new(hier, compile_id); + } + + ret = maat_hierarchy_compile_add_literal(compile, &literal_id, not_flag, clause_index); + if (ret < 0) { + log_error(hier->logger, MODULE_HIERARCHY, + "Add group %d vt_id %d to clause %d of compile %d failed, group is already exisited.", + group_id, vt_id, clause_index, compile_id); + ret = -1; + } else { + ret = 0; + group->ref_by_compile_cnt++; + } + pthread_rwlock_unlock(&hier->rwlock); + + return ret; +} + +int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, + int not_flag, int clause_index, int compile_id) +{ + struct maat_hierarchy_group *group = NULL; + struct maat_hierarchy_literal_id literal_id = {group_id, vt_id}; + struct maat_hierarchy_compile *compile = NULL; + int ret = 0; + + pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag = 1; + HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); + if (!group) { + log_error(hier->logger, MODULE_HIERARCHY, + "Remove group %d from compile %d failed, group is not exisited.", + group_id, compile_id); + goto error_out; + } + + HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile); + if (!compile) { + log_error(hier->logger, MODULE_HIERARCHY, + "Remove group %d from compile %d failed, compile is not exisited.", + group_id, compile_id); + goto error_out; + } + + ret = maat_hierarchy_compile_remove_literal(compile, &literal_id, clause_index); + if (ret < 0) { + log_error(hier->logger, MODULE_HIERARCHY, + "Remove group %d vt_id %d from clause %d of compile %d failed, literal is not in compile.", + group_id, vt_id, clause_index, compile_id); + goto error_out; + } + + if (0 == compile->actual_clause_num && !compile->user_data) { + HASH_DEL(hier->hash_compile_by_id, compile); + maat_garbage_bagging(hier->ref_garbage_bin, compile, (void (*)(void*))maat_hierarchy_compile_free); + } + pthread_rwlock_unlock(&hier->rwlock); + return 0; + +error_out: + pthread_rwlock_unlock(&hier->rwlock); + return -1; +} + +int maat_hierarchy_add_group_to_group(struct maat_hierarchy *hier, int group_id, int superior_group_id) +{ + igraph_integer_t edge_id; + struct maat_hierarchy_group *group = NULL, *superior_group = NULL; + + pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag = 1; + HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); + if (!group) { + group = maat_hierarchy_group_new(hier, group_id); + } + + HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group); + if (!superior_group) { + superior_group = maat_hierarchy_group_new(hier, superior_group_id); + } + + int ret = igraph_get_eid(&hier->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); + //No duplicated edges between two groups. + if (edge_id > 0) { + log_error(hier->logger, MODULE_HIERARCHY, + "Add group %d to group %d failed, relation already exisited.", + group->group_id, superior_group->group_id); + ret = -1; + } else { + igraph_add_edge(&hier->group_graph, group->vertex_id, superior_group->vertex_id); + group->ref_by_superior_group_cnt++; + superior_group->ref_by_subordinate_group_cnt++; + ret = 0; + } + pthread_rwlock_unlock(&hier->rwlock); + + return ret; +} + +int maat_hierarchy_remove_group_from_group(struct maat_hierarchy *hier, int group_id, int superior_group_id) +{ + struct maat_hierarchy_group *group = NULL, *superior_group = NULL; + + //No hash write operation, LOCK protection is unnecessary. + hier->changed_flag = 1; + HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); + if (NULL == group) { + log_error(hier->logger, MODULE_HIERARCHY, + "Del group %d from group %d failed, group %d not exisited.", + group_id, superior_group_id, group_id); + return -1; + } + + HASH_FIND(hh_group_id, hier->hash_group_by_id, &superior_group_id, sizeof(superior_group_id), superior_group); + if (NULL == superior_group) { + log_error(hier->logger, MODULE_HIERARCHY, + "Del group %d from group %d failed, superior group %d not exisited.", + group_id, superior_group_id, superior_group_id); + return -1; + } + + igraph_es_t es; + igraph_integer_t edge_num_before = 0, edge_num_after = 0; + + edge_num_before = igraph_ecount(&hier->group_graph); + // The edges between the given pairs of vertices will be included in the edge selection. + //The vertex pairs must be given as the arguments of the function call, the third argument + //is the first vertex of the first edge, the fourth argument is the second vertex of the + //first edge, the fifth is the first vertex of the second edge and so on. The last element + //of the argument list must be -1 to denote the end of the argument list. + //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small + int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); + assert(ret == IGRAPH_SUCCESS); + // ignore no such edge to abort(). + igraph_set_error_handler(igraph_error_handler_ignore); + ret = igraph_delete_edges(&hier->group_graph, es); + edge_num_after = igraph_ecount(&hier->group_graph); + igraph_es_destroy(&es); + + if (ret != IGRAPH_SUCCESS || edge_num_before - edge_num_after != 1) { + assert(0); + return -1; + } + + group->ref_by_superior_group_cnt--; + superior_group->ref_by_subordinate_group_cnt--; + + return 0; +} + +int maat_hierarchy_add_region_to_group(struct maat_hierarchy *hier, int group_id, int region_id, + int table_id, void *user_data) +{ + //A region rule belongs to ONE group only. + struct maat_hierarchy_group *group = NULL; + struct maat_hierarchy_region *region = NULL; + int ret = 0; + + pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag = 1; + HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); + if (!group) { + group = maat_hierarchy_group_new(hier, group_id); + } + + HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); + if (region) { + log_error(hier->logger, MODULE_HIERARCHY, + "Add region %d to group %d failed, region already in group %d.", + region_id, group_id, region->ref_parent_group->group_id); + ret = -1; + } else { + region = maat_hierarchy_region_new(hier, region_id, group_id, table_id, group, user_data); + ret = 0; + } + pthread_rwlock_unlock(&hier->rwlock); + + return ret; +} + +int maat_hierarchy_remove_region_from_group(struct maat_hierarchy *hier, int group_id, int region_id) +{ + struct maat_hierarchy_group *group = NULL; + struct maat_hierarchy_region *region = NULL; + + pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag = 1; + HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); + if (!group) { + log_error(hier->logger, MODULE_HIERARCHY, + "Remove region %d from group %d failed, group is not existed.", + region_id, group_id); + goto error_out; + } + + HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); + if (!region) { + log_error(hier->logger, MODULE_HIERARCHY, + "Remove region %d from group %d failed, region is not exisited.", + region_id, group_id); + goto error_out; + } + + assert(region->group_id == group->group_id); + maat_hierarchy_region_free(hier, region); + pthread_rwlock_unlock(&hier->rwlock); + return 0; + +error_out: + pthread_rwlock_unlock(&hier->rwlock); + return -1; +} \ No newline at end of file diff --git a/src/maat_redis_monitor.cpp b/src/maat_redis_monitor.cpp index e912649..536ff7c 100644 --- a/src/maat_redis_monitor.cpp +++ b/src/maat_redis_monitor.cpp @@ -112,7 +112,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in } int foreign_columns[8]; - int n_foreign_column = plugin_table_schema_get_foreign_column(table_schema, foreign_columns); + int n_foreign_column = table_schema_get_foreign_column(table_schema, foreign_columns); if (0 == n_foreign_column) { continue; } diff --git a/src/maat_rule.cpp b/src/maat_rule.cpp index 2b029c0..785f9d4 100644 --- a/src/maat_rule.cpp +++ b/src/maat_rule.cpp @@ -23,27 +23,81 @@ #include "maat_rule.h" #include "maat_config_monitor.h" #include "maat_redis_monitor.h" - #include "maat_table_runtime.h" #include "maat_table_schema.h" +#include "maat_hierarchy.h" +#include "alignment.h" #define MODULE_MAAT_RULE module_name_str("maat.rule") +void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, + const char *srv_def, int srv_def_len) +{ + memcpy(rule, rule_head, sizeof(struct maat_rule_head)); + memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN)); +} + +void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, void *ex_data, + const struct compile_ex_data_schema *ex_schema) +{ + struct maat_rule rule; + + memset(&rule, 0, sizeof(rule)); + fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); + ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, ex_schema->argl, ex_schema->argp); +} + +void destroy_compile_rule(struct maat_compile_rule *compile_rule) +{ + size_t n_compile_ex_schema = table_schema_compile_rule_ex_data_schema_count(compile_rule->ref_table); + assert(compile_rule->magic_num == COMPILE_RULE_MAGIC); + + for (size_t i = 0; i < n_compile_ex_schema; i++) { + struct compile_ex_data_schema *compile_ex_schema = table_schema_get_compile_rule_ex_data_schema(compile_rule->ref_table, i); + rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, compile_ex_schema); + void *ex_data = compile_rule->ex_data + 1; + ex_data = NULL; + } + free(compile_rule->ex_data); + + compile_rule->is_valid = 0; + compile_rule->declared_clause_num = -1; + FREE(compile_rule->service_defined); + FREE(compile_rule); +} + +void maat_region_inner_free(struct maat_region_inner *region) +{ + assert(region->magic_num == REGION_RULE_MAGIC); + assert(region->expr_id_cnt == 0 ||region->expr_id_cnt == (region->expr_id_ub - region->expr_id_lb + 1)); + region->magic_num = 0; + FREE(region); +} + struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance) { struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); maat_rt->version = version; maat_rt->table_rt_mgr = table_runtime_manager_create(maat_instance->table_schema_mgr, - maat_instance->nr_worker_thread, - maat_instance->garbage_bin); + maat_instance->nr_worker_thread, + maat_instance->garbage_bin); maat_rt->max_table_num = table_schema_manager_get_size(maat_instance->table_schema_mgr); maat_rt->max_thread_num = maat_instance->nr_worker_thread; + maat_rt->hier = maat_hierarchy_new(maat_instance->nr_worker_thread, maat_instance->garbage_bin, + maat_instance->logger); + maat_hierarchy_set_compile_user_data_free_func(maat_rt->hier, (void (*)(void*))destroy_compile_rule); + maat_hierarchy_set_region_user_data_free_func(maat_rt->hier, (void (*)(void*))maat_region_inner_free); + maat_rt->logger = maat_instance->logger; + maat_rt->ref_garbage_bin = maat_instance->garbage_bin; + maat_rt->region_result_buff = ALLOC(struct scan_result, MAX_SCANNER_HIT_NUM * maat_instance->nr_worker_thread); + maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->nr_worker_thread); + return maat_rt; } -void maat_runtime_commit(struct maat_runtime *maat_rt) +void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger) { for (size_t i = 0; i < maat_rt->max_table_num; i++) { struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i); @@ -51,7 +105,7 @@ void maat_runtime_commit(struct maat_runtime *maat_rt) continue; } - table_runtime_commit(table_rt, maat_rt->max_thread_num); + table_runtime_commit(table_rt, maat_rt->max_thread_num, logger); } maat_rt->last_update_time = time(NULL); @@ -159,13 +213,13 @@ void maat_finish_cb(void *u_param) if (maat_instance->creating_maat_rt != NULL) { maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); - maat_runtime_commit(maat_instance->creating_maat_rt); + maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger); log_info(maat_instance->logger, MODULE_MAAT_RULE, "Full config version %llu load %d entries complete\n", maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->rule_num); } else if (maat_instance->maat_rt != NULL) { maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt); maat_instance->maat_rt->version = maat_instance->maat_version; - maat_runtime_commit(maat_instance->maat_rt); + maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger); log_info(maat_instance->logger, MODULE_MAAT_RULE, "Inc config version %llu load %d entries complete\n", maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); } @@ -267,7 +321,7 @@ void *rule_monitor_loop(void *arg) time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time; if ((updating_flag > 0) && (time_window >= maat_instance->rule_effect_interval_ms / 1000)) { - maat_runtime_commit(maat_instance->maat_rt); + maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger); log_info(maat_instance->logger,MODULE_MAAT_RULE, "Actual update config version %u, %d entries load to rulescan after postpone.", maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); diff --git a/src/maat_table_runtime.cpp b/src/maat_table_runtime.cpp index a6061c3..0dcc032 100644 --- a/src/maat_table_runtime.cpp +++ b/src/maat_table_runtime.cpp @@ -23,6 +23,7 @@ #include "adapter_hs.h" #include "rcu_hash.h" #include "IPMatcher.h" +#include "alignment.h" #define MODULE_TABLE_RUNTIME module_name_str("maat.table_runtime") @@ -69,6 +70,10 @@ struct table_runtime { }; struct maat_garbage_bin *ref_garbage_bin; + + long long *scan_cnt; + long long *hit_cnt; + long long *stream_num; //ex_data_rt //table相关指针 }; @@ -144,6 +149,10 @@ struct table_runtime *table_runtime_new(struct table_schema *table_schema, int m break; } + table_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + table_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); + table_rt->stream_num = alignment_int64_array_alloc(max_thread_num); + return table_rt; } @@ -232,7 +241,7 @@ size_t table_runtime_rule_count(struct table_runtime *table_rt) enum table_type table_runtime_get_type(struct table_runtime* table_rt) { if (NULL == table_rt) { - return TABLE_TYPE_MAX; + return TABLE_TYPE_INVALID; } return table_rt->table_type; @@ -286,7 +295,7 @@ enum IP_TYPE ip_type_transform(enum ip_type type) } int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, - int results[], size_t *n_result) + struct scan_result *results, size_t *n_result, size_t n_result_array) { if (NULL == table_rt) { return -1; @@ -294,7 +303,7 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct int i = 0; int n_hit_result = 0; - struct scan_result scan_results[table_rt->rule_num] = {0}; + struct scan_result scan_results[n_result_array] = {0}; struct ip_data ip; ip.type = ip_type_transform(data->ip_type); @@ -304,7 +313,11 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct memcpy(ip.ipv6, data->ipv6, sizeof(data->ipv6)); } - n_hit_result = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, table_rt->rule_num); + n_hit_result = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, n_result_array); + if (n_hit_result > (int)n_result_array) { + n_hit_result = n_result_array; + } + int index = 0; /* filter rule_id by port */ for (i = 0; i < n_hit_result; i++) { @@ -312,7 +325,7 @@ int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct if (data->port < port_range->min_port || data->port > port_range->max_port) { continue; } - results[index++] = scan_results[i].rule_id; + results[index++].rule_id = scan_results[i].rule_id; } *n_result = index; @@ -473,8 +486,8 @@ void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_sc { struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); ex_container->ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); - int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema); - size_t cb_count = plugin_table_schema_callback_count(table_schema); + int set_flag = table_schema_ex_data_schema_flag(table_schema); + size_t cb_count = table_schema_callback_count(table_schema); /* already set plugin_table_schema's ex_data_schema */ if (1 == set_flag) { @@ -489,7 +502,7 @@ void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_sc /* plugin table schema has callback */ if (cb_count > 0) { - plugin_table_schema_all_cb_update(table_schema, row); + table_schema_all_cb_update(table_schema, row); } if ((0 == set_flag) && (0 == cb_count)) { @@ -510,7 +523,7 @@ void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct struct ex_container_ctx *ctx = ex_data_runtime_get_ex_container_ctx(ex_data_rt); ctx->custom_data_free = free; - int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema); + int set_flag = table_schema_ex_data_schema_flag(table_schema); if (1 == set_flag) { if (0 == is_valid) { @@ -576,7 +589,8 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t } } -int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread) +int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, + struct log_handle *logger) { struct expr_runtime *expr_rt = &(table_rt->expr_rt); void **ex_data_array = NULL; @@ -594,8 +608,12 @@ int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread) struct adapter_hs *new_adapter_hs = NULL; struct adapter_hs *old_adapter_hs = NULL; if (rule_cnt > 0) { + log_info(logger, MODULE_TABLE_RUNTIME, + "committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt); new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt); if (NULL == new_adapter_hs) { + log_error(logger, MODULE_TABLE_RUNTIME, + "rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt); ret = -1; } } @@ -615,7 +633,7 @@ int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread) return ret; } -int ip_plus_runtime_commit(struct table_runtime *table_rt) +int ip_plus_runtime_commit(struct table_runtime *table_rt, struct log_handle *logger) { struct ex_data_container **ex_container = NULL; struct ip_rule *rules = NULL; @@ -636,8 +654,12 @@ int ip_plus_runtime_commit(struct table_runtime *table_rt) size_t mem_used = 0; if (rule_cnt > 0) { + log_info(logger, MODULE_TABLE_RUNTIME, + "committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_ip_matcher) { + log_error(logger, MODULE_TABLE_RUNTIME, + "rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt); ret = -1; } } @@ -662,7 +684,7 @@ int plugin_runtime_commit(struct table_runtime *table_rt) return 0; } -int ip_plugin_runtime_commit(struct table_runtime *table_rt) +int ip_plugin_runtime_commit(struct table_runtime *table_rt, struct log_handle *logger) { struct ex_data_container **ex_container = NULL; struct ip_rule *rules = NULL; @@ -683,8 +705,12 @@ int ip_plugin_runtime_commit(struct table_runtime *table_rt) size_t mem_used = 0; if (rule_cnt > 0) { + log_info(logger, MODULE_TABLE_RUNTIME, + "committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_ip_matcher) { + log_error(logger, MODULE_TABLE_RUNTIME, + "rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt); ret = -1; } } @@ -737,20 +763,21 @@ int table_runtime_updating_flag(struct table_runtime *table_rt) return updating_flag; } -void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread) +void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, + struct log_handle *logger) { switch (table_rt->table_type) { case TABLE_TYPE_EXPR: - expr_runtime_commit(table_rt, nr_worker_thread); + expr_runtime_commit(table_rt, nr_worker_thread, logger); break; case TABLE_TYPE_IP_PLUS: - ip_plus_runtime_commit(table_rt); + ip_plus_runtime_commit(table_rt, logger); break; case TABLE_TYPE_PLUGIN: plugin_runtime_commit(table_rt); break; case TABLE_TYPE_IP_PLUGIN: - ip_plugin_runtime_commit(table_rt); + ip_plugin_runtime_commit(table_rt, logger); break; default: break; @@ -813,9 +840,9 @@ struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table } void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, - struct log_handle *logger) + int nr_worker_thread, struct log_handle *logger) { - struct ex_data_schema *ex_data_schema = plugin_table_schema_get_ex_data_schema(table_schema); + struct ex_data_schema *ex_data_schema = table_schema_get_ex_data_schema(table_schema); struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt); ex_data_runtime_set_schema(ex_data_rt, ex_data_schema); struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1); @@ -840,5 +867,5 @@ void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct } ex_data_runtime_clear_row_cache(ex_data_rt); - table_runtime_commit(table_rt, 0); + table_runtime_commit(table_rt, nr_worker_thread, logger); } diff --git a/src/maat_table_schema.cpp b/src/maat_table_schema.cpp index 3261413..6f41103 100644 --- a/src/maat_table_schema.cpp +++ b/src/maat_table_schema.cpp @@ -29,6 +29,20 @@ #define MAX_FOREIGN_CLMN_NUM 8 #define MAX_CONJUNCTION_TABLE_NUM 8 +#define MAX_COMPILE_EX_DATA_NUM 2 + +enum user_region_encode { + USER_REGION_ENCODE_NONE=0, + USER_REGION_ENCODE_ESCAPE, + USER_REGION_ENCODE_BASE64 +}; + +struct compile_table_schema { + enum user_region_encode user_region_encoding; + size_t n_ex_schema; + struct compile_ex_data_schema ex_schema[MAX_COMPILE_EX_DATA_NUM]; +}; + struct expr_table_schema { int item_id_column; int group_id_column; @@ -41,23 +55,6 @@ struct expr_table_schema { enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ }; -struct ip_table_schema { - int item_id_column; - int group_id_column; - int addr_type_column; - int sip_column; - int sip_mask_column; - int sport_column; - int sport_mask_column; - int dip_column; - int dip_mask_column; - int dport_column; - int dport_mask_column; - int proto_column; - int direction_column; - int is_valid_column; -}; - struct ip_plus_table_schema { int item_id_column; int group_id_column; @@ -109,7 +106,7 @@ struct virtual_table_schema { struct composition_table_schema { - int component_table_id[COMPOSITION_TYPE_MAX]; + int component_table_id[COMPONENT_TABLE_TYPE_MAX]; }; struct table_schema { @@ -119,8 +116,8 @@ struct table_schema { char table_name[MAX_CONJUNCTION_TABLE_NUM][NAME_MAX]; enum table_type table_type; union { + struct compile_table_schema compile; struct expr_table_schema expr; - struct ip_table_schema ip; struct ip_plus_table_schema ip_plus; struct plugin_table_schema plugin; struct ip_plugin_table_schema ip_plugin; @@ -247,127 +244,7 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable, return 0; } -#if 0 -int read_ip_table_schema(cJSON *root, struct table_schema *ptable, struct log_handle *logger) -{ - size_t read_cnt = 0; - cJSON *json = NULL; - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, "ip table name %s length too long", - json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "item_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->ip.item_id_column = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "group_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->ip.group_id_column = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "custom"); - if (NULL == json || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE_SCHEMA, "ip table %s has no custom column", - ptable->table_name[0]); - return -1; - } - - cJSON *item = cJSON_GetObjectItem(json, "addr_type"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.addr_type_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "sip"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.sip_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "sip_mask"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.sip_mask_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "sport"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.sport_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "sport_mask"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.sport_mask_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "dip"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.dip_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "dip_mask"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.dip_mask_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "dport"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.dport_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "dport_mask"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.dport_mask_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "proto"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.proto_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "direction"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.direction_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "is_valid"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip.is_valid_column = item->valueint; - read_cnt++; - } - - if (read_cnt < 16) { - return -1; - } - - return 0; -} -#endif int read_ip_plus_table_schema(cJSON *root, struct table_schema *ptable, struct log_handle *logger) { @@ -606,7 +483,8 @@ int read_composition_table_schema(cJSON *root, struct table_schema *ptable, cJSON *item = cJSON_GetObjectItem(json, "source"); if (item != NULL && item->type == cJSON_String) { /* physical table should already exist */ - ret = maat_kv_read(tablename2id_map, item->valuestring, &composition_schema->component_table_id[COMPOSITION_TYPE_SIP]); + ret = maat_kv_read(tablename2id_map, item->valuestring, + &composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SIP]); if (ret < 0) { log_error(logger, MODULE_TABLE_SCHEMA, "Child table %s of table %s (id=%d) are not defined", @@ -618,7 +496,8 @@ int read_composition_table_schema(cJSON *root, struct table_schema *ptable, item = cJSON_GetObjectItem(json, "dest"); if (item != NULL && item->type == cJSON_String) { /* physical table should already exist */ - ret = maat_kv_read(tablename2id_map, item->valuestring, &composition_schema->component_table_id[COMPOSITION_TYPE_DIP]); + ret = maat_kv_read(tablename2id_map, item->valuestring, + &composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DIP]); if (ret < 0) { log_error(logger, MODULE_TABLE_SCHEMA, "Child table %s of table %s (id=%d) are not defined", @@ -630,7 +509,8 @@ int read_composition_table_schema(cJSON *root, struct table_schema *ptable, item = cJSON_GetObjectItem(json, "session"); if (item != NULL && item->type == cJSON_String) { /* physical table should already exist */ - ret = maat_kv_read(tablename2id_map, item->valuestring, &composition_schema->component_table_id[COMPOSITION_TYPE_SESSION]); + ret = maat_kv_read(tablename2id_map, item->valuestring, + &composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION]); if (ret < 0) { log_error(logger, MODULE_TABLE_SCHEMA, "Child table %s of table %s (id=%d) are not defined", @@ -833,13 +713,6 @@ int table_schema_populate(cJSON *json, struct table_schema **table_array, return -1; } break; - /* TODO: delete - case TABLE_TYPE_IP: - ret = read_ip_table_schema(json, ptable); - if (ret < 0) { - return -1; - } - break; */ case TABLE_TYPE_IP_PLUS: ret = read_ip_plus_table_schema(json, ptable, logger); if (ret < 0) { @@ -912,7 +785,6 @@ table_schema_manager_create(const char *table_info_path, struct log_handle *logg /* register table type reserved word */ maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); - //maat_kv_register(reserved_word_map, "ip", TABLE_TYPE_IP); maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); @@ -1023,7 +895,7 @@ int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_ } int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id, - enum table_composition_type type) + enum component_table_type type) { struct table_schema* table_schema = table_schema_get(table_schema_mgr, parent_table_id); if (table_schema->table_type != TABLE_TYPE_COMPOSITION) { @@ -1036,11 +908,11 @@ int table_schema_manager_get_child_table_id(struct table_schema_manager *table_s enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int id) { if (NULL == table_schema_mgr) { - return TABLE_TYPE_MAX; + return TABLE_TYPE_INVALID; } - if (table_schema_mgr->schema_table[id] == NULL) { - return TABLE_TYPE_MAX; + if (NULL == table_schema_mgr->schema_table[id]) { + return TABLE_TYPE_INVALID; } return table_schema_mgr->schema_table[id]->table_type; @@ -1205,7 +1077,7 @@ struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager * enum table_type table_schema_get_table_type(struct table_schema *table_schema) { if (NULL == table_schema) { - return TABLE_TYPE_MAX; + return TABLE_TYPE_INVALID; } return table_schema->table_type; @@ -1239,7 +1111,6 @@ enum scan_type table_schema_get_scan_type(struct table_schema *table_schema) case TABLE_TYPE_INTERVAL_PLUS: ret = SCAN_TYPE_INTERVAL; break; - //case TABLE_TYPE_IP: case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_COMPOSITION: ret = SCAN_TYPE_IP; @@ -1530,7 +1401,7 @@ int populate_ip_plus_table_item(const char *line, struct ip_plus_table_schema *i direction = ip_plus_item->ipv4.direction; } else { //ipv6 - ret = ip_format2range(ip_plus_item->addr_type, IP_FORMAT_MASK, sip1_str, sip2_str, + ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip); if (ret < 0) { return -1; @@ -1747,12 +1618,12 @@ enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_sch return table_schema->expr.scan_mode; } -int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema, - maat_plugin_ex_new_func_t *new_func, - maat_plugin_ex_free_func_t *free_func, - maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp, - struct log_handle *logger) +int table_schema_set_ex_data_schema(struct table_schema *table_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger) { if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) { assert(0); @@ -1760,7 +1631,7 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema, return -1; } - struct ex_data_schema *ex_schema = plugin_table_schema_get_ex_data_schema(table_schema); + struct ex_data_schema *ex_schema = table_schema_get_ex_data_schema(table_schema); if (NULL == ex_schema) { log_error(logger, MODULE_TABLE_SCHEMA, "Error: %s, target table is not a valid plugin table", __FUNCTION__); @@ -1784,7 +1655,7 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema, return 0; } -struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schema *table_schema) +struct ex_data_schema *table_schema_get_ex_data_schema(struct table_schema *table_schema) { if (NULL == table_schema) { return NULL; @@ -1806,7 +1677,77 @@ struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schem return ex_schema; } -int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema) +int table_schema_set_compile_rule_ex_data_schema(struct table_schema *table_schema, + maat_rule_ex_new_func_t *new_func, + maat_rule_ex_free_func_t *free_func, + maat_rule_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger) +{ + if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) { + assert(0); + log_error(logger, MODULE_TABLE_SCHEMA, "%s failed: invalid parameter", __FUNCTION__); + return -1; + } + + if (table_schema->table_type != TABLE_TYPE_COMPILE) { + log_error(logger, MODULE_TABLE_SCHEMA, + "table type must TABLE_TYPE_COMPILE when set compile rule ex_data schema"); + return -1; + } + + struct compile_table_schema* compile_schema = &(table_schema->compile); + if (compile_schema->n_ex_schema == MAX_COMPILE_EX_DATA_NUM) { + log_error(logger, MODULE_TABLE_SCHEMA, + "compile ex schema num reach maxium, can't set anymore"); + return -1; + } + + int idx = compile_schema->n_ex_schema; + compile_schema->ex_schema[idx].idx = idx; + compile_schema->ex_schema[idx].table_id = table_schema->table_id; + compile_schema->ex_schema[idx].argl = argl; + compile_schema->ex_schema[idx].argp = argp; + compile_schema->ex_schema[idx].new_func = new_func; + compile_schema->ex_schema[idx].free_func = free_func; + compile_schema->ex_schema[idx].dup_func = dup_func; + compile_schema->n_ex_schema++; + + return idx; +} + +struct compile_ex_data_schema * +table_schema_get_compile_rule_ex_data_schema(struct table_schema *table_schema, size_t idx) +{ + if (NULL == table_schema) { + return NULL; + } + + if (table_schema->table_type != TABLE_TYPE_COMPILE) { + return NULL; + } + + if (idx < table_schema->compile.n_ex_schema) { + return (table_schema->compile.ex_schema + idx); + } + + return NULL; +} + +size_t table_schema_compile_rule_ex_data_schema_count(struct table_schema *table_schema) +{ + if (NULL == table_schema) { + return 0; + } + + if (table_schema->table_type != TABLE_TYPE_COMPILE) { + return 0; + } + + return table_schema->compile.n_ex_schema; +} + +int table_schema_ex_data_schema_flag(struct table_schema *table_schema) { if (NULL == table_schema) { return -1; @@ -1828,7 +1769,7 @@ int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema) return ex_schema->set_flag; } -int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_mgr, int table_id, +int table_schema_add_callback(struct table_schema_manager* table_schema_mgr, int table_id, maat_start_callback_t *start, maat_update_callback_t *update, maat_finish_callback_t *finish, @@ -1872,7 +1813,7 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m return 0; } -size_t plugin_table_schema_callback_count(struct table_schema *table_schema) +size_t table_schema_callback_count(struct table_schema *table_schema) { if (NULL == table_schema) { return 0; @@ -1885,7 +1826,7 @@ size_t plugin_table_schema_callback_count(struct table_schema *table_schema) } } -void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const char *row) +void table_schema_all_cb_update(struct table_schema* table_schema, const char *row) { if (NULL == table_schema || NULL == row) { return; @@ -1911,7 +1852,7 @@ void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const } } -int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns) +int table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns) { if (NULL == table_schema) { return -1; diff --git a/src/maat_utils.cpp b/src/maat_utils.cpp index f86cf5b..dc73344 100644 --- a/src/maat_utils.cpp +++ b/src/maat_utils.cpp @@ -52,7 +52,7 @@ int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len char *str = NULL; char *saveptr = NULL; char *subtoken = NULL; - const char *seps = " \t"; + const char *seps = "\t"; char *dup_line = maat_strdup(line); for (str = dup_line; ; str = NULL) { diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index e289a2b..942e979 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -1,5 +1,6 @@ #include +#include "utils.h" #include "maat/maat.h" #include "maat_rule.h" #include "maat_utils.h" @@ -7,175 +8,26 @@ #include "maat_table_runtime.h" #include "maat_command.h" #include "IPMatcher.h" +#include "json2iris.h" +#include "maat_config_monitor.h" struct maat *g_maat_instance = NULL; const char *table_info_path = "./table_info.conf"; -const char *rule_full_path = "./rule/full/index"; -const char *rule_inc_path = "./rule/inc/index"; const char *json_path="./maat_json.json"; -const char *iris_file = "./HTTP_URL.000001"; -#if 0 -TEST(maat_scan_string, hit_one_expr) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); - char data[64] = ""; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, -1); - - char data1[64] = "baidu.com"; - ret = maat_scan_string(g_maat_instance, table_id, 0, data1, strlen(data1), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 0); +const char *json_filename = "maat_json.json"; - char data2[64] = "hello"; - ret = maat_scan_string(g_maat_instance, table_id, 0, data2, strlen(data2), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 101); - -/* - memset(result_array, 0, sizeof(result_array)); - char data3[64] = "maat"; - ret = maat_scan_string(g_maat_instance, table_id, 0, data3, strlen(data3), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 102); */ - -/* - memset(result_array, 0, sizeof(result_array)); - char data4[64] = "world"; - ret = maat_scan_string(g_maat_instance, table_id, 0, data4, strlen(data4), result_array, &n_result_array, NULL); - EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 103); */ -} - -TEST(maat_scan_string, hit_two_expr) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); - char data[64] = "hello maat"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 2); - EXPECT_EQ(result_array[0], 102); - EXPECT_EQ(result_array[1], 101); - - memset(result_array, 0, sizeof(result_array)); - char data1[64] = "maat world"; - ret = maat_scan_string(g_maat_instance, table_id, 0, data1, strlen(data1), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 2); - EXPECT_EQ(result_array[0], 103); - EXPECT_EQ(result_array[1], 102); -} - -TEST(maat_scan_string, hit_three_expr) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); - char data[64] = "hello world"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 3); - EXPECT_EQ(result_array[0], 104); - EXPECT_EQ(result_array[1], 103); - EXPECT_EQ(result_array[2], 101); -} - -TEST(maat_scan_string, hit_four_expr) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); - char data[64] = "9hello world"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 4); - EXPECT_EQ(result_array[0], 105); - EXPECT_EQ(result_array[1], 104); - EXPECT_EQ(result_array[2], 103); - EXPECT_EQ(result_array[3], 101); -} - -TEST(maat_scan_string, hit_five_expr) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); - char data[64] = "9hello 8world"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 5); - EXPECT_EQ(result_array[0], 106); - EXPECT_EQ(result_array[1], 105); - EXPECT_EQ(result_array[2], 104); - EXPECT_EQ(result_array[3], 103); - EXPECT_EQ(result_array[4], 101); -} - -TEST(maat_scan_string, config_dynamic_update) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); - - char data[128] = "www.baidu.com"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 0); - - /* generate inc update config */ - system_cmd_mkdir("./rule/inc/index"); - char new_rule[128] = "207 7 baidu 0 0 0 1"; - - FILE *fp = fopen("./rule/inc/HTTP_URL.000002", "w+"); - EXPECT_NE(fp, nullptr); - fprintf(fp, "%s\n", "0000000001"); - fprintf(fp, "%s\n", new_rule); - fclose(fp); - - char new_rule_idx[128] = "HTTP_URL 1 ./rule/inc/HTTP_URL.000002"; - fp = fopen("./rule/inc/index/inc_config_index.000002", "w+"); - EXPECT_NE(fp, nullptr); - fprintf(fp, "%s\n", new_rule_idx); - fclose(fp); - - /* if updated rule already valid */ - memset(result_array, 0, sizeof(result_array)); - sleep(2); - ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 207); - - /* if the old rules are still valid */ - memset(result_array, 0, sizeof(result_array)); - char data1[64] = "9hello 8world"; - ret = maat_scan_string(g_maat_instance, table_id, 0, data1, strlen(data1), result_array, &n_result_array, NULL); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 5); - EXPECT_EQ(result_array[0], 106); - EXPECT_EQ(result_array[1], 105); - EXPECT_EQ(result_array[2], 104); - EXPECT_EQ(result_array[3], 103); - EXPECT_EQ(result_array[4], 101); -} -#endif TEST(maat_scan_string, hit_one_expr) { struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); char data[128] = "i.ytimg.com"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + int results[5] = {0}; + size_t n_result = 0; + struct maat_state *state = NULL; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 30); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(results[0], 30); } TEST(maat_scan_string, hit_two_expr) { @@ -183,13 +35,14 @@ TEST(maat_scan_string, hit_two_expr) { int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); char data[128] = "should hit aaa bbb"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + int results[5] = {0}; + size_t n_result = 0; + struct maat_state *state = NULL; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 2); - EXPECT_EQ(result_array[0], 28); - EXPECT_EQ(result_array[1], 27); + EXPECT_EQ(n_result, 2); + EXPECT_EQ(results[0], 28); + EXPECT_EQ(results[1], 27); } TEST(maat_scan_string, hit_three_expr) { @@ -197,14 +50,15 @@ TEST(maat_scan_string, hit_three_expr) { int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); char data[128] = "should hit aaa bbb C#中国"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + int results[5] = {0}; + size_t n_result = 0; + struct maat_state *state = NULL; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 3); - EXPECT_EQ(result_array[0], 28); - EXPECT_EQ(result_array[1], 27); - EXPECT_EQ(result_array[2], 18); + EXPECT_EQ(n_result, 3); + EXPECT_EQ(results[0], 28); + EXPECT_EQ(results[1], 27); + EXPECT_EQ(results[2], 18); } TEST(maat_scan_ipv4, hit_ip_and_port) { @@ -220,14 +74,16 @@ TEST(maat_scan_ipv4, hit_ip_and_port) { int results[3] = {-1}; size_t n_result = 0; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + struct maat_state *state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 1); EXPECT_EQ(results[0], 7); port = 20001; addr.ipv4.sport = htons(port); - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 0); } @@ -245,7 +101,8 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) { int results[3] = {-1}; size_t n_result = 0; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + struct maat_state *state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 1); EXPECT_EQ(results[0], 4); @@ -254,7 +111,8 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) { addr.ipv4.sport = htons(port); memset(results, 0, sizeof(results)); n_result = 0; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 1); EXPECT_EQ(results[0], 4); @@ -263,7 +121,8 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) { addr.ipv4.sport = htons(port); memset(results, 0, sizeof(results)); n_result = 0; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 0); } @@ -284,7 +143,8 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { int results[3] = {-1}; size_t n_result = 0; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + struct maat_state *state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 1); EXPECT_EQ(results[0], 8); @@ -292,7 +152,8 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); EXPECT_EQ(ret, 1); addr.ipv4.sport = htons(port2); - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 1); EXPECT_EQ(results[0], 8); @@ -300,7 +161,8 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { ret = inet_pton(AF_INET, ip_str3, &addr.ipv4.sip); EXPECT_EQ(ret, 1); addr.ipv4.sport = htons(port2); - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 0); } @@ -319,14 +181,16 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) { int results[3] = {-1}; size_t n_result = 0; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + struct maat_state *state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 1); EXPECT_EQ(results[0], 50); ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); EXPECT_EQ(ret, 1); - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 0); } @@ -344,7 +208,8 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { int results[3] = {-1}; size_t n_result = 0; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + struct maat_state *state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 2); EXPECT_EQ(results[0], 63); @@ -352,7 +217,8 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { port = 442; addr.ipv4.sport = htons(port); - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 0); } @@ -370,31 +236,205 @@ TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) { int results[3] = {-1}; size_t n_result = 0; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + struct maat_state *state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 1); EXPECT_EQ(results[0], 47); port = 442; addr.ipv6.sport = htons(port); - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 0); } +TEST(maat_scan_string, dynamic_config) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + + char data[128] = "hello world"; + int results[5] = {0}; + size_t n_result = 0; + struct maat_state *state = NULL; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + const char *table_name = "HTTP_URL"; + const char *table_line = "9999\t8888\thello world\t0\t0\t0\t1\t"; + struct maat_cmd_line line_rule; + line_rule.rule_id = 101; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + ret = maat_cmd_set_line(g_maat_instance, &line_rule); + EXPECT_EQ(ret, 1); + + sleep(2); + state = NULL; + ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(results[0], 9999); +} + +TEST(maat_scan_ip, dynamic_config) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG"); + + char ip_str[32] = "10.0.6.201"; + uint16_t port = 443; + struct addr_4tuple addr; + addr.type = IP_TYPE_V4; + int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip); + EXPECT_EQ(ret, 1); + addr.ipv4.sport = htons(port); + + int results[3] = {-1}; + size_t n_result = 0; + struct maat_state *state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + const char *table_name = "IP_PLUS_CONFIG"; + const char *table_line = "9998\t8887\t4\trange\t10.0.6.201\t255.255.0.0\trange\t0\t65535\t6\t0\t1"; + struct maat_cmd_line line_rule; + line_rule.rule_id = 101; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + ret = maat_cmd_set_line(g_maat_instance, &line_rule); + EXPECT_EQ(ret, 0); + + sleep(2); + state = NULL; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(results[0], 9998); +} + +int count_line_num_cb(const char *table_name, const char *line, void *u_para) +{ + (*((unsigned int *)u_para))++; + return 0; +} + +int line_idx = 0; +long long absolute_expire_time=0; +int make_serial_rule(const char *table_name, const char *line, void *u_para) +{ + struct serial_rule *s_rule=(struct serial_rule *)u_para; + int rule_id = 0; + char *buff = ALLOC(char, strlen(line) + 1); + + memcpy(buff, line, strlen(line) + 1); + + while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { + buff[strlen(buff) - 1] = '\0'; + } + + int j = 0; + char *str1 = NULL; + char *token = NULL; + char *saveptr1 = NULL; + + for (j = 0,str1 = buff; ; j++, str1 = NULL) { + token = strtok_r(str1, "\t ", &saveptr1); + if (token == NULL) + break; + if (j == 0) { + sscanf(token,"%d", &rule_id); + } + } + + memcpy(buff, line, strlen(line)+1); + while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') { + buff[strlen(buff)-1]='\0'; + } + + maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time); + line_idx++; + FREE(str1); + + return 0; +} + int main(int argc, char ** argv) { int ret=0; ::testing::InitGoogleTest(&argc, argv); - +/* struct maat_options *opts = maat_options_new(); maat_options_set_json_file(opts, json_path); g_maat_instance = maat_new(opts, table_info_path); - EXPECT_NE(g_maat_instance, nullptr); + EXPECT_NE(g_maat_instance, nullptr);*/ + + char json_iris_path[128] = {0}; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + struct log_handle *logger = log_handle_create("./tmp.log", 0); + + snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); + + redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger); + EXPECT_NE(c, nullptr); + + redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb"); + EXPECT_NE(reply, nullptr); + + if ((access(json_iris_path, F_OK)) < 0) { + char tmp_iris_path[128] = {0}; + char *json_buff = NULL; + size_t json_buff_sz = 0; + + int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); + EXPECT_NE(ret, -1); + + ret = json2iris(json_buff, json_filename, c, tmp_iris_path, + sizeof(tmp_iris_path), NULL, NULL, logger); + EXPECT_NE(ret, -1); + } + + size_t total_line_cnt = 0; + char tmp_iris_full_idx_path[128] = {0}; + snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path); + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger); + + struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); + long long server_time = maat_cmd_redis_server_time_s(c); + EXPECT_NE(server_time, -1); + + absolute_expire_time = server_time + 300; + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger); + + int success_cnt = 0; + do { + success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger); + } while (success_cnt < 0); + + EXPECT_EQ(success_cnt, (int)total_line_cnt); + + for (size_t i = 0; i < total_line_cnt; i++) { + maat_cmd_clear_rule_cache(s_rule + i); + } + FREE(s_rule); + redisFree(c); + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis_ip(opts, redis_ip); + maat_options_set_redis_port(opts, redis_port); + maat_options_set_logger(opts, logger); + + g_maat_instance = maat_new(opts, table_info_path); ret=RUN_ALL_TESTS(); - + + log_handle_destroy(g_maat_instance->logger); maat_free(g_maat_instance); return ret; diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 9451b29..29d0e05 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -57,8 +57,7 @@ TEST(iris_mode, maat_scan_string) { int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz); EXPECT_NE(ret, -1); - ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, NULL, - tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL); + ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL); EXPECT_NE(ret, -1); } @@ -158,8 +157,7 @@ TEST(redis_mode, maat_scan_string) { int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); EXPECT_NE(ret, -1); - ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, c, tmp_iris_path, - sizeof(tmp_iris_path), NULL, NULL, logger); + ret = json2iris(json_buff, json_filename, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger); EXPECT_NE(ret, -1); } diff --git a/test/maat_json.json b/test/maat_json.json index 42cf591..7b7bab2 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -21,58 +21,62 @@ { "group_name": "ASN2345", "regions": [ - { - "table_name": "AS_NUMBER", - "table_type": "expr", - "table_content": { - "keywords": "AS2345", - "expr_type": "none", - "match_method": "exact", - "format": "uncase plain" - } - } + { + "table_name": "AS_NUMBER", + "table_type": "expr", + "table_content": { + "keywords": "AS2345", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } ] }, { "group_name": "financial-department-ip", "regions": [ - { - "table_name": "IP_CONFIG", - "table_type": "ip", - "table_content": { - "addr_type": "ipv4", - "src_ip": "192.168.40.88", - "mask_src_ip": "255.255.255.255", - "src_port": "0", - "mask_src_port": "65535", - "dst_ip": "0.0.0.0", - "mask_dst_ip": "255.255.255.255", - "dst_port": "0", - "mask_dst_port": "65535", - "protocol": 6, - "direction": "double" - } - } - ] + { + "table_name": "IP_PLUS_CONFIG", + "table_type": "ip_plus", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "mask", + "src_ip1": "192.168.40.88", + "src_ip2": "255.255.255.255", + "sport_format": "mask", + "src_port1": "0", + "src_port2": "65535", + "daddr_format": "mask", + "dst_ip1": "0.0.0.0", + "dst_ip2": "255.255.255.255", + "dport_format": "mask", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ] }, { - "group_name": "Country-Sparta-IP", - "regions": [ - { - "table_name": "GeoLocation", - "table_type": "expr", - "table_content": { - "keywords": "Greece.Sparta", - "expr_type": "none", - "match_method": "exact", - "format": "uncase plain" - } - } - ] + "group_name": "Country-Sparta-IP", + "regions": [ + { + "table_name": "GeoLocation", + "table_type": "expr", + "table_content": { + "keywords": "Greece.Sparta", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] }, { - "group_name": "IPv4-composition-source-only", - "regions": [ + "group_name": "IPv4-composition-source-only", + "regions": [ { "table_type": "ip_plus", "table_name": "IP_PLUS_CONFIG", @@ -97,37 +101,37 @@ ] }, { - "group_name": "FQDN_OBJ1", - "regions": [ - { - "table_name": "KEYWORDS_TABLE", - "table_type": "expr", - "table_content": { - "keywords": "sports.example.com", - "expr_type": "none", - "match_method": "exact", - "format": "uncase plain" - } - } - ] + "group_name": "FQDN_OBJ1", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "sports.example.com", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] }, { - "group_name": "FQDN_CAT1", - "regions": [ - { - "table_name": "INTERGER_PLUS", - "table_type": "interval_plus", - "table_content": { - "district": "fqdn_cat_id", - "low_boundary": 1724, - "up_boundary": 1724 - } - } - ] + "group_name": "FQDN_CAT1", + "regions": [ + { + "table_name": "INTERGER_PLUS", + "table_type": "interval_plus", + "table_content": { + "district": "fqdn_cat_id", + "low_boundary": 1724, + "up_boundary": 1724 + } + } + ] }, - { - "group_name": "IPv4-composition-NOT-client-ip", - "regions": [ + { + "group_name": "IPv4-composition-NOT-client-ip", + "regions": [ { "table_type": "ip_plus", "table_name": "IP_PLUS_CONFIG", @@ -152,8 +156,8 @@ ] }, { - "group_name": "IPv4-composition-NOT-server-ip", - "regions": [ + "group_name": "IPv4-composition-NOT-server-ip", + "regions": [ { "table_type": "ip_plus", "table_name": "IP_PLUS_CONFIG", @@ -192,35 +196,43 @@ "group_name": "123_IP_group", "regions": [ { - "table_name": "IP_CONFIG", - "table_type": "ip", + "table_name": "IP_PLUS_CONFIG", + "table_type": "ip_plus", "table_content": { "addr_type": "ipv4", - "src_ip": "10.0.6.201", - "mask_src_ip": "255.255.0.0", - "src_port": "0", - "mask_src_port": "65535", - "dst_ip": "0.0.0.0", - "mask_dst_ip": "255.255.255.255", - "dst_port": "0", - "mask_dst_port": "65535", + "saddr_format": "mask", + "src_ip1": "10.0.6.201", + "src_ip2": "255.255.0.0", + "sport_format": "mask", + "src_port1": "0", + "src_port2": "65535", + "daddr_format":"mask", + "dst_ip1": "0.0.0.0", + "dst_ip2": "255.255.255.255", + "dport_format":"mask", + "dst_port1": "0", + "dst_port2": "65535", "protocol": 6, "direction": "double" } }, { - "table_name": "IP_CONFIG", - "table_type": "ip", + "table_name": "IP_PLUS_CONFIG", + "table_type": "ip_plus", "table_content": { "addr_type": "ipv6", - "src_ip": "2001:da8:205:1::101", - "mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000", - "src_port": "0", - "mask_src_port": "65535", - "dst_ip": "0::0", - "mask_dst_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - "dst_port": "0", - "mask_dst_port": "65535", + "saddr_format":"mask", + "src_ip1": "2001:da8:205:1::101", + "src_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000", + "sport_format":"mask", + "src_port1": "0", + "src_port2": "65535", + "daddr_format":"mask", + "dst_ip1": "0::0", + "dst_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "dport_format":"mask", + "dst_port1": "0", + "dst_port2": "65535", "protocol": 6, "direction": "double" } @@ -941,6 +953,7 @@ } ] }, + { "compile_id": 150, "service": 0, @@ -1999,7 +2012,6 @@ } ] }, - { "compile_id": 184, "user_region": "APP_ID=6006740;Liumengyan-Bugreport-20210515", @@ -2015,20 +2027,24 @@ "regions": [ { - "table_name": "IP_CONFIG", - "table_type": "ip", - "table_content": { - "protocol": 0, + "table_name": "IP_PLUS_CONFIG", + "table_type": "ip_plus", + "table_content": { "addr_type": "ipv6", - "direction": "double", - "src_ip": "::", - "dst_ip": "2620:100:3000::", - "src_port": "0", - "dst_port": "0", - "mask_src_port": "65535", - "mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", - "mask_dst_port": "65535", - "mask_dst_ip": "ffff:ffff:ff00:0000:0000:0000:0000:0000" + "saddr_format":"mask", + "src_ip1": "::", + "src_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "sport_format":"mask", + "src_port1": "0", + "src_port2": "65535", + "daddr_format":"mask", + "dst_ip1": "2620:100:3000::", + "dst_ip2": "ffff:ffff:ff00:0000:0000:0000:0000:0000", + "dport_format":"mask", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 0, + "direction": "double" } } ] diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 428d40c..68306c6 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -308,14 +308,13 @@ int main(int argc, char * argv[]) printf("open %s failed.\n", json_file); } - ret = json2iris(json_buff, json_file, NULL, NULL, NULL, c, tmp_iris_path, - sizeof(tmp_iris_path), NULL, NULL, NULL); + ret = json2iris(json_buff, json_file, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger); if (ret < 0) { printf("Invalid json format.\n"); } size_t total_line_cnt = 0; - config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL); + config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger); printf("Serialize %s to %zu lines, write temp file to %s .\n", json_file, total_line_cnt, tmp_iris_path); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); @@ -328,13 +327,13 @@ int main(int argc, char * argv[]) absolute_expire_time = server_time + timeout; } - config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, NULL); + config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, logger); printf("Timeout = %lld\n", absolute_expire_time); ret = 0; int success_cnt = 0; do { - success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, NULL); + success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger); } while(success_cnt < 0); if (success_cnt != (int)total_line_cnt) { diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index aa549f0..78859fa 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -82,4 +82,21 @@ file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) add_library(hiredis-static STATIC IMPORTED GLOBAL) add_dependencies(hiredis-static hiredis) set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a) -set_property(TARGET hiredis-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) \ No newline at end of file +set_property(TARGET hiredis-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) + +#igraph-0.7.1 +ExternalProject_Add(igraph PREFIX igraph + URL ${CMAKE_CURRENT_SOURCE_DIR}/igraph-0.7.1.tar.gz + URL_MD5 4f6e7c16b45fce8ed423516a9786e4e8 + CONFIGURE_COMMAND ./configure --prefix= --disable-shared --with-pic + BUILD_COMMAND make + INSTALL_COMMAND make install + BUILD_IN_SOURCE 1) + +ExternalProject_Get_Property(igraph INSTALL_DIR) +file(MAKE_DIRECTORY ${INSTALL_DIR}/include) + +add_library(igraph-static STATIC IMPORTED GLOBAL) +add_dependencies(igraph-static igraph) +set_property(TARGET igraph-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libigraph.a) +set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) \ No newline at end of file diff --git a/vendor/igraph-0.7.1.tar.gz b/vendor/igraph-0.7.1.tar.gz new file mode 100644 index 0000000..9340f6b Binary files /dev/null and b/vendor/igraph-0.7.1.tar.gz differ