diff --git a/deps/log/log.c b/deps/log/log.c index ac82b33..f21813e 100644 --- a/deps/log/log.c +++ b/deps/log/log.c @@ -136,7 +136,8 @@ static int log_create_path(const char *file_path) { return 0; } - + fclose(fp); + return 1; } diff --git a/include/maat/maat.h b/include/maat/maat.h index 83def00..623167c 100644 --- a/include/maat/maat.h +++ b/include/maat/maat.h @@ -27,11 +27,6 @@ extern "C" /* maat instance handle */ struct maat; -enum ip_type { - IP_TYPE_V4, - IP_TYPE_V6 -}; - struct maat_hit_path { int Nth_scan; int item_id; @@ -41,7 +36,7 @@ struct maat_hit_path { int compile_id; }; -struct maat_matched { +struct maat_hit_object { int virtual_table_id; int group_id; }; @@ -49,30 +44,17 @@ struct maat_matched { enum maat_scan_opt { MAAT_SET_SCAN_DISTRICT = 1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default. - MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination. - MAAT_GET_SCAN_HIT_PATH, //VALUE is struct maat_hit_path*, an array of struct maat_hit_path, SIZE= sizeof(struct maat_hit_path)*N, + MAAT_SET_SCAN_LAST_ITEM, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination. + MAAT_SET_SCAN_COMPILE_TABLE_ID, //Caller can specify which compile table to use + MAAT_GET_SCAN_HIT_PATH, //VALUE is struct maat_hit_path*, an array of struct maat_hit_path, SIZE= sizeof(struct maat_hit_path)*N, //Maat_get_scan_status returns actual got number. - MAAT_GET_SCAN_MATCHED //VALUE is struct maat_matched*, an array of struct maat_matched, SIZE= sizeof(struct maat_matched)*N, + MAAT_GET_SCAN_HIT_OBJECTS //VALUE is struct maat_hit_object*, an array of struct maat_hit_object, SIZE= sizeof(struct maat_hit_object)*N, }; -/* network order */ -struct ipv4_2tuple { - uint32_t sip; - uint32_t dip; -}; - -struct ipv6_2tuple { - uint8_t sip[16]; - uint8_t dip[16]; -}; - -struct addr_2tuple { - enum ip_type type; - union { - struct ipv4_2tuple ipv4; - struct ipv6_2tuple ipv6; - }; -}; +#define MAAT_OK 0 //scan but not hit(group or compile) +#define MAAT_ERR -1 //scan error +#define MAAT_HALF_HIT 1 //half hit: hit group, not hit compile +#define MAAT_HIT 2 //scan hit compile #define MAAT_RULE_UPDATE_TYPE_FULL 1 #define MAAT_RULE_UPDATE_TYPE_INC 2 @@ -93,8 +75,11 @@ typedef void maat_rule_ex_dup_func_t(int idx, void *to, void *from, long argl, v /* maat_instance options API */ struct maat_options; -struct maat_options* maat_options_new(void); +struct maat_options *maat_options_new(void); +void maat_options_free(struct maat_options *opts); + int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread); +int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags); int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms); int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms); int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms); @@ -110,8 +95,9 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path); void maat_free(struct maat *instance); /* maat table API */ +/* return table_id(>=0) if success,otherwise return -1 */ int maat_table_get_id(struct maat *instance, const char *table_name); - +/* return 0 if success, otherwise return -1 */ int maat_table_callback_register(struct maat *instance, int table_id, maat_start_callback_t *start, maat_update_callback_t *update, @@ -129,17 +115,33 @@ void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id, const char *key, size_t key_len); /* maat scan API */ struct maat_state; -int maat_scan_integer(struct maat *instance, int table_id, int thread_id, - unsigned int intval, int results[], size_t *n_result, - struct maat_state **state); -int maat_scan_ip(struct maat *instance, int table_id, int thread_id, - struct addr_2tuple *addr, int results[], size_t *n_result, - struct maat_state **state); +/** + * @param instance: maat instance created by maat_new() + * @param table_id: the id of table which to be scanned + * @param thread_id: thread index + * @param results: array to store hit compile id + * @param n_result: the array size + * @param n_hit_result: the number of hit compile id + * @param state: scan mid status + * + * @retval MAAT_ERR/MAAT_OK/MAAT_HALF_HIT +*/ +int maat_scan_integer(struct maat *instance, int table_id, int thread_id, + unsigned int intval, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state); + +int maat_scan_ipv4(struct maat *instance, int table_id, int thread_id, + uint32_t ip_addr, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state); + +int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id, + uint8_t *ip_addr, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state); int maat_scan_string(struct maat *instance, int table_id, int thread_id, - const char *data, size_t data_len, int results[], size_t *n_result, - struct maat_state **state); + const char *data, size_t data_len, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state); struct maat_stream; struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id); @@ -150,15 +152,20 @@ int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* dat void maat_scan_stream_close(struct maat_stream **stream); /* maat state API */ -int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, const void *value, int size); +int maat_state_set_scan_district(struct maat *instance, struct maat_state **state, const char *district, size_t district_len); -//return >=0 if success, return -1 when failed; -int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, void *value, int size); +int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state); + +int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, int compile_table_id); + +int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path); + +int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj); void maat_state_free(struct maat_state **state); -/* return matched compile_id */ -int maat_matched_compile_id(struct maat *instance, struct maat_matched *matched); +/* return hit object compile_id */ +int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj); #ifdef __cpluscplus } diff --git a/scanner/CMakeLists.txt b/scanner/CMakeLists.txt index f472437..3e1fb3b 100644 --- a/scanner/CMakeLists.txt +++ b/scanner/CMakeLists.txt @@ -3,6 +3,7 @@ add_definitions(-fPIC) include_directories(${PROJECT_SOURCE_DIR}/deps) include_directories(${PROJECT_SOURCE_DIR}/include) +include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) add_library(adapter-static adapter_hs.cpp bool_matcher.cpp) diff --git a/scanner/adapter_hs.cpp b/scanner/adapter_hs.cpp index 3205358..fc3c8b9 100644 --- a/scanner/adapter_hs.cpp +++ b/scanner/adapter_hs.cpp @@ -17,8 +17,11 @@ #include "uthash/utarray.h" #include "uthash/uthash.h" #include "utils.h" +#include "maat_utils.h" #include "bool_matcher.h" +#define MODULE_ADAPTER_HS module_name_str("maat.adapter_hs") + struct adpt_hs_compile_data { unsigned int *ids; unsigned int *flags; @@ -56,7 +59,8 @@ struct adapter_hs_stream { UT_array *pattern_id_set; }; -int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_threads, int max_pattern_type) +static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_threads, int max_pattern_type, + struct log_handle *logger) { hs_database_t *database = NULL; hs_rt->scratchs = ALLOC(hs_scratch_t *, nr_worker_threads); @@ -68,7 +72,7 @@ int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_thr } if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) { - fprintf(stderr, "ERROR: Unable to allocate scratch space. Exiting.\n"); + log_error(logger, MODULE_ADAPTER_HS, "ERROR: Unable to allocate scratch space. Exiting."); hs_free_database(database); return -1; } @@ -76,14 +80,14 @@ int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_thr for (size_t i = 1; i < nr_worker_threads; i++) { hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]); if (err != HS_SUCCESS) { - fprintf(stderr, "Unable to clone scratch prototype\n"); + log_error(logger, MODULE_ADAPTER_HS, "Unable to clone scratch prototype"); hs_free_database(database); return -1; } err = hs_scratch_size(hs_rt->scratchs[i], &hs_rt->scratch_size); if (err != HS_SUCCESS) { - fprintf(stderr, "Unable to query scratch size\n"); + log_error(logger, MODULE_ADAPTER_HS, "Unable to query scratch size"); hs_free_database(database); return -1; } @@ -97,10 +101,10 @@ int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_thr * * @retval 0(success) -1(failed) */ -int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, - struct adpt_hs_compile_data *literal_cd, - struct adpt_hs_compile_data *regex_cd, - int scan_mode) +static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, + struct adpt_hs_compile_data *literal_cd, + struct adpt_hs_compile_data *regex_cd, + int scan_mode, struct log_handle *logger) { hs_error_t err; hs_compile_error_t *compile_err = NULL; @@ -115,7 +119,7 @@ int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, scan_mode, NULL, &hs_rt->literal_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { - fprintf(stderr, "%s compile error: %s\n", __func__, compile_err->message); + log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message); } hs_free_compile_error(compile_err); @@ -129,7 +133,7 @@ int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, scan_mode, NULL, &hs_rt->regex_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { - fprintf(stderr, "%s compile error: %s\n", __func__, compile_err->message); + log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message); } hs_free_compile_error(compile_err); goto error; @@ -182,11 +186,12 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt FREE(hs_cd); } -struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array) +struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array, + struct log_handle *logger) { if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) || 0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) { - fprintf(stderr, "%s input parameters illegal!\n", __func__); + log_error(logger, MODULE_ADAPTER_HS, "%s input parameters illegal!", __func__); return NULL; } @@ -196,7 +201,8 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads for (size_t i = 0; i < n_expr_array; i++) { if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) { - fprintf(stderr, "the number of patterns in one expression should less than %d\n", MAX_EXPR_PATTERN_NUM); + log_error(logger, MODULE_ADAPTER_HS, + "the number of patterns in one expression should less than %d", MAX_EXPR_PATTERN_NUM); return NULL; } @@ -206,7 +212,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads } else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) { regex_pattern_num++; } else { - fprintf(stderr, "unknown pattern type: %d\n", expr_array[i].patterns[j].type); + log_error(logger, MODULE_ADAPTER_HS, "unknown pattern type: %d", expr_array[i].patterns[j].type); return NULL; } } @@ -285,16 +291,17 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads /* create bool matcher */ hs_instance->hs_rt->bm = bool_matcher_new(exprs, n_expr_array, &mem_size); if (hs_instance->hs_rt->bm != NULL) { - fprintf(stdout, "Adapter_hs module: build bool matcher of %zu expressions with %zu bytes memory\n", - n_expr_array, mem_size); + log_info(logger, MODULE_ADAPTER_HS, + "Adapter_hs module: build bool matcher of %zu expressions with %zu bytes memory", + n_expr_array, mem_size); } else { - fprintf(stderr, "Adapter_hs module: build bool matcher failed\n"); + log_error(logger, MODULE_ADAPTER_HS, "Adapter_hs module: build bool matcher failed"); goto error; } FREE(exprs); /* build hs database */ - ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode); + ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode, logger); if (ret < 0) { goto error; } @@ -314,7 +321,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads max_patterns_type = PATTERN_TYPE_REG; } - ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type); + ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type, logger); if (ret < 0) { goto error; } diff --git a/scanner/adapter_hs.h b/scanner/adapter_hs.h index 1679b61..a484263 100644 --- a/scanner/adapter_hs.h +++ b/scanner/adapter_hs.h @@ -19,6 +19,8 @@ extern "C" #include #include +#include "log/log.h" + #define MAX_EXPR_PATTERN_NUM 8 struct adapter_hs; @@ -63,7 +65,7 @@ typedef struct { * * @retval the pointer to adapter_hs instance */ -struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array); +struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array, struct log_handle *logger); /** * @brief scan input data to match logic AND expression, return all matched expr_id diff --git a/scanner/adapter_hs_gtest.cpp b/scanner/adapter_hs_gtest.cpp index da27127..96e5c99 100644 --- a/scanner/adapter_hs_gtest.cpp +++ b/scanner/adapter_hs_gtest.cpp @@ -2,7 +2,6 @@ #include "utils.h" #include "adapter_hs.h" -#include "maat_table_schema.h" int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr) { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4aa78c9..df304cc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,8 +9,9 @@ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp - maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp - maat_command.cpp maat_redis_monitor.cpp maat_hierarchy.cpp) + maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp + maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp + maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_virtual.cpp) set(LIB_SOURCE_FILES ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c) @@ -29,14 +30,14 @@ set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z) # Shared Library Output -add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) -set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX) -set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe) -set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) -set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}) -set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION}) +#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) +#set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX) +#set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe) +#set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) +#set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}) +#set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION}) -target_link_libraries(maat_frame_shared hyperscan_static) +#target_link_libraries(maat_frame_shared hyperscan_static) # install set(CMAKE_INSTALL_PREFIX /opt/MESA/) diff --git a/src/inc_internal/maat_common.h b/src/inc_internal/maat_common.h index 17646c2..f05ae81 100644 --- a/src/inc_internal/maat_common.h +++ b/src/inc_internal/maat_common.h @@ -23,8 +23,10 @@ extern "C" struct maat_options { char instance_name[NAME_MAX]; - char compile_tablename[NAME_MAX]; size_t nr_worker_threads; + + const char *accept_tags; + int rule_effect_interval_ms; int rule_update_checking_interval_ms; int gc_timeout_ms; diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h new file mode 100644 index 0000000..ced29c2 --- /dev/null +++ b/src/inc_internal/maat_compile.h @@ -0,0 +1,90 @@ +/* +********************************************************************************************** +* File: maat_compile.h +* Description: +* Authors: Zheng Chao +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_COMPILE_H_ +#define _MAAT_COMPILE_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include "log/log.h" +#include "cJSON/cJSON.h" +#include "maat.h" +#include "maat_kv.h" +#include "maat_rule.h" + +struct compile_ex_data_schema { + maat_rule_ex_new_func_t *new_func; + maat_rule_ex_free_func_t *free_func; + maat_rule_ex_dup_func_t *dup_func; + long argl; + void *argp; + int idx; + int table_id; +}; + +/* compile schema API */ +void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void compile_schema_free(void *compile_schema); + +void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void group2compile_schema_free(void *g2c_schema); + +int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id, + maat_rule_ex_new_func_t *new_func, + maat_rule_ex_free_func_t *free_func, + maat_rule_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger); +struct compile_ex_data_schema * +compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, size_t idx); + +size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema); + +/* compile runtime API */ +void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void compile_runtime_free(void *compile_runtime); + +int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, + int valid_column); +int compile_runtime_commit(void *compile_runtime); + +int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids, + int *compile_ids, size_t compile_ids_size, struct maat_state *state); + +size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct maat_group_topology *group_topo, + struct maat_compile_state *compile_state, + struct maat_hit_path *hit_paths, size_t hit_path_siz); +/* group2compile runtime API */ +void *group2compile_runtime_new(void *g2c_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void group2compile_runtime_free(void *g2c_runtime); + +int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line, + int valid_column); + +/* maat compile state API */ +struct maat_compile_state; +struct maat_compile_state *maat_compile_state_new(int thread_id); +void maat_compile_state_free(struct maat_compile_state *compile_state); +void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id, + int Nth_scan, int Nth_item_result); +void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, struct maat_compile **compile_hash, + int group_id, int virtual_table_id); +int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_ex_data.h b/src/inc_internal/maat_ex_data.h index f269bca..0a641dc 100644 --- a/src/inc_internal/maat_ex_data.h +++ b/src/inc_internal/maat_ex_data.h @@ -16,6 +16,7 @@ extern "C" { #endif +#include "maat/maat.h" #include "rcu_hash.h" struct ex_data_container { @@ -23,6 +24,12 @@ struct ex_data_container { void *custom_data; }; +struct ex_container_ctx { + int table_id; + void (*custom_data_free)(void *custom_data); + struct ex_data_schema *ex_schema; +}; + struct ex_data_runtime; /* ex_data_runtime API */ @@ -42,6 +49,12 @@ size_t ex_data_runtime_cached_row_count(struct ex_data_runtime *ex_data_rt); void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt); /* set schema API */ +struct ex_data_schema *ex_data_schema_new(maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp); +void ex_data_schema_free(struct ex_data_schema *ex_schema); + void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema); /* set user_ctx API */ @@ -49,16 +62,24 @@ void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, st struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt); +struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data); +void ex_data_container_free(void *ctx, void *data); + /* ex_data_runtime ex data API */ -void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len); +void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, + const char *row, const char *key, size_t key_len); -void ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct ex_data_container *ex_container); +int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len, + struct ex_data_container *ex_container); -void ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct log_handle *logger); +int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len); -size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container); +size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, + struct ex_data_container ***ex_container); -void *ex_data_runtime_dup_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); +void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h new file mode 100644 index 0000000..e6b0ba1 --- /dev/null +++ b/src/inc_internal/maat_expr.h @@ -0,0 +1,54 @@ +/* +********************************************************************************************** +* File: maat_expr.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_EXPR_H_ +#define _MAAT_EXPR_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include "log/log.h" +#include "cJSON/cJSON.h" + +struct expr_runtime; + +void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void expr_schema_free(void *expr_schema); + +/* expr runtime API */ +void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); +void expr_runtime_free(void *expr_runtime); + +int expr_runtime_updating_flag(struct expr_runtime *expr_rt); +int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column); +int expr_runtime_commit(void *expr_runtime); + +/* expr runtime scan API */ +/** + * @brief scan string to get hit group_ids + * + * @retval the num of hit group_id +*/ +int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, + const char *data, size_t data_len, int group_ids[], + size_t max_hit_num, int virtual_table_id, + struct maat_state *state); + +void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); +int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, int results[], size_t *n_result); +void expr_runtime_stream_close(struct expr_runtime *expr_rt); + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/src/inc_internal/maat_group.h b/src/inc_internal/maat_group.h index 511b4f9..cb7479c 100644 --- a/src/inc_internal/maat_group.h +++ b/src/inc_internal/maat_group.h @@ -16,15 +16,10 @@ extern "C" { #endif -struct maat_item { - int item_id; - int group_id; - - struct maat_group *ref_parent_group; - UT_hash_handle hh; - - void *user_data; -}; +#include "cJSON/cJSON.h" +#include "uthash/uthash.h" +#include "igraph/igraph.h" +#include "maat_kv.h" struct maat_group { igraph_integer_t vertex_id; @@ -32,16 +27,16 @@ struct maat_group { int ref_by_compile_cnt; int ref_by_superior_group_cnt; int ref_by_subordinate_group_cnt; - int ref_by_item_cnt; size_t top_group_cnt; int *top_group_ids; + UT_hash_handle hh_group_id; UT_hash_handle hh_vertex_id; }; +/* maat group topology API */ struct maat_group_topology; - struct maat_group_topology *maat_group_topology_new(struct log_handle *logger); void maat_group_topology_free(struct maat_group_topology *group_topo); @@ -59,6 +54,23 @@ int maat_group_topology_remove_group_from_group(struct maat_group_topology *grou /* build top groups */ int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo); + +/* group2group schema API */ +void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void group2group_schema_free(void *g2g_schema); + +/* group2group runtime API */ +struct group2group_runtime * +group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void group2group_runtime_free(void *g2g_runtime); + +int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column); +int group2group_runtime_commit(void *g2g_runtime); + +int group2group_runtime_get_top_groups(struct group2group_runtime *g2g_rt, int *group_ids, size_t n_group_ids, + int *top_group_ids); + #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_hierarchy.h b/src/inc_internal/maat_hierarchy.h deleted file mode 100644 index 44f4121..0000000 --- a/src/inc_internal/maat_hierarchy.h +++ /dev/null @@ -1,67 +0,0 @@ -/* -********************************************************************************************** -* File: maat_hierarchy.h -* Description: -* Authors: Zheng Chao -* Date: 2022-10-31 -* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. -*********************************************************************************************** -*/ - -#ifndef _MAAT_HIERARCHY_H_ -#define _MAAT_HIERARCHY_H_ - -#ifdef __cpluscplus -extern "C" -{ -#endif - -#include "maat_garbage_collection.h" - -/* maat hierarchy API */ -struct maat_hierarchy; -struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger); -void maat_hierarchy_free(struct maat_hierarchy *hier); -int maat_hierarchy_rebuild(struct maat_hierarchy *hier); - -size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - struct maat_hit_path *hit_paths, size_t n_path); - -/* maat hierarchy compile mid API */ -struct maat_hierarchy_compile_mid; -struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id); -void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid); -void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - int item_id, int virtual_table_id, int Nth_scan, int Nth_item_result); -int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid); - -/* maat hierarchy compile API */ -int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, int declared_clause_num, void* user_data); -int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id); - -void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)); -void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id); -void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id); -void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param); - -/* maat hierarchy group2compile API */ -int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag, - int clause_index, int compile_id); -int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, int not_flag, - int clause_index, int compile_id); - -/* maat hierarchy item API */ -int maat_hierarchy_item_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - int is_last_compile, void **user_data_array, size_t ud_array_sz); -void *maat_hierarchy_item_dettach_user_data(struct maat_hierarchy *hier, int item_id); -void maat_hierarchy_set_item_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)); - -/* maat hierarchy item2group API */ -int maat_hierarchy_add_item_to_group(struct maat_hierarchy *hier, int group_id, int item_id, void* user_data); -int maat_hierarchy_remove_item_from_group(struct maat_hierarchy *hier, int group_id, int item_id); - -#ifdef __cpluscplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h new file mode 100644 index 0000000..91651f8 --- /dev/null +++ b/src/inc_internal/maat_ip.h @@ -0,0 +1,42 @@ +/* +********************************************************************************************** +* File: maat_ip.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_IP_H_ +#define _MAAT_IP_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +struct ip_plus_runtime; + +void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void ip_plus_schema_free(void *ip_plus_schema); + +/* ip plus runtime API */ +void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); +void ip_plus_runtime_free(void *ip_plus_runtime); + +int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line); +int ip_plus_runtime_commit(void *ip_plus_runtime); + +struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt); + +/* ip runtime scan API */ +int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *data, + int *group_id_array, size_t n_group_id_array, int virtual_table_id, + struct maat_state *state); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_ip_plugin.h b/src/inc_internal/maat_ip_plugin.h new file mode 100644 index 0000000..39f9fc5 --- /dev/null +++ b/src/inc_internal/maat_ip_plugin.h @@ -0,0 +1,49 @@ +/* +********************************************************************************************** +* File: maat_ip_plugin.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_IP_PLUGIN_H_ +#define _MAAT_IP_PLUGIN_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +struct ip_plugin_runtime; + +/* ip plugin schema API */ +void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void ip_plugin_schema_free(void *ip_plugin_schema); + +/* ip plugin table ex data API */ +struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema); + +int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp); + +/* ip plugin runtime API */ +void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void ip_plugin_runtime_free(void *ip_plugin_runtime); + +int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt); +int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line); +int ip_plugin_runtime_commit(void *ip_plugin_runtime); + +struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_kv.h b/src/inc_internal/maat_kv.h index 7133b2d..c8108d0 100644 --- a/src/inc_internal/maat_kv.h +++ b/src/inc_internal/maat_kv.h @@ -18,13 +18,17 @@ extern "C" struct maat_kv_store; -struct maat_kv_store* maat_kv_store_new(void); +struct maat_kv_store *maat_kv_store_new(void); -void maat_kv_store_free(struct maat_kv_store* store); +void maat_kv_store_free(struct maat_kv_store *store); -int maat_kv_register(struct maat_kv_store* store, const char* key, int value); +int maat_kv_register(struct maat_kv_store *store, const char *key, int value); -int maat_kv_read(struct maat_kv_store* store, const char* key, int* value); +int maat_kv_read(struct maat_kv_store *store, const char *key, int *value); + +int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, int *value); + +struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_limits.h b/src/inc_internal/maat_limits.h new file mode 100644 index 0000000..517c900 --- /dev/null +++ b/src/inc_internal/maat_limits.h @@ -0,0 +1,25 @@ +/* +********************************************************************************************** +* File: maat_limits.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_LIMITS_H_ +#define _MAAT_LIMITS_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#define MAX_KEYWORDS_STR 1024 + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_plugin.h b/src/inc_internal/maat_plugin.h new file mode 100644 index 0000000..3b00153 --- /dev/null +++ b/src/inc_internal/maat_plugin.h @@ -0,0 +1,60 @@ +/* +********************************************************************************************** +* File: maat_plugin.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_PLUGIN_H_ +#define _MAAT_PLUGIN_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#define MAX_FOREIGN_CLMN_NUM 8 + +/* plugin schema API */ +void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void plugin_schema_free(void *plugin_schema); + +struct plugin_schema; +/* plugin table callback API */ +int plugin_table_add_callback(void *plugin_schema, int table_id, + maat_start_callback_t *start, + maat_update_callback_t *update, + maat_finish_callback_t *finish, + void *u_para, struct log_handle *logger); + +void plugin_table_all_callback_start(struct plugin_schema *plugin_schema, int update_type); +void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema); + +int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns); + +/* plugin table ex data API */ +int plugin_table_set_ex_data_schema(void *custom_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp); +struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema); + +/* plugin runtime API */ +struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime); + +int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt); + +int plugin_runtime_update(struct plugin_runtime *plugin_rt, struct plugin_item *plugin_item, + struct plugin_schema *plugin_schema, const char *row, + struct log_handle *logger); +int plugin_runtime_commit(void *plugin_runtime); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index b4e22e6..d866eed 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -27,9 +27,12 @@ extern "C" #include "log/log.h" #include "hiredis/hiredis.h" #include "uthash/uthash.h" -#include "maat_table_schema.h" #include "maat_command.h" #include "IPMatcher.h" +#include "maat_kv.h" +#include "maat_table.h" + +#define MAX_TABLE_NUM 256 struct maat_rule_head { int config_id; @@ -64,8 +67,15 @@ struct maat_item_inner { int expr_id_ub; //up boundary }; +struct maat_item { + int item_id; + int group_id; + UT_hash_handle hh; + void *user_data; +}; + #define COMPILE_RULE_MAGIC 0x1a2b3c4d -struct maat_compile_rule { +struct compile_rule { long long magic_num; int compile_id; struct maat_rule_head head;// fix len of Maat_rule_t @@ -73,7 +83,7 @@ struct maat_compile_rule { int is_valid; int declared_clause_num; double evaluation_order; - struct table_schema *ref_table; + struct compile_schema *ref_table; void **ex_data; pthread_rwlock_t rwlock; }; @@ -83,8 +93,9 @@ struct group2compile_rule { int compile_id; int is_valid; int not_flag; - int virtual_table_id; + int vt_id; //virtual_table_id int clause_index; + int associated_compile_table_id; }; struct group2group_rule { @@ -100,14 +111,19 @@ struct maat_runtime { time_t last_update_time; long long *ref_cnt; - struct table_runtime_manager *table_rt_mgr; + struct table_manager *ref_tbl_mgr; //share with maat instance size_t max_table_num; size_t max_thread_num; uint32_t rule_num; struct maat_garbage_bin *ref_garbage_bin; - struct scan_result *item_result_buff; + + struct maat_kv_store *district_map; + struct maat_kv_store *tmp_district_map; + + unsigned int district_num; + struct log_handle *logger; }; @@ -179,7 +195,7 @@ struct maat { struct maat_runtime *maat_rt; struct maat_runtime *creating_maat_rt; - struct table_schema_manager *table_schema_mgr; + struct table_manager *tbl_mgr; enum data_source input_mode; union { @@ -187,9 +203,6 @@ struct maat { struct source_json_ctx json_ctx; struct source_redis_ctx mr_ctx; }; - - struct rule_tag *accept_tags; - int n_accept_tag; struct log_handle *logger; int deferred_load; @@ -210,10 +223,8 @@ struct maat { struct maat_garbage_bin *garbage_bin; - char compile_tablename[NAME_MAX]; - //char group_tn[NAME_MAX]; - //char group2compile_tn[NAME_MAX]; - //char group2group_tn[NAME_MAX]; + int default_compile_table_id; + int g2g_table_id; //group2group table id char decrypt_key[NAME_MAX]; char decrypt_algo[NAME_MAX]; @@ -232,6 +243,29 @@ struct maat { long long scan_err_cnt; }; +struct maat_state { + struct maat *maat_instance; + int16_t thread_id; + int compile_table_id; //caller can select compile table to scan + unsigned char is_set_district; + unsigned char is_last_scan; + int district_id; //-1: Any District; -2: Unkonwn District; + int scan_cnt; + struct maat_compile_state *compile_mid; +}; + +int parse_accept_tag(const char *value, struct rule_tag **result, void *logger); + +int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag); + +struct maat_item *maat_item_new(int item_id, int group_id, void *user_data); + +void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)); + +struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id); + +void maat_item_inner_free(struct maat_item_inner *item); + void maat_start_cb(long long new_version, int update_type, void *u_para); int maat_update_cb(const char *table_name, const char *line, void *u_para); diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h new file mode 100644 index 0000000..ba4938a --- /dev/null +++ b/src/inc_internal/maat_table.h @@ -0,0 +1,68 @@ +/* +********************************************************************************************** +* File: maat_table.h +* Description: maat table schema and runtime +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_TABLE_H_ +#define _MAAT_TABLE_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include +#include + +enum table_type { + TABLE_TYPE_INVALID = -1, + TABLE_TYPE_EXPR = 0, + TABLE_TYPE_EXPR_PLUS, + TABLE_TYPE_IP_PLUS, + TABLE_TYPE_INTERVAL, + TABLE_TYPE_INTERVAL_PLUS, + TABLE_TYPE_DIGEST, + TABLE_TYPE_SIMILARITY, + TABLE_TYPE_CONJUNCTION, + TABLE_TYPE_PLUGIN, + TABLE_TYPE_IP_PLUGIN, + TABLE_TYPE_FQDN_PLUGIN, + TABLE_TYPE_BOOL_PLUGIN, + //above are physical table + TABLE_TYPE_VIRTUAL, + TABLE_TYPE_COMPILE, + TABLE_TYPE_GROUP2GROUP, + TABLE_TYPE_GROUP2COMPILE, + TABLE_TYPE_MAX +}; + +struct table_manager; + +struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags, + struct log_handle *logger); +int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *garbage_bin); +int table_manager_deinit(struct table_manager *tbl_mgr); +void table_manager_destroy(struct table_manager *tbl_mgr); + +size_t table_manager_table_count(struct table_manager *tbl_mgr); +int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name); +enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id); +int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id); +int table_manager_accept_tags_match(const char *tags); + +void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); + +void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); +void table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line); +void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_table_runtime.h b/src/inc_internal/maat_table_runtime.h deleted file mode 100644 index fa59ae2..0000000 --- a/src/inc_internal/maat_table_runtime.h +++ /dev/null @@ -1,86 +0,0 @@ -/* -********************************************************************************************** -* File: maat_table_runtime.h -* Description: maat table runtime entry -* Authors: Liu WenTan -* Date: 2022-10-31 -* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. -*********************************************************************************************** -*/ - -#ifndef _MAAT_TABLE_RUNTIME_H_ -#define _MAAT_TABLE_RUNTIME_H_ - -#ifdef __cpluscplus -extern "C" -{ -#endif - -#include "maat/maat.h" -#include "maat_table_schema.h" -#include "maat_garbage_collection.h" - -struct ip_addr { - enum ip_type ip_type; - union { - uint32_t ipv4; - uint32_t ipv6[4]; - }; -}; - -struct table_item; -struct table_runtime; -struct table_runtime_manager; - -/* table runtime manager API */ -struct table_runtime_manager * -table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num, - struct maat_garbage_bin *bin, struct log_handle *logger); - -void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr); - -/* table runtime API */ -struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id); - -size_t table_runtime_rule_count(struct table_runtime *table_rt); - -enum table_type table_runtime_get_type(struct table_runtime* table_rt); - -void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, - const char *line, struct table_item *table_item, struct log_handle *logger); - -/** - * @brief if table_runtime is updating - * - * @retval 1(yes) 0(no) -*/ -int table_runtime_updating_flag(struct table_runtime *table_rt); - -void table_runtime_commit(struct table_runtime *table_rt, long long version, size_t nr_worker_thread, struct log_handle *logger); - -/* table runtime scan API */ -int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, const char *data, size_t data_len, - int results[], size_t *n_result); - -void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id); -int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result); -void table_runtime_stream_close(struct table_runtime *table_rt); - -int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct ip_addr *data, struct scan_result *results, size_t *n_result, size_t n_result_array); - -/* table runtime cached row API */ -size_t table_runtime_cached_row_count(struct table_runtime *table_rt); - -const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq); - -/* table runtime ex data API */ -struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt); - -void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, - int nr_worker_thread, long long version, struct log_handle *logger); - -#ifdef __cpluscplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/inc_internal/maat_table_schema.h b/src/inc_internal/maat_table_schema.h deleted file mode 100644 index 0a4c8d9..0000000 --- a/src/inc_internal/maat_table_schema.h +++ /dev/null @@ -1,310 +0,0 @@ -/* -********************************************************************************************** -* File: maat_table_schema.h -* Description: maat table schema entry -* Authors: Liu WenTan -* Date: 2022-10-31 -* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. -*********************************************************************************************** -*/ - -#ifndef _MAAT_TABLE_SCHEMA_H_ -#define _MAAT_TABLE_SCHEMA_H_ - -#ifdef __cpluscplus -extern "C" -{ -#endif - -#include -#include - -#include "maat/maat.h" -#include "adapter_hs.h" - -#define MAX_TABLE_NUM 256 -#define MAX_DISTRICT_STR 128 -#define MAX_IP_STR 128 -#define MAX_KEYWORDS_STR 1024 -#define MAX_FOREIGN_CLMN_NUM 8 -#define MAX_TABLE_LINE_SIZE (1024 * 16) -#define MAX_COMPILE_EX_DATA_NUM 2 - -enum component_table_type { - COMPONENT_TABLE_TYPE_NONE = -1, - COMPONENT_TABLE_TYPE_SIP = 0, - COMPONENT_TABLE_TYPE_DIP, - COMPONENT_TABLE_TYPE_SESSION, - COMPONENT_TABLE_TYPE_MAX -}; - -enum table_type { - TABLE_TYPE_INVALID = -1, - TABLE_TYPE_EXPR = 0, - TABLE_TYPE_EXPR_PLUS, - TABLE_TYPE_IP_PLUS, - TABLE_TYPE_INTERVAL, - TABLE_TYPE_INTERVAL_PLUS, - TABLE_TYPE_DIGEST, - TABLE_TYPE_SIMILARITY, - TABLE_TYPE_PLUGIN, - TABLE_TYPE_IP_PLUGIN, - TABLE_TYPE_FQDN_PLUGIN, - TABLE_TYPE_BOOL_PLUGIN, - //above are physical table - TABLE_TYPE_VIRTUAL, - TABLE_TYPE_COMPOSITION, - TABLE_TYPE_COMPILE, - TABLE_TYPE_GROUP, - TABLE_TYPE_GROUP2GROUP, - TABLE_TYPE_GROUP2COMPILE -}; - -enum expr_type { - EXPR_TYPE_STRING = 0, - EXPR_TYPE_AND, - EXPR_TYPE_REGEX, - EXPR_TYPE_MAX -}; - -enum scan_type { - SCAN_TYPE_INVALID = -1, - SCAN_TYPE_NONE = 0, - SCAN_TYPE_PLUGIN, - SCAN_TYPE_IP_PLUGIN, - SCAN_TYPE_FQDN_PLUGIN, - SCAN_TYPE_BOOL_PLUGIN, - SCAN_TYPE_IP, - SCAN_TYPE_INTERVAL, - SCAN_TYPE_STRING, - SCAN_TYPE_MAX -}; - -enum match_method { - MATCH_METHOD_SUB = 0, - MATCH_METHOD_RIGHT, - MATCH_METHOD_LEFT, - MATCH_METHOD_COMPLETE, - MATCH_METHOD_MAX -}; - -struct compile_item { - int compile_id; - int service_id; - int action; - int do_blacklist; - int do_log; - char user_region[MAX_TABLE_LINE_SIZE]; - int is_valid; - int clause_num; - int evaluation_order; -}; - -struct group2compile_item { - int group_id; - int compile_id; - int is_valid; - int not_flag; - int virtual_table_id; - int clause_index; - int associated_compile_table_id; -}; - -struct group2group_item { - int group_id; - int superior_group_id; - int is_valid; -}; - -struct expr_item { - int item_id; - int group_id; - char district[MAX_DISTRICT_STR]; - char keywords[MAX_KEYWORDS_STR]; - enum expr_type expr_type; - enum match_method match_method; - int is_hexbin; - int is_case_sensitive; - int is_valid; -}; - -struct ipv4_item_rule { - uint32_t min_sip; /* 源地址下界；0表示忽略本字段 */ - uint32_t max_sip; /* 源地址上界；0表示固定IP=min_saddr */ - uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */ - uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */ - uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP；0表示忽略本字段 */ - uint16_t direction; /* 方向，0表示双向，1表示单向 */ -}; - -struct ipv6_item_rule { - uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */ - uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */ - uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */ - uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */ - uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP，无限制默认为0 */ - uint16_t direction; /* 方向，0表示双向，1表示单向 */ -}; - -struct ip_plus_item { - int item_id; - int group_id; - int addr_type; - union { - struct ipv4_item_rule ipv4; - struct ipv6_item_rule ipv6; - }; - int is_valid; -}; - -struct plugin_item { - char key[MAX_KEYWORDS_STR]; - size_t key_len; - int is_valid; -}; - -struct ip_plugin_item { - int item_id; - int ip_type; - char start_ip[MAX_IP_STR]; - char end_ip[MAX_IP_STR]; - int is_valid; - int rule_tag; -}; - -struct table_item { - enum table_type table_type; - union { - struct compile_item compile_item; - struct group2compile_item g2c_item; - struct group2group_item g2g_item; - struct expr_item expr_item; - struct ip_plus_item ip_plus_item; - struct plugin_item plugin_item; - struct ip_plugin_item ip_plugin_item; - }; -}; - -struct plugin_table_callback_schema -{ - maat_start_callback_t *start; - maat_update_callback_t *update; - maat_finish_callback_t *finish; - void *u_para; -}; - -struct ex_data_schema -{ - maat_plugin_ex_new_func_t *new_func; - maat_plugin_ex_free_func_t *free_func; - maat_plugin_ex_dup_func_t *dup_func; - //Maat_plugin_EX_key2index_func_t* key2index_func; - long argl; - void *argp; - int set_flag; -}; - -struct compile_ex_data_schema { - maat_rule_ex_new_func_t *new_func; - maat_rule_ex_free_func_t *free_func; - maat_rule_ex_dup_func_t *dup_func; - long argl; - void *argp; - int idx; - int table_id; -}; - -struct table_schema; -struct table_schema_manager; - -/* table schema manager API */ -struct table_schema_manager *table_schema_manager_create(const char *table_info_path, struct log_handle *logger); -void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr); - -int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name); - -/** - * @brief get composition table's child table(specified by type) id -*/ -int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id, - enum component_table_type type); - -enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id); - -size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_mgr); - -void table_schema_manager_all_plugin_cb_start(struct table_schema_manager* table_schema_mgr, int update_type); -void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* table_schema_mgr); - -/* table schema generic API */ -struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id); - -struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr, - int table_id, enum scan_type type, int *virtual_table_id); - -enum table_type table_schema_get_table_type(struct table_schema *table_schema); - -int table_schema_get_table_id(struct table_schema *table_schema); - -/* get group2compile table's associated compile table id */ -int table_schema_get_associated_table_id(struct table_schema *table_schema); - -enum scan_type table_schema_get_scan_type(struct table_schema *table_schema); - -struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema, - struct rule_tag *accept_tags, int n_accept_tag, struct log_handle *logger); -void table_item_free(struct table_item *table_item); - -int table_schema_get_valid_flag_column(struct table_schema *table_schema); - -void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name); -const char *table_schema_get_updating_name(struct table_schema *table_schema); - -/* expr table schema API */ -enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema); - -/* table schema ex API */ -int table_schema_set_ex_data_schema(struct table_schema *table_schema, - maat_plugin_ex_new_func_t *new_func, - maat_plugin_ex_free_func_t *free_func, - maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp, - struct log_handle *logger); - -struct ex_data_schema *table_schema_get_ex_data_schema(struct table_schema *table_schema); - -/* table schema compile rule ex API */ -int table_schema_set_compile_rule_ex_data_schema(struct table_schema *table_schema, - maat_rule_ex_new_func_t *new_func, - maat_rule_ex_free_func_t *free_func, - maat_rule_ex_dup_func_t *dup_func, - long argl, void *argp, - struct log_handle *logger); -struct compile_ex_data_schema *table_schema_get_compile_rule_ex_data_schema(struct table_schema *table_schema, size_t idx); -size_t table_schema_compile_rule_ex_data_schema_count(struct table_schema *table_schema); -/** - * @brief if plugin table schema's ex data schema set - * - * @retval 1(already Set) 0(Not set yet) -*/ -int table_schema_ex_data_schema_flag(struct table_schema *table_schema); - -int table_schema_add_callback(struct table_schema_manager *table_schema_mgr, int table_id, - maat_start_callback_t *start, - maat_update_callback_t *update, - maat_finish_callback_t *finish, - void *u_para, struct log_handle *logger); -/** - * @brief the number of callback function stored in plugin table schema -*/ -size_t table_schema_callback_count(struct table_schema *table_schema); - -void table_schema_all_cb_update(struct table_schema *table_schema, const char *row); - -int table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns); - -#ifdef __cpluscplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/inc_internal/maat_utils.h b/src/inc_internal/maat_utils.h index 9542d35..8d2c58d 100644 --- a/src/inc_internal/maat_utils.h +++ b/src/inc_internal/maat_utils.h @@ -31,7 +31,9 @@ extern "C" #define MIN(a, b) (((a) < (b)) ? (a) : (b)) #endif -#define MAX_SCANNER_HIT_NUM 4096 +#define MAX_SCANNER_HIT_COMPILE_NUM 4096 +#define MAX_SCANNER_HIT_GROUP_NUM 4096 +#define MAX_SCANNER_HIT_ITEM_NUM 4096 enum maat_ip_format { IP_FORMAT_RANGE, @@ -59,6 +61,9 @@ char *maat_strdup(const char *s); int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len); +/* the column value must be integer */ +int get_column_value(const char *line, int column_seq); + int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz); char *strtok_r_esc(char *s, const char delim, char **save_ptr); diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h new file mode 100644 index 0000000..b687df5 --- /dev/null +++ b/src/inc_internal/maat_virtual.h @@ -0,0 +1,38 @@ +/* +********************************************************************************************** +* File: maat_virtual.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_VIRTUAL_H_ +#define _MAAT_VIRTUAL_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +enum scan_type { + SCAN_TYPE_INVALID = -1, + SCAN_TYPE_NONE = 0, + SCAN_TYPE_PLUGIN, + SCAN_TYPE_IP_PLUGIN, + SCAN_TYPE_FQDN_PLUGIN, + SCAN_TYPE_BOOL_PLUGIN, + SCAN_TYPE_IP, + SCAN_TYPE_INTERVAL, + SCAN_TYPE_STRING, + SCAN_TYPE_MAX +}; + +void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/rcu_hash.h b/src/inc_internal/rcu_hash.h index b5186bb..1ef32ed 100644 --- a/src/inc_internal/rcu_hash.h +++ b/src/inc_internal/rcu_hash.h @@ -51,6 +51,7 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len); size_t rcu_hash_count(struct rcu_hash_table *htable); +size_t rcu_hash_updating_count(struct rcu_hash_table *htable); /** * @brief make add/del effective diff --git a/src/json2iris.cpp b/src/json2iris.cpp index 890bdd1..2b1c3e0 100644 --- a/src/json2iris.cpp +++ b/src/json2iris.cpp @@ -20,7 +20,6 @@ #include "cJSON/cJSON.h" #include "maat_kv.h" #include "maat_utils.h" -#include "maat_table_schema.h" #include "maat_rule.h" #include "uthash/uthash.h" diff --git a/src/maat_api.cpp b/src/maat_api.cpp index 28e3b77..96ef3fd 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -21,31 +21,22 @@ #include "maat_kv.h" #include "maat_command.h" #include "maat_ex_data.h" -#include "maat_table_schema.h" -#include "maat_table_runtime.h" +#include "maat_table.h" #include "maat_config_monitor.h" #include "maat_redis_monitor.h" -#include "maat_hierarchy.h" +#include "maat_compile.h" #include "alignment.h" +#include "maat_plugin.h" +#include "maat_ip_plugin.h" #define MODULE_MAAT_API module_name_str("maat.api") #define DISTRICT_ANY -1 #define DISTRICT_UNKNOWN -2 -struct maat_state { - struct maat *maat_instance; - int16_t thread_id; - unsigned char is_set_district; - unsigned char is_last_scan; - int district_id; //-1: Any District; -2: Unkonwn District; - int scan_cnt; - struct maat_hierarchy_compile_mid *compile_mid; -}; - struct scan_item_hit_wrapper { int Nth_scan; - struct maat_item_inner* wrapped_items[MAX_SCANNER_HIT_NUM]; + struct maat_item_inner* wrapped_items[MAX_SCANNER_HIT_COMPILE_NUM]; size_t n_wrapped_item; int *virtual_table_ids; @@ -56,7 +47,7 @@ struct scan_item_hit_wrapper { inline int scan_state_should_compile_NOT(struct maat_state *mid) { if (mid && mid->is_last_scan==1 && mid->compile_mid && - maat_hierarchy_compile_mid_has_NOT_clause(mid->compile_mid)) { + maat_compile_state_has_NOT_clause(mid->compile_mid)) { return 1; } else { return 0; @@ -78,6 +69,19 @@ struct maat_options* maat_options_new(void) return options; } +void maat_options_free(struct maat_options *opts) +{ + if (NULL == opts) { + return; + } + + if (opts->accept_tags != NULL) { + FREE(opts->accept_tags); + } + + FREE(opts); +} + int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread) { opts->nr_worker_threads = n_thread; @@ -85,6 +89,13 @@ int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_th return 0; } +int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags) +{ + opts->accept_tags = maat_strdup(accept_tags); + + return 0; +} + int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms) { opts->rule_effect_interval_ms = interval_ms; @@ -244,15 +255,15 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) } maat_instance->logger = log_handle_create(log_path, opts->log_level); } - - maat_instance->table_schema_mgr = table_schema_manager_create(table_info_path, maat_instance->logger); - if (NULL == maat_instance->table_schema_mgr) { + + maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags, maat_instance->logger); + if (NULL == maat_instance->tbl_mgr) { goto failed; } + //TODO: by luis + //maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr); + maat_instance->default_compile_table_id = -1; - if (opts->compile_tablename != NULL) { - memcpy(maat_instance->compile_tablename, opts->compile_tablename, strlen(opts->compile_tablename)); - } maat_instance->input_mode = opts->input_mode; switch (maat_instance->input_mode) { case DATA_SOURCE_REDIS: @@ -317,8 +328,8 @@ int maat_table_get_id(struct maat *maat_instance, const char *table_name) { int table_id = -1; - struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr; - table_id = table_schema_manager_get_table_id(table_schema_mgr, table_name); + struct table_manager *table_mgr = maat_instance->tbl_mgr; + table_id = table_manager_get_table_id(table_mgr, table_name); return table_id; } @@ -333,6 +344,24 @@ inline void maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id) alignment_int64_array_add(maat_rt->ref_cnt, thread_id, -1); } +void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, + const char *srv_def, int srv_def_len) +{ + memcpy(rule, rule_head, sizeof(struct maat_rule_head)); + memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN)); +} + +size_t generic_plugin_runtime_cached_row_count(void *custom_rt, enum table_type table_type) +{ + +} + +const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type, size_t row_id) +{ + +} + +/* must be plugin table */ int maat_table_callback_register(struct maat *maat_instance, int table_id, maat_start_callback_t *start, maat_update_callback_t *update, @@ -342,8 +371,9 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id, int ret = -1; pthread_mutex_lock(&(maat_instance->background_update_mutex)); - ret = table_schema_add_callback(maat_instance->table_schema_mgr, table_id, - start, update, finish, u_para, maat_instance->logger); + void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + ret = plugin_table_add_callback(schema, table_id, start, update, finish, + u_para, maat_instance->logger); if (ret < 0) { pthread_mutex_unlock(&(maat_instance->background_update_mutex)); return -1; @@ -354,15 +384,17 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id, return 0; } - struct table_runtime *table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id); - size_t row_count = table_runtime_cached_row_count(table_rt); - if (row_count > 0) { + void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + size_t row_cnt = generic_plugin_runtime_cached_row_count(runtime, table_type); + + if (row_cnt > 0) { if (start != NULL) { start(MAAT_RULE_UPDATE_TYPE_FULL, u_para); } for (size_t i = 0; i < row_count; i++) { - const char *line = table_runtime_get_cached_row(table_rt, i); + const char *line = generic_plugin_runtime_get_cached_row(runtime, table_type, i); if (NULL == line) { break; } @@ -380,26 +412,132 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id, return 0; } +int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int table_id, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp) +{ + if (NULL == tbl_mgr || NULL == new_func || NULL == free_func || NULL == dup_func) { + assert(0); + log_error(tbl_mgr->logger, MODULE_MAAT_API, + "table(table_id:%d) %s failed: invalid parameter", __FUNCTION__); + return -1; + } + + void *schema = table_manager_get_schema(tbl_mgr, table_id); + if (NULL == schema) { + log_error(tbl_mgr->logger, MODULE_MAAT_API, + "Error: %s, table(table_id:%d) is not registered", __FUNCTION__, table_id); + return -1; + } + + struct ex_data_schema *ex_schema = NULL; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + switch (table_type) { + case TABLE_TYPE_PLUGIN: + ex_schema = plugin_table_get_ex_data_schema(schema); + if (NULL == ex_schema) { + log_error(tbl_mgr->logger, MODULE_MAAT_API, + "Error: %s, table(table_id:%d) is not a valid plugin table", + __FUNCTION__, table_id); + return -1; + } + plugin_table_set_ex_data_schema(schema, new_func, free_func, + dup_func, argl, argp, tbl_mgr->logger); + break; + case TABLE_TYPE_IP_PLUGIN: + ex_schema = ip_plugin_table_get_ex_data_schema(schema); + if (NULL == ex_schema) { + log_error(tbl_mgr->logger, MODULE_MAAT_API, + "Error: %s, table(table_id:%d) is not a valid ip_plugin table", + __FUNCTION__, table_id); + return -1; + } + ip_plugin_table_set_ex_data_schema(schema, new_func, free_func, + dup_func, argl, argp); + break; + default: + break; + } + + return 0; +} + +void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, enum table_type table_type, + int nr_worker_thread, long long version, struct log_handle *logger) +{ + struct ex_data_schema *ex_data_schema = NULL; + struct ex_data_runtime *ex_data_rt = NULL; + + switch (table_type) { + case TABLE_TYPE_PLUGIN: + ex_data_schema = plugin_table_get_ex_data_schema(schema); + ex_data_rt = plugin_runtime_get_ex_data_rt(runtime); + break; + case TABLE_TYPE_IP_PLUGIN: + ex_data_schema = ip_plugin_table_get_ex_data_schema(schema); + ex_data_rt = ip_plugin_runtime_get_ex_data_rt(runtime); + break; + default: + break; + } + + ex_data_runtime_set_schema(ex_data_rt, ex_data_schema); + struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1); + + ctx->table_id = table_schema_get_table_id(custom_schema); + ctx->ex_schema = ex_data_schema; + ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx); + + size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt); + for (size_t i = 0; i < n_cached_row; i++) { + const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i); + switch (table_rt->table_type) { + case TABLE_TYPE_PLUGIN: + plugin_runtime_update_row(custom_rt, custom_schema, row, NULL, 0, 1); + break; + case TABLE_TYPE_IP_PLUGIN: + ip_plugin_runtime_update_row(custom_rt, custom_schema, NULL, NULL, 0, NULL, 1); + break; + default: + break; + } + } + ex_data_runtime_clear_row_cache(ex_data_rt); + + switch (table_type) { + case TABLE_TYPE_PLUGIN: + plugin_runtime_commit(custom_rt); + break; + case TABLE_TYPE_IP_PLUGIN: + ip_plugin_runtime_commit(custom_rt); + break; + default: + break; + } +} + int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_id, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, long argl, void *argp) { - struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); pthread_mutex_lock(&(maat_instance->background_update_mutex)); - int ret = table_schema_set_ex_data_schema(table_schema, new_func, free_func, dup_func, - argl, argp, maat_instance->logger); + int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id, + new_func, free_func, dup_func, argl, argp); if (ret < 0) { pthread_mutex_unlock(&(maat_instance->background_update_mutex)); return -1; } - struct table_runtime *table_rt = NULL; if (maat_instance->maat_rt != NULL) { - table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id); - table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->nr_worker_thread, - maat_instance->maat_rt->version, maat_instance->logger); + void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + generic_plugin_runtime_commit_ex_schema(runtime, schema, table_type, maat_instance->nr_worker_thread, + maat_instance->maat_rt->version, maat_instance->logger); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); @@ -413,22 +551,29 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, co return NULL; } - struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); - if (NULL == table_schema) { + enum table_type table_type = table_manager_get_table_type(maat_rt->ref_tbl_mgr, table_id); + void *runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, table_id); + if (NULL == runtime) { return NULL; } - struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); - if (NULL == table_rt) { - return NULL; + struct ex_data_runtime *ex_data_rt = NULL; + switch (table_type) { + case TABLE_TYPE_PLUGIN: + ex_data_rt = plugin_runtime_get_ex_data_rt(runtime); + break; + case TABLE_TYPE_IP_PLUGIN: + ex_data_rt = ip_plugin_runtime_get_ex_data_rt(runtime); + break; + default: + break; } - - struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt); + if (NULL == ex_data_rt) { return NULL; } - return ex_data_runtime_dup_ex_data(ex_data_rt, key, key_len); + return ex_data_runtime_get_ex_data(ex_data_rt, key, key_len); } static void scan_count_inc(struct maat_state *mid) @@ -444,11 +589,12 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id) outer_state->maat_instance = maat_instance; outer_state->district_id = DISTRICT_ANY; outer_state->thread_id = (signed short)thread_id; + outer_state->compile_table_id = -1; //-1 means caller not specify compile table, use default compile table return outer_state; } -struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instance, int thread_id, int is_hit_item) +struct maat_state *grab_state(struct maat_state **state, struct maat *maat_instance, int thread_id) { struct maat_state *mid = *state; @@ -466,13 +612,12 @@ struct maat_state *grab_mid(struct maat_state **state, struct maat *maat_instanc alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1); } - if (is_hit_item == 1) { - if (NULL == mid->compile_mid) { - mid->compile_mid = maat_hierarchy_compile_mid_new(maat_instance->maat_rt->hier, thread_id); - alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1); - } - } - return mid; + if (NULL == mid->compile_mid) { + mid->compile_mid = maat_compile_state_new(thread_id); + alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1); + } + + return mid; } void scan_item_hit_wrapper_build(struct scan_item_hit_wrapper* wrapper, struct scan_result* results, size_t n_result, @@ -497,186 +642,27 @@ void scan_item_hit_wrapper_build(struct scan_item_hit_wrapper* wrapper, struct s wrapper->virtual_table_ids = NULL; } -struct compile_sort_para { - double evaluation_order; - int declared_clause_num; - int compile_id; - void *user; -}; - -static void compile_sort_para_set(struct compile_sort_para *para, - const struct maat_compile_rule *compile_relation, void *user) -{ - para->compile_id=compile_relation->compile_id; - para->evaluation_order=compile_relation->evaluation_order; - para->declared_clause_num=compile_relation->declared_clause_num; - para->user = user; -} - -static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b) -{ - //If both of compile rule's evaluation order are specified, compile rule with small evaluation order is priority. - if(a->evaluation_order!=0 && b->evaluation_order!=0) - { - if(a->evaluation_order - b->evaluation_order <0) - { - return -1; - } - else if(a->evaluation_order - b->evaluation_order >0) - { - return 1; - } - } - //If one of compile rule's evaluation order is zero, compile rule with big evaluation order is priority. - else if(a->evaluation_order + b->evaluation_order!= 0) - { - return (a->evaluation_order - b->evaluation_order >0) ? -1 : 1; - } - //If compile rule's execute sequences are not specified or equal. - if(a->declared_clause_num!=b->declared_clause_num) - { - return (a->declared_clause_num-b->declared_clause_num); - } - else - { - return (b->compile_id-a->compile_id); - } - -} - -static int compare_compile_rule(const void *a, const void *b) -{ - const struct maat_compile_rule *ra=*(const struct Maat_compile_rule **)a; - const struct maat_compile_rule *rb=*(const struct Maat_compile_rule **)b; - - struct compile_sort_para sa, sb; - compile_sort_para_set(&sa, ra, NULL); - compile_sort_para_set(&sb, rb, NULL); - - return compile_sort_para_compare(&sa, &sb); -} - -int item_compile(struct maat *maat_instance, struct maat_hierarchy_compile_mid *compile_mid, - const struct scan_item_hit_wrapper *item_hit_wrapper, int *result, - int size, int thread_id) -{ - int is_last_item = item_hit_wrapper->is_last_item; - size_t item_hit_num = item_hit_wrapper->n_wrapped_item; - - int scan_ret=0; - int i=0; - - struct maat_compile_rule *compile_rule_array[size]; - struct maat_compile_rule *compile_rule = NULL; - int virtual_table_id = 0; - - struct maat_item_inner *item = NULL; - - for (i = 0; (size_t)i < item_hit_num;i++) { - item = item_hit_wrapper->wrapped_items[i]; - assert(item->magic_num == ITEM_RULE_MAGIC); - if (item_hit_wrapper->virtual_table_ids) { - virtual_table_id = item_hit_wrapper->virtual_table_ids[i]; - } else { - virtual_table_id = item_hit_wrapper->virtual_table_id; - } - maat_hierarchy_compile_mid_update(feather->scanner->hier, compile_mid, item->item_id, virtual_table_id, item_hit_wrapper->Nth_scan, i); - } - - scan_ret = maat_hierarchy_item_compile(feather->scanner->hier, compile_mid, is_last_item, (void **)compile_rule_array, size); - //Maat_hierarchy is rwlock protected, it always returns non-NULL compile_rule. - if (scan_ret > 1) { - qsort(compile_rule_array, scan_ret, sizeof(struct maat_compile_rule *), - compare_compile_rule); - } - for (i = 0; i < scan_ret && i < size; i++) { - compile_rule = compile_rule_array[i]; - assert(compile_rule->magic_num == COMPILE_RULE_MAGIC); - result[i] = compile_rule->compile_id; - /* - fill_maat_rule(&(result[i]), &(compile_rule->head), compile_rule->service_defined, - compile_rule->head.serv_def_len);*/ - } - - if (scan_ret > 0) { - alignment_int64_array_add(feather->hit_cnt, thread_id, 1); - } - - if (item_hit_num == 0 && scan_ret > 0) { - alignment_int64_array_add(feather->not_grp_hit_cnt, thread_id, 1); - } - - return MIN(scan_ret, size); -} - int maat_scan_integer(struct maat *instance, int table_id, int thread_id, - unsigned int intval, int results[], size_t *n_result, - struct maat_state **state) + unsigned int intval, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state) { return 0; } -static int ip_scan_data_set(struct ip_addr *scan_data, struct addr_2tuple *addr, enum component_table_type child_type) +static int ip_scan_data_set(struct ip_addr *scan_data, const char *ip_addr) { - switch (addr->type) { - case IP_TYPE_V4: - scan_data->ip_type = IP_TYPE_V4; - switch (child_type) { - case COMPONENT_TABLE_TYPE_SIP: - scan_data->ipv4 = ntohl(addr->ipv4.sip); - break; - case COMPONENT_TABLE_TYPE_DIP: - scan_data->ipv4 = ntohl(addr->ipv4.dip); - break; - default: - assert(0); - return -1; - } - break; - case IP_TYPE_V6: - scan_data->ip_type = IP_TYPE_V6; - switch (child_type) { - case COMPONENT_TABLE_TYPE_SIP: - memcpy(scan_data->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip)); - ipv6_ntoh(scan_data->ipv6); - break; - case COMPONENT_TABLE_TYPE_DIP: - memcpy(scan_data->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip)); - ipv6_ntoh(scan_data->ipv6); - break; - default: - assert(0); - return -1; - } - break; - default: - assert(0); - return -1; - } - return 0; } -static int ip_composition_scan(int thread_id, struct addr_2tuple *addr, - int parent_table_id, enum component_table_type child_type, - int *virtual_table_id, +static int ip_composition_scan(int thread_id, const char *ip_addr, + int parent_table_id, int *virtual_table_id, struct table_schema_manager *table_schema_mgr, struct table_runtime_manager *table_rt_mgr, - struct scan_result *item_results, size_t n_result_array) + int *group_id_array, size_t n_group_id_array, + struct maat_state *state) { int child_table_id = 0; - if (child_type == COMPONENT_TABLE_TYPE_NONE) { - child_table_id = parent_table_id; - child_type = COMPONENT_TABLE_TYPE_SESSION; - } else { - child_table_id = table_schema_manager_get_child_table_id(table_schema_mgr, parent_table_id, child_type); - } - - if (child_table_id < 0) { - return 0; - } - struct table_schema *real_table = table_schema_get_by_scan_type(table_schema_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id); if (NULL == real_table) { return 0; @@ -688,7 +674,7 @@ static int ip_composition_scan(int thread_id, struct addr_2tuple *addr, } int table_id = table_schema_get_table_id(real_table); - struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); + struct table_runtime *table_rt = table_manager_get_runtime(table_rt_mgr, table_id); size_t rule_num = table_runtime_rule_count(table_rt); if (0 == rule_num) { return 0; @@ -697,122 +683,150 @@ static int ip_composition_scan(int thread_id, struct addr_2tuple *addr, struct ip_addr scan_data; memset(&scan_data, 0, sizeof(struct ip_addr)); - ip_scan_data_set(&scan_data, addr, child_type); - size_t hit_cnt = 0; - int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, item_results, &hit_cnt, n_result_array); + ip_scan_data_set(&scan_data, ip_addr); + size_t hit_group_cnt = 0; + int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, group_id_array, n_group_id_array, *virtual_table_id, state); if (ret < 0) { return -1; } - return hit_cnt; + return hit_group_cnt; } -int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_2tuple *addr, - int results[], size_t *n_result, struct maat_state **state) +int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, + uint32_t ip_addr, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state) { if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || - (thread_id < 0) || (NULL == addr) || (NULL == results) || (NULL == n_result) || + (thread_id < 0) || (NULL == ip_addr) || (NULL == results) || (0 == n_result) || (NULL == state)) { return -1; } struct maat_state *mid = NULL; - mid = grab_mid(state, maat_instance, thread_id, 0); + mid = grab_state(state, maat_instance, thread_id); scan_count_inc(mid); - - struct maat_runtime *maat_rt = maat_instance->maat_rt; - if (NULL == maat_rt) { - return 0; - } - struct table_runtime_manager *table_rt_mgr = maat_rt->table_rt_mgr; - struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); - enum table_type table_type = table_runtime_get_type(table_rt); - if (table_type == TABLE_TYPE_INVALID) { - maat_instance->scan_err_cnt++; - return -1; - } - - int item_ret = 0; int virtual_table_id = 0; - struct scan_result *item_result = maat_rt->item_result_buff + thread_id * MAX_SCANNER_HIT_NUM; - int item_hit_cnt = 0; - int item_result_virtual_table_ids[MAX_SCANNER_HIT_NUM]; - alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - maat_runtime_ref_inc(maat_rt, thread_id); + struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, + SCAN_TYPE_IP, &virtual_table_id); + if (NULL == table_schema) { + return MAAT_ERR; + } - if (table_type == TABLE_TYPE_COMPOSITION) { - /* - item_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_SIP, &virtual_table_id, - maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, - item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt); - item_hit_cnt += item_ret; */ - enum component_table_type childs[3] = {COMPONENT_TABLE_TYPE_SIP, COMPONENT_TABLE_TYPE_DIP, COMPONENT_TABLE_TYPE_SESSION}; - for (int i = 0; i < 3; i++) { - item_ret = ip_composition_scan(thread_id, addr, table_id, childs[i], &virtual_table_id, - maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, - item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt); - if (item_ret < 0) { - maat_instance->scan_err_cnt++; - } else { - for (int j = 0; j < item_ret; j++) { - item_result_virtual_table_ids[item_hit_cnt++] = virtual_table_id; - } - } - } + struct maat_runtime *maat_rt = maat_instance->maat_rt; + int group_result[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + int group_hit_cnt = 0; + int group_result_virtual_table_ids[MAX_SCANNER_HIT_GROUP_NUM]; + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + int group_ret = -1; + + group_ret = ip_composition_scan(thread_id, ip_addr, table_id, &virtual_table_id, + maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, + group_result + group_hit_cnt, MAX_SCANNER_HIT_COMPILE_NUM - group_hit_cnt, mid); + if (group_ret < 0) { + maat_instance->scan_err_cnt++; + } else if (0 == group_ret) { + return MAAT_OK; } else { - item_ret = ip_composition_scan(thread_id, addr, table_id, COMPONENT_TABLE_TYPE_NONE, &virtual_table_id, - maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, - item_result + item_hit_cnt, MAX_SCANNER_HIT_NUM - item_hit_cnt); - if (item_ret < 0) { - maat_instance->scan_err_cnt++; - } else { - item_hit_cnt += item_ret; - } + group_hit_cnt += group_ret; } - int compile_ret = 0; - struct scan_item_hit_wrapper item_hit_wrapper; - if (item_hit_cnt > 0 || scan_state_should_compile_NOT(mid)) { - mid = grab_mid(state, maat_instance, thread_id, 1); - scan_item_hit_wrapper_build(&item_hit_wrapper, item_result, item_hit_cnt, -1, - mid->is_last_scan, virtual_table_id, mid->scan_cnt); - if (table_type == TABLE_TYPE_COMPOSITION) { - item_hit_wrapper.virtual_table_ids = item_result_virtual_table_ids; - } + // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT - compile_ret = item_compile(maat_instance, mid->compile_mid, - &item_hit_wrapper, - results, item_hit_cnt, - thread_id); - assert(mid->is_last_scan < 2); - if (mid->is_last_scan == 1) { - mid->is_last_scan = 2; - } + struct table_runtime *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); + int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + int n_top_group_ids = group2group_runtime_get_top_groups(g2g_rt, group_result, group_hit_cnt, top_group_ids); + size_t n_all_group_ids = 0; + if (n_top_group_ids > 0) { + n_all_group_ids = group_hit_cnt + n_top_group_ids; + } else { + n_all_group_ids = group_hit_cnt; } - maat_runtime_ref_dec(maat_rt, thread_id); + + int all_group_ids[n_all_group_ids] = {-1}; + // maat state find compile_table_id, if not found, maat_instance->default_compile_table_id + struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id); + int n_compile_ids = compile_runtime_match(compile_table_rt, all_group_ids, n_all_group_ids, results, n_result, mid); + *n_hit_result = n_compile_ids; - if (compile_ret == 0 && item_hit_cnt > 0) { - return -2; - } - *n_result = compile_ret; - return compile_ret; + if (n_compile_ids > 0) { + return MAAT_HIT; + } else { + return MAAT_HALF_HIT; + } +} + + +int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id, + uint8_t *ip_addr, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state) +{ + return 0; } int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, - const char *data, size_t data_len, int results[], size_t *n_result, - struct maat_state **state) + const char *data, size_t data_len, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state) { if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || (thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) || - (NULL == n_result) || (NULL == state)) { - return -1; + (0 == n_result) || (NULL == state)) { + return MAAT_ERR; } - struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr; - struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); + struct maat_state *mid = NULL; + mid = grab_state(state, maat_instance, thread_id); + scan_count_inc(mid); + + int virtual_table_id = 0; + struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, + SCAN_TYPE_STRING, &virtual_table_id); + if (NULL == table_schema) { + return MAAT_ERR; + } + + struct table_runtime *table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - return table_runtime_scan_string(table_rt, thread_id, data, data_len, results, n_result); + int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + int group_hit_cnt = table_runtime_scan_string(table_rt, thread_id, data, data_len, group_ids, MAX_SCANNER_HIT_GROUP_NUM, + virtual_table_id, mid); + if (group_hit_cnt <= 0) { + return MAAT_OK; + } + + // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT + + struct table_runtime *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); + int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + int n_top_group_ids = group2group_runtime_get_top_groups(g2g_rt, group_ids, group_hit_cnt, top_group_ids); + size_t n_all_group_ids = 0; + if (n_top_group_ids > 0) { + n_all_group_ids = group_hit_cnt + n_top_group_ids; + } else { + n_all_group_ids = group_hit_cnt; + } + + int all_group_ids[n_all_group_ids] = {-1}; + int i = 0, j = 0, k = 0; + for (i = 0; i < group_hit_cnt; i++) { + all_group_ids[i] = group_ids[i]; + } + for (j = i; j < n_all_group_ids; j++, k++) { + all_group_ids[j] = top_group_ids[k]; + } + + // maat state find compile_table_id, if not found, maat_instance->default_compile_table_id + struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id); + int n_compile_ids = compile_runtime_match(compile_table_rt, all_group_ids, n_all_group_ids, results, n_result, mid); + + *n_hit_result = n_compile_ids; + + if (n_compile_ids > 0) { + return MAAT_HIT; + } else { + return MAAT_HALF_HIT; + } } struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id) @@ -831,25 +845,112 @@ void maat_scan_stream_close(struct maat_stream **stream) } -int maat_state_set(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, - const void *value, int size) +int maat_state_set_scan_district(struct maat *maat_instance, struct maat_state **state, const char *district, size_t district_len) { + if (NULL == maat_instance->maat_rt || NULL == district || district_len <= 0) { + return -1; + } + struct maat_state *mid = grab_state(state, maat_instance, -1); + int map_ret = maat_kv_read_unNull(maat_instance->maat_rt->district_map, district, district_len, &(mid->district_id)); + if (map_ret < 0) { + mid->district_id = DISTRICT_UNKNOWN; + } + + mid->is_set_district = 1; + + return 0; } -//return >=0 if success, return -1 when failed; -int maat_state_get(struct maat *instance, struct maat_state **mid, enum maat_scan_opt opt, - void *value, int size) +int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state) { + if (NULL == maat_instance->maat_rt) { + return -1; + } + struct maat_state *mid = grab_state(state, maat_instance, -1); + assert(mid->is_last_scan == 0); + mid->is_last_scan = 1; + + return 0; +} + +int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, int compile_table_id) +{ + if (NULL == maat_instance->maat_rt) { + return -1; + } + + struct maat_state *mid = grab_state(state, maat_instance, -1); + mid->compile_table_id = compile_table_id; + + return 0; +} + +size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid, struct maat_hit_path *paths, size_t n_path) +{ + int compile_table_id = -1; + + if (mid->compile_table_id == -1) { + compile_table_id = maat_instance->default_compile_table_id; + } else { + compile_table_id = mid->compile_table_id; + } + + compile_runtime_get_hit_paths(maat_instance->maat_rt->table_rt_mgr, compile_table_id, maat_instance->g2g_table_id, mid, paths, n_path); + + return 0; +} + +size_t maat_get_hit_objects(struct maat_compile_state *compile_state, struct maat_hit_object *objs, size_t n_objs) +{ + return 0; +} + +int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path) +{ + struct maat_state *mid = NULL; + struct maat_hit_path *paths; + + mid = grab_state(state, maat_instance, 0); + if (NULL == mid->compile_mid || NULL == maat_instance->maat_rt) { + return 0; + } + + int n_read = maat_get_hit_paths(mid->maat_instance, mid, paths, n_path); + + return n_read; +} + +int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj) +{ + return 0; } void maat_state_free(struct maat_state **state) { + struct maat_state *mid = NULL; + if (NULL == *state) { + return; + } + mid = *state; + if (mid->thread_id >= 0) { + alignment_int64_array_add(mid->maat_instance->outer_mid_cnt, mid->thread_id, -1); + } + + if (mid->compile_mid != NULL) { + maat_compile_state_free(mid->compile_mid); + mid->compile_mid = NULL; + alignment_int64_array_add(mid->maat_instance->compile_mid_cnt, mid->thread_id, -1); + } + + mid->maat_instance = NULL; + free(mid); + *state = NULL; } -int maat_matched_compile_id(struct maat *instance, struct maat_matched *matched) +int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj) { - + return 0; } \ No newline at end of file diff --git a/src/maat_command.cpp b/src/maat_command.cpp index f8d914f..ba5a379 100644 --- a/src/maat_command.cpp +++ b/src/maat_command.cpp @@ -19,7 +19,6 @@ #include "maat_rule.h" #include "hiredis/hiredis.h" #include "maat_config_monitor.h" -#include "maat_table_schema.h" #define MODULE_MAAT_COMMAND module_name_str("maat.command") @@ -305,7 +304,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li struct serial_rule *s_rule = ALLOC(struct serial_rule, 1); - int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, line_rule->table_name); if (table_id < 0) { log_error(maat_instance->logger, MODULE_MAAT_COMMAND, "Command set line id %d failed: unknown table %s", line_rule->rule_id, line_rule->table_name); @@ -313,14 +312,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li return -1; } - struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); - if (!table_schema) { - FREE(s_rule); - return -1; - } - - int valid_flag_column = table_schema_get_valid_flag_column(table_schema); - if (valid_flag_column < 0) { + int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); + if (valid_column < 0) { log_error(maat_instance->logger, MODULE_MAAT_COMMAND, "Command set line id %d failed: table %s is not a plugin or ip_plugin table", line_rule->rule_id, line_rule->table_name); @@ -328,9 +321,9 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li return -1; } - enum table_type table_type = table_schema_get_table_type(table_schema); - int valid_offset = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_flag_column); - int is_valid = atoi(line_rule->table_line+valid_offset); + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + int valid_offset = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_column); + int is_valid = atoi(line_rule->table_line + valid_offset); if (line_rule->expire_after > 0) { absolute_expire_time = server_time + line_rule->expire_after; } diff --git a/src/maat_compile.cpp b/src/maat_compile.cpp new file mode 100644 index 0000000..7c48a5f --- /dev/null +++ b/src/maat_compile.cpp @@ -0,0 +1,1580 @@ +/* +********************************************************************************************** +* File: maat_hierarchy.cpp +* Description: +* Authors: Zheng Chao +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include +#include + +#include "utils.h" +#include "maat_utils.h" +#include "log/log.h" +#include "uthash/utarray.h" +#include "uthash/uthash.h" +#include "bool_matcher.h" +#include "igraph/igraph.h" +#include "maat_compile.h" +#include "maat_garbage_collection.h" +#include "maat_group.h" +#include "maat/maat.h" +#include "rcu_hash.h" + +#define MODULE_COMPILE module_name_str("maat.compile") +#define MAX_TABLE_LINE_SIZE (1024 * 16) + +enum user_region_encode { + USER_REGION_ENCODE_NONE=0, + USER_REGION_ENCODE_ESCAPE, + USER_REGION_ENCODE_BASE64 +}; + +struct compile_schema { + int compile_id_column; + int service_id_column; + int action_column; + int do_blacklist_column; + int do_log_column; + int tags_column; + int user_region_column; + int clause_num_column; + int evaluation_order_column; + enum user_region_encode user_region_encoding; + size_t n_ex_schema; + struct compile_ex_data_schema ex_schema[MAX_COMPILE_EX_DATA_NUM]; + int table_id; //ugly +}; + +struct group2compile_schema { + int group_id_column; + int compile_id_column; + int not_flag_column; + int virtual_table_name_column; + int clause_index_column; + char associated_compile_table_id; + int table_id;//ugly +}; + +struct compile_item { + int compile_id; + int service_id; + int action; + int do_blacklist; + int do_log; + char user_region[MAX_TABLE_LINE_SIZE]; + int clause_num; + int evaluation_order; +}; + +struct group2compile_item { + int group_id; + int compile_id; + int not_flag; + int vt_id; //virtual_table_id + int clause_index; + int associated_compile_table_id; +}; + +/* compile_runtime and group2compile_runtime share compile_hash_map */ +struct compile_runtime { + struct bool_matcher *bm; + struct maat_compile *compile_hash; // + unsigned long long clause_id_generator; + + uint32_t rule_num; + uint32_t updating_rule_num; + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; +}; + +struct group2compile_runtime { + long long not_flag_group; + struct compile_runtime *ref_compile_rt; + struct group2group_runtime *ref_g2g_rt; +}; + +struct maat_clause_state { + unsigned long long clause_id; + char not_flag; + char in_use; + UT_array *literal_ids; +}; + +struct maat_literal_id { + int group_id; + int vt_id; +}; + +struct maat_clause { + unsigned long long clause_id; + size_t n_literal_id; + struct maat_literal_id *literal_ids; + UT_hash_handle hh; +}; + +struct compile_sort_para { + double evaluation_order; + int declared_clause_num; + int compile_id; + void *user; +}; + +#define MAAT_COMPILE_MAGIC 0x4a5b6c7d +struct maat_compile { + unsigned int magic; + int compile_id; + int actual_clause_num; + int declared_clause_num; + int not_clause_cnt; + void *user_data; + void (*user_data_free)(void *); + UT_hash_handle hh; + struct maat_clause_state clause_states[MAX_ITEMS_PER_BOOL_EXPR]; +}; + +struct maat_internal_hit_path { + int Nth_scan; + int Nth_hit_item; + int item_id; + int group_id; + int virtual_table_id; +}; + +struct maat_compile_state { + int thread_id; + int Nth_scan; + time_t hier_ver; + size_t this_scan_item_hit_cnt; + int not_clause_hitted_flag; + int is_no_count_scan; + size_t hit_path_cnt; + + UT_array *internal_hit_paths; + UT_array *all_hit_clause_array; + UT_array *this_scan_hit_clause_ids; +}; + +int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id, + maat_rule_ex_new_func_t *new_func, + maat_rule_ex_free_func_t *free_func, + maat_rule_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger) +{ + if (compile_schema->n_ex_schema == MAX_COMPILE_EX_DATA_NUM) { + log_error(logger, MODULE_COMPILE, "compile ex schema num reach maxium, can't set anymore"); + return -1; + } + + int idx = compile_schema->n_ex_schema; + compile_schema->ex_schema[idx].idx = idx; + compile_schema->ex_schema[idx].table_id = table_id; + compile_schema->ex_schema[idx].argl = argl; + compile_schema->ex_schema[idx].argp = argp; + compile_schema->ex_schema[idx].new_func = new_func; + compile_schema->ex_schema[idx].free_func = free_func; + compile_schema->ex_schema[idx].dup_func = dup_func; + compile_schema->n_ex_schema++; + + return idx; +} + +struct compile_ex_data_schema * +compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, size_t idx) +{ + if (NULL == compile_schema) { + return NULL; + } + + if (idx < compile_schema->n_ex_schema) { + return (compile_schema->ex_schema + idx); + } + + return NULL; +} + +size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema) +{ + if (NULL == compile_schema) { + return 0; + } + + return compile_schema->n_ex_schema; +} + +UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL}; +UT_icd ut_clause_id_icd = {sizeof(uint64_t), NULL, NULL, NULL}; +UT_icd ut_hit_path_icd = {sizeof(struct maat_internal_hit_path), NULL, NULL, NULL}; + +void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +{ + int read_cnt = 0; + + struct compile_schema *compile_schema = ALLOC(struct compile_schema, 1); + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + goto error; + } + compile_schema->table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "custom"); + if (item == NULL || item->type != cJSON_Object) { + log_error(logger, MODULE_COMPILE, "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "compile_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + compile_schema->compile_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "service_id"); + if (item != NULL && item->type == cJSON_Number) { + compile_schema->service_id_column = item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "action"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + compile_schema->action_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "do_blacklist"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + compile_schema->do_blacklist_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "do_log"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + compile_schema->do_log_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "tags"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + compile_schema->tags_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "user_region"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + compile_schema->user_region_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "clause_num"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + compile_schema->clause_num_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "evaluation_order"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + compile_schema->evaluation_order_column = custom_item->valueint; + read_cnt++; + } + + if (read_cnt < 9) { + goto error; + } + + return compile_schema; +error: + FREE(compile_schema); + return NULL; +} + +void compile_schema_free(void *compile_schema) +{ + FREE(compile_schema); +} + +void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +{ + int read_cnt = 0; + struct group2compile_schema *g2c_schema = ALLOC(struct group2compile_schema, 1); + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + goto error; + } + g2c_schema->table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "custom"); + if (item == NULL || item->type != cJSON_Object) { + log_error(logger, MODULE_COMPILE, "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "associated_compile_table_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + g2c_schema->associated_compile_table_id = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "group_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + g2c_schema->group_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "compile_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + g2c_schema->compile_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "not_flag"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + g2c_schema->not_flag_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "virtual_table_name"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + g2c_schema->virtual_table_name_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "clause_index"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + g2c_schema->clause_index_column = custom_item->valueint; + read_cnt++; + } + + if (read_cnt < 6) { + goto error; + } + + return g2c_schema; +error: + FREE(g2c_schema); + return NULL; +} + +void group2compile_schema_free(void *g2c_schema) +{ + FREE(g2c_schema); +} + +struct compile_item * +compile_item_new(const char *line, struct compile_schema *compile_schema, struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + + struct compile_item *compile_item = ALLOC(struct compile_item, 1); + + int ret = get_column_pos(line, compile_schema->compile_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no compile_id", + compile_schema->table_id, line); + goto error; + } + compile_item->compile_id = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->service_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no service_id", + compile_schema->table_id, line); + goto error; + } + compile_item->service_id = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->action_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no action", + compile_schema->table_id, line); + goto error; + } + compile_item->action = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->do_blacklist_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no do_blacklist", + compile_schema->table_id, line); + goto error; + } + compile_item->do_blacklist = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->do_log_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no do_log", + compile_schema->table_id, line); + goto error; + } + compile_item->do_log = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->tags_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no tags", + compile_schema->table_id, line); + goto error; + } + + char tag_str[MAX_TABLE_LINE_SIZE] = {0}; + memcpy(tag_str, (line + column_offset), column_len); + + if (n_accept_tag > 0 && strlen(tag_str) > 2) { + str_unescape(tag_str); + ret = compare_accept_tag(tag_str, accept_tags, n_accept_tag); + ret = table_manager_accept_tag_match(tag_str); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s is invalid tag", + compile_schema->table_id, line); + goto error; + } + + if (0 == ret) { //not matched + //TODO: by luis + //table_schema->unmatched_tag_cnt++; + } + } + + ret = get_column_pos(line, compile_schema->user_region_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s has no user_region", + compile_schema->table_id, line); + goto error; + } + + if (column_len > MAX_TABLE_LINE_SIZE) { + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s user_region too long", + compile_schema->table_id, line); + goto error; + } + + memcpy(compile_item->user_region, (line + column_offset), column_len); + + switch (compile_schema->user_region_encoding) { + case USER_REGION_ENCODE_ESCAPE: + str_unescape(compile_item->user_region); + break; + default: + break; + } + + ret = get_column_pos(line, compile_schema->clause_num_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s has no clause_num", + compile_schema->table_id, line); + goto error; + } + compile_item->clause_num = atoi(line + column_offset); + + ret = get_column_pos(line, compile_schema->evaluation_order_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s has no evaluation_order", + compile_schema->table_id, line); + goto error; + } + compile_item->evaluation_order = atoi(line + column_offset); + + return compile_item; +error: + FREE(compile_item); + return NULL; +} + +void compile_item_free(struct compile_item *compile_item) +{ + if (NULL == compile_item) { + return; + } + + FREE(compile_item); +} + +void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == compile_schema) { + return NULL; + } + + struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1); + + compile_rt->logger = logger; + compile_rt->ref_garbage_bin = garbage_bin; + + return compile_rt; +} + +void compile_runtime_free(void *compile_runtime) +{ + if (NULL == compile_runtime) { + return; + } + + struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; + if (compile_rt->bm != NULL) { + bool_matcher_free(compile_rt->bm); + } + + if (compile_rt->compile_hash != NULL) { + maat_compile_hash_free(&(compile_rt->compile_hash)); + } + + FREE(compile_rt); +} + +void *group2compile_runtime_new(void *g2c_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == g2c_schema) { + return NULL; + } + + struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1); + return g2c_rt; +} + +void group2compile_runtime_free(void *g2c_runtime) +{ + if (NULL == g2c_runtime) { + return; + } + + FREE(g2c_runtime); +} + +int is_valid_table_name(const char *str) +{ + size_t integer_cnt=0; + for (size_t i = 0; i < strlen(str); i++) { + if (str[i] >= '0' && str[i] <= '9') { + integer_cnt++; + } + } + + if (strlen(str) == 0 || integer_cnt == strlen(str) || 0 == strcasecmp(str, "null")) { + return 0; + } + + return 1; +} + +struct group2compile_item * +group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1); + + int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "group2compile table(table_id:%d) line:%s has no group_id", + g2c_schema->table_id, line); + goto error; + } + g2c_item->group_id = atoi(line + column_offset); + + ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "group2compile table(table_id:%d) line:%s has no compile_id", + g2c_schema->table_id, line); + goto error; + } + g2c_item->compile_id = atoi(line + column_offset); + + ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "group2compile table(table_id:%d) line:%s has no NOT_flag", + g2c_schema->table_id, line); + goto error; + } + g2c_item->not_flag = atoi(line + column_offset); + + ret = get_column_pos(line, g2c_schema->virtual_table_name_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "group2compile table(table_id:%d) line:%s has no virtual_table_name", + g2c_schema->table_id, line); + goto error; + } + + if (column_len > NAME_MAX) { + log_error(logger, MODULE_COMPILE, + "group2compile table(table_id:%d) line:%s virtual_table_name length too long", + g2c_schema->table_id, line); + goto error; + } + char virtual_table_name[NAME_MAX] = {0}; + memcpy(virtual_table_name, (line + column_offset), column_len); + + if (is_valid_table_name(virtual_table_name)) { + g2c_item->vt_id = table_manager_get_table_id(tbl_mgr, virtual_table_name); + if (g2c_item->vt_id < 0) { + log_error(logger, MODULE_COMPILE, + "group2compile table(table_id:%d) line:%s unknown virtual table:%s", + g2c_schema->table_id, line, virtual_table_name); + goto error; + } + } + + ret = get_column_pos(line, g2c_schema->clause_index_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "group2compile table(table_id:%d) line:%s has no clause_index", + g2c_schema->table_id, line); + goto error; + } + + g2c_item->clause_index = atoi(line + column_offset); + + return g2c_item; +error: + FREE(g2c_item); + return NULL; +} + +void group2compile_item_free(struct group2compile_item *g2c_item) +{ + if (NULL == g2c_item) { + return; + } + + FREE(g2c_item); +} + +#define MAAT_HIER_COMPILE_MAGIC 0x4a5b6c7d +struct maat_compile *maat_compile_new(int compile_id) +{ + struct maat_compile *compile = ALLOC(struct maat_compile, 1); + + compile->magic = MAAT_HIER_COMPILE_MAGIC; + compile->compile_id = compile_id; + + for(int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + utarray_new(compile->clause_states[i].literal_ids, &ut_literal_id_icd); + compile->clause_states[i].in_use=0; + } + + return compile; +} + +int maat_compile_set(struct maat_compile *compile, int declared_clause_num, void *user_data, void (*user_data_free)(void *)) +{ + if (user_data != NULL && NULL == user_data_free) { + return -1; + } + + compile->declared_clause_num = declared_clause_num; + compile->user_data = user_data; + compile->user_data_free = user_data_free; + + return 0; +} + +void maat_compile_free(struct maat_compile *compile) +{ + struct maat_clause_state *clause_state = NULL; + if (compile->user_data && compile->user_data_free) { + compile->user_data_free(compile->user_data); + } + + for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states + i; + utarray_free(clause_state->literal_ids); + clause_state->literal_ids = NULL; + clause_state->in_use = 0; + } + compile->magic = 0; + free(compile); +} + +int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, struct maat_compile *compile) +{ + int ret = 0; + struct maat_compile *tmp_compile = NULL; + + HASH_FIND_INT(*compile_hash, &compile_id, tmp_compile); + if (!tmp_compile) { + // not found + assert(compile->declared_clause_num >= 0); + HASH_ADD_INT(*compile_hash, compile_id, compile); + } else { + // already exist + if (tmp_compile->user_data != NULL) { + ret = -1; + } else { + maat_compile_set(tmp_compile, compile->declared_clause_num, compile->user_data, compile->user_data_free); + } + } + + //TODO:mytest need to delete +#if 0 + size_t compile_cnt = HASH_COUNT(*compile_hash); + struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL; + HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) { + printf("compile->compile_id:%d, compile_cnt:%zu\n", compile1->compile_id, compile_cnt); + } +#endif + return ret; +} + +int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id, struct maat_garbage_bin *garbage_bin) +{ + struct maat_compile *compile = NULL; + + HASH_FIND_INT(*compile_hash, &compile_id, compile); + if (!compile) { + return -1; + } + + if (0 == compile->actual_clause_num) { + HASH_DEL(*compile_hash, compile); + maat_garbage_bagging(garbage_bin, compile, (void (*)(void *))maat_compile_free); + } + + //TODO:mytest need to delete +#if 0 + size_t compile_cnt = HASH_COUNT(*compile_hash); + struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL; + HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) { + printf("compile->compile_id:%d, compile_cnt:%zu\n", compile1->compile_id, compile_cnt); + } +#endif + return 0; +} + +struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash, int compile_id) +{ + struct maat_compile *compile = NULL; + + HASH_FIND(hh, *compile_hash, &compile_id, sizeof(compile_id), compile); + + return compile; +} + +size_t maat_compile_hash_count(struct maat_compile *compile_hash) +{ + return HASH_COUNT(compile_hash); +} + +void maat_compile_hash_free(struct maat_compile **compile_hash) +{ + struct maat_compile *compile = NULL, *tmp_compile = NULL; + + HASH_ITER(hh, *compile_hash, compile, tmp_compile) { + HASH_DEL(*compile_hash, compile); + maat_compile_free(compile); + } + assert(*compile_hash == NULL); +} + +int compare_literal_id(const void *pa, const void *pb) +{ + struct maat_literal_id *la = (struct maat_literal_id *)pa; + struct maat_literal_id *lb = (struct maat_literal_id *)pb; + + int ret = la->vt_id - lb->vt_id; + if (0 == ret) { + ret = la->group_id - lb->group_id; + } + + return ret; +} + +int maat_compile_clause_add_literal(struct maat_compile *compile, struct maat_literal_id *literal_id, + int clause_index, int clause_not_flag) +{ + struct maat_clause_state *clause_state = compile->clause_states + clause_index; + + clause_state->not_flag = clause_not_flag; + if (!clause_state->in_use) { + clause_state->in_use = 1; + compile->actual_clause_num++; + } + + struct maat_literal_id *tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); + if (tmp) { + assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); + return -1; + } else { + utarray_push_back(clause_state->literal_ids, literal_id); + utarray_sort(clause_state->literal_ids, compare_literal_id); + } + + return 0; +} + +int maat_compile_clause_remove_literal(struct maat_compile *compile, struct maat_literal_id *literal_id, int clause_index) +{ + struct maat_clause_state* clause_state = compile->clause_states + clause_index; + struct maat_literal_id *tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); + if (tmp) { + assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); + } else { + return -1; + } + + size_t remove_idx = utarray_eltidx(clause_state->literal_ids, tmp); + utarray_erase(clause_state->literal_ids, remove_idx, 1); + + if (0 == utarray_len(clause_state->literal_ids)) { + clause_state->in_use=0; + compile->actual_clause_num--; + } + + return 0; +} + +static const struct maat_clause * +maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, unsigned long long *clause_id_generator, + struct maat_literal_id *literal_ids, size_t n_literal_id) +{ + struct maat_clause *clause = NULL; + + HASH_FIND(hh, *clause_hash, literal_ids, n_literal_id * sizeof(struct maat_literal_id), clause); + if (!clause) { + clause = ALLOC(struct maat_clause, 1); + clause->clause_id = *clause_id_generator; + clause->n_literal_id = n_literal_id; + clause->literal_ids = ALLOC(struct maat_literal_id, n_literal_id); + memcpy(clause->literal_ids, literal_ids, n_literal_id * sizeof(struct maat_literal_id)); + + (*clause_id_generator)++; + HASH_ADD_KEYPTR(hh, *clause_hash, clause->literal_ids, n_literal_id * sizeof(struct maat_literal_id), clause); + } + + return clause; +} + +static void maat_clause_hash_free(struct maat_clause *clause_hash) +{ + struct maat_clause *clause = NULL, *tmp_clause = NULL; + + HASH_ITER (hh, clause_hash, clause, tmp_clause) { + HASH_DELETE(hh, clause_hash, clause); + free(clause->literal_ids); + clause->n_literal_id = 0; + free(clause); + } +} + +struct bool_matcher *maat_compile_bool_matcher_new(struct maat_compile *compile_hash, unsigned long long *clause_id_generator, struct log_handle *logger) +{ + if (NULL == compile_hash || NULL == logger) { + return NULL; + } + + size_t i = 0, j = 0; + int has_clause_num = 0; + struct bool_matcher *bm = NULL; + struct maat_clause_state *clause_state = NULL; + const struct maat_clause *clause = NULL; + struct maat_clause *clause_hash = NULL; // + + //STEP 1, update clause_id of each compile and literal + struct maat_compile *compile = NULL, *tmp_compile = NULL; + struct maat_literal_id *literal_ids = NULL; + size_t n_literal_id = 0; + HASH_ITER(hh, compile_hash, compile, tmp_compile) { + has_clause_num = 0; + for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states + i; + clause_state->clause_id = 0; + if (!clause_state->in_use) { + continue; + } + + has_clause_num++; + literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->literal_ids, 0); + n_literal_id = utarray_len(clause_state->literal_ids); + clause = maat_clause_hash_fetch_clause(&clause_hash, clause_id_generator, literal_ids, n_literal_id); + clause_state->clause_id = clause->clause_id; + } + assert(has_clause_num == compile->actual_clause_num); + } + + //STEP 2, serial compile clause states to a bool expression array + size_t expr_cnt = 0; + size_t compile_cnt = maat_compile_hash_count(compile_hash); + struct bool_expr *bool_expr_array = ALLOC(struct bool_expr, compile_cnt); + HASH_ITER(hh, compile_hash, compile, tmp_compile) { + for (i = 0, j = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + if (compile->clause_states[i].in_use) { + if (compile->clause_states[i].not_flag) { + compile->not_clause_cnt++; + } + + //TODO:mytest need to delete + #if 0 + struct maat_literal_id *p = NULL; + for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) { + printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n", + compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vt_id); + } + #endif + bool_expr_array[expr_cnt].items[j].item_id = compile->clause_states[i].clause_id; + bool_expr_array[expr_cnt].items[j].not_flag = compile->clause_states[i].not_flag; + + j++; + } + } + //some compile may have zero groups, e.g. default policy. + if (j == (size_t)compile->declared_clause_num && j > 0) { + bool_expr_array[expr_cnt].expr_id = compile->compile_id; + bool_expr_array[expr_cnt].user_tag = compile; + bool_expr_array[expr_cnt].item_num = j; + expr_cnt++; + } + } + + //size_t expr_index = 0, item_index = 0; + + // STEP 3, build bool matcher + size_t mem_size = 0; + if (0 == expr_cnt) { + log_error(logger, MODULE_COMPILE, "No bool expression to build bool matcher."); + goto error; + } + + //TODO:mytest need to delete + #if 0 + printf("bool_matcher_new....................\n"); + for (expr_index = 0; expr_index < expr_cnt; expr_index++) { + printf("bool_expr_array[%zu].expr_id:%llu, item_num:%zu\n", expr_index, bool_expr_array[expr_index].expr_id, + bool_expr_array[expr_index].item_num); + for (item_index = 0; item_index < bool_expr_array[expr_index].item_num; item_index++) { + printf("bool_expr_array[%zu].items[%zu]:%llu, not_flag:%d\n", expr_index, item_index, + bool_expr_array[expr_index].items[item_index].item_id, + bool_expr_array[expr_index].items[item_index].not_flag); + } + printf("\n"); + } + #endif + + bm = bool_matcher_new(bool_expr_array, expr_cnt, &mem_size); + if (bm != NULL) { + log_info(logger, MODULE_COMPILE, + "Build bool matcher of %zu expressions with %zu bytes memory.", expr_cnt, mem_size); + } else { + log_error(logger, MODULE_COMPILE, "Build bool matcher failed!"); + } + +error: + maat_clause_hash_free(clause_hash); + FREE(bool_expr_array); + + return bm; +} + +void maat_compile_bool_matcher_free(struct bool_matcher *bm) +{ + bool_matcher_free(bm); +} + +size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_compile_state *compile_state, void **user_data_array, size_t ud_array_size) +{ + struct maat_compile *compile = NULL; + struct bool_expr_match *expr_match = ALLOC(struct bool_expr_match, MAX_SCANNER_HIT_COMPILE_NUM); + size_t ud_result_cnt = 0; + + //TODO:mytest need to delete + #if 0 + unsigned long long *p; + for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clause_array); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clause_array, p)) + { + printf(" before bool_matcher_match compile_state clause_id:%llu\n", *p); + } + #endif + + int bool_match_ret = bool_matcher_match(bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clause_array, 0), + utarray_len(compile_state->all_hit_clause_array), expr_match, MAX_SCANNER_HIT_COMPILE_NUM); + for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) { + compile = (struct maat_compile *)expr_match[i].user_tag; + assert(compile->magic == MAAT_COMPILE_MAGIC); + assert((unsigned long long)compile->compile_id == expr_match[i].expr_id); + if (0 == compile->actual_clause_num) { + continue; + } + + //TODO: not_clause + if (compile->user_data) { + user_data_array[ud_result_cnt] = compile->user_data; + ud_result_cnt++; + } + } + + return ud_result_cnt; +} + +int maat_add_group_to_compile(struct maat_compile **compile_hash, struct maat_group_topology *group_topo, int group_id, int vt_id, + int clause_not_flag, int clause_index, int compile_id, struct log_handle *logger) +{ + struct maat_group *group = maat_group_topology_find_group(group_topo, group_id); + if (!group) { + group = maat_group_topology_add_group(group_topo, group_id); + } + + int ret = -1; + struct maat_compile *compile = maat_compile_hash_find(compile_hash, compile_id); + if (!compile) { + compile = maat_compile_new(compile_id); + ret = maat_compile_hash_add(compile_hash, compile_id, compile); + if (ret < 0) { + return -1; + } + } + + struct maat_literal_id literal_id = {group_id, vt_id}; + ret = maat_compile_clause_add_literal(compile, &literal_id, clause_index, clause_not_flag); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, "add literal_id{group_id:%d, vt_id:%d} to clause %d of compile %d failed", + group_id, vt_id, clause_index, compile_id); + ret = -1; + } else { + group->ref_by_compile_cnt++; + ret = 0; + } + + return ret; +} + +int maat_remove_group_from_compile(struct maat_compile **compile_hash, struct maat_group_topology *group_topo, int group_id, int vt_id, + int clause_not_flag, int clause_index, int compile_id, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + struct maat_group *group = maat_group_topology_find_group(group_topo, group_id); + if (!group) { + log_error(logger, MODULE_COMPILE, "Remove group %d from compile %d failed, group is not exisited.", + group_id, compile_id); + return -1; + } + + struct maat_compile *compile = NULL; + HASH_FIND(hh, *compile_hash, &compile_id, sizeof(compile_id), compile); + if (!compile) { + log_error(logger, MODULE_COMPILE, "Remove group %d from compile %d failed, compile is not exisited.", + group_id, compile_id); + return -1; + } + + struct maat_literal_id literal_id = {group_id, vt_id}; + int ret = maat_compile_clause_remove_literal(compile, &literal_id, clause_index); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "Remove group %d vt_id %d from clause %d of compile %d failed, literal is not in compile.", + group_id, vt_id, clause_index, compile_id); + return -1; + } + + if (0 == compile->actual_clause_num && NULL == compile->user_data) { + HASH_DEL(*compile_hash, compile); + maat_garbage_bagging(garbage_bin, compile, (void (*)(void*))maat_compile_free); + } + + return 0; +} + +static inline int compare_clause_id(const void *a, const void *b) +{ + long long ret = *(const unsigned long long *)a - *(const unsigned long long *)b; + + if (0 == ret) { + return 0; + } else if(ret < 0) { + return -1; + } else { + return 1; + } +} + +struct maat_compile_state *maat_compile_state_new(int thread_id) +{ + struct maat_compile_state *compile_state = ALLOC(struct maat_compile_state, 1); + + compile_state->thread_id = thread_id; + //compile_state->hier_ver = hier->version; + utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd); + utarray_new(compile_state->all_hit_clause_array, &ut_clause_id_icd); + utarray_new(compile_state->this_scan_hit_clause_ids, &ut_clause_id_icd); + + return compile_state; +} + +void maat_compile_state_free(struct maat_compile_state *compile_state) +{ + utarray_free(compile_state->internal_hit_paths); + utarray_free(compile_state->all_hit_clause_array); + utarray_free(compile_state->this_scan_hit_clause_ids); + free(compile_state); +} + +static int maat_compile_hit_path_add(UT_array* hit_paths, int item_id, int group_id, int virtual_table_id, int Nth_scan, int Nth_item_result) +{ + struct maat_internal_hit_path new_path; + + new_path.item_id = item_id; + new_path.Nth_hit_item = Nth_item_result; + new_path.Nth_scan = Nth_scan; + new_path.group_id = group_id; + new_path.virtual_table_id = virtual_table_id; + + utarray_push_back(hit_paths, &new_path); + return 1; +} + +static int maat_compile_has_literal(struct maat_compile* compile, struct maat_literal_id *literal_id) +{ + int i = 0; + struct maat_literal_id *tmp = NULL; + struct maat_clause_state *clause_state = NULL; + + for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states+i; + if(!clause_state->in_use) { + continue; + } + + tmp = (struct maat_literal_id*)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); + if (tmp) { + assert(tmp->group_id == literal_id->group_id && tmp->vt_id == literal_id->vt_id); + return 1; + } + } + + return 0; +} + +static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths, size_t n_path, const struct maat_hit_path *find) +{ + for (size_t i = 0; i < n_path; i++) { + if (0 == memcmp(hit_paths + i, find, sizeof(*find))) { + return 1; + } + } + + return 0; +} + +size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct maat_group_topology *group_topo, + struct maat_compile_state *compile_state, + struct maat_hit_path *hit_paths, size_t hit_path_size) +{ + size_t i = 0, j = 0; + struct maat_internal_hit_path *internal_path = NULL; + struct maat_group *group = NULL; + size_t n_made_by_item = 0, n_made_by_compile = 0; + + for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) { + internal_path = (struct maat_internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i); + group = maat_group_topology_find_group(group_topo, internal_path->group_id); + + if (0 == group->top_group_cnt && n_made_by_item < hit_path_size) { + //group not referenced by compile + hit_paths[n_made_by_item].Nth_scan = internal_path->Nth_scan; + hit_paths[n_made_by_item].item_id = internal_path->item_id; + hit_paths[n_made_by_item].sub_group_id = group->group_id; + hit_paths[n_made_by_item].top_group_id = -1; + hit_paths[n_made_by_item].virtual_table_id = internal_path->virtual_table_id; + hit_paths[n_made_by_item].compile_id = -1; + n_made_by_item++; + } else { + for (j = 0; j < group->top_group_cnt && n_made_by_item < hit_path_size; j++, n_made_by_item++) { + hit_paths[n_made_by_item].Nth_scan = internal_path->Nth_scan; + hit_paths[n_made_by_item].item_id = internal_path->item_id; + hit_paths[n_made_by_item].sub_group_id = group->group_id; + hit_paths[n_made_by_item].top_group_id = group->top_group_ids[j]; + hit_paths[n_made_by_item].virtual_table_id = internal_path->virtual_table_id; + hit_paths[n_made_by_item].compile_id = -1; + } + } + } + + struct maat_compile *compile = NULL; + struct maat_literal_id literal_id = {0, 0}; + struct maat_hit_path tmp_path; + struct bool_expr_match *expr_match = ALLOC(struct bool_expr_match, MAX_SCANNER_HIT_COMPILE_NUM); + int bool_match_ret = bool_matcher_match(compile_rt->bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clause_array, 0), + utarray_len(compile_state->all_hit_clause_array), expr_match, MAX_SCANNER_HIT_COMPILE_NUM); + for (int idx = 0; idx < bool_match_ret; idx++) { + compile = (struct maat_compile *)expr_match[idx].user_tag; + assert(compile->magic == MAAT_COMPILE_MAGIC); + assert((unsigned long long)compile->compile_id == expr_match[idx].expr_id); + if (0 == compile->actual_clause_num) { + continue; + } + + for (j = 0; j < n_made_by_item && (n_made_by_item+n_made_by_compile) < hit_path_size; j++) { + if (hit_paths[j].top_group_id<0) { + continue; + } + + literal_id.group_id = hit_paths[j].top_group_id; + literal_id.vt_id = hit_paths[j].virtual_table_id; + if (maat_compile_has_literal(compile, &literal_id)) { + if (hit_paths[j].compile_id < 0) { + hit_paths[j].compile_id = compile->compile_id; + } else { + tmp_path = hit_paths[j]; + tmp_path.compile_id = compile->compile_id; + if(maat_compile_is_hit_path_existed(hit_paths, n_made_by_item + n_made_by_compile, &tmp_path)) { + hit_paths[n_made_by_item + n_made_by_compile] = tmp_path; + n_made_by_compile++; + } + } + } + } + } + + return n_made_by_item + n_made_by_compile; +} + +void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id, + int Nth_scan, int Nth_item_result) +{ + if (compile_state->Nth_scan != Nth_scan) { + assert(compile_state->this_scan_item_hit_cnt == 0); + compile_state->Nth_scan = Nth_scan; + utarray_clear(compile_state->this_scan_hit_clause_ids); + } + + int ret = maat_compile_hit_path_add(compile_state->internal_hit_paths, item_id, group_id, virtual_table_id, Nth_scan, Nth_item_result); + if (!ret) { + return; + } + + compile_state->hit_path_cnt++; + compile_state->this_scan_item_hit_cnt++; +} + +void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, struct maat_compile **compile_hash, + int group_id, int vt_id) +{ + struct maat_compile *compile = NULL, *tmp_compile = NULL; + struct maat_clause_state *clause_state = NULL; + struct maat_literal_id literal_id = {group_id, vt_id}; + + HASH_ITER(hh, *compile_hash, compile, tmp_compile) { + for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states + i; + if (!clause_state->in_use) { + continue; + } + + struct maat_literal_id *tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, &literal_id, compare_literal_id); + if (tmp) { + //Deduplication + if (utarray_find(compile_state->all_hit_clause_array, &(clause_state->clause_id), compare_clause_id)) { + continue; + } + utarray_push_back(compile_state->all_hit_clause_array, &(clause_state->clause_id)); + utarray_sort(compile_state->all_hit_clause_array, compare_clause_id); + } + } + } +} + +int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state) +{ + return compile_state->not_clause_hitted_flag; +} + +int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, + int valid_column) +{ + if (NULL == compile_runtime || NULL == compile_schema || NULL == line) { + return -1; + } + + int ret = -1; + struct maat_compile *compile = NULL; + struct compile_item *compile_item = NULL; + struct compile_schema *schema = (struct compile_schema *)compile_schema; + struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; + int is_valid = get_column_value(line, valid_column); + int compile_id = get_column_value(line, schema->compile_id_column); + if (is_valid < 0) { + return -1; + } else if (0 == is_valid) { + //delete + ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id, compile_rt->ref_garbage_bin); + if (ret < 0) { + log_error(logger, MODULE_COMPILE, + "remove compile table(table_id:%d) compile(compile_id:%d) from compile_hash failed", + schema->table_id, compile_id); + return -1; + } + } else { + //add + compile_item = compile_item_new(line, schema, compile_rt->logger); + struct compile_rule *compile_rule = ALLOC(struct compile_rule, 1); + compile_item_to_compile_rule(compile_item, schema, compile_rule); + compile_item_free(compile_item); + compile_item = NULL; + + compile = maat_compile_new(compile_rule->compile_id); + if (NULL == compile) { + destroy_compile_rule(compile_rule); + log_error(logger, MODULE_COMPILE, + "maat_compile_new failed, compile_table(table_id:%d) compile_id:%d", + schema->table_id, compile_item->compile_id); + return -1; + } + + maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule, (void (*)(void *))destroy_compile_rule); + ret = maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile); + if (ret < 0) { + maat_compile_free(compile); + log_error(logger, MODULE_TABLE_RUNTIME, + "add compile table(table_id:%d) compile(compile_id:%d) to compile_hash failed", + schema->table_id, compile_id); + return -1; + } + } + + return 0; +} + +int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line, + int valid_column) +{ + if (NULL == g2c_runtime || NULL == g2c_schema || NULL == line) { + return -1; + } + + struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema; + struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; + struct compile_runtime *compile_rt = g2c_rt->ref_compile_rt; + struct group2group_runtime *g2g_rt = g2c_rt->ref_g2g_rt; + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } + + int ret = -1; + struct group2compile_item *g2c_item = group2compile_item_new(line, g2c_schema, compile_rt->logger); + if (0 == is_valid) { + //delete + ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2g_rt->group_topo, g2c_item->group_id, + g2c_item->vt_id, g2c_item->not_flag, g2c_item->clause_index, + g2c_item->compile_id, compile_rt->ref_garbage_bin, compile_rt->logger); + if (0 == ret) { + if (g2c_item->not_flag) { + g2c_rt->not_flag_group--; + } + } + } else { + //add + ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2g_rt->group_topo, g2c_item->group_id, + g2c_item->vt_id, g2c_item->not_flag, g2c_item->clause_index, + g2c_item->compile_id, compile_rt->logger); + if (0 == ret) { + if (g2c_item->not_flag) { + g2c_rt->not_flag_group++; + } + } + } + + group2compile_item_free(g2c_item); + return ret; +} + +int compile_runtime_commit(void *compile_runtime) +{ + struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; + int ret = 0; + + size_t compile_cnt = maat_compile_hash_count(compile_rt->compile_hash); + if (0 == compile_cnt) { + return 0; + } + + struct bool_matcher *old_bool_matcher = NULL; + struct bool_matcher *new_bool_matcher = NULL; + + log_info(logger, MODULE_TABLE_RUNTIME, + "committing %zu compile rules for rebuilding compile bool_matcher engine", compile_cnt); + new_bool_matcher = maat_compile_bool_matcher_new(compile_rt->compile_hash, &compile_rt->clause_id_generator, logger); + if (NULL == new_bool_matcher) { + log_error(logger, MODULE_TABLE_RUNTIME, + "rebuild compile bool_matcher engine failed when update %zu compile rules", compile_cnt); + ret = -1; + } + + old_bool_matcher = compile_rt->bm; + compile_rt->bm = new_bool_matcher; + maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))maat_compile_bool_matcher_free); + + compile_rt->rule_num = compile_cnt; + + return ret; +} + +void *rule_ex_data_new(const struct maat_rule_head *rule_head, const char *srv_def, const struct compile_ex_data_schema *ex_schema) +{ + void *ex_data = NULL; + struct maat_rule rule; + + fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); + ex_schema->new_func(ex_schema->idx, &rule, srv_def, &ex_data, ex_schema->argl, ex_schema->argp); + + return ex_data; +} + +void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, void *ex_data, + const struct compile_ex_data_schema *ex_schema) +{ + struct maat_rule rule; + + memset(&rule, 0, sizeof(rule)); + fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); + ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, ex_schema->argl, ex_schema->argp); +} + +void compile_item_to_compile_rule(struct compile_item *compile_item, + struct compile_schema *compile_schema, + struct compile_rule *compile_rule) +{ + struct maat_rule_head rule_head; + rule_head.config_id = compile_item->compile_id; + rule_head.service_id = compile_item->service_id; + rule_head.action = compile_item->action; + rule_head.do_blacklist = compile_item->do_blacklist; + rule_head.do_log = compile_item->do_log; + + compile_rule->magic_num = COMPILE_RULE_MAGIC; + compile_rule->head = rule_head; + compile_rule->declared_clause_num = compile_item->clause_num; + compile_rule->ex_data = ALLOC(void *, MAX_COMPILE_EX_DATA_NUM); + + compile_rule->ref_table = compile_schema; + compile_rule->head.serv_def_len = strlen(compile_item->user_region); + compile_rule->service_defined = ALLOC(char, compile_rule->head.serv_def_len); + memcpy(compile_rule->service_defined, compile_item->user_region, compile_rule->head.serv_def_len); + compile_rule->evaluation_order = compile_item->evaluation_order; + + size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(compile_schema); + for (size_t i = 0; i < n_rule_ex_schema; i++) { + struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(compile_schema, i); + compile_rule->ex_data[i] = rule_ex_data_new(&compile_rule->head, compile_rule->service_defined, ex_schema); + } + compile_rule->is_valid = 1; + compile_rule->compile_id = compile_item->compile_id; + pthread_rwlock_init(&compile_rule->rwlock, NULL); +} + +void destroy_compile_rule(struct compile_rule *compile_rule) +{ + struct table_schema *table_schema = compile_rule->ref_table; + assert(compile_rule->magic_num==COMPILE_RULE_MAGIC); + + size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(table_schema); + for (size_t i = 0; i < n_rule_ex_schema; i++) { + struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(table_schema, i); + rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, ex_schema); + compile_rule->ex_data[i] = NULL; + } + free(compile_rule->ex_data); + + compile_rule->is_valid = 0; + compile_rule->declared_clause_num = -1; + free(compile_rule->service_defined); + compile_rule->service_defined = NULL; + + free(compile_rule); +} + +struct group2compile_rule *g2c_item_to_g2c_rule(struct group2compile_item *g2c_item) +{ + struct group2compile_rule *g2c_rule = ALLOC(struct group2compile_rule, 1); + + g2c_rule->group_id = g2c_item->group_id; + g2c_rule->compile_id = g2c_item->compile_id; + g2c_rule->is_valid = g2c_item->is_valid; + g2c_rule->not_flag = g2c_item->not_flag; + g2c_rule->vt_id = g2c_item->vt_id; + g2c_rule->clause_index = g2c_item->clause_index; + g2c_rule->associated_compile_table_id = g2c_item->associated_compile_table_id; + + return g2c_rule; +} + +static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b) +{ + //If both of compile rule's evaluation order are specified, compile rule with small evaluation order is priority. + if (a->evaluation_order != 0 && b->evaluation_order != 0) { + if (a->evaluation_order - b->evaluation_order < 0) { + return -1; + } else if(a->evaluation_order - b->evaluation_order > 0) { + return 1; + } + } else if(a->evaluation_order + b->evaluation_order!= 0) { + //If one of compile rule's evaluation order is zero, compile rule with big evaluation order is priority. + return (a->evaluation_order - b->evaluation_order > 0) ? -1 : 1; + } + + //If compile rule's execute sequences are not specified or equal. + if (a->declared_clause_num != b->declared_clause_num) { + return (a->declared_clause_num - b->declared_clause_num); + } else { + return (b->compile_id - a->compile_id); + } +} + +static void compile_sort_para_set(struct compile_sort_para *para, const struct compile_rule *compile_relation, void *user) +{ + para->compile_id = compile_relation->compile_id; + para->evaluation_order = compile_relation->evaluation_order; + para->declared_clause_num = compile_relation->declared_clause_num; + para->user = user; +} + +static int compare_compile_rule(const void *a, const void *b) +{ + const struct compile_rule *ra = *(const struct compile_rule **)a; + const struct compile_rule *rb = *(const struct compile_rule **)b; + + struct compile_sort_para sa, sb; + compile_sort_para_set(&sa, ra, NULL); + compile_sort_para_set(&sb, rb, NULL); + + return compile_sort_para_compare(&sa, &sb); +} + +int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids, + int *compile_ids, size_t compile_ids_size, struct maat_state *state) +{ + if (NULL == compile_rt || table_rt->table_type != TABLE_TYPE_COMPILE) { + return -1; + } + + struct compile_runtime *compile_rt = &(table_rt->compile_rt); + struct maat_compile_state *compile_state = state->compile_mid; + struct compile_rule *compile_rule_array[compile_ids_size]; + + // all hit clause_id -> compile_id + size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt->bm, compile_state, (void **)compile_rule_array, compile_ids_size); + if (bool_match_ret > 0) { + qsort(compile_rule_array, bool_match_ret, sizeof(struct compile_rule *), compare_compile_rule); + } + + for (size_t i = 0; i < bool_match_ret; i++) { + compile_ids[i] = compile_rule_array[i]->compile_id; + } + + return MIN(bool_match_ret, compile_ids_size); +} \ No newline at end of file diff --git a/src/maat_ex_data.cpp b/src/maat_ex_data.cpp index 02c2a1e..8b19f1c 100644 --- a/src/maat_ex_data.cpp +++ b/src/maat_ex_data.cpp @@ -17,11 +17,18 @@ #include "rcu_hash.h" #include "utils.h" #include "maat_utils.h" -#include "maat_table_schema.h" #include "maat_ex_data.h" #define MODULE_EX_DATA module_name_str("maat.ex_data") +struct ex_data_schema { + maat_plugin_ex_new_func_t *new_func; + maat_plugin_ex_free_func_t *free_func; + maat_plugin_ex_dup_func_t *dup_func; + long argl; + void *argp; +}; + struct ex_data_runtime { UT_array *cache_rows; size_t cache_row_num; @@ -30,6 +37,8 @@ struct ex_data_runtime { struct rcu_hash_table *htable; struct ex_data_schema *ex_schema; int table_id; + + struct log_handle *logger; }; void cache_row_free(void *p) @@ -39,13 +48,15 @@ void cache_row_free(void *p) UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free}; -struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn) +struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, + struct log_handle *logger) { struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1); utarray_new(ex_data_rt->cache_rows, &ut_cache_row_icd); ex_data_rt->htable = rcu_hash_new(data_free_fn); ex_data_rt->table_id = table_id; + ex_data_rt->logger = logger; return ex_data_rt; } @@ -117,6 +128,27 @@ void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt) ex_data_rt->cache_size = 0; } +struct ex_data_schema *ex_data_schema_new(maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp) +{ + struct ex_data_schema *ex_schema = ALLOC(struct ex_data_schema, 1); + + ex_schema->new_func = new_func; + ex_schema->free_func = free_func; + ex_schema->dup_func = dup_func; + ex_schema->argl = argl; + ex_schema->argp = argp; + + return ex_schema; +} + +void ex_data_schema_free(struct ex_data_schema *ex_schema) +{ + FREE(ex_schema); +} + void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema) { ex_data_rt->ex_schema = schema; @@ -141,29 +173,73 @@ void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char return ex_data; } -void ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct ex_data_container *ex_container) +struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data) { - struct ex_data_container *tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); - if (tmp_container != NULL) { + struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); + + ex_container->ex_data = ex_data; + ex_container->custom_data = custom_data; + + return ex_container; +} + +void ex_data_container_free(void *ctx, void *data) +{ + if (NULL == ctx || NULL == data) { return; } + struct ex_container_ctx *container_ctx = (struct ex_container_ctx *)ctx; + long argl = container_ctx->ex_schema->argl; + void *argp = container_ctx->ex_schema->argp; + + struct ex_data_container *ex_container = (struct ex_data_container *)data; + if (ex_container->ex_data != NULL && container_ctx->ex_schema->free_func != NULL) { + container_ctx->ex_schema->free_func(container_ctx->table_id, &(ex_container->ex_data), argl, argp); + } + + if (ex_container->custom_data != NULL && container_ctx->custom_data_free != NULL) { + container_ctx->custom_data_free(ex_container->custom_data); + } + + FREE(ex_container); +} + +int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len, + struct ex_data_container *ex_container) +{ + struct ex_data_container *tmp_container = NULL; + + tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + if (tmp_container != NULL) { + log_error(ex_data_rt->logger, MODULE_EX_DATA, + "ex_data_runtime add ex container error: already exist same key:%s", key); + return -1; + } + rcu_hash_add(ex_data_rt->htable, key, key_len, ex_container); + + return 0; } -void ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, - struct log_handle *logger) +int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len) { - struct ex_data_container *tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + struct ex_data_container *tmp_container = NULL; + tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); if (NULL == tmp_container) { - log_error(logger, MODULE_EX_DATA, "ex data del error: no such key:%s", key); - return; + log_error(ex_data_rt->logger, MODULE_EX_DATA, + "ex_data_runtime del ex container error: no such key:%s", key); + return -1; } rcu_hash_del(ex_data_rt->htable, key, key_len); + + return 0; } -void *ex_data_runtime_dup_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) +void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) { struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); if (NULL == ex_container) { diff --git a/src/maat_expr.cpp b/src/maat_expr.cpp new file mode 100644 index 0000000..413487b --- /dev/null +++ b/src/maat_expr.cpp @@ -0,0 +1,699 @@ +/* +********************************************************************************************** +* File: maat_expr.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include +#include + +#include "utils.h" +#include "maat_expr.h" +#include "adapter_hs.h" +#include "maat_utils.h" +#include "maat_kv.h" +#include "maat_limits.h" +#include "rcu_hash.h" +#include "maat_rule.h" +#include "maat_garbage_collection.h" + +#define MAX_DISTRICT_STR 128 + +#define MODULE_EXPR module_name_str("maat.expr") + +struct expr_schema { + int item_id_column; + int group_id_column; + int district_column; + int keywords_column; + int expr_type_column; + int match_method_column; + int is_hexbin_column; + enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ +}; + +enum expr_type { + EXPR_TYPE_STRING = 0, + EXPR_TYPE_AND, + EXPR_TYPE_REGEX, + EXPR_TYPE_MAX +}; + +enum match_method { + MATCH_METHOD_SUB = 0, + MATCH_METHOD_RIGHT, + MATCH_METHOD_LEFT, + MATCH_METHOD_COMPLETE, + MATCH_METHOD_MAX +}; + +struct expr_item { + int item_id; + int group_id; + char district[MAX_DISTRICT_STR]; + char keywords[MAX_KEYWORDS_STR]; + enum expr_type expr_type; + enum match_method match_method; + int is_hexbin; + int is_case_sensitive; +}; + +struct expr_runtime { + enum hs_scan_mode scan_mode; + struct adapter_hs *hs; + struct adapter_hs_stream *hs_stream; + struct rcu_hash_table *htable; + struct group2group_runtime *ref_g2g_rt; + + uint32_t rule_num; + uint32_t updating_rule_num; + struct maat_item *item_hash; + void (*item_user_data_free)(void *); + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; + + // long long *scan_cnt; + // long long *hit_cnt; + // long long *not_grp_hit_cnt; + // long long *stream_num; +}; + +enum expr_type int_to_expr_type(int expr_type) +{ + enum expr_type type = EXPR_TYPE_MAX; + + switch (expr_type) { + case 0: + type = EXPR_TYPE_STRING; + break; + case 1: + type = EXPR_TYPE_AND; + break; + case 2: + type = EXPR_TYPE_REGEX; + break; + default: + break; + } + + return type; +} + +enum match_method int_to_match_method_type(int match_method_type) +{ + enum match_method type = MATCH_METHOD_MAX; + + switch (match_method_type) { + case 0: + type = MATCH_METHOD_SUB; + break; + case 1: + type = MATCH_METHOD_RIGHT; + break; + case 2: + type = MATCH_METHOD_LEFT; + break; + case 3: + type = MATCH_METHOD_COMPLETE; + break; + default: + break; + } + + return type; +} + +struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schema, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + int db_hexbin = -1; + int expr_type = -1; + int match_method_type = -1; + struct expr_item *expr_item = ALLOC(struct expr_item, 1); + + int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len); + if (ret < 0) { + goto error; + } + expr_item->item_id = atoi(line + column_offset); + + ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len); + if (ret < 0) { + goto error; + } + expr_item->group_id = atoi(line + column_offset); + + //TODO + #if 0 + if (table_item->table_type == TABLE_TYPE_EXPR_PLUS) { + ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + + if (column_len >= MAX_DISTRICT_STR) { + log_error(logger, MODULE_EXPR, + "update error: expr table[%s]:item_id[%d] district length too long", + table_name, expr_item->item_id); + return -1; + } + memcpy(expr_item->district, (line + column_offset), column_len); + } + #endif + ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); + if (ret < 0) { + goto error; + } + + if (column_len >= MAX_KEYWORDS_STR) { + log_error(logger, MODULE_EXPR, + "update error: expr table[%s]:item_id[%d] keywords length too long", + table_name, expr_item->item_id); + goto error; + } + memcpy(expr_item->keywords, (line + column_offset), column_len); + + ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len); + if (ret < 0) { + goto error; + } + + expr_type = atoi(line + column_offset); + expr_item->expr_type = int_to_expr_type(expr_type); + + ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len); + if (ret < 0) { + goto error; + } + + match_method_type = atoi(line + column_offset); + expr_item->match_method = int_to_match_method_type(match_method_type); + + ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len); + if (ret < 0) { + goto error; + } + db_hexbin = atoi(line + column_offset); + + switch (db_hexbin) { + case 0: + expr_item->is_hexbin = FALSE; + expr_item->is_case_sensitive = FALSE; + break; + case 1: + expr_item->is_hexbin = TRUE; + expr_item->is_case_sensitive = TRUE; + break; + case 2: + expr_item->is_hexbin = FALSE; + expr_item->is_case_sensitive = TRUE; + break; + default: + log_error(logger, MODULE_EXPR, + "update error: expr table[%s]:item_id[%d] invalid hexbin value:%d", + table_name, expr_item->item_id, db_hexbin); + goto error; + } + + return expr_item; +error: + FREE(expr_item); + return NULL; +} + +void expr_item_free(struct expr_item *expr_item) +{ + FREE(expr_item); +} + +void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +{ + int read_cnt = 0; + struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1); + struct maat_kv_store *scan_mode_map = maat_kv_store_new(); + + maat_kv_register(scan_mode_map, "block", HS_SCAN_MODE_BLOCK); + maat_kv_register(scan_mode_map, "stream", HS_SCAN_MODE_STREAM); + + int ret = -1; + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "custom"); + if (item == NULL || item->type != cJSON_Object) { + log_error(logger, MODULE_EXPR, "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "scan_mode"); + if (custom_item != NULL && custom_item->type == cJSON_String) { + ret = maat_kv_read(scan_mode_map, custom_item->valuestring, (int*)&(expr_schema->scan_mode)); + if (ret < 0) { + log_error(logger, MODULE_EXPR, "scan_mode %s illegal", custom_item->valuestring); + goto error; + } + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "item_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + expr_schema->item_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "group_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + expr_schema->group_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "keywords"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + expr_schema->keywords_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "district"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + expr_schema->district_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "expr_type"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + expr_schema->expr_type_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "match_method"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + expr_schema->match_method_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "is_hexbin"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + expr_schema->is_hexbin_column = custom_item->valueint; + read_cnt++; + } + + if (read_cnt < 8) { + goto error; + } + + maat_kv_store_free(scan_mode_map); + return expr_schema; +error: + maat_kv_store_free(scan_mode_map); + FREE(expr_schema); + return NULL; +} + +void expr_schema_free(void *expr_schema) +{ + FREE(expr_schema); +} + +void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == expr_schema) { + return NULL; + } + + struct expr_schema *schema = (struct expr_schema *)expr_schema; + struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1); + + expr_rt->htable = rcu_hash_new(expr_ex_data_free); + expr_rt->scan_mode = schema->scan_mode; + expr_rt->item_user_data_free = maat_item_inner_free; + expr_rt->ref_garbage_bin = garbage_bin; + expr_rt->logger = logger; + + // expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + // expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); + // expr_rt->not_grp_hit_cnt = alignment_int64_array_alloc(max_thread_num); + // expr_rt->stream_num = alignment_int64_array_alloc(max_thread_num); + + return expr_rt; +} + +void expr_runtime_free(void *expr_runtime) +{ + if (NULL == expr_runtime) { + return; + } + + struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; + if (expr_rt->hs != NULL) { + adapter_hs_destroy(expr_rt->hs); + expr_rt->hs = NULL; + } + + if (expr_rt->hs_stream != NULL) { + adapter_hs_stream_close(expr_rt->hs_stream); + expr_rt->hs_stream = NULL; + } + + if (expr_rt->htable != NULL) { + rcu_hash_free(expr_rt->htable); + expr_rt->htable = NULL; + } + + struct maat_item *item = NULL, *tmp = NULL; + HASH_ITER(hh, expr_rt->item_hash, item, tmp) { + HASH_DELETE(hh, expr_rt->item_hash, item); + maat_item_free(item, expr_rt->item_user_data_free); + } + FREE(expr_rt); +} + +int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len, + and_expr_t *expr_rule, int is_valid, struct log_handle *logger) +{ + void *data = NULL; + + if (0 == is_valid) { + //delete + data = rcu_hash_find(expr_rt->htable, key, key_len); + if (NULL == data) { + log_error(logger, MODULE_EXPR, + "the key of expr rule not exist, can't be deleted, expr_id:%d", expr_rule->expr_id); + return -1; + } + rcu_hash_del(expr_rt->htable, key, key_len); + } else { + //add + data = rcu_hash_find(expr_rt->htable, key, key_len); + if (data != NULL) { + log_error(logger, MODULE_EXPR, + "the key of expr rule already exist, can't be added, expr_id:%d", expr_rule->expr_id); + return -1; + } + rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule); + } + + return 0; +} + +and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger) +{ + size_t i = 0; + size_t sub_expr_cnt = 0; + char *pos = NULL; + char *saveptr = NULL; + char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM]; + and_expr_t *expr_rule = ALLOC(and_expr_t, 1); + + switch (expr_item->expr_type) { + case EXPR_TYPE_AND: + case EXPR_TYPE_REGEX: + for (i = 0, pos = expr_item->keywords; ; i++, pos = NULL) { + char *tmp = strtok_r_esc(pos, '&', &saveptr); + if (NULL == tmp) { + break; + } + + if (i >= MAAT_MAX_EXPR_ITEM_NUM) { + log_error(logger, MODULE_TABLE_RUNTIME, "expr item_id:%d too many patterns", + expr_item->item_id); + return NULL; + } + + sub_key_array[i] = tmp; + if (expr_item->expr_type == EXPR_TYPE_REGEX) { + sub_key_array[i] = str_unescape_and(sub_key_array[i]); + } else { + sub_key_array[i] = str_unescape(sub_key_array[i]); + } + } + sub_expr_cnt = i; + break; + case EXPR_TYPE_STRING: + sub_expr_cnt = 1; + sub_key_array[0] = expr_item->keywords; + sub_key_array[0] = str_unescape(sub_key_array[0]); + break; + default: + break; + } + + for (i = 0; i < sub_expr_cnt; i++) { + expr_rule->expr_id = expr_item->item_id; + expr_rule->patterns[i].pat = ALLOC(char, strlen(sub_key_array[i])); + memcpy(expr_rule->patterns[i].pat, sub_key_array[i], strlen(sub_key_array[i])); + expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]); + expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); + } + expr_rule->n_patterns = sub_expr_cnt; + + return expr_rule; +} + +int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column) +{ + if (NULL == expr_runtime || NULL == expr_schema) { + return -1; + } + + int ret = -1; + struct maat_item_inner *u_para = NULL; + struct maat_item *item = NULL; + and_expr_t *expr_rule = NULL; + struct expr_schema *schema = (struct expr_schema *)expr_schema; + struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; + + int item_id = get_column_value(line, schema->item_id_column); + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } else if (0 == is_valid) { + //delete + HASH_FIND_INT(expr_rt->item_hash, &item_id, item); + if (NULL == item) { + return -1; + } + + u_para = (struct maat_item_inner *)item->user_data; + item->user_data = NULL; + + if (NULL == u_para) { + return -1; + } + + HASH_DELETE(hh, expr_rt->item_hash, item); + maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free); + } else { + //add + HASH_FIND_INT(expr_rt->item_hash, &item_id, item); + if (item) { + log_error(expr_rt->logger, MODULE_EXPR, + "expr runtime add item %d to item_hash failed, already exist", item_id); + return -1; + } + + struct expr_item *expr_item = expr_item_new(line, schema, expr_rt->logger); + if (NULL == expr_item) { + log_error(expr_rt->logger, MODULE_EXPR, "expr line %s to item failed", line); + return -1; + } + + // TODO: by luis + //int district_id = get_district_id(maat_rt, expr_item->district); + int district_id = -1; + u_para = maat_item_inner_new(expr_item->group_id, item_id, district_id); + item = maat_item_new(item_id, group_id, u_para); + HASH_ADD_INT(expr_rt->item_hash, item_id, item); + + expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger); + expr_item_free(expr_item); + if (NULL == expr_rule) { + log_error(expr_rt->logger, MODULE_EXPR, "transform expr table:%s item to expr_rule failed, item_id:%d", + table_name, item_id); + return -1; + } + } + + char *key = (char *)&item_id; + ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, is_valid, expr_rt->logger); + if (ret < 0) { + if (expr_rule != NULL) { + expr_rule_free(expr_rule); + } + return -1; + } else { + if (0 == is_valid) { + expr_rt->rule_num--; + } else { + expr_rt->rule_num++; + } + } + + return 0; +} + +int expr_runtime_commit(void *expr_runtime) +{ + if (NULL == expr_runtime) { + return -1; + } + + int ret = 0; + struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; + void **ex_data_array = NULL; + + size_t rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array); + if (0 == rule_cnt) { + FREE(ex_data_array); + return 0; + } + + and_expr_t *rules = ALLOC(and_expr_t, rule_cnt); + + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(and_expr_t *)ex_data_array[i]; + } + + struct adapter_hs *new_adapter_hs = NULL; + struct adapter_hs *old_adapter_hs = NULL; + + log_info(expr_rt->logger, MODULE_EXPR, + "committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt); + new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt, expr_rt->logger); + if (NULL == new_adapter_hs) { + log_error(expr_rt->logger, MODULE_EXPR, + "rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt); + ret = -1; + } + + old_adapter_hs = expr_rt->hs; + expr_rt->hs = new_adapter_hs; + + maat_garbage_bagging(table_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy); + rcu_hash_commit(expr_rt->htable); + expr_rt->rule_num = rcu_hash_count(expr_rt->htable); + rule_cnt = rcu_hash_updating_count(expr_rt->htable); + assert(rule_cnt == 0); + + FREE(rules); + FREE(ex_data_array); + + return ret; +} + +int expr_runtime_updating_flag(struct expr_runtime *expr_rt) +{ + return rcu_hash_updating_flag(expr_rt->htable); +} + +void expr_rule_free(and_expr_t *expr_rule) +{ + if (NULL == expr_rule) { + return; + } + + for (size_t i = 0; i < expr_rule->n_patterns; i++) { + FREE(expr_rule->patterns[i].pat); + } + + FREE(expr_rule); +} + +void expr_ex_data_free(void *user_ctx, void *data) +{ + and_expr_t *expr_rule = (and_expr_t *)data; + expr_rule_free(expr_rule); +} + +int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len, + int group_id_array[], size_t n_group_id_array, int virtual_table_id, struct maat_state *state) +{ + if (NULL == table_rt) { + return -1; + } + + int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; + size_t n_hit_item = 0; + int ret = adapter_hs_scan(table_rt->expr_rt.hs, thread_id, data, data_len, hit_item_ids, &n_hit_item); + if (ret < 0) { + return -1; + } + if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { + n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; + } + + struct maat_compile_state *compile_state = state->compile_mid; + //tranform item_id to group_id + struct maat_item *item = NULL; + size_t n_group_id = 0; + size_t i = 0; + for (i = 0; i < n_hit_item; i++) { + HASH_FIND_INT(table_rt->item_hash, &(hit_item_ids[i]), item); + assert(item != NULL); + if (!item) { + // should not come here + continue; + } + + if (n_group_id >= n_group_id_array) { + n_group_id = n_group_id_array; + //Prevent group_id_array out of bounds + } else { + group_id_array[n_group_id++] = item->group_id; + } + + maat_compile_state_update_hit_path(compile_state, hit_item_ids[i], item->group_id, virtual_table_id, state->scan_cnt, i); + } + + // literal_id{group_id,vt_id} to clause_id + // STEP 1: get compile table runtime + int compile_table_id = -1; + if (state->compile_table_id == -1) { + compile_table_id = state->maat_instance->default_compile_table_id; + } else { + compile_table_id = state->compile_table_id; + } + struct maat_runtime *maat_rt = state->maat_instance->maat_rt; + struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, compile_table_id); + assert(compile_table_rt->table_type == TABLE_TYPE_COMPILE); + + // STEP 2: get the specified compile table's hit clause_id array by literal_id + for (i = 0; i < n_group_id; i++) { + maat_compile_state_update_hit_clause(compile_state, &(compile_table_rt->compile_rt.compile_hash), group_id_array[i], virtual_table_id); + } + + return n_group_id; +} + +void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) +{ + if (NULL == table_rt) { + return; + } + + struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(table_rt->expr_rt.hs, thread_id); + table_rt->expr_rt.hs_stream = hs_stream; +} + +int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, + int result[], size_t *n_result) +{ + if (NULL == table_rt) { + return -1; + } + + return adapter_hs_scan_stream(table_rt->expr_rt.hs_stream, data, data_len, result, n_result); +} + +void expr_runtime_stream_close(struct expr_runtime *expr_rt) +{ + if (table_rt != NULL) { + adapter_hs_stream_close(table_rt->expr_rt.hs_stream); + table_rt->expr_rt.hs_stream = NULL; + } +} diff --git a/src/maat_group.cpp b/src/maat_group.cpp index 1f33255..a5979e4 100644 --- a/src/maat_group.cpp +++ b/src/maat_group.cpp @@ -20,6 +20,27 @@ #define MODULE_GROUP module_name_str("maat.group") +struct group2group_item { + int group_id; + int superior_group_id; +}; + +struct group2group_schema { + int group_id_column; + int superior_group_id_column; + int table_id;//ugly +}; + +struct group2group_runtime { + struct maat_group_topology *group_topo; + + uint32_t rule_num; + uint32_t updating_rule_num; + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; +}; + struct maat_group_topology { struct maat_group *hash_group_by_id; //key: group_id, value: struct maat_group *. struct maat_group *hash_group_by_vertex; //key: vetex_id, value: struct maat_group *. Multimap (Items with multiple keys). @@ -32,6 +53,116 @@ struct maat_group_topology { struct log_handle *logger; }; +void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +{ + int read_cnt = 0; + struct group2group_schema *g2g_schema = ALLOC(struct group2group_schema, 1); + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + goto error; + } + g2g_schema->table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "custom"); + if (item == NULL || item->type != cJSON_Object) { + log_error(logger, MODULE_GROUP, "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "group_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + g2g_schema->group_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "superior_group_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + g2g_schema->superior_group_id_column = custom_item->valueint; + read_cnt++; + } + + if (read_cnt < 2) { + goto error; + } + + return g2g_schema; +error: + FREE(g2c_schema); + return NULL; +} + +void group2group_schema_free(void *g2g_schema) +{ + FREE(g2g_schema); +} + +struct group2group_runtime * +group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + struct group2group_runtime *g2g_rt = ALLOC(struct group2group_runtime, 1); + + g2g_rt->group_topo = maat_group_topology_new(logger); + g2g_rt->ref_garbage_bin = garbage_bin; + g2g_rt->logger = logger; + + return g2g_rt; +} + +void group2group_runtime_free(void *g2g_runtime) +{ + if (NULL == g2g_runtime) { + return; + } + + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + + if (g2g_rt->group_topo != NULL) { + maat_group_topology_free(g2g_rt->group_topo); + } + + FREE(g2g_rt); +} + +struct group2group_item * +group2group_item_new(const char *line, struct group2group_schema *g2g_schema, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + struct group2group_item *g2g_item = ALLOC(struct group2group_item, 1); + + int ret = get_column_pos(line, g2g_schema->group_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_GROUP, + "group2group table(table_id:%d) line:%s has no group_id", + g2g_schema->table_id, line); + goto error; + } + g2g_item->group_id = atoi(line + column_offset); + + ret = get_column_pos(line, g2g_schema->superior_group_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_GROUP, + "group2group table(table_id:%d) line:%s has no superior_group_id", + g2g_schema->table_id, line); + goto error; + } + g2g_item->superior_group_id = atoi(line + column_offset); + + return g2g_item; +error: + FREE(g2g_item); + return NULL; +} + +void group2group_item_free(struct group2group_item *g2g_item) +{ + FREE(g2g_item); +} + void group_vertex_free(struct maat_group *group) { free(group->top_group_ids); @@ -260,8 +391,7 @@ int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo) //Orphan, Not reference by any one, free it. if (0 == group->ref_by_compile_cnt && 0 == group->ref_by_superior_group_cnt - && 0 == group->ref_by_subordinate_group_cnt - && 0 == group->ref_by_item_cnt) { + && 0 == group->ref_by_subordinate_group_cnt) { FREE(group->top_group_ids); maat_group_topology_remove_group(group_topo, group); @@ -269,8 +399,7 @@ int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo) } //A group is need to build top groups when it has items and referenced by superior groups or compiles. - if (group->ref_by_item_cnt > 0 && - (group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0)) { + if (group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0) { if (0 == group->ref_by_superior_group_cnt) { //fast path, group is only referenced by compile rules. top_group_cnt = 1; @@ -310,4 +439,74 @@ int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo) igraph_vector_destroy(&group_topo->dfs_vids); return 0; +} + +int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, + int valid_column) +{ + if (NULL == g2g_runtime || NULL == g2g_schema || NULL == line) { + return -1; + } + + int ret = -1; + struct group2group_item *g2g_item = NULL; + struct group2group_schema *schema = (struct group2group_schema *)g2g_schema; + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + int item_id = get_column_value(line, schema->table_id); + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } else if (0 == is_valid) { + //delete + ret = maat_group_topology_remove_group_from_group(g2g_rt->group_topo, g2g_item->group_id, g2g_item->superior_group_id); + } else { + //add + g2g_item = group2group_item_new(line, schema, logger); + } + + if (1 == is_valid) { + //add + ret = maat_group_topology_add_group_to_group(g2g_rt->group_topo, g2g_item->group_id, g2g_item->superior_group_id); + } else { + //delete + + } + + return ret; +} + +int group2group_runtime_commit(void *g2g_runtime) +{ + if (NULL == g2g_runtime) { + return -1; + } + + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + int ret = maat_group_topology_build_top_groups(g2g_rt->group_topo); + + return ret; +} + +int group2group_runtime_get_top_groups(struct group2group_runtime *g2g_rt, int *group_ids, size_t n_group_ids, + int *top_group_ids) +{ + if (NULL == table_rt || NULL == group_ids || 0 == n_group_ids || + table_rt->table_type != TABLE_TYPE_GROUP2GROUP) { + return -1; + } + + size_t top_group_index = 0; + struct group2group_runtime *g2g_rt = &(table_rt->g2g_rt); + for (size_t i = 0; i < n_group_ids; i++) { + struct maat_group *group = maat_group_topology_find_group(g2g_rt->group_topo, group_ids[i]); + if (!group) { + continue; + } + + for (size_t j = 0; j < group->top_group_cnt; j++) { + top_group_ids[top_group_index++] = group->top_group_ids[j]; + } + } + + return top_group_index; } \ No newline at end of file diff --git a/src/maat_hierarchy.cpp b/src/maat_hierarchy.cpp deleted file mode 100644 index e43ddfb..0000000 --- a/src/maat_hierarchy.cpp +++ /dev/null @@ -1,1149 +0,0 @@ -/* -********************************************************************************************** -* File: maat_hierarchy.cpp -* Description: -* Authors: Zheng Chao -* Date: 2022-10-31 -* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. -*********************************************************************************************** -*/ - -#include -#include - -#include "utils.h" -#include "maat_utils.h" -#include "log/log.h" -#include "uthash/utarray.h" -#include "uthash/uthash.h" -#include "bool_matcher.h" -#include "igraph/igraph.h" -#include "maat_hierarchy.h" -#include "maat_garbage_collection.h" -#include "maat_group.h" -#include "maat/maat.h" - -#define MODULE_HIERARCHY module_name_str("maat.hierarchy") - -struct maat_hierarchy_clause_state { - unsigned long long clause_id; - char not_flag; - char in_use; - UT_array *literal_ids; -}; - -struct maat_hierarchy_compile { - unsigned int magic; - int compile_id; - int actual_clause_num; - int declared_clause_num; - int not_clause_cnt; - void *user_data; - UT_hash_handle hh; - struct maat_hierarchy_clause_state clause_states[MAX_ITEMS_PER_BOOL_EXPR]; -}; - -struct maat_hierarchy_literal_id { - int group_id; - int vt_id; -}; - -struct maat_hierarchy_clause { - long long clause_id; - size_t n_literal_id; - struct maat_hierarchy_literal_id *literal_ids; - UT_hash_handle hh; -}; - -struct item2clause_key { - int item_id; - int vt_id; -}; - -struct group2item { - int group_id; - UT_array *item_ids; - UT_hash_handle hh; //index to -}; - -struct item2clause_value { - struct item2clause_key key; - UT_array *clause_ids; - int group_id; - UT_hash_handle hh; //index to -}; - -struct maat_hierarchy_internal_hit_path { - int Nth_scan; - int Nth_hit_item; - int item_id; - int virtual_table_id; -}; - -struct maat_hierarchy { - pthread_rwlock_t rwlock; - time_t version; //After full update, clause id may indicate a different clause. Comparing hier->version and mid->hier_ver can prevent false positive match. - int changed_flag; - - struct maat_hierarchy_compile *hash_compile_by_id; //key: compile_id, value: struct maat_hierarchy_compile*. - void (* compile_user_data_free)(void *compile_ud); - - struct maat_hierarchy_item *hash_item_by_id; //key: item_id, value: struct maat_hierarchy_item*. - - struct maat_hierarchy_clause *hash_dedup_clause_by_literals; //key: literal combination, value: struct maat_hierarchy_clause*. For generating unique clause_id. - unsigned long long clause_id_generator; //Increasing number. - - void (* item_user_data_free)(void *item_ud); - - /*Following members are accessed from scan threads.*/ - struct item2clause_value *hash_item2clause; //key: item_id+virtual_table_id, value: struct item2clause_value. - struct bool_matcher *bm; - - struct maat_group_topology *ref_group_topo; - int thread_num; - struct maat_garbage_bin *ref_garbage_bin; - struct log_handle *logger; - struct bool_expr_match *expr_match_buff; -}; - -struct maat_hierarchy_compile_mid { - int thread_id; - int Nth_scan; - time_t hier_ver; - size_t this_scan_item_hit_cnt; - int not_clause_hitted_flag; - int is_no_count_scan; - size_t hit_path_cnt; - - UT_array *internal_hit_paths; - UT_array *all_hit_clause_array; - UT_array *this_scan_hit_clause_ids; -}; - -struct maat_hierarchy *maat_hierarchy_new(int thread_num, struct maat_garbage_bin *bin, struct log_handle *logger) -{ - struct maat_hierarchy *hier = ALLOC(struct maat_hierarchy, 1); - UNUSED int ret = 0; - hier->logger = logger; - hier->thread_num = thread_num; - hier->version = time(NULL); - - hier->hash_compile_by_id = NULL; - hier->hash_item2clause = NULL; - hier->hash_item_by_id = NULL; - hier->hash_dedup_clause_by_literals = NULL; - hier->clause_id_generator = 0; - hier->ref_garbage_bin = bin; - hier->expr_match_buff = ALLOC(struct bool_expr_match, thread_num * MAX_SCANNER_HIT_NUM); - - ret = pthread_rwlock_init(&hier->rwlock, NULL); - assert(ret == 0); - - return hier; -} - -UT_icd ut_literal_id_icd = {sizeof(struct maat_hierarchy_literal_id), NULL, NULL, NULL}; -UT_icd ut_clause_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; -UT_icd ut_item_id_icd = {sizeof(int), NULL, NULL, NULL}; -UT_icd ut_hit_path_icd = {sizeof(struct maat_hierarchy_internal_hit_path), NULL, NULL, NULL}; - -#define MAAT_HIER_COMPILE_MAGIC 0x4a5b6c7d -static struct maat_hierarchy_compile *maat_hierarchy_compile_new(struct maat_hierarchy *hier, int compile_id) -{ - struct maat_hierarchy_compile *compile = NULL; - compile = ALLOC(struct maat_hierarchy_compile, 1); - compile->magic = MAAT_HIER_COMPILE_MAGIC; - compile->compile_id = compile_id; - HASH_ADD_INT(hier->hash_compile_by_id, compile_id, compile); - - for(int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - utarray_new(compile->clause_states[i].literal_ids, &ut_literal_id_icd); - compile->clause_states[i].in_use=0; - } - return compile; -} - -static void maat_hierarchy_compile_free(struct maat_hierarchy_compile *compile) -{ - struct maat_hierarchy_clause_state *clause_state = NULL; - //user_data must be freed before calling this function. - assert(compile->user_data == NULL); - - for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - clause_state = compile->clause_states + i; - utarray_free(clause_state->literal_ids); - clause_state->literal_ids = NULL; - clause_state->in_use = 0; - } - compile->magic = 0; - free(compile); -} - -void maat_hierarchy_free_item2clause_hash(struct item2clause_value* hash) -{ - struct item2clause_value *i2c_val = NULL, *tmp_i2c_val = NULL; - HASH_ITER(hh, hash, i2c_val, tmp_i2c_val) { - HASH_DEL(hash, i2c_val); - utarray_free(i2c_val->clause_ids); - free(i2c_val); - } - assert(hash == NULL); -} - -static struct maat_hierarchy_item * -maat_hierarchy_item_new(struct maat_hierarchy *hier, int item_id, int group_id, - struct maat_group *parent_group, void *user_data) -{ - struct maat_hierarchy_item *item = NULL; - item = ALLOC(struct maat_hierarchy_item, 1); - item->group_id = group_id; - item->item_id = item_id; - item->ref_parent_group = parent_group; - item->user_data = user_data; - HASH_ADD_INT(hier->hash_item_by_id, item_id, item); - parent_group->ref_by_item_cnt++; - - return item; -} - -static void maat_hierarchy_item_free(struct maat_hierarchy *hier, struct maat_hierarchy_item *item) -{ - HASH_DELETE(hh, hier->hash_item_by_id, item); - item->ref_parent_group->ref_by_item_cnt--; - - if (hier->item_user_data_free && item->user_data) { - hier->item_user_data_free(item->user_data); - item->user_data = NULL; - } - free(item); -} - -static const struct maat_hierarchy_clause * -maat_hierarchy_clause_fetch(struct maat_hierarchy *hier, struct maat_hierarchy_literal_id *literal_ids, - size_t n_literal_id) -{ - struct maat_hierarchy_clause *clause = NULL; - - HASH_FIND(hh, hier->hash_dedup_clause_by_literals, literal_ids, - n_literal_id * sizeof(struct maat_hierarchy_literal_id), clause); - if (!clause) { - clause = ALLOC(struct maat_hierarchy_clause, 1); - clause->clause_id = hier->clause_id_generator; - clause->n_literal_id = n_literal_id; - clause->literal_ids = ALLOC(struct maat_hierarchy_literal_id, n_literal_id); - memcpy(clause->literal_ids, literal_ids, n_literal_id * sizeof(struct maat_hierarchy_literal_id)); - - hier->clause_id_generator++; - HASH_ADD_KEYPTR(hh, hier->hash_dedup_clause_by_literals, clause->literal_ids, - n_literal_id * sizeof(struct maat_hierarchy_literal_id), clause); - } - - return clause; -} - -static void maat_hierarchy_clause_free(struct maat_hierarchy *hier, struct maat_hierarchy_clause *clause) -{ - HASH_DELETE(hh, hier->hash_dedup_clause_by_literals, clause); - free(clause->literal_ids); - clause->n_literal_id = 0; - free(clause); -} - -void maat_hierarchy_free(struct maat_hierarchy *hier) -{ - struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; - struct item2clause_value *i2c_val = NULL, *tmp_i2c_val = NULL; - struct maat_hierarchy_item *item = NULL, *tmp_item = NULL; - struct maat_hierarchy_clause *clause = NULL, *tmp_clause = NULL; - pthread_rwlock_wrlock(&hier->rwlock); - - //Reference: https://troydhanson.github.io/uthash/userguide.html#_what_can_it_do - //Some have asked how uthash cleans up its internal memory. - //The answer is simple: when you delete the final item from a hash table, - //uthash releases all the internal memory associated with that hash table, - //and sets its pointer to NULL. - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { - if (hier->compile_user_data_free && compile->user_data) { - hier->compile_user_data_free(compile->user_data); - compile->user_data = NULL; - } - HASH_DEL(hier->hash_compile_by_id, compile); - maat_hierarchy_compile_free(compile); - } - assert(hier->hash_compile_by_id == NULL); - - HASH_ITER(hh, hier->hash_item2clause, i2c_val, tmp_i2c_val) { - HASH_DEL(hier->hash_item2clause, i2c_val); - utarray_free(i2c_val->clause_ids); - free(i2c_val); - } - - maat_hierarchy_free_item2clause_hash(hier->hash_item2clause); - - HASH_ITER(hh, hier->hash_item_by_id, item, tmp_item) { - maat_hierarchy_item_free(hier, item); - } - - HASH_ITER(hh, hier->hash_dedup_clause_by_literals, clause, tmp_clause) { - maat_hierarchy_clause_free(hier, clause); - } - - bool_matcher_free(hier->bm); - hier->bm = NULL; - - pthread_rwlock_unlock(&hier->rwlock); - pthread_rwlock_destroy(&hier->rwlock); - - FREE(hier->expr_match_buff); - FREE(hier); -} - -size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) -{ - int printed = 0; - - for (long int i = 0; i < igraph_vector_size(v); i++) { - printed += snprintf(buff + printed, sz - printed, " %li", (long int)VECTOR(*v)[i]); - } - - return printed; -} - -static struct bool_matcher *maat_hierarchy_build_bool_matcher(struct maat_hierarchy *hier) -{ - struct bool_matcher *bm = NULL; - size_t compile_num = 0, expr_cnt = 0; - struct bool_expr *bool_expr_array = NULL; - struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; - - struct maat_hierarchy_clause_state *clause_state = NULL; - const struct maat_hierarchy_clause *clause = NULL; - size_t i = 0, j = 0; - int has_clause_num = 0; - compile_num = HASH_COUNT(hier->hash_compile_by_id); - - if (0 == compile_num) { - log_error(hier->logger, MODULE_HIERARCHY, "No compile to build."); - return NULL; - } - - //STEP 1, update clause_id of each compile and literal - struct maat_hierarchy_literal_id *literal_ids = NULL; - size_t n_literal_id = 0; - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { - has_clause_num = 0; - for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - clause_state = compile->clause_states + i; - clause_state->clause_id = 0; - if (!clause_state->in_use) { - continue; - } - - has_clause_num++; - literal_ids = (struct maat_hierarchy_literal_id *)utarray_eltptr(clause_state->literal_ids, 0); - n_literal_id = utarray_len(clause_state->literal_ids); - clause = maat_hierarchy_clause_fetch(hier, literal_ids, n_literal_id); - clause_state->clause_id = clause->clause_id; - } - assert(has_clause_num == compile->actual_clause_num); - } - - //STEP 2, serial compile clause states to a bool expression array. - compile_num = HASH_COUNT(hier->hash_compile_by_id); - bool_expr_array = ALLOC(struct bool_expr, compile_num); - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { - for (i = 0, j = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - if (compile->clause_states[i].in_use) { - if (compile->clause_states[i].not_flag) { - compile->not_clause_cnt++; - } - bool_expr_array[expr_cnt].items[j].item_id = compile->clause_states[i].clause_id; - bool_expr_array[expr_cnt].items[j].not_flag = compile->clause_states[i].not_flag; - j++; - } - } - //some compile may have zero groups, e.g. default policy. - if (j == (size_t)compile->declared_clause_num && j > 0) { - bool_expr_array[expr_cnt].expr_id = compile->compile_id; - bool_expr_array[expr_cnt].user_tag = compile; - bool_expr_array[expr_cnt].item_num = j; - expr_cnt++; - } - } - - //STEP 3, build the bool matcher. - size_t mem_size = 0; - if (0 == expr_cnt) { - log_error(hier->logger, MODULE_HIERARCHY, "No bool expression to build."); - goto error_out; - } - - bm = bool_matcher_new(bool_expr_array, expr_cnt, &mem_size); - if (bm != NULL) { - log_info(hier->logger, MODULE_HIERARCHY, - "Build bool matcher of %zu expressions with %zu bytes memory.", expr_cnt, mem_size); - } else { - log_error(hier->logger, MODULE_HIERARCHY, "Build bool matcher failed!"); - } - -error_out: - FREE(bool_expr_array); - - return bm; -} - -static inline int compare_clause_id(const void *a, const void *b) -{ - long long ret = *(const unsigned long long *)a - *(const unsigned long long *)b; - - if (0 == ret) { - return 0; - } else if(ret < 0) { - return -1; - } else { - return 1; - } -} - -static inline int compare_item_id(const void *a, const void *b) -{ - return (*(int*)a - *(int*)b); -} - -struct item2clause_value *maat_hierarchy_build_item2clause_hash(struct maat_hierarchy *hier) -{ - size_t i = 0, j = 0, k = 0; - struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; - struct maat_hierarchy_literal_id *literal_id = NULL; - struct maat_hierarchy_clause_state *clause_state = NULL; - - struct maat_hierarchy_item *item = NULL, *tmp_item = NULL; - struct maat_group *group = NULL; - struct group2item *g2i_hash = NULL, *g2i = NULL, *g2i_tmp = NULL; - struct item2clause_value *item2clause_hash = NULL, *i2c_val = NULL; - struct item2clause_key i2c_key; - - //Build a temporary hash that maps group to its items. - HASH_ITER(hh, hier->hash_item_by_id, item, tmp_item) { - group = item->ref_parent_group; - for (i = 0; i < group->top_group_cnt; i++) { - HASH_FIND_INT(g2i_hash, group->top_group_ids+i, g2i); - if (!g2i) { - g2i = ALLOC(struct group2item, 1); - utarray_new(g2i->item_ids, &ut_item_id_icd); - utarray_reserve(g2i->item_ids, group->ref_by_item_cnt); - g2i->group_id = group->top_group_ids[i]; - HASH_ADD_INT(g2i_hash, group_id, g2i); - } - - //One item belongs to one group, one group may have many items. So duplicate item check is unnecessary. - //if(utarray_find(g2i->item_ids, &(item->item_id), compare_item_id)) assert(0); - - utarray_push_back(g2i->item_ids, &(item->item_id)); - //utarray_sort(g2i->item_ids, compare_item_id); - } - } - - //Build short cut hash that maps item_id+vt_id to clause_ids. - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { - for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - clause_state = compile->clause_states + i; - if (!clause_state->in_use) { - continue; - } - - for (j = 0; j < utarray_len(clause_state->literal_ids); j++) { - literal_id = (struct maat_hierarchy_literal_id *)utarray_eltptr(clause_state->literal_ids, j); - HASH_FIND(hh_group_id, hier->hash_group_by_id, &(literal_id->group_id), sizeof(literal_id->group_id), group); - if (!group) { - continue; - } - HASH_FIND_INT(g2i_hash, &(group->group_id), g2i); - - //group declared by compile, but has no subordinate or item. - if (!g2i) { - continue; - } - - for (k = 0; k < utarray_len(g2i->item_ids); k++) { - i2c_key.item_id = *((int*)utarray_eltptr(g2i->item_ids, k)); - i2c_key.vt_id = literal_id->vt_id; - HASH_FIND(hh, item2clause_hash, &i2c_key, sizeof(i2c_key), i2c_val); - if (!i2c_val) { - i2c_val = ALLOC(struct item2clause_value, 1); - i2c_val->key = i2c_key; - i2c_val->group_id = g2i->group_id; - utarray_new(i2c_val->clause_ids, &ut_clause_id_icd); - HASH_ADD(hh, item2clause_hash, key, sizeof(i2c_val->key), i2c_val); - } - - if (utarray_find(i2c_val->clause_ids, &(clause_state->clause_id), compare_clause_id)) { - continue; - } - utarray_push_back(i2c_val->clause_ids, &(clause_state->clause_id)); - utarray_sort(i2c_val->clause_ids, compare_clause_id); - } - } - } - } - - int tmp1 = 0, tmp2 = 0; - HASH_ITER(hh, g2i_hash, g2i, g2i_tmp) { - HASH_DEL(g2i_hash, g2i); - //Sanity Check - utarray_sort(g2i->item_ids, compare_item_id); - for (i = 1; i < utarray_len(g2i->item_ids); i++) { - tmp1 = *((int*)utarray_eltptr(g2i->item_ids, i-1)); - tmp2 = *((int*)utarray_eltptr(g2i->item_ids, i)); - assert(tmp1!=tmp2); - } - utarray_free(g2i->item_ids); - g2i->item_ids = NULL; - free(g2i); - } - - log_info(hier->logger, MODULE_HIERARCHY, "Build item2clause hash with %llu element.", - HASH_COUNT(item2clause_hash)); - return item2clause_hash; -} - -int maat_hierarchy_rebuild(struct maat_hierarchy *hier) -{ - int ret=0; - struct bool_matcher *new_bm = NULL, *old_bm = NULL; - struct item2clause_value *new_item2clause_hash = NULL, *old_item2clause_hash = NULL; - - //Read hier from update thread is OK. - if (!hier->changed_flag) { - return ret; - } - - ret = maat_group_topology_build_top_groups(hier->ref_group_topo); - new_bm = maat_hierarchy_build_bool_matcher(hier); - new_item2clause_hash = maat_hierarchy_build_item2clause_hash(hier); - - pthread_rwlock_wrlock(&hier->rwlock); - - old_bm = hier->bm; - old_item2clause_hash = hier->hash_item2clause; - - hier->bm = new_bm; - hier->hash_item2clause = new_item2clause_hash; - hier->changed_flag = 0; - pthread_rwlock_unlock(&hier->rwlock); - - maat_garbage_bagging(hier->ref_garbage_bin, old_bm, (void (*)(void*))bool_matcher_free); - maat_garbage_bagging(hier->ref_garbage_bin, old_item2clause_hash, - (void (*)(void*))maat_hierarchy_free_item2clause_hash); - - return ret; -} - -static int maat_hierarchy_compile_has_literal(struct maat_hierarchy_compile *compile, struct maat_hierarchy_literal_id *literal_id) -{ - if (NULL == compile || NULL == literal_id) { - return 0; - } - - for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - struct maat_hierarchy_clause_state *clause_state = compile->clause_states + i; - if (!clause_state->in_use) { - continue; - } - - struct maat_hierarchy_literal_id *tmp = (struct maat_hierarchy_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); - if (tmp) { - assert(tmp->group_id == literal_id->group_id && tmp->vt_id == literal_id->vt_id); - return 1; - } - } - - return 0; -} - -static int maat_hierarchy_is_hit_path_existed(const struct maat_hit_path *hit_paths, size_t n_path, const struct maat_hit_path *find) -{ - if (NULL == hit_paths || NULL == find) { - return 0; - } - - for (size_t i = 0; i < n_path; i++) { - if (0 == memcmp(hit_paths + i, find, sizeof(*find))) { - return 1; - } - } - - return 0; -} - -size_t maat_hierarchy_get_hit_paths(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - struct maat_hit_path *hit_paths, size_t n_path) -{ - struct maat_hierarchy_internal_hit_path *p = NULL; - struct maat_hierarchy_item *item = NULL; - struct maat_group *group = NULL; - struct maat_hierarchy_compile *compile = NULL; - struct maat_hierarchy_literal_id literal_id = {0, 0}; - size_t n_made_by_item = 0, n_made_by_compile = 0; - size_t i = 0, j = 0, bool_match_ret = 0; - struct bool_expr_match *expr_match = hier->expr_match_buff + mid->thread_id * MAX_SCANNER_HIT_NUM; - struct maat_hit_path tmp_path; - - if (hier->version != mid->hier_ver) { - return 0; - } - pthread_rwlock_rdlock(&hier->rwlock); - - for (i = 0; i < utarray_len(mid->internal_hit_paths); i++) { - p = (struct maat_hierarchy_internal_hit_path *)utarray_eltptr(mid->internal_hit_paths, i); - - HASH_FIND_INT(hier->hash_item_by_id, &(p->item_id), item); - if (!item) { - continue; - } - - group = item->ref_parent_group; - if (group->top_group_cnt == 0 && n_made_by_item < n_path) { - hit_paths[n_made_by_item].Nth_scan = p->Nth_scan; - hit_paths[n_made_by_item].item_id = p->item_id; - hit_paths[n_made_by_item].sub_group_id = group->group_id; - hit_paths[n_made_by_item].top_group_id = -1; - hit_paths[n_made_by_item].virtual_table_id = p->virtual_table_id; - hit_paths[n_made_by_item].compile_id = -1; - n_made_by_item++; - } else { - for (j = 0; j < group->top_group_cnt && n_made_by_item < n_path; j++, n_made_by_item++) { - hit_paths[n_made_by_item].Nth_scan = p->Nth_scan; - hit_paths[n_made_by_item].item_id = p->item_id; - hit_paths[n_made_by_item].sub_group_id = group->group_id; - hit_paths[n_made_by_item].top_group_id = group->top_group_ids[j]; - hit_paths[n_made_by_item].virtual_table_id = p->virtual_table_id; - hit_paths[n_made_by_item].compile_id = -1; - } - } - } - - bool_match_ret = bool_matcher_match(hier->bm, (unsigned long long *)utarray_eltptr(mid->all_hit_clause_array, 0), - utarray_len(mid->all_hit_clause_array), - expr_match, MAX_SCANNER_HIT_NUM); - for (i = 0; i < bool_match_ret; i++) { - compile = (struct maat_hierarchy_compile *)expr_match[i].user_tag; - assert(compile->magic == MAAT_HIER_COMPILE_MAGIC); - assert((unsigned long long)compile->compile_id == expr_match[i].expr_id); - if (compile->actual_clause_num == 0 || !compile->user_data) { - continue; - } - - for (j = 0; j < n_made_by_item && n_made_by_item + n_made_by_compile < n_path; j++) { - if (hit_paths[j].top_group_id < 0) { - continue; - } - - literal_id.group_id = hit_paths[j].top_group_id; - literal_id.vt_id = hit_paths[j].virtual_table_id; - if (maat_hierarchy_compile_has_literal(compile, &literal_id)) { - if (hit_paths[j].compile_id < 0) { - hit_paths[j].compile_id = compile->compile_id; - } else { - tmp_path = hit_paths[j]; - tmp_path.compile_id=compile->compile_id; - if (maat_hierarchy_is_hit_path_existed(hit_paths, n_made_by_item + n_made_by_compile, &tmp_path)) { - hit_paths[n_made_by_item + n_made_by_compile] = tmp_path; - n_made_by_compile++; - } - } - } - } - } - - pthread_rwlock_unlock(&hier->rwlock); - return (n_made_by_item + n_made_by_compile); -} - -void maat_hierarchy_set_compile_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)) -{ - hier->compile_user_data_free = func; -} - -void maat_hierarchy_set_item_user_data_free_func(struct maat_hierarchy *hier, void (* func)(void *)) -{ - hier->item_user_data_free = func; -} - -struct maat_hierarchy_compile_mid *maat_hierarchy_compile_mid_new(struct maat_hierarchy *hier, int thread_id) -{ - struct maat_hierarchy_compile_mid *mid = ALLOC(struct maat_hierarchy_compile_mid, 1); - - mid->thread_id = thread_id; - mid->hier_ver = hier->version; - utarray_new(mid->internal_hit_paths, &ut_hit_path_icd); - utarray_new(mid->all_hit_clause_array, &ut_clause_id_icd); - utarray_new(mid->this_scan_hit_clause_ids, &ut_clause_id_icd); - - return mid; -} - -void maat_hierarchy_compile_mid_free(struct maat_hierarchy_compile_mid *mid) -{ - utarray_free(mid->internal_hit_paths); - utarray_free(mid->all_hit_clause_array); - utarray_free(mid->this_scan_hit_clause_ids); - free(mid); -} - -static int maat_hierarchy_hit_path_add(UT_array *hit_paths, int item_id, int virtual_table_id, - int Nth_scan, int Nth_item_result) -{ - struct maat_hierarchy_internal_hit_path new_path; - - new_path.item_id = item_id; - new_path.Nth_hit_item = Nth_item_result; - new_path.Nth_scan = Nth_scan; - new_path.virtual_table_id = virtual_table_id; - - utarray_push_back(hit_paths, &new_path); - return 1; -} - -void maat_hierarchy_compile_mid_update(struct maat_hierarchy *hier, - struct maat_hierarchy_compile_mid *mid, - int item_id, int virtual_table_id, - int Nth_scan, int Nth_item_result) -{ - if (mid->Nth_scan != Nth_scan) { - assert(mid->this_scan_item_hit_cnt == 0); - mid->Nth_scan = Nth_scan; - utarray_clear(mid->this_scan_hit_clause_ids); - } - - int ret = maat_hierarchy_hit_path_add(mid->internal_hit_paths, item_id, virtual_table_id, - Nth_scan, Nth_item_result); - if (!ret) { - return; - } - mid->hit_path_cnt++; - mid->this_scan_item_hit_cnt++; - - struct item2clause_value* i2c_val = NULL; - struct item2clause_key i2c_key; - i2c_key.item_id = item_id; - i2c_key.vt_id = virtual_table_id; - - HASH_FIND(hh, hier->hash_item2clause, &i2c_key, sizeof(i2c_key), i2c_val); - if (!i2c_val) { - return; - } - - size_t i = 0; - unsigned long long *clause_id = 0; - size_t new_clause_idx = utarray_len(mid->this_scan_hit_clause_ids); - for (size_t i = 0; i < utarray_len(r2c_val->clause_ids); i++) { - clause_id = (unsigned long long *)utarray_eltptr(r2c_val->clause_ids, i); - if (utarray_find(mid->all_hit_clause_array, clause_id, compare_clause_id)) { - continue; - } - utarray_push_back(mid->this_scan_hit_clause_ids, clause_id); - } - - if (utarray_len(mid->this_scan_hit_clause_ids) - new_clause_idx) { - utarray_reserve(mid->all_hit_clause_array, utarray_len(mid->this_scan_hit_clause_ids) - new_clause_idx); - for (i = new_clause_idx; i < utarray_len(mid->this_scan_hit_clause_ids); i++) { - clause_id = (unsigned long long *)utarray_eltptr(mid->this_scan_hit_clause_ids, i); - utarray_push_back(mid->all_hit_clause_array, clause_id); - } - utarray_sort(mid->all_hit_clause_array, compare_clause_id); - } -} - -int maat_hierarchy_compile_mid_has_NOT_clause(struct maat_hierarchy_compile_mid *mid) -{ - return mid->not_clause_hitted_flag; -} - -int maat_hierarchy_compile_add(struct maat_hierarchy *hier, int compile_id, - int declared_clause_num, void *user_data) -{ - int ret = 0; - struct maat_hierarchy_compile *compile = NULL; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag = 1; - HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); - if (!compile) { - assert(declared_clause_num >= 0); - compile = maat_hierarchy_compile_new(hier, compile_id); - compile->declared_clause_num = declared_clause_num; - compile->user_data = user_data; - } else { - if (compile->user_data != NULL) { - log_error(hier->logger, MODULE_HIERARCHY, - "Add compile %d failed, compile is already exisited.", compile_id); - ret = -1; - } else { - compile->declared_clause_num = declared_clause_num; - compile->user_data = user_data; - } - } - pthread_rwlock_unlock(&hier->rwlock); - - return ret; -} - -int maat_hierarchy_compile_remove(struct maat_hierarchy *hier, int compile_id) -{ - int ret = 0; - struct maat_hierarchy_compile *compile = NULL; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag = 1; - HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); - if (compile) { - if (hier->compile_user_data_free && compile->user_data) { - hier->compile_user_data_free(compile->user_data); - compile->user_data = NULL; - } - - if (0 == compile->actual_clause_num) { - HASH_DEL(hier->hash_compile_by_id, compile); - maat_garbage_bagging(hier->ref_garbage_bin, compile, (void (*)(void*))maat_hierarchy_compile_free); - } - ret = 0; - } else { - log_error(hier->logger, MODULE_HIERARCHY, - "Remove compile %d failed, compile is not exisited.", compile_id); - ret = -1; - } - pthread_rwlock_unlock(&hier->rwlock); - - return ret; -} - -static void *maat_hier_compile_get_user_data(struct maat_hierarchy* hier, int compile_id, int is_dettach) -{ - struct maat_hierarchy_compile *compile = NULL; - void *ret = NULL; - - pthread_rwlock_rdlock(&hier->rwlock); - HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); - if (compile) { - ret = compile->user_data; - if (is_dettach) { - compile->user_data = NULL; - } - } - pthread_rwlock_unlock(&hier->rwlock); - - return ret; -} - -void *maat_hierarchy_compile_dettach_user_data(struct maat_hierarchy *hier, int compile_id) -{ - return maat_hier_compile_get_user_data(hier, compile_id, 1); -} - -void *maat_hierarchy_compile_read_user_data(struct maat_hierarchy *hier, int compile_id) -{ - return maat_hier_compile_get_user_data(hier, compile_id, 0); -} - -void maat_hierarchy_compile_user_data_iterate(struct maat_hierarchy *hier, void (*callback)(void *user_data, void *param), void *param) -{ - struct maat_hierarchy_compile *compile = NULL, *tmp_compile = NULL; - - pthread_rwlock_rdlock(&hier->rwlock); - HASH_ITER(hh, hier->hash_compile_by_id, compile, tmp_compile) { - if (compile->user_data) { - callback(compile->user_data, param); - } - } - pthread_rwlock_unlock(&hier->rwlock); -} - -static int maat_hierarchy_compile_has_clause(struct maat_hierarchy_compile* compile, unsigned long long clause_id) -{ - struct maat_hierarchy_clause_state *clause_state = NULL; - - for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - clause_state = compile->clause_states + i; - if (!clause_state->in_use) { - continue; - } - - if (clause_state->clause_id == clause_id) { - return 1; - } - } - - return 0; -} - -static size_t maat_hierarchy_compile_mid_if_new_hit_compile(struct maat_hierarchy_compile_mid *mid, - struct maat_hierarchy_compile *compile) -{ - size_t r_in_c_cnt = 0; - - unsigned long long new_hit_clause_id = 0; - for(size_t i = 0; ithis_scan_hit_clause_ids); i++) { - new_hit_clause_id = *(unsigned long long*)utarray_eltptr(mid->this_scan_hit_clause_ids, i); - int ret = maat_hierarchy_compile_has_clause(compile, new_hit_clause_id); - if (ret) { - r_in_c_cnt++; - } - } - return r_in_c_cnt; -} - -int maat_hierarchy_item_compile(struct maat_hierarchy *hier, struct maat_hierarchy_compile_mid *mid, - int is_last_compile, void **user_data_array, size_t ud_array_sz) -{ - int bool_match_ret = 0, i = 0; - struct maat_hierarchy_compile *compile = NULL; - struct bool_expr_match *expr_match = hier->expr_match_buff + (mid->thread_id * MAX_SCANNER_HIT_NUM); - - size_t r_in_c_cnt = 0; - size_t ud_result_cnt = 0; - size_t this_scan_item_hits = mid->this_scan_item_hit_cnt; - - if (!hier->bm || 0 == utarray_len(mid->all_hit_clause_array) || hier->version != mid->hier_ver) { - mid->this_scan_item_hit_cnt = 0; - return 0; - } - - bool_match_ret = bool_matcher_match(hier->bm, (unsigned long long *)utarray_eltptr(mid->all_hit_clause_array, 0), - utarray_len(mid->all_hit_clause_array), expr_match, MAX_SCANNER_HIT_NUM); - for (i = 0; i < bool_match_ret && ud_result_cnt < ud_array_sz; i++) { - compile = (struct maat_hierarchy_compile *)expr_match[i].user_tag; - assert(compile->magic == MAAT_HIER_COMPILE_MAGIC); - assert((unsigned long long)compile->compile_id == expr_match[i].expr_id); - if (0 == compile->actual_clause_num) { - continue; - } - - r_in_c_cnt = maat_hierarchy_compile_mid_if_new_hit_compile(mid, compile); - if (compile->not_clause_cnt > 0 && !is_last_compile) { - mid->not_clause_hitted_flag = 1; - } else if(compile->user_data) { - //For compile may be dettached by Maat_hierarchy_compile_dettach_user_data, only return non-NULL userdata. - if (r_in_c_cnt > 0 || //compile hitted becasue of new reigon - this_scan_item_hits == 0) //or hit a compile that refer a NOT-logic group in previous scan. - { - user_data_array[ud_result_cnt]=compile->user_data; - ud_result_cnt++; - } - } - } - - mid->this_scan_item_hit_cnt = 0; - - return ud_result_cnt; -} - -void *maat_hierarchy_item_dettach_user_data(struct maat_hierarchy *hier, int item_id) -{ - struct maat_hierarchy_item *item = NULL; - void *ret = NULL; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag = 1; - HASH_FIND_INT(hier->hash_item_by_id, &item_id, item); - if (item) { - ret = item->user_data; - item->user_data = NULL; - } - pthread_rwlock_unlock(&hier->rwlock); - - return ret; -} - -int compare_literal_id(const void *pa, const void *pb) -{ - struct maat_hierarchy_literal_id *la = (struct maat_hierarchy_literal_id *)pa; - struct maat_hierarchy_literal_id *lb = (struct maat_hierarchy_literal_id *)pb; - - int ret = la->vt_id - lb->vt_id; - if (ret == 0) { - ret = la->group_id - lb->group_id; - } - - return ret; -} - -static int maat_hierarchy_compile_add_literal(struct maat_hierarchy_compile *compile, - struct maat_hierarchy_literal_id *literal_id, - int not_flag, int clause_index) -{ - struct maat_hierarchy_literal_id *tmp = NULL; - struct maat_hierarchy_clause_state *clause_state = compile->clause_states + clause_index; - clause_state->not_flag = not_flag; - - if (!clause_state->in_use) { - clause_state->in_use = 1; - compile->actual_clause_num++; - } - - tmp = (struct maat_hierarchy_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); - if (tmp) { - assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); - return -1; - } else { - utarray_push_back(clause_state->literal_ids, literal_id); - utarray_sort(clause_state->literal_ids, compare_literal_id); - } - - return 0; -} - -static int maat_hierarchy_compile_remove_literal(struct maat_hierarchy_compile *compile, - struct maat_hierarchy_literal_id *literal_id, - int clause_index) -{ - struct maat_hierarchy_literal_id *tmp = NULL; - struct maat_hierarchy_clause_state *clause_state = compile->clause_states + clause_index; - - tmp = (struct maat_hierarchy_literal_id *)utarray_find(clause_state->literal_ids, literal_id , compare_literal_id); - if (tmp) { - assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); - } else { - return -1; - } - - size_t remove_idx = utarray_eltidx(clause_state->literal_ids, tmp); - utarray_erase(clause_state->literal_ids, remove_idx, 1); - if (0 == utarray_len(clause_state->literal_ids)) { - clause_state->in_use = 0; - compile->actual_clause_num--; - } - - return 0; -} - -int maat_hierarchy_add_group_to_compile(struct maat_hierarchy *hier, int group_id, int vt_id, - int not_flag, int clause_index, int compile_id) -{ - int ret = 0; - struct maat_group *group = NULL; - struct maat_hierarchy_literal_id literal_id = {group_id, vt_id}; - struct maat_hierarchy_compile *compile = NULL; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag = 1; - - HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile); - if (!compile) { - compile = maat_hierarchy_compile_new(hier, compile_id); - } - - ret = maat_hierarchy_compile_add_literal(compile, &literal_id, not_flag, clause_index); - if (ret < 0) { - log_error(hier->logger, MODULE_HIERARCHY, - "Add group %d vt_id %d to clause %d of compile %d failed, group is already exisited.", - group_id, vt_id, clause_index, compile_id); - ret = -1; - } else { - ret = 0; - } - pthread_rwlock_unlock(&hier->rwlock); - - return ret; -} - -int maat_hierarchy_remove_group_from_compile(struct maat_hierarchy *hier, int group_id, int vt_id, - int not_flag, int clause_index, int compile_id) -{ - struct maat_hierarchy_literal_id literal_id = {group_id, vt_id}; - struct maat_hierarchy_compile *compile = NULL; - int ret = 0; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag = 1; - - HASH_FIND(hh, hier->hash_compile_by_id, &compile_id, sizeof(compile_id), compile); - if (!compile) { - log_error(hier->logger, MODULE_HIERARCHY, - "Remove group %d from compile %d failed, compile is not exisited.", - group_id, compile_id); - goto error_out; - } - - ret = maat_hierarchy_compile_remove_literal(compile, &literal_id, clause_index); - if (ret < 0) { - log_error(hier->logger, MODULE_HIERARCHY, - "Remove group %d vt_id %d from clause %d of compile %d failed, literal is not in compile.", - group_id, vt_id, clause_index, compile_id); - goto error_out; - } - - if (0 == compile->actual_clause_num && !compile->user_data) { - HASH_DEL(hier->hash_compile_by_id, compile); - maat_garbage_bagging(hier->ref_garbage_bin, compile, (void (*)(void*))maat_hierarchy_compile_free); - } - pthread_rwlock_unlock(&hier->rwlock); - return 0; - -error_out: - pthread_rwlock_unlock(&hier->rwlock); - return -1; -} - -int maat_hierarchy_add_item_to_group(struct maat_hierarchy *hier, int group_id, int item_id, void *user_data) -{ - //A item rule belongs to ONE group only. - struct maat_group *group = NULL; - struct maat_hierarchy_item *item = NULL; - int ret = 0; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag = 1; - /* - HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); - if (!group) { - group = maat_hierarchy_group_new(hier, group_id); - }*/ - - HASH_FIND_INT(hier->hash_item_by_id, &item_id, item); - if (item) { - log_error(hier->logger, MODULE_HIERARCHY, - "Add item %d to group %d failed, item already in group %d.", - item_id, group_id, item->ref_parent_group->group_id); - ret = -1; - } else { - item = maat_hierarchy_item_new(hier, item_id, group_id, group, user_data); - ret = 0; - } - pthread_rwlock_unlock(&hier->rwlock); - - return ret; -} - -int maat_hierarchy_remove_item_from_group(struct maat_hierarchy *hier, int group_id, int item_id) -{ - struct maat_group *group = NULL; - struct maat_hierarchy_item *item = NULL; - - pthread_rwlock_wrlock(&hier->rwlock); - hier->changed_flag = 1; - HASH_FIND(hh_group_id, hier->ref_group_topo->hash_group_by_id, &group_id, sizeof(group_id), group); - if (!group) { - log_error(hier->logger, MODULE_HIERARCHY, - "Remove item %d from group %d failed, group is not existed.", - item_id, group_id); - goto error_out; - } - - HASH_FIND_INT(hier->hash_item_by_id, &item_id, item); - if (!item) { - log_error(hier->logger, MODULE_HIERARCHY, - "Remove item %d from group %d failed, item is not exisited.", - item_id, group_id); - goto error_out; - } - - assert(item->group_id == group->group_id); - maat_hierarchy_item_free(hier, item); - pthread_rwlock_unlock(&hier->rwlock); - return 0; - -error_out: - pthread_rwlock_unlock(&hier->rwlock); - return -1; -} \ No newline at end of file diff --git a/src/maat_ip.cpp b/src/maat_ip.cpp new file mode 100644 index 0000000..75883db --- /dev/null +++ b/src/maat_ip.cpp @@ -0,0 +1,708 @@ +/* +********************************************************************************************** +* File: maat_ip.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include + +#include "utils.h" +#include "log/log.h" +#include "cJSON/cJSON.h" +#include "utils.h" +#include "maat_utils.h" +#include "maat_ex_data.h" +#include "IPMatcher.h" +#include "maat_ip.h" +#include "maat_rule.h" +#include "maat_garbage_collection.h" + +#define MODULE_IP module_name_str("maat.ip") + +struct port_range { + uint16_t min_port; + uint16_t max_port; +}; + +struct ip_plus_schema { + int item_id_column; + int group_id_column; + int addr_type_column; + int saddr_format_column; + int sip1_column; + int sip2_column; + int sport_format_column; + int sport1_column; + int sport2_column; + int daddr_format_column; + int dip1_column; + int dip2_column; + int dport_format_column; + int dport1_column; + int dport2_column; + int proto_column; + int direction_column; + int table_id; //ugly +}; + +struct ipv4_item_rule { + uint32_t min_sip; /* 源地址下界；0表示忽略本字段 */ + uint32_t max_sip; /* 源地址上界；0表示固定IP=min_saddr */ + uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */ + uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */ + uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP；0表示忽略本字段 */ + uint16_t direction; /* 方向，0表示双向，1表示单向 */ +}; + +struct ipv6_item_rule { + uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */ + uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */ + uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */ + uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */ + uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP，无限制默认为0 */ + uint16_t direction; /* 方向，0表示双向，1表示单向 */ +}; + +struct ip_plus_item { + int item_id; + int group_id; + int addr_type; + union { + struct ipv4_item_rule ipv4; + struct ipv6_item_rule ipv6; + }; +}; + +struct ip_plus_runtime { + struct ip_matcher* ip_matcher; + struct ex_data_runtime* ex_data_rt; + + uint32_t rule_num; + uint32_t updating_rule_num; + struct maat_item *item_hash; + void (*item_user_data_free)(void *); + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; + // long long *scan_cnt; + // long long *hit_cnt; + // long long *not_grp_hit_cnt; + // long long *stream_num; +}; + +void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +{ + size_t read_cnt = 0; + struct ip_plus_schema *ip_plus_schema = ALLOC(struct ip_plus_schema, 1); + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + goto error; + } + ip_plus_schema->table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "custom"); + if (NULL == item || item->type != cJSON_Object) { + log_error(logger, MODULE_IP, "ip_plus table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "item_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->item_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "group_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->group_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "addr_type"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->addr_type_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "saddr_format"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->saddr_format_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "sip1"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->sip1_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "sip2"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->sip2_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "sport_format"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->sport_format_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "sport1"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->sport1_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "sport2"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->sport2_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "daddr_format"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->daddr_format_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "dip1"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->dip1_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "dip2"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->dip2_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "dport_format"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->dport_format_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "dport1"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->dport1_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "dport2"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->dport2_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "proto"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->proto_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(json, "direction"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plus_schema->direction_column = custom_item->valueint; + read_cnt++; + } + + if (read_cnt < 17) { + goto error; + } + + return ip_plus_schema; +error: + FREE(ip_plus_schema); + return NULL; +} + +void ip_plus_schema_free(void *ip_plus_schema) +{ + FREE(ip_plus_schema); +} + +void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == ip_plus_schema) { + return NULL; + } + + struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema; + struct ip_plus_runtime *ip_plus_rt = ALLOC(struct ip_plus_runtime, 1); + + ip_plus_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free); + ip_plus_rt->item_user_data_free = maat_item_inner_free; + ip_plus_rt->ref_garbage_bin = garbage_bin; + ip_plus_rt->logger = logger; + + return ip_plus_rt; +} + +void ip_plus_runtime_free(void *ip_plus_runtime) +{ + if (NULL == ip_plus_runtime) { + return; + } + + struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; + if (ip_plus_rt->ip_matcher != NULL) { + ip_matcher_free(ip_plus_rt->ip_matcher); + } + + if (ip_plus_rt->ex_data_rt != NULL) { + ex_data_runtime_free(ip_plus_rt->ex_data_rt); + } + + struct maat_item *item = NULL, *tmp_item = NULL; + HASH_ITER(hh, ip_plus_rt->item_hash, item, tmp_item) { + HASH_DELETE(hh, ip_plus_rt->item_hash, item); + maat_item_free(item, ip_plus_rt->item_user_data_free); + } + + FREE(ip_plus_rt); +} + +struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *ip_plus_schema, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + char saddr_format[16] = {0}; + char sport_format[16] = {0}; + char sip1_str[40] = {0}; + char sip2_str[40] = {0}; + uint16_t sport1 = 0; + uint16_t sport2 = 0; + uint16_t protocol = 0; + uint16_t direction = 0; + struct ip_plus_item *ip_plus_item = ALLOC(struct ip_plus_item, 1); + + int ret = get_column_pos(line, ip_plus_schema->item_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no item_id", + ip_plus_schema->table_id, line); + goto error; + } + ip_plus_item->item_id = atoi(line + column_offset); + + ret = get_column_pos(line, ip_plus_schema->group_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no group_id", + ip_plus_schema->table_id, line); + goto error; + } + ip_plus_item->group_id = atoi(line + column_offset); + + ret = get_column_pos(line, ip_plus_schema->addr_type_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no addr_type", + ip_plus_schema->table_id, line); + goto error; + } + ip_plus_item->addr_type = atoi(line + column_offset); + + if (ip_plus_item->addr_type != 4 && ip_plus_item->addr_type != 6) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has invalid addr type:%d", + ip_plus_schema->table_id, line, ip_plus_item->addr_type); + goto error; + } + + ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + memcpy(saddr_format, (line + column_offset), column_len); + if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) { + log_error(logger, MODULE_IP, + "ip_plus table(table_id:%d) line:%s has invalid saddr_format, should be range/mask/CIDR", + ip_plus_schema->table_id, line); + goto error; + } + + ret = get_column_pos(line, ip_plus_schema->sip1_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip1", + ip_plus_schema->table_id, line); + goto error; + } + memcpy(sip1_str, (line + column_offset), column_len); + + ret = get_column_pos(line, ip_plus_schema->sip2_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip2", + ip_plus_schema->table_id, line); + goto error; + } + memcpy(sip2_str, (line + column_offset), column_len); + + ret = get_column_pos(line, ip_plus_schema->sport_format_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport_format", + ip_plus_schema->table_id, line); + goto error; + } + memcpy(sport_format, (line + column_offset), column_len); + if (IP_FORMAT_UNKNOWN == ip_format_str2int(sport_format)) { + log_error(logger, MODULE_IP, + "ip_plus table(table_id:%d) line:%s has invalid sport_format, should be range/mask/CIDR", + ip_plus_schema->table_id, line); + goto error; + } + + ret = get_column_pos(line, ip_plus_schema->sport1_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport1", + ip_plus_schema->table_id, line); + goto error; + } + sport1 = atoi(line + column_offset); + + ret = get_column_pos(line, ip_plus_schema->sport2_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport2", + ip_plus_schema->table_id, line); + goto error; + } + sport2 = atoi(line + column_offset); + + if (4 == ip_plus_item->addr_type) { + ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, + &ip_plus_item->ipv4.min_sip, &ip_plus_item->ipv4.max_sip); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip4) failed", + ip_plus_schema->table_id, line); + goto error; + } + + if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { + ip_plus_item->ipv4.min_sport = sport1 & sport2; + ip_plus_item->ipv4.max_sport = sport1 | ~sport2; + } else { + ip_plus_item->ipv4.min_sport = sport1; + ip_plus_item->ipv4.max_sport = sport2; + } + + ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto", + ip_plus_schema->table_id, line); + goto error; + } + ip_plus_item->ipv4.proto = atoi(line + column_offset); + protocol = ip_plus_item->ipv4.proto; + + ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction", + ip_plus_schema->table_id, line); + goto error; + } + ip_plus_item->ipv4.direction = atoi(line + column_offset); + direction = ip_plus_item->ipv4.direction; + } else { + //ipv6 + ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, + ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip6) failed", + ip_plus_schema->table_id, line); + goto error; + } + + if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { + ip_plus_item->ipv6.min_sport = sport1 & sport2; + ip_plus_item->ipv6.max_sport = sport1 | ~sport2; + } else { + ip_plus_item->ipv6.min_sport = sport1; + ip_plus_item->ipv6.max_sport = sport2; + } + + ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto", + ip_plus_schema->table_id, line); + goto error; + } + ip_plus_item->ipv6.proto = atoi(line + column_offset); + protocol = ip_plus_item->ipv6.proto; + + ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction", + ip_plus_schema->table_id, line); + goto error; + } + ip_plus_item->ipv6.direction = atoi(line + column_offset); + direction = ip_plus_item->ipv6.direction; + } + + if (protocol > 65535 || protocol < 0) { + log_error(logger, MODULE_IP, + "ip_plus table(table_id:%d) line:%s has invalid proto:%d", + ip_plus_schema->table_id, line, protocol); + goto error; + } + + if (direction != 0 && direction != 1) { + log_error(logger, MODULE_IP, + "ip_plus table(table_id:%d) line:%s has invalid direction:%d", + ip_plus_schema->table_id, line, direction); + goto error; + } + + return ip_plus_item; +error: + FREE(ip_plus_item); + return NULL; +} + +void ip_plus_item_free(struct ip_plus_item *ip_plus_item) +{ + FREE(ip_plus_item); +} + +void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule) +{ + struct port_range *sport_range = ALLOC(struct port_range, 1); + if (4 == item->addr_type) { + rule->type = IPv4; + sport_range->min_port = item->ipv4.min_sport; + sport_range->max_port = item->ipv4.max_sport; + rule->ipv4_rule.start_ip = item->ipv4.min_sip; + rule->ipv4_rule.end_ip = item->ipv4.max_sip; + } else { + rule->type = IPv6; + sport_range->min_port = item->ipv6.min_sport; + sport_range->max_port = item->ipv6.max_sport; + memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, sizeof(item->ipv6.min_sip)); + memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, sizeof(item->ipv6.max_sip)); + } + rule->rule_id = item->item_id; + rule->user_tag = sport_range; +} + +struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt) +{ + return ip_plus_rt->ex_data_rt; +} + +int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key_len, + struct ip_plus_item *item, int is_valid) +{ + int ret = -1; + struct ex_data_runtime *ex_data_rt = rt->ex_data_rt; + + if (0 == is_valid) { + // delete + ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len); + if (ret < 0) { + return -1; + } + } else { + // add + struct ex_data_container *ex_container = ex_data_container_new(NULL, (void *)item); + ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); + if (ret < 0) { + return -1; + } + } + + return 0; +} + +int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line, + int valid_column) +{ + if (NULL == ip_plus_runtime || NULL == ip_plus_schema || NULL == line) { + return -1; + } + + struct maat_item *item = NULL; + struct ip_plus_item *ip_plus_item = NULL; + struct maat_item_inner *u_para = NULL; + struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema; + struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; + int item_id = get_column_value(line, schema->item_id_column); + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } else if (0 == is_valid) { + //delete + HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item); + if (NULL == item) { + return -1; + } + + u_para = (struct maat_item_inner *)item->user_data; + item->user_data = NULL; + + if (NULL == u_para) { + return -1; + } + + HASH_DELETE(hh, ip_plus_rt->item_hash, item); + maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free); + } else { + //add + HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item); + if (item) { + log_error(ip_plus_rt->logger, MODULE_IP, + "ip_plus runtime add item %d to item_hash failed, already exist", item_id); + return -1; + } + + ip_plus_item = ip_plus_item_new(line, schema, ip_plus_rt->logger); + if (NULL == ip_plus_item) { + log_error(ip_plus_rt->logger, MODULE_IP, "ip_plus line:%s to item failed", line); + return -1; + } + + u_para = maat_item_inner_new(ip_plus_item->group_id, item_id, 0); + item = maat_item_new(item_id, group_id, u_para); + HASH_ADD_INT(ip_plus_rt->item_hash, item_id, item); + } + + char *key = (char *)&item_id; + int ret = ip_plus_runtime_update_row(ip_plus_rt, key, sizeof(int), ip_plus_item, is_valid); + if (ret < 0) { + if (ip_plus_item != NULL) { + ip_plus_item_free(ip_plus_item); + ip_plus_item = NULL; + } + return -1; + } else { + if (0 == is_valid) { + ip_plus_rt->rule_num--; + } else { + ip_plus_rt->rule_num++; + } + } + + return 0; +} + +int ip_plus_runtime_commit(void *ip_plus_runtime) +{ + if (NULL == ip_plus_runtime) { + return -1; + } + + int ret = 0; + struct ex_data_container **ex_container = NULL; + struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; + struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt; + + size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); + if (0 == rule_cnt) { + FREE(ex_container); + return 0; + } + + struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt); + + for (size_t i = 0; i < rule_cnt; i++) { + struct ip_plus_item *item = (struct ip_plus_item *)ex_container[i]->custom_data; + ip_plus_item_to_ip_rule(item, &rules[i]); + } + + struct ip_matcher *new_ip_matcher = NULL; + struct ip_matcher *old_ip_matcher = NULL; + size_t mem_used = 0; + + if (rule_cnt > 0) { + log_info(logger, MODULE_IP, + "committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt); + new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); + if (NULL == new_ip_matcher) { + log_error(logger, MODULE_IP, + "rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt); + ret = -1; + } + } + + old_ip_matcher = ip_plus_rt->ip_matcher; + ip_plus_rt->ip_matcher = new_ip_matcher; + maat_garbage_bagging(garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); + ex_data_runtime_commit(ex_data_rt); + ip_plus_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt); + + FREE(rules); + FREE(ex_container); + + return ret; +} + +int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *data, + int *group_id_array, size_t n_group_id_array, int virtual_table_id, + struct maat_state *state) +{ + if (NULL == table_rt) { + return -1; + } + + int n_hit_item = 0; + struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; + + struct ip_data ip; + ip.type = ip_type_transform(scan_data->ip_type); + if (ip.type == IPv4) { + ip.ipv4 = scan_data->ipv4; + } else { + memcpy(ip.ipv6, scan_data->ipv6, sizeof(scan_data->ipv6)); + } + + n_hit_item = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, MAX_SCANNER_HIT_ITEM_NUM); + if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { + n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; + } + + struct maat_compile_state *compile_state = state->compile_mid; + //tranform item_id to group_id + struct maat_item *item = NULL; + size_t n_group_id = 0; + int i = 0; + for (i = 0; i < n_hit_item; i++) { + HASH_FIND_INT(table_rt->item_hash, &(scan_results[i].rule_id), item); + assert(item != NULL); + if (!item) { + // should not come here + continue; + } + + if (n_group_id >= n_group_id_array) { + n_group_id = n_group_id_array; + //Prevent group_id_array out of bounds + } else { + group_id_array[n_group_id++] = item->group_id; + } + + // update hit path + maat_compile_state_update_hit_path(compile_state, scan_results[i].rule_id, item->group_id, virtual_table_id, state->scan_cnt, i); + } + + // update hit clause: literal_id{group_id,vt_id} to clause_id + int compile_table_id = -1; + if (state->compile_table_id == -1) { + compile_table_id = state->maat_instance->default_compile_table_id; + } else { + compile_table_id = state->compile_table_id; + } + struct maat_runtime *maat_rt = state->maat_instance->maat_rt; + struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, compile_table_id); + assert(compile_table_rt->table_type == TABLE_TYPE_COMPILE); + + for (size_t idx = 0; idx < n_group_id; idx++) { + maat_compile_state_update_hit_clause(compile_state, &(compile_table_rt->compile_rt.compile_hash), group_id_array[idx], virtual_table_id); + } + + return n_group_id; +} \ No newline at end of file diff --git a/src/maat_ip_plugin.cpp b/src/maat_ip_plugin.cpp new file mode 100644 index 0000000..e577a10 --- /dev/null +++ b/src/maat_ip_plugin.cpp @@ -0,0 +1,448 @@ +/* +********************************************************************************************** +* File: maat_ip_plugin.cpp +* Description: +* Authors: Liu wentan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include "maat_ip_plugin.h" +#include "cJSON/cJSON.h" +#include "log/log.h" +#include "utils.h" +#include "maat_utils.h" +#include "maat_ex_data.h" +#include "IPMatcher.h" +#include "maat_rule.h" + +#define MODULE_IP_PLUGIN module_name_str("maat.ip_plugin") +#define MAX_IP_STR 128 + +struct ip_plugin_item { + int item_id; + int ip_type; + char start_ip[MAX_IP_STR]; + char end_ip[MAX_IP_STR]; + int rule_tag; +}; + +struct ip_plugin_schema { + int item_id_column; + int ip_type_column; + int start_ip_column; + int end_ip_column; + int rule_tag_column; + struct ex_data_schema *ex_schema; + int table_id; //ugly +}; + +struct ip_plugin_runtime { + struct ip_matcher *ip_matcher; + struct ex_data_runtime *ex_data_rt; + + uint32_t rule_num; + uint32_t updating_rule_num; + struct maat_item *item_hash; + void (*item_user_data_free)(void *); + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; + + // long long *scan_cnt; + // long long *hit_cnt; + // long long *not_grp_hit_cnt; + // long long *stream_num; +}; + +void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +{ + size_t read_cnt = 0; + struct ip_plugin_schema *ip_plugin_schema = ALLOC(struct ip_plugin_schema, 1); + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + goto error; + } + ip_plugin_schema->table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "custom"); + if (NULL == item || item->type != cJSON_Object) { + log_error(logger, MODULE_IP_PLUGIN, "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "item_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plugin_schema->item_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "ip_type"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plugin_schema->ip_type_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "start_ip"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plugin_schema->start_ip_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "end_ip"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + ip_plugin_schema->end_ip_column = custom_item->valueint; + read_cnt++; + } + + if (read_cnt < 4) { + goto error; + } + + return ip_plugin_schema; +error: + FREE(ip_plugin_schema); + return NULL; +} + +void ip_plugin_schema_free(void *ip_plugin_schema) +{ + if (NULL == ip_plugin_schema) { + return; + } + struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; + if (schema->ex_schema != NULL) { + ex_data_schema_free(schema->ex_schema); + schema->ex_schema = NULL; + } + + FREE(schema); +} + +struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema) +{ + if (NULL == ip_plugin_schema) { + return NULL; + } + + struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; + + return schema->ex_schema; +} + +struct ip_plugin_item *ip_plugin_item_new(const char *line, struct ip_plugin_schema *ip_plugin_schema, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + struct ip_plugin_item *ip_plugin_item = ALLOC(struct ip_plugin_item, 1); + + int ret = get_column_pos(line, ip_plugin_schema->item_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no item_id", + ip_plugin_schema->table_id, line); + goto error; + } + ip_plugin_item->item_id = atoi(line + column_offset); + + ret = get_column_pos(line, ip_plugin_schema->ip_type_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no ip_type", + ip_plugin_schema->table_id, line); + goto error; + } + ip_plugin_item->ip_type = atoi(line + column_offset); + if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) { + log_error(logger, MODULE_IP_PLUGIN, + "ip_plugin table(table_id:%d) line:%s ip_type[%d] invalid", + ip_plugin_schema->table_id, line, ip_plugin_item->ip_type); + goto error; + } + + ret = get_column_pos(line, ip_plugin_schema->start_ip_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP_PLUGIN, + "ip_plugin table(table_id:%d) line:%s has no start_ip", + ip_plugin_schema->table_id, line); + goto error; + } + strncpy(ip_plugin_item->start_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->start_ip))); + + ret = get_column_pos(line, ip_plugin_schema->end_ip_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP_PLUGIN, + "ip_plugin table(table_id:%d) line:%s has no end_ip", + ip_plugin_schema->table_id, line); + goto error; + } + strncpy(ip_plugin_item->end_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->end_ip))); + + return ip_plugin_item; +error: + FREE(ip_plugin_item); + return NULL; +} + +void ip_plugin_item_free(struct ip_plugin_item *item) +{ + FREE(item); +} + +int ip_plugin_table_ex_data_schema_flag(struct ip_plugin_schema *ip_plugin_schema) +{ + +} + +int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp) +{ + struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; + struct ex_data_schema *ex_schema = schema->ex_schema; + + if (ex_schema->set_flag) { + assert(0); + log_error(logger, MODULE_TABLE, "Error: %s, EX data schema already registed", + __FUNCTION__); + return -1; + } + + ex_schema->new_func = new_func; + ex_schema->free_func = free_func; + ex_schema->dup_func = dup_func; + ex_schema->argl = argl; + ex_schema->argp = argp; + //ex_schema->set_flag = 1; + + return 0; +} + +int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_schema *schema, + const char *row, char *key, size_t key_len, struct ip_plugin_item *item, + int is_valid) +{ + int ret = -1; + struct ex_data_runtime *ex_data_rt = rt->ex_data_rt; + int set_flag = ip_plugin_table_schema_ex_data_schema_flag(schema); + + if (1 == set_flag) { + if (0 == is_valid) { + //delete + ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len); + if (ret < 0) { + return -1; + } + } else { + //add + void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); + struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)item); + ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); + if (ret < 0) { + return -1; + } + } + } else { + ex_data_runtime_cache_row_put(ex_data_rt, row); + } + + return 0; +} + +void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == ip_plugin_schema) { + return NULL; + } + + struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; + struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1); + + ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free); + ip_plugin_rt->item_user_data_free = maat_item_inner_free; + ip_plugin_rt->ref_garbage_bin = garbage_bin; + ip_plugin_rt->logger = logger; + + return ip_plugin_rt; +} + +void ip_plugin_runtime_free(void *ip_plugin_runtime) +{ + if (NULL == ip_plugin_runtime) { + return; + } + + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + if (ip_plugin_rt->ip_matcher != NULL) { + ip_matcher_free(ip_plugin_rt->ip_matcher); + } + + if (ip_plugin_rt->ex_data_rt != NULL) { + ex_data_runtime_free(ip_plugin_rt->ex_data_rt); + } + + struct maat_item *item = NULL, *tmp_item = NULL; + HASH_ITER(hh, ip_plugin_rt->item_hash, item, tmp_item) { + HASH_DELETE(hh, ip_plugin_rt->item_hash, item); + maat_item_free(item, ip_plugin_rt->item_user_data_free); + } + + FREE(ip_plugin_rt); +} + +int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line, + int valid_column) +{ + if (NULL == ip_plugin_runtime || NULL == ip_plugin_schema || NULL == line) { + return -1; + } + + struct maat_item *item = NULL; + struct ip_plugin_item *ip_plugin_item = NULL; + struct maat_item_inner *u_para = NULL; + struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + int item_id = get_column_value(line, schema->item_id_column); + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } else if (0 == is_valid) { + //delete + HASH_FIND_INT(ip_plugin_rt->item_hash, &item_id, item); + if (NULL == item) { + return -1; + } + + u_para = (struct maat_item_inner *)item->user_data; + item->user_data = NULL; + + if (NULL == u_para) { + return -1; + } + + HASH_DELETE(hh, ip_plugin_rt->item_hash, item); + maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free); + } else { + //add + HASH_FIND_INT(ip_plugin_rt->item_hash, &item_id, item); + if (item) { + log_error(ip_plus_rt->logger, MODULE_IP_PLUGIN, + "ip_plugin runtime add item %d to item_hash failed, already exist", item_id); + return -1; + } + + ip_plugin_item = ip_plugin_item_new(line, schema, ip_plugin_rt->logger); + if (NULL == ip_plugin_item) { + log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, "ip_plugin line:%s to item failed", line); + return -1; + } + + u_para = maat_item_inner_new(ip_plugin_item->group_id, item_id, 0); + item = maat_item_new(item_id, group_id, u_para); + HASH_ADD_INT(ip_plugin_rt->item_hash, item_id, item); + } + + char *key = (char *)&item_id; + int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, row, key, sizeof(int), ip_plugin_item, is_valid); + if (ret < 0) { + if (ip_plugin_item != NULL) { + ip_plugin_item_free(ip_plugin_item); + ip_plugin_item = NULL; + } + return -1; + } else { + if (0 == is_valid) { + ip_plugin_rt->rule_num--; + } else { + ip_plugin_rt->rule_num++; + } + } + + return 0; +} + +void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule) +{ + if (4 == item->ip_type) { + rule->type = IPv4; + ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, &(rule->ipv4_rule.start_ip), &(rule->ipv4_rule.end_ip)); + } else { + rule->type = IPv6; + ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, &(rule->ipv6_rule.start_ip), &(rule->ipv6_rule.end_ip)); + } + + rule->rule_id = item->item_id; + rule->user_tag = NULL; +} + +int ip_plugin_runtime_commit(void *ip_plugin_runtime) +{ + if (NULL == ip_plugin_runtime) { + return -1; + } + + int ret = 0; + struct ex_data_container **ex_container = NULL; + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt; + + size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); + if (0 == rule_cnt) { + FREE(ex_container); + return 0; + } + + struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt); + + for (size_t i = 0; i < rule_cnt; i++) { + struct ip_plugin_item *item = (struct ip_plugin_item *)ex_container[i]->custom_data; + ip_plugin_item_to_ip_rule(item, &rules[i]); + } + + struct ip_matcher *new_ip_matcher = NULL; + struct ip_matcher *old_ip_matcher = NULL; + size_t mem_used = 0; + + if (rule_cnt > 0) { + log_info(ip_plugin_rt->logger, MODULE_TABLE_RUNTIME, + "committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt); + new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); + if (NULL == new_ip_matcher) { + log_error(ip_plugin_rt->logger, MODULE_TABLE_RUNTIME, + "rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt); + ret = -1; + } + } + + old_ip_matcher = ip_plugin_rt->ip_matcher; + ip_plugin_rt->ip_matcher = new_ip_matcher; + maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); + ex_data_runtime_commit(ex_data_rt); + ip_plugin_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt); + + FREE(rules); + FREE(ex_container); + + return ret; +} + +int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt) +{ + return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt); +} + +struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime) +{ + if (NULL == ip_plugin_runtime) { + return NULL; + } + + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + + return ip_plugin_rt->ex_data_rt; +} \ No newline at end of file diff --git a/src/maat_kv.cpp b/src/maat_kv.cpp index 027dbda..055b427 100644 --- a/src/maat_kv.cpp +++ b/src/maat_kv.cpp @@ -130,4 +130,17 @@ int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t key int maat_kv_read(struct maat_kv_store * store, const char * key, int * value) { return maat_kv_read_unNull(store, key, strlen(key), value); +} + +struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map) +{ + struct maat_kv_store *target = maat_kv_store_new(); + struct maat_kv_pair *kv = NULL, *tmp_kv = NULL, *copy_kv = NULL; + + HASH_ITER (hh, origin_map->hash, kv, tmp_kv) { + copy_kv = maat_kv_pair_new(kv->key, kv->keylen, kv->val); + HASH_ADD_KEYPTR(hh, target->hash, copy_kv->key, copy_kv->keylen, copy_kv); + } + + return target; } \ No newline at end of file diff --git a/src/maat_plugin.cpp b/src/maat_plugin.cpp new file mode 100644 index 0000000..d525675 --- /dev/null +++ b/src/maat_plugin.cpp @@ -0,0 +1,375 @@ +/* +********************************************************************************************** +* File: maat_plugin.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include + +#include "log/log.h" +#include "cJSON/cJSON.h" +#include "utils.h" +#include "maat_utils.h" +#include "maat.h" +#include "maat_plugin.h" +#include "maat_ex_data.h" +#include "maat_limits.h" + +#define MODULE_PLUGIN module_name_str("maat.plugin") + +struct plugin_callback_schema { + maat_start_callback_t *start; + maat_update_callback_t *update; + maat_finish_callback_t *finish; + void *u_para; +}; + +struct plugin_item { + char key[MAX_KEYWORDS_STR]; + size_t key_len; +}; + +struct plugin_runtime { + uint64_t acc_line_num; + struct ex_data_runtime *ex_data_rt; + + uint32_t rule_num; + uint32_t updating_rule_num; + + struct maat_garbage_bin *ref_garbage_bin; +}; + +#define MAX_PLUGIN_PER_TABLE 32 +struct plugin_schema { + int item_id_column; + int key_column; + int rule_tag_column; + int n_foreign; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; + size_t cb_cnt; + struct plugin_callback_schema cb[MAX_PLUGIN_PER_TABLE]; + struct ex_data_schema *ex_schema; + int table_id; //ugly +}; + +static int read_integer_array(char *string, int *array, int size) +{ + int i = 0; + char *token = NULL, *sub_token = NULL, *saveptr; + + for (i = 0, token = string; i < size; token= NULL, i++) { + sub_token = strtok_r(token, ",", &saveptr); + if (sub_token == NULL) { + break; + } + + sscanf(sub_token, "%d", array + i); + } + + return i; +} + +void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +{ + size_t read_cnt = 0; + struct plugin_schema *plugin_schema = ALLOC(struct plugin_schema, 1); + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + goto error; + } + plugin_schema->table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "custom"); + if (item == NULL || item->type != cJSON_Object) { + log_error(logger, MODULE_PLUGIN, "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "item_id"); + if (custom_item == NULL || custom_item->type != cJSON_Object) { + plugin_schema->item_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "key"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + plugin_schema->key_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "tag"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + plugin_schema->rule_tag_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "foreign"); + if (custom_item != NULL) { + read_cnt++; + if (custom_item->type == cJSON_String) { + plugin_schema->n_foreign = read_integer_array(custom_item->valuestring, + plugin_schema->foreign_columns, + MAX_FOREIGN_CLMN_NUM); + } else if (custom_item->type == cJSON_Array) { + plugin_schema->n_foreign = cJSON_GetArraySize(custom_item); + for (int i = 0; i < plugin_schema->n_foreign; i++) { + cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i); + assert(foreign_item->type == cJSON_Number); + plugin_schema->foreign_columns[i] = foreign_item->valueint; + } + } + } + + if (read_cnt < 4) { + goto error; + } + + return plugin_schema; +error: + FREE(plugin_schema); + return NULL; +} + +void plugin_schema_free(void *plugin_schema) +{ + if (NULL == plugin_schema) { + return; + } + + struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; + if (schema->ex_schema != NULL) { + ex_data_schema_free(schema->ex_schema); + schema->ex_schema = NULL; + } + + FREE(schema); +} + +int plugin_table_add_callback(void *plugin_schema, int table_id, + maat_start_callback_t *start, + maat_update_callback_t *update, + maat_finish_callback_t *finish, + void *u_para, struct log_handle *logger) +{ + if (NULL == plugin_schema) { + return -1; + } + + struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; + size_t idx = schema->cb_cnt; + + if (idx == MAX_PLUGIN_PER_TABLE) { + log_error(logger, MODULE_PLUGIN, "the plugin number of table_id: %d exceed maxium:%d", + table_id, MAX_PLUGIN_PER_TABLE); + return -1; + } + + schema->cb_cnt++; + schema->cb[idx].start = start; + schema->cb[idx].update = update; + schema->cb[idx].finish = finish; + schema->cb[idx].u_para = u_para; + + return 0; +} + +void plugin_table_all_callback_start(struct plugin_schema *plugin_schema, int update_type) +{ + for (size_t i = 0; i < plugin_schema->cb_cnt; i++) { + if (plugin_schema->cb[i].start != NULL) { + plugin_schema->cb[i].start(update_type, plugin_schema->cb[i].u_para); + } + } +} + +void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema) +{ + for (size_t i = 0; i < plugin_schema->cb_cnt; i++) { + if (plugin_schema->cb[i].finish != NULL) { + plugin_schema->cb[i].finish(plugin_schema->cb[i].u_para); + } + } +} + +struct plugin_item *plugin_item_new(const char *line, struct plugin_schema *plugin_schema, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + struct plugin_item *plugin_item = ALLOC(struct plugin_item, 1); + + int ret = get_column_pos(line, plugin_schema->key_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_PLUGIN, + "plugin table(table_id:%d) line:%s has no key", + plugin_schema->table_id, line); + goto error; + } + + if (column_len > MAX_KEYWORDS_STR) { + log_error(logger, MODULE_PLUGIN, + "plugin table(table_id:%d): key:%s length:%zu too long, exceed %d", + plugin_schema->table_id, (line + column_offset), column_len, MAX_KEYWORDS_STR); + goto error; + } + memcpy(plugin_item->key, (line + column_offset), column_len); + plugin_item->key_len = column_len; + + return plugin_item; +error: + FREE(plugin_item); + return NULL; +} + +void plugin_item_free(struct plugin_item *plugin_item) +{ + FREE(plugin_item); +} + +int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns) +{ + if (NULL == plugin_schema) { + return -1; + } + + int n_foreign = plugin_schema->n_foreign; + for (int i = 0; i < n_foreign; i++) { + foreign_columns[i] = plugin_schema->foreign_columns[i]; + } + + return n_foreign; +} + +int plugin_table_set_ex_data_schema(void *plugin_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger) +{ + if (NULL == plugin_schema) { + return -1; + } + + struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; + if (schema->ex_schema != NULL) { + assert(0); + log_error(logger, MODULE_PLUGIN, + "Error: %s, EX data schema already registed", __FUNCTION__); + return -1; + } + schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp); + + return 0; +} + +struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema) +{ + if (NULL == plugin_schema) { + return NULL; + } + + struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; + return schema->ex_schema; +} + +int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_schema *plugin_schema, + const char *row, char *key, size_t key_len, int is_valid) +{ + int ret = -1; + void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); + struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL); + struct ex_data_schema *ex_schema = plugin_schema->ex_schema; + + /* already set plugin_table_schema's ex_data_schema */ + if (ex_schema != NULL) { + if (is_valid == 0) { + // delete + ret = ex_data_runtime_del_ex_container(plugin_rt->ex_data_rt, key, key_len); + if (ret < 0) { + return -1; + } + } else { + // add + ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container); + if (ret < 0) { + return -1; + } + } + } + + /* plugin table schema has callback */ + size_t cb_count = plugin_schema->cb_cnt; + if (cb_count > 0) { + for (size_t i = 0; i < cb_count; i++) { + plugin_schema->cb[i].update(plugin_schema->table_id, row, plugin_schema->cb[i].u_para); + } + } + + if ((NULL == ex_schema) && (0 == cb_count)) { + ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row); + } + + plugin_rt->acc_line_num++; + + return 0; +} + +int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line, + int valid_column) +{ + if (NULL == plugin_runtime || NULL == plugin_runtime) { + return -1; + } + + struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; + struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; + int item_id = get_column_value(line, schema->item_id_column); + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } + char *key = (char *)&item_id; + + int ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid); + if (ret < 0) { + + } else { + + } + return 0; +} + +int plugin_runtime_commit(void *plugin_runtime) +{ + if (NULL == plugin_runtime) { + return -1; + } + + struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; + ex_data_runtime_commit(plugin_rt->ex_data_rt); + //table_rt->rule_num = ex_data_runtime_ex_container_count(table_rt->plugin_rt.ex_data_rt); + + return 0; +} + +int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt) +{ + return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt); +} + +struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime) +{ + if (NULL == plugin_runtime) { + return NULL; + } + + struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; + return plugin_rt->ex_data_rt; +} \ No newline at end of file diff --git a/src/maat_redis_monitor.cpp b/src/maat_redis_monitor.cpp index 536ff7c..9d71d4d 100644 --- a/src/maat_redis_monitor.cpp +++ b/src/maat_redis_monitor.cpp @@ -17,7 +17,7 @@ #include "maat_command.h" #include "maat_config_monitor.h" #include "maat_redis_monitor.h" -#include "maat_table_schema.h" +#include "maat_plugin.h" #define MODULE_REDIS_MONITOR module_name_str("maat.redis_monitor") @@ -97,22 +97,22 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_instance, const char *dir) { int rule_with_foreign_key = 0; - struct table_schema *table_schema = NULL; + void *schema = NULL; for (int i = 0; i < rule_num; i++) { if (NULL == rule_list[i].table_line) { continue; } - int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, rule_list[i].table_name); - table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); - enum table_type table_type = table_schema_get_table_type(table_schema); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, rule_list[i].table_name); + schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); if (!table_schema || table_type != TABLE_TYPE_PLUGIN) { continue; } int foreign_columns[8]; - int n_foreign_column = table_schema_get_foreign_column(table_schema, foreign_columns); + int n_foreign_column = plugin_table_get_foreign_column((struct plugin_schema *)schema, foreign_columns); if (0 == n_foreign_column) { continue; } @@ -467,7 +467,7 @@ int recovery_history_version(const struct serial_rule *current, int current_num, } int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, - long long *new_version, struct table_schema_manager* table_schema_mgr, + long long *new_version, struct table_manager *tbl_mgr, struct serial_rule **list, int *update_type, int cumulative_off, struct log_handle *logger) { @@ -608,7 +608,7 @@ FULL_UPDATE: } if (table_schema_mgr) { - int table_id = table_schema_manager_get_table_id(table_schema_mgr, s_rule_array[full_idx].table_name); + int table_id = table_manager_get_table_id(tbl_mgr, s_rule_array[full_idx].table_name); //Unrecognized table. if (table_id < 0) { continue; @@ -1277,7 +1277,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, int valid_column = -1; enum table_type table_type; enum scan_type scan_type; - struct table_schema *table_schema = NULL; + void *table_schema = NULL; struct maat *maat_instance = (struct maat *)u_param; //authorized to write @@ -1369,19 +1369,19 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, continue; } - table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, rule_list[i].table_name); + table_id = table_manager_get_table_id(maat_instance->tbl_mgr, rule_list[i].table_name); //Unrecognized table. if (table_id < 0) { no_table_num++; continue; } - table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); if (rule_list[i].op == MAAT_OP_DEL) { - scan_type = table_schema_get_scan_type(table_schema); - table_type = table_schema_get_table_type(table_schema); - table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, scan_type, NULL); - valid_column = table_schema_get_valid_flag_column(table_schema); + //TODO: by luis + //scan_type = table_schema_get_scan_type(table_schema); + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + //table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, scan_type, NULL); + valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); ret = invalidate_line(rule_list[i].table_line, table_type, valid_column); if (ret < 0) { log_error(maat_instance->logger, MODULE_REDIS_MONITOR, diff --git a/src/maat_rule.cpp b/src/maat_rule.cpp index 5d56f64..32b0e9f 100644 --- a/src/maat_rule.cpp +++ b/src/maat_rule.cpp @@ -23,28 +23,223 @@ #include "maat_rule.h" #include "maat_config_monitor.h" #include "maat_redis_monitor.h" -#include "maat_table_runtime.h" -#include "maat_table_schema.h" -#include "maat_hierarchy.h" +#include "maat_table.h" +#include "maat_compile.h" +#include "maat_plugin.h" #include "alignment.h" #define MODULE_MAAT_RULE module_name_str("maat.rule") +struct maat_item *maat_item_new(int item_id, int group_id, void *user_data) +{ + struct maat_item *item = NULL; + item = ALLOC(struct maat_item, 1); + item->group_id = group_id; + item->item_id = item_id; + item->user_data = user_data; + + return item; +} + +void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)) +{ + if (item_user_data_free && item->user_data) { + item_user_data_free(item->user_data); + item->user_data = NULL; + } + + free(item); +} + +static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, int n_accept_tag) +{ + if (NULL == tag_obj || NULL == accept_tags) { + return -1; + } + + cJSON *tab_name_obj = cJSON_GetObjectItem(tag_obj, "tag"); + if (NULL == tab_name_obj || tab_name_obj->type != cJSON_String) { + return -1; + } + + const char *tag_name = tab_name_obj->valuestring; + cJSON *tag_vals_array = cJSON_GetObjectItem(tag_obj, "value"); + if (NULL == tag_vals_array || tag_vals_array->type != cJSON_Array) { + return -1; + } + + int name_matched = 0; + int n_val = cJSON_GetArraySize(tag_vals_array); + for (int i = 0; i < n_accept_tag; i++) { + if (0 != strcmp(accept_tags[i].tag_name, tag_name)) { + continue; + } + name_matched++; + + for (int j = 0; j < n_val; j++) { + cJSON *tag_val_obj = cJSON_GetArrayItem(tag_vals_array, j); + if (NULL == tag_val_obj || tag_val_obj->type != cJSON_String) { + return -1; + } + + const char *tag_val = tag_val_obj->valuestring; + // compare a/b/c with a/b/c/d is a miss. + if (strlen(accept_tags[i].tag_val) < strlen(tag_val)) { + continue; + } + + // compare a1a2/b1/c1 with a1a2/b/ is a miss. + //make sure the overlap is ended with a '/' + if (0 == strncmp(accept_tags[i].tag_val, tag_val, strlen(tag_val)) && + (strlen(accept_tags[i].tag_val) == strlen(tag_val) || accept_tags[i].tag_val[strlen(tag_val)] == '/')) { + return 1; + } + } + } + + //no matched name is considered as a + if (name_matched > 0) { + return 0; + } else { + return 1; + } +} + +//@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}] +static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_tags, int n_accept_tag) +{ + int matched = 0; + + int n_tag = cJSON_GetArraySize(tag_set); + for (int i = 0; i < n_tag; i++) { + cJSON *tag_obj = cJSON_GetArrayItem(tag_set, i); + if (NULL == tag_obj || tag_obj->type != cJSON_Object) { + goto error; + } + + int ret = compare_each_tag(tag_obj, accept_tags, n_accept_tag); + if (ret < 0) { + return -1; + } + + if(1 == ret) { + matched++; + } + } + + if (matched == n_tag) { + return 1; + } else { + return 0; + } + +error: + return -1; +} + +//@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]} +int parse_accept_tag(const char *value, struct rule_tag **result, void *logger) +{ + cJSON *json = JSON_Parse(value); + if (!json) { + log_error(logger, MODULE_MAAT_RULE, + "parse accept tag Error before: %-200.200s", cJSON_GetErrorPtr()); + return 0; + } + + cJSON *tag = NULL, *tmp = NULL; + cJSON *array = cJSON_GetObjectItem(json, "tags"); + int n_tag = cJSON_GetArraySize(array); + struct rule_tag *p = ALLOC(struct rule_tag, n_tag); + + for (int i = 0; i < n_tag; i++) { + tag = cJSON_GetArrayItem(array, i); + tmp = cJSON_GetObjectItem(tag, "tag"); + p[i].tag_name = maat_strdup(tmp->valuestring); + tmp = cJSON_GetObjectItem(tag, "value"); + p[i].tag_val = maat_strdup(tmp->valuestring); + } + + cJSON_Delete(json); + *result = p; + return n_tag; +} + +//@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]} +//@return 1 on match, 0 on not match, -1 on error. +int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag) +{ + int ret = -1; + int n_set = 0; + cJSON *tag_set = NULL; + cJSON *tag_set_array = NULL; + + cJSON *root = cJSON_Parse(value); + if (NULL == root) { + goto error; + } + + tag_set_array = cJSON_GetObjectItem(root, "tag_sets"); + if (NULL == tag_set_array || tag_set_array->type != cJSON_Array) { + goto error; + } + + n_set = cJSON_GetArraySize(tag_set_array); + for (int i = 0; i < n_set; i++) { + tag_set = cJSON_GetArrayItem(tag_set_array, i); + if (NULL == tag_set || tag_set->type != cJSON_Array) { + goto error; + } + + ret = compare_each_tag_set(tag_set, accept_tags, n_tag); + //match or error occurs. + if (ret != 0) { + break; + } + } + +error: + cJSON_Delete(root); + return ret; +} + +struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id) +{ + struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1); + item->magic_num = ITEM_RULE_MAGIC; + item->item_id = item_id; + item->group_id = group_id; + item->district_id = district_id; + + return item; +} + +void maat_item_inner_free(struct maat_item_inner *item) +{ + assert(item->magic_num == ITEM_RULE_MAGIC); + assert(item->expr_id_cnt == 0 || item->expr_id_cnt == item->expr_id_ub - item->expr_id_lb + 1); + item->magic_num = 0; + + free(item); +} + struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance) { struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); maat_rt->version = version; - maat_rt->table_rt_mgr = table_runtime_manager_create(maat_instance->table_schema_mgr, - maat_instance->nr_worker_thread, - maat_instance->garbage_bin, - maat_instance->logger); - maat_rt->max_table_num = table_schema_manager_get_size(maat_instance->table_schema_mgr); + int ret = table_manager_init(maat_instance->tbl_mgr, maat_instance->garbage_bin); + if (ret < 0) { + FREE(maat_rt); + return NULL; + } + + maat_rt->ref_tbl_mgr = maat_instance->tbl_mgr; + maat_rt->max_table_num = table_manager_table_count(maat_instance->table_mgr); maat_rt->max_thread_num = maat_instance->nr_worker_thread; maat_rt->logger = maat_instance->logger; maat_rt->ref_garbage_bin = maat_instance->garbage_bin; - maat_rt->item_result_buff = ALLOC(struct scan_result, MAX_SCANNER_HIT_NUM * maat_instance->nr_worker_thread); maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->nr_worker_thread); return maat_rt; @@ -53,12 +248,12 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger) { for (size_t i = 0; i < maat_rt->max_table_num; i++) { - struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i); - if (NULL == table_rt) { + void *runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, i); + if (NULL == runtime) { continue; } - table_runtime_commit(table_rt, maat_rt->version, maat_rt->max_thread_num, logger); + table_manager_commit_runtime(runtime, maat_rt->version, maat_rt->max_thread_num, logger); } maat_rt->last_update_time = time(NULL); @@ -71,8 +266,8 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt) } if (maat_rt->table_rt_mgr != NULL) { - table_runtime_manager_destroy(maat_rt->table_rt_mgr); - maat_rt->table_rt_mgr = NULL; + table_manager_runtime_destroy(maat_rt->ref_tbl_mgr); + maat_rt->ref_table_mgr = NULL; } FREE(maat_rt); @@ -81,7 +276,7 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt) int maat_runtime_updating_flag(struct maat_runtime *maat_rt) { for (size_t i = 0; i < maat_rt->max_table_num; i++) { - struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, i); + struct table_runtime *table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, i); if (NULL == table_rt) { continue; } @@ -105,27 +300,41 @@ void maat_start_cb(long long new_version, int update_type, void *u_param) maat_instance->maat_version = new_version; } - table_schema_manager_all_plugin_cb_start(maat_instance->table_schema_mgr, update_type); + int table_id = -1; + enum table_type table_type = TABLE_TYPE_MAX; + size_t table_cnt = table_manager_table_count(maat_instance->tbl_mgr); + for (size_t i = 0; i < table_cnt; i++) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i); + if (table_type != TABLE_TYPE_PLUGIN) { + continue; + } + + void *schema = table_manager_get_schema(maat_instance->tbl_mgr, i); + plugin_table_all_callback_start((struct plugin_schema *)schema, update_type); + } } int maat_update_cb(const char *table_name, const char *line, void *u_param) { + if (NULL == table_name || NULL == line || NULL == u_param) { + return 0; + } + struct maat *maat_instance =(struct maat *)u_param; struct maat_runtime* maat_rt = NULL; - int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, table_name); if (table_id < 0) { - log_error(maat_instance->logger, MODULE_MAAT_RULE, "update warning, unknown table name %s", table_name); + log_error(maat_instance->logger, MODULE_MAAT_RULE, + "update warning, unknown table name %s", table_name); return -1; } - struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); - if (NULL == table_schema) { + void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + if (NULL == schema) { log_error(maat_instance->logger, MODULE_MAAT_RULE, "update warning, table name %s doesn't have table schema", table_name); return -1; } - - table_schema_set_updating_name(table_schema, table_name); if (maat_instance->creating_maat_rt != NULL) { maat_rt = maat_instance->creating_maat_rt; @@ -133,14 +342,16 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param) maat_rt = maat_instance->maat_rt; } - struct table_item *table_item = table_schema_line_to_item(line, table_schema, maat_instance->accept_tags, - maat_instance->n_accept_tag, maat_instance->logger); - if (table_item != NULL) { - struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); - table_runtime_update(table_rt, table_schema, line, table_item, maat_instance->logger); - table_item_free(table_item); + void *runtime = table_manager_get_runtime(maat_rt->tbl_mgr, table_id); + int ret = table_manager_update_runtime(runtime, schema, line); + if (ret < 0) { + log_error(maat_instance->logger, MODULE_MAAT_RULE, + "table manager update runtime error, table_name:%s", table_name); + return -1; } - + + //TODO: by luis + //int is_valid = table_manager_get_valid return 0; } @@ -150,7 +361,7 @@ uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt) struct table_runtime *table_rt = NULL; for (size_t i = 0; i < maat_rt->max_table_num; i++) { - table_rt = table_runtime_get(maat_rt->table_rt_mgr, i); + table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, i); if (table_rt != NULL) { total += table_runtime_rule_count(table_rt); } @@ -163,8 +374,8 @@ void maat_finish_cb(void *u_param) { struct maat *maat_instance = (struct maat *)u_param; - table_schema_manager_all_plugin_cb_finish(maat_instance->table_schema_mgr); - + //table_manager_all_plugin_cb_finish(maat_instance->table_schema_mgr); + plugin_table_all_callback_finish(maat_) if (maat_instance->creating_maat_rt != NULL) { maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger); @@ -290,7 +501,7 @@ void *rule_monitor_loop(void *arg) maat_runtime_destroy(maat_instance->maat_rt); maat_garbage_bin_free(maat_instance->garbage_bin); - table_schema_manager_destroy(maat_instance->table_schema_mgr); + table_manager_destroy(maat_instance->tbl_mgr); if (maat_instance->input_mode == DATA_SOURCE_REDIS) { if (maat_instance->mr_ctx.read_ctx != NULL) { @@ -304,6 +515,12 @@ void *rule_monitor_loop(void *arg) } } + for (int i = 0; i < maat_instance->n_accept_tag; i++) { + FREE(maat_instance->accept_tags[i].tag_name); + FREE(maat_instance->accept_tags[i].tag_val); + } + FREE(maat_instance->accept_tags); + FREE(maat_instance); return NULL; diff --git a/src/maat_table.cpp b/src/maat_table.cpp new file mode 100644 index 0000000..5d2ac0e --- /dev/null +++ b/src/maat_table.cpp @@ -0,0 +1,759 @@ +/* +********************************************************************************************** +* File: maat_table.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include +#include + +#include "log/log.h" +#include "maat_utils.h" +#include "maat_table.h" +#include "maat_rule.h" +#include "maat_garbage_collection.h" +#include "maat_kv.h" +#include "maat_expr.h" +#include "maat_ip.h" +#include "maat_compile.h" +#include "maat_group.h" +#include "maat_plugin.h" +#include "maat_ip_plugin.h" + +#define MODULE_TABLE module_name_str("maat.table") + +struct table_item { + enum table_type table_type; + void *custom_item; +}; + +struct maat_table { + int table_id; + char table_name[NAME_MAX]; + enum table_type table_type; + int valid_column; + void *schema; + void *runtime; +}; + +struct table_manager { + struct maat_table *tbl[MAX_TABLE_NUM]; + size_t n_table; + + struct rule_tag *accept_tags; + int n_accept_tag; + + struct maat_kv_store *tablename2id_map; + struct log_handle *logger; +}; + +struct table_operations { + enum table_type type; + void *(*new_schema)(cJSON *json, const char *table_name, struct log_handle *logger); + void (*free_schema)(void *schema); + + void *(*new_runtime)(void *schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); + void (*free_runtime)(void *runtime); + + int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column); + int (*commit_runtime)(void *runtime); +}; + +struct table_operations table_ops[TABLE_TYPE_MAX] = { + { + .type = TABLE_TYPE_EXPR, + .new_schema = expr_schema_new, + .free_schema = expr_schema_free, + .new_runtime = expr_runtime_new, + .free_runtime = expr_runtime_free, + .update_runtime = expr_runtime_update, + .commit_runtime = expr_runtime_commit + }, + { + .type = TABLE_TYPE_EXPR_PLUS, + .new_schema = expr_schema_new, + .free_schema = expr_schema_free, + .new_runtime = expr_runtime_new, + .free_runtime = expr_runtime_free, + .update_runtime = expr_runtime_update, + .commit_runtime = expr_runtime_commit + }, + { + .type = TABLE_TYPE_IP_PLUS, + .new_schema = ip_plus_schema_new, + .free_schema = ip_plus_schema_free, + .new_runtime = ip_plus_runtime_new, + .free_runtime = ip_plus_runtime_free, + .update_runtime = ip_plus_runtime_update, + .commit_runtime = ip_plus_runtime_commit + }, + { + .type = TABLE_TYPE_INTERVAL, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, + { + .type = TABLE_TYPE_INTERVAL_PLUS, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, + { + .type = TABLE_TYPE_DIGEST, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, + { + .type = TABLE_TYPE_SIMILARITY, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, + { + .type = TABLE_TYPE_CONJUNCTION, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, + { + .type = TABLE_TYPE_PLUGIN, + .new_schema = plugin_schema_new, + + }, + { + .type = TABLE_TYPE_IP_PLUGIN, + .new_schema = ip_plugin_schema_new, + .free_schema = ip_plugin_schema_free, + .new_runtime = ip_plugin_runtime_new, + .free_runtime = ip_plugin_runtime_free, + .update_runtime = ip_plugin_runtime_update, + .commit_runtime = ip_plugin_runtime_commit + }, + { + .type = TABLE_TYPE_FQDN_PLUGIN, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, + { + .type = TABLE_TYPE_BOOL_PLUGIN, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, + { + .type = TABLE_TYPE_VIRTUAL, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, + { + .type = TABLE_TYPE_COMPILE, + .new_schema = compile_schema_new, + .free_schema = compile_schema_free, + .new_runtime = compile_runtime_new, + .free_runtime = compile_runtime_free, + .update_runtime = compile_runtime_update, + .commit_runtime = compile_runtime_commit + }, + { + .type = TABLE_TYPE_GROUP2COMPILE, + .new_schema = group2compile_schema_new, + .free_schema = group2compile_schema_free, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = group2compile_runtime_update, + .commit_runtime = NULL + }, + { + .type = TABLE_TYPE_GROUP2GROUP, + .new_schema = group2group_schema_new, + .free_schema = group2group_schema_free, + .new_runtime = group2group_runtime_new, + .free_runtime = group2group_runtime_free, + .update_runtime = group2group_runtime_update, + .commit_runtime = group2group_runtime_commit + } +}; + +void *maat_table_schema_new(cJSON *json, const char *table_name, enum table_type table_type) +{ + void *schema = NULL; + + if (table_ops[table_type].new_schema != NULL) { + schema = table_ops[table_type].new_schema(json, table_name, logger); + } + + return schema; +} + +void maat_table_schema_free(void *schema, enum table_type table_type) +{ + if (NULL == schema) { + return; + } + + if (table_ops[table_type].free_schema != NULL) { + table_ops[table_type].free_schema(schema); + } +} + +static void register_reserved_word(struct maat_kv_store *reserved_word_map) +{ + maat_kv_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE); + maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); + maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); + maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); + maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); + maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); + maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); + maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); + maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); +} + +static void register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map, + struct log_handle *logger) +{ + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + return; + } + int table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "table_name"); + if (NULL == item || item->type != cJSON_String) { + log_error(logger, MODULE_TABLE, + "table(table_id:%d) has no table name", table_id); + return; + } + + if (strlen(item->valuestring) >= NAME_MAX) { + log_error(logger, MODULE_TABLE_SCHEMA, + "table(table_id:%d) name %s length too long", table_id, item->valuestring); + return; + } + + maat_kv_register(tablename2id_map, item->valuestring, table_id); +} + +struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_word_map, + struct log_handle *logger) +{ + struct maat_table *ptable = ALLOC(struct maat_table, 1); + + int ret = -1; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + goto error; + } + + if (item->valueint >= MAX_TABLE_NUM) { + log_error(logger, MODULE_TABLE, + "table(table_id:%d) exceed maxium %d", MAX_TABLE_NUM); + goto error; + } + ptable->table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "table_name"); + if (NULL == item || item->type != cJSON_String) { + log_error(logger, MODULE_TABLE, + "table(table_id:%d) has no table name", ptable->table_id); + goto error; + } + + if (strlen(item->valuestring) >= NAME_MAX) { + log_error(logger, MODULE_TABLE, + "table(table_id:%d) name %s length too long", + ptable->table_id, item->valuestring); + goto error; + } + memcpy(ptable->table_name, item->valuestring, strlen(item->valuestring)); + + item = cJSON_GetObjectItem(json, "table_type"); + if (NULL == item || item->type != cJSON_String) { + goto error; + } + + ret = maat_kv_read(reserved_word_map, item->valuestring, (int*)&(ptable->table_type)); + if (ret < 0) { + log_error(logger, MODULE_TABLE, "table_type %s is illegal", item->valuestring); + goto error; + } + + item = cJSON_GetObjectItem(json, "valid_column"); + if (NULL == item && item->type != cJSON_Number) { + goto error; + } + ptable->valid_column = item->valueint; + + return ptable; +error: + FREE(ptable); + return NULL; +} + +void maat_table_free(struct maat_table *maat_tbl) +{ + if (NULL == maat_tbl) { + return; + } + + if (maat_tbl->schema != NULL) { + maat_table_schema_free(maat_tbl->schema); + maat_tbl->schema = NULL; + } + + if (maat_tbl->runtime != NULL) { + maat_table_runtime_free(maat_tbl->runtime); + maat_tbl->runtime = NULL; + } + + FREE(maat_tbl); +} + +struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags, + struct log_handle *logger) +{ + if (NULL == table_info_path) { + return NULL; + } + + unsigned char *json_buff = NULL; + size_t json_buff_sz = 0; + int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz); + if (ret < 0) { + log_error(logger, MODULE_TABLE, "Maat read table info %s error.", table_info_path); + return NULL; + } + + cJSON *root = NULL; + cJSON *json = NULL; + root = cJSON_Parse((const char *)json_buff); + if (!root) { + log_error(logger, MODULE_TABLE, "Error before: %-200.200s", cJSON_GetErrorPtr()); + FREE(json_buff); + return NULL; + } + + int json_array_size = cJSON_GetArraySize(root); + if (json_array_size <= 0) { + log_error(logger, MODULE_TABLE, "invalid json content in %s", table_info_path); + free(json_buff); + return NULL; + } + + struct maat_kv_store *reserved_word_map = maat_kv_store_new(); + register_reserved_word(reserved_word_map); + + struct table_manager *tbl_mgr = ALLOC(struct table_manager, 1); + tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger); + tbl_mgr->logger = logger; + tbl_mgr->tablename2id_map = maat_kv_store_new(); + + int default_compile_table_id = MAX_TABLE_NUM; + + for (int i = 0; i < json_array_size; i++) { + json = cJSON_GetArrayItem(root, i); + + if (json != NULL && json->type == cJSON_Object) { + register_tablename2id(json, tbl_mgr->tablename2id_map, logger); + } + } + + for (int i = 0; i < json_array_size; i++) { + json = cJSON_GetArrayItem(root, i); + + if (json != NULL && json->type == cJSON_Object) { + struct maat_table *maat_tbl = maat_table_new(json, reserved_word_map, logger); + if (NULL == maat_tbl) { + log_error(logger, MODULE_TABLE, "Maat table new failed."); + continue; + } + + maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type); + if (NULL == maat_tbl->schema) { + log_error(logger, MODULE_TABLE, "Maat table schema new failed, table_name:%d", + maat_tbl->table_name); + maat_table_free(maat_tbl); + continue; + } + + if (maat_tbl->table_type == TABLE_TYPE_COMPILE) { + if (maat_tbl->table_id < default_compile_table_id) { + default_compile_table_id = maat_tbl->table_id; + } + } + + tbl_mgr->tbl[maat_tbl->table_id] = maat_tbl; + tbl_mgr->n_table++; + } + } + + assert(default_compile_table_id != MAX_TABLE_NUM); + tbl_mgr->default_compile_table_id = default_compile_table_id; + log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id); + + maat_kv_store_free(reserved_word_map); + cJSON_Delete(root); + FREE(json_buff); + + return tbl_mgr; +} + +void *maat_table_runtime_new(void *schema, enum table_type table_type, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + void *runtime = NULL; + + if (table_ops[table_type].new_runtime != NULL) { + runtime = table_ops[table_type].new_runtime(schema, garbage_bin, logger); + } + + return runtime; + + #if 0 + switch (table_rt->table_type) { + case TABLE_TYPE_COMPILE: + break; + case TABLE_TYPE_GROUP2COMPILE: + break; + case TABLE_TYPE_GROUP2GROUP: + table_rt->custom_rt = group2group_runtime_new(logger); + table_rt->g2g_rt.group_topo = maat_group_topology_new(logger); + break; + case TABLE_TYPE_EXPR: + table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free); + table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema); + break; + case TABLE_TYPE_IP_PLUS: + table_rt->ip_plus_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); + ex_container_ctx = ALLOC(struct ex_container_ctx, 1); + ex_container_ctx->custom_data_free = free; + //ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx); + break; + case TABLE_TYPE_PLUGIN: + table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); + break; + case TABLE_TYPE_IP_PLUGIN: + table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); + ex_container_ctx = ALLOC(struct ex_container_ctx, 1); + ex_container_ctx->custom_data_free = free; + //ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx); + break; + default: + break; + } + #endif +} + + +void maat_table_runtime_free(void *runtime, enum table_type table_type) +{ + if (NULL == runtime) { + return; + } + + table_ops[table_type].free_runtime(runtime); +#if 0 + switch (table_rt->table_type) { + case TABLE_TYPE_COMPILE: + bool_matcher_free(table_rt->compile_rt.bm); + maat_compile_hash_free(&(table_rt->compile_rt.compile_hash)); + break; + case TABLE_TYPE_GROUP2COMPILE: + break; + case TABLE_TYPE_GROUP2GROUP: + maat_group_topology_free(table_rt->g2g_rt.group_topo); + break; + case TABLE_TYPE_EXPR: + adapter_hs_destroy(table_rt->expr_rt.hs); + rcu_hash_free(table_rt->expr_rt.htable); + break; + case TABLE_TYPE_PLUGIN: + ex_data_runtime_free(table_rt->plugin_rt.ex_data_rt); + break; + case TABLE_TYPE_IP_PLUGIN: + ip_matcher_free(table_rt->ip_plugin_rt.ip_matcher); + ex_data_runtime_free(table_rt->ip_plugin_rt.ex_data_rt); + break; + default: + break; + } +#endif +} + +int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *garbage_bin) +{ + if (NULL == tbl_mgr) { + return -1; + } + + assert(tbl_mgr->n_table != 0); + + size_t i = 0; + int g2g_group_id = MAX_TABLE_NUM; + enum table_type table_type = TABLE_TYPE_MAX; + + for (i = 0; i < MAX_TABLE_NUM; i++) { + void *schema = table_manager_get_schema(tbl_mgr, i); + if (NULL == schema) { + continue; + } + + table_type = table_manager_get_table_type(tbl_mgr, i); + if (table_type == TABLE_TYPE_GROUP2GROUP) { + g2g_group_id = i; + } + + tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, garbage_bin, logger); + } + + assert(g2g_group_id != MAX_TABLE_NUM); + + /* group2compile runtime depends on associated compile runtime, + must make sure associated compile runtime already exist */ + for (i = 0; i < MAX_TABLE_NUM; i++) { + void *runtime = tbl_mgr->tbl[i]->runtime; + if (NULL == runtime) { + continue; + } + + table_type = table_manager_get_table_type(tbl_mgr, i); + if (table_type != TABLE_TYPE_GROUP2COMPILE) { + continue; + } + + + void *schema = table_manager_get_schema(tbl_mgr, i); + //int associated_compile_table_id = table_schema_get_associated_table_id(table_schema); + //TODO: by luis + int associated_compile_table_id = -1; + void *compile_rt = table_manager_get_runtime(tbl_mgr, associated_compile_table_id); + void *g2g_rt = table_manager_get_runtime(tbl_mgr, g2g_group_id); + + table_rt->g2c_rt.ref_compile_rt = &(compile_table_rt->compile_rt); + table_rt->g2c_rt.ref_g2g_rt = &(g2g_table_rt->g2g_rt); + assert(table_rt->g2c_rt.ref_compile_rt != NULL); + assert(table_rt->g2c_rt.ref_g2g_rt != NULL); + } + + return 0; +} + +void table_manager_deinit(struct table_manager *tbl_mgr) +{ + if (NULL == tbl_mgr) { + return; + } + + for(size_t i = 0; i < MAX_TABLE_NUM; i++) { + maat_table_runtime_free(tbl_mgr->tbl[i]->runtime, tbl_mgr->tbl[i]->table_type); + tbl_mgr->tbl[i]->runtime = NULL; + } +} + +void table_manager_destroy(struct table_manager *tbl_mgr) +{ + if (NULL == tbl_mgr) { + return; + } + + for (size_t i = 0; i < MAX_TABLE_NUM; i++) { + assert(NULL == tbl_mgr->tbl[i]->runtime); + + if (NULL == tbl_mgr->tbl[i]->schema) { + continue; + } + + table_schema_free(tbl_mgr->tbl[i]->schema); + tbl_mgr->tbl[i]->schema = NULL; + } + + maat_kv_store_free(tbl_mgr->tablename2id_map); + FREE(tbl_mgr); +} + +size_t table_manager_table_count(struct table_manager *tbl_mgr) +{ + return MAX_TABLE_NUM; +} + +int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) +{ + +} + +enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id) +{ + +} + +int table_manager_get_logger(struct table_manager *tbl_mgr) +{ + +} + +void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id) +{ + if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) { + return NULL; + } + + if (NULL == tbl_mgr->tbl[table_id]) { + return NULL; + } + + return tbl_mgr->tbl[table_id]->schema; +} + +struct ex_data_schema *table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id) +{ + +} + +int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id) +{ + if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) { + return -1; + } + + if (NULL == tbl_mgr->tbl[table_id]) { + return -1; + } + + return tbl_mgr->tbl[table_id]->valid_column; +} + +int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags) +{ + +} + +void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id) +{ + if (NULL == tbl_mgr || (table_id < 0) || (table_id >= MAX_TABLE_NUM)) { + return NULL; + } + + assert(table_id < (int)tbl_mgr->n_table); + + if (NULL == tbl_mgr->tbl[table_id]) { + return NULL; + } + + return tbl_mgr->tbl[table_id]->runtime; +} + +int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line) +{ + void *schema = table_manager_get_schema(tbl_mgr, table_id); + if (NULL == schema) { + return -1; + } + + void *runtime = table_manager_get_runtime(tbl_mgr, table_id); + if (NULL == runtime) { + return -1; + } + + int valid_column = table_manager_get_valid_column(tbl_mgr, table_id); + if (valid_column < 0) { + return -1; + } + + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + if (table_type == TABLE_TYPE_MAX) { + return -1; + } + + if (NULL == table_ops[table_type].update_runtime) { + return -1; + } + + return table_ops[table_type].update_runtime(runtime, schema, line, valid_column); + + #if 0 + switch (table_rt->table_type) { + case TABLE_TYPE_COMPILE: + ret = compile_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, table_rt->ref_garbage_bin, logger); + break; + case TABLE_TYPE_GROUP2COMPILE: + ret = group2compile_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger); + break; + case TABLE_TYPE_GROUP2GROUP: + ret = group2group_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, logger); + break; + case TABLE_TYPE_EXPR: + ret = expr_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger); + break; + case TABLE_TYPE_IP_PLUS: + ret = ip_plus_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger); + break; + case TABLE_TYPE_PLUGIN: + ret = plugin_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, row, logger); + break; + case TABLE_TYPE_IP_PLUGIN: + ret = ip_plugin_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, row, logger); + break; + default: + break; + } + + if (ret < 0) { + return; + } + #endif +#if 0 + if (is_valid == 0) { + table_rt->rule_num--; + } else { + table_rt->rule_num++; + } +#endif +} + +void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) +{ + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + if (table_type == TABLE_TYPE_MAX) { + return; + } + + void *runtime = table_manager_get_runtime(tbl_mgr, table_id); + if (NULL == runtime) { + return; + } + + table_ops[table_type].commit_runtime(runtime); +} \ No newline at end of file diff --git a/src/maat_table_runtime.cpp b/src/maat_table_runtime.cpp deleted file mode 100644 index b8baece..0000000 --- a/src/maat_table_runtime.cpp +++ /dev/null @@ -1,1290 +0,0 @@ -/* -********************************************************************************************** -* File: maat_table_runtime.cpp -* Description: -* Authors: Liu WenTan -* Date: 2022-10-31 -* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. -*********************************************************************************************** -*/ - -#include -#include -#include -#include -#include - -#include "utils.h" -#include "log/log.h" -#include "cJSON/cJSON.h" -#include "maat_utils.h" -#include "maat_table_schema.h" -#include "maat_table_runtime.h" -#include "uthash/uthash.h" -#include "maat_ex_data.h" -#include "adapter_hs.h" -#include "rcu_hash.h" -#include "IPMatcher.h" -#include "alignment.h" -#include "maat_hierarchy.h" -#include "maat_rule.h" -#include "igraph/igraph.h" -#include "maat_group.h" - -#define MODULE_TABLE_RUNTIME module_name_str("maat.table_runtime") - -#define MAAT_MAX_EXPR_ITEM_NUM 8 - -struct port_range { - uint16_t min_port; - uint16_t max_port; -}; - -struct ex_container_ctx { - int table_id; - void (*custom_data_free)(void *custom_data); - struct ex_data_schema *ex_schema; -}; - -struct compile_runtime { - struct maat_hierarchy *hier; -}; - -struct group2compile_runtime { - long long not_flag_group; - struct compile_runtime *ref_compile_rt; - struct group2group_runtime *ref_g2g_rt; -}; - -struct group2group_runtime { - struct maat_group_topology *group_topo; -}; - -struct expr_runtime { - enum hs_scan_mode scan_mode; - struct adapter_hs *hs; - struct adapter_hs_stream *hs_stream; - struct rcu_hash_table *htable; -}; - -struct plugin_runtime { - uint64_t acc_line_num; - struct ex_data_runtime *ex_data_rt; -}; - -struct ip_plugin_runtime { - struct ip_matcher* ip_matcher; - struct ex_data_runtime* ex_data_rt; -}; - -struct table_runtime { - uint32_t rule_num; - uint32_t updating_rule_num; - enum table_type table_type; - union { - struct compile_runtime compile_rt; - struct group2compile_runtime g2c_rt; - struct group2group_runtime g2g_rt; - struct expr_runtime expr_rt; - struct ip_plugin_runtime ip_plus_rt; - struct plugin_runtime plugin_rt; - struct ip_plugin_runtime ip_plugin_rt; - }; - - struct maat_garbage_bin *ref_garbage_bin; - - long long *scan_cnt; - long long *hit_cnt; - long long *stream_num; - //ex_data_rt - //table相关指针 -}; - -struct table_runtime_manager { - struct table_runtime **table_rt; - size_t n_table_rt; - - struct maat_item *hash_item_by_id; - struct maat_garbage_bin *garbage_bin; -}; - -void ex_data_container_free(void *ctx, void *data) -{ - if (NULL == ctx || NULL == data) { - return; - } - - struct ex_container_ctx *container_ctx = (struct ex_container_ctx *)ctx; - long argl = container_ctx->ex_schema->argl; - void *argp = container_ctx->ex_schema->argp; - - struct ex_data_container *ex_container = (struct ex_data_container *)data; - if (ex_container->ex_data != NULL && container_ctx->ex_schema->free_func != NULL) { - container_ctx->ex_schema->free_func(container_ctx->table_id, &(ex_container->ex_data), argl, argp); - } - - if (ex_container->custom_data != NULL && container_ctx->custom_data_free != NULL) { - container_ctx->custom_data_free(ex_container->custom_data); - } - - FREE(ex_container); -} - -void expr_rule_free(and_expr_t *expr_rule) -{ - if (NULL == expr_rule) { - return; - } - - for (size_t i = 0; i < expr_rule->n_patterns; i++) { - FREE(expr_rule->patterns[i].pat); - } - - FREE(expr_rule); -} -void expr_ex_data_free(void *user_ctx, void *data) -{ - and_expr_t *expr_rule = (and_expr_t *)data; - expr_rule_free(expr_rule); -} - -void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, - const char *srv_def, int srv_def_len) -{ - memcpy(rule, rule_head, sizeof(struct maat_rule_head)); - memcpy(rule->service_defined, srv_def, MIN(srv_def_len, MAX_SERVICE_DEFINE_LEN)); -} - -void *rule_ex_data_new(const struct maat_rule_head *rule_head, const char *srv_def, const struct compile_ex_data_schema *ex_schema) -{ - void *ex_data = NULL; - struct maat_rule rule; - - fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); - ex_schema->new_func(ex_schema->idx, &rule, srv_def, &ex_data, ex_schema->argl, ex_schema->argp); - - return ex_data; -} - -void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, void *ex_data, - const struct compile_ex_data_schema *ex_schema) -{ - struct maat_rule rule; - - memset(&rule, 0, sizeof(rule)); - fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); - ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, ex_schema->argl, ex_schema->argp); -} - -void destroy_compile_rule(struct maat_compile_rule *compile_rule) -{ - size_t n_compile_ex_schema = table_schema_compile_rule_ex_data_schema_count(compile_rule->ref_table); - assert(compile_rule->magic_num == COMPILE_RULE_MAGIC); - - for (size_t i = 0; i < n_compile_ex_schema; i++) { - struct compile_ex_data_schema *compile_ex_schema = table_schema_get_compile_rule_ex_data_schema(compile_rule->ref_table, i); - rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, compile_ex_schema); - void *ex_data = compile_rule->ex_data + 1; - ex_data = NULL; - } - free(compile_rule->ex_data); - - compile_rule->is_valid = 0; - compile_rule->declared_clause_num = -1; - FREE(compile_rule->service_defined); - FREE(compile_rule); -} - -void maat_item_inner_free(struct maat_item_inner *item) -{ - assert(item->magic_num == ITEM_RULE_MAGIC); - assert(item->expr_id_cnt == 0 || item->expr_id_cnt == (item->expr_id_ub - item->expr_id_lb + 1)); - item->magic_num = 0; - FREE(item); -} - -struct table_runtime *table_runtime_new(struct table_schema *table_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, - struct log_handle *logger) -{ - struct ex_container_ctx *ex_container_ctx = NULL; - int table_id = table_schema_get_table_id(table_schema); - struct table_runtime *table_rt = ALLOC(struct table_runtime, 1); - table_rt->ref_garbage_bin = garbage_bin; - table_rt->table_type = table_schema_get_table_type(table_schema); - - switch (table_rt->table_type) { - case TABLE_TYPE_COMPILE: - table_rt->compile_rt.hier = maat_hierarchy_new(max_thread_num, garbage_bin, logger); - maat_hierarchy_set_compile_user_data_free_func(table_rt->compile_rt.hier, (void (*)(void*))destroy_compile_rule); - maat_hierarchy_set_item_user_data_free_func(table_rt->compile_rt.hier, (void (*)(void*))maat_item_inner_free); - break; - case TABLE_TYPE_GROUP2COMPILE: - break; - case TABLE_TYPE_GROUP2GROUP: - table_rt->g2g_rt.group_topo = maat_group_topology_new(logger); - break; - case TABLE_TYPE_EXPR: - table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free); - table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema); - break; - case TABLE_TYPE_IP_PLUS: - table_rt->ip_plus_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); - ex_container_ctx = ALLOC(struct ex_container_ctx, 1); - ex_container_ctx->custom_data_free = free; - //ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx); - break; - case TABLE_TYPE_PLUGIN: - table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); - break; - case TABLE_TYPE_IP_PLUGIN: - table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); - ex_container_ctx = ALLOC(struct ex_container_ctx, 1); - ex_container_ctx->custom_data_free = free; - //ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx); - break; - default: - break; - } - - table_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); - table_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); - table_rt->stream_num = alignment_int64_array_alloc(max_thread_num); - - return table_rt; -} - -void table_runtime_free(struct table_runtime *table_rt) -{ - if (NULL == table_rt) { - return; - } - - switch (table_rt->table_type) { - case TABLE_TYPE_COMPILE: - maat_hierarchy_free(table_rt->compile_rt.hier); - break; - case TABLE_TYPE_GROUP2COMPILE: - break; - case TABLE_TYPE_GROUP2GROUP: - maat_group_topology_free(table_rt->group2group_rt.group_topo); - break; - case TABLE_TYPE_EXPR: - adapter_hs_destroy(table_rt->expr_rt.hs); - rcu_hash_free(table_rt->expr_rt.htable); - break; - case TABLE_TYPE_PLUGIN: - ex_data_runtime_free(table_rt->plugin_rt.ex_data_rt); - break; - case TABLE_TYPE_IP_PLUGIN: - ip_matcher_free(table_rt->ip_plugin_rt.ip_matcher); - ex_data_runtime_free(table_rt->ip_plugin_rt.ex_data_rt); - break; - default: - break; - } - - FREE(table_rt); -} - -struct table_runtime_manager * -table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num, - struct maat_garbage_bin *garbage_bin, struct log_handle *logger) -{ - if (NULL == table_schema_mgr) { - return NULL; - } - - struct table_runtime_manager *table_rt_mgr = ALLOC(struct table_runtime_manager, 1); - table_rt_mgr->n_table_rt = table_schema_manager_get_size(table_schema_mgr); - table_rt_mgr->table_rt = ALLOC(struct table_runtime *, table_rt_mgr->n_table_rt); - - size_t i = 0; - for (i = 0; i < table_rt_mgr->n_table_rt; i++) { - struct table_schema *table_schema = table_schema_get(table_schema_mgr, i); - if (NULL == table_schema) { - continue; - } - table_rt_mgr->table_rt[i] = table_runtime_new(table_schema, max_thread_num, garbage_bin, logger); - } - - /* group2compile table_rt depends on associated compile table_rt, - must make sure associated compile table_rt already exist */ - for (i = 0; i < table_rt_mgr->n_table_rt; i++) { - struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, i); - if (NULL == table_rt) { - continue; - } - - enum table_type table_type = table_runtime_get_type(table_rt); - if (table_type != TABLE_TYPE_GROUP2COMPILE) { - continue; - } - - struct table_schema *table_schema = table_schema_get(table_schema_mgr, i); - int associated_compile_table_id = table_schema_get_associated_table_id(table_schema); - struct table_runtime *compile_table_rt = table_runtime_get(table_rt_mgr, associated_compile_table_id); - table_rt->group2compile_rt.ref_compile_rt = &compile_table_rt->compile_rt; - assert(table_rt->group2compile_rt.ref_compile_rt != NULL); - } - - return table_rt_mgr; -} - -void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr) -{ - if (NULL == table_rt_mgr) { - return; - } - - for(size_t i = 0; i < table_rt_mgr->n_table_rt; i++) { - table_runtime_free(table_rt_mgr->table_rt[i]); - table_rt_mgr->table_rt[i] = NULL; - } - - FREE(table_rt_mgr->table_rt); - FREE(table_rt_mgr); -} - -struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id) -{ - if (NULL == table_rt_mgr || (table_id < 0) || (table_id >= MAX_TABLE_NUM)) { - return NULL; - } - - assert(table_id < (int)table_rt_mgr->n_table_rt); - return table_rt_mgr->table_rt[table_id]; -} - -size_t table_runtime_rule_count(struct table_runtime *table_rt) -{ - if (NULL == table_rt) { - return 0; - } - - return table_rt->rule_num; -} - -enum table_type table_runtime_get_type(struct table_runtime* table_rt) -{ - if (NULL == table_rt) { - return TABLE_TYPE_INVALID; - } - - return table_rt->table_type; -} - -int table_runtime_scan_string(struct table_runtime* table_rt, int thread_id, const char *data, size_t data_len, - int result[], size_t *n_result) -{ - if (NULL == table_rt) { - return -1; - } - - return adapter_hs_scan(table_rt->expr_rt.hs, thread_id, data, data_len, result, n_result); -} - -void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id) -{ - if (NULL == table_rt) { - return; - } - - struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(table_rt->expr_rt.hs, thread_id); - table_rt->expr_rt.hs_stream = hs_stream; -} - -int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, - int result[], size_t *n_result) -{ - if (NULL == table_rt) { - return -1; - } - - return adapter_hs_scan_stream(table_rt->expr_rt.hs_stream, data, data_len, result, n_result); -} - -void table_runtime_stream_close(struct table_runtime *table_rt) -{ - if (table_rt != NULL) { - adapter_hs_stream_close(table_rt->expr_rt.hs_stream); - table_rt->expr_rt.hs_stream = NULL; - } -} - -enum IP_TYPE ip_type_transform(enum ip_type type) -{ - if (type == IP_TYPE_V4) { - return IPv4; - } else { - return IPv6; - } -} - -int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct ip_addr *scan_data, - struct scan_result *results, size_t *n_result, size_t n_result_array) -{ - if (NULL == table_rt) { - return -1; - } - - int i = 0; - int n_hit_result = 0; - struct scan_result scan_results[n_result_array] = {0}; - - struct ip_data ip; - ip.type = ip_type_transform(scan_data->ip_type); - if (ip.type == IPv4) { - ip.ipv4 = scan_data->ipv4; - } else { - memcpy(ip.ipv6, scan_data->ipv6, sizeof(scan_data->ipv6)); - } - - n_hit_result = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, n_result_array); - if (n_hit_result > (int)n_result_array) { - n_hit_result = n_result_array; - } - - int index = 0; - for (i = 0; i < n_hit_result; i++) { - results[index++].rule_id = scan_results[i].rule_id; - } - *n_result = index; - - return 0; -} - -void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule) -{ - struct port_range *sport_range = ALLOC(struct port_range, 1); - if (4 == item->addr_type) { - rule->type = IPv4; - sport_range->min_port = item->ipv4.min_sport; - sport_range->max_port = item->ipv4.max_sport; - } else { - rule->type = IPv6; - sport_range->min_port = item->ipv6.min_sport; - sport_range->max_port = item->ipv6.max_sport; - } - rule->rule_id = item->item_id; - rule->user_tag = sport_range; - rule->ipv4_rule.start_ip = item->ipv4.min_sip; - rule->ipv4_rule.end_ip = item->ipv4.max_sip; - memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, sizeof(item->ipv6.min_sip)); - memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, sizeof(item->ipv6.max_sip)); -} - -void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule) -{ - -} - -enum pattern_type expr_type2pattern_type(enum expr_type expr_type) -{ - enum pattern_type pattern_type = PATTERN_TYPE_STR; - - switch (expr_type) { - case EXPR_TYPE_STRING: - case EXPR_TYPE_AND: - break; - case EXPR_TYPE_REGEX: - pattern_type = PATTERN_TYPE_REG; - break; - default: - break; - } - - return pattern_type; -} - -and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger) -{ - size_t i = 0; - size_t sub_expr_cnt = 0; - char *pos = NULL; - char *saveptr = NULL; - char *sub_key_array[MAAT_MAX_EXPR_ITEM_NUM]; - and_expr_t *expr_rule = ALLOC(and_expr_t, 1); - - switch (expr_item->expr_type) { - case EXPR_TYPE_AND: - case EXPR_TYPE_REGEX: - for (i = 0, pos = expr_item->keywords; ; i++, pos = NULL) { - char *tmp = strtok_r_esc(pos, '&', &saveptr); - if (NULL == tmp) { - break; - } - - if (i >= MAAT_MAX_EXPR_ITEM_NUM) { - log_error(logger, MODULE_TABLE_RUNTIME, "expr item_id:%d too many patterns", - expr_item->item_id); - return NULL; - } - - sub_key_array[i] = tmp; - if (expr_item->expr_type == EXPR_TYPE_REGEX) { - sub_key_array[i] = str_unescape_and(sub_key_array[i]); - } else { - sub_key_array[i] = str_unescape(sub_key_array[i]); - } - } - sub_expr_cnt = i; - break; - case EXPR_TYPE_STRING: - sub_expr_cnt = 1; - sub_key_array[0] = expr_item->keywords; - sub_key_array[0] = str_unescape(sub_key_array[0]); - break; - default: - break; - } - - for (i = 0; i < sub_expr_cnt; i++) { - expr_rule->expr_id = expr_item->item_id; - expr_rule->patterns[i].pat = ALLOC(char, strlen(sub_key_array[i])); - memcpy(expr_rule->patterns[i].pat, sub_key_array[i], strlen(sub_key_array[i])); - expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]); - expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); - } - expr_rule->n_patterns = sub_expr_cnt; - - return expr_rule; -} - -struct maat_item_inner* maat_item_inner_new(int group_id, int item_id, int district_id) -{ - struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1); - item->magic_num = ITEM_RULE_MAGIC; - item->item_id = item_id; - item->group_id = group_id; - item->district_id = district_id; - - return item; -} - -void maat_item_inner_free(struct maat_item_inner *item) -{ - assert(item->magic_num == ITEM_RULE_MAGIC); - assert(item->expr_id_cnt == 0 || item->expr_id_cnt == item->expr_id_ub - item->expr_id_lb + 1); - item->magic_num = 0; - - free(item); -} - -int compile_runtime_update_row(struct compile_runtime *compile_rt, const char *table_name, int compile_id, - struct maat_compile_rule *compile_rule, struct maat_garbage_bin *garbage_bin, - int is_valid, struct log_handle *logger) -{ - int ret = -1; - struct maat_hierarchy *maat_hier = compile_rt->hier; - - if (0 == is_valid) { - // delete - struct maat_compile_rule *p_compile = (struct maat_compile_rule *)maat_hierarchy_compile_dettach_user_data(compile_rt->hier, compile_id); - if (p_compile != NULL) { - ret = maat_hierarchy_compile_remove(compile_rt->hier, compile_id); - assert(ret == 0); - maat_garbage_bagging(garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule); - } - } else { - // add - ret = maat_hierarchy_compile_add(compile_rt->hier, compile_id, compile_rule->declared_clause_num, compile_rule); - if (ret != 0) { - log_info(logger, MODULE_TABLE_RUNTIME, "duplicate config of compile table %s compile_id %d", - table_name, compile_id); - destroy_compile_rule(compile_rule); - } - } - - return ret; -} - -int group2compile_runtime_update_row(struct group2compile_runtime *g2c_rt, - int compile_id, struct group2compile_item *g2c_item, - int is_valid, struct log_handle *logger) -{ - int ret = -1; - struct compile_runtime *compile_rt = g2c_rt->ref_compile_rt; - struct group2group_runtime *g2g_rt = g2c_rt->ref_g2g_rt; - struct maat_group *group = NULL; - - if (0 == is_valid) { - /*TODO: by luis*/ - group = maat_group_topology_find_group(g2g_rt->group_topo, g2c_item->group_id); - if (NULL == group) { - log_error(logger, MODULE_TABLE_RUNTIME, - "Remove group %d from compile %d failed, group is not exisited.", - g2c_item->group_id, compile_id); - return ret; - } - - ret = maat_hierarchy_remove_group_from_compile(compile_rt->hier, g2c_item->group_id, g2c_item->virtual_table_id, - g2c_item->not_flag, g2c_item->clause_index, g2c_item->compile_id); - if (0 == ret) { - group->ref_by_compile_cnt--; - if (g2c_item->not_flag) { - g2c_rt->not_flag_group--; - } - } - } else { - group = maat_group_topology_find_group(g2g_rt->group_topo, g2c_item->group_id); - if (NULL == group) { - group = maat_group_topology_add_group(g2g_rt->group_topo, g2c_item->group_id); - } - - ret = maat_hierarchy_add_group_to_compile(compile_rt->hier, g2c_item->group_id, g2c_item->virtual_table_id, - g2c_item->not_flag, g2c_item->clause_index, g2c_item->compile_id); - if (0 == ret) { - group->ref_by_compile_cnt++; - if (g2c_item->not_flag) { - g2c_rt->not_flag_group++; - } - } - } - - return ret; -} - -int group2group_runtime_update_row(struct group2group_runtime *g2g_rt, int group_id, - struct group2group_item *g2g_item, - int is_valid, struct log_handle *logger) -{ - int ret = -1; - - if (0 == is_valid) { - //delete - ret = maat_group_topology_add_group_to_group(g2g_rt->group_topo, group_id, g2g_item->superior_group_id); - } else { - //add - ret = maat_group_topology_remove_group_from_group(g2g_rt->group_topo, group_id, g2g_item->superior_group_id); - } - - return ret; -} - -void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len, - and_expr_t *expr_rule, int is_valid, struct log_handle *logger) -{ - void *data = NULL; - - if (0 == is_valid) { - //delete - data = rcu_hash_find(expr_rt->htable, key, key_len); - if (NULL == data) { - log_error(logger, MODULE_TABLE_RUNTIME, - "the key of expr rule not exist, can't be deleted, expr_id:%d", expr_rule->expr_id); - return; - } - rcu_hash_del(expr_rt->htable, key, key_len); - } else { - //add - data = rcu_hash_find(expr_rt->htable, key, key_len); - if (data != NULL) { - log_error(logger, MODULE_TABLE_RUNTIME, - "the key of expr rule already exist, can't be added, expr_id:%d", expr_rule->expr_id); - return; - } - rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule); - } -} - -void ip_plus_runtime_update_row(struct ip_plugin_runtime *ip_plus_rt, char *key, size_t key_len, - struct ip_plus_item *ip_plus_item, int is_valid, - struct log_handle *logger) -{ - struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt; - - if (0 == is_valid) { - // delete - ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger); - } else { - // add - struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); - ex_container->ex_data = NULL; - ex_container->custom_data = ip_plus_item; - ex_data_runtime_add_ex_container(ip_plus_rt->ex_data_rt, key, key_len, ex_container); - } -} - -void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_schema *table_schema, - const char *row, char *key, size_t key_len, int is_valid, - struct log_handle *logger) -{ - struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); - ex_container->ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); - int set_flag = table_schema_ex_data_schema_flag(table_schema); - size_t cb_count = table_schema_callback_count(table_schema); - - /* already set plugin_table_schema's ex_data_schema */ - if (1 == set_flag) { - if (is_valid == 0) { - // delete - ex_data_runtime_del_ex_container(plugin_rt->ex_data_rt, key, key_len, logger); - } else { - // add - ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container); - } - } - - /* plugin table schema has callback */ - if (cb_count > 0) { - table_schema_all_cb_update(table_schema, row); - } - - if ((0 == set_flag) && (0 == cb_count)) { - ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row); - } - - plugin_rt->acc_line_num++; -} - -void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct table_schema *table_schema, - const char *row, char *key, size_t key_len, struct ip_plugin_item *ip_plugin_item, - int is_valid, struct log_handle *logger) -{ - struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt; - int set_flag = table_schema_ex_data_schema_flag(table_schema); - - if (1 == set_flag) { - if (0 == is_valid) { - //delete - ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger); - } else { - //add - struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); - ex_container->ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); - ex_container->custom_data = ip_plugin_item; - ex_data_runtime_add_ex_container(ip_plugin_rt->ex_data_rt, key, key_len, ex_container); - } - } else { - ex_data_runtime_cache_row_put(ip_plugin_rt->ex_data_rt, row); - } -} - -struct maat_compile_rule *compile_item_to_compile_rule(struct compile_item *compile_item, struct table_schema *table_schema, struct log_handle *logger) -{ - struct maat_rule_head rule_head; - rule_head.config_id = compile_item->compile_id; - rule_head.service_id = compile_item->service_id; - rule_head.action = compile_item->action; - rule_head.do_blacklist = compile_item->do_blacklist; - rule_head.do_log = compile_item->do_log; - - struct maat_compile_rule *p = ALLOC(struct maat_compile_rule, 1); - - p->magic_num = COMPILE_RULE_MAGIC; - p->head = rule_head; - p->declared_clause_num = compile_item->clause_num; - p->ex_data = ALLOC(void *, MAX_COMPILE_EX_DATA_NUM); - - p->ref_table = table_schema; - p->head.serv_def_len = strlen(compile_item->user_region); - p->service_defined = ALLOC(char, p->head.serv_def_len); - memcpy(p->service_defined, compile_item->user_region, p->head.serv_def_len); - p->evaluation_order = compile_item->evaluation_order; - - size_t n_rule_ex_schema = table_schema_compile_rule_ex_data_schema_count(table_schema); - for (size_t i = 0; i < n_rule_ex_schema; i++) { - struct compile_ex_data_schema *ex_schema = table_schema_get_compile_rule_ex_data_schema(table_schema, i); - p->ex_data[i] = rule_ex_data_new(&p->head, p->service_defined, ex_schema); - } - p->is_valid = 1; - p->compile_id = compile_item->compile_id; - pthread_rwlock_init(&p->rwlock, NULL); - - return p; -} - -int compile_runtime_update(struct compile_runtime *compile_rt, struct compile_item *compile_item, - struct table_schema *table_schema, const char *table_name, - struct maat_garbage_bin *garbage_bin, int is_valid, struct log_handle *logger) -{ - int ret = -1; - - if (NULL == compile_rt || NULL == compile_item) { - return ret; - } - - struct maat_compile_rule *compile_rule = NULL; - if (1 == is_valid) { - compile_rule = compile_item_to_compile_rule(compile_item, table_schema, logger); - if (NULL == compile_rule) { - log_error(logger, MODULE_TABLE_RUNTIME, - "transform compile table:%s item to compile_rule failed, compile_id:%d", - table_name, compile_item->compile_id); - return ret; - } - } - - ret = compile_runtime_update_row(compile_rt, table_name, compile_item->compile_id, compile_rule, garbage_bin, is_valid, logger); - return ret; -} - -int group2compile_runtime_update(struct group2compile_runtime *g2c_rt, struct group2compile_item *g2c_item, - const char *table_name, int is_valid, struct log_handle *logger) -{ - int ret = -1; - - if (NULL == g2c_rt || NULL == g2c_item) { - return ret; - } - - ret = group2compile_runtime_update_row(g2c_rt, g2c_item->compile_id, g2c_item, is_valid, logger); - return ret; -} - -int group2group_runtime_update(struct group2group_runtime *g2g_rt, struct group2group_item *g2g_item, - const char *table_name, int is_valid, struct log_handle *logger) -{ - int ret = -1; - - if (NULL == g2g_rt || NULL == g2g_item) { - return ret; - } - - ret = group2group_runtime_update_row(g2g_rt, g2g_item->group_id, g2g_item, is_valid, logger); - return ret; -} - -int expr_runtime_update(struct expr_runtime *expr_rt, struct expr_item *expr_item, - const char *table_name, int is_valid, struct log_handle *logger) -{ - if (NULL == expr_rt || NULL == expr_item) { - return -1; - } - - /* add item to group */ - struct maat_item_inner *u_para = NULL; - u_para = maat_item_inner_new(expr_item->group_id, expr_item->item_id, -1); - int ret = maat_hierarchy_add_item_to_group(scanner->hier, expr_item->group_id, expr_item->item_id, u_para); - if (ret != 0) { - maat_item_inner_free(u_para); - u_para = NULL; - return -1; - } - - and_expr_t *expr_rule = NULL; - if (1 == is_valid) { - expr_rule = expr_item_to_expr_rule(expr_item, logger); - if (NULL == expr_rule) { - log_error(logger, MODULE_TABLE_RUNTIME, "transform expr table:%s item to expr_rule failed, item_id:%d", - table_name, expr_item->item_id); - return -1; - } - } - - char *key = (char *)&(expr_item->item_id); - expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, is_valid, logger); - - return 0; -} - -int ip_plus_runtime_update(struct ip_plugin_runtime *ip_plus_rt, struct ip_plus_item *ip_plus_item, - const char *table_name, int is_valid, struct log_handle *logger) -{ - if (NULL == ip_plus_rt || NULL == ip_plus_item) { - return -1; - } - - struct ip_plus_item *item = NULL; - if (1 == is_valid) { - item = ALLOC(struct ip_plus_item, 1); - memcpy(item, ip_plus_item, sizeof(struct ip_plus_item)); - } - - char *key = (char *)&(ip_plus_item->item_id); - ip_plus_runtime_update_row(ip_plus_rt, key, sizeof(int), ip_plus_item, is_valid, logger); - - return 0; -} - -int plugin_runtime_update(struct plugin_runtime *plugin_rt, struct plugin_item *plugin_item, - struct table_schema *table_schema, const char *row, int is_valid, - struct log_handle *logger) -{ - if (NULL == plugin_rt || NULL == plugin_item) { - return -1; - } - - char *key = plugin_item->key; - size_t key_len = plugin_item->key_len; - - plugin_runtime_update_row(plugin_rt, table_schema, row, key, key_len, is_valid, logger); - - return 0; -} - -int ip_plugin_runtime_update(struct ip_plugin_runtime *ip_plugin_rt, - struct ip_plugin_item *ip_plugin_item, - struct table_schema *table_schema, const char *row, - int is_valid, struct log_handle *logger) -{ - if (NULL == ip_plugin_rt || NULL == ip_plugin_item) { - return -1; - } - - struct ip_plugin_item *item = NULL; - if (1 == is_valid) { - item = ALLOC(struct ip_plugin_item, 1); - memcpy(item, ip_plugin_item, sizeof(struct ip_plugin_item)); - } - - char *key = (char *)&(ip_plugin_item->item_id); - ip_plugin_runtime_update_row(ip_plugin_rt, table_schema, row, key, sizeof(int), - item, is_valid, logger); - - return 0; -} - -void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, - const char *row, struct table_item *table_item, struct log_handle *logger) -{ - int ret = -1; - int is_valid = -1; - const char *table_name = table_schema_get_updating_name(table_schema); - int table_id = table_schema_get_table_id(table_schema); - - switch (table_rt->table_type) { - case TABLE_TYPE_COMPILE: - is_valid = table_item->compile_item.is_valid; - ret = compile_runtime_update(&(table_rt->compile_rt), &(table_item->compile_item), table_schema, table_name, table_rt->ref_garbage_bin, is_valid, logger); - break; - case TABLE_TYPE_GROUP2COMPILE: - is_valid = table_item->g2c_item.is_valid; - ret = group2compile_runtime_update(&(table_rt->g2c_rt), &(table_item->g2c_item), table_name, is_valid, logger); - break; - case TABLE_TYPE_GROUP2GROUP: - is_valid = table_item->g2g_item.is_valid; - ret = group2group_runtime_update(&(table_rt->g2g_rt), &(table_item->g2g_item), table_name, is_valid, logger); - break; - case TABLE_TYPE_EXPR: - is_valid = table_item->expr_item.is_valid; - ret = expr_runtime_update(&(table_rt->expr_rt), &(table_item->expr_item), table_name, is_valid, logger); - break; - case TABLE_TYPE_IP_PLUS: - is_valid = table_item->ip_plus_item.is_valid; - ret = ip_plus_runtime_update(&(table_rt->ip_plus_rt), &(table_item->ip_plus_item), table_name, is_valid, logger); - break; - case TABLE_TYPE_PLUGIN: - is_valid = table_item->plugin_item.is_valid; - ret = plugin_runtime_update(&(table_rt->plugin_rt), &(table_item->plugin_item), table_schema, row, is_valid, logger); - break; - case TABLE_TYPE_IP_PLUGIN: - is_valid = table_item->ip_plugin_item.is_valid; - ret = ip_plugin_runtime_update(&(table_rt->ip_plugin_rt), &(table_item->ip_plugin_item), table_schema, row, is_valid, logger); - break; - default: - break; - } - - if (ret < 0) { - return; - } - - if (is_valid == 0) { - table_rt->rule_num--; - } else { - table_rt->rule_num++; - } -} - -int compile_runtime_commit(struct table_runtime *table_rt, long long version, struct log_handle *logger) -{ - int ret = -1; - - ret = maat_hierarchy_rebuild(table_rt->compile_rt.hier); - if (ret != 0) { - log_error(logger, MODULE_TABLE_RUNTIME, "version %lld hierarchy rebuild failed.", - version); - return -1; - } else { - log_info(logger, MODULE_TABLE_RUNTIME, "version %lld hierarchy rebuild success.", - version); - return 0; - } -} - -int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread, - struct log_handle *logger) -{ - struct expr_runtime *expr_rt = &(table_rt->expr_rt); - void **ex_data_array = NULL; - and_expr_t *rules = NULL; - size_t rule_cnt = 0; - int ret = 0; - - rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array); - rules = ALLOC(and_expr_t, rule_cnt); - - for (size_t i = 0; i < rule_cnt; i++) { - rules[i] = *(and_expr_t *)ex_data_array[i]; - } - - struct adapter_hs *new_adapter_hs = NULL; - struct adapter_hs *old_adapter_hs = NULL; - if (rule_cnt > 0) { - log_info(logger, MODULE_TABLE_RUNTIME, - "committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt); - new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt); - if (NULL == new_adapter_hs) { - log_error(logger, MODULE_TABLE_RUNTIME, - "rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt); - ret = -1; - } - } - - old_adapter_hs = expr_rt->hs; - expr_rt->hs = new_adapter_hs; - - maat_garbage_bagging(table_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy); - rcu_hash_commit(expr_rt->htable); - table_rt->rule_num = rcu_hash_count(expr_rt->htable); - rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array); - assert(rule_cnt == 0); - - FREE(rules); - FREE(ex_data_array); - - return ret; -} - -int ip_plus_runtime_commit(struct table_runtime *table_rt, struct log_handle *logger) -{ - struct ex_data_container **ex_container = NULL; - struct ip_rule *rules = NULL; - size_t rule_cnt = 0; - int ret = 0; - struct ip_plugin_runtime *ip_plus_rt = &(table_rt->ip_plus_rt); - - rule_cnt = ex_data_runtime_list_updating_ex_container(ip_plus_rt->ex_data_rt, &ex_container); - rules = ALLOC(struct ip_rule, rule_cnt); - - for (size_t i = 0; i < rule_cnt; i++) { - struct ip_plus_item *item = (struct ip_plus_item *)ex_container[i]->custom_data; - ip_plus_item_to_ip_rule(item, &rules[i]); - } - - struct ip_matcher *new_ip_matcher = NULL; - struct ip_matcher *old_ip_matcher = NULL; - size_t mem_used = 0; - - if (rule_cnt > 0) { - log_info(logger, MODULE_TABLE_RUNTIME, - "committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt); - new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); - if (NULL == new_ip_matcher) { - log_error(logger, MODULE_TABLE_RUNTIME, - "rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt); - ret = -1; - } - } - - old_ip_matcher = ip_plus_rt->ip_matcher; - ip_plus_rt->ip_matcher = new_ip_matcher; - maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); - ex_data_runtime_commit(ip_plus_rt->ex_data_rt); - table_rt->rule_num = ex_data_runtime_ex_container_count(ip_plus_rt->ex_data_rt); - - FREE(rules); - FREE(ex_container); - - return ret; -} - -int plugin_runtime_commit(struct table_runtime *table_rt) -{ - ex_data_runtime_commit(table_rt->plugin_rt.ex_data_rt); - table_rt->rule_num = ex_data_runtime_ex_container_count(table_rt->plugin_rt.ex_data_rt); - - return 0; -} - -int ip_plugin_runtime_commit(struct table_runtime *table_rt, struct log_handle *logger) -{ - struct ex_data_container **ex_container = NULL; - struct ip_rule *rules = NULL; - size_t rule_cnt = 0; - int ret = 0; - struct ip_plugin_runtime *ip_plugin_rt = &(table_rt->ip_plugin_rt); - - rule_cnt = ex_data_runtime_list_updating_ex_container(ip_plugin_rt->ex_data_rt, &ex_container); - rules = ALLOC(struct ip_rule, rule_cnt); - - for (size_t i = 0; i < rule_cnt; i++) { - struct ip_plugin_item *item = (struct ip_plugin_item *)ex_container[i]->custom_data; - ip_plugin_item_to_ip_rule(item, &rules[i]); - } - - struct ip_matcher *new_ip_matcher = NULL; - struct ip_matcher *old_ip_matcher = NULL; - size_t mem_used = 0; - - if (rule_cnt > 0) { - log_info(logger, MODULE_TABLE_RUNTIME, - "committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt); - new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); - if (NULL == new_ip_matcher) { - log_error(logger, MODULE_TABLE_RUNTIME, - "rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt); - ret = -1; - } - } - - old_ip_matcher = ip_plugin_rt->ip_matcher; - ip_plugin_rt->ip_matcher = new_ip_matcher; - maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); - ex_data_runtime_commit(ip_plugin_rt->ex_data_rt); - table_rt->rule_num = ex_data_runtime_ex_container_count(ip_plugin_rt->ex_data_rt); - - FREE(rules); - FREE(ex_container); - - return ret; -} - -int expr_runtime_updating_flag(struct expr_runtime *expr_rt) -{ - return rcu_hash_updating_flag(expr_rt->htable); -} - -int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt) -{ - return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt); -} - -int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt) -{ - return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt); -} - -int table_runtime_updating_flag(struct table_runtime *table_rt) -{ - int updating_flag = 0; - - switch (table_rt->table_type) { - case TABLE_TYPE_EXPR: - updating_flag = expr_runtime_updating_flag(&(table_rt->expr_rt)); - break; - case TABLE_TYPE_PLUGIN: - updating_flag = plugin_runtime_updating_flag(&(table_rt->plugin_rt)); - break; - case TABLE_TYPE_IP_PLUGIN: - updating_flag = ip_plugin_runtime_updating_flag(&(table_rt->ip_plugin_rt)); - break; - default: - break; - } - - return updating_flag; -} - -void table_runtime_commit(struct table_runtime *table_rt, long long version, size_t nr_worker_thread, - struct log_handle *logger) -{ - switch (table_rt->table_type) { - case TABLE_TYPE_COMPILE: - case TABLE_TYPE_GROUP2COMPILE: - case TABLE_TYPE_GROUP2GROUP: - compile_runtime_commit(table_rt, version, logger); - break; - case TABLE_TYPE_EXPR: - expr_runtime_commit(table_rt, nr_worker_thread, logger); - break; - case TABLE_TYPE_IP_PLUS: - ip_plus_runtime_commit(table_rt, logger); - break; - case TABLE_TYPE_PLUGIN: - plugin_runtime_commit(table_rt); - break; - case TABLE_TYPE_IP_PLUGIN: - ip_plugin_runtime_commit(table_rt, logger); - break; - default: - break; - } -} - -size_t table_runtime_cached_row_count(struct table_runtime* table_rt) -{ - size_t row_count = 0; - struct ex_data_runtime *ex_data_rt = NULL; - - switch (table_rt->table_type) { - case TABLE_TYPE_IP_PLUGIN: - ex_data_rt = table_rt->ip_plugin_rt.ex_data_rt; - break; - default: - break; - } - row_count = ex_data_runtime_cached_row_count(ex_data_rt); - - return row_count; -} - -const char* table_runtime_get_cached_row(struct table_runtime* table_rt, size_t row_seq) -{ - const char *line = NULL; - struct ex_data_runtime *ex_data_rt = NULL; - - switch (table_rt->table_type) { - case TABLE_TYPE_IP_PLUGIN: - ex_data_rt = table_rt->ip_plugin_rt.ex_data_rt; - break; - default: - break; - } - line = ex_data_runtime_cached_row_get(ex_data_rt, row_seq); - - return line; -} - -struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt) -{ - struct ex_data_runtime *ex_data_rt = NULL; - - switch (table_rt->table_type) { - case TABLE_TYPE_IP_PLUS: - ex_data_rt = table_rt->ip_plus_rt.ex_data_rt; - break; - case TABLE_TYPE_PLUGIN: - ex_data_rt = table_rt->plugin_rt.ex_data_rt; - break; - case TABLE_TYPE_IP_PLUGIN: - ex_data_rt = table_rt->ip_plugin_rt.ex_data_rt; - break; - default: - break; - } - - return ex_data_rt; -} - -void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, - int nr_worker_thread, long long version, struct log_handle *logger) -{ - struct ex_data_schema *ex_data_schema = table_schema_get_ex_data_schema(table_schema); - struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt); - ex_data_runtime_set_schema(ex_data_rt, ex_data_schema); - struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1); - - ctx->table_id = table_schema_get_table_id(table_schema); - ctx->ex_schema = ex_data_schema; - ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx); - - size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt); - for (size_t i = 0; i < n_cached_row; i++) { - const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i); - switch (table_rt->table_type) { - case TABLE_TYPE_PLUGIN: - plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, NULL, 0, 1, logger); - break; - case TABLE_TYPE_IP_PLUGIN: - ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, NULL, NULL, 0, NULL, 1, logger); - break; - default: - break; - } - } - ex_data_runtime_clear_row_cache(ex_data_rt); - - table_runtime_commit(table_rt, version, nr_worker_thread, logger); -} diff --git a/src/maat_table_schema.cpp b/src/maat_table_schema.cpp deleted file mode 100644 index 196e5d4..0000000 --- a/src/maat_table_schema.cpp +++ /dev/null @@ -1,2481 +0,0 @@ -/* -********************************************************************************************** -* File: maat_table_schema.cpp -* Description: -* Authors: Liu WenTan -* Date: 2022-10-31 -* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. -*********************************************************************************************** -*/ - -#include -#include -#include -#include -#include - -#include "utils.h" -#include "log/log.h" -#include "maat_utils.h" -#include "maat_table_schema.h" -#include "maat_kv.h" -#include "cJSON/cJSON.h" -#include "adapter_hs.h" -#include "maat_ex_data.h" - -#define MODULE_TABLE_SCHEMA module_name_str("maat.table_schema") - -#define MAX_FOREIGN_CLMN_NUM 8 -#define MAX_CONJUNCTION_TABLE_NUM 8 - -enum user_region_encode { - USER_REGION_ENCODE_NONE=0, - USER_REGION_ENCODE_ESCAPE, - USER_REGION_ENCODE_BASE64 -}; - -struct compile_table_schema { - int compile_id_column; - int service_id_column; - int action_column; - int do_blacklist_column; - int do_log_column; - int tags_column; - int user_region_column; - int is_valid_column; - int clause_num_column; - int evaluation_order_column; - enum user_region_encode user_region_encoding; - size_t n_ex_schema; - struct compile_ex_data_schema ex_schema[MAX_COMPILE_EX_DATA_NUM]; -}; - -struct group2compile_table_schema { - int group_id_column; - int compile_id_column; - int is_valid_column; - int not_flag_column; - int virtual_table_name_column; - int clause_index_column; - char associated_compile_table_id; -}; - -struct group2group_table_schema { - int group_id_column; - int superior_group_id_column; - int is_valid_column; -}; - -struct expr_table_schema { - int item_id_column; - int group_id_column; - int district_column; - int keywords_column; - int expr_type_column; - int match_method_column; - int is_hexbin_column; - int is_valid_column; /* valid means add, invalid means delete */ - enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ -}; - -struct ip_plus_table_schema { - int item_id_column; - int group_id_column; - int addr_type_column; - int saddr_format_column; - int sip1_column; - int sip2_column; - int sport_format_column; - int sport1_column; - int sport2_column; - int daddr_format_column; - int dip1_column; - int dip2_column; - int dport_format_column; - int dport1_column; - int dport2_column; - int proto_column; - int direction_column; - int is_valid_column; -}; - -#define MAX_PLUGIN_PER_TABLE 32 -struct plugin_table_schema { - int item_id_column; - int key_column; - int is_valid_column; - int rule_tag_column; - int n_foreign; - int foreign_columns[MAX_FOREIGN_CLMN_NUM]; - size_t cb_plug_cnt; - struct plugin_table_callback_schema cb_plug[MAX_PLUGIN_PER_TABLE]; - struct ex_data_schema ex_schema; -}; - -struct ip_plugin_table_schema { - int item_id_column; - int ip_type_column; - int start_ip_column; - int end_ip_column; - int is_valid_column; - int rule_tag_column; - int have_exdata; - struct ex_data_schema ex_schema; -}; - -struct virtual_table_schema { - int physical_table_id[SCAN_TYPE_MAX]; -}; - -struct composition_table_schema -{ - int component_table_id[COMPONENT_TABLE_TYPE_MAX]; -}; - -struct table_schema { - int table_id; - int conj_cnt; - int updating_name; - char table_name[MAX_CONJUNCTION_TABLE_NUM][NAME_MAX]; - enum table_type table_type; - union { - struct compile_table_schema compile; - struct group2compile_table_schema group2compile; - struct group2group_table_schema group2group; - struct expr_table_schema expr; - struct ip_plus_table_schema ip_plus; - struct plugin_table_schema plugin; - struct ip_plugin_table_schema ip_plugin; - struct virtual_table_schema virtual_table; - struct composition_table_schema composition; - }; - - uint64_t update_err_cnt; - uint64_t unmatched_tag_cnt; -}; - -struct table_schema_manager { - struct table_schema *schema_table[MAX_TABLE_NUM]; - size_t n_schema_table; - size_t n_active_plugin_table; - char default_compile_table[NAME_MAX]; - struct maat_kv_store *tablename2id_map; - struct log_handle *logger; -}; - -struct table_schema *table_schema_new(void) -{ - struct table_schema *ptable = ALLOC(struct table_schema, 1); - ptable->conj_cnt = 1; - - return ptable; -} - -void table_schema_free(struct table_schema *ptable) -{ - FREE(ptable); -} - -int read_compile_table_schema(cJSON *root, struct table_schema *ptable, - struct maat_kv_store* reserved_word_map, - struct log_handle *logger) -{ - int read_cnt = 0; - cJSON *json = NULL; - - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, - "compile table name %s length too long", json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "custom"); - if (json == NULL || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column", - ptable->table_name[0]); - return -1; - } - - cJSON *item = cJSON_GetObjectItem(json, "compile_id"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.compile_id_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "service_id"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.service_id_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "action"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.action_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "do_blacklist"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.do_blacklist_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "do_log"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.do_log_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "tags"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.tags_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "user_region"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.user_region_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "is_valid"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.is_valid_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "clause_num"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.clause_num_column = item->valueint; - read_cnt++; - } - item = cJSON_GetObjectItem(json, "evaluation_order"); - if (item != NULL && item->type == cJSON_Number) { - ptable->compile.evaluation_order_column = item->valueint; - read_cnt++; - } - - if (read_cnt < 12) { - return -1; - } - - return 0; -} - -int read_group2compile_table_schema(cJSON *root, struct table_schema *ptable, - struct maat_kv_store* reserved_word_map, - struct log_handle *logger) -{ - int read_cnt = 0; - cJSON *json = NULL; - - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, - "group2compile table name %s length too long", json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "associated_compile_table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->group2compile.associated_compile_table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "custom"); - if (json == NULL || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column", - ptable->table_name[0]); - return -1; - } - - cJSON *item = cJSON_GetObjectItem(json, "group_id"); - if (item != NULL && item->type == cJSON_Number) { - ptable->group2compile.group_id_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "compile_id"); - if (item != NULL && item->type == cJSON_Number) { - ptable->group2compile.compile_id_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "is_valid"); - if (item != NULL && item->type == cJSON_Number) { - ptable->group2compile.is_valid_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "not_flag"); - if (item != NULL && item->type == cJSON_Number) { - ptable->group2compile.not_flag_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "virtual_table_name"); - if (item != NULL && item->type == cJSON_Number) { - ptable->group2compile.virtual_table_name_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "clause_index"); - if (item != NULL && item->type == cJSON_Number) { - ptable->group2compile.clause_index_column = item->valueint; - read_cnt++; - } - - if (read_cnt < 8) { - return -1; - } - - return 0; -} - -int read_group2group_table_schema(cJSON *root, struct table_schema *ptable, - struct maat_kv_store* reserved_word_map, - struct log_handle *logger) -{ - int read_cnt = 0; - cJSON *json = NULL; - - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, - "group2group table name %s length too long", json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "custom"); - if (json == NULL || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column", - ptable->table_name[0]); - return -1; - } - - cJSON *item = cJSON_GetObjectItem(json, "group_id"); - if (item != NULL && item->type == cJSON_Number) { - ptable->group2group.group_id_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "superior_group_id"); - if (item != NULL && item->type == cJSON_Number) { - ptable->group2group.superior_group_id_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "is_valid"); - if (item != NULL && item->type == cJSON_Number) { - ptable->group2group.is_valid_column = item->valueint; - read_cnt++; - } - - if (read_cnt < 5) { - return -1; - } - - return 0; -} - -int read_expr_table_schema(cJSON *root, struct table_schema *ptable, - struct maat_kv_store* reserved_word_map, - struct log_handle *logger) -{ - int read_cnt = 0; - cJSON *json = NULL; - - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, "expr table name %s length too long", - json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "scan_mode"); - if (json != NULL && json->type == cJSON_String) { - int ret = maat_kv_read(reserved_word_map, json->valuestring, (int*)&(ptable->expr.scan_mode)); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, "scan_mode %s illegal", json->valuestring); - return -1; - } - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "item_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->expr.item_id_column = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "group_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->expr.group_id_column = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "custom"); - if (json == NULL || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column", - ptable->table_name[0]); - return -1; - } - - cJSON *item = cJSON_GetObjectItem(json, "keywords"); - if (item != NULL && item->type == cJSON_Number) { - ptable->expr.keywords_column = item->valueint; - read_cnt++; - } - - if (ptable->table_type == TABLE_TYPE_EXPR_PLUS) { - item = cJSON_GetObjectItem(json, "district"); - if (item != NULL && item->type == cJSON_Number) { - ptable->expr.district_column = item->valueint; - read_cnt++; - } - } - - item = cJSON_GetObjectItem(json, "expr_type"); - if (item != NULL && item->type == cJSON_Number) { - ptable->expr.expr_type_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "match_method"); - if (item != NULL && item->type == cJSON_Number) { - ptable->expr.match_method_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "is_hexbin"); - if (item != NULL && item->type == cJSON_Number) { - ptable->expr.is_hexbin_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "is_valid"); - if (item != NULL && item->type == cJSON_Number) { - ptable->expr.is_valid_column = item->valueint; - read_cnt++; - } - - if (read_cnt < 10) { - return -1; - } - - return 0; -} - -int read_ip_plus_table_schema(cJSON *root, struct table_schema *ptable, - struct log_handle *logger) -{ - size_t read_cnt = 0; - cJSON *json = NULL; - - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, "ip_plus table name %s length too long", - json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "item_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->ip_plus.item_id_column = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "group_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->ip_plus.group_id_column = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "custom"); - if (NULL == json || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE_SCHEMA, "ip_plus table %s has no custom column", - ptable->table_name[0]); - return -1; - } - - cJSON *item = cJSON_GetObjectItem(json, "addr_type"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.addr_type_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "saddr_format"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.saddr_format_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "sip1"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.sip1_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "sip2"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.sip2_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "sport_format"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.sport_format_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "sport1"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.sport1_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "sport2"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.sport2_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "daddr_format"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.daddr_format_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "dip1"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.dip1_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "dip2"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.dip2_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "dport_format"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.dport_format_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "dport1"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.dport1_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "dport2"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.dport2_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "proto"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.proto_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "direction"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.direction_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "is_valid"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plus.is_valid_column = item->valueint; - read_cnt++; - } - - if (read_cnt < 20) { - return -1; - } - - return 0; -} - -int read_virtual_table_schema(cJSON *root, struct table_schema **table_array, - struct table_schema *ptable, - struct maat_kv_store *tablename2id_map, - struct log_handle *logger) -{ - size_t read_cnt = 0; - cJSON *json = NULL; - - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, "virtual table name %s length too long", - json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "physical_table"); - if (NULL == json || json->type != cJSON_Array) { - log_error(logger, MODULE_TABLE_SCHEMA, "virtual table %s has no physical_table column", - ptable->table_name[0]); - return -1; - } - read_cnt++; - - int cnt = cJSON_GetArraySize(json); - for (int i = 0; i < cnt; i++) { - cJSON *item = cJSON_GetArrayItem(json, i); - if (item != NULL && item->type == cJSON_String) { - int table_id = -1; - /* physical table should already exist */ - int ret = maat_kv_read(tablename2id_map, item->valuestring, &table_id); - if (ret < 0) { - return -1; - } - - enum scan_type table_scan_type = table_schema_get_scan_type(table_array[table_id]); - ptable->virtual_table.physical_table_id[table_scan_type]= table_id; - } - } - - if (read_cnt < 3) { - return -1; - } - - return 0; -} - -int read_composition_table_schema(cJSON *root, struct table_schema *ptable, - struct maat_kv_store *tablename2id_map, - struct log_handle *logger) -{ - size_t read_cnt = 0; - cJSON *json = NULL; - - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, "composition table name %s length too long", - json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "composition_table"); - if (NULL == json || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no composition_table column", - ptable->table_name[0]); - return -1; - } - read_cnt++; - - int ret = 0; - struct composition_table_schema *composition_schema = &(ptable->composition); - cJSON *item = cJSON_GetObjectItem(json, "source"); - if (item != NULL && item->type == cJSON_String) { - /* physical table should already exist */ - ret = maat_kv_read(tablename2id_map, item->valuestring, - &composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SIP]); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "Child table %s of table %s (id=%d) are not defined", - item->valuestring, ptable->table_name[0], ptable->table_id); - return -1; - } - } - - item = cJSON_GetObjectItem(json, "dest"); - if (item != NULL && item->type == cJSON_String) { - /* physical table should already exist */ - ret = maat_kv_read(tablename2id_map, item->valuestring, - &composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DIP]); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "Child table %s of table %s (id=%d) are not defined", - item->valuestring, ptable->table_name[0], ptable->table_id); - return -1; - } - } - - item = cJSON_GetObjectItem(json, "session"); - if (item != NULL && item->type == cJSON_String) { - /* physical table should already exist */ - ret = maat_kv_read(tablename2id_map, item->valuestring, - &composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION]); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "Child table %s of table %s (id=%d) are not defined", - item->valuestring, ptable->table_name[0], ptable->table_id); - return -1; - } - } - - if (read_cnt < 3) { - return -1; - } - - return 0; -} - -static int read_integer_array(char *string, int *array, int size) -{ - int i = 0; - char *token = NULL, *sub_token = NULL, *saveptr; - - for (i = 0, token = string; i < size; token= NULL, i++) { - sub_token = strtok_r(token, ",", &saveptr); - if (sub_token == NULL) { - break; - } - - sscanf(sub_token, "%d", array + i); - } - - return i; -} - -int read_plugin_table_schema(cJSON *root, struct table_schema *ptable, struct log_handle *logger) -{ - size_t read_cnt = 0; - cJSON *json = NULL; - - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, "plugin table name %s length too long", - json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "item_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->plugin.item_id_column = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "custom"); - if (json == NULL || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column", - ptable->table_name[0]); - return -1; - } - - cJSON *item = cJSON_GetObjectItem(json, "key"); - if (item != NULL && item->type == cJSON_Number) { - ptable->plugin.key_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "is_valid"); - if (item != NULL && item->type == cJSON_Number) { - ptable->plugin.is_valid_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "tag"); - if (item != NULL && item->type == cJSON_Number) { - ptable->plugin.rule_tag_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "foreign"); - if (item != NULL) { - read_cnt++; - if (item->type == cJSON_String) { - ptable->plugin.n_foreign = read_integer_array(item->valuestring, - ptable->plugin.foreign_columns, - MAX_FOREIGN_CLMN_NUM); - } else if (item->type == cJSON_Array) { - ptable->plugin.n_foreign = cJSON_GetArraySize(item); - for (int i = 0; i < ptable->plugin.n_foreign; i++) { - cJSON *foreign_item = cJSON_GetArrayItem(item, i); - assert(foreign_item->type == cJSON_Number); - ptable->plugin.foreign_columns[i] = foreign_item->valueint; - } - } - } - - if (read_cnt < 7) { - return -1; - } - - return 0; -} - -int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable, struct log_handle *logger) -{ - size_t read_cnt = 0; - cJSON *json = NULL; - - json = cJSON_GetObjectItem(root, "table_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->table_id = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "table_name"); - if (json != NULL && json->type == cJSON_String) { - if (strlen(json->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, "ip_plugin table name %s length too long", - json->valuestring); - return -1; - } - memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "item_id"); - if (json != NULL && json->type == cJSON_Number) { - ptable->ip_plugin.item_id_column = json->valueint; - read_cnt++; - } - - json = cJSON_GetObjectItem(root, "custom"); - if (NULL == json || json->type != cJSON_Object) { - log_error(logger, MODULE_TABLE_SCHEMA, "table %s has no custom column", - ptable->table_name[0]); - return -1; - } - - cJSON *item = cJSON_GetObjectItem(json, "ip_type"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plugin.ip_type_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "start_ip"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plugin.start_ip_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "end_ip"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plugin.end_ip_column = item->valueint; - read_cnt++; - } - - item = cJSON_GetObjectItem(json, "is_valid"); - if (item != NULL && item->type == cJSON_Number) { - ptable->ip_plugin.is_valid_column = item->valueint; - read_cnt++; - } - - if (read_cnt < 7) { - return -1; - } - - return 0; -} - -int table_schema_populate(cJSON *json, struct table_schema **table_array, - struct table_schema *ptable, - struct maat_kv_store *reserved_word_map, - struct maat_kv_store *tablename2id_map, - struct log_handle *logger) -{ - int ret = -1; - cJSON *item = NULL; - - item = cJSON_GetObjectItem(json, "table_type"); - if (item != NULL && item->type == cJSON_String) { - ret = maat_kv_read(reserved_word_map, item->valuestring, (int*)&(ptable->table_type)); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, "table_type %s is illegal", - item->valuestring); - return -1; - } - } - - switch (ptable->table_type) { - case TABLE_TYPE_COMPILE: - ret = read_compile_table_schema(json, ptable, reserved_word_map, logger); - break; - case TABLE_TYPE_GROUP2COMPILE: - ret = read_group2compile_table_schema(json, ptable, reserved_word_map, logger); - break; - case TABLE_TYPE_GROUP2GROUP: - ret = read_group2group_table_schema(json, ptable, reserved_word_map, logger); - break; - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - ret = read_expr_table_schema(json, ptable, reserved_word_map, logger); - break; - case TABLE_TYPE_IP_PLUS: - ret = read_ip_plus_table_schema(json, ptable, logger); - break; - case TABLE_TYPE_PLUGIN: - ret = read_plugin_table_schema(json, ptable, logger); - break; - case TABLE_TYPE_IP_PLUGIN: - ret = read_ip_plugin_table_schema(json, ptable, logger); - break; - case TABLE_TYPE_VIRTUAL: - ret = read_virtual_table_schema(json, table_array, ptable, tablename2id_map, logger); - break; - case TABLE_TYPE_COMPOSITION: - ret = read_composition_table_schema(json, ptable, tablename2id_map, logger); - break; - default: - break; - } - - if (ret < 0) { - return -1; - } - - return 0; -} - -struct table_schema_manager * -table_schema_manager_create(const char *table_info_path, struct log_handle *logger) -{ - unsigned char *json_buff = NULL; - size_t json_buff_sz = 0; - - if (NULL == table_info_path) { - return NULL; - } - - int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, "Maat read table info %s error.", table_info_path); - return NULL; - } - - cJSON *root = NULL; - root = cJSON_Parse((const char *)json_buff); - if (!root) { - log_error(logger, MODULE_TABLE_SCHEMA, "Error before: %-200.200s", cJSON_GetErrorPtr()); - FREE(json_buff); - return NULL; - } - - int json_array_size = cJSON_GetArraySize(root); - if (json_array_size <= 0) { - log_error(logger, MODULE_TABLE_SCHEMA, "invalid json content in %s", table_info_path); - free(json_buff); - return NULL; - } - - struct maat_kv_store* reserved_word_map = maat_kv_store_new(); - /* register table type reserved word */ - maat_kv_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE); - maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); - maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); - maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); - maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); - maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); - maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); - maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); - maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); - maat_kv_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION); - - maat_kv_register(reserved_word_map, "block", HS_SCAN_MODE_BLOCK); - maat_kv_register(reserved_word_map, "stream", HS_SCAN_MODE_STREAM); - - struct table_schema_manager *table_schema_mgr = ALLOC(struct table_schema_manager, 1); - table_schema_mgr->logger = logger; - struct table_schema **table_array = table_schema_mgr->schema_table; - table_schema_mgr->tablename2id_map = maat_kv_store_new(); - - struct table_schema *conj_table = NULL; - for (int i = 0; i < json_array_size; i++) { - cJSON *json = cJSON_GetArrayItem(root, i); - - if (json != NULL && json->type == cJSON_Object) { - struct table_schema *ptable = table_schema_new(); - int ret = table_schema_populate(json, table_array, ptable, reserved_word_map, - table_schema_mgr->tablename2id_map, logger); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "Maat populate table info error, table_id:%d", ptable->table_id); - goto free_table; - } - - if (ptable->table_id >= MAX_TABLE_NUM) { - log_error(logger, MODULE_TABLE_SCHEMA, - "Maat read table info %s:%d error: table id %u > %d", - table_info_path, i, ptable->table_id, MAX_TABLE_NUM); - goto free_table; - } - - ret = maat_kv_register(table_schema_mgr->tablename2id_map, ptable->table_name[0], ptable->table_id); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "Duplicate table %s of table id %d", - ptable->table_name[0], ptable->table_id); - goto free_table; - } - - if (table_array[ptable->table_id] != NULL) { - conj_table = table_array[ptable->table_id]; - if (conj_table->conj_cnt == MAX_CONJUNCTION_TABLE_NUM) { - log_error(logger, MODULE_TABLE_SCHEMA, - "Maat read table info %s error: reach table_id %d conjunction upper limit", - table_info_path, ptable->table_id); - goto free_table; - } - memcpy(conj_table->table_name[conj_table->conj_cnt], ptable->table_name[0], NAME_MAX); - conj_table->conj_cnt++; - log_info(logger, MODULE_TABLE_SCHEMA, - "Maat read table info %s: conjunction %s with %s (id=%d,total=%d)", - table_info_path, ptable->table_name[0], conj_table->table_name[0], - conj_table->table_id, conj_table->conj_cnt); - goto free_table; - } - - table_array[ptable->table_id] = ptable; - table_schema_mgr->n_schema_table++; - continue; - free_table: - table_schema_free(ptable); - ptable = NULL; - } - } - - maat_kv_store_free(reserved_word_map); - FREE(json_buff); - - return table_schema_mgr; -} - -void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr) -{ - if (NULL == table_schema_mgr) { - return; - } - - for (size_t i = 0; i < MAX_TABLE_NUM; i++) { - if (NULL == table_schema_mgr->schema_table[i]) { - continue; - } - table_schema_free(table_schema_mgr->schema_table[i]); - table_schema_mgr->schema_table[i] = NULL; - } - - maat_kv_store_free(table_schema_mgr->tablename2id_map); - FREE(table_schema_mgr); -} - -int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name) -{ - if (NULL == table_schema_mgr || NULL == table_name) { - return -1; - } - - int table_id = -1; - - int ret = maat_kv_read(table_schema_mgr->tablename2id_map, table_name, &table_id); - if (ret > 0) { - return table_id; - } else { - return -1; - } -} - -int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id, - enum component_table_type type) -{ - struct table_schema* table_schema = table_schema_get(table_schema_mgr, parent_table_id); - if (table_schema->table_type != TABLE_TYPE_COMPOSITION) { - return -1; - } - - return table_schema->composition.component_table_id[type]; -} - -enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int id) -{ - if (NULL == table_schema_mgr) { - return TABLE_TYPE_INVALID; - } - - if (NULL == table_schema_mgr->schema_table[id]) { - return TABLE_TYPE_INVALID; - } - - return table_schema_mgr->schema_table[id]->table_type; -} - -size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_mgr) -{ - return MAX_TABLE_NUM; -} - -void table_schema_manager_all_plugin_cb_start(struct table_schema_manager *table_schema_mgr, int update_type) -{ - if (NULL == table_schema_mgr) { - return; - } - - struct table_schema *ptable = NULL; - struct plugin_table_schema *plugin_schema = NULL; - - for (size_t i = 0; i < MAX_TABLE_NUM; i++) { - ptable = table_schema_mgr->schema_table[i]; - if (NULL == ptable) { - continue; - } - - plugin_schema = &(ptable->plugin); - if (ptable->table_type != TABLE_TYPE_PLUGIN || 0 == plugin_schema->cb_plug_cnt) { - continue; - } - - table_schema_mgr->n_active_plugin_table++; - for (size_t j = 0; j < plugin_schema->cb_plug_cnt; j++) { - if (plugin_schema->cb_plug[j].start != NULL) { - plugin_schema->cb_plug[j].start(update_type, plugin_schema->cb_plug[j].u_para); - } - } - } -} - -void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* table_schema_mgr) -{ - if (NULL == table_schema_mgr) { - return; - } - - struct table_schema *ptable = NULL; - struct plugin_table_schema *plugin_schema = NULL; - - for (size_t i = 0; i < MAX_TABLE_NUM; i++) { - ptable = table_schema_mgr->schema_table[i]; - if (NULL == ptable) { - continue; - } - - if (ptable->table_type != TABLE_TYPE_PLUGIN) { - continue; - } - - plugin_schema = &(ptable->plugin); - for (size_t j = 0; j < plugin_schema->cb_plug_cnt; j++) { - if (plugin_schema->cb_plug[j].finish != NULL) { - plugin_schema->cb_plug[j].finish(plugin_schema->cb_plug[j].u_para); - } - } - } - table_schema_mgr->n_active_plugin_table = 0; -} - -enum expr_type int_to_expr_type(int expr_type) -{ - enum expr_type type = EXPR_TYPE_MAX; - - switch (expr_type) { - case 0: - type = EXPR_TYPE_STRING; - break; - case 1: - type = EXPR_TYPE_AND; - break; - case 2: - type = EXPR_TYPE_REGEX; - break; - default: - break; - } - - return type; -} - -enum match_method int_to_match_method_type(int match_method_type) -{ - enum match_method type = MATCH_METHOD_MAX; - - switch (match_method_type) { - case 0: - type = MATCH_METHOD_SUB; - break; - case 1: - type = MATCH_METHOD_RIGHT; - break; - case 2: - type = MATCH_METHOD_LEFT; - break; - case 3: - type = MATCH_METHOD_COMPLETE; - break; - default: - break; - } - - return type; -} - -struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id) -{ - if ((NULL == table_schema_mgr) || (table_id < 0)) { - return NULL; - } - - return table_schema_mgr->schema_table[table_id]; -} - -struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr, - int table_id, enum scan_type scan_type, int *virtual_table_id) -{ - enum scan_type table_scan_type; - struct table_schema **pptable = table_schema_mgr->schema_table; - size_t n_table = MAX_TABLE_NUM; - - if ((unsigned int)table_id > n_table) { - return NULL; - } - - if (NULL == pptable[table_id]) { - return NULL; - } - - struct table_schema *ptable = pptable[table_id]; - if (NULL == ptable) { - return NULL; - } - - struct table_schema *p_physical_table = NULL; - if (ptable->table_type == TABLE_TYPE_VIRTUAL) { - p_physical_table = pptable[ptable->virtual_table.physical_table_id[scan_type]]; - *virtual_table_id = table_id; - } else { - p_physical_table = ptable; - if(virtual_table_id) { - *virtual_table_id = 0; - } - } - - table_scan_type = table_schema_get_scan_type(p_physical_table); - if (table_scan_type != scan_type) { - return NULL; - } - - return p_physical_table; -} - -enum table_type table_schema_get_table_type(struct table_schema *table_schema) -{ - if (NULL == table_schema) { - return TABLE_TYPE_INVALID; - } - - return table_schema->table_type; -} - -int table_schema_get_table_id(struct table_schema *table_schema) -{ - if (NULL == table_schema) { - return -1; - } - - return table_schema->table_id; -} - -int table_schema_get_associated_table_id(struct table_schema *table_schema) -{ - if (NULL == table_schema || table_schema->table_type != TABLE_TYPE_GROUP2COMPILE) { - return -1; - } - - return table_schema->group2compile.associated_compile_table_id; -} - -enum scan_type table_schema_get_scan_type(struct table_schema *table_schema) -{ - enum scan_type ret = SCAN_TYPE_INVALID; - - if (NULL == table_schema) { - return ret; - } - - switch (table_schema->table_type) { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - case TABLE_TYPE_SIMILARITY: - case TABLE_TYPE_DIGEST: - ret = SCAN_TYPE_STRING; - break; - case TABLE_TYPE_INTERVAL: - case TABLE_TYPE_INTERVAL_PLUS: - ret = SCAN_TYPE_INTERVAL; - break; - case TABLE_TYPE_IP_PLUS: - case TABLE_TYPE_COMPOSITION: - ret = SCAN_TYPE_IP; - break; - case TABLE_TYPE_PLUGIN: - ret = SCAN_TYPE_PLUGIN; - break; - case TABLE_TYPE_IP_PLUGIN: - ret = SCAN_TYPE_IP; - break; - case TABLE_TYPE_FQDN_PLUGIN: - ret = SCAN_TYPE_FQDN_PLUGIN; - break; - case TABLE_TYPE_BOOL_PLUGIN: - ret = SCAN_TYPE_BOOL_PLUGIN; - break; - case TABLE_TYPE_COMPILE: - ret = SCAN_TYPE_NONE; - break; - default: - break; - } - - return ret; -} - -int table_schema_get_valid_flag_column(struct table_schema *table_schema) -{ - int valid_flag_column = -1; - - if (NULL == table_schema) { - return valid_flag_column; - } - - switch (table_schema->table_type) { - case TABLE_TYPE_COMPILE: - valid_flag_column = table_schema->compile.is_valid_column; - break; - case TABLE_TYPE_GROUP2COMPILE: - valid_flag_column = table_schema->group2compile.is_valid_column; - break; - case TABLE_TYPE_GROUP2GROUP: - valid_flag_column = table_schema->group2group.is_valid_column; - break; - case TABLE_TYPE_EXPR: - valid_flag_column = table_schema->expr.is_valid_column; - break; - case TABLE_TYPE_PLUGIN: - valid_flag_column = table_schema->plugin.is_valid_column; - break; - case TABLE_TYPE_IP_PLUGIN: - valid_flag_column = table_schema->ip_plugin.is_valid_column; - break; - default: - valid_flag_column = -1; - break; - } - - return valid_flag_column; -} - -void table_schema_set_updating_name(struct table_schema *table_schema, const char *table_name) -{ - if (NULL == table_schema) { - return; - } - - int i = 0; - for (i = 0; i < table_schema->conj_cnt; i++) { - if (0 == strcmp(table_schema->table_name[i], table_name)) { - table_schema->updating_name = i; - } - } - assert(i <= table_schema->conj_cnt); -} - -const char *table_schema_get_updating_name(struct table_schema *table_schema) -{ - return table_schema->table_name[table_schema->updating_name]; -} - -static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, int n_accept_tag) -{ - cJSON *tab_name_obj = cJSON_GetObjectItem(tag_obj, "tag"); - if (NULL == tab_name_obj || tab_name_obj->type != cJSON_String) { - goto error; - } - - const char *tag_name = tab_name_obj->valuestring; - cJSON *tag_vals_array = cJSON_GetObjectItem(tag_obj, "value"); - if (NULL == tag_vals_array || tag_vals_array->type != cJSON_Array) { - goto error; - } - - int name_matched = 0; - int n_val = cJSON_GetArraySize(tag_vals_array); - - for (int i = 0; i < n_accept_tag; i++) { - if (0 != strcmp(accept_tags[i].tag_name, tag_name)) { - continue; - } - name_matched++; - - for (int j = 0; j < n_val; j++) { - cJSON *tag_val_obj = cJSON_GetArrayItem(tag_vals_array, j); - if (NULL == tag_val_obj || tag_val_obj->type != cJSON_String) { - goto error; - } - - const char *tag_val = tag_val_obj->valuestring; - // compare a/b/c with a/b/c/d is a miss. - if (strlen(accept_tags[i].tag_val) < strlen(tag_val)) { - continue; - } - - // compare a1a2/b1/c1 with a1a2/b/ is a miss. - //make sure the overlap is ended with a '/' - size_t compare_len = - if (0 == strncmp(accept_tags[i].tag_val, tag_val, strlen(tag_val)) && - (strlen(accept_tags[i].tag_val) == strlen(tag_val) || accept_tags[i].tag_val[strlen(tag_val)] == '/')) { - return 1; - } - } - } - - //no matched name is considered as a - if (name_matched > 0) { - return 0; - } else { - return 1; - } - -error: - return -1; -} - -//@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}] -static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_tags, int n_accept_tag) -{ - int matched = 0; - - int n_tag = cJSON_GetArraySize(tag_set); - for (int i = 0; i < n_tag; i++) { - cJSON *tag_obj = cJSON_GetArrayItem(tag_set, i); - if (NULL == tag_obj || tag_obj->type != cJSON_Object) { - goto error; - } - - int ret = compare_each_tag(tag_obj, accept_tags, n_accept_tag); - if (ret < 0) { - return -1; - } - - if(1 == ret) { - matched++; - } - } - - if (matched == n_tag) { - return 1; - } else { - return 0; - } - -error: - return -1; -} - -//@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]} -//@return 1 on match, 0 on not match, -1 on error. -static int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag) -{ - int ret = -1; - int n_set = 0; - cJSON *tag_set = NULL; - cJSON *tag_set_array = NULL; - - cJSON *root = cJSON_Parse(value); - if (NULL == root) { - goto next; - } - - tag_set_array = cJSON_GetObjectItem(root, "tag_sets"); - if (NULL == tag_set_array || tag_set_array->type != cJSON_Array) { - goto next; - } - - n_set = cJSON_GetArraySize(tag_set_array); - for (int i = 0; i < n_set; i++) { - tag_set = cJSON_GetArrayItem(tag_set_array, i); - if (NULL == tag_set || tag_set->type != cJSON_Array) { - goto next; - } - - ret = compare_each_tag_set(tag_set, accept_tags, n_tag); - //match or error occurs. - if (ret != 0) { - break; - } - } - -error: - cJSON_Delete(root); - return ret; -} - -int populate_compile_table_item(const char *line, struct table_schema *table_schema, struct table_item *table_item, - const char *table_name, const struct rule_tag *accept_tags, int n_accept_tag, - struct log_handle *logger) -{ - size_t column_offset = 0; - size_t column_len = 0; - struct compile_table_schema *compile_schema = &(table_schema->compile); - struct compile_item *compile_item = &(table_item->compile_item); - - int ret = get_column_pos(line, compile_schema->compile_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - compile_item->compile_id = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->service_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - compile_item->service_id = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->action_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - compile_item->action = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->do_blacklist_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - compile_item->do_blacklist = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->do_log_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - compile_item->do_log = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->tags_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - - char tag_str[MAX_TABLE_LINE_SIZE] = {0}; - memcpy(tag_str, (line + column_offset), column_len); - - if (n_accept_tag > 0 && strlen(tag_str) > 2) { - str_unescape(tag_str); - ret = compare_accept_tag(tag_str, accept_tags, n_accept_tag); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error, invalid tag format for compile table[%s]:compile_id[%d]", - table_name, compile_item->compile_id); - return -1; - } - - if (0 == ret) { - table_schema->unmatched_tag_cnt++; - return -1; - } - } - - ret = get_column_pos(line, compile_schema->user_region_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - - if (column_len > MAX_TABLE_LINE_SIZE) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: compile table[%s]:compile_id[%d] user_region length too long", - table_name, compile_item->compile_id); - return -1; - } - - memcpy(compile_item->user_region, (line + column_offset), column_len); - - switch (compile_schema->user_region_encoding) { - case USER_REGION_ENCODE_ESCAPE: - str_unescape(compile_item->user_region); - break; - default: - break; - } - - ret = get_column_pos(line, compile_schema->is_valid_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - compile_item->is_valid = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->clause_num_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - compile_item->clause_num = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->evaluation_order_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - compile_item->evaluation_order = atoi(line + column_offset); - - return 0; -} - -int populate_group2compile_table_item(const char *line, struct group2compile_table_schema *group2compile_schema, - struct table_item *table_item, const char *table_name, - struct log_handle *logger) -{ - size_t column_offset = 0; - size_t column_len = 0; - struct group2compile_item *group2compile_item = &(table_item->group2compile_item); - - int ret = get_column_pos(line, group2compile_schema->group_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - group2compile_item->group_id = atoi(line + column_offset); - - ret = get_column_pos(line, group2compile_schema->compile_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - group2compile_item->compile_id = atoi(line + column_offset); - - ret = get_column_pos(line, group2compile_schema->is_valid_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - group2compile_item->is_valid = atoi(line + column_offset); - - ret = get_column_pos(line, group2compile_schema->not_flag_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - group2compile_item->not_flag = atoi(line + column_offset); - - ret = get_column_pos(line, group2compile_schema->virtual_table_name_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - - if (column_len > NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: group2compile table[%s]:group_id[%d] virtual table name length too long", - table_name, group2compile_item->group_id); - return -1; - } - memcpy(group2compile_item->virtual_table_name, (line + column_offset), column_len); - - ret = get_column_pos(line, group2compile_schema->clause_index_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - - group2compile_item->clause_index = atoi(line + column_offset); - - return 0; -} - -int populate_group2group_table_item(const char *line, struct group2group_table_schema *group2group_schema, - struct table_item *table_item, const char *table_name, - struct log_handle *logger) -{ - size_t column_offset = 0; - size_t column_len = 0; - struct group2group_item *group2group_item = &(table_item->group2group_item); - - int ret = get_column_pos(line, group2group_schema->group_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - group2group_item->group_id = atoi(line + column_offset); - - ret = get_column_pos(line, group2group_schema->superior_group_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - group2group_item->superior_group_id = atoi(line + column_offset); - - ret = get_column_pos(line, group2group_schema->is_valid_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - group2group_item->is_valid = atoi(line + column_offset); - - return 0; -} - -int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, - struct table_item *table_item, const char *table_name, - struct log_handle *logger) -{ - size_t column_offset = 0; - size_t column_len = 0; - struct expr_item *expr_item = &(table_item->expr_item); - - int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - expr_item->item_id = atoi(line + column_offset); - - ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - expr_item->group_id = atoi(line + column_offset); - - if (table_item->table_type == TABLE_TYPE_EXPR_PLUS) { - ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - - if (column_len >= MAX_DISTRICT_STR) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: expr table[%s]:item_id[%d] district length too long", - table_name, expr_item->item_id); - return -1; - } - memcpy(expr_item->district, (line + column_offset), column_len); - } - - ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - - if (column_len >= MAX_KEYWORDS_STR) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: expr table[%s]:item_id[%d] keywords length too long", - table_name, expr_item->item_id); - return -1; - } - memcpy(expr_item->keywords, (line + column_offset), column_len); - - ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - - int expr_type = atoi(line + column_offset); - expr_item->expr_type = int_to_expr_type(expr_type); - - ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - - int match_method_type = atoi(line + column_offset); - expr_item->match_method = int_to_match_method_type(match_method_type); - - ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - int db_hexbin = atoi(line + column_offset); - switch (db_hexbin) { - case 0: - expr_item->is_hexbin = FALSE; - expr_item->is_case_sensitive = FALSE; - break; - case 1: - expr_item->is_hexbin = TRUE; - expr_item->is_case_sensitive = TRUE; - break; - case 2: - expr_item->is_hexbin = FALSE; - expr_item->is_case_sensitive = TRUE; - break; - default: - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: expr table[%s]:item_id[%d] invalid hexbin value:%d", - table_name, expr_item->item_id, db_hexbin); - return -1; - } - ret = get_column_pos(line, expr_schema->is_valid_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - expr_item->is_valid = atoi(line + column_offset); - - return 0; -} - -int populate_ip_plus_table_item(const char *line, struct ip_plus_table_schema *ip_plus_schema, - struct table_item *table_item, const char *table_name, - struct log_handle *logger) -{ - size_t column_offset = 0; - size_t column_len = 0; - struct ip_plus_item *ip_plus_item = &(table_item->ip_plus_item); - - int ret = get_column_pos(line, ip_plus_schema->item_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plus_item->item_id = atoi(line + column_offset); - - ret = get_column_pos(line, ip_plus_schema->group_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plus_item->group_id = atoi(line + column_offset); - - ret = get_column_pos(line, ip_plus_schema->addr_type_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plus_item->addr_type = atoi(line + column_offset); - if (ip_plus_item->addr_type != 4 && ip_plus_item->addr_type != 6) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: ip_plus table[%s]:item_id[%d] invalid addr type:%d", - table_name, ip_plus_item->item_id, ip_plus_item->addr_type); - return -1; - } - - char saddr_format[16] = {0}; - char sport_format[16] = {0}; - char sip1_str[40] = {0}; - char sip2_str[40] = {0}; - uint16_t sport1 = 0; - uint16_t sport2 = 0; - - ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - memcpy(saddr_format, (line + column_offset), column_len); - if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: ip_plus table[%s]:item_id[%d] invalid saddr_format, should be range/mask/CIDR", - table_name, ip_plus_item->item_id); - return -1; - } - - ret = get_column_pos(line, ip_plus_schema->sip1_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - memcpy(sip1_str, (line + column_offset), column_len); - - ret = get_column_pos(line, ip_plus_schema->sip2_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - memcpy(sip2_str, (line + column_offset), column_len); - - ret = get_column_pos(line, ip_plus_schema->sport_format_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - memcpy(sport_format, (line + column_offset), column_len); - if (IP_FORMAT_UNKNOWN == ip_format_str2int(sport_format)) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: ip_plus table[%s]:item_id[%d] invalid sport_format, should be range/mask/CIDR", - table_name, ip_plus_item->item_id); - return -1; - } - - ret = get_column_pos(line, ip_plus_schema->sport1_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - sport1 = atoi(line + column_offset); - - ret = get_column_pos(line, ip_plus_schema->sport2_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - sport2 = atoi(line + column_offset); - - uint16_t protocol = 0; - uint16_t direction = 0; - if (4 == ip_plus_item->addr_type) { - ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, - &ip_plus_item->ipv4.min_sip, &ip_plus_item->ipv4.max_sip); - if (ret < 0) { - return -1; - } - - if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { - ip_plus_item->ipv4.min_sport = sport1 & sport2; - ip_plus_item->ipv4.max_sport = sport1 | ~sport2; - } else { - ip_plus_item->ipv4.min_sport = sport1; - ip_plus_item->ipv4.max_sport = sport2; - } - - ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plus_item->ipv4.proto = atoi(line + column_offset); - protocol = ip_plus_item->ipv4.proto; - - ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plus_item->ipv4.direction = atoi(line + column_offset); - direction = ip_plus_item->ipv4.direction; - } else { - //ipv6 - ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, - ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip); - if (ret < 0) { - return -1; - } - - if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { - ip_plus_item->ipv6.min_sport = sport1 & sport2; - ip_plus_item->ipv6.max_sport = sport1 | ~sport2; - } else { - ip_plus_item->ipv6.min_sport = sport1; - ip_plus_item->ipv6.max_sport = sport2; - } - - ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plus_item->ipv6.proto = atoi(line + column_offset); - protocol = ip_plus_item->ipv6.proto; - - ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plus_item->ipv6.direction = atoi(line + column_offset); - direction = ip_plus_item->ipv6.direction; - } - - if (protocol > 65535 || protocol < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: ip_plus table[%s]:item_id[%d] invalid protocol:%d", - table_name, ip_plus_item->item_id, protocol); - return -1; - } - - if (direction != 0 && direction != 1) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: ip_plus table[%s]:item_id[%d] invalid direction:%d", - table_name, ip_plus_item->item_id, direction); - return -1; - } - - ret = get_column_pos(line, ip_plus_schema->is_valid_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plus_item->is_valid = atoi(line + column_offset); - - return 0; -} - -int populate_plugin_table_item(const char *line, struct plugin_table_schema *plugin_schema, - struct table_item *table_item, const char *table_name, - struct log_handle *logger) -{ - size_t column_offset = 0; - size_t column_len = 0; - struct plugin_item *plugin_item = &(table_item->plugin_item); - - int ret = get_column_pos(line, plugin_schema->key_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - - if (column_len > MAX_KEYWORDS_STR) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: plugin table[%s]: key:%s length:%zu too long, exceed %d", - table_name, (line + column_offset), column_len, MAX_KEYWORDS_STR); - return -1; - } - memcpy(plugin_item->key, (line + column_offset), column_len); - plugin_item->key_len = column_len; - - ret = get_column_pos(line, plugin_schema->is_valid_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - plugin_item->is_valid = atoi(line + column_offset); - - return 0; -} - -int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schema *ip_plugin_schema, - struct table_item *table_item, const char *table_name, - struct log_handle *logger) -{ - size_t column_offset = 0; - size_t column_len = 0; - struct ip_plugin_item *ip_plugin_item = &(table_item->ip_plugin_item); - - int ret = get_column_pos(line, ip_plugin_schema->item_id_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plugin_item->item_id = atoi(line + column_offset); - - ret = get_column_pos(line, ip_plugin_schema->is_valid_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: ip_plugin table[%s]:item_id[%d] not found is_valid value", - table_name, ip_plugin_item->item_id); - return -1; - } - ip_plugin_item->is_valid = atoi(line + column_offset); - - ret = get_column_pos(line, ip_plugin_schema->ip_type_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } - ip_plugin_item->ip_type = atoi(line + column_offset); - if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: ip_plugin table[%s]:item_id[%d] ip_type[%d] invalid", - table_name, ip_plugin_item->item_id, ip_plugin_item->ip_type); - return -1; - } - - ret = get_column_pos(line, ip_plugin_schema->start_ip_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: ip_plugin table[%s]:item_id[%d] no found start_ip value", - table_name, ip_plugin_item->item_id, ip_plugin_item->start_ip); - return -1; - } - strncpy(ip_plugin_item->start_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->start_ip))); - - ret = get_column_pos(line, ip_plugin_schema->end_ip_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "update error: ip_plugin table[%s]:item_id[%d] no found end_ip value", - table_name, ip_plugin_item->item_id, ip_plugin_item->end_ip); - return -1; - } - strncpy(ip_plugin_item->end_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->end_ip))); - - return 0; -} - -void table_item_free(struct table_item *table_item) -{ - if (NULL == table_item) { - return; - } - - switch (table_item->table_type) { - case TABLE_TYPE_COMPILE: - case TABLE_TYPE_GROUP2COMPILE: - case TABLE_TYPE_GROUP2GROUP: - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - case TABLE_TYPE_IP_PLUS: - case TABLE_TYPE_PLUGIN: - break; - case TABLE_TYPE_IP_PLUGIN: - /* TODO: free ex_data */ - break; - default: - break; - } - - FREE(table_item); -} - -struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema, - struct rule_tag *accept_tags, int n_accept_tag, struct log_handle *logger) -{ - if (NULL == line || NULL == table_schema) { - return NULL; - } - - int ret = -1; - const char *table_name = table_schema_get_updating_name(table_schema); - struct table_item *table_item = ALLOC(struct table_item, 1); - - switch (table_schema->table_type) { - case TABLE_TYPE_COMPILE: - table_item->table_type = table_schema->table_type; - ret = populate_compile_table_item(line, table_schema, table_item, table_name, accept_tags, n_accept_tag, logger); - break; - case TABLE_TYPE_GROUP2COMPILE: - table_item->table_type = table_schema->table_type; - ret = populate_group2compile_table_item(line, &table_schema->group2compile, table_item, table_name, logger); - break; - case TABLE_TYPE_GROUP2GROUP: - table_item->table_type = table_schema->table_type; - ret = populate_group2group_table_item(line, &table_schema->group2group, table_item, table_name, logger); - break; - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - table_item->table_type = table_schema->table_type; - ret = populate_expr_table_item(line, &table_schema->expr, table_item, table_name, logger); - break; - case TABLE_TYPE_IP_PLUS: - table_item->table_type = table_schema->table_type; - ret = populate_ip_plus_table_item(line, &table_schema->ip_plus, table_item, table_name, logger); - break; - case TABLE_TYPE_PLUGIN: - table_item->table_type = TABLE_TYPE_PLUGIN; - ret = populate_plugin_table_item(line, &table_schema->plugin, table_item, table_name, logger); - break; - case TABLE_TYPE_IP_PLUGIN: - table_item->table_type = TABLE_TYPE_IP_PLUGIN; - ret = populate_ip_plugin_table_item(line, &table_schema->ip_plugin, table_item, table_name, logger); - break; - default: - break; - } - - if (ret < 0) { - log_error(logger, MODULE_TABLE_SCHEMA, - "abandon config: invalid format of table %s:%s, transform line to item failed. ", - table_name, line); - table_schema->update_err_cnt++; - goto error; - } - return table_item; - -error: - table_item_free(table_item); - return NULL; -} - -enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema) -{ - if (NULL == table_schema) { - return HS_SCAN_MODE_MAX; - } - - if (table_schema->table_type != TABLE_TYPE_EXPR) { - return HS_SCAN_MODE_MAX; - } - - return table_schema->expr.scan_mode; -} - -int table_schema_set_ex_data_schema(struct table_schema *table_schema, - maat_plugin_ex_new_func_t *new_func, - maat_plugin_ex_free_func_t *free_func, - maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp, - struct log_handle *logger) -{ - if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) { - assert(0); - log_error(logger, MODULE_TABLE_SCHEMA, "%s failed: invalid parameter", __FUNCTION__); - return -1; - } - - struct ex_data_schema *ex_schema = table_schema_get_ex_data_schema(table_schema); - if (NULL == ex_schema) { - log_error(logger, MODULE_TABLE_SCHEMA, - "Error: %s, target table is not a valid plugin table", __FUNCTION__); - return -1; - } - - if (ex_schema->set_flag) { - assert(0); - log_error(logger, MODULE_TABLE_SCHEMA, "Error: %s, EX data schema already registed", - __FUNCTION__); - return -1; - } - - ex_schema->new_func = new_func; - ex_schema->free_func = free_func; - ex_schema->dup_func = dup_func; - ex_schema->argl = argl; - ex_schema->argp = argp; - ex_schema->set_flag = 1; - - return 0; -} - -struct ex_data_schema *table_schema_get_ex_data_schema(struct table_schema *table_schema) -{ - if (NULL == table_schema) { - return NULL; - } - - struct ex_data_schema *ex_schema = NULL; - - switch (table_schema->table_type) { - case TABLE_TYPE_PLUGIN: - ex_schema = &(table_schema->plugin.ex_schema); - break; - case TABLE_TYPE_IP_PLUGIN: - ex_schema = &(table_schema->ip_plugin.ex_schema); - break; - default: - break; - } - - return ex_schema; -} - -int table_schema_set_compile_rule_ex_data_schema(struct table_schema *table_schema, - maat_rule_ex_new_func_t *new_func, - maat_rule_ex_free_func_t *free_func, - maat_rule_ex_dup_func_t *dup_func, - long argl, void *argp, - struct log_handle *logger) -{ - if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) { - assert(0); - log_error(logger, MODULE_TABLE_SCHEMA, "%s failed: invalid parameter", __FUNCTION__); - return -1; - } - - if (table_schema->table_type != TABLE_TYPE_COMPILE) { - log_error(logger, MODULE_TABLE_SCHEMA, - "table type must TABLE_TYPE_COMPILE when set compile rule ex_data schema"); - return -1; - } - - struct compile_table_schema* compile_schema = &(table_schema->compile); - if (compile_schema->n_ex_schema == MAX_COMPILE_EX_DATA_NUM) { - log_error(logger, MODULE_TABLE_SCHEMA, - "compile ex schema num reach maxium, can't set anymore"); - return -1; - } - - int idx = compile_schema->n_ex_schema; - compile_schema->ex_schema[idx].idx = idx; - compile_schema->ex_schema[idx].table_id = table_schema->table_id; - compile_schema->ex_schema[idx].argl = argl; - compile_schema->ex_schema[idx].argp = argp; - compile_schema->ex_schema[idx].new_func = new_func; - compile_schema->ex_schema[idx].free_func = free_func; - compile_schema->ex_schema[idx].dup_func = dup_func; - compile_schema->n_ex_schema++; - - return idx; -} - -struct compile_ex_data_schema * -table_schema_get_compile_rule_ex_data_schema(struct table_schema *table_schema, size_t idx) -{ - if (NULL == table_schema) { - return NULL; - } - - if (table_schema->table_type != TABLE_TYPE_COMPILE) { - return NULL; - } - - if (idx < table_schema->compile.n_ex_schema) { - return (table_schema->compile.ex_schema + idx); - } - - return NULL; -} - -size_t table_schema_compile_rule_ex_data_schema_count(struct table_schema *table_schema) -{ - if (NULL == table_schema) { - return 0; - } - - if (table_schema->table_type != TABLE_TYPE_COMPILE) { - return 0; - } - - return table_schema->compile.n_ex_schema; -} - -int table_schema_ex_data_schema_flag(struct table_schema *table_schema) -{ - if (NULL == table_schema) { - return -1; - } - - struct ex_data_schema *ex_schema = NULL; - - switch (table_schema->table_type) { - case TABLE_TYPE_PLUGIN: - ex_schema = &(table_schema->plugin.ex_schema); - break; - case TABLE_TYPE_IP_PLUGIN: - ex_schema = &(table_schema->ip_plugin.ex_schema); - break; - default: - break; - } - - return ex_schema->set_flag; -} - -int table_schema_add_callback(struct table_schema_manager* table_schema_mgr, int table_id, - maat_start_callback_t *start, - maat_update_callback_t *update, - maat_finish_callback_t *finish, - void *u_para, struct log_handle *logger) -{ - if ((NULL == table_schema_mgr) || (table_id < 0)) { - return -1; - } - - struct table_schema *ptable = table_schema_get(table_schema_mgr, table_id); - if (NULL == ptable) { - log_error(logger, MODULE_TABLE_SCHEMA, - "table_id:%d unregistered, can't add callback func", table_id); - return -1; - } - - switch (ptable->table_type) { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - case TABLE_TYPE_IP_PLUS: - log_error(logger, MODULE_TABLE_SCHEMA, "table_id: %d is not plugin type", table_id); - return -1; - default: - break; - } - - struct plugin_table_schema *plugin_schema = &(ptable->plugin); - size_t idx = plugin_schema->cb_plug_cnt; - if (idx == MAX_PLUGIN_PER_TABLE) { - log_error(logger, MODULE_TABLE_SCHEMA, "the plugin number of table_id: %d exceed maxium:%d", - ptable->table_id, MAX_PLUGIN_PER_TABLE); - return -1; - } - - plugin_schema->cb_plug_cnt++; - plugin_schema->cb_plug[idx].start = start; - plugin_schema->cb_plug[idx].update = update; - plugin_schema->cb_plug[idx].finish = finish; - plugin_schema->cb_plug[idx].u_para = u_para; - - return 0; -} - -size_t table_schema_callback_count(struct table_schema *table_schema) -{ - if (NULL == table_schema) { - return 0; - } - - if (table_schema->table_type == TABLE_TYPE_PLUGIN) { - return table_schema->plugin.cb_plug_cnt; - } else { - return 0; - } -} - -void table_schema_all_cb_update(struct table_schema* table_schema, const char *row) -{ - if (NULL == table_schema || NULL == row) { - return; - } - - struct plugin_table_schema *plugin_schema = NULL; - - switch (table_schema->table_type) { - case TABLE_TYPE_EXPR: - case TABLE_TYPE_EXPR_PLUS: - case TABLE_TYPE_IP_PLUGIN: - break; - case TABLE_TYPE_PLUGIN: - plugin_schema = &(table_schema->plugin); - if (plugin_schema->cb_plug_cnt > 0) { - for (size_t i = 0; i < plugin_schema->cb_plug_cnt; i++) { - plugin_schema->cb_plug[i].update(table_schema->table_id, row, plugin_schema->cb_plug[i].u_para); - } - } - break; - default: - break; - } -} - -int table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns) -{ - if (NULL == table_schema) { - return -1; - } - - if (table_schema->table_type != TABLE_TYPE_PLUGIN) { - return 0; - } - - int n_foreign = table_schema->plugin.n_foreign; - for (int i = 0; i < n_foreign; i++) { - foreign_columns[i] = table_schema->plugin.foreign_columns[i]; - } - - return n_foreign; -} \ No newline at end of file diff --git a/src/maat_utils.cpp b/src/maat_utils.cpp index dc73344..e44854e 100644 --- a/src/maat_utils.cpp +++ b/src/maat_utils.cpp @@ -45,6 +45,19 @@ char *maat_strdup(const char *s) return d; } +int get_column_value(const char *line, int column_seq) +{ + size_t column_offset = 0; + size_t column_len = 0; + + int ret = get_column_pos(line, column_seq, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + + return atoi(line + column_offset); +} + int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len) { int i = 0; diff --git a/src/maat_virtual.cpp b/src/maat_virtual.cpp new file mode 100644 index 0000000..5941b6e --- /dev/null +++ b/src/maat_virtual.cpp @@ -0,0 +1,59 @@ +/* +********************************************************************************************** +* File: maat_virtual.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include "cJSON/cJSON.h" +#include "maat_kv.h" +#include "utils.h" +#include "maat_utils.h" +#include "log/log.h" +#include "maat_virtual.h" + +#define MODULE_VIRTUAL module_name_str("maat.virtual") + +struct virtual_schema { + int physical_table_id[SCAN_TYPE_MAX]; +}; + +void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +{ + size_t read_cnt = 0; + + cJSON *item = cJSON_GetObjectItem(json, "physical_table"); + if (NULL == item || item->type != cJSON_Array) { + log_error(logger, MODULE_VIRTUAL, "virtual table %s has no physical_table column", table_name); + return NULL; + } + read_cnt++; +#if 0 + struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); + int cnt = cJSON_GetArraySize(item); + for (int i = 0; i < cnt; i++) { + cJSON *tmp_item = cJSON_GetArrayItem(item, i); + if (tmp_item != NULL && tmp_item->type == cJSON_String) { + int table_id = -1; + /* physical table should already exist */ + int ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &table_id); + if (ret < 0) { + return -1; + } + + enum scan_type table_scan_type = table_schema_get_scan_type(table_array[table_id]); + vt_schema->physical_table_id[table_scan_type]= table_id; + } + } + + if (read_cnt < 3) { + FREE(vt_schema); + return NULL; + } + + return vt_schema; +#endif +} diff --git a/src/rcu_hash.cpp b/src/rcu_hash.cpp index 15439a2..93d3382 100644 --- a/src/rcu_hash.cpp +++ b/src/rcu_hash.cpp @@ -267,6 +267,19 @@ size_t rcu_hash_count(struct rcu_hash_table *htable) } } +size_t rcu_hash_updating_count(struct rcu_hash_table *htable) +{ + if (NULL == htable) { + return 0; + } + + if (htable->effective_hash == 'a') { + return HASH_CNT(hh_b, htable->hashmap_b); + } else { + return HASH_CNT(hh_a, htable->hashmap_a); + } +} + void rcu_hash_commit(struct rcu_hash_table *htable) { if (NULL == htable) { diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 6105e9b..caf0acc 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -4,8 +4,6 @@ #include "maat/maat.h" #include "maat_rule.h" #include "maat_utils.h" -#include "maat_table_schema.h" -#include "maat_table_runtime.h" #include "maat_command.h" #include "IPMatcher.h" #include "json2iris.h" @@ -17,250 +15,238 @@ const char *json_path="./maat_json.json"; const char *json_filename = "maat_json.json"; TEST(maat_scan_string, hit_one_expr) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); - char data[128] = "i.ytimg.com"; + char scan_data[128] = "hello"; int results[5] = {0}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); - EXPECT_EQ(results[0], 30); + int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 191); + + struct maat_hit_path hit_path[128] = {0}; + int n_read = 0; + n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, sizeof(hit_path)); + maat_state_free(&state); } TEST(maat_scan_string, hit_two_expr) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); char data[128] = "should hit aaa bbb"; int results[5] = {0}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 2); + EXPECT_EQ(n_hit_result, 2); EXPECT_EQ(results[0], 28); EXPECT_EQ(results[1], 27); + + maat_state_free(&state); } TEST(maat_scan_string, hit_three_expr) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); char data[128] = "should hit aaa bbb C#中国"; int results[5] = {0}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 3); + EXPECT_EQ(n_hit_result, 3); EXPECT_EQ(results[0], 28); EXPECT_EQ(results[1], 27); EXPECT_EQ(results[2], 18); + + maat_state_free(&state); } TEST(maat_scan_ipv4, hit_ip_and_port) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG"); + int table_id = maat_table_get_id(g_maat_instance, "IP_PLUS_CONFIG"); char ip_str[32] = "192.168.58.19"; - uint16_t port = 20000; - struct addr_4tuple addr; - addr.type = IP_TYPE_V4; - int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip); + uint32_t sip; + int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); - addr.ipv4.sport = htons(port); int results[3] = {-1}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); + EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 7); + maat_state_free(&state); - port = 20001; - addr.ipv4.sport = htons(port); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); + EXPECT_EQ(n_hit_result, 0); + + maat_state_free(&state); } TEST(maat_scan_ipv4, hit_ip_and_port_range) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG"); + int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "192.168.50.24"; - uint16_t port = 1; - struct addr_4tuple addr; + struct addr_2tuple addr; addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip); EXPECT_EQ(ret, 1); - addr.ipv4.sport = htons(port); int results[3] = {-1}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); + EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 4); + maat_state_free(&state); - port = 40000; - addr.ipv4.sport = htons(port); memset(results, 0, sizeof(results)); - n_result = 0; + n_hit_result = 0; state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); + EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 4); + maat_state_free(&state); - port = 40001; - addr.ipv4.sport = htons(port); memset(results, 0, sizeof(results)); - n_result = 0; + n_hit_result = 0; state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); + EXPECT_EQ(n_hit_result, 0); } TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG"); + int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str1[32] = "10.0.1.20"; char ip_str2[32] = "10.0.1.25"; char ip_str3[32] = "10.0.1.26"; - uint16_t port1 = 1; - uint16_t port2 = 443; - struct addr_4tuple addr; + struct addr_2tuple addr; addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str1, &addr.ipv4.sip); EXPECT_EQ(ret, 1); - addr.ipv4.sport = htons(port1); int results[3] = {-1}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); + EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 8); + maat_state_free(&state); ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); EXPECT_EQ(ret, 1); - addr.ipv4.sport = htons(port2); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); + EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 8); + maat_state_free(&state); ret = inet_pton(AF_INET, ip_str3, &addr.ipv4.sip); EXPECT_EQ(ret, 1); - addr.ipv4.sport = htons(port2); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); + EXPECT_EQ(n_hit_result, 0); + maat_state_free(&state); } TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG"); + int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str1[32] = "192.168.0.1"; char ip_str2[32] = "192.168.0.0"; - uint16_t port = 5210; - struct addr_4tuple addr; + struct addr_2tuple addr; addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str1, &addr.ipv4.sip); EXPECT_EQ(ret, 1); - addr.ipv4.sport = htons(port); int results[3] = {-1}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); + EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 50); + maat_state_free(&state); ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); EXPECT_EQ(ret, 1); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); + EXPECT_EQ(n_hit_result, 0); + maat_state_free(&state); } TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG"); + int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "192.168.40.10"; - uint16_t port = 443; - struct addr_4tuple addr; + struct addr_2tuple addr; addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip); EXPECT_EQ(ret, 1); - addr.ipv4.sport = htons(port); int results[3] = {-1}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 2); + EXPECT_EQ(n_hit_result, 2); EXPECT_EQ(results[0], 63); EXPECT_EQ(results[1], 67); + maat_state_free(&state); - port = 442; - addr.ipv4.sport = htons(port); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); + EXPECT_EQ(n_hit_result, 0); + maat_state_free(&state); } TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG"); + int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "1001:da8:205:1::101"; - uint16_t port = 5210; - struct addr_4tuple addr; + struct addr_2tuple addr; addr.type = IP_TYPE_V6; int ret = inet_pton(AF_INET6, ip_str, &addr.ipv6.sip); EXPECT_EQ(ret, 1); - addr.ipv6.sport = htons(port); int results[3] = {-1}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); + EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 47); + maat_state_free(&state); - port = 442; - addr.ipv6.sport = htons(port); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); + EXPECT_EQ(n_hit_result, 0); + maat_state_free(&state); } TEST(maat_scan_string, dynamic_config) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "HTTP_URL"); char data[128] = "hello world"; int results[5] = {0}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); + EXPECT_EQ(n_hit_result, 0); + maat_state_free(&state); const char *table_name = "HTTP_URL"; const char *table_line = "9999\t8888\thello world\t0\t0\t0\t1\t"; @@ -273,30 +259,29 @@ TEST(maat_scan_string, dynamic_config) { sleep(2); state = NULL; - ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, &n_result, &state); + ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); + EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 9999); + maat_state_free(&state); } TEST(maat_scan_ip, dynamic_config) { - struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG"); + int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "10.0.6.201"; - uint16_t port = 443; - struct addr_4tuple addr; + struct addr_2tuple addr; addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip); EXPECT_EQ(ret, 1); - addr.ipv4.sport = htons(port); int results[3] = {-1}; - size_t n_result = 0; + size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 0); + EXPECT_EQ(n_hit_result, 0); + maat_state_free(&state); const char *table_name = "IP_PLUS_CONFIG"; const char *table_line = "9998\t8887\t4\trange\t10.0.6.201\t255.255.0.0\trange\t0\t65535\t6\t0\t1"; @@ -309,10 +294,11 @@ TEST(maat_scan_ip, dynamic_config) { sleep(2); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, &state); + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result, 1); + EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 9998); + maat_state_free(&state); } int count_line_num_cb(const char *table_name, const char *line, void *u_para) @@ -424,7 +410,8 @@ int main(int argc, char ** argv) maat_options_set_logger(opts, logger); g_maat_instance = maat_new(opts, table_info_path); - + maat_options_free(opts); + ret=RUN_ALL_TESTS(); log_handle_destroy(g_maat_instance->logger); diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index d37a7db..456bc85 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -2,7 +2,6 @@ #include "maat/maat.h" #include "maat_utils.h" #include "maat_rule.h" -#include "maat_table_schema.h" #include "json2iris.h" #include "maat_config_monitor.h" @@ -31,17 +30,19 @@ TEST(json_mode, maat_scan_string) { struct maat *maat_instance = maat_new(opts, table_info_path); EXPECT_NE(maat_instance, nullptr); - struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL"); - char data[128] = "i.ytimg.com"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + char scan_data[128] = "hello"; + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 30); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 191); + maat_options_free(opts); + maat_state_free(&state); maat_free(maat_instance); } @@ -72,17 +73,19 @@ TEST(iris_mode, maat_scan_string) { struct maat *maat_instance = maat_new(opts, table_info_path); EXPECT_NE(maat_instance, nullptr); - struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL"); - char data[128] = "i.ytimg.com"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + char scan_data[128] = "hello"; + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 30); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 191); + maat_options_free(opts); + maat_state_free(&state); maat_free(maat_instance); } @@ -190,17 +193,19 @@ TEST(redis_mode, maat_scan_string) { maat_options_set_logger(opts, logger); struct maat *maat_instance = maat_new(opts, table_info_path); - struct table_schema_manager *table_schema_mgr = maat_instance->table_schema_mgr; - int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr "HTTP_URL"); - char data[128] = "i.ytimg.com"; - int result_array[5] = {0}; - size_t n_result_array = 0; - int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + char scan_data[128] = "hello"; + int results[5] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 30); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 191); + maat_options_free(opts); + maat_state_free(&state); log_handle_destroy(maat_instance->logger); maat_free(maat_instance); } diff --git a/test/maat_json.json b/test/maat_json.json index 7b7bab2..94971d4 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -2283,6 +2283,32 @@ ] } ] + }, + { + "compile_id": 191, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "hello", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] } ], "plugin_table": [ diff --git a/test/table_info.conf b/test/table_info.conf index 0697fc9..a653496 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -1,50 +1,123 @@ [ { "table_id":1, - "table_name":"HTTP_URL", - "table_type":"expr", - "scan_mode":"block", - "item_id":1, - "group_id":2, + "table_name":"COMPILE", + "table_type":"compile", + "valid_column":8, "custom": { - "keywords":3, - "expr_type":4, - "match_method":5, - "is_hexbin":6, - "is_valid":7 + "compile_id":1, + "service_id":2, + "action":3, + "do_blacklist":4, + "do_log":5, + "tags":6, + "user_region":7, + "clause_num":9, + "evaluation_order":10 } - }, + }, { "table_id":2, - "table_name":"IP_PLUGIN_TABLE", - "table_type":"ip_plugin", - "item_id":1, + "table_name":"GROUP2COMPILE", + "table_type":"group2compile", + "valid_column":3, "custom": { - "ip_type":2, - "start_ip":3, - "end_ip":4, - "is_valid":5 + "associated_compile_table_id":1, + "group_id":1, + "compile_id":2, + "not_flag":4, + "virtual_table_name":5, + "clause_index":6 } }, { "table_id":3, - "table_name":"PLUGIN_TABLE", - "table_type":"plugin", - "item_id":1, + "table_name":"COMPILE_2", + "table_type":"compile", + "valid_column":8, "custom": { - "key":2, - "tag":3, - "is_valid":4, - "foreign":[6,8,10] + "compile_id":1, + "service_id":2, + "action":3, + "do_blacklist":4, + "do_log":5, + "tags":6, + "user_region":7, + "clause_num":9, + "evaluation_order":10 } }, { "table_id":4, + "table_name":"GROUP2COMPILE_2", + "table_type":"group2compile", + "associated_compile_table_id":3, + "valid_column":3, + "custom": { + "group_id":1, + "compile_id":2, + "not_flag":4, + "virtual_table_name":5, + "clause_index":6 + } + }, + { + "table_id":5, + "table_name":"GROUP2GROUP", + "table_type":"group2group", + "valid_column":3, + "custom": { + "group_id":1, + "superior_group_id":2 + } + }, + { + "table_id":6, + "table_name":"HTTP_URL", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":7, + "table_name":"IP_PLUGIN_TABLE", + "table_type":"ip_plugin", + "valid_column":5, + "custom": { + "item_id":1, + "ip_type":2, + "start_ip":3, + "end_ip":4 + } + }, + { + "table_id":8, + "table_name":"PLUGIN_TABLE", + "table_type":"plugin", + "valid_column":4, + "custom": { + "item_id":1, + "key":2, + "tag":3, + "foreign":[6,8,10] + } + }, + { + "table_id":9, "table_name":"IP_PLUS_CONFIG", "table_type":"ip_plus", - "item_id":1, - "group_id":2, + "valid_column":18, "custom": { + "item_id":1, + "group_id":2, "addr_type":3, "saddr_format":4, "sip1":5, @@ -59,108 +132,13 @@ "dport1":14, "dport2":15, "proto":16, - "direction":17, - "is_valid":18 - } - }, - { - "table_id":5, - "table_name":"COMPOSITION_IP_SOURCE", - "table_type":"virtual", - "physical_table":["IP_PLUS_CONFIG"] - }, - { - "table_id":6, - "table_name":"COMPOSITION_IP_DEST", - "table_type":"virtual", - "physical_table":["IP_PLUS_CONFIG"] - }, - { - "table_id":7, - "table_name":"COMPOSITION_IP_SESSION", - "table_type":"virtual", - "physical_table":["IP_PLUS_CONFIG"] - }, - { - "table_id":8, - "table_name":"COMPOSITION_IP", - "table_type":"composition", - "composition_table": { - "source": "COMPOSITION_IP_SOURCE", - "dest": "COMPOSITION_IP_DEST", - "session": "COMPOSITION_IP_SESSION" - } - }, - { - "table_id":9, - "table_name":"COMPILE_1", - "table_type":"compile", - "custom": { - "compile_id":1, - "service_id":2, - "action":3, - "do_blacklist":4, - "do_log":5, - "tags":6, - "user_region":7, - "is_valid":8, - "clause_num":9, - "evaluation_order":10 + "direction":17 } }, { "table_id":10, - "table_name":"GROUP2COMPILE_1", - "table_type":"group2compile", - "associated_compile_table_id":9, - "custom": { - "group_id":1, - "compile_id":2, - "is_valid":3, - "not_flag":4, - "virtual_table_name":5, - "clause_index":6 - } - }, - { - "table_id":11, - "table_name":"COMPILE_2", - "table_type":"compile", - "custom": { - "compile_id":1, - "service_id":2, - "action":3, - "do_blacklist":4, - "do_log":5, - "tags":6, - "user_region":7, - "is_valid":8, - "clause_num":9, - "evaluation_order":10 - } - }, - { - "table_id":12, - "table_name":"GROUP2COMPILE_2", - "table_type":"group2compile", - "associated_compile_table_id":11, - "custom": { - "group_id":1, - "compile_id":2, - "is_valid":3, - "not_flag":4, - "virtual_table_name":5, - "clause_index":6 - } - }, - { - "table_id":13, - "table_name":"GROUP2GROUP", - "table_type":"group2group", - "custom": { - "group_id":1, - "superior_group_id":2, - "is_valid":3 - } + "table_name":"VIRTUAL_IP_PLUS_TABLE", + "table_type":"virtual", + "physical_table": ["IP_PLUS_CONFIG"] } ] \ No newline at end of file