diff --git a/include/maat/maat.h b/include/maat/maat.h index 026942c..9b4894d 100644 --- a/include/maat/maat.h +++ b/include/maat/maat.h @@ -36,13 +36,28 @@ enum ip_type { IP_TYPE_V6 }; -typedef struct ip_data_s { +/* network order */ +struct ipv4_4tuple { + uint32_t sip; + uint32_t dip; + uint16_t sport; + uint16_t dport; +}; + +struct ipv6_4tuple { + uint8_t sip[16]; + uint8_t dip[16]; + uint16_t sport; + uint16_t dport; +}; + +struct addr_4tuple { enum ip_type type; union { - uint32_t ipv4; - uint32_t ipv6[4]; + struct ipv4_4tuple ipv4; + struct ipv6_4tuple ipv6; }; -} ip_data_t; +}; #define MAAT_RULE_UPDATE_TYPE_FULL 1 #define MAAT_RULE_UPDATE_TYPE_INC 2 @@ -69,7 +84,8 @@ int maat_options_set_iris_inc_index_dir(struct maat_options *opts, const char *i int maat_options_set_json_file(struct maat_options *opts, const char *json_filename); int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip); int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port); -int maat_options_set_redis_db_index(struct maat_options *opts, int db_index); +int maat_options_set_redis_db(struct maat_options *opts, int db_index); +int maat_options_set_logger(struct maat_options *opts, void *logger); /* maat_instance API */ struct maat *maat_new(struct maat_options *opts, const char *table_info_path); @@ -90,7 +106,7 @@ int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id, maat_plugin_ex_dup_func_t *dup_func, long argl, void *argp); -void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id, +void *maat_plugin_table_dup_ex_data(struct maat *instance, int table_id, const char *key, size_t key_len); /* maat scan API */ struct maat_state; @@ -99,7 +115,7 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id, struct maat_state *state); int maat_scan_ip(struct maat *instance, int table_id, int thread_id, - const ip_data_t *ip, int results[], size_t *n_result, + struct addr_4tuple *addr, int results[], size_t *n_result, struct maat_state *state); int maat_scan_string(struct maat *instance, int table_id, int thread_id, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a6c25d3..db6ad50 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,7 +13,7 @@ set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_command.cpp maat_redis_monitor.cpp) set(LIB_SOURCE_FILES - ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c) + ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c) include_directories(${PROJECT_SOURCE_DIR}/include/) include_directories(${PROJECT_SOURCE_DIR}/deps/) diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index bfeca07..ff73ef4 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -20,7 +20,8 @@ extern "C" int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, - char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo); + char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, + struct log_handle *logger); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_common.h b/src/inc_internal/maat_common.h index 61aa499..62c997f 100644 --- a/src/inc_internal/maat_common.h +++ b/src/inc_internal/maat_common.h @@ -18,14 +18,18 @@ extern "C" #include +#include "log/log.h" #include "maat_rule.h" struct maat_options { + char instance_name[NAME_MAX]; size_t nr_worker_threads; int rule_effect_interval_ms; int rule_update_checking_interval_ms; int gc_timeout_ms; int deferred_load_on; + int log_level; + struct log_handle *logger; enum data_source input_mode; union { struct source_iris_ctx iris_ctx; diff --git a/src/inc_internal/maat_config_monitor.h b/src/inc_internal/maat_config_monitor.h index eb3de29..5d17dbe 100644 --- a/src/inc_internal/maat_config_monitor.h +++ b/src/inc_internal/maat_config_monitor.h @@ -25,8 +25,8 @@ extern "C" void config_monitor_traverse(long long version, const char *idx_dir, void (*start_fn)(long long, int, void *), int (*update_fn)(const char *, const char *, void *), - void (*finish_fn)(void *), - void *u_param); + void (*finish_fn)(void *), void *u_param, + struct log_handle *logger); int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz); diff --git a/src/inc_internal/maat_ex_data.h b/src/inc_internal/maat_ex_data.h index 7eae24b..f269bca 100644 --- a/src/inc_internal/maat_ex_data.h +++ b/src/inc_internal/maat_ex_data.h @@ -18,6 +18,11 @@ extern "C" #include "rcu_hash.h" +struct ex_data_container { + void *ex_data; + void *custom_data; +}; + struct ex_data_runtime; /* ex_data_runtime API */ @@ -40,20 +45,24 @@ void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt); void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema); /* set user_ctx API */ -void ex_data_runtime_set_user_ctx(struct ex_data_runtime *ex_data_rt, void *user_ctx); +void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx); + +struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt); /* ex_data_runtime ex data API */ void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len); -void ex_data_runtime_add_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, void *data); +void ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct ex_data_container *ex_container); -void ex_data_runtime_del_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); +void ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct log_handle *logger); -void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); +size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container); -size_t ex_data_runtime_ex_data_count(struct ex_data_runtime *ex_data_rt); +void *ex_data_runtime_dup_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); -size_t ex_data_runtime_list_updating_ex_data(struct ex_data_runtime *ex_data_rt, void ***ex_data_array); +void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); + +size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt); int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt); diff --git a/src/inc_internal/maat_redis_monitor.h b/src/inc_internal/maat_redis_monitor.h index 7bb4800..971c86f 100644 --- a/src/inc_internal/maat_redis_monitor.h +++ b/src/inc_internal/maat_redis_monitor.h @@ -23,8 +23,7 @@ extern "C" void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, void (*start_fn)(long long, int, void *), int (*update_fn)(const char *, const char *, void *), - void (*finish_fn)(void *), - void *u_param); + void (*finish_fn)(void *), void *u_param); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 706d047..0c62dba 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -24,6 +24,7 @@ extern "C" #include #include +#include "log/log.h" #include "hiredis/hiredis.h" #include "uthash/uthash.h" #include "maat_table_schema.h" @@ -114,6 +115,7 @@ struct maat { struct source_redis_ctx mr_ctx; }; + struct log_handle *logger; int deferred_load; int is_running; @@ -159,7 +161,7 @@ void *rule_monitor_loop(void *arg); void maat_read_full_config(struct maat *maat_instance); /* maat command API for internal */ -redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db); +redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger); redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...); @@ -173,19 +175,24 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len); -int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time); +int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, + long long server_time, struct log_handle *logger); void maat_cmd_clear_rule_cache(struct serial_rule *s_rule); int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long *new_version, struct table_schema_manager* table_schema_mgr, - struct serial_rule **list, int *update_type, int cumulative_off); + struct serial_rule **list, int *update_type, int cumulative_off, + struct log_handle *logger); -int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process); +int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process, + struct log_handle *logger); -int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num, const char* dir); +int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num, + const char *dir, struct log_handle *logger); -void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn); +void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn, + struct log_handle *logger); void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule); diff --git a/src/inc_internal/maat_table_runtime.h b/src/inc_internal/maat_table_runtime.h index bc6db8c..56a67a4 100644 --- a/src/inc_internal/maat_table_runtime.h +++ b/src/inc_internal/maat_table_runtime.h @@ -16,16 +16,27 @@ extern "C" { #endif +#include "maat/maat.h" #include "maat_table_schema.h" #include "maat_garbage_collection.h" +struct table_rt_2tuple { + enum ip_type ip_type; + uint16_t port; + union { + uint32_t ipv4; + uint32_t ipv6[4]; + }; +}; + struct table_item; struct table_runtime; struct table_runtime_manager; /* table runtime manager API */ -struct table_runtime_manager *table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num, - struct maat_garbage_bin *bin); +struct table_runtime_manager * +table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num, + struct maat_garbage_bin *bin); void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr); @@ -36,7 +47,8 @@ size_t table_runtime_rule_count(struct table_runtime *table_rt); enum table_type table_runtime_get_type(struct table_runtime* table_rt); -void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, const char *line, struct table_item *table_item); +void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, + const char *line, struct table_item *table_item, struct log_handle *logger); /** * @brief if table_runtime is updating @@ -52,19 +64,21 @@ int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, con int results[], size_t *n_result); void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id); -int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, - int result[], size_t n_result); +int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result); void table_runtime_stream_close(struct table_runtime *table_rt); +int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, int results[], size_t *n_result); + /* table runtime cached row API */ size_t table_runtime_cached_row_count(struct table_runtime *table_rt); const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq); /* table runtime ex data API */ -void *table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema, const char *key, size_t key_len); +struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt); -void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema); +void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, + struct log_handle *logger); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_table_schema.h b/src/inc_internal/maat_table_schema.h index 1624bcd..e01363f 100644 --- a/src/inc_internal/maat_table_schema.h +++ b/src/inc_internal/maat_table_schema.h @@ -27,6 +27,13 @@ extern "C" #define MAX_KEYWORDS_STR 1024 #define MAX_FOREIGN_CLMN_NUM 8 +enum table_composition_type { + COMPOSITION_TYPE_SIP = 0, + COMPOSITION_TYPE_DIP, + COMPOSITION_TYPE_SESSION, + COMPOSITION_TYPE_MAX +}; + enum table_type { TABLE_TYPE_EXPR = 0, TABLE_TYPE_EXPR_PLUS, @@ -90,6 +97,35 @@ struct expr_item { int is_valid; }; +struct ipv4_item_rule { + uint32_t min_sip; /* 源地址下界;0表示忽略本字段 */ + uint32_t max_sip; /* 源地址上界;0表示固定IP=min_saddr */ + uint16_t min_sport; /* 源端口范围下界;0表示忽略本字段 */ + uint16_t max_sport; /* 源端口范围上界;0表示固定端口=min_sport */ + uint16_t proto; /* 传输层协议,6表示TCP,17表示UDP;0表示忽略本字段 */ + uint16_t direction; /* 方向,0表示双向,1表示单向 */ +}; + +struct ipv6_item_rule { + uint32_t min_sip[4]; /* 源地址下界;全0表示忽略本字段 */ + uint32_t max_sip[4]; /* 源地址上界;全0表示固定IP=min_saddr */ + uint16_t min_sport; /* 源端口范围下界;0表示忽略本字段 */ + uint16_t max_sport; /* 源端口范围上界;0表示固定端口=min_sport */ + uint16_t proto; /* 传输层协议,6表示TCP,17表示UDP,无限制默认为0 */ + uint16_t direction; /* 方向,0表示双向,1表示单向 */ +}; + +struct ip_plus_item { + int item_id; + int group_id; + int addr_type; + union { + struct ipv4_item_rule ipv4; + struct ipv6_item_rule ipv6; + }; + int is_valid; +}; + struct plugin_item { char key[MAX_KEYWORDS_STR]; size_t key_len; @@ -111,6 +147,7 @@ struct table_item { enum table_type table_type; union { struct expr_item expr_item; + struct ip_plus_item ip_plus_item; struct plugin_item plugin_item; struct ip_plugin_item ip_plugin_item; }; @@ -139,10 +176,17 @@ struct table_schema; struct table_schema_manager; /* table schema manager API */ -struct table_schema_manager *table_schema_manager_create(const char *table_info_path); +struct table_schema_manager *table_schema_manager_create(const char *table_info_path, struct log_handle *logger); void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr); int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name); + +/** + * @brief get composition table's child table(specified by type) id +*/ +int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id, + enum table_composition_type type); + enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id); size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_mgr); @@ -162,7 +206,8 @@ int table_schema_get_table_id(struct table_schema *table_schema); enum scan_type table_schema_get_scan_type(struct table_schema *table_schema); -struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema); +struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema, + struct log_handle *logger); int table_schema_get_valid_flag_column(struct table_schema *table_schema); diff --git a/src/inc_internal/maat_utils.h b/src/inc_internal/maat_utils.h index 69d9298..4f99bc8 100644 --- a/src/inc_internal/maat_utils.h +++ b/src/inc_internal/maat_utils.h @@ -18,6 +18,7 @@ extern "C" #include #include +#include #define TRUE 1 #define FALSE 0 @@ -30,8 +31,28 @@ extern "C" #define MIN(a, b) (((a) < (b)) ? (a) : (b)) #endif +enum maat_ip_format { + IP_FORMAT_RANGE, + IP_FORMAT_MASK, + IP_FORMAT_CIDR, + IP_FORMAT_UNKNOWN +}; + +enum maat_ip_format ip_format_str2int(const char *format); +int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, const char *ip2, + uint32_t range_begin[], uint32_t range_end[]); + +inline void ipv6_ntoh(unsigned int *v6_addr) +{ + for (unsigned int i = 0; i < 4; i++) { + v6_addr[i] = ntohl(v6_addr[i]); + } +} + #define UNUSED __attribute__((unused)) +const char *module_name_str(const char *name); + char *maat_strdup(const char *s); int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len); diff --git a/src/inc_internal/rcu_hash.h b/src/inc_internal/rcu_hash.h index dfb8255..b5186bb 100644 --- a/src/inc_internal/rcu_hash.h +++ b/src/inc_internal/rcu_hash.h @@ -29,6 +29,8 @@ void rcu_hash_free(struct rcu_hash_table *htable); void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx); +void *rcu_hash_get_user_ctx(struct rcu_hash_table *htable); + /** * @brief just means add to the updating nodes * after call rcu_hash_commit, they become effective nodes diff --git a/src/json2iris.cpp b/src/json2iris.cpp index 005a958..475bf74 100644 --- a/src/json2iris.cpp +++ b/src/json2iris.cpp @@ -24,6 +24,8 @@ #include "maat_rule.h" #include "uthash/uthash.h" +#define MODULE_JSON2IRIS module_name_str("maat.json2iris") + #define MAX_COLUMN_NUM 32 #define mr_region_id_var "SEQUENCE_REGION" @@ -219,22 +221,24 @@ int create_tmp_dir(struct iris_description *p) return 0; } -int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_description *p_iris) +int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_description *p_iris, + struct log_handle *logger) { cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name"); if (NULL == item || item->type != cJSON_String) { - fprintf(stderr, "The %d plugin_table's table_name not defined or format error\n", sequence); + log_error(logger, MODULE_JSON2IRIS, + "The %d plugin_table's table_name not defined or format error", sequence); return -1; } const char *table_name = item->valuestring; cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content"); if (NULL == table_content || table_content->type != cJSON_Array) { - fprintf(stderr, "%d plugin_table's table_content not defined or format error\n", sequence); + log_error(logger, MODULE_JSON2IRIS, + "%d plugin_table's table_content not defined or format error", sequence); return -1; } int line_cnt = cJSON_GetArraySize(table_content); - struct iris_table *table_info = query_table_info(p_iris, table_name, TABLE_TYPE_PLUGIN); cJSON *each_line = NULL; @@ -242,7 +246,8 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript for (int i = 0; i < line_cnt; i++) { each_line = cJSON_GetArrayItem(table_content, i); if (NULL == each_line || each_line->type != cJSON_String) { - fprintf(stderr, "plugin_table %s's line %d format error\n", table_info->table_name, i+1); + log_error(logger, MODULE_JSON2IRIS, "plugin_table %s's line %d format error", + table_info->table_name, i + 1); continue; } @@ -314,7 +319,8 @@ static int get_region_seq(struct iris_description *iris_cfg) return sequence; } -int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct translate_command *cmd, int cmd_cnt, struct iris_table *table) +int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct translate_command *cmd, int cmd_cnt, + struct iris_table *table, struct log_handle *logger) { int i = 0; int ret = -1; @@ -331,7 +337,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla } if (NULL == item || item->type != cmd[i].json_type) { - fprintf(stderr, "%s not defined or wrong format\n", cmd[i].json_string); + log_error(logger, MODULE_JSON2IRIS, "%s not defined or wrong format", cmd[i].json_string); ret = -1; goto error_out; } @@ -341,7 +347,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla char *p = item->valuestring; ret = maat_kv_read(str2int, p, &int_value); if (ret < 0) { - fprintf(stderr, "%s's value %s is not valid format\n", cmd[i].json_string, p); + log_error(logger, MODULE_JSON2IRIS, "%s's value %s is not valid format", cmd[i].json_string, p); FREE(p); ret = -1; goto error_out; @@ -380,7 +386,8 @@ error_out: return ret; } -int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, + struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -423,10 +430,11 @@ int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_ip_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +int write_ip_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, + struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -510,10 +518,11 @@ int write_ip_line(cJSON *region_json, struct iris_description *p_iris, struct ir json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, + struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -621,10 +630,11 @@ int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, stru json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, + struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -656,10 +666,11 @@ int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struc json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, + struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -689,10 +700,11 @@ int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struc json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_similar_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +int write_similar_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, + struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -718,22 +730,24 @@ int write_similar_line(cJSON *region_json, struct iris_description *p_iris, stru json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); } -int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_description *p_iris) +int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_description *p_iris, + struct log_handle *logger) { cJSON *item = cJSON_GetObjectItem(region_json, "table_name"); if (NULL == item || item->type != cJSON_String) { - fprintf(stderr, "compile rule %d's table_name not defined or format error\n", compile_id); + log_error(logger, MODULE_JSON2IRIS, "compile rule %d's table_name not defined or format error", + compile_id); return -1; } const char *table_name = item->valuestring; item = cJSON_GetObjectItem(region_json, "table_type"); if (NULL == item || item->type != cJSON_String) { - fprintf(stderr, "compile rule %d's table name %s's table_type not defined or format error\n", - compile_id, table_name); + log_error(logger, MODULE_JSON2IRIS, + "compile rule %d's table name %s's table_type not defined or format error", compile_id, table_name); return -1; } @@ -741,15 +755,15 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i enum table_type table_type = TABLE_TYPE_EXPR; int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type)); if (ret != 1) { - fprintf(stderr, "compile rule %d table name %s's table_type %s invalid\n", - compile_id, table_name, table_type_str); + log_error(logger, MODULE_JSON2IRIS, "compile rule %d table name %s's table_type %s invalid", + compile_id, table_name, table_type_str); return -1; } cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content"); if (NULL == table_content || table_content->type != cJSON_Object) { - fprintf(stderr, "compile rule %d table name %s's table_content not defined or format error\n", - compile_id, table_name); + log_error(logger, MODULE_JSON2IRIS, + "compile rule %d table name %s's table_content not defined or format error", compile_id, table_name); return -1; } @@ -763,23 +777,23 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - ret = write_expr_line(table_content, p_iris, table_info); + ret = write_expr_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_IP: - ret = write_ip_line(table_content, p_iris, table_info); + ret = write_ip_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_IP_PLUS: - ret = write_ip_plus_line(table_content, p_iris, table_info); + ret = write_ip_plus_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_INTERVAL: case TABLE_TYPE_INTERVAL_PLUS: - ret = write_intval_line(table_content, p_iris, table_info); + ret = write_intval_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_DIGEST: - ret = write_digest_line(table_content, p_iris, table_info); + ret = write_digest_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_SIMILARITY: - ret = write_similar_line(table_content, p_iris, table_info); + ret = write_similar_line(table_content, p_iris, table_info, logger); break; default: assert(0); @@ -814,7 +828,7 @@ int write_group2group_line(int group_id, int superior_group_id, struct iris_desc } int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, - int Nth_group, struct iris_description *p_iris) + int Nth_group, struct iris_description *p_iris, struct log_handle *logger) { int ret = 0; int group_not_flag = 0; @@ -863,21 +877,22 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac if (region_json != NULL) { cJSON *region_rule = NULL; cJSON_ArrayForEach(region_rule, region_json) { - ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris); + ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris, logger); if (ret < 0) { - fprintf(stderr, "compile rule %d write region error\n", tracking_compile_id); + log_error(logger, MODULE_JSON2IRIS, "compile rule %d write region error", + tracking_compile_id); return -1; } } } - cJSON *sub_groups = cJSON_GetObjectItem(group_json,"sub_groups"); + cJSON *sub_groups = cJSON_GetObjectItem(group_json, "sub_groups"); if (sub_groups != NULL) { //recursively int i = 0; cJSON_ArrayForEach(item, sub_groups) { i++; - ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris); + ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris, logger); if (ret < 0) { return -1; } @@ -885,8 +900,9 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac } if (NULL == region_json && NULL == sub_groups) { - fprintf(stdout, "A group of compile rule %d has neither regions, sub groups, nor refered another exisited group\n", - tracking_compile_id); + log_info(logger, MODULE_JSON2IRIS, + "A group of compile rule %d has neither regions, sub groups, nor refered another exisited group", + tracking_compile_id); } } @@ -897,18 +913,18 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac } if (ret < 0) { - fprintf(stderr, "%s rule %d write group error\n", str_parent_type[parent_type], parent_id); + log_error(logger, MODULE_JSON2IRIS, "%s rule %d write group error", str_parent_type[parent_type], parent_id); return -1; } return 0; } -int write_compile_line(cJSON *compile, struct iris_description *p_iris) +int write_compile_line(cJSON *compile, struct iris_description *p_iris, struct log_handle *logger) { cJSON *item=cJSON_GetObjectItem(compile, "compile_id"); if (item->type != cJSON_Number) { - fprintf(stderr, "compile_id format not number\n"); + log_error(logger, MODULE_JSON2IRIS, "compile_id format not number"); return -1; } int compile_id = item->valueint; @@ -1002,7 +1018,7 @@ int write_compile_line(cJSON *compile, struct iris_description *p_iris) table_info = query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE); } - int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, table_info); + int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, table_info, logger); if (ret < 0) { return -1; } @@ -1041,11 +1057,12 @@ void write_table_idx(struct iris_description *p_iris, struct iris_table *table) FREE(buff); } -int write_index_file(struct iris_description *p_iris) +int write_index_file(struct iris_description *p_iris, struct log_handle *logger) { p_iris->idx_fp = fopen(p_iris->index_path, "w"); if (NULL == p_iris->idx_fp) { - fprintf(stderr, "index file %s fopen error %s\n",p_iris->index_path, strerror(errno)); + log_error(logger, MODULE_JSON2IRIS, "index file %s fopen error %s", + p_iris->index_path, strerror(errno)); return -1; } @@ -1061,7 +1078,7 @@ int write_index_file(struct iris_description *p_iris) return 0; } -int write_iris(cJSON *json, struct iris_description *p_iris) +int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *logger) { int i=0; int ret=0; @@ -1071,7 +1088,7 @@ int write_iris(cJSON *json, struct iris_description *p_iris) if (plug_tables != NULL) { cJSON *each_plug_table = NULL; cJSON_ArrayForEach(each_plug_table, plug_tables) { - write_plugin_line(each_plug_table, i, p_iris); + write_plugin_line(each_plug_table, i, p_iris, logger); i++; } } @@ -1093,7 +1110,7 @@ int write_iris(cJSON *json, struct iris_description *p_iris) parent_group = group_info_add_unsafe(p_iris, parent_group_name); } - ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris); + ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris, logger); if (ret < 0) { return -1; } @@ -1109,28 +1126,28 @@ int write_iris(cJSON *json, struct iris_description *p_iris) if (compile_cnt > 0) { cJSON *compile_obj = NULL; cJSON_ArrayForEach(compile_obj, compile_array) { - int compile_id = write_compile_line(compile_obj,p_iris); + int compile_id = write_compile_line(compile_obj, p_iris, logger); if (compile_id < 0) { - fprintf(stderr, "In %d compile rule\n", i); + log_error(logger, MODULE_JSON2IRIS, "In %d compile rule", i); return -1; } group_array = cJSON_GetObjectItem(compile_obj, "groups"); if (NULL == group_array) { - fprintf(stderr, "compile rule %d have no group\n",compile_id); + log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no group", compile_id); return -1; } int group_cnt = cJSON_GetArraySize(group_array); if (group_cnt <= 0) { - fprintf(stderr, "compile rule %d have no groups\n", compile_id); + log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no groups", compile_id); return -1; } i = 0; cJSON *group_obj = NULL; cJSON_ArrayForEach(group_obj, group_array) { - ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris); + ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris, logger); if (ret < 0) { return -1; } @@ -1139,7 +1156,7 @@ int write_iris(cJSON *json, struct iris_description *p_iris) } } - ret = write_index_file(p_iris); + ret = write_index_file(p_iris, logger); if (ret < 0) { return -1; } @@ -1148,7 +1165,8 @@ int write_iris(cJSON *json, struct iris_description *p_iris) int json2iris(const char *json_buff, const char *json_filename, const char *compile_tn, const char *group2compile_tn, const char *group2group_tn, redisContext *redis_write_ctx, - char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo) + char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, + struct log_handle *logger) { int ret = -1; cJSON *tmp_obj = NULL; @@ -1158,7 +1176,7 @@ int json2iris(const char *json_buff, const char *json_filename, const char *comp cJSON *json = cJSON_Parse(json_buff); if (!json) { - fprintf(stderr, "Error before: %-200.200s\n", cJSON_GetErrorPtr()); + log_error(logger, MODULE_JSON2IRIS, "Error before: %-200.200s", cJSON_GetErrorPtr()); goto error_out; } @@ -1186,11 +1204,11 @@ int json2iris(const char *json_buff, const char *json_filename, const char *comp ret = create_tmp_dir(&iris_cfg); if (ret < 0) { - fprintf(stderr, "create tmp folder %s error\n", iris_cfg.tmp_iris_dir); + log_error(logger, MODULE_JSON2IRIS, "create tmp folder %s error", iris_cfg.tmp_iris_dir); goto error_out; } - ret = write_iris(json, &iris_cfg); + ret = write_iris(json, &iris_cfg, logger); if (ret < 0) { goto error_out; } diff --git a/src/maat_api.cpp b/src/maat_api.cpp index bba2dc8..88d39ff 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -10,19 +10,24 @@ #include #include +#include #include "utils.h" +#include "maat_utils.h" #include "json2iris.h" #include "maat/maat.h" #include "maat_rule.h" #include "maat_common.h" #include "maat_kv.h" #include "maat_command.h" +#include "maat_ex_data.h" #include "maat_table_schema.h" #include "maat_table_runtime.h" #include "maat_config_monitor.h" #include "maat_redis_monitor.h" +#define MODULE_MAAT_API module_name_str("maat.api") + struct maat_options* maat_options_new(void) { struct maat_options *options = ALLOC(struct maat_options, 1); @@ -33,6 +38,7 @@ struct maat_options* maat_options_new(void) options->rule_update_checking_interval_ms = 1 * 1000; options->gc_timeout_ms = 10 * 1000; options->input_mode = DATA_SOURCE_NONE; + options->log_level = 0; return options; } @@ -65,6 +71,13 @@ int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms) return 0; } +int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len) +{ + memcpy(opts->instance_name, instance_name, name_len); + + return 0; +} + int maat_options_set_deferred_load_on(struct maat_options *opts) { opts->deferred_load_on = 1; @@ -119,13 +132,20 @@ int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port) return 0; } -int maat_options_set_redis_db_index(struct maat_options *opts, int db_index) +int maat_options_set_redis_db(struct maat_options *opts, int db_index) { opts->redis_ctx.redis_db = db_index; return 0; } +int maat_options_set_logger(struct maat_options *opts, void *logger) +{ + opts->logger = (struct log_handle *)logger; + + return 0; +} + void maat_read_full_config(struct maat *maat_instance) { int ret = -1; @@ -135,9 +155,10 @@ void maat_read_full_config(struct maat *maat_instance) switch (maat_instance->input_mode) { case DATA_SOURCE_REDIS: mr_ctx = &(maat_instance->mr_ctx); - fprintf(stdout, "Maat initiate from Redis %s:%hu db%d\n", - mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); - mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); + log_info(maat_instance->logger, MODULE_MAAT_API, "Maat initiate from Redis %s:%hu db%d", + mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); + mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, + mr_ctx->redis_db, maat_instance->logger); if (mr_ctx->read_ctx != NULL) { redis_monitor_traverse(maat_instance->maat_version, mr_ctx, maat_start_cb, maat_update_cb, maat_finish_cb, @@ -145,34 +166,37 @@ void maat_read_full_config(struct maat *maat_instance) } if (NULL == maat_instance->creating_maat_rt) { - fprintf(stderr, "At initiation: NO effective rule in redis %s:%hu db%d\n", - mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); + log_error(maat_instance->logger, MODULE_MAAT_API, + "At initiation: NO effective rule in redis %s:%hu db%d", + mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); } break; case DATA_SOURCE_IRIS_FILE: config_monitor_traverse(maat_instance->maat_version, maat_instance->iris_ctx.full_idx_dir, maat_start_cb, maat_update_cb, maat_finish_cb, - maat_instance); + maat_instance, maat_instance->logger); if (NULL == maat_instance->creating_maat_rt) { - fprintf(stderr, "At initiation: NO effective rule in %s\n", - maat_instance->iris_ctx.full_idx_dir); + log_error(maat_instance->logger, MODULE_MAAT_API, + "At initiation: NO effective rule in %s", + maat_instance->iris_ctx.full_idx_dir); } break; case DATA_SOURCE_JSON_FILE: ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); if (ret < 0) { - fprintf(stderr, "Maat re-initiate with JSON file %s failed: %s\n", maat_instance->json_ctx.json_file, err_str); + log_error(maat_instance->logger, MODULE_MAAT_API, "Maat re-initiate with JSON file %s failed: %s", + maat_instance->json_ctx.json_file, err_str); return; } config_monitor_traverse(maat_instance->maat_version, maat_instance->json_ctx.iris_file, maat_start_cb, maat_update_cb, maat_finish_cb, - maat_instance); + maat_instance, maat_instance->logger); if (NULL == maat_instance->creating_maat_rt) { - fprintf(stderr, "At initiation: NO effective rule in %s\n", - maat_instance->json_ctx.iris_file); + log_error(maat_instance->logger, MODULE_MAAT_API, "At initiation: NO effective rule in %s", + maat_instance->json_ctx.iris_file); } break; default: @@ -197,7 +221,19 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) int garbage_gc_timeout_s = 0; struct maat *maat_instance = ALLOC(struct maat, 1); - maat_instance->table_schema_mgr = table_schema_manager_create(table_info_path); + if (opts->logger != NULL) { + maat_instance->logger = opts->logger; + } else { + char log_path[1024] = {0}; + if (strlen(maat_instance->instance_name) > 0) { + snprintf(log_path, sizeof(log_path), "%s.log", maat_instance->instance_name); + } else { + snprintf(log_path, sizeof(log_path), "maat.log"); + } + maat_instance->logger = log_handle_create(log_path, opts->log_level); + } + + maat_instance->table_schema_mgr = table_schema_manager_create(table_info_path, maat_instance->logger); if (NULL == maat_instance->table_schema_mgr) { goto failed; } @@ -217,7 +253,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file)); break; default: - fprintf(stderr, "data source unsupported:%d\n", maat_instance->input_mode); + log_error(maat_instance->logger, MODULE_MAAT_API, "data source unsupported:%d", maat_instance->input_mode); goto failed; } @@ -330,14 +366,14 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i struct table_runtime *table_rt = NULL; if (maat_instance->maat_rt != NULL) { table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id); - table_runtime_commit_ex_data_schema(table_rt, table_schema); + table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->logger); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); return 0; } -void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, +void *maat_plugin_table_dup_ex_data(struct maat *maat_instance, int table_id, const char *key, size_t key_len) { struct maat_runtime *maat_rt = maat_instance->maat_rt; @@ -346,9 +382,21 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, } struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); - struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); + if (NULL == table_schema) { + return NULL; + } - return table_runtime_get_ex_data(table_rt, table_schema, key, key_len); + struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); + if (NULL == table_rt) { + return NULL; + } + + struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt); + if (NULL == ex_data_rt) { + return NULL; + } + + return ex_data_runtime_dup_ex_data(ex_data_rt, key, key_len); } int maat_scan_integer(struct maat *instance, int table_id, int thread_id, @@ -358,27 +406,141 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id, return 0; } -int maat_scan_ip(struct maat *instance, int table_id, int thread_id, - const struct ip_data *ip, int results[], size_t n_result, - struct maat_state *state) +static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4tuple *addr, enum table_composition_type child_type) { + switch (addr->type) { + case IP_TYPE_V4: + switch (child_type) { + case COMPOSITION_TYPE_SIP: + table_rt_addr->ipv4 = ntohl(addr->ipv4.sip); + table_rt_addr->port = ntohs(addr->ipv4.sport); + break; + case COMPOSITION_TYPE_DIP: + table_rt_addr->ipv4 = ntohl(addr->ipv4.dip); + table_rt_addr->port = ntohs(addr->ipv4.dport); + break; + default: + assert(0); + return -1; + } + break; + case IP_TYPE_V6: + switch (child_type) { + case COMPOSITION_TYPE_SIP: + memcpy(table_rt_addr->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip)); + ipv6_ntoh(table_rt_addr->ipv6); + table_rt_addr->port = ntohs(addr->ipv6.sport); + break; + case COMPOSITION_TYPE_DIP: + memcpy(table_rt_addr->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip)); + ipv6_ntoh(table_rt_addr->ipv6); + table_rt_addr->port = ntohs(addr->ipv6.dport); + break; + default: + assert(0); + return -1; + } + break; + default: + assert(0); + return -1; + } + return 0; } +static int ip_composition_scan(int thread_id, struct addr_4tuple *addr, + int parent_table_id, enum table_composition_type child_type, + int *virtual_table_id, + struct table_schema_manager *table_schema_mgr, + struct table_runtime_manager *table_rt_mgr, int results[], size_t *n_result) +{ + int child_table_id = 0; + + if (child_type == COMPOSITION_TYPE_MAX) { + child_table_id = parent_table_id; + child_type = COMPOSITION_TYPE_SIP; + } else { + child_table_id = table_schema_manager_get_child_table_id(table_schema_mgr, parent_table_id, child_type); + } + + if (child_table_id < 0) { + return 0; + } + + struct table_schema *real_table = table_schema_get_by_scan_type(table_schema_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id); + if (NULL == real_table) { + return 0; + } + + enum table_type table_type = table_schema_get_table_type(real_table); + if (table_type != TABLE_TYPE_IP_PLUS) { + return -1; + } + + int table_id = table_schema_get_table_id(real_table); + struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); + size_t rule_num = table_runtime_rule_count(table_rt); + if (0 == rule_num) { + return 0; + } + + struct table_rt_2tuple scan_data; + memset(&scan_data, 0, sizeof(struct table_rt_2tuple)); + + ip_scan_data_set(&scan_data, addr, child_type); + + return table_runtime_scan_ip(table_rt, thread_id, &scan_data, results, n_result); +} + +int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_4tuple *addr, + int results[], size_t *n_result, struct maat_state *state) +{ + if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || + (thread_id < 0) || (NULL == addr) || (NULL == results) || (NULL == n_result)) { + return -1; + } + + struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr; + struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); + enum table_type table_type = table_runtime_get_type(table_rt); + + int region_ret = 0; + int virtual_table_id = 0; + + if (table_type == TABLE_TYPE_COMPOSITION) { + /* + enum table_composition_type childs[3] = {COMPOSITION_TYPE_SIP, COMPOSITION_TYPE_DIP, COMPOSITION_TYPE_SESSION}; + for (int i = 0; i < 3; i++) { + region_ret = ip_composition_scan(table_rt, thread_id, ip, table_id, childs[i], maat_instance->table_schema_mgr, + maat_instance->maat_rt->table_rt_mgr, results, n_result); + }*/ + region_ret = ip_composition_scan(thread_id, addr, table_id, COMPOSITION_TYPE_SIP, &virtual_table_id, + maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, + results, n_result); + } else { + region_ret = ip_composition_scan(thread_id, addr, table_id, COMPOSITION_TYPE_MAX, &virtual_table_id, + maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, + results, n_result); + } + + return region_ret; +} + int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, - const char *data, size_t data_len, int results[], size_t *n_results, + const char *data, size_t data_len, int results[], size_t *n_result, struct maat_state *state) { if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || (thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) || - (NULL == n_results)) { + (NULL == n_result)) { return -1; } struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr; struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); - return table_runtime_scan_string(table_rt, thread_id, data, data_len, results, n_results); + return table_runtime_scan_string(table_rt, thread_id, data, data_len, results, n_result); } struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id) diff --git a/src/maat_command.cpp b/src/maat_command.cpp index 92b71aa..ba51636 100644 --- a/src/maat_command.cpp +++ b/src/maat_command.cpp @@ -21,6 +21,8 @@ #include "maat_config_monitor.h" #include "maat_table_schema.h" +#define MODULE_MAAT_COMMAND module_name_str("maat.command") + extern const char *foreign_source_prefix; extern const char *mr_key_prefix; @@ -51,7 +53,7 @@ redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ... return (redisReply *)reply; } -redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db) +redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger) { struct timeval connect_timeout; connect_timeout.tv_sec = 0; @@ -59,8 +61,8 @@ redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int r redisContext *c = redisConnectWithTimeout(redis_ip, redis_port, connect_timeout); if (NULL == c || c->err) { - fprintf(stderr, "Unable to connect redis server %s:%d db%d, error: %s\n", - redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr); + log_error(logger, MODULE_MAAT_COMMAND, "Unable to connect redis server %s:%d db%d, error: %s", + redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr); if (c != NULL) { redisFree(c); @@ -109,10 +111,10 @@ void maat_cmd_clear_rule_cache(struct serial_rule *s_rule) memset(s_rule, 0, sizeof(struct serial_rule)); } -int connect_redis_for_write(struct source_redis_ctx *mr_ctx) +int connect_redis_for_write(struct source_redis_ctx *mr_ctx, struct log_handle *logger) { assert(mr_ctx->write_ctx == NULL); - mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); + mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, logger); if (NULL == mr_ctx->write_ctx) { return -1; } else { @@ -123,7 +125,7 @@ int connect_redis_for_write(struct source_redis_ctx *mr_ctx) redisContext *get_redis_ctx_for_write(struct maat *maat_instance) { if (NULL == maat_instance->mr_ctx.write_ctx) { - int ret = connect_redis_for_write(&(maat_instance->mr_ctx)); + int ret = connect_redis_for_write(&(maat_instance->mr_ctx), maat_instance->logger); if(ret!=0) { return NULL; @@ -302,7 +304,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name); if (table_id < 0) { - fprintf(stderr, "Command set line id %d failed: unknown table %s\n", line_rule->rule_id, line_rule->table_name); + log_error(maat_instance->logger, MODULE_MAAT_COMMAND, "Command set line id %d failed: unknown table %s", + line_rule->rule_id, line_rule->table_name); FREE(s_rule); return -1; } @@ -315,8 +318,9 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li int valid_flag_column = table_schema_get_valid_flag_column(table_schema); if (valid_flag_column < 0) { - fprintf(stderr, "Command set line id %d failed: table %s is not a plugin or ip_plugin table\n", - line_rule->rule_id, line_rule->table_name); + log_error(maat_instance->logger, MODULE_MAAT_COMMAND, + "Command set line id %d failed: table %s is not a plugin or ip_plugin table", + line_rule->rule_id, line_rule->table_name); FREE(s_rule); return -1; } @@ -331,7 +335,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name, line_rule->table_line, absolute_expire_time); - int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time); + int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_instance->logger); if (success_cnt != 1) { ret = -1; goto error_out; diff --git a/src/maat_config_monitor.cpp b/src/maat_config_monitor.cpp index 71f37c8..ce3bd1a 100644 --- a/src/maat_config_monitor.cpp +++ b/src/maat_config_monitor.cpp @@ -22,6 +22,8 @@ #include "json2iris.h" #include "maat_config_monitor.h" +#define MODULE_CONFIG_MONITOR module_name_str("maat.config_monitor") + #define CM_MAX_TABLE_NUM 256 #define MAX_CONFIG_LINE (1024 * 16) @@ -215,14 +217,15 @@ int filter_fn(const struct dirent *ent) strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0); } -int get_new_idx_path(long long current_version, const char *file_dir, char ***idx_path, size_t *idx_num) +int get_new_idx_path(long long current_version, const char *file_dir, + char ***idx_path, size_t *idx_num, struct log_handle *logger) { struct dirent **namelist = NULL; int update_type = CM_UPDATE_TYPE_NONE; int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort); if (n < 0) { - //log_error("scan dir error"); + log_error(logger, MODULE_CONFIG_MONITOR, "scan dir error"); return update_type; } @@ -241,15 +244,17 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id } if (strlen(namelist[i]->d_name) > 42) { - fprintf(stderr, "config file %s filename too long, should like full_config_index.000000000001\n", - namelist[i]->d_name); + log_error(logger, MODULE_CONFIG_MONITOR, + "config file %s filename too long, should like full_config_index.000000000001", + namelist[i]->d_name); continue; } int ret = sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld", update_str, &config_seq); if (ret != 2) { - fprintf(stderr, "config file %s filename error, should like full_config_index.000000000001\n", - namelist[i]->d_name); + log_error(logger, MODULE_CONFIG_MONITOR, + "config file %s filename error, should like full_config_index.000000000001", + namelist[i]->d_name); continue; } @@ -267,7 +272,8 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id } } } else { - fprintf(stderr, "config file %s, not full or inc config\n", namelist[i]->d_name); + log_error(logger, MODULE_CONFIG_MONITOR, "config file %s, not full or inc config", + namelist[i]->d_name); } } @@ -306,8 +312,8 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id void config_monitor_traverse(long long current_version, const char *idx_dir, void (*start_fn)(long long, int, void *), int (*update_fn)(const char *, const char *, void *), - void (*finish_fn)(void *), - void *u_param) + void (*finish_fn)(void *), void *u_param, + struct log_handle *logger) { size_t i = 0; long long new_version = 0; @@ -317,12 +323,13 @@ void config_monitor_traverse(long long current_version, const char *idx_dir, memset(table_array, 0, sizeof(table_array)); - int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num); + int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num, logger); if (update_type != CM_UPDATE_TYPE_NONE) { for (i = 0; i < idx_path_num; i++) { + log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]); int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM); if (table_num < 0) { - fprintf(stderr, "load %s failed, abandon update\n", idx_path_array[i]); + log_error(logger, MODULE_CONFIG_MONITOR, "load %s failed, abandon update", idx_path_array[i]); break; } @@ -350,7 +357,8 @@ void config_monitor_traverse(long long current_version, const char *idx_dir, FREE(idx_path_array); } -int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz) +int load_maat_json_file(struct maat *maat_instance, const char *json_filename, + char *err_str, size_t err_str_sz) { int ret = 0; struct stat fstat_buf; @@ -361,13 +369,16 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c size_t decrypted_buff_sz = 0; size_t uncompressed_buff_sz = 0; - fprintf(stdout, "Maat initial with JSON file %s, formating...\n", json_filename); + log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, + "Maat initial with JSON file %s, formating...", json_filename); if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) { ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo, - (unsigned char **)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz); + (unsigned char **)&decrypted_buff, &decrypted_buff_sz, + err_str, err_str_sz); if (ret < 0) { - fprintf(stderr, "Decrypt Maat JSON file %s failed\n", json_filename); + log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, + "Decrypt Maat JSON file %s failed", json_filename); return -1; } @@ -379,7 +390,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz); FREE(json_buff); if (ret < 0) { - fprintf(stderr, "Uncompress Maat JSON file %s failed\n", json_filename); + log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, + "Uncompress Maat JSON file %s failed", json_filename); return -1; } @@ -391,18 +403,18 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c if (NULL == json_buff) { ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz); if (ret < 0) { - fprintf(stderr, "Read Maat JSON file %s failed\n", json_filename); + log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, + "Read Maat JSON file %s failed", json_filename); return -1; } } - ret = json2iris((const char*)json_buff, json_filename, - maat_instance->compile_tn, maat_instance->group2compile_tn, maat_instance->group2group_tn, - NULL, - maat_instance->json_ctx.iris_file, - sizeof(maat_instance->json_ctx.iris_file), - strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, - strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL); + ret = json2iris((const char*)json_buff, json_filename, maat_instance->compile_tn, + maat_instance->group2compile_tn, maat_instance->group2group_tn, NULL, + maat_instance->json_ctx.iris_file, sizeof(maat_instance->json_ctx.iris_file), + strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, + strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL, + maat_instance->logger); FREE(json_buff); json_buff = NULL; @@ -418,10 +430,10 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim; md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5); - fprintf(stdout, "JSON file %s md5: %s, generate index file %s OK\n", - maat_instance->json_ctx.json_file, - maat_instance->json_ctx.effective_json_md5, - maat_instance->json_ctx.iris_file); + log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, "JSON file %s md5: %s, generate index file %s OK", + maat_instance->json_ctx.json_file, + maat_instance->json_ctx.effective_json_md5, + maat_instance->json_ctx.iris_file); maat_instance->input_mode = DATA_SOURCE_JSON_FILE; return 0; diff --git a/src/maat_ex_data.cpp b/src/maat_ex_data.cpp index cf697e2..02c2a1e 100644 --- a/src/maat_ex_data.cpp +++ b/src/maat_ex_data.cpp @@ -13,12 +13,15 @@ #include "uthash/uthash.h" #include "uthash/utarray.h" +#include "log/log.h" #include "rcu_hash.h" #include "utils.h" #include "maat_utils.h" #include "maat_table_schema.h" #include "maat_ex_data.h" +#define MODULE_EX_DATA module_name_str("maat.ex_data") + struct ex_data_runtime { UT_array *cache_rows; size_t cache_row_num; @@ -119,9 +122,14 @@ void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_da ex_data_rt->ex_schema = schema; } -void ex_data_runtime_set_user_ctx(struct ex_data_runtime *ex_data_rt, void *user_ctx) +void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx) { - rcu_hash_set_user_ctx(ex_data_rt->htable, user_ctx); + rcu_hash_set_user_ctx(ex_data_rt->htable, container_ctx); +} + +struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt) +{ + return (struct ex_container_ctx *)rcu_hash_get_user_ctx(ex_data_rt->htable); } void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len) @@ -133,52 +141,59 @@ void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char return ex_data; } -void ex_data_runtime_add_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, void *data) +void ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct ex_data_container *ex_container) { - void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len); - if (tmp_data != NULL) { + struct ex_data_container *tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + if (tmp_container != NULL) { return; } - rcu_hash_add(ex_data_rt->htable, key, key_len, data); + rcu_hash_add(ex_data_rt->htable, key, key_len, ex_container); } -void ex_data_runtime_del_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) +void ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, + struct log_handle *logger) { - void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len); - if (NULL == tmp_data) { - fprintf(stderr, "ex data del error: no such key:%s\n", key); + struct ex_data_container *tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + if (NULL == tmp_container) { + log_error(logger, MODULE_EX_DATA, "ex data del error: no such key:%s", key); return; } rcu_hash_del(ex_data_rt->htable, key, key_len); } -void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) +void *ex_data_runtime_dup_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) { - void *ex_data = rcu_hash_find(ex_data_rt->htable, key, key_len); - if (NULL == ex_data) { + struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + if (NULL == ex_container) { return NULL; } void *dup_ex_data = NULL; - ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, &ex_data, + ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, &(ex_container->ex_data), ex_data_rt->ex_schema->argl, ex_data_rt->ex_schema->argp); return dup_ex_data; } -size_t ex_data_runtime_ex_data_count(struct ex_data_runtime *ex_data_rt) +void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) +{ + struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + if (NULL == ex_container) { + return NULL; + } + + return ex_container->custom_data; +} + +size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt) { return rcu_hash_count(ex_data_rt->htable); } -size_t ex_data_runtime_list_updating_ex_data(struct ex_data_runtime *ex_data_rt, void ***ex_data_array) +size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container) { - if (NULL == ex_data_rt->ex_schema) { - return 0; - } - - return rcu_hash_list_updating_data(ex_data_rt->htable, ex_data_array); + return rcu_hash_list_updating_data(ex_data_rt->htable, (void ***)ex_container); } int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt) diff --git a/src/maat_redis_monitor.cpp b/src/maat_redis_monitor.cpp index 2d05489..e912649 100644 --- a/src/maat_redis_monitor.cpp +++ b/src/maat_redis_monitor.cpp @@ -19,6 +19,8 @@ #include "maat_redis_monitor.h" #include "maat_table_schema.h" +#define MODULE_REDIS_MONITOR module_name_str("maat.redis_monitor") + const time_t MAAT_REDIS_RECONNECT_INTERVAL_S = 5; const static int MAAT_REDIS_SYNC_TIME = 30 * 60; @@ -47,7 +49,8 @@ char *get_foreign_cont_filename(const char *table_name, int rule_id, const char return filename; } -void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_foreign, const char *dir) +void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_foreign, + const char *dir, struct log_handle *logger) { int foreign_key_size = 0; p_rule->f_keys = ALLOC(struct foreign_key, n_foreign); @@ -55,8 +58,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f for (int i = 0; i < n_foreign; i++) { const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size); if (NULL == p_foreign) { - fprintf(stderr, "Get %s,%lu foreign keys failed: No %dth column\n", - p_rule->table_name, p_rule->rule_id, foreign_columns[i]); + log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign keys failed: No %dth column", + p_rule->table_name, p_rule->rule_id, foreign_columns[i]); continue; } @@ -66,8 +69,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f } if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) { - fprintf(stderr, "Get %s,%lu foreign key failed: Invalid source prefix %s\n", - p_rule->table_name, p_rule->rule_id, p_foreign); + log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign key failed: Invalid source prefix %s", + p_rule->table_name, p_rule->rule_id, p_foreign); continue; } @@ -76,8 +79,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f p_foreign += strlen(foreign_source_prefix); if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) { - fprintf(stdout, "%s, %lu foreign key prefix %s is not recommended\n", - p_rule->table_name, p_rule->rule_id, p_foreign); + log_info(logger, MODULE_REDIS_MONITOR, "%s, %lu foreign key prefix %s is not recommended", + p_rule->table_name, p_rule->rule_id, p_foreign); } p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size+1); @@ -114,7 +117,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in continue; } - _get_foregin_keys(rule_list+i, foreign_columns, n_foreign_column, dir); + _get_foregin_keys(rule_list+i, foreign_columns, n_foreign_column, dir, maat_instance->logger); rule_with_foreign_key++; } @@ -122,7 +125,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in } int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, - int rule_num, const char* dir) + int rule_num, const char* dir, struct log_handle *logger) { int j = 0; int foreign_key_size = 0; @@ -146,7 +149,7 @@ int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *r } while (p_foreign != NULL && n_foreign < MAX_FOREIGN_CLMN_NUM); if (n_foreign > 0) { - _get_foregin_keys(rule_list+i, foreign_columns, n_foreign, dir); + _get_foregin_keys(rule_list+i, foreign_columns, n_foreign, dir, logger); rule_with_foreign_key++; } } @@ -160,7 +163,8 @@ struct foreign_conts_track int foreign_idx; }; -int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num) +int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, + struct log_handle *logger) { int i = 0; int failed_cnt = 0; @@ -181,8 +185,9 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru for (i = 0; i < rule_num; i++) { ret = maat_cmd_wrap_redis_get_reply(c, &reply); if (ret == REDIS_ERR) { - fprintf(stderr, "Redis GET %s:%s,%lu failed, redis server error\n", mr_key_prefix[rule_list[i].op], - rule_list[i].table_name, rule_list[i].rule_id); + log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed, redis server error", + mr_key_prefix[rule_list[i].op], + rule_list[i].table_name, rule_list[i].rule_id); error_happened = 1; break; } @@ -194,8 +199,9 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru retry_ids[failed_cnt] = i; failed_cnt++; } else { - fprintf(stderr, "Redis GET %s:%s,%lu failed\n",mr_key_prefix[rule_list[i].op], - rule_list[i].table_name, rule_list[i].rule_id); + log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed", + mr_key_prefix[rule_list[i].op], + rule_list[i].table_name, rule_list[i].rule_id); error_happened = 1; } } @@ -222,7 +228,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru idx = retry_ids[i]; ret = maat_cmd_wrap_redis_get_reply(c, &reply); if (ret == REDIS_ERR) { - fprintf(stderr, "redis command %s failed, redis server error\n", redis_cmd); + log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, redis server error", redis_cmd); FREE(retry_ids); return -1; } @@ -231,10 +237,12 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru rule_list[idx].table_line = maat_strdup(reply->str); } else if(reply->type==REDIS_REPLY_ERROR) { //Deal with Redis response: "Loading Redis is loading the database in memory" - fprintf(stderr, "redis command %s error, reply type=%d, error str=%s\n", redis_cmd, reply->type, reply->str); + log_error(logger, MODULE_REDIS_MONITOR, "redis command %s error, reply type=%d, error str=%s", + redis_cmd, reply->type, reply->str); } else { //Handle type "nil" - fprintf(stderr, "redis command %s failed, reply type=%d\n", redis_cmd, reply->type); + log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, reply type=%d", + redis_cmd, reply->type); } freeReplyObject(reply); @@ -246,7 +254,8 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru return 0; } -int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process) +int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, + int print_process, struct log_handle *logger) { int max_redis_batch = 4096; int success_cnt = 0; @@ -254,7 +263,7 @@ int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int while (success_cnt < rule_num) { int batch_cnt = MIN(rule_num-success_cnt, max_redis_batch); - int ret = _get_maat_redis_value(c, rule_list+success_cnt, batch_cnt); + int ret = _get_maat_redis_value(c, rule_list+success_cnt, batch_cnt, logger); if (ret < 0) { return -1; } else { @@ -277,14 +286,15 @@ int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int } int get_inc_key_list(long long instance_version, long long target_version, - redisContext *c, struct serial_rule **list) + redisContext *c, struct serial_rule **list, struct log_handle *logger) { //Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version. //The elements are considered to be ordered from low to high scores(instance_version). redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", mr_status_sset, instance_version,target_version); if (NULL == reply) { - fprintf(stderr, "GET %s failed with a NULL reply, error: %s\n", mr_status_sset, c->errstr); + log_error(logger, MODULE_REDIS_MONITOR, "GET %s failed with a NULL reply, error: %s", + mr_status_sset, c->errstr); return -1; } @@ -299,8 +309,8 @@ int get_inc_key_list(long long instance_version, long long target_version, redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", mr_status_sset, reply->element[0]->str); if (tmp_reply->type != REDIS_REPLY_STRING) { - fprintf(stderr, "ZSCORE %s %s failed Version: %lld->%lld\n", mr_status_sset, - reply->element[0]->str, instance_version, target_version); + log_error(logger, MODULE_REDIS_MONITOR, "ZSCORE %s %s failed Version: %lld->%lld", + mr_status_sset, reply->element[0]->str, instance_version, target_version); freeReplyObject(tmp_reply); tmp_reply = NULL; freeReplyObject(reply); @@ -317,8 +327,8 @@ int get_inc_key_list(long long instance_version, long long target_version, } if (nearest_rule_version != instance_version + 1) { - fprintf(stdout, "Noncontinuous VERSION Redis: %lld MAAT: %lld\n", - nearest_rule_version, instance_version); + log_info(logger, MODULE_REDIS_MONITOR, "Noncontinuous VERSION Redis: %lld MAAT: %lld", + nearest_rule_version, instance_version); } int i = 0; @@ -331,7 +341,7 @@ int get_inc_key_list(long long instance_version, long long target_version, int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", op_str, s_rule[j].table_name, &(s_rule[j].rule_id)); if (ret != 3 || s_rule[i].rule_id < 0) { - fprintf(stderr, "Invalid Redis Key: %s\n", reply->element[i]->str); + log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str); continue; } @@ -340,7 +350,7 @@ int get_inc_key_list(long long instance_version, long long target_version, } else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) { s_rule[j].op = MAAT_OP_DEL; } else { - fprintf(stderr, "Invalid Redis Key: %s\n", reply->element[i]->str); + log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str); continue; } j++; @@ -458,7 +468,8 @@ int recovery_history_version(const struct serial_rule *current, int current_num, int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long *new_version, struct table_schema_manager* table_schema_mgr, - struct serial_rule **list, int *update_type, int cumulative_off) + struct serial_rule **list, int *update_type, int cumulative_off, + struct log_handle *logger) { int rule_num = 0; long long target_version = 0; @@ -467,20 +478,20 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION"); if (reply != NULL) { if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { - fprintf(stderr, "GET MAAT_VERSION failed, maybe Redis is busy\n"); + log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed, maybe Redis is busy"); freeReplyObject(reply); reply = NULL; return -1; } } else { - fprintf(stderr, "GET MAAT_VERSION failed with NULL reply, error: %s\n", c->errstr); + log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed with NULL reply, error: %s", c->errstr); return -1; } long long redis_version = maat_cmd_read_redis_integer(reply); if (redis_version < 0) { if (reply->type == REDIS_REPLY_ERROR) { - fprintf(stderr, "Redis Communication error: %s\n", reply->str); + log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", reply->str); } return -1; } @@ -497,7 +508,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l } if (redis_version < instance_version) { - fprintf(stderr, "VERSION roll back MAAT: %lld -> Redis: %lld\n", instance_version, redis_version); + log_error(logger, MODULE_REDIS_MONITOR, "VERSION roll back MAAT: %lld -> Redis: %lld", + instance_version, redis_version); goto FULL_UPDATE; } @@ -509,7 +521,7 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l do { target_version++; - rule_num = get_inc_key_list(instance_version, target_version, c, &s_rule_array); + rule_num = get_inc_key_list(instance_version, target_version, c, &s_rule_array, logger); if (rule_num > 0) { break; } else if (rule_num < 0) { @@ -521,13 +533,13 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l } while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off); if (0 == rule_num) { - fprintf(stdout, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s\n", mr_status_sset, - instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON"); + log_info(logger, MODULE_REDIS_MONITOR, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s", + mr_status_sset, instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON"); return 0; } - fprintf(stdout, "Inc Update from instance_version %lld to %lld (%d entries)\n", - instance_version, target_version, rule_num); + log_info(logger, MODULE_REDIS_MONITOR, "Inc Update from instance_version %lld to %lld (%d entries)", + instance_version, target_version, rule_num); *list = s_rule_array; *update_type = CM_UPDATE_TYPE_INC; @@ -535,8 +547,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l return rule_num; FULL_UPDATE: - fprintf(stdout, "Initiate full update from instance_version %lld to %lld\n", instance_version, - desired_version == 0 ? redis_version : desired_version); + log_info(logger, MODULE_REDIS_MONITOR, "Initiate full update from instance_version %lld to %lld", + instance_version, desired_version == 0 ? redis_version : desired_version); size_t append_cmd_cnt = 0; int ret = redisAppendCommand(c, "MULTI"); append_cmd_cnt++; @@ -557,12 +569,12 @@ FULL_UPDATE: reply = maat_cmd_wrap_redis_command(c, "EXEC"); if (NULL == reply) { - fprintf(stderr, "Redis Communication error: %s\n", c->errstr); + log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", c->errstr); return -1; } if (reply->type != REDIS_REPLY_ARRAY) { - fprintf(stderr, "Invalid Redis Key List type %d\n", reply->type); + log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", reply->type); freeReplyObject(reply); reply = NULL; return -1; @@ -571,7 +583,7 @@ FULL_UPDATE: *new_version = maat_cmd_read_redis_integer(reply->element[0]); redisReply *sub_reply = reply->element[1]; if (sub_reply->type != REDIS_REPLY_ARRAY) { - fprintf(stderr, "Invalid Redis Key List type %d\n", sub_reply->type); + log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", sub_reply->type); freeReplyObject(reply); reply = NULL; return -1; @@ -581,7 +593,7 @@ FULL_UPDATE: s_rule_array = ALLOC(struct serial_rule, sub_reply->elements); for (i = 0, full_idx = 0; i < sub_reply->elements; i++) { if (sub_reply->element[i]->type != REDIS_REPLY_STRING) { - fprintf(stderr, "Invalid Redis Key Type: %d\n", sub_reply->element[i]->type); + log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Type: %d", sub_reply->element[i]->type); continue; } @@ -591,7 +603,7 @@ FULL_UPDATE: s_rule_array[full_idx].op = MAAT_OP_ADD; if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || strlen(s_rule_array[full_idx].table_name) == 0) { - fprintf(stderr, "Invalid Redis Key Format: %s\n", sub_reply->element[i]->str); + log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Format: %s", sub_reply->element[i]->str); continue; } @@ -611,13 +623,13 @@ FULL_UPDATE: if (desired_version != 0) { struct serial_rule *changed_rule_array = NULL; - int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array); + int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array, logger); if (changed_rule_num < 0) { - fprintf(stderr, "Recover history version %lld faild where as redis version is %lld\n", - desired_version, redis_version); + log_error(logger, MODULE_REDIS_MONITOR, "Recover history version %lld faild where as redis version is %lld", + desired_version, redis_version); } else if(0 == changed_rule_num) { - fprintf(stderr, "Nothing to recover from history version %lld to redis version is %lld\n", - desired_version, redis_version); + log_error(logger, MODULE_REDIS_MONITOR, "Nothing to recover from history version %lld to redis version is %lld", + desired_version, redis_version); } else { struct serial_rule *history_rule_array = NULL; ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array); @@ -626,8 +638,8 @@ FULL_UPDATE: s_rule_array = history_rule_array; rule_num = ret; *new_version = desired_version; - fprintf(stdout, "Successfully recovered from history version %lld to redis version is %lld\n", - desired_version, redis_version); + log_info(logger, MODULE_REDIS_MONITOR, "Successfully recovered from history version %lld to redis version is %lld", + desired_version, redis_version); } } FREE(changed_rule_array); @@ -635,12 +647,13 @@ FULL_UPDATE: *list = s_rule_array; *update_type = CM_UPDATE_TYPE_FULL; - fprintf(stdout, "Full update %d keys of version %lld\n", rule_num, *new_version); + log_info(logger, MODULE_REDIS_MONITOR, "Full update %d keys of version %lld", rule_num, *new_version); return rule_num ; } -void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn) +void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn, + struct log_handle *logger) { int i = 0; int j = 0; @@ -663,8 +676,8 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule ret = remove(rule_list[i].f_keys[j].filename); if (ret == -1) { - fprintf(stderr, "Foreign content file %s remove failed\n", - rule_list[i].f_keys[j].filename); + log_error(logger, MODULE_REDIS_MONITOR, "Foreign content file %s remove failed", + rule_list[i].f_keys[j].filename); } } } else { @@ -694,25 +707,28 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule for (i = 0; i < key_num; i++) { ret = maat_cmd_wrap_redis_get_reply(c, &reply); if (ret == REDIS_ERR) { - fprintf(stderr, "Get %s,%lu foreign key %s content failed, redis server error\n", - rule_list[track[i].rule_idx].table_name, - rule_list[track[i].rule_idx].rule_id, - rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); + log_error(logger, MODULE_REDIS_MONITOR, + "Get %s,%lu foreign key %s content failed, redis server error", + rule_list[track[i].rule_idx].table_name, + rule_list[track[i].rule_idx].rule_id, + rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); break; } if (reply->type != REDIS_REPLY_STRING) { - fprintf(stderr, "Get %s,%lu foreign key %s content failed\n", - rule_list[track[i].rule_idx].table_name, - rule_list[track[i].rule_idx].rule_id, - rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); + log_error(logger, MODULE_REDIS_MONITOR, + "Get %s,%lu foreign key %s content failed", + rule_list[track[i].rule_idx].table_name, + rule_list[track[i].rule_idx].rule_id, + rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); continue; } else { s_rule = rule_list+track[i].rule_idx; FILE *fp = fopen(s_rule->f_keys[track[i].foreign_idx].filename, "w"); if (NULL == fp) { - fprintf(stderr, "Write foreign content failed: fopen %s error\n", - s_rule->f_keys[track[i].foreign_idx].filename); + log_error(logger, MODULE_REDIS_MONITOR, + "Write foreign content failed: fopen %s error", + s_rule->f_keys[track[i].foreign_idx].filename); } else { fwrite(reply->str, 1, reply->len, fp); fclose(fp); @@ -731,14 +747,15 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule return; } -void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn) +void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn, + struct log_handle *logger) { int max_redis_batch = 4096; int success_cnt = 0; while (success_cnt < rule_num) { int batch_cnt = MIN(rule_num - success_cnt, max_redis_batch); - _get_foreign_conts(c, rule_list + success_cnt, batch_cnt, print_fn); + _get_foreign_conts(c, rule_list + success_cnt, batch_cnt, print_fn, logger); success_cnt += batch_cnt; } } @@ -1036,7 +1053,8 @@ int mr_operation_success(redisReply *actual_reply, struct expected_reply *expect return 0; } -int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time) +int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, + long long server_time, struct log_handle *logger) { size_t i = 0; size_t rule_seq = 0; @@ -1085,16 +1103,20 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri assert(transaction_reply->elements == multi_cmd_cnt); for (i = 0; i < multi_cmd_cnt; i++) { p = transaction_reply->element[i]; - //failed is acceptable - //or transaciton is success + //failed is acceptable + //or transaciton is success //or continuation of last failed - if (expected_reply[i].s_rule_seq == -1 || 1 == mr_operation_success(p, expected_reply+i) || last_failed == expected_reply[i].s_rule_seq) { + if (expected_reply[i].s_rule_seq == -1 || + 1 == mr_operation_success(p, expected_reply+i) || + last_failed == expected_reply[i].s_rule_seq) { continue; } + rule_seq = expected_reply[i].s_rule_seq; - fprintf(stderr, "%s %s %lu failed, rule id maybe conflict or not exist\n", - mr_op_str[s_rule[rule_seq].op], s_rule[rule_seq].table_name, - s_rule[rule_seq].rule_id); + log_error(logger, MODULE_REDIS_MONITOR, + "%s %s %lu failed, rule id maybe conflict or not exist", + mr_op_str[s_rule[rule_seq].op], s_rule[rule_seq].table_name, + s_rule[rule_seq].rule_id); success_cnt--; last_failed = rule_seq; } @@ -1103,9 +1125,10 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri } if (transaction_version > 0) { - transaction_finished_version = maat_cmd_read_redis_integer(transaction_reply->element[multi_cmd_cnt-1]); - fprintf(stdout, "Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld\n", - transaction_version, transaction_finished_version); + transaction_finished_version = maat_cmd_read_redis_integer(transaction_reply->element[multi_cmd_cnt - 1]); + log_info(logger, MODULE_REDIS_MONITOR, + "Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld", + transaction_version, transaction_finished_version); } freeReplyObject(transaction_reply); @@ -1115,9 +1138,8 @@ error_out: if (renew_num > 0 && renew_allowed != 1) { for (i = 0; i < (unsigned int)serial_rule_num; i++) { if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) { - fprintf(stdout, "%s %s %lu is not allowed due to lock contention\n", - mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, - s_rule[i].rule_id); + log_error(logger, MODULE_REDIS_MONITOR, "%s %s %lu is not allowed due to lock contention", + mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, s_rule[i].rule_id); } } @@ -1131,7 +1153,7 @@ error_out: return success_cnt; } -void cleanup_update_status(redisContext *c) +void cleanup_update_status(redisContext *c, struct log_handle *logger) { long long version_upper_bound = 0; long long version_lower_bound = 0; @@ -1191,18 +1213,17 @@ void cleanup_update_status(redisContext *c) freeReplyObject(reply); reply = NULL; - fprintf(stdout, "Clean up update status from version %lld to %lld (%lld versions, %lld entries)\n", - version_lower_bound, version_upper_bound, version_num, entry_num); + log_info(logger, MODULE_REDIS_MONITOR, + "Clean up update status from version %lld to %lld (%lld versions, %lld entries)", + version_lower_bound, version_upper_bound, version_num, entry_num); return; error_out: freeReplyObject(reply); reply = NULL; - - return; } -void check_maat_expiration(redisContext *c) +void check_maat_expiration(redisContext *c, struct log_handle *logger) { UNUSED int ret = 0; @@ -1229,14 +1250,14 @@ void check_maat_expiration(redisContext *c) freeReplyObject(data_reply); data_reply = NULL; - int success_cnt = maat_cmd_write_rule(c, s_rule, s_rule_num, server_time); + int success_cnt = maat_cmd_write_rule(c, s_rule, s_rule_num, server_time, logger); if (success_cnt < 0) { - fprintf(stderr, "maat_cmd_write_rule failed.\n"); + log_error(logger, MODULE_REDIS_MONITOR, "maat_cmd_write_rule failed."); } else if (success_cnt == (int)s_rule_num) { - fprintf(stdout, "Succesfully expired %zu rules in Redis\n", s_rule_num); + log_info(logger, MODULE_REDIS_MONITOR, "Succesfully expired %zu rules in Redis", s_rule_num); } else { - fprintf(stderr, "Failed to expired %d of %zu rules in Redis, try later\n", - s_rule_num - success_cnt, s_rule_num); + log_error(logger, MODULE_REDIS_MONITOR, "Failed to expired %d of %zu rules in Redis, try later", + s_rule_num - success_cnt, s_rule_num); } FREE(s_rule); @@ -1245,8 +1266,7 @@ void check_maat_expiration(redisContext *c) void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, void (*start_fn)(long long, int, void *), int (*update_fn)(const char *, const char *, void *), - void (*finish_fn)(void *), - void *u_param) + void (*finish_fn)(void *), void *u_param) { int i = 0; int ret = 0; @@ -1258,13 +1278,14 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, enum table_type table_type; enum scan_type scan_type; struct table_schema *table_schema = NULL; + struct maat *maat_instance = (struct maat *)u_param; //authorized to write if (mr_ctx->write_ctx != NULL && mr_ctx->write_ctx->err == 0) { //For thread safe, deliberately use redis_read_ctx but not redis_write_ctx. if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout_ms)) { - check_maat_expiration(mr_ctx->read_ctx); - cleanup_update_status(mr_ctx->read_ctx); + check_maat_expiration(mr_ctx->read_ctx, maat_instance->logger); + cleanup_update_status(mr_ctx->read_ctx, maat_instance->logger); redlock_unlock(mr_ctx->read_ctx, mr_expire_lock); } } @@ -1278,9 +1299,10 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, if (mr_ctx->read_ctx != NULL) { redisFree(mr_ctx->read_ctx); } - fprintf(stdout, "Reconnecting...\n"); + log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Reconnecting..."); - mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); + mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, + maat_instance->logger); if (NULL == mr_ctx->read_ctx) { return; } else { @@ -1288,14 +1310,14 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, } } - struct maat *maat_instance = (struct maat *)u_param; struct serial_rule *rule_list = NULL; long long new_version = 0; int update_type = CM_UPDATE_TYPE_INC; int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version, &new_version, maat_instance->table_schema_mgr, &rule_list, - &update_type, maat_instance->cumulative_update_off); + &update_type, maat_instance->cumulative_update_off, + maat_instance->logger); //redis communication error if (rule_num < 0) { redisFree(mr_ctx->read_ctx); @@ -1310,12 +1332,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, } if (rule_num > 0) { - ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, 0); + ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger); //redis communication error if (ret < 0) { redisFree(mr_ctx->read_ctx); mr_ctx->read_ctx = NULL; - fprintf(stderr, "Get Redis value failed, abandon update and close connection\n"); + log_error(maat_instance->logger, MODULE_REDIS_MONITOR, + "Get Redis value failed, abandon update and close connection"); goto clean_up; } @@ -1326,19 +1349,20 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, } if (empty_value_num == rule_num) { - fprintf(stdout, "All %d rules are empty, abandon update\n", empty_value_num); + log_info(maat_instance->logger, MODULE_REDIS_MONITOR, + "All %d rules are empty, abandon update", empty_value_num); goto clean_up; } ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num, maat_instance, maat_instance->foreign_cont_dir); if (ret > 0) { - maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0); + maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger); } } start_fn(new_version, update_type, u_param); - fprintf(stdout, "Start %s update: %lld -> %lld (%d entries)\n", - update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num); + log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Start %s update: %lld -> %lld (%d entries)", + update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num); for (i = 0; i < rule_num; i++) { if (NULL == rule_list[i].table_line) { @@ -1360,7 +1384,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, valid_column = table_schema_get_valid_flag_column(table_schema); ret = invalidate_line(rule_list[i].table_line, table_type, valid_column); if (ret < 0) { - fprintf(stdout, "Invalidate line failed, invaid format %s\n", rule_list[i].table_line); + log_error(maat_instance->logger, MODULE_REDIS_MONITOR, + "Invalidate line failed, invaid format %s", rule_list[i].table_line); continue; } } @@ -1376,8 +1401,9 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, finish_fn(u_param); if (call_update_num < rule_num) { - fprintf(stdout, "Load %d entries to match engine, no table: %d, empty value: %d\n", - call_update_num, no_table_num, empty_value_num); + log_error(maat_instance->logger, MODULE_REDIS_MONITOR, + "Load %d entries to match engine, no table: %d, empty value: %d", + call_update_num, no_table_num, empty_value_num); } clean_up: diff --git a/src/maat_rule.cpp b/src/maat_rule.cpp index 647e791..fdf1f15 100644 --- a/src/maat_rule.cpp +++ b/src/maat_rule.cpp @@ -27,6 +27,8 @@ #include "maat_table_runtime.h" #include "maat_table_schema.h" +#define MODULE_MAAT_RULE module_name_str("maat.rule") + struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance) { struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); @@ -105,13 +107,14 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param) struct maat_runtime* maat_rt = NULL; int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name); if (table_id < 0) { - fprintf(stderr, "update warning, unknown table name %s\n", table_name); + log_error(maat_instance->logger, MODULE_MAAT_RULE, "update warning, unknown table name %s", table_name); return -1; } struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); if (NULL == table_schema) { - fprintf(stderr, "update warning, table name %s doesn't have table schema\n", table_name); + log_error(maat_instance->logger, MODULE_MAAT_RULE, + "update warning, table name %s doesn't have table schema", table_name); return -1; } @@ -123,10 +126,10 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param) maat_rt = maat_instance->maat_rt; } - struct table_item *table_item = table_schema_line_to_item(line, table_schema); + struct table_item *table_item = table_schema_line_to_item(line, table_schema, maat_instance->logger); if (table_item != NULL) { struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); - table_runtime_update(table_rt, table_schema, line, table_item); + table_runtime_update(table_rt, table_schema, line, table_item, maat_instance->logger); FREE(table_item); } @@ -157,16 +160,14 @@ void maat_finish_cb(void *u_param) if (maat_instance->creating_maat_rt != NULL) { maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_runtime_commit(maat_instance->creating_maat_rt); - fprintf(stdout, "Full config version %llu load %d entries complete\n", - maat_instance->creating_maat_rt->version, - maat_instance->creating_maat_rt->rule_num); + log_info(maat_instance->logger, MODULE_MAAT_RULE, "Full config version %llu load %d entries complete\n", + maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->rule_num); } else if (maat_instance->maat_rt != NULL) { maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt); maat_instance->maat_rt->version = maat_instance->maat_version; maat_runtime_commit(maat_instance->maat_rt); - fprintf(stdout, "Inc config version %llu load %d entries complete\n", - maat_instance->maat_rt->version, - maat_instance->maat_rt->rule_num); + log_info(maat_instance->logger, MODULE_MAAT_RULE, "Inc config version %llu load %d entries complete\n", + maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); } } @@ -189,7 +190,7 @@ void *rule_monitor_loop(void *arg) pthread_mutex_lock(&(maat_instance->background_update_mutex)); /* if deferred load on */ if (maat_instance->deferred_load != 0) { - fprintf(stdout, "Deferred Loading ON, updating in %s\n", __func__); + log_info(maat_instance->logger, MODULE_MAAT_RULE, "Deferred Loading ON, updating in %s", __func__); maat_read_full_config(maat_instance); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); @@ -204,18 +205,14 @@ void *rule_monitor_loop(void *arg) case DATA_SOURCE_REDIS: redis_monitor_traverse(maat_instance->maat_version, &(maat_instance->mr_ctx), - maat_start_cb, - maat_update_cb, - maat_finish_cb, + maat_start_cb, maat_update_cb, maat_finish_cb, maat_instance); break; case DATA_SOURCE_IRIS_FILE: config_monitor_traverse(maat_instance->maat_version, maat_instance->iris_ctx.inc_idx_dir, - maat_start_cb, - maat_update_cb, - maat_finish_cb, - maat_instance); + maat_start_cb, maat_update_cb, maat_finish_cb, + maat_instance, maat_instance->logger); break; case DATA_SOURCE_JSON_FILE: memset(md5_tmp, 0, sizeof(md5_tmp)); @@ -224,18 +221,19 @@ void *rule_monitor_loop(void *arg) maat_instance->json_ctx.last_md5_time = attrib.st_ctim; md5_file(maat_instance->json_ctx.json_file, md5_tmp); if (0 != strcmp(md5_tmp, maat_instance->json_ctx.effective_json_md5)) { - ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); + ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, + err_str, sizeof(err_str)); if (ret < 0) { - fprintf(stdout, "Maat re-initiate with JSON file %s (md5=%s)failed: %s\n", - maat_instance->json_ctx.json_file, md5_tmp, err_str); + log_error(maat_instance->logger, MODULE_MAAT_RULE, + "Maat re-initiate with JSON file %s (md5=%s)failed: %s\n", + maat_instance->json_ctx.json_file, md5_tmp, err_str); } else { config_monitor_traverse(0, maat_instance->json_ctx.iris_file, - maat_start_cb, - maat_update_cb, - maat_finish_cb, - maat_instance); - fprintf(stdout, "Maat re-initiate with JSON file %s success, md5: %s\n", - maat_instance->json_ctx.json_file, md5_tmp); + maat_start_cb, maat_update_cb, maat_finish_cb, + maat_instance, maat_instance->logger); + log_info(maat_instance->logger, MODULE_MAAT_RULE, + "Maat re-initiate with JSON file %s success, md5: %s\n", + maat_instance->json_ctx.json_file, md5_tmp); } } } @@ -250,11 +248,11 @@ void *rule_monitor_loop(void *arg) if (old_maat_rt != NULL) { if (maat_instance->maat_rt->version > old_maat_rt->version) { - fprintf(stdout, "Maat version updated %lld -> %lld\n", - old_maat_rt->version, maat_instance->maat_rt->version); + log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version updated %lld -> %lld\n", + old_maat_rt->version, maat_instance->maat_rt->version); } else { - fprintf(stdout, "Maat version roll back %lld -> %lld\n", - old_maat_rt->version, maat_instance->maat_rt->version); + log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version roll back %lld -> %lld\n", + old_maat_rt->version, maat_instance->maat_rt->version); } maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy); } diff --git a/src/maat_table_runtime.cpp b/src/maat_table_runtime.cpp index 2fd1fb6..8178157 100644 --- a/src/maat_table_runtime.cpp +++ b/src/maat_table_runtime.cpp @@ -15,6 +15,7 @@ #include #include "utils.h" +#include "log/log.h" #include "maat_utils.h" #include "maat_table_runtime.h" #include "uthash/uthash.h" @@ -23,15 +24,18 @@ #include "rcu_hash.h" #include "IPMatcher.h" +#define MODULE_TABLE_RUNTIME module_name_str("maat.table_runtime") + #define MAAT_MAX_EXPR_ITEM_NUM 8 -struct plugin_ex_data { - void *data; - void *user_data; +struct port_range { + uint16_t min_port; + uint16_t max_port; }; -struct plugin_user_ctx { +struct ex_container_ctx { int table_id; + void (*custom_data_free)(void *custom_data); struct ex_data_schema *ex_schema; }; @@ -42,11 +46,6 @@ struct expr_runtime { struct rcu_hash_table *htable; }; -struct ip_runtime { - struct ip_matcher *ip_matcher; - struct rcu_hash_table *htable; -}; - struct plugin_runtime { uint64_t acc_line_num; struct ex_data_runtime *ex_data_rt; @@ -63,7 +62,8 @@ struct table_runtime { enum table_type table_type; union { struct expr_runtime expr_rt; - struct ip_runtime ip_rt; + //struct ip_plugin_runtime ip_rt; + struct ip_plugin_runtime ip_plus_rt; struct plugin_runtime plugin_rt; struct ip_plugin_runtime ip_plugin_rt; }; @@ -80,16 +80,26 @@ struct table_runtime_manager { struct maat_garbage_bin *garbage_bin; }; -void plugin_ex_data_free(void *user_ctx, void *data) +void ex_data_container_free(void *ctx, void *data) { - if (NULL == user_ctx || NULL == data) { + if (NULL == ctx || NULL == data) { return; } - struct plugin_user_ctx *ctx = (struct plugin_user_ctx *)user_ctx; - long argl = ctx->ex_schema->argl; - void *argp = ctx->ex_schema->argp; - ctx->ex_schema->free_func(ctx->table_id, &data, argl, argp); + struct ex_container_ctx *container_ctx = (struct ex_container_ctx *)ctx; + long argl = container_ctx->ex_schema->argl; + void *argp = container_ctx->ex_schema->argp; + + struct ex_data_container *ex_container = (struct ex_data_container *)data; + if (ex_container->ex_data != NULL && container_ctx->ex_schema->free_func != NULL) { + container_ctx->ex_schema->free_func(container_ctx->table_id, &(ex_container->ex_data), argl, argp); + } + + if (ex_container->custom_data != NULL && container_ctx->custom_data_free != NULL) { + container_ctx->custom_data_free(ex_container->custom_data); + } + + FREE(ex_container); } void expr_rule_free(and_expr_t *expr_rule) @@ -121,11 +131,14 @@ struct table_runtime *table_runtime_new(struct table_schema *table_schema, int m table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free); table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema); break; + case TABLE_TYPE_IP_PLUS: + table_rt->ip_plus_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); + break; case TABLE_TYPE_PLUGIN: - table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, plugin_ex_data_free); + table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); break; case TABLE_TYPE_IP_PLUGIN: - table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, plugin_ex_data_free); + table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); break; default: break; @@ -263,9 +276,72 @@ void table_runtime_stream_close(struct table_runtime *table_rt) } } -struct ip_rule *ip_plugin_item_to_ip_rule(struct ip_plugin_item *ip_plugin_item) +enum IP_TYPE ip_type_transform(enum ip_type type) { - return NULL; + if (type == IP_TYPE_V4) { + return IPv4; + } else { + return IPv6; + } +} + +int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, + int results[], size_t *n_result) +{ + if (NULL == table_rt) { + return -1; + } + + int i = 0; + int n_hit_result = 0; + struct scan_result scan_results[table_rt->rule_num] = {0}; + + struct ip_data ip; + ip.type = ip_type_transform(data->ip_type); + if (ip.type == IPv4) { + ip.ipv4 = data->ipv4; + } else { + memcpy(ip.ipv6, data->ipv6, sizeof(data->ipv6)); + } + + n_hit_result = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, table_rt->rule_num); + int index = 0; + /* filter rule_id by port */ + for (i = 0; i < n_hit_result; i++) { + struct port_range *port_range = (struct port_range *)scan_results[i].tag; + if (data->port < port_range->min_port || data->port > port_range->max_port) { + continue; + } + results[index++] = scan_results[i].rule_id; + } + *n_result = index; + + return 0; +} + +void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule) +{ + struct port_range *sport_range = ALLOC(struct port_range, 1); + if (4 == item->addr_type) { + rule->type = IPv4; + sport_range->min_port = item->ipv4.min_sport; + sport_range->max_port = item->ipv4.max_sport; + } else { + rule->type = IPv6; + sport_range->min_port = item->ipv6.min_sport; + sport_range->max_port = item->ipv6.max_sport; + } + rule->rule_id = item->item_id; + rule->user_tag = sport_range; + rule->ipv4_rule.start_ip = item->ipv4.min_sip; + rule->ipv4_rule.end_ip = item->ipv4.max_sip; + memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, sizeof(item->ipv6.min_sip)); + memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, sizeof(item->ipv6.max_sip)); +} + +void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule) +{ + } enum pattern_type expr_type2pattern_type(enum expr_type expr_type) @@ -286,7 +362,7 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type) return pattern_type; } -and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item) +and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger) { size_t i = 0; size_t sub_expr_cnt = 0; @@ -305,7 +381,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item) } if (i >= MAAT_MAX_EXPR_ITEM_NUM) { - fprintf(stderr, "item_id:%d too many patterns\n", expr_item->item_id); + log_error(logger, MODULE_TABLE_RUNTIME, "item_id:%d too many patterns", + expr_item->item_id); return NULL; } @@ -340,7 +417,7 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item) } void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len, - and_expr_t *expr_rule, int is_valid) + and_expr_t *expr_rule, int is_valid, struct log_handle *logger) { void *data = NULL; @@ -348,7 +425,7 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key //delete data = rcu_hash_find(expr_rt->htable, key, key_len); if (NULL == data) { - fprintf(stderr, "the key:%s not exist, so can't be deleted\n", key); + log_error(logger, MODULE_TABLE_RUNTIME, "the key:%s not exist, so can't be deleted", key); return; } rcu_hash_del(expr_rt->htable, key, key_len); @@ -356,7 +433,7 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key //add data = rcu_hash_find(expr_rt->htable, key, key_len); if (data != NULL) { - fprintf(stderr, "the key:%s already exist, so can't be added\n", key); + log_error(logger, MODULE_TABLE_RUNTIME, "the key:%s already exist, so can't be added", key); return; } and_expr_t *data = ALLOC(and_expr_t, 1); @@ -369,11 +446,33 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key } } -void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_schema *table_schema, - const char *row, char *key, size_t key_len, int is_valid) +void ip_plus_runtime_update_row(struct ip_plugin_runtime *ip_plus_rt, struct table_schema *table_schema, + const char *row, char *key, size_t key_len, struct ip_plus_item *ip_plus_item, + int is_valid, struct log_handle *logger) { - struct plugin_ex_data *ex_data = ALLOC(struct plugin_ex_data, 1); - ex_data->data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); + struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); + struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt; + ex_container->ex_data = NULL; + ex_container->custom_data = ip_plus_item; + struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1); + ctx->custom_data_free = free; + ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx); + + if (0 == is_valid) { + // delete + ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger); + } else { + // add + ex_data_runtime_add_ex_container(ip_plus_rt->ex_data_rt, key, key_len, ex_container); + } +} + +void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_schema *table_schema, + const char *row, char *key, size_t key_len, int is_valid, + struct log_handle *logger) +{ + struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); + ex_container->ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema); size_t cb_count = plugin_table_schema_callback_count(table_schema); @@ -381,10 +480,10 @@ void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_sc if (1 == set_flag) { if (is_valid == 0) { // delete - ex_data_runtime_del_ex_data(plugin_rt->ex_data_rt, key, key_len); + ex_data_runtime_del_ex_container(plugin_rt->ex_data_rt, key, key_len, logger); } else { // add - ex_data_runtime_add_ex_data(plugin_rt->ex_data_rt, key, key_len, (void *)ex_data); + ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container); } } @@ -401,57 +500,70 @@ void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_sc } void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct table_schema *table_schema, - const char *row, char *key, size_t key_len, - struct ip_rule *ip_rule, int is_valid) + const char *row, char *key, size_t key_len, struct ip_plugin_item *ip_plugin_item, + int is_valid, struct log_handle *logger) { - struct plugin_ex_data *ex_data = ALLOC(struct plugin_ex_data, 1); + struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt; - ex_data->data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); - ex_data->user_data = ip_rule; + ex_container->ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); + ex_container->custom_data = ip_plugin_item; + struct ex_container_ctx *ctx = ex_data_runtime_get_ex_container_ctx(ex_data_rt); + ctx->custom_data_free = free; + int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema); if (1 == set_flag) { if (0 == is_valid) { //delete - ex_data_runtime_del_ex_data(ex_data_rt, key, key_len); + ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger); } else { //add - ex_data_runtime_add_ex_data(ip_plugin_rt->ex_data_rt, key, key_len, ex_data); + ex_data_runtime_add_ex_container(ip_plugin_rt->ex_data_rt, key, key_len, ex_container); } } else { ex_data_runtime_cache_row_put(ip_plugin_rt->ex_data_rt, row); } - } -void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, const char *row, struct table_item *table_item) +void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, + const char *row, struct table_item *table_item, struct log_handle *logger) { int is_valid = -1; char *key = NULL; size_t key_len = 0; and_expr_t *expr_rule = NULL; - struct ip_rule *ip_rule = NULL; + struct ip_plus_item *ip_plus_item = NULL; + struct ip_plugin_item *ip_plugin_item = NULL; switch (table_rt->table_type) { case TABLE_TYPE_EXPR: is_valid = table_item->expr_item.is_valid; - expr_rule = expr_item_to_expr_rule(&table_item->expr_item); + expr_rule = expr_item_to_expr_rule(&table_item->expr_item, logger); key = (char *)&(table_item->expr_item.item_id); - expr_runtime_update_row(&(table_rt->expr_rt), key, sizeof(int), expr_rule, is_valid); + expr_runtime_update_row(&(table_rt->expr_rt), key, sizeof(int), expr_rule, is_valid, logger); expr_rule_free(expr_rule); break; + case TABLE_TYPE_IP_PLUS: + is_valid = table_item->ip_plus_item.is_valid; + ip_plus_item = ALLOC(struct ip_plus_item, 1); + memcpy(ip_plus_item, &(table_item->ip_plus_item), sizeof(table_item->ip_plus_item)); + key = (char *)&(table_item->ip_plus_item.item_id); + ip_plus_runtime_update_row(&(table_rt->ip_plus_rt), table_schema, row, key, sizeof(int), + ip_plus_item, is_valid, logger); + break; case TABLE_TYPE_PLUGIN: is_valid = table_item->plugin_item.is_valid; key = table_item->plugin_item.key; key_len = table_item->plugin_item.key_len; - plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, key, key_len, is_valid); + plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, key, key_len, is_valid, logger); break; case TABLE_TYPE_IP_PLUGIN: is_valid = table_item->ip_plugin_item.is_valid; - ip_rule = ip_plugin_item_to_ip_rule(&table_item->ip_plugin_item); + ip_plugin_item = ALLOC(struct ip_plugin_item, 1); + memcpy(ip_plugin_item, &(table_item->ip_plugin_item), sizeof(table_item->ip_plugin_item)); key = (char *)&(table_item->ip_plugin_item.item_id); - ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int), ip_rule, is_valid); - FREE(ip_rule); + ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int), + ip_plugin_item, is_valid, logger); break; default: break; @@ -503,27 +615,67 @@ int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread) return ret; } +int ip_plus_runtime_commit(struct table_runtime *table_rt) +{ + struct ex_data_container **ex_container = NULL; + struct ip_rule *rules = NULL; + size_t rule_cnt = 0; + int ret = 0; + struct ip_plugin_runtime *ip_plus_rt = &(table_rt->ip_plus_rt); + + rule_cnt = ex_data_runtime_list_updating_ex_container(ip_plus_rt->ex_data_rt, &ex_container); + rules = ALLOC(struct ip_rule, rule_cnt); + + for (size_t i = 0; i < rule_cnt; i++) { + struct ip_plus_item *item = (struct ip_plus_item *)ex_container[i]->custom_data; + ip_plus_item_to_ip_rule(item, &rules[i]); + } + + struct ip_matcher *new_ip_matcher = NULL; + struct ip_matcher *old_ip_matcher = NULL; + size_t mem_used = 0; + + if (rule_cnt > 0) { + new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); + if (NULL == new_ip_matcher) { + ret = -1; + } + } + + old_ip_matcher = ip_plus_rt->ip_matcher; + ip_plus_rt->ip_matcher = new_ip_matcher; + maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); + ex_data_runtime_commit(ip_plus_rt->ex_data_rt); + table_rt->rule_num = ex_data_runtime_ex_container_count(ip_plus_rt->ex_data_rt); + + FREE(rules); + FREE(ex_container); + + return ret; +} + int plugin_runtime_commit(struct table_runtime *table_rt) { ex_data_runtime_commit(table_rt->plugin_rt.ex_data_rt); - table_rt->rule_num = ex_data_runtime_ex_data_count(table_rt->plugin_rt.ex_data_rt); + table_rt->rule_num = ex_data_runtime_ex_container_count(table_rt->plugin_rt.ex_data_rt); return 0; } int ip_plugin_runtime_commit(struct table_runtime *table_rt) { - struct plugin_ex_data **ex_data_array = NULL; + struct ex_data_container **ex_container = NULL; struct ip_rule *rules = NULL; size_t rule_cnt = 0; int ret = 0; struct ip_plugin_runtime *ip_plugin_rt = &(table_rt->ip_plugin_rt); - rule_cnt = ex_data_runtime_list_updating_ex_data(ip_plugin_rt->ex_data_rt, (void ***)&ex_data_array); + rule_cnt = ex_data_runtime_list_updating_ex_container(ip_plugin_rt->ex_data_rt, &ex_container); rules = ALLOC(struct ip_rule, rule_cnt); for (size_t i = 0; i < rule_cnt; i++) { - rules[i] = *(struct ip_rule *)ex_data_array[i]->user_data; + struct ip_plugin_item *item = (struct ip_plugin_item *)ex_container[i]->custom_data; + ip_plugin_item_to_ip_rule(item, &rules[i]); } struct ip_matcher *new_ip_matcher = NULL; @@ -541,10 +693,10 @@ int ip_plugin_runtime_commit(struct table_runtime *table_rt) ip_plugin_rt->ip_matcher = new_ip_matcher; maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); ex_data_runtime_commit(ip_plugin_rt->ex_data_rt); - table_rt->rule_num = ex_data_runtime_ex_data_count(ip_plugin_rt->ex_data_rt); + table_rt->rule_num = ex_data_runtime_ex_container_count(ip_plugin_rt->ex_data_rt); FREE(rules); - FREE(ex_data_array); + FREE(ex_container); return ret; } @@ -591,8 +743,12 @@ void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_threa case TABLE_TYPE_EXPR: expr_runtime_commit(table_rt, nr_worker_thread); break; + case TABLE_TYPE_IP_PLUS: + ip_plus_runtime_commit(table_rt); + break; case TABLE_TYPE_PLUGIN: plugin_runtime_commit(table_rt); + break; case TABLE_TYPE_IP_PLUGIN: ip_plugin_runtime_commit(table_rt); break; @@ -635,35 +791,14 @@ const char* table_runtime_get_cached_row(struct table_runtime* table_rt, size_t return line; } -void *table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema, - const char *key, size_t key_len) -{ - void *ex_data = NULL; - int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema); - if (0 == set_flag) { - assert(0); - return NULL; - } - - enum table_type table_type = table_schema_get_table_type(table_schema); - switch (table_type) { - case TABLE_TYPE_PLUGIN: - ex_data = ex_data_runtime_get_ex_data(table_rt->plugin_rt.ex_data_rt, key, key_len); - break; - case TABLE_TYPE_IP_PLUGIN: - break; - default: - break; - } - - return ex_data; -} - struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt) { struct ex_data_runtime *ex_data_rt = NULL; switch (table_rt->table_type) { + case TABLE_TYPE_IP_PLUS: + ex_data_rt = table_rt->ip_plus_rt.ex_data_rt; + break; case TABLE_TYPE_PLUGIN: ex_data_rt = table_rt->plugin_rt.ex_data_rt; break; @@ -677,26 +812,27 @@ struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table return ex_data_rt; } -void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema) +void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema, + struct log_handle *logger) { struct ex_data_schema *ex_data_schema = plugin_table_schema_get_ex_data_schema(table_schema); struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt); ex_data_runtime_set_schema(ex_data_rt, ex_data_schema); - struct plugin_user_ctx *user_ctx = ALLOC(struct plugin_user_ctx, 1); + struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1); - user_ctx->table_id = table_schema_get_table_id(table_schema); - user_ctx->ex_schema = ex_data_schema; - ex_data_runtime_set_user_ctx(ex_data_rt, user_ctx); + ctx->table_id = table_schema_get_table_id(table_schema); + ctx->ex_schema = ex_data_schema; + ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx); size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt); for (size_t i = 0; i < n_cached_row; i++) { const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i); switch (table_rt->table_type) { case TABLE_TYPE_PLUGIN: - plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, NULL, 0, 1); + plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, NULL, 0, 1, logger); break; case TABLE_TYPE_IP_PLUGIN: - ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, NULL, NULL, 0, NULL, 1); + ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, NULL, NULL, 0, NULL, 1, logger); break; default: break; diff --git a/src/maat_table_schema.cpp b/src/maat_table_schema.cpp index 50bc05f..22b9bba 100644 --- a/src/maat_table_schema.cpp +++ b/src/maat_table_schema.cpp @@ -12,8 +12,10 @@ #include #include #include +#include #include "utils.h" +#include "log/log.h" #include "maat_utils.h" #include "maat_table_schema.h" #include "maat_kv.h" @@ -21,6 +23,8 @@ #include "adapter_hs.h" #include "maat_ex_data.h" +#define MODULE_TABLE_SCHEMA module_name_str("maat.table_schema") + #define MAX_TABLE_LINE_SIZE (1024 * 16) #define MAX_FOREIGN_CLMN_NUM 8 #define MAX_CONJUNCTION_TABLE_NUM 8 @@ -37,6 +41,44 @@ struct expr_table_schema { enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ }; +struct ip_table_schema { + int item_id_column; + int group_id_column; + int addr_type_column; + int sip_column; + int sip_mask_column; + int sport_column; + int sport_mask_column; + int dip_column; + int dip_mask_column; + int dport_column; + int dport_mask_column; + int proto_column; + int direction_column; + int is_valid_column; +}; + +struct ip_plus_table_schema { + int item_id_column; + int group_id_column; + int addr_type_column; + int saddr_format_column; + int sip1_column; + int sip2_column; + int sport_format_column; + int sport1_column; + int sport2_column; + int daddr_format_column; + int dip1_column; + int dip2_column; + int dport_format_column; + int dport1_column; + int dport2_column; + int proto_column; + int direction_column; + int is_valid_column; +}; + #define MAX_PLUGIN_PER_TABLE 32 struct plugin_table_schema { int item_id_column; @@ -65,6 +107,11 @@ struct virtual_table_schema { int physical_table_id[SCAN_TYPE_MAX]; }; +struct composition_table_schema +{ + int component_table_id[COMPOSITION_TYPE_MAX]; +}; + struct table_schema { int table_id; int conj_cnt; @@ -73,9 +120,12 @@ struct table_schema { enum table_type table_type; union { struct expr_table_schema expr; + struct ip_table_schema ip; + struct ip_plus_table_schema ip_plus; struct plugin_table_schema plugin; struct ip_plugin_table_schema ip_plugin; struct virtual_table_schema virtual_table; + struct composition_table_schema composition; }; }; @@ -84,6 +134,7 @@ struct table_schema_manager { size_t n_schema_table; size_t n_active_plugin_table; struct maat_kv_store *tablename2id_map; + struct log_handle *logger; }; struct table_schema *table_schema_new(void) @@ -143,9 +194,9 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable, read_cnt++; } - json = cJSON_GetObjectItem(root, "rule"); + json = cJSON_GetObjectItem(root, "custom"); if (json == NULL || json->type != cJSON_Object) { - fprintf(stderr, "table %s has no rule\n", ptable->table_name[0]); + fprintf(stderr, "table %s has no custom column\n", ptable->table_name[0]); return -1; } @@ -189,9 +240,391 @@ int read_expr_table_schema(cJSON *root, struct table_schema *ptable, if (read_cnt < 10) { return -1; - } else { - return 0; } + + return 0; +} + +int read_ip_table_schema(cJSON *root, struct table_schema *ptable) +{ + size_t read_cnt = 0; + cJSON *json = NULL; + + json = cJSON_GetObjectItem(root, "table_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->table_id = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "table_name"); + if (json != NULL && json->type == cJSON_String) { + if (strlen(json->valuestring) >= NAME_MAX) { + fprintf(stderr, "table name %s length too long\n", json->valuestring); + return -1; + } + memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "item_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->ip.item_id_column = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "group_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->ip.group_id_column = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "custom"); + if (NULL == json || json->type != cJSON_Object) { + fprintf(stderr, "table %s has no custom column\n", ptable->table_name[0]); + return -1; + } + + cJSON *item = cJSON_GetObjectItem(json, "addr_type"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.addr_type_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "sip"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.sip_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "sip_mask"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.sip_mask_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "sport"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.sport_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "sport_mask"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.sport_mask_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "dip"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.dip_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "dip_mask"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.dip_mask_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "dport"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.dport_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "dport_mask"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.dport_mask_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "proto"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.proto_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "direction"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.direction_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "is_valid"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip.is_valid_column = item->valueint; + read_cnt++; + } + + if (read_cnt < 16) { + return -1; + } + + return 0; +} + +int read_ip_plus_table_schema(cJSON *root, struct table_schema *ptable) +{ + size_t read_cnt = 0; + cJSON *json = NULL; + + json = cJSON_GetObjectItem(root, "table_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->table_id = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "table_name"); + if (json != NULL && json->type == cJSON_String) { + if (strlen(json->valuestring) >= NAME_MAX) { + fprintf(stderr, "table name %s length too long\n", json->valuestring); + return -1; + } + memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "item_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->ip_plus.item_id_column = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "group_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->ip_plus.group_id_column = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "custom"); + if (NULL == json || json->type != cJSON_Object) { + fprintf(stderr, "table %s has no custom column\n", ptable->table_name[0]); + return -1; + } + + cJSON *item = cJSON_GetObjectItem(json, "addr_type"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.addr_type_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "saddr_format"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.saddr_format_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "sip1"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.sip1_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "sip2"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.sip2_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "sport_format"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.sport_format_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "sport1"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.sport1_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "sport2"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.sport2_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "daddr_format"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.daddr_format_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "dip1"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.dip1_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "dip2"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.dip2_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "dport_format"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.dport_format_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "dport1"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.dport1_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "dport2"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.dport2_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "proto"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.proto_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "direction"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.direction_column = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "is_valid"); + if (item != NULL && item->type == cJSON_Number) { + ptable->ip_plus.is_valid_column = item->valueint; + read_cnt++; + } + + if (read_cnt < 20) { + return -1; + } + + return 0; +} + +int read_virtual_table_schema(cJSON *root, struct table_schema **table_array, struct table_schema *ptable, + struct maat_kv_store *tablename2id_map) +{ + size_t read_cnt = 0; + cJSON *json = NULL; + + json = cJSON_GetObjectItem(root, "table_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->table_id = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "table_name"); + if (json != NULL && json->type == cJSON_String) { + if (strlen(json->valuestring) >= NAME_MAX) { + fprintf(stderr, "table name %s length too long\n", json->valuestring); + return -1; + } + memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "physical_table"); + if (NULL == json || json->type != cJSON_Array) { + fprintf(stderr, "table %s has no physical_table column\n", ptable->table_name[0]); + return -1; + } + read_cnt++; + + int cnt = cJSON_GetArraySize(json); + for (int i = 0; i < cnt; i++) { + cJSON *item = cJSON_GetArrayItem(json, i); + if (item != NULL && item->type == cJSON_String) { + int table_id = -1; + /* physical table should already exist */ + int ret = maat_kv_read(tablename2id_map, item->valuestring, &table_id); + if (ret < 0) { + return -1; + } + + enum scan_type table_scan_type = table_schema_get_scan_type(table_array[table_id]); + ptable->virtual_table.physical_table_id[table_scan_type]= table_id; + } + } + + if (read_cnt < 3) { + return -1; + } + + return 0; +} + +int read_composition_table_schema(cJSON *root, struct table_schema *ptable, struct maat_kv_store *tablename2id_map) +{ + size_t read_cnt = 0; + cJSON *json = NULL; + + json = cJSON_GetObjectItem(root, "table_id"); + if (json != NULL && json->type == cJSON_Number) { + ptable->table_id = json->valueint; + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "table_name"); + if (json != NULL && json->type == cJSON_String) { + if (strlen(json->valuestring) >= NAME_MAX) { + fprintf(stderr, "table name %s length too long\n", json->valuestring); + return -1; + } + memcpy(ptable->table_name[0], json->valuestring, strlen(json->valuestring)); + read_cnt++; + } + + json = cJSON_GetObjectItem(root, "composition_table"); + if (NULL == json || json->type != cJSON_Object) { + fprintf(stderr, "table %s has no composition_table column\n", ptable->table_name[0]); + return -1; + } + read_cnt++; + + int ret = 0; + struct composition_table_schema *composition_schema = &(ptable->composition); + cJSON *item = cJSON_GetObjectItem(json, "source"); + if (item != NULL && item->type == cJSON_String) { + /* physical table should already exist */ + ret = maat_kv_read(tablename2id_map, item->valuestring, &composition_schema->component_table_id[COMPOSITION_TYPE_SIP]); + if (ret < 0) { + fprintf(stderr, "Child table %s of table %s (id=%d) are not defined", + item->valuestring, ptable->table_name[0], ptable->table_id); + return -1; + } + } + + item = cJSON_GetObjectItem(json, "dest"); + if (item != NULL && item->type == cJSON_String) { + /* physical table should already exist */ + ret = maat_kv_read(tablename2id_map, item->valuestring, &composition_schema->component_table_id[COMPOSITION_TYPE_DIP]); + if (ret < 0) { + fprintf(stderr, "Child table %s of table %s (id=%d) are not defined", + item->valuestring, ptable->table_name[0], ptable->table_id); + return -1; + } + } + + item = cJSON_GetObjectItem(json, "session"); + if (item != NULL && item->type == cJSON_String) { + /* physical table should already exist */ + ret = maat_kv_read(tablename2id_map, item->valuestring, &composition_schema->component_table_id[COMPOSITION_TYPE_SESSION]); + if (ret < 0) { + fprintf(stderr, "Child table %s of table %s (id=%d) are not defined", + item->valuestring, ptable->table_name[0], ptable->table_id); + return -1; + } + } + + if (read_cnt < 3) { + return -1; + } + + return 0; } static int read_integer_array(char *string, int *array, int size) @@ -238,9 +671,9 @@ int read_plugin_table_schema(cJSON *root, struct table_schema *ptable) read_cnt++; } - json = cJSON_GetObjectItem(root, "rule"); + json = cJSON_GetObjectItem(root, "custom"); if (json == NULL || json->type != cJSON_Object) { - fprintf(stderr, "table %s has no rule\n", ptable->table_name[0]); + fprintf(stderr, "table %s has no custom column\n", ptable->table_name[0]); return -1; } @@ -281,9 +714,9 @@ int read_plugin_table_schema(cJSON *root, struct table_schema *ptable) if (read_cnt < 7) { return -1; - } else { - return 0; } + + return 0; } int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable) @@ -313,9 +746,9 @@ int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable) read_cnt++; } - json = cJSON_GetObjectItem(root, "rule"); + json = cJSON_GetObjectItem(root, "custom"); if (NULL == json || json->type != cJSON_Object) { - fprintf(stderr, "table %s has no rule\n", ptable->table_name[0]); + fprintf(stderr, "table %s has no custom column\n", ptable->table_name[0]); return -1; } @@ -345,13 +778,13 @@ int read_ip_plugin_table_schema(cJSON *root, struct table_schema *ptable) if (read_cnt < 7) { return -1; - } else { - return 0; } + + return 0; } -int table_schema_populate(cJSON *json, struct table_schema *ptable, - struct maat_kv_store* reserved_word_map) +int table_schema_populate(cJSON *json, struct table_schema **table_array, struct table_schema *ptable, + struct maat_kv_store *reserved_word_map, struct maat_kv_store *tablename2id_map) { int ret = -1; cJSON *item = NULL; @@ -373,6 +806,19 @@ int table_schema_populate(cJSON *json, struct table_schema *ptable, return -1; } break; + /* TODO: delete + case TABLE_TYPE_IP: + ret = read_ip_table_schema(json, ptable); + if (ret < 0) { + return -1; + } + break; */ + case TABLE_TYPE_IP_PLUS: + ret = read_ip_plus_table_schema(json, ptable); + if (ret < 0) { + return -1; + } + break; case TABLE_TYPE_PLUGIN: ret = read_plugin_table_schema(json, ptable); if (ret < 0) { @@ -385,6 +831,18 @@ int table_schema_populate(cJSON *json, struct table_schema *ptable, return -1; } break; + case TABLE_TYPE_VIRTUAL: + ret = read_virtual_table_schema(json, table_array, ptable, tablename2id_map); + if (ret < 0) { + return -1; + } + break; + case TABLE_TYPE_COMPOSITION: + ret = read_composition_table_schema(json, ptable, tablename2id_map); + if (ret < 0) { + return -1; + } + break; default: break; } @@ -392,7 +850,7 @@ int table_schema_populate(cJSON *json, struct table_schema *ptable, return 0; } -struct table_schema_manager *table_schema_manager_create(const char *table_info_path) +struct table_schema_manager *table_schema_manager_create(const char *table_info_path, struct log_handle *logger) { unsigned char *json_buff = NULL; size_t json_buff_sz = 0; @@ -403,21 +861,21 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_ int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz); if (ret < 0) { - fprintf(stderr, "Maat read table info %s error.\n", table_info_path); + log_error(logger, MODULE_TABLE_SCHEMA, "Maat read table info %s error.", table_info_path); return NULL; } cJSON *root = NULL; root = cJSON_Parse((const char *)json_buff); if (!root) { - fprintf(stderr, "Error before: %-200.200s\n", cJSON_GetErrorPtr()); + log_error(logger, MODULE_TABLE_SCHEMA, "Error before: %-200.200s", cJSON_GetErrorPtr()); FREE(json_buff); return NULL; } int json_array_size = cJSON_GetArraySize(root); if (json_array_size <= 0) { - fprintf(stderr, "invalid json content in %s\n", table_info_path); + log_error(logger, MODULE_TABLE_SCHEMA, "invalid json content in %s", table_info_path); free(json_buff); return NULL; } @@ -426,14 +884,18 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_ /* register table type reserved word */ maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); - maat_kv_register(reserved_word_map, "ip", TABLE_TYPE_IP); + //maat_kv_register(reserved_word_map, "ip", TABLE_TYPE_IP); + maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); + maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); + maat_kv_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION); maat_kv_register(reserved_word_map, "block", HS_SCAN_MODE_BLOCK); maat_kv_register(reserved_word_map, "stream", HS_SCAN_MODE_STREAM); struct table_schema_manager *table_schema_mgr = ALLOC(struct table_schema_manager, 1); + table_schema_mgr->logger = logger; struct table_schema **table_array = table_schema_mgr->schema_table; table_schema_mgr->tablename2id_map = maat_kv_store_new(); @@ -443,37 +905,43 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_ if (json != NULL && json->type == cJSON_Object) { struct table_schema *ptable = table_schema_new(); - int ret = table_schema_populate(json, ptable, reserved_word_map); + int ret = table_schema_populate(json, table_array, ptable, reserved_word_map, + table_schema_mgr->tablename2id_map); if (ret < 0) { - fprintf(stderr, "Maat populate table info error, table_id:%d\n", ptable->table_id); + log_error(logger, MODULE_TABLE_SCHEMA, + "Maat populate table info error, table_id:%d", ptable->table_id); goto free_table; } if (ptable->table_id >= MAX_TABLE_NUM) { - fprintf(stderr, "Maat read table info %s:%d error: table id %u > %d\n", - table_info_path, i, ptable->table_id, MAX_TABLE_NUM); + log_error(logger, MODULE_TABLE_SCHEMA, + "Maat read table info %s:%d error: table id %u > %d", + table_info_path, i, ptable->table_id, MAX_TABLE_NUM); goto free_table; } ret = maat_kv_register(table_schema_mgr->tablename2id_map, ptable->table_name[0], ptable->table_id); if (ret < 0) { - fprintf(stderr, "Duplicate table %s of table id %d\n", ptable->table_name[0], ptable->table_id); + log_error(logger, MODULE_TABLE_SCHEMA, + "Duplicate table %s of table id %d", + ptable->table_name[0], ptable->table_id); goto free_table; } - /* TODO: conjunction table*/ if (table_array[ptable->table_id] != NULL) { conj_table = table_array[ptable->table_id]; if (conj_table->conj_cnt == MAX_CONJUNCTION_TABLE_NUM) { - fprintf(stderr, "Maat read table info %s error: reach table_id %d conjunction upper limit\n", - table_info_path, ptable->table_id); + log_error(logger, MODULE_TABLE_SCHEMA, + "Maat read table info %s error: reach table_id %d conjunction upper limit", + table_info_path, ptable->table_id); goto free_table; } memcpy(conj_table->table_name[conj_table->conj_cnt], ptable->table_name[0], NAME_MAX); conj_table->conj_cnt++; - fprintf(stdout, "Maat read table info %s: conjunction %s with %s (id=%d,total=%d)\n", - table_info_path, ptable->table_name[0], conj_table->table_name[0], - conj_table->table_id, conj_table->conj_cnt); + log_info(logger, MODULE_TABLE_SCHEMA, + "Maat read table info %s: conjunction %s with %s (id=%d,total=%d)", + table_info_path, ptable->table_name[0], conj_table->table_name[0], + conj_table->table_id, conj_table->conj_cnt); goto free_table; } @@ -526,6 +994,17 @@ int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_ } } +int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id, + enum table_composition_type type) +{ + struct table_schema* table_schema = table_schema_get(table_schema_mgr, parent_table_id); + if (table_schema->table_type != TABLE_TYPE_COMPOSITION) { + return -1; + } + + return table_schema->composition.component_table_id[type]; +} + enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int id) { if (NULL == table_schema_mgr) { @@ -732,7 +1211,7 @@ enum scan_type table_schema_get_scan_type(struct table_schema *table_schema) case TABLE_TYPE_INTERVAL_PLUS: ret = SCAN_TYPE_INTERVAL; break; - case TABLE_TYPE_IP: + //case TABLE_TYPE_IP: case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_COMPOSITION: ret = SCAN_TYPE_IP; @@ -807,10 +1286,13 @@ void table_schema_set_updating_name(struct table_schema *table_schema, const cha assert(i <= table_schema->conj_cnt); } -int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, struct expr_item *expr_item) +int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, + struct table_item *table_item, const char *table_name, + struct log_handle *logger) { size_t column_offset = 0; size_t column_len = 0; + struct expr_item *expr_item = &(table_item->expr_item); int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len); if (ret < 0) { @@ -824,19 +1306,20 @@ int populate_expr_table_item(const char *line, struct expr_table_schema *expr_sc } expr_item->group_id = atoi(line + column_offset); - /* TODO: luis */ - #if 0 - ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); - if (ret < 0) { - return -1; - } + if (table_item->table_type == TABLE_TYPE_EXPR_PLUS) { + ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } - if (column_len >= MAX_DISTRICT_STR) { - //log_error - return -1; + if (column_len >= MAX_DISTRICT_STR) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: expr table[%s]:item_id[%d] district length too long", + table_name, expr_item->item_id); + return -1; + } + memcpy(expr_item->district, (line + column_offset), column_len); } - memcpy(expr_item->district, (line + column_offset), column_len); - #endif ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); if (ret < 0) { @@ -844,7 +1327,9 @@ int populate_expr_table_item(const char *line, struct expr_table_schema *expr_sc } if (column_len >= MAX_KEYWORDS_STR) { - //log_error + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: expr table[%s]:item_id[%d] keywords length too long", + table_name, expr_item->item_id); return -1; } memcpy(expr_item->keywords, (line + column_offset), column_len); @@ -877,14 +1362,16 @@ int populate_expr_table_item(const char *line, struct expr_table_schema *expr_sc break; case 1: expr_item->is_hexbin = TRUE; - expr_item->is_case_sensitive = FALSE; + expr_item->is_case_sensitive = TRUE; break; case 2: expr_item->is_hexbin = FALSE; expr_item->is_case_sensitive = TRUE; break; default: - fprintf(stderr, "invalid hexbin value of expr table\n"); + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: expr table[%s]:item_id[%d] invalid hexbin value:%d", + table_name, expr_item->item_id, db_hexbin); return -1; } ret = get_column_pos(line, expr_schema->is_valid_column, &column_offset, &column_len); @@ -896,10 +1383,184 @@ int populate_expr_table_item(const char *line, struct expr_table_schema *expr_sc return 0; } -int populate_plugin_table_item(const char *line, struct plugin_table_schema *plugin_schema, struct plugin_item *plugin_item) +int populate_ip_plus_table_item(const char *line, struct ip_plus_table_schema *ip_plus_schema, + struct table_item *table_item, const char *table_name, + struct log_handle *logger) { size_t column_offset = 0; size_t column_len = 0; + struct ip_plus_item *ip_plus_item = &(table_item->ip_plus_item); + + int ret = get_column_pos(line, ip_plus_schema->item_id_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + ip_plus_item->item_id = atoi(line + column_offset); + + ret = get_column_pos(line, ip_plus_schema->group_id_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + ip_plus_item->group_id = atoi(line + column_offset); + + ret = get_column_pos(line, ip_plus_schema->addr_type_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + ip_plus_item->addr_type = atoi(line + column_offset); + if (ip_plus_item->addr_type != 4 && ip_plus_item->addr_type != 6) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: ip_plus table[%s]:item_id[%d] invalid addr type:%d", + table_name, ip_plus_item->item_id, ip_plus_item->addr_type); + return -1; + } + + char saddr_format[16] = {0}; + char sport_format[16] = {0}; + char sip1_str[40] = {0}; + char sip2_str[40] = {0}; + uint16_t sport1 = 0; + uint16_t sport2 = 0; + + ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + memcpy(saddr_format, (line + column_offset), column_len); + if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: ip_plus table[%s]:item_id[%d] invalid saddr_format, should be range/mask/CIDR", + table_name, ip_plus_item->item_id); + return -1; + } + + ret = get_column_pos(line, ip_plus_schema->sip1_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + memcpy(sip1_str, (line + column_offset), column_len); + + ret = get_column_pos(line, ip_plus_schema->sip2_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + memcpy(sip2_str, (line + column_offset), column_len); + + ret = get_column_pos(line, ip_plus_schema->sport_format_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + memcpy(sport_format, (line + column_offset), column_len); + if (IP_FORMAT_UNKNOWN == ip_format_str2int(sport_format)) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: ip_plus table[%s]:item_id[%d] invalid sport_format, should be range/mask/CIDR", + table_name, ip_plus_item->item_id); + return -1; + } + + ret = get_column_pos(line, ip_plus_schema->sport1_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + sport1 = atoi(line + column_offset); + + ret = get_column_pos(line, ip_plus_schema->sport2_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + sport2 = atoi(line + column_offset); + + uint16_t protocol = 0; + uint16_t direction = 0; + if (4 == ip_plus_item->addr_type) { + ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, + &ip_plus_item->ipv4.min_sip, &ip_plus_item->ipv4.max_sip); + if (ret < 0) { + return -1; + } + + if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { + ip_plus_item->ipv4.min_sport = sport1 & sport2; + ip_plus_item->ipv4.max_sport = sport1 | ~sport2; + } else { + ip_plus_item->ipv4.min_sport = sport1; + ip_plus_item->ipv4.max_sport = sport2; + } + + ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + ip_plus_item->ipv4.proto = atoi(line + column_offset); + protocol = ip_plus_item->ipv4.proto; + + ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + ip_plus_item->ipv4.direction = atoi(line + column_offset); + direction = ip_plus_item->ipv4.direction; + } else { + //ipv6 + ret = ip_format2range(ip_plus_item->addr_type, IP_FORMAT_MASK, sip1_str, sip2_str, + ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip); + if (ret < 0) { + return -1; + } + + if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { + ip_plus_item->ipv6.min_sport = sport1 & sport2; + ip_plus_item->ipv6.max_sport = sport1 | ~sport2; + } else { + ip_plus_item->ipv6.min_sport = sport1; + ip_plus_item->ipv6.max_sport = sport2; + } + + ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + ip_plus_item->ipv6.proto = atoi(line + column_offset); + protocol = ip_plus_item->ipv6.proto; + + ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + ip_plus_item->ipv6.direction = atoi(line + column_offset); + direction = ip_plus_item->ipv6.direction; + } + + if (protocol > 65535 || protocol < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: ip_plus table[%s]:item_id[%d] invalid protocol:%d", + table_name, ip_plus_item->item_id, protocol); + return -1; + } + + if (direction != 0 && direction != 1) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: ip_plus table[%s]:item_id[%d] invalid direction:%d", + table_name, ip_plus_item->item_id, direction); + return -1; + } + + ret = get_column_pos(line, ip_plus_schema->is_valid_column, &column_offset, &column_len); + if (ret < 0) { + return -1; + } + ip_plus_item->is_valid = atoi(line + column_offset); + + return 0; +} + +int populate_plugin_table_item(const char *line, struct plugin_table_schema *plugin_schema, + struct table_item *table_item, const char *table_name, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + struct plugin_item *plugin_item = &(table_item->plugin_item); int ret = get_column_pos(line, plugin_schema->key_column, &column_offset, &column_len); if (ret < 0) { @@ -907,7 +1568,9 @@ int populate_plugin_table_item(const char *line, struct plugin_table_schema *plu } if (column_len > MAX_KEYWORDS_STR) { - fprintf(stderr, "plugin table's key len:%zu, exceed %d\n", column_len, MAX_KEYWORDS_STR); + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: plugin table[%s]: key:%s length:%zu too long, exceed %d", + table_name, (line + column_offset), column_len, MAX_KEYWORDS_STR); return -1; } memcpy(plugin_item->key, (line + column_offset), column_len); @@ -922,11 +1585,14 @@ int populate_plugin_table_item(const char *line, struct plugin_table_schema *plu return 0; } -int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schema *ip_plugin_schema, struct ip_plugin_item *ip_plugin_item) +int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schema *ip_plugin_schema, + struct table_item *table_item, const char *table_name, + struct log_handle *logger) { size_t column_offset = 0; size_t column_len = 0; - + struct ip_plugin_item *ip_plugin_item = &(table_item->ip_plugin_item); + int ret = get_column_pos(line, ip_plugin_schema->item_id_column, &column_offset, &column_len); if (ret < 0) { return -1; @@ -935,6 +1601,9 @@ int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schem ret = get_column_pos(line, ip_plugin_schema->is_valid_column, &column_offset, &column_len); if (ret < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: ip_plugin table[%s]:item_id[%d] not found is_valid value", + table_name, ip_plugin_item->item_id); return -1; } ip_plugin_item->is_valid = atoi(line + column_offset); @@ -945,17 +1614,26 @@ int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schem } ip_plugin_item->ip_type = atoi(line + column_offset); if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: ip_plugin table[%s]:item_id[%d] ip_type[%d] invalid", + table_name, ip_plugin_item->item_id, ip_plugin_item->ip_type); return -1; } ret = get_column_pos(line, ip_plugin_schema->start_ip_column, &column_offset, &column_len); if (ret < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: ip_plugin table[%s]:item_id[%d] no found start_ip value", + table_name, ip_plugin_item->item_id, ip_plugin_item->start_ip); return -1; } strncpy(ip_plugin_item->start_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->start_ip))); ret = get_column_pos(line, ip_plugin_schema->end_ip_column, &column_offset, &column_len); if (ret < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "update error: ip_plugin table[%s]:item_id[%d] no found end_ip value", + table_name, ip_plugin_item->item_id, ip_plugin_item->end_ip); return -1; } strncpy(ip_plugin_item->end_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->end_ip))); @@ -964,7 +1642,7 @@ int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schem } struct table_item * -table_schema_line_to_item(const char *line, struct table_schema *table_schema) +table_schema_line_to_item(const char *line, struct table_schema *table_schema, struct log_handle *logger) { if (NULL == line || NULL == table_schema) { return NULL; @@ -975,23 +1653,47 @@ table_schema_line_to_item(const char *line, struct table_schema *table_schema) switch (table_schema->table_type) { case TABLE_TYPE_EXPR: - table_item->table_type = TABLE_TYPE_EXPR; - ret = populate_expr_table_item(line, &table_schema->expr, &table_item->expr_item); + case TABLE_TYPE_EXPR_PLUS: + table_item->table_type = table_schema->table_type; + ret = populate_expr_table_item(line, &table_schema->expr, table_item, + table_schema->table_name[table_schema->updating_name], logger); if (ret < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "abandon config: invalid format of expr table %s:%s", + table_schema->table_name[table_schema->updating_name], line); + goto error; + } + break; + case TABLE_TYPE_IP_PLUS: + table_item->table_type = table_schema->table_type; + ret = populate_ip_plus_table_item(line, &table_schema->ip_plus, table_item, + table_schema->table_name[table_schema->updating_name], logger); + if (ret < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "abandon config: invalid format of ip_plus table %s:%s", + table_schema->table_name[table_schema->updating_name], line); goto error; } break; case TABLE_TYPE_PLUGIN: table_item->table_type = TABLE_TYPE_PLUGIN; - ret = populate_plugin_table_item(line, &table_schema->plugin, &table_item->plugin_item); + ret = populate_plugin_table_item(line, &table_schema->plugin, table_item, + table_schema->table_name[table_schema->updating_name], logger); if (ret < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "abandon config: invalid format of plugin table %s:%s", + table_schema->table_name[table_schema->updating_name], line); goto error; } break; case TABLE_TYPE_IP_PLUGIN: table_item->table_type = TABLE_TYPE_IP_PLUGIN; - ret = populate_ip_plugin_table_item(line, &table_schema->ip_plugin, &table_item->ip_plugin_item); + ret = populate_ip_plugin_table_item(line, &table_schema->ip_plugin, table_item, + table_schema->table_name[table_schema->updating_name], logger); if (ret < 0) { + log_error(logger, MODULE_TABLE_SCHEMA, + "abandon config: invalid format of ip_plugin table %s:%s", + table_schema->table_name[table_schema->updating_name], line); goto error; } break; @@ -1114,7 +1816,7 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m switch (ptable->table_type) { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - case TABLE_TYPE_IP: + case TABLE_TYPE_IP_PLUS: fprintf(stderr, "table_id: %d is not plugin type\n", table_id); return -1; default: diff --git a/src/maat_utils.cpp b/src/maat_utils.cpp index 7a21cfc..f86cf5b 100644 --- a/src/maat_utils.cpp +++ b/src/maat_utils.cpp @@ -10,7 +10,9 @@ #include #include +#include #include +#include #include #include #include @@ -18,6 +20,19 @@ #include "utils.h" #include "maat_utils.h" +pid_t gettid() +{ + return syscall(SYS_gettid); +} + +const char *module_name_str(const char *name) +{ + static __thread char module[64]; + snprintf(module,sizeof(module),"%s(%d)", name, gettid()); + + return module; +} + char *maat_strdup(const char *s) { if (NULL == s) { @@ -378,3 +393,129 @@ size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_ return n_src; } + +enum maat_ip_format ip_format_str2int(const char *format) +{ + if (0 == strcasecmp(format, "range")) { + return IP_FORMAT_RANGE; + } else if(0 == strcasecmp(format, "mask")) { + return IP_FORMAT_MASK; + } else if(0 == strcasecmp(format, "CIDR")) { + return IP_FORMAT_CIDR; + } else { + assert(0); + } + return IP_FORMAT_UNKNOWN; +} + +int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, const char *ip2, + uint32_t range_begin[], uint32_t range_end[]) +{ + int cidr = 0; + int ret = 0; + + if (ip_type != 4 && ip_type != 6) { + assert(0); + return -1; + } + + if (ip_type == 4) { + uint32_t ipv4_addr = 0; + ret = inet_pton(AF_INET, ip1, &ipv4_addr); + if (ret <= 0) { + return -1; + } + + ipv4_addr = ntohl(ipv4_addr); + uint32_t ipv4_range_end = 0; + uint32_t ipv4_mask = 0; + switch (format) { + case IP_FORMAT_RANGE: + range_begin[0] = ipv4_addr; + ret = inet_pton(AF_INET, ip2, &ipv4_range_end); + if (ret <= 0) { + return -1; + } + ipv4_range_end = ntohl(ipv4_range_end); + range_end[0] = ipv4_range_end; + break; + case IP_FORMAT_MASK: + ret = inet_pton(AF_INET, ip2, &ipv4_mask); + if (ret <= 0) { + return -1; + } + ipv4_mask = ntohl(ipv4_mask); + range_begin[0] = ipv4_addr & ipv4_mask; + range_end[0] = ipv4_addr | ~ipv4_mask; + break; + case IP_FORMAT_CIDR: + cidr = atoi(ip2); + if (cidr > 32 || cidr < 0) { + return -1; + } + ipv4_mask = (0xFFFFFFFFUL << (32 - cidr)) & 0xFFFFFFFFUL; + range_begin[0] = ipv4_addr & ipv4_mask; + range_end[0] = ipv4_addr | ~ipv4_mask; + break; + default: + assert(0); + } + } else { + //ipv6 + int i = 0; + uint32_t ipv6_addr[4] = {0}; + uint32_t ipv6_mask[4] = {0}; + uint32_t ipv6_range_end[4] = {0}; + + ret = inet_pton(AF_INET6, ip1, ipv6_addr); + if (ret <= 0) { + return -1; + } + ipv6_ntoh(ipv6_addr); + + switch (format) { + case IP_FORMAT_RANGE: + ret = inet_pton(AF_INET6, ip2, ipv6_range_end); + if (ret <= 0) { + return -1; + } + ipv6_ntoh(ipv6_range_end); + + memcpy(range_begin, ipv6_addr, sizeof(ipv6_addr)); + memcpy(range_end, ipv6_range_end, sizeof(ipv6_range_end)); + break; + case IP_FORMAT_MASK: + ret = inet_pton(AF_INET6, ip2, ipv6_mask); + if (ret <= 0) { + return -1; + } + ipv6_ntoh(ipv6_mask); + + for (i = 0; i < 4; i++) { + range_begin[i]=ipv6_addr[i] & ipv6_mask[i]; + range_end[i] = ipv6_addr[i] | ~ipv6_mask[i]; + } + break; + case IP_FORMAT_CIDR: + cidr = atoi(ip2); + if (cidr > 128 || cidr < 0) { + return -1; + } + + for (i = 0; i < 4; i++) { + int bit32 = 128 - cidr - 32 * (3 - i); + if (bit32 < 0) { + bit32 = 0; + } + + ipv6_mask[i] = (0xFFFFFFFFUL << bit32) & 0xFFFFFFFFUL; + range_begin[i] = ipv6_addr[i] & ipv6_mask[i]; + range_end[i] = ipv6_addr[i] | ~ipv6_mask[i]; + } + break; + default: + assert(0); + } + } + return 0; +} \ No newline at end of file diff --git a/src/rcu_hash.cpp b/src/rcu_hash.cpp index 8ce7634..a907071 100644 --- a/src/rcu_hash.cpp +++ b/src/rcu_hash.cpp @@ -142,6 +142,11 @@ void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx) htable->user_ctx = user_ctx; } +void *rcu_hash_get_user_ctx(struct rcu_hash_table *htable) +{ + return htable->user_ctx; +} + void rcu_hash_commit_prepare(struct rcu_hash_table *htable) { struct rcu_hash_node *node = NULL; diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 981815f..39c49cf 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -6,6 +6,7 @@ #include "maat_table_schema.h" #include "maat_table_runtime.h" #include "maat_command.h" +#include "IPMatcher.h" struct maat *g_maat_instance = NULL; const char *table_info_path = "./table_info.conf"; @@ -206,6 +207,24 @@ TEST(maat_scan_string, hit_three_expr) { EXPECT_EQ(result_array[2], 18); } +TEST(maat_scan_ip, hit_ip) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG"); + char ip_str[32] = "192.168.50.24"; + struct addr_4tuple addr; + addr.type = IP_TYPE_V4; + int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip); + EXPECT_EQ(ret, 1); + addr.ipv4.sport = htons(1); + + int results[3] = {-1}; + size_t n_result = 0; + ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(results[0], 4); +} + int main(int argc, char ** argv) { int ret=0; diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 7995480..8774d45 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -57,7 +57,8 @@ TEST(iris_mode, maat_scan_string) { int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz); EXPECT_NE(ret, -1); - ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, NULL, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL); + ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, NULL, + tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL); EXPECT_NE(ret, -1); } @@ -141,7 +142,7 @@ TEST(redis_mode, maat_scan_string) { snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); - redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db); + redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, NULL); EXPECT_NE(c, nullptr); redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb"); @@ -155,25 +156,26 @@ TEST(redis_mode, maat_scan_string) { int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); EXPECT_NE(ret, -1); - ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL); + ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, c, tmp_iris_path, + sizeof(tmp_iris_path), NULL, NULL, NULL); EXPECT_NE(ret, -1); } size_t total_line_cnt = 0; char tmp_iris_full_idx_path[128] = {0}; snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path); - config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt); + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); long long server_time = maat_cmd_redis_server_time_s(c); EXPECT_NE(server_time, -1); absolute_expire_time = server_time + 300; - config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule); + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, NULL); int success_cnt = 0; do { - success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time); + success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, NULL); } while (success_cnt < 0); EXPECT_EQ(success_cnt, (int)total_line_cnt); diff --git a/test/table_info.conf b/test/table_info.conf index 6e50c04..d613789 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -6,20 +6,20 @@ "scan_mode":"block", "item_id":1, "group_id":2, - "rule": { + "custom": { "keywords":3, "expr_type":4, "match_method":5, "is_hexbin":6, "is_valid":7 } - }, + }, { "table_id":2, "table_name":"IP_PLUGIN_TABLE", "table_type":"ip_plugin", "item_id":1, - "rule": { + "custom": { "ip_type":2, "start_ip":3, "end_ip":4, @@ -31,11 +31,64 @@ "table_name":"PLUGIN_TABLE", "table_type":"plugin", "item_id":1, - "rule": { + "custom": { "key":2, "tag":3, "is_valid":4, "foreign":[6,8,10] } + }, + { + "table_id":4, + "table_name":"IP_PLUS_CONFIG", + "table_type":"ip_plus", + "item_id":1, + "group_id":2, + "custom": { + "addr_type":3, + "saddr_format":4, + "sip1":5, + "sip2":6, + "sport_format":7, + "sport1":8, + "sport2":9, + "daddr_format":10, + "dip1":11, + "dip2":12, + "dport_format":13, + "dport1":14, + "dport2":15, + "proto":16, + "direction":17, + "is_valid":18 + } + }, + { + "table_id":5, + "table_name":"COMPOSITION_IP_SOURCE", + "table_type":"virtual", + "physical_table":["IP_PLUS_CONFIG"] + }, + { + "table_id":6, + "table_name":"COMPOSITION_IP_DEST", + "table_type":"virtual", + "physical_table":["IP_PLUS_CONFIG"] + }, + { + "table_id":7, + "table_name":"COMPOSITION_IP_SESSION", + "table_type":"virtual", + "physical_table":["IP_PLUS_CONFIG"] + }, + { + "table_id":8, + "table_name":"COMPOSITION_IP", + "table_type":"composition", + "composition_table": { + "source": "COMPOSITION_IP_SOURCE", + "dest": "COMPOSITION_IP_DEST", + "session": "COMPOSITION_IP_SESSION" + } } ] \ No newline at end of file diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 2d40b4a..428d40c 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -65,7 +65,7 @@ int set_file_rulenum(const char *path, int rule_num) return 0; } -void read_rule_from_redis(redisContext *c, long long desire_version, const char* output_path) +void read_rule_from_redis(redisContext *c, long long desire_version, const char *output_path, struct log_handle *logger) { int i = 0; int ret = 0; @@ -80,7 +80,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char* FILE *index_fp = NULL; struct serial_rule *rule_list = NULL; - int rule_num = maat_cmd_get_rm_key_list(c, 0, desire_version, &version, NULL, &rule_list, &update_type, 0); + int rule_num = maat_cmd_get_rm_key_list(c, 0, desire_version, &version, NULL, &rule_list, &update_type, 0, logger); if (0 == rule_num) { if (desire_version != 0) { printf("Read desired version %lld failed.\n", desire_version); @@ -102,7 +102,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char* } printf("Reading value: \n"); - ret = maat_cmd_get_redis_value(c, rule_list, rule_num, 1); + ret = maat_cmd_get_redis_value(c, rule_list, rule_num, 1, logger); if (ret < 0) { goto clean_up; } @@ -123,10 +123,10 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char* } } - ret = maat_cmd_get_foreign_keys_by_prefix(c, rule_list, rule_num, foreign_files_dir); + ret = maat_cmd_get_foreign_keys_by_prefix(c, rule_list, rule_num, foreign_files_dir, logger); if (ret > 0) { printf("%d lines has foreign content.\n", ret); - maat_cmd_get_foreign_conts(c, rule_list, rule_num, 1); + maat_cmd_get_foreign_conts(c, rule_list, rule_num, 1, NULL); } snprintf(index_path,sizeof(index_path), "%s/full_config_index.%020lld", output_path, version); @@ -248,11 +248,13 @@ int main(int argc, char * argv[]) int redis_db = 0; char dump_dir[128] = {0}; char json_file[128] = {0}; + char log_path[128] = "./maat_redis_tool.log"; int timeout = 0; long long desired_version = 0; strncpy(redis_ip, "127.0.0.1", sizeof(redis_ip)); strncpy(dump_dir, redis_dump_dir, sizeof(dump_dir)); - + struct log_handle *logger = log_handle_create(log_path, 0); + while ((oc = getopt(argc,argv,"h:p:n:d:v:f:j:t:")) != -1) { switch (oc) { case 'h': @@ -289,7 +291,7 @@ int main(int argc, char * argv[]) } } - redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db); + redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger); if (NULL == c) { return -1; } @@ -297,7 +299,7 @@ int main(int argc, char * argv[]) char tmp_iris_path[128] = {0}; if (model == WORK_MODE_DUMP) { printf("Reading key list from %s:%d db%d.\n", redis_ip, redis_port, redis_db); - read_rule_from_redis(c, desired_version, dump_dir); + read_rule_from_redis(c, desired_version, dump_dir, logger); } else if(model == WORK_MODE_JSON) { char *json_buff = NULL; size_t json_buff_sz = 0; @@ -306,13 +308,14 @@ int main(int argc, char * argv[]) printf("open %s failed.\n", json_file); } - ret = json2iris(json_buff, json_file, NULL, NULL, NULL, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL); + ret = json2iris(json_buff, json_file, NULL, NULL, NULL, c, tmp_iris_path, + sizeof(tmp_iris_path), NULL, NULL, NULL); if (ret < 0) { printf("Invalid json format.\n"); } size_t total_line_cnt = 0; - config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt); + config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL); printf("Serialize %s to %zu lines, write temp file to %s .\n", json_file, total_line_cnt, tmp_iris_path); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); @@ -325,13 +328,13 @@ int main(int argc, char * argv[]) absolute_expire_time = server_time + timeout; } - config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule); + config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, NULL); printf("Timeout = %lld\n", absolute_expire_time); ret = 0; int success_cnt = 0; do { - success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time); + success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, NULL); } while(success_cnt < 0); if (success_cnt != (int)total_line_cnt) {