gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support scan ip

Browse Source
This commit is contained in:
liuwentan
2022-12-09 17:12:18 +08:00
parent 6ba2f6241e
commit 0536083cbe
27 changed files with 1894 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
include/maat/maat.h
View File

@@ -36,13 +36,28 @@ enum ip_type {
IP_TYPE_V6 IP_TYPE_V6
}; };
typedef struct ip_data_s { /* network order */
struct ipv4_4tuple {
uint32_t sip;
uint32_t dip;
uint16_t sport;
uint16_t dport;
};
struct ipv6_4tuple {
uint8_t sip[16];
uint8_t dip[16];
uint16_t sport;
uint16_t dport;
};
struct addr_4tuple {
enum ip_type type; enum ip_type type;
union { union {
uint32_t ipv4; struct ipv4_4tuple ipv4;
uint32_t ipv6[4]; struct ipv6_4tuple ipv6;
}; };
} ip_data_t; };
#define MAAT_RULE_UPDATE_TYPE_FULL 1 #define MAAT_RULE_UPDATE_TYPE_FULL 1
#define MAAT_RULE_UPDATE_TYPE_INC 2 #define MAAT_RULE_UPDATE_TYPE_INC 2
@@ -69,7 +84,8 @@ int maat_options_set_iris_inc_index_dir(struct maat_options *opts, const char *i
int maat_options_set_json_file(struct maat_options *opts, const char *json_filename); int maat_options_set_json_file(struct maat_options *opts, const char *json_filename);
int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip); int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip);
int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port); int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port);
int maat_options_set_redis_db_index(struct maat_options *opts, int db_index); int maat_options_set_redis_db(struct maat_options *opts, int db_index);
int maat_options_set_logger(struct maat_options *opts, void *logger);
/* maat_instance API */ /* maat_instance API */
struct maat *maat_new(struct maat_options *opts, const char *table_info_path); struct maat *maat_new(struct maat_options *opts, const char *table_info_path);
@@ -90,7 +106,7 @@ int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id,
maat_plugin_ex_dup_func_t *dup_func, maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp); long argl, void *argp);
void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id, void *maat_plugin_table_dup_ex_data(struct maat *instance, int table_id,
const char *key, size_t key_len); const char *key, size_t key_len);
/* maat scan API */ /* maat scan API */
struct maat_state; struct maat_state;
@@ -99,7 +115,7 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
struct maat_state *state); struct maat_state *state);
int maat_scan_ip(struct maat *instance, int table_id, int thread_id, int maat_scan_ip(struct maat *instance, int table_id, int thread_id,
const ip_data_t *ip, int results[], size_t *n_result, struct addr_4tuple *addr, int results[], size_t *n_result,
struct maat_state *state); struct maat_state *state);
int maat_scan_string(struct maat *instance, int table_id, int thread_id, int maat_scan_string(struct maat *instance, int table_id, int thread_id,

2
src/CMakeLists.txt
View File

@@ -13,7 +13,7 @@ set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp
maat_command.cpp maat_redis_monitor.cpp) maat_command.cpp maat_redis_monitor.cpp)
set(LIB_SOURCE_FILES set(LIB_SOURCE_FILES
${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c) ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c)
include_directories(${PROJECT_SOURCE_DIR}/include/) include_directories(${PROJECT_SOURCE_DIR}/include/)
include_directories(${PROJECT_SOURCE_DIR}/deps/) include_directories(${PROJECT_SOURCE_DIR}/deps/)

3
src/inc_internal/json2iris.h
View File

@@ -20,7 +20,8 @@ extern "C"
int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn,
const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx,
char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo); char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo,
struct log_handle *logger);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

4
src/inc_internal/maat_common.h
View File

@@ -18,14 +18,18 @@ extern "C"
#include <stddef.h> #include <stddef.h>
#include "log/log.h"
#include "maat_rule.h" #include "maat_rule.h"
struct maat_options { struct maat_options {
char instance_name[NAME_MAX];
size_t nr_worker_threads; size_t nr_worker_threads;
int rule_effect_interval_ms; int rule_effect_interval_ms;
int rule_update_checking_interval_ms; int rule_update_checking_interval_ms;
int gc_timeout_ms; int gc_timeout_ms;
int deferred_load_on; int deferred_load_on;
int log_level;
struct log_handle *logger;
enum data_source input_mode; enum data_source input_mode;
union { union {
struct source_iris_ctx iris_ctx; struct source_iris_ctx iris_ctx;

4
src/inc_internal/maat_config_monitor.h
View File

@@ -25,8 +25,8 @@ extern "C"
void config_monitor_traverse(long long version, const char *idx_dir, void config_monitor_traverse(long long version, const char *idx_dir,
void (*start_fn)(long long, int, void *), void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *), int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void (*finish_fn)(void *), void *u_param,
void *u_param); struct log_handle *logger);
int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz); int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz);

21
src/inc_internal/maat_ex_data.h
View File

@@ -18,6 +18,11 @@ extern "C"
#include "rcu_hash.h" #include "rcu_hash.h"
struct ex_data_container {
void *ex_data;
void *custom_data;
};
struct ex_data_runtime; struct ex_data_runtime;
/* ex_data_runtime API */ /* ex_data_runtime API */
@@ -40,20 +45,24 @@ void ex_data_runtime_clear_row_cache(struct ex_data_runtime *ex_data_rt);
void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema); void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema);
/* set user_ctx API */ /* set user_ctx API */
void ex_data_runtime_set_user_ctx(struct ex_data_runtime *ex_data_rt, void *user_ctx); void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx);
struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt);
/* ex_data_runtime ex data API */ /* ex_data_runtime ex data API */
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len); void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len);
void ex_data_runtime_add_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, void *data); void ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct ex_data_container *ex_container);
void ex_data_runtime_del_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); void ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct log_handle *logger);
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container);
size_t ex_data_runtime_ex_data_count(struct ex_data_runtime *ex_data_rt); void *ex_data_runtime_dup_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
size_t ex_data_runtime_list_updating_ex_data(struct ex_data_runtime *ex_data_rt, void ***ex_data_array); void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len);
size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt);
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt); int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt);

3
src/inc_internal/maat_redis_monitor.h
View File

@@ -23,8 +23,7 @@ extern "C"
void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx,
void (*start_fn)(long long, int, void *), void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *), int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void (*finish_fn)(void *), void *u_param);
void *u_param);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

19
src/inc_internal/maat_rule.h
View File

@@ -24,6 +24,7 @@ extern "C"
#include <sys/queue.h> #include <sys/queue.h>
#include <openssl/md5.h> #include <openssl/md5.h>
#include "log/log.h"
#include "hiredis/hiredis.h" #include "hiredis/hiredis.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
#include "maat_table_schema.h" #include "maat_table_schema.h"
@@ -114,6 +115,7 @@ struct maat {
struct source_redis_ctx mr_ctx; struct source_redis_ctx mr_ctx;
}; };
struct log_handle *logger;
int deferred_load; int deferred_load;
int is_running; int is_running;
@@ -159,7 +161,7 @@ void *rule_monitor_loop(void *arg);
void maat_read_full_config(struct maat *maat_instance); void maat_read_full_config(struct maat *maat_instance);
/* maat command API for internal */ /* maat command API for internal */
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db); redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger);
redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...); redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...);
@@ -173,19 +175,24 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type,
const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len); const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len);
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time); int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num,
long long server_time, struct log_handle *logger);
void maat_cmd_clear_rule_cache(struct serial_rule *s_rule); void maat_cmd_clear_rule_cache(struct serial_rule *s_rule);
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version,
long long *new_version, struct table_schema_manager* table_schema_mgr, long long *new_version, struct table_schema_manager* table_schema_mgr,
struct serial_rule **list, int *update_type, int cumulative_off); struct serial_rule **list, int *update_type, int cumulative_off,
struct log_handle *logger);
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process); int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process,
struct log_handle *logger);
int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num, const char* dir); int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num,
const char *dir, struct log_handle *logger);
void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn); void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn,
struct log_handle *logger);
void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule); void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule);

28
src/inc_internal/maat_table_runtime.h
View File

@@ -16,16 +16,27 @@ extern "C"
{ {
#endif #endif
#include "maat/maat.h"
#include "maat_table_schema.h" #include "maat_table_schema.h"
#include "maat_garbage_collection.h" #include "maat_garbage_collection.h"
struct table_rt_2tuple {
enum ip_type ip_type;
uint16_t port;
union {
uint32_t ipv4;
uint32_t ipv6[4];
};
};
struct table_item; struct table_item;
struct table_runtime; struct table_runtime;
struct table_runtime_manager; struct table_runtime_manager;
/* table runtime manager API */ /* table runtime manager API */
struct table_runtime_manager *table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num, struct table_runtime_manager *
struct maat_garbage_bin *bin); table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int max_thread_num,
struct maat_garbage_bin *bin);
void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr); void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr);
@@ -36,7 +47,8 @@ size_t table_runtime_rule_count(struct table_runtime *table_rt);
enum table_type table_runtime_get_type(struct table_runtime* table_rt); enum table_type table_runtime_get_type(struct table_runtime* table_rt);
void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, const char *line, struct table_item *table_item); void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema,
const char *line, struct table_item *table_item, struct log_handle *logger);
/** /**
* @brief if table_runtime is updating * @brief if table_runtime is updating
@@ -52,19 +64,21 @@ int table_runtime_scan_string(struct table_runtime *table_rt, int thread_id, con
int results[], size_t *n_result); int results[], size_t *n_result);
void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id); void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id);
int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int results[], size_t *n_result);
int result[], size_t n_result);
void table_runtime_stream_close(struct table_runtime *table_rt); void table_runtime_stream_close(struct table_runtime *table_rt);
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data, int results[], size_t *n_result);
/* table runtime cached row API */ /* table runtime cached row API */
size_t table_runtime_cached_row_count(struct table_runtime *table_rt); size_t table_runtime_cached_row_count(struct table_runtime *table_rt);
const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq); const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq);
/* table runtime ex data API */ /* table runtime ex data API */
void *table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema, const char *key, size_t key_len); struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt);
void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema); void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema,
struct log_handle *logger);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

49
src/inc_internal/maat_table_schema.h
View File

@@ -27,6 +27,13 @@ extern "C"
#define MAX_KEYWORDS_STR 1024 #define MAX_KEYWORDS_STR 1024
#define MAX_FOREIGN_CLMN_NUM 8 #define MAX_FOREIGN_CLMN_NUM 8
enum table_composition_type {
COMPOSITION_TYPE_SIP = 0,
COMPOSITION_TYPE_DIP,
COMPOSITION_TYPE_SESSION,
COMPOSITION_TYPE_MAX
};
enum table_type { enum table_type {
TABLE_TYPE_EXPR = 0, TABLE_TYPE_EXPR = 0,
TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_EXPR_PLUS,
@@ -90,6 +97,35 @@ struct expr_item {
int is_valid; int is_valid;
}; };
struct ipv4_item_rule {
uint32_t min_sip; /* 源地址下界；0表示忽略本字段 */
uint32_t max_sip; /* 源地址上界；0表示固定IP=min_saddr */
uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */
uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */
uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP；0表示忽略本字段 */
uint16_t direction; /* 方向，0表示双向，1表示单向 */
};
struct ipv6_item_rule {
uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */
uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */
uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */
uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */
uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP，无限制默认为0 */
uint16_t direction; /* 方向，0表示双向，1表示单向 */
};
struct ip_plus_item {
int item_id;
int group_id;
int addr_type;
union {
struct ipv4_item_rule ipv4;
struct ipv6_item_rule ipv6;
};
int is_valid;
};
struct plugin_item { struct plugin_item {
char key[MAX_KEYWORDS_STR]; char key[MAX_KEYWORDS_STR];
size_t key_len; size_t key_len;
@@ -111,6 +147,7 @@ struct table_item {
enum table_type table_type; enum table_type table_type;
union { union {
struct expr_item expr_item; struct expr_item expr_item;
struct ip_plus_item ip_plus_item;
struct plugin_item plugin_item; struct plugin_item plugin_item;
struct ip_plugin_item ip_plugin_item; struct ip_plugin_item ip_plugin_item;
}; };
@@ -139,10 +176,17 @@ struct table_schema;
struct table_schema_manager; struct table_schema_manager;
/* table schema manager API */ /* table schema manager API */
struct table_schema_manager *table_schema_manager_create(const char *table_info_path); struct table_schema_manager *table_schema_manager_create(const char *table_info_path, struct log_handle *logger);
void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr); void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr);
int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name); int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name);
/**
* @brief get composition table's child table(specified by type) id
*/
int table_schema_manager_get_child_table_id(struct table_schema_manager *table_schema_mgr, int parent_table_id,
enum table_composition_type type);
enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id); enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int table_id);
size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_mgr); size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_mgr);
@@ -162,7 +206,8 @@ int table_schema_get_table_id(struct table_schema *table_schema);
enum scan_type table_schema_get_scan_type(struct table_schema *table_schema); enum scan_type table_schema_get_scan_type(struct table_schema *table_schema);
struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema); struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema,
struct log_handle *logger);
int table_schema_get_valid_flag_column(struct table_schema *table_schema); int table_schema_get_valid_flag_column(struct table_schema *table_schema);

21
src/inc_internal/maat_utils.h
View File

@@ -18,6 +18,7 @@ extern "C"
#include <stdlib.h> #include <stdlib.h>
#include <stddef.h> #include <stddef.h>
#include <arpa/inet.h>
#define TRUE 1 #define TRUE 1
#define FALSE 0 #define FALSE 0
@@ -30,8 +31,28 @@ extern "C"
#define MIN(a, b) (((a) < (b)) ? (a) : (b)) #define MIN(a, b) (((a) < (b)) ? (a) : (b))
#endif #endif
enum maat_ip_format {
IP_FORMAT_RANGE,
IP_FORMAT_MASK,
IP_FORMAT_CIDR,
IP_FORMAT_UNKNOWN
};
enum maat_ip_format ip_format_str2int(const char *format);
int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, const char *ip2,
uint32_t range_begin[], uint32_t range_end[]);
inline void ipv6_ntoh(unsigned int *v6_addr)
{
for (unsigned int i = 0; i < 4; i++) {
v6_addr[i] = ntohl(v6_addr[i]);
}
}
#define UNUSED __attribute__((unused)) #define UNUSED __attribute__((unused))
const char *module_name_str(const char *name);
char *maat_strdup(const char *s); char *maat_strdup(const char *s);
int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len); int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len);

2
src/inc_internal/rcu_hash.h
View File

@@ -29,6 +29,8 @@ void rcu_hash_free(struct rcu_hash_table *htable);
void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx); void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx);
void *rcu_hash_get_user_ctx(struct rcu_hash_table *htable);
/** /**
* @brief just means add to the updating nodes * @brief just means add to the updating nodes
* after call rcu_hash_commit, they become effective nodes * after call rcu_hash_commit, they become effective nodes

138
src/json2iris.cpp
View File

@@ -24,6 +24,8 @@
#include "maat_rule.h" #include "maat_rule.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
#define MODULE_JSON2IRIS module_name_str("maat.json2iris")
#define MAX_COLUMN_NUM 32 #define MAX_COLUMN_NUM 32
#define mr_region_id_var "SEQUENCE_REGION" #define mr_region_id_var "SEQUENCE_REGION"
@@ -219,22 +221,24 @@ int create_tmp_dir(struct iris_description *p)
return 0; return 0;
} }
int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_description *p_iris) int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_description *p_iris,
struct log_handle *logger)
{ {
cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name"); cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
fprintf(stderr, "The %d plugin_table's table_name not defined or format error\n", sequence); log_error(logger, MODULE_JSON2IRIS,
"The %d plugin_table's table_name not defined or format error", sequence);
return -1; return -1;
} }
const char *table_name = item->valuestring; const char *table_name = item->valuestring;
cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content"); cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content");
if (NULL == table_content || table_content->type != cJSON_Array) { if (NULL == table_content || table_content->type != cJSON_Array) {
fprintf(stderr, "%d plugin_table's table_content not defined or format error\n", sequence); log_error(logger, MODULE_JSON2IRIS,
"%d plugin_table's table_content not defined or format error", sequence);
return -1; return -1;
} }
int line_cnt = cJSON_GetArraySize(table_content); int line_cnt = cJSON_GetArraySize(table_content);
struct iris_table *table_info = query_table_info(p_iris, table_name, TABLE_TYPE_PLUGIN); struct iris_table *table_info = query_table_info(p_iris, table_name, TABLE_TYPE_PLUGIN);
cJSON *each_line = NULL; cJSON *each_line = NULL;
@@ -242,7 +246,8 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript
for (int i = 0; i < line_cnt; i++) { for (int i = 0; i < line_cnt; i++) {
each_line = cJSON_GetArrayItem(table_content, i); each_line = cJSON_GetArrayItem(table_content, i);
if (NULL == each_line || each_line->type != cJSON_String) { if (NULL == each_line || each_line->type != cJSON_String) {
fprintf(stderr, "plugin_table %s's line %d format error\n", table_info->table_name, i+1); log_error(logger, MODULE_JSON2IRIS, "plugin_table %s's line %d format error",
table_info->table_name, i + 1);
continue; continue;
} }
@@ -314,7 +319,8 @@ static int get_region_seq(struct iris_description *iris_cfg)
return sequence; return sequence;
} }
int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct translate_command *cmd, int cmd_cnt, struct iris_table *table) int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct translate_command *cmd, int cmd_cnt,
struct iris_table *table, struct log_handle *logger)
{ {
int i = 0; int i = 0;
int ret = -1; int ret = -1;
@@ -331,7 +337,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
} }
if (NULL == item || item->type != cmd[i].json_type) { if (NULL == item || item->type != cmd[i].json_type) {
fprintf(stderr, "%s not defined or wrong format\n", cmd[i].json_string); log_error(logger, MODULE_JSON2IRIS, "%s not defined or wrong format", cmd[i].json_string);
ret = -1; ret = -1;
goto error_out; goto error_out;
} }
@@ -341,7 +347,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla
char *p = item->valuestring; char *p = item->valuestring;
ret = maat_kv_read(str2int, p, &int_value); ret = maat_kv_read(str2int, p, &int_value);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "%s's value %s is not valid format\n", cmd[i].json_string, p); log_error(logger, MODULE_JSON2IRIS, "%s's value %s is not valid format", cmd[i].json_string, p);
FREE(p); FREE(p);
ret = -1; ret = -1;
goto error_out; goto error_out;
@@ -380,7 +386,8 @@ error_out:
return ret; return ret;
} }
int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -423,10 +430,11 @@ int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
} }
int write_ip_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) int write_ip_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -510,10 +518,11 @@ int write_ip_line(cJSON *region_json, struct iris_description *p_iris, struct ir
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
} }
int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -621,10 +630,11 @@ int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, stru
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
} }
int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -656,10 +666,11 @@ int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struc
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
} }
int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -689,10 +700,11 @@ int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struc
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
} }
int write_similar_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) int write_similar_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table,
struct log_handle *logger)
{ {
struct translate_command json_cmd[MAX_COLUMN_NUM]; struct translate_command json_cmd[MAX_COLUMN_NUM];
int cmd_cnt = 0; int cmd_cnt = 0;
@@ -718,22 +730,24 @@ int write_similar_line(cJSON *region_json, struct iris_description *p_iris, stru
json_cmd[cmd_cnt].json_type = cJSON_Number; json_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++; cmd_cnt++;
return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);
} }
int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_description *p_iris) int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_description *p_iris,
struct log_handle *logger)
{ {
cJSON *item = cJSON_GetObjectItem(region_json, "table_name"); cJSON *item = cJSON_GetObjectItem(region_json, "table_name");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
fprintf(stderr, "compile rule %d's table_name not defined or format error\n", compile_id); log_error(logger, MODULE_JSON2IRIS, "compile rule %d's table_name not defined or format error",
compile_id);
return -1; return -1;
} }
const char *table_name = item->valuestring; const char *table_name = item->valuestring;
item = cJSON_GetObjectItem(region_json, "table_type"); item = cJSON_GetObjectItem(region_json, "table_type");
if (NULL == item || item->type != cJSON_String) { if (NULL == item || item->type != cJSON_String) {
fprintf(stderr, "compile rule %d's table name %s's table_type not defined or format error\n", log_error(logger, MODULE_JSON2IRIS,
compile_id, table_name); "compile rule %d's table name %s's table_type not defined or format error", compile_id, table_name);
return -1; return -1;
} }
@@ -741,15 +755,15 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
enum table_type table_type = TABLE_TYPE_EXPR; enum table_type table_type = TABLE_TYPE_EXPR;
int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type)); int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type));
if (ret != 1) { if (ret != 1) {
fprintf(stderr, "compile rule %d table name %s's table_type %s invalid\n", log_error(logger, MODULE_JSON2IRIS, "compile rule %d table name %s's table_type %s invalid",
compile_id, table_name, table_type_str); compile_id, table_name, table_type_str);
return -1; return -1;
} }
cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content"); cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content");
if (NULL == table_content || table_content->type != cJSON_Object) { if (NULL == table_content || table_content->type != cJSON_Object) {
fprintf(stderr, "compile rule %d table name %s's table_content not defined or format error\n", log_error(logger, MODULE_JSON2IRIS,
compile_id, table_name); "compile rule %d table name %s's table_content not defined or format error", compile_id, table_name);
return -1; return -1;
} }
@@ -763,23 +777,23 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i
{ {
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS: case TABLE_TYPE_EXPR_PLUS:
ret = write_expr_line(table_content, p_iris, table_info); ret = write_expr_line(table_content, p_iris, table_info, logger);
break; break;
case TABLE_TYPE_IP: case TABLE_TYPE_IP:
ret = write_ip_line(table_content, p_iris, table_info); ret = write_ip_line(table_content, p_iris, table_info, logger);
break; break;
case TABLE_TYPE_IP_PLUS: case TABLE_TYPE_IP_PLUS:
ret = write_ip_plus_line(table_content, p_iris, table_info); ret = write_ip_plus_line(table_content, p_iris, table_info, logger);
break; break;
case TABLE_TYPE_INTERVAL: case TABLE_TYPE_INTERVAL:
case TABLE_TYPE_INTERVAL_PLUS: case TABLE_TYPE_INTERVAL_PLUS:
ret = write_intval_line(table_content, p_iris, table_info); ret = write_intval_line(table_content, p_iris, table_info, logger);
break; break;
case TABLE_TYPE_DIGEST: case TABLE_TYPE_DIGEST:
ret = write_digest_line(table_content, p_iris, table_info); ret = write_digest_line(table_content, p_iris, table_info, logger);
break; break;
case TABLE_TYPE_SIMILARITY: case TABLE_TYPE_SIMILARITY:
ret = write_similar_line(table_content, p_iris, table_info); ret = write_similar_line(table_content, p_iris, table_info, logger);
break; break;
default: default:
assert(0); assert(0);
@@ -814,7 +828,7 @@ int write_group2group_line(int group_id, int superior_group_id, struct iris_desc
} }
int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id,
int Nth_group, struct iris_description *p_iris) int Nth_group, struct iris_description *p_iris, struct log_handle *logger)
{ {
int ret = 0; int ret = 0;
int group_not_flag = 0; int group_not_flag = 0;
@@ -863,21 +877,22 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
if (region_json != NULL) { if (region_json != NULL) {
cJSON *region_rule = NULL; cJSON *region_rule = NULL;
cJSON_ArrayForEach(region_rule, region_json) { cJSON_ArrayForEach(region_rule, region_json) {
ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris); ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris, logger);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "compile rule %d write region error\n", tracking_compile_id); log_error(logger, MODULE_JSON2IRIS, "compile rule %d write region error",
tracking_compile_id);
return -1; return -1;
} }
} }
} }
cJSON *sub_groups = cJSON_GetObjectItem(group_json,"sub_groups"); cJSON *sub_groups = cJSON_GetObjectItem(group_json, "sub_groups");
if (sub_groups != NULL) { if (sub_groups != NULL) {
//recursively //recursively
int i = 0; int i = 0;
cJSON_ArrayForEach(item, sub_groups) { cJSON_ArrayForEach(item, sub_groups) {
i++; i++;
ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris); ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -885,8 +900,9 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
} }
if (NULL == region_json && NULL == sub_groups) { if (NULL == region_json && NULL == sub_groups) {
fprintf(stdout, "A group of compile rule %d has neither regions, sub groups, nor refered another exisited group\n", log_info(logger, MODULE_JSON2IRIS,
tracking_compile_id); "A group of compile rule %d has neither regions, sub groups, nor refered another exisited group",
tracking_compile_id);
} }
} }
@@ -897,18 +913,18 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
} }
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "%s rule %d write group error\n", str_parent_type[parent_type], parent_id); log_error(logger, MODULE_JSON2IRIS, "%s rule %d write group error", str_parent_type[parent_type], parent_id);
return -1; return -1;
} }
return 0; return 0;
} }
int write_compile_line(cJSON *compile, struct iris_description *p_iris) int write_compile_line(cJSON *compile, struct iris_description *p_iris, struct log_handle *logger)
{ {
cJSON *item=cJSON_GetObjectItem(compile, "compile_id"); cJSON *item=cJSON_GetObjectItem(compile, "compile_id");
if (item->type != cJSON_Number) { if (item->type != cJSON_Number) {
fprintf(stderr, "compile_id format not number\n"); log_error(logger, MODULE_JSON2IRIS, "compile_id format not number");
return -1; return -1;
} }
int compile_id = item->valueint; int compile_id = item->valueint;
@@ -1002,7 +1018,7 @@ int write_compile_line(cJSON *compile, struct iris_description *p_iris)
table_info = query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE); table_info = query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE);
} }
int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, table_info); int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, table_info, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1041,11 +1057,12 @@ void write_table_idx(struct iris_description *p_iris, struct iris_table *table)
FREE(buff); FREE(buff);
} }
int write_index_file(struct iris_description *p_iris) int write_index_file(struct iris_description *p_iris, struct log_handle *logger)
{ {
p_iris->idx_fp = fopen(p_iris->index_path, "w"); p_iris->idx_fp = fopen(p_iris->index_path, "w");
if (NULL == p_iris->idx_fp) { if (NULL == p_iris->idx_fp) {
fprintf(stderr, "index file %s fopen error %s\n",p_iris->index_path, strerror(errno)); log_error(logger, MODULE_JSON2IRIS, "index file %s fopen error %s",
p_iris->index_path, strerror(errno));
return -1; return -1;
} }
@@ -1061,7 +1078,7 @@ int write_index_file(struct iris_description *p_iris)
return 0; return 0;
} }
int write_iris(cJSON *json, struct iris_description *p_iris) int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *logger)
{ {
int i=0; int i=0;
int ret=0; int ret=0;
@@ -1071,7 +1088,7 @@ int write_iris(cJSON *json, struct iris_description *p_iris)
if (plug_tables != NULL) { if (plug_tables != NULL) {
cJSON *each_plug_table = NULL; cJSON *each_plug_table = NULL;
cJSON_ArrayForEach(each_plug_table, plug_tables) { cJSON_ArrayForEach(each_plug_table, plug_tables) {
write_plugin_line(each_plug_table, i, p_iris); write_plugin_line(each_plug_table, i, p_iris, logger);
i++; i++;
} }
} }
@@ -1093,7 +1110,7 @@ int write_iris(cJSON *json, struct iris_description *p_iris)
parent_group = group_info_add_unsafe(p_iris, parent_group_name); parent_group = group_info_add_unsafe(p_iris, parent_group_name);
} }
ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris); ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1109,28 +1126,28 @@ int write_iris(cJSON *json, struct iris_description *p_iris)
if (compile_cnt > 0) { if (compile_cnt > 0) {
cJSON *compile_obj = NULL; cJSON *compile_obj = NULL;
cJSON_ArrayForEach(compile_obj, compile_array) { cJSON_ArrayForEach(compile_obj, compile_array) {
int compile_id = write_compile_line(compile_obj,p_iris); int compile_id = write_compile_line(compile_obj, p_iris, logger);
if (compile_id < 0) { if (compile_id < 0) {
fprintf(stderr, "In %d compile rule\n", i); log_error(logger, MODULE_JSON2IRIS, "In %d compile rule", i);
return -1; return -1;
} }
group_array = cJSON_GetObjectItem(compile_obj, "groups"); group_array = cJSON_GetObjectItem(compile_obj, "groups");
if (NULL == group_array) { if (NULL == group_array) {
fprintf(stderr, "compile rule %d have no group\n",compile_id); log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no group", compile_id);
return -1; return -1;
} }
int group_cnt = cJSON_GetArraySize(group_array); int group_cnt = cJSON_GetArraySize(group_array);
if (group_cnt <= 0) { if (group_cnt <= 0) {
fprintf(stderr, "compile rule %d have no groups\n", compile_id); log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no groups", compile_id);
return -1; return -1;
} }
i = 0; i = 0;
cJSON *group_obj = NULL; cJSON *group_obj = NULL;
cJSON_ArrayForEach(group_obj, group_array) { cJSON_ArrayForEach(group_obj, group_array) {
ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris); ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1139,7 +1156,7 @@ int write_iris(cJSON *json, struct iris_description *p_iris)
} }
} }
ret = write_index_file(p_iris); ret = write_index_file(p_iris, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -1148,7 +1165,8 @@ int write_iris(cJSON *json, struct iris_description *p_iris)
int json2iris(const char *json_buff, const char *json_filename, const char *compile_tn, int json2iris(const char *json_buff, const char *json_filename, const char *compile_tn,
const char *group2compile_tn, const char *group2group_tn, redisContext *redis_write_ctx, const char *group2compile_tn, const char *group2group_tn, redisContext *redis_write_ctx,
char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo) char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo,
struct log_handle *logger)
{ {
int ret = -1; int ret = -1;
cJSON *tmp_obj = NULL; cJSON *tmp_obj = NULL;
@@ -1158,7 +1176,7 @@ int json2iris(const char *json_buff, const char *json_filename, const char *comp
cJSON *json = cJSON_Parse(json_buff); cJSON *json = cJSON_Parse(json_buff);
if (!json) { if (!json) {
fprintf(stderr, "Error before: %-200.200s\n", cJSON_GetErrorPtr()); log_error(logger, MODULE_JSON2IRIS, "Error before: %-200.200s", cJSON_GetErrorPtr());
goto error_out; goto error_out;
} }
@@ -1186,11 +1204,11 @@ int json2iris(const char *json_buff, const char *json_filename, const char *comp
ret = create_tmp_dir(&iris_cfg); ret = create_tmp_dir(&iris_cfg);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "create tmp folder %s error\n", iris_cfg.tmp_iris_dir); log_error(logger, MODULE_JSON2IRIS, "create tmp folder %s error", iris_cfg.tmp_iris_dir);
goto error_out; goto error_out;
} }
ret = write_iris(json, &iris_cfg); ret = write_iris(json, &iris_cfg, logger);
if (ret < 0) { if (ret < 0) {
goto error_out; goto error_out;
} }

212
src/maat_api.cpp
View File

@@ -10,19 +10,24 @@
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <assert.h>
#include "utils.h" #include "utils.h"
#include "maat_utils.h"
#include "json2iris.h" #include "json2iris.h"
#include "maat/maat.h" #include "maat/maat.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_common.h" #include "maat_common.h"
#include "maat_kv.h" #include "maat_kv.h"
#include "maat_command.h" #include "maat_command.h"
#include "maat_ex_data.h"
#include "maat_table_schema.h" #include "maat_table_schema.h"
#include "maat_table_runtime.h" #include "maat_table_runtime.h"
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
#include "maat_redis_monitor.h" #include "maat_redis_monitor.h"
#define MODULE_MAAT_API module_name_str("maat.api")
struct maat_options* maat_options_new(void) struct maat_options* maat_options_new(void)
{ {
struct maat_options *options = ALLOC(struct maat_options, 1); struct maat_options *options = ALLOC(struct maat_options, 1);
@@ -33,6 +38,7 @@ struct maat_options* maat_options_new(void)
options->rule_update_checking_interval_ms = 1 * 1000; options->rule_update_checking_interval_ms = 1 * 1000;
options->gc_timeout_ms = 10 * 1000; options->gc_timeout_ms = 10 * 1000;
options->input_mode = DATA_SOURCE_NONE; options->input_mode = DATA_SOURCE_NONE;
options->log_level = 0;
return options; return options;
} }
@@ -65,6 +71,13 @@ int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms)
return 0; return 0;
} }
int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len)
{
memcpy(opts->instance_name, instance_name, name_len);
return 0;
}
int maat_options_set_deferred_load_on(struct maat_options *opts) int maat_options_set_deferred_load_on(struct maat_options *opts)
{ {
opts->deferred_load_on = 1; opts->deferred_load_on = 1;
@@ -119,13 +132,20 @@ int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port)
return 0; return 0;
} }
int maat_options_set_redis_db_index(struct maat_options *opts, int db_index) int maat_options_set_redis_db(struct maat_options *opts, int db_index)
{ {
opts->redis_ctx.redis_db = db_index; opts->redis_ctx.redis_db = db_index;
return 0; return 0;
} }
int maat_options_set_logger(struct maat_options *opts, void *logger)
{
opts->logger = (struct log_handle *)logger;
return 0;
}
void maat_read_full_config(struct maat *maat_instance) void maat_read_full_config(struct maat *maat_instance)
{ {
int ret = -1; int ret = -1;
@@ -135,9 +155,10 @@ void maat_read_full_config(struct maat *maat_instance)
switch (maat_instance->input_mode) { switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS: case DATA_SOURCE_REDIS:
mr_ctx = &(maat_instance->mr_ctx); mr_ctx = &(maat_instance->mr_ctx);
fprintf(stdout, "Maat initiate from Redis %s:%hu db%d\n", log_info(maat_instance->logger, MODULE_MAAT_API, "Maat initiate from Redis %s:%hu db%d",
mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port,
mr_ctx->redis_db, maat_instance->logger);
if (mr_ctx->read_ctx != NULL) { if (mr_ctx->read_ctx != NULL) {
redis_monitor_traverse(maat_instance->maat_version, mr_ctx, redis_monitor_traverse(maat_instance->maat_version, mr_ctx,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
@@ -145,34 +166,37 @@ void maat_read_full_config(struct maat *maat_instance)
} }
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in redis %s:%hu db%d\n", log_error(maat_instance->logger, MODULE_MAAT_API,
mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); "At initiation: NO effective rule in redis %s:%hu db%d",
mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
} }
break; break;
case DATA_SOURCE_IRIS_FILE: case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version, config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.full_idx_dir, maat_instance->iris_ctx.full_idx_dir,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance); maat_instance, maat_instance->logger);
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in %s\n", log_error(maat_instance->logger, MODULE_MAAT_API,
maat_instance->iris_ctx.full_idx_dir); "At initiation: NO effective rule in %s",
maat_instance->iris_ctx.full_idx_dir);
} }
break; break;
case DATA_SOURCE_JSON_FILE: case DATA_SOURCE_JSON_FILE:
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str));
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Maat re-initiate with JSON file %s failed: %s\n", maat_instance->json_ctx.json_file, err_str); log_error(maat_instance->logger, MODULE_MAAT_API, "Maat re-initiate with JSON file %s failed: %s",
maat_instance->json_ctx.json_file, err_str);
return; return;
} }
config_monitor_traverse(maat_instance->maat_version, config_monitor_traverse(maat_instance->maat_version,
maat_instance->json_ctx.iris_file, maat_instance->json_ctx.iris_file,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_instance); maat_instance, maat_instance->logger);
if (NULL == maat_instance->creating_maat_rt) { if (NULL == maat_instance->creating_maat_rt) {
fprintf(stderr, "At initiation: NO effective rule in %s\n", log_error(maat_instance->logger, MODULE_MAAT_API, "At initiation: NO effective rule in %s",
maat_instance->json_ctx.iris_file); maat_instance->json_ctx.iris_file);
} }
break; break;
default: default:
@@ -197,7 +221,19 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
int garbage_gc_timeout_s = 0; int garbage_gc_timeout_s = 0;
struct maat *maat_instance = ALLOC(struct maat, 1); struct maat *maat_instance = ALLOC(struct maat, 1);
maat_instance->table_schema_mgr = table_schema_manager_create(table_info_path); if (opts->logger != NULL) {
maat_instance->logger = opts->logger;
} else {
char log_path[1024] = {0};
if (strlen(maat_instance->instance_name) > 0) {
snprintf(log_path, sizeof(log_path), "%s.log", maat_instance->instance_name);
} else {
snprintf(log_path, sizeof(log_path), "maat.log");
}
maat_instance->logger = log_handle_create(log_path, opts->log_level);
}
maat_instance->table_schema_mgr = table_schema_manager_create(table_info_path, maat_instance->logger);
if (NULL == maat_instance->table_schema_mgr) { if (NULL == maat_instance->table_schema_mgr) {
goto failed; goto failed;
} }
@@ -217,7 +253,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file)); memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file));
break; break;
default: default:
fprintf(stderr, "data source unsupported:%d\n", maat_instance->input_mode); log_error(maat_instance->logger, MODULE_MAAT_API, "data source unsupported:%d", maat_instance->input_mode);
goto failed; goto failed;
} }
@@ -330,14 +366,14 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
struct table_runtime *table_rt = NULL; struct table_runtime *table_rt = NULL;
if (maat_instance->maat_rt != NULL) { if (maat_instance->maat_rt != NULL) {
table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id); table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id);
table_runtime_commit_ex_data_schema(table_rt, table_schema); table_runtime_commit_ex_data_schema(table_rt, table_schema, maat_instance->logger);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return 0; return 0;
} }
void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, void *maat_plugin_table_dup_ex_data(struct maat *maat_instance, int table_id,
const char *key, size_t key_len) const char *key, size_t key_len)
{ {
struct maat_runtime *maat_rt = maat_instance->maat_rt; struct maat_runtime *maat_rt = maat_instance->maat_rt;
@@ -346,9 +382,21 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
} }
struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); if (NULL == table_schema) {
return NULL;
}
return table_runtime_get_ex_data(table_rt, table_schema, key, key_len); struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
if (NULL == table_rt) {
return NULL;
}
struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt);
if (NULL == ex_data_rt) {
return NULL;
}
return ex_data_runtime_dup_ex_data(ex_data_rt, key, key_len);
} }
int maat_scan_integer(struct maat *instance, int table_id, int thread_id, int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
@@ -358,27 +406,141 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
return 0; return 0;
} }
int maat_scan_ip(struct maat *instance, int table_id, int thread_id, static int ip_scan_data_set(struct table_rt_2tuple *table_rt_addr, struct addr_4tuple *addr, enum table_composition_type child_type)
const struct ip_data *ip, int results[], size_t n_result,
struct maat_state *state)
{ {
switch (addr->type) {
case IP_TYPE_V4:
switch (child_type) {
case COMPOSITION_TYPE_SIP:
table_rt_addr->ipv4 = ntohl(addr->ipv4.sip);
table_rt_addr->port = ntohs(addr->ipv4.sport);
break;
case COMPOSITION_TYPE_DIP:
table_rt_addr->ipv4 = ntohl(addr->ipv4.dip);
table_rt_addr->port = ntohs(addr->ipv4.dport);
break;
default:
assert(0);
return -1;
}
break;
case IP_TYPE_V6:
switch (child_type) {
case COMPOSITION_TYPE_SIP:
memcpy(table_rt_addr->ipv6, addr->ipv6.sip, sizeof(addr->ipv6.sip));
ipv6_ntoh(table_rt_addr->ipv6);
table_rt_addr->port = ntohs(addr->ipv6.sport);
break;
case COMPOSITION_TYPE_DIP:
memcpy(table_rt_addr->ipv6, addr->ipv6.dip, sizeof(addr->ipv6.dip));
ipv6_ntoh(table_rt_addr->ipv6);
table_rt_addr->port = ntohs(addr->ipv6.dport);
break;
default:
assert(0);
return -1;
}
break;
default:
assert(0);
return -1;
}
return 0; return 0;
} }
static int ip_composition_scan(int thread_id, struct addr_4tuple *addr,
int parent_table_id, enum table_composition_type child_type,
int *virtual_table_id,
struct table_schema_manager *table_schema_mgr,
struct table_runtime_manager *table_rt_mgr, int results[], size_t *n_result)
{
int child_table_id = 0;
if (child_type == COMPOSITION_TYPE_MAX) {
child_table_id = parent_table_id;
child_type = COMPOSITION_TYPE_SIP;
} else {
child_table_id = table_schema_manager_get_child_table_id(table_schema_mgr, parent_table_id, child_type);
}
if (child_table_id < 0) {
return 0;
}
struct table_schema *real_table = table_schema_get_by_scan_type(table_schema_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id);
if (NULL == real_table) {
return 0;
}
enum table_type table_type = table_schema_get_table_type(real_table);
if (table_type != TABLE_TYPE_IP_PLUS) {
return -1;
}
int table_id = table_schema_get_table_id(real_table);
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id);
size_t rule_num = table_runtime_rule_count(table_rt);
if (0 == rule_num) {
return 0;
}
struct table_rt_2tuple scan_data;
memset(&scan_data, 0, sizeof(struct table_rt_2tuple));
ip_scan_data_set(&scan_data, addr, child_type);
return table_runtime_scan_ip(table_rt, thread_id, &scan_data, results, n_result);
}
int maat_scan_ip(struct maat *maat_instance, int table_id, int thread_id, struct addr_4tuple *addr,
int results[], size_t *n_result, struct maat_state *state)
{
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == addr) || (NULL == results) || (NULL == n_result)) {
return -1;
}
struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr;
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id);
enum table_type table_type = table_runtime_get_type(table_rt);
int region_ret = 0;
int virtual_table_id = 0;
if (table_type == TABLE_TYPE_COMPOSITION) {
/*
enum table_composition_type childs[3] = {COMPOSITION_TYPE_SIP, COMPOSITION_TYPE_DIP, COMPOSITION_TYPE_SESSION};
for (int i = 0; i < 3; i++) {
region_ret = ip_composition_scan(table_rt, thread_id, ip, table_id, childs[i], maat_instance->table_schema_mgr,
maat_instance->maat_rt->table_rt_mgr, results, n_result);
}*/
region_ret = ip_composition_scan(thread_id, addr, table_id, COMPOSITION_TYPE_SIP, &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
results, n_result);
} else {
region_ret = ip_composition_scan(thread_id, addr, table_id, COMPOSITION_TYPE_MAX, &virtual_table_id,
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr,
results, n_result);
}
return region_ret;
}
int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
const char *data, size_t data_len, int results[], size_t *n_results, const char *data, size_t data_len, int results[], size_t *n_result,
struct maat_state *state) struct maat_state *state)
{ {
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) || (thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) ||
(NULL == n_results)) { (NULL == n_result)) {
return -1; return -1;
} }
struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr; struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr;
struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id);
return table_runtime_scan_string(table_rt, thread_id, data, data_len, results, n_results); return table_runtime_scan_string(table_rt, thread_id, data, data_len, results, n_result);
} }
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id) struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id)

24
src/maat_command.cpp
View File

@@ -21,6 +21,8 @@
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
#include "maat_table_schema.h" #include "maat_table_schema.h"
#define MODULE_MAAT_COMMAND module_name_str("maat.command")
extern const char *foreign_source_prefix; extern const char *foreign_source_prefix;
extern const char *mr_key_prefix; extern const char *mr_key_prefix;
@@ -51,7 +53,7 @@ redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...
return (redisReply *)reply; return (redisReply *)reply;
} }
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db) redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger)
{ {
struct timeval connect_timeout; struct timeval connect_timeout;
connect_timeout.tv_sec = 0; connect_timeout.tv_sec = 0;
@@ -59,8 +61,8 @@ redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int r
redisContext *c = redisConnectWithTimeout(redis_ip, redis_port, connect_timeout); redisContext *c = redisConnectWithTimeout(redis_ip, redis_port, connect_timeout);
if (NULL == c || c->err) { if (NULL == c || c->err) {
fprintf(stderr, "Unable to connect redis server %s:%d db%d, error: %s\n", log_error(logger, MODULE_MAAT_COMMAND, "Unable to connect redis server %s:%d db%d, error: %s",
redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr); redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr);
if (c != NULL) { if (c != NULL) {
redisFree(c); redisFree(c);
@@ -109,10 +111,10 @@ void maat_cmd_clear_rule_cache(struct serial_rule *s_rule)
memset(s_rule, 0, sizeof(struct serial_rule)); memset(s_rule, 0, sizeof(struct serial_rule));
} }
int connect_redis_for_write(struct source_redis_ctx *mr_ctx) int connect_redis_for_write(struct source_redis_ctx *mr_ctx, struct log_handle *logger)
{ {
assert(mr_ctx->write_ctx == NULL); assert(mr_ctx->write_ctx == NULL);
mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, logger);
if (NULL == mr_ctx->write_ctx) { if (NULL == mr_ctx->write_ctx) {
return -1; return -1;
} else { } else {
@@ -123,7 +125,7 @@ int connect_redis_for_write(struct source_redis_ctx *mr_ctx)
redisContext *get_redis_ctx_for_write(struct maat *maat_instance) redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
{ {
if (NULL == maat_instance->mr_ctx.write_ctx) { if (NULL == maat_instance->mr_ctx.write_ctx) {
int ret = connect_redis_for_write(&(maat_instance->mr_ctx)); int ret = connect_redis_for_write(&(maat_instance->mr_ctx), maat_instance->logger);
if(ret!=0) if(ret!=0)
{ {
return NULL; return NULL;
@@ -302,7 +304,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name); int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name);
if (table_id < 0) { if (table_id < 0) {
fprintf(stderr, "Command set line id %d failed: unknown table %s\n", line_rule->rule_id, line_rule->table_name); log_error(maat_instance->logger, MODULE_MAAT_COMMAND, "Command set line id %d failed: unknown table %s",
line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
return -1; return -1;
} }
@@ -315,8 +318,9 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
int valid_flag_column = table_schema_get_valid_flag_column(table_schema); int valid_flag_column = table_schema_get_valid_flag_column(table_schema);
if (valid_flag_column < 0) { if (valid_flag_column < 0) {
fprintf(stderr, "Command set line id %d failed: table %s is not a plugin or ip_plugin table\n", log_error(maat_instance->logger, MODULE_MAAT_COMMAND,
line_rule->rule_id, line_rule->table_name); "Command set line id %d failed: table %s is not a plugin or ip_plugin table",
line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
return -1; return -1;
} }
@@ -331,7 +335,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name, maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name,
line_rule->table_line, absolute_expire_time); line_rule->table_line, absolute_expire_time);
int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time); int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_instance->logger);
if (success_cnt != 1) { if (success_cnt != 1) {
ret = -1; ret = -1;
goto error_out; goto error_out;

68
src/maat_config_monitor.cpp
View File

@@ -22,6 +22,8 @@
#include "json2iris.h" #include "json2iris.h"
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
#define MODULE_CONFIG_MONITOR module_name_str("maat.config_monitor")
#define CM_MAX_TABLE_NUM 256 #define CM_MAX_TABLE_NUM 256
#define MAX_CONFIG_LINE (1024 * 16) #define MAX_CONFIG_LINE (1024 * 16)
@@ -215,14 +217,15 @@ int filter_fn(const struct dirent *ent)
strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0); strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0);
} }
int get_new_idx_path(long long current_version, const char *file_dir, char ***idx_path, size_t *idx_num) int get_new_idx_path(long long current_version, const char *file_dir,
char ***idx_path, size_t *idx_num, struct log_handle *logger)
{ {
struct dirent **namelist = NULL; struct dirent **namelist = NULL;
int update_type = CM_UPDATE_TYPE_NONE; int update_type = CM_UPDATE_TYPE_NONE;
int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort); int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort);
if (n < 0) { if (n < 0) {
//log_error("scan dir error"); log_error(logger, MODULE_CONFIG_MONITOR, "scan dir error");
return update_type; return update_type;
} }
@@ -241,15 +244,17 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id
} }
if (strlen(namelist[i]->d_name) > 42) { if (strlen(namelist[i]->d_name) > 42) {
fprintf(stderr, "config file %s filename too long, should like full_config_index.000000000001\n", log_error(logger, MODULE_CONFIG_MONITOR,
namelist[i]->d_name); "config file %s filename too long, should like full_config_index.000000000001",
namelist[i]->d_name);
continue; continue;
} }
int ret = sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld", update_str, &config_seq); int ret = sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld", update_str, &config_seq);
if (ret != 2) { if (ret != 2) {
fprintf(stderr, "config file %s filename error, should like full_config_index.000000000001\n", log_error(logger, MODULE_CONFIG_MONITOR,
namelist[i]->d_name); "config file %s filename error, should like full_config_index.000000000001",
namelist[i]->d_name);
continue; continue;
} }
@@ -267,7 +272,8 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id
} }
} }
} else { } else {
fprintf(stderr, "config file %s, not full or inc config\n", namelist[i]->d_name); log_error(logger, MODULE_CONFIG_MONITOR, "config file %s, not full or inc config",
namelist[i]->d_name);
} }
} }
@@ -306,8 +312,8 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id
void config_monitor_traverse(long long current_version, const char *idx_dir, void config_monitor_traverse(long long current_version, const char *idx_dir,
void (*start_fn)(long long, int, void *), void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *), int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void (*finish_fn)(void *), void *u_param,
void *u_param) struct log_handle *logger)
{ {
size_t i = 0; size_t i = 0;
long long new_version = 0; long long new_version = 0;
@@ -317,12 +323,13 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
memset(table_array, 0, sizeof(table_array)); memset(table_array, 0, sizeof(table_array));
int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num); int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num, logger);
if (update_type != CM_UPDATE_TYPE_NONE) { if (update_type != CM_UPDATE_TYPE_NONE) {
for (i = 0; i < idx_path_num; i++) { for (i = 0; i < idx_path_num; i++) {
log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]);
int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM); int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM);
if (table_num < 0) { if (table_num < 0) {
fprintf(stderr, "load %s failed, abandon update\n", idx_path_array[i]); log_error(logger, MODULE_CONFIG_MONITOR, "load %s failed, abandon update", idx_path_array[i]);
break; break;
} }
@@ -350,7 +357,8 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
FREE(idx_path_array); FREE(idx_path_array);
} }
int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz) int load_maat_json_file(struct maat *maat_instance, const char *json_filename,
char *err_str, size_t err_str_sz)
{ {
int ret = 0; int ret = 0;
struct stat fstat_buf; struct stat fstat_buf;
@@ -361,13 +369,16 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c
size_t decrypted_buff_sz = 0; size_t decrypted_buff_sz = 0;
size_t uncompressed_buff_sz = 0; size_t uncompressed_buff_sz = 0;
fprintf(stdout, "Maat initial with JSON file %s, formating...\n", json_filename); log_info(maat_instance->logger, MODULE_CONFIG_MONITOR,
"Maat initial with JSON file %s, formating...", json_filename);
if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) { if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) {
ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo, ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo,
(unsigned char **)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz); (unsigned char **)&decrypted_buff, &decrypted_buff_sz,
err_str, err_str_sz);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Decrypt Maat JSON file %s failed\n", json_filename); log_error(maat_instance->logger, MODULE_CONFIG_MONITOR,
"Decrypt Maat JSON file %s failed", json_filename);
return -1; return -1;
} }
@@ -379,7 +390,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c
ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz); ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz);
FREE(json_buff); FREE(json_buff);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Uncompress Maat JSON file %s failed\n", json_filename); log_error(maat_instance->logger, MODULE_CONFIG_MONITOR,
"Uncompress Maat JSON file %s failed", json_filename);
return -1; return -1;
} }
@@ -391,18 +403,18 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c
if (NULL == json_buff) { if (NULL == json_buff) {
ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz); ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz);
if (ret < 0) { if (ret < 0) {
fprintf(stderr, "Read Maat JSON file %s failed\n", json_filename); log_error(maat_instance->logger, MODULE_CONFIG_MONITOR,
"Read Maat JSON file %s failed", json_filename);
return -1; return -1;
} }
} }
ret = json2iris((const char*)json_buff, json_filename, ret = json2iris((const char*)json_buff, json_filename, maat_instance->compile_tn,
maat_instance->compile_tn, maat_instance->group2compile_tn, maat_instance->group2group_tn, maat_instance->group2compile_tn, maat_instance->group2group_tn, NULL,
NULL, maat_instance->json_ctx.iris_file, sizeof(maat_instance->json_ctx.iris_file),
maat_instance->json_ctx.iris_file, strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL,
sizeof(maat_instance->json_ctx.iris_file), strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL,
strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, maat_instance->logger);
strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL);
FREE(json_buff); FREE(json_buff);
json_buff = NULL; json_buff = NULL;
@@ -418,10 +430,10 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, c
maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim; maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim;
md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5); md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5);
fprintf(stdout, "JSON file %s md5: %s, generate index file %s OK\n", log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, "JSON file %s md5: %s, generate index file %s OK",
maat_instance->json_ctx.json_file, maat_instance->json_ctx.json_file,
maat_instance->json_ctx.effective_json_md5, maat_instance->json_ctx.effective_json_md5,
maat_instance->json_ctx.iris_file); maat_instance->json_ctx.iris_file);
maat_instance->input_mode = DATA_SOURCE_JSON_FILE; maat_instance->input_mode = DATA_SOURCE_JSON_FILE;
return 0; return 0;

57
src/maat_ex_data.cpp
View File

@@ -13,12 +13,15 @@
#include "uthash/uthash.h" #include "uthash/uthash.h"
#include "uthash/utarray.h" #include "uthash/utarray.h"
#include "log/log.h"
#include "rcu_hash.h" #include "rcu_hash.h"
#include "utils.h" #include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_table_schema.h" #include "maat_table_schema.h"
#include "maat_ex_data.h" #include "maat_ex_data.h"
#define MODULE_EX_DATA module_name_str("maat.ex_data")
struct ex_data_runtime { struct ex_data_runtime {
UT_array *cache_rows; UT_array *cache_rows;
size_t cache_row_num; size_t cache_row_num;
@@ -119,9 +122,14 @@ void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_da
ex_data_rt->ex_schema = schema; ex_data_rt->ex_schema = schema;
} }
void ex_data_runtime_set_user_ctx(struct ex_data_runtime *ex_data_rt, void *user_ctx) void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx)
{ {
rcu_hash_set_user_ctx(ex_data_rt->htable, user_ctx); rcu_hash_set_user_ctx(ex_data_rt->htable, container_ctx);
}
struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt)
{
return (struct ex_container_ctx *)rcu_hash_get_user_ctx(ex_data_rt->htable);
} }
void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len) void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len)
@@ -133,52 +141,59 @@ void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char
return ex_data; return ex_data;
} }
void ex_data_runtime_add_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, void *data) void ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, struct ex_data_container *ex_container)
{ {
void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len); struct ex_data_container *tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
if (tmp_data != NULL) { if (tmp_container != NULL) {
return; return;
} }
rcu_hash_add(ex_data_rt->htable, key, key_len, data); rcu_hash_add(ex_data_rt->htable, key, key_len, ex_container);
} }
void ex_data_runtime_del_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) void ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len,
struct log_handle *logger)
{ {
void *tmp_data = rcu_hash_find(ex_data_rt->htable, key, key_len); struct ex_data_container *tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
if (NULL == tmp_data) { if (NULL == tmp_container) {
fprintf(stderr, "ex data del error: no such key:%s\n", key); log_error(logger, MODULE_EX_DATA, "ex data del error: no such key:%s", key);
return; return;
} }
rcu_hash_del(ex_data_rt->htable, key, key_len); rcu_hash_del(ex_data_rt->htable, key, key_len);
} }
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) void *ex_data_runtime_dup_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
{ {
void *ex_data = rcu_hash_find(ex_data_rt->htable, key, key_len); struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
if (NULL == ex_data) { if (NULL == ex_container) {
return NULL; return NULL;
} }
void *dup_ex_data = NULL; void *dup_ex_data = NULL;
ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, &ex_data, ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, &(ex_container->ex_data),
ex_data_rt->ex_schema->argl, ex_data_rt->ex_schema->argp); ex_data_rt->ex_schema->argl, ex_data_rt->ex_schema->argp);
return dup_ex_data; return dup_ex_data;
} }
size_t ex_data_runtime_ex_data_count(struct ex_data_runtime *ex_data_rt) void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len)
{
struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len);
if (NULL == ex_container) {
return NULL;
}
return ex_container->custom_data;
}
size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt)
{ {
return rcu_hash_count(ex_data_rt->htable); return rcu_hash_count(ex_data_rt->htable);
} }
size_t ex_data_runtime_list_updating_ex_data(struct ex_data_runtime *ex_data_rt, void ***ex_data_array) size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container)
{ {
if (NULL == ex_data_rt->ex_schema) { return rcu_hash_list_updating_data(ex_data_rt->htable, (void ***)ex_container);
return 0;
}
return rcu_hash_list_updating_data(ex_data_rt->htable, ex_data_array);
} }
int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt) int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt)

244
src/maat_redis_monitor.cpp
View File

@@ -19,6 +19,8 @@
#include "maat_redis_monitor.h" #include "maat_redis_monitor.h"
#include "maat_table_schema.h" #include "maat_table_schema.h"
#define MODULE_REDIS_MONITOR module_name_str("maat.redis_monitor")
const time_t MAAT_REDIS_RECONNECT_INTERVAL_S = 5; const time_t MAAT_REDIS_RECONNECT_INTERVAL_S = 5;
const static int MAAT_REDIS_SYNC_TIME = 30 * 60; const static int MAAT_REDIS_SYNC_TIME = 30 * 60;
@@ -47,7 +49,8 @@ char *get_foreign_cont_filename(const char *table_name, int rule_id, const char
return filename; return filename;
} }
void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_foreign, const char *dir) void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_foreign,
const char *dir, struct log_handle *logger)
{ {
int foreign_key_size = 0; int foreign_key_size = 0;
p_rule->f_keys = ALLOC(struct foreign_key, n_foreign); p_rule->f_keys = ALLOC(struct foreign_key, n_foreign);
@@ -55,8 +58,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
for (int i = 0; i < n_foreign; i++) { for (int i = 0; i < n_foreign; i++) {
const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size); const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size);
if (NULL == p_foreign) { if (NULL == p_foreign) {
fprintf(stderr, "Get %s,%lu foreign keys failed: No %dth column\n", log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign keys failed: No %dth column",
p_rule->table_name, p_rule->rule_id, foreign_columns[i]); p_rule->table_name, p_rule->rule_id, foreign_columns[i]);
continue; continue;
} }
@@ -66,8 +69,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
} }
if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) { if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) {
fprintf(stderr, "Get %s,%lu foreign key failed: Invalid source prefix %s\n", log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign key failed: Invalid source prefix %s",
p_rule->table_name, p_rule->rule_id, p_foreign); p_rule->table_name, p_rule->rule_id, p_foreign);
continue; continue;
} }
@@ -76,8 +79,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
p_foreign += strlen(foreign_source_prefix); p_foreign += strlen(foreign_source_prefix);
if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) { if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) {
fprintf(stdout, "%s, %lu foreign key prefix %s is not recommended\n", log_info(logger, MODULE_REDIS_MONITOR, "%s, %lu foreign key prefix %s is not recommended",
p_rule->table_name, p_rule->rule_id, p_foreign); p_rule->table_name, p_rule->rule_id, p_foreign);
} }
p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size+1); p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size+1);
@@ -114,7 +117,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in
continue; continue;
} }
_get_foregin_keys(rule_list+i, foreign_columns, n_foreign_column, dir); _get_foregin_keys(rule_list+i, foreign_columns, n_foreign_column, dir, maat_instance->logger);
rule_with_foreign_key++; rule_with_foreign_key++;
} }
@@ -122,7 +125,7 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in
} }
int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, const char* dir) int rule_num, const char* dir, struct log_handle *logger)
{ {
int j = 0; int j = 0;
int foreign_key_size = 0; int foreign_key_size = 0;
@@ -146,7 +149,7 @@ int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *r
} while (p_foreign != NULL && n_foreign < MAX_FOREIGN_CLMN_NUM); } while (p_foreign != NULL && n_foreign < MAX_FOREIGN_CLMN_NUM);
if (n_foreign > 0) { if (n_foreign > 0) {
_get_foregin_keys(rule_list+i, foreign_columns, n_foreign, dir); _get_foregin_keys(rule_list+i, foreign_columns, n_foreign, dir, logger);
rule_with_foreign_key++; rule_with_foreign_key++;
} }
} }
@@ -160,7 +163,8 @@ struct foreign_conts_track
int foreign_idx; int foreign_idx;
}; };
int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num) int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num,
struct log_handle *logger)
{ {
int i = 0; int i = 0;
int failed_cnt = 0; int failed_cnt = 0;
@@ -181,8 +185,9 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
for (i = 0; i < rule_num; i++) { for (i = 0; i < rule_num; i++) {
ret = maat_cmd_wrap_redis_get_reply(c, &reply); ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) { if (ret == REDIS_ERR) {
fprintf(stderr, "Redis GET %s:%s,%lu failed, redis server error\n", mr_key_prefix[rule_list[i].op], log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed, redis server error",
rule_list[i].table_name, rule_list[i].rule_id); mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1; error_happened = 1;
break; break;
} }
@@ -194,8 +199,9 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
retry_ids[failed_cnt] = i; retry_ids[failed_cnt] = i;
failed_cnt++; failed_cnt++;
} else { } else {
fprintf(stderr, "Redis GET %s:%s,%lu failed\n",mr_key_prefix[rule_list[i].op], log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed",
rule_list[i].table_name, rule_list[i].rule_id); mr_key_prefix[rule_list[i].op],
rule_list[i].table_name, rule_list[i].rule_id);
error_happened = 1; error_happened = 1;
} }
} }
@@ -222,7 +228,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
idx = retry_ids[i]; idx = retry_ids[i];
ret = maat_cmd_wrap_redis_get_reply(c, &reply); ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) { if (ret == REDIS_ERR) {
fprintf(stderr, "redis command %s failed, redis server error\n", redis_cmd); log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, redis server error", redis_cmd);
FREE(retry_ids); FREE(retry_ids);
return -1; return -1;
} }
@@ -231,10 +237,12 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
rule_list[idx].table_line = maat_strdup(reply->str); rule_list[idx].table_line = maat_strdup(reply->str);
} else if(reply->type==REDIS_REPLY_ERROR) { } else if(reply->type==REDIS_REPLY_ERROR) {
//Deal with Redis response: "Loading Redis is loading the database in memory" //Deal with Redis response: "Loading Redis is loading the database in memory"
fprintf(stderr, "redis command %s error, reply type=%d, error str=%s\n", redis_cmd, reply->type, reply->str); log_error(logger, MODULE_REDIS_MONITOR, "redis command %s error, reply type=%d, error str=%s",
redis_cmd, reply->type, reply->str);
} else { } else {
//Handle type "nil" //Handle type "nil"
fprintf(stderr, "redis command %s failed, reply type=%d\n", redis_cmd, reply->type); log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, reply type=%d",
redis_cmd, reply->type);
} }
freeReplyObject(reply); freeReplyObject(reply);
@@ -246,7 +254,8 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru
return 0; return 0;
} }
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process) int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num,
int print_process, struct log_handle *logger)
{ {
int max_redis_batch = 4096; int max_redis_batch = 4096;
int success_cnt = 0; int success_cnt = 0;
@@ -254,7 +263,7 @@ int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int
while (success_cnt < rule_num) { while (success_cnt < rule_num) {
int batch_cnt = MIN(rule_num-success_cnt, max_redis_batch); int batch_cnt = MIN(rule_num-success_cnt, max_redis_batch);
int ret = _get_maat_redis_value(c, rule_list+success_cnt, batch_cnt); int ret = _get_maat_redis_value(c, rule_list+success_cnt, batch_cnt, logger);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} else { } else {
@@ -277,14 +286,15 @@ int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int
} }
int get_inc_key_list(long long instance_version, long long target_version, int get_inc_key_list(long long instance_version, long long target_version,
redisContext *c, struct serial_rule **list) redisContext *c, struct serial_rule **list, struct log_handle *logger)
{ {
//Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version. //Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version.
//The elements are considered to be ordered from low to high scores(instance_version). //The elements are considered to be ordered from low to high scores(instance_version).
redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", mr_status_sset, redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", mr_status_sset,
instance_version,target_version); instance_version,target_version);
if (NULL == reply) { if (NULL == reply) {
fprintf(stderr, "GET %s failed with a NULL reply, error: %s\n", mr_status_sset, c->errstr); log_error(logger, MODULE_REDIS_MONITOR, "GET %s failed with a NULL reply, error: %s",
mr_status_sset, c->errstr);
return -1; return -1;
} }
@@ -299,8 +309,8 @@ int get_inc_key_list(long long instance_version, long long target_version,
redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", mr_status_sset, reply->element[0]->str); redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", mr_status_sset, reply->element[0]->str);
if (tmp_reply->type != REDIS_REPLY_STRING) { if (tmp_reply->type != REDIS_REPLY_STRING) {
fprintf(stderr, "ZSCORE %s %s failed Version: %lld->%lld\n", mr_status_sset, log_error(logger, MODULE_REDIS_MONITOR, "ZSCORE %s %s failed Version: %lld->%lld",
reply->element[0]->str, instance_version, target_version); mr_status_sset, reply->element[0]->str, instance_version, target_version);
freeReplyObject(tmp_reply); freeReplyObject(tmp_reply);
tmp_reply = NULL; tmp_reply = NULL;
freeReplyObject(reply); freeReplyObject(reply);
@@ -317,8 +327,8 @@ int get_inc_key_list(long long instance_version, long long target_version,
} }
if (nearest_rule_version != instance_version + 1) { if (nearest_rule_version != instance_version + 1) {
fprintf(stdout, "Noncontinuous VERSION Redis: %lld MAAT: %lld\n", log_info(logger, MODULE_REDIS_MONITOR, "Noncontinuous VERSION Redis: %lld MAAT: %lld",
nearest_rule_version, instance_version); nearest_rule_version, instance_version);
} }
int i = 0; int i = 0;
@@ -331,7 +341,7 @@ int get_inc_key_list(long long instance_version, long long target_version,
int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", op_str, int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", op_str,
s_rule[j].table_name, &(s_rule[j].rule_id)); s_rule[j].table_name, &(s_rule[j].rule_id));
if (ret != 3 || s_rule[i].rule_id < 0) { if (ret != 3 || s_rule[i].rule_id < 0) {
fprintf(stderr, "Invalid Redis Key: %s\n", reply->element[i]->str); log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str);
continue; continue;
} }
@@ -340,7 +350,7 @@ int get_inc_key_list(long long instance_version, long long target_version,
} else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) { } else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) {
s_rule[j].op = MAAT_OP_DEL; s_rule[j].op = MAAT_OP_DEL;
} else { } else {
fprintf(stderr, "Invalid Redis Key: %s\n", reply->element[i]->str); log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str);
continue; continue;
} }
j++; j++;
@@ -458,7 +468,8 @@ int recovery_history_version(const struct serial_rule *current, int current_num,
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version,
long long *new_version, struct table_schema_manager* table_schema_mgr, long long *new_version, struct table_schema_manager* table_schema_mgr,
struct serial_rule **list, int *update_type, int cumulative_off) struct serial_rule **list, int *update_type, int cumulative_off,
struct log_handle *logger)
{ {
int rule_num = 0; int rule_num = 0;
long long target_version = 0; long long target_version = 0;
@@ -467,20 +478,20 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION"); redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION");
if (reply != NULL) { if (reply != NULL) {
if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) {
fprintf(stderr, "GET MAAT_VERSION failed, maybe Redis is busy\n"); log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed, maybe Redis is busy");
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return -1; return -1;
} }
} else { } else {
fprintf(stderr, "GET MAAT_VERSION failed with NULL reply, error: %s\n", c->errstr); log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed with NULL reply, error: %s", c->errstr);
return -1; return -1;
} }
long long redis_version = maat_cmd_read_redis_integer(reply); long long redis_version = maat_cmd_read_redis_integer(reply);
if (redis_version < 0) { if (redis_version < 0) {
if (reply->type == REDIS_REPLY_ERROR) { if (reply->type == REDIS_REPLY_ERROR) {
fprintf(stderr, "Redis Communication error: %s\n", reply->str); log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", reply->str);
} }
return -1; return -1;
} }
@@ -497,7 +508,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
} }
if (redis_version < instance_version) { if (redis_version < instance_version) {
fprintf(stderr, "VERSION roll back MAAT: %lld -> Redis: %lld\n", instance_version, redis_version); log_error(logger, MODULE_REDIS_MONITOR, "VERSION roll back MAAT: %lld -> Redis: %lld",
instance_version, redis_version);
goto FULL_UPDATE; goto FULL_UPDATE;
} }
@@ -509,7 +521,7 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
do { do {
target_version++; target_version++;
rule_num = get_inc_key_list(instance_version, target_version, c, &s_rule_array); rule_num = get_inc_key_list(instance_version, target_version, c, &s_rule_array, logger);
if (rule_num > 0) { if (rule_num > 0) {
break; break;
} else if (rule_num < 0) { } else if (rule_num < 0) {
@@ -521,13 +533,13 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
} while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off); } while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off);
if (0 == rule_num) { if (0 == rule_num) {
fprintf(stdout, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s\n", mr_status_sset, log_info(logger, MODULE_REDIS_MONITOR, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s",
instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON"); mr_status_sset, instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON");
return 0; return 0;
} }
fprintf(stdout, "Inc Update from instance_version %lld to %lld (%d entries)\n", log_info(logger, MODULE_REDIS_MONITOR, "Inc Update from instance_version %lld to %lld (%d entries)",
instance_version, target_version, rule_num); instance_version, target_version, rule_num);
*list = s_rule_array; *list = s_rule_array;
*update_type = CM_UPDATE_TYPE_INC; *update_type = CM_UPDATE_TYPE_INC;
@@ -535,8 +547,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l
return rule_num; return rule_num;
FULL_UPDATE: FULL_UPDATE:
fprintf(stdout, "Initiate full update from instance_version %lld to %lld\n", instance_version, log_info(logger, MODULE_REDIS_MONITOR, "Initiate full update from instance_version %lld to %lld",
desired_version == 0 ? redis_version : desired_version); instance_version, desired_version == 0 ? redis_version : desired_version);
size_t append_cmd_cnt = 0; size_t append_cmd_cnt = 0;
int ret = redisAppendCommand(c, "MULTI"); int ret = redisAppendCommand(c, "MULTI");
append_cmd_cnt++; append_cmd_cnt++;
@@ -557,12 +569,12 @@ FULL_UPDATE:
reply = maat_cmd_wrap_redis_command(c, "EXEC"); reply = maat_cmd_wrap_redis_command(c, "EXEC");
if (NULL == reply) { if (NULL == reply) {
fprintf(stderr, "Redis Communication error: %s\n", c->errstr); log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", c->errstr);
return -1; return -1;
} }
if (reply->type != REDIS_REPLY_ARRAY) { if (reply->type != REDIS_REPLY_ARRAY) {
fprintf(stderr, "Invalid Redis Key List type %d\n", reply->type); log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", reply->type);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return -1; return -1;
@@ -571,7 +583,7 @@ FULL_UPDATE:
*new_version = maat_cmd_read_redis_integer(reply->element[0]); *new_version = maat_cmd_read_redis_integer(reply->element[0]);
redisReply *sub_reply = reply->element[1]; redisReply *sub_reply = reply->element[1];
if (sub_reply->type != REDIS_REPLY_ARRAY) { if (sub_reply->type != REDIS_REPLY_ARRAY) {
fprintf(stderr, "Invalid Redis Key List type %d\n", sub_reply->type); log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", sub_reply->type);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return -1; return -1;
@@ -581,7 +593,7 @@ FULL_UPDATE:
s_rule_array = ALLOC(struct serial_rule, sub_reply->elements); s_rule_array = ALLOC(struct serial_rule, sub_reply->elements);
for (i = 0, full_idx = 0; i < sub_reply->elements; i++) { for (i = 0, full_idx = 0; i < sub_reply->elements; i++) {
if (sub_reply->element[i]->type != REDIS_REPLY_STRING) { if (sub_reply->element[i]->type != REDIS_REPLY_STRING) {
fprintf(stderr, "Invalid Redis Key Type: %d\n", sub_reply->element[i]->type); log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Type: %d", sub_reply->element[i]->type);
continue; continue;
} }
@@ -591,7 +603,7 @@ FULL_UPDATE:
s_rule_array[full_idx].op = MAAT_OP_ADD; s_rule_array[full_idx].op = MAAT_OP_ADD;
if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || strlen(s_rule_array[full_idx].table_name) == 0) { if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || strlen(s_rule_array[full_idx].table_name) == 0) {
fprintf(stderr, "Invalid Redis Key Format: %s\n", sub_reply->element[i]->str); log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Format: %s", sub_reply->element[i]->str);
continue; continue;
} }
@@ -611,13 +623,13 @@ FULL_UPDATE:
if (desired_version != 0) { if (desired_version != 0) {
struct serial_rule *changed_rule_array = NULL; struct serial_rule *changed_rule_array = NULL;
int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array); int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array, logger);
if (changed_rule_num < 0) { if (changed_rule_num < 0) {
fprintf(stderr, "Recover history version %lld faild where as redis version is %lld\n", log_error(logger, MODULE_REDIS_MONITOR, "Recover history version %lld faild where as redis version is %lld",
desired_version, redis_version); desired_version, redis_version);
} else if(0 == changed_rule_num) { } else if(0 == changed_rule_num) {
fprintf(stderr, "Nothing to recover from history version %lld to redis version is %lld\n", log_error(logger, MODULE_REDIS_MONITOR, "Nothing to recover from history version %lld to redis version is %lld",
desired_version, redis_version); desired_version, redis_version);
} else { } else {
struct serial_rule *history_rule_array = NULL; struct serial_rule *history_rule_array = NULL;
ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array); ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array);
@@ -626,8 +638,8 @@ FULL_UPDATE:
s_rule_array = history_rule_array; s_rule_array = history_rule_array;
rule_num = ret; rule_num = ret;
*new_version = desired_version; *new_version = desired_version;
fprintf(stdout, "Successfully recovered from history version %lld to redis version is %lld\n", log_info(logger, MODULE_REDIS_MONITOR, "Successfully recovered from history version %lld to redis version is %lld",
desired_version, redis_version); desired_version, redis_version);
} }
} }
FREE(changed_rule_array); FREE(changed_rule_array);
@@ -635,12 +647,13 @@ FULL_UPDATE:
*list = s_rule_array; *list = s_rule_array;
*update_type = CM_UPDATE_TYPE_FULL; *update_type = CM_UPDATE_TYPE_FULL;
fprintf(stdout, "Full update %d keys of version %lld\n", rule_num, *new_version); log_info(logger, MODULE_REDIS_MONITOR, "Full update %d keys of version %lld", rule_num, *new_version);
return rule_num ; return rule_num ;
} }
void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn) void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn,
struct log_handle *logger)
{ {
int i = 0; int i = 0;
int j = 0; int j = 0;
@@ -663,8 +676,8 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
ret = remove(rule_list[i].f_keys[j].filename); ret = remove(rule_list[i].f_keys[j].filename);
if (ret == -1) { if (ret == -1) {
fprintf(stderr, "Foreign content file %s remove failed\n", log_error(logger, MODULE_REDIS_MONITOR, "Foreign content file %s remove failed",
rule_list[i].f_keys[j].filename); rule_list[i].f_keys[j].filename);
} }
} }
} else { } else {
@@ -694,25 +707,28 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
for (i = 0; i < key_num; i++) { for (i = 0; i < key_num; i++) {
ret = maat_cmd_wrap_redis_get_reply(c, &reply); ret = maat_cmd_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) { if (ret == REDIS_ERR) {
fprintf(stderr, "Get %s,%lu foreign key %s content failed, redis server error\n", log_error(logger, MODULE_REDIS_MONITOR,
rule_list[track[i].rule_idx].table_name, "Get %s,%lu foreign key %s content failed, redis server error",
rule_list[track[i].rule_idx].rule_id, rule_list[track[i].rule_idx].table_name,
rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); rule_list[track[i].rule_idx].rule_id,
rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key);
break; break;
} }
if (reply->type != REDIS_REPLY_STRING) { if (reply->type != REDIS_REPLY_STRING) {
fprintf(stderr, "Get %s,%lu foreign key %s content failed\n", log_error(logger, MODULE_REDIS_MONITOR,
rule_list[track[i].rule_idx].table_name, "Get %s,%lu foreign key %s content failed",
rule_list[track[i].rule_idx].rule_id, rule_list[track[i].rule_idx].table_name,
rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); rule_list[track[i].rule_idx].rule_id,
rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key);
continue; continue;
} else { } else {
s_rule = rule_list+track[i].rule_idx; s_rule = rule_list+track[i].rule_idx;
FILE *fp = fopen(s_rule->f_keys[track[i].foreign_idx].filename, "w"); FILE *fp = fopen(s_rule->f_keys[track[i].foreign_idx].filename, "w");
if (NULL == fp) { if (NULL == fp) {
fprintf(stderr, "Write foreign content failed: fopen %s error\n", log_error(logger, MODULE_REDIS_MONITOR,
s_rule->f_keys[track[i].foreign_idx].filename); "Write foreign content failed: fopen %s error",
s_rule->f_keys[track[i].foreign_idx].filename);
} else { } else {
fwrite(reply->str, 1, reply->len, fp); fwrite(reply->str, 1, reply->len, fp);
fclose(fp); fclose(fp);
@@ -731,14 +747,15 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule
return; return;
} }
void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn) void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn,
struct log_handle *logger)
{ {
int max_redis_batch = 4096; int max_redis_batch = 4096;
int success_cnt = 0; int success_cnt = 0;
while (success_cnt < rule_num) { while (success_cnt < rule_num) {
int batch_cnt = MIN(rule_num - success_cnt, max_redis_batch); int batch_cnt = MIN(rule_num - success_cnt, max_redis_batch);
_get_foreign_conts(c, rule_list + success_cnt, batch_cnt, print_fn); _get_foreign_conts(c, rule_list + success_cnt, batch_cnt, print_fn, logger);
success_cnt += batch_cnt; success_cnt += batch_cnt;
} }
} }
@@ -1036,7 +1053,8 @@ int mr_operation_success(redisReply *actual_reply, struct expected_reply *expect
return 0; return 0;
} }
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time) int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num,
long long server_time, struct log_handle *logger)
{ {
size_t i = 0; size_t i = 0;
size_t rule_seq = 0; size_t rule_seq = 0;
@@ -1085,16 +1103,20 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri
assert(transaction_reply->elements == multi_cmd_cnt); assert(transaction_reply->elements == multi_cmd_cnt);
for (i = 0; i < multi_cmd_cnt; i++) { for (i = 0; i < multi_cmd_cnt; i++) {
p = transaction_reply->element[i]; p = transaction_reply->element[i];
//failed is acceptable //failed is acceptable
//or transaciton is success //or transaciton is success
//or continuation of last failed //or continuation of last failed
if (expected_reply[i].s_rule_seq == -1 || 1 == mr_operation_success(p, expected_reply+i) || last_failed == expected_reply[i].s_rule_seq) { if (expected_reply[i].s_rule_seq == -1 ||
1 == mr_operation_success(p, expected_reply+i) ||
last_failed == expected_reply[i].s_rule_seq) {
continue; continue;
} }
rule_seq = expected_reply[i].s_rule_seq; rule_seq = expected_reply[i].s_rule_seq;
fprintf(stderr, "%s %s %lu failed, rule id maybe conflict or not exist\n", log_error(logger, MODULE_REDIS_MONITOR,
mr_op_str[s_rule[rule_seq].op], s_rule[rule_seq].table_name, "%s %s %lu failed, rule id maybe conflict or not exist",
s_rule[rule_seq].rule_id); mr_op_str[s_rule[rule_seq].op], s_rule[rule_seq].table_name,
s_rule[rule_seq].rule_id);
success_cnt--; success_cnt--;
last_failed = rule_seq; last_failed = rule_seq;
} }
@@ -1103,9 +1125,10 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri
} }
if (transaction_version > 0) { if (transaction_version > 0) {
transaction_finished_version = maat_cmd_read_redis_integer(transaction_reply->element[multi_cmd_cnt-1]); transaction_finished_version = maat_cmd_read_redis_integer(transaction_reply->element[multi_cmd_cnt - 1]);
fprintf(stdout, "Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld\n", log_info(logger, MODULE_REDIS_MONITOR,
transaction_version, transaction_finished_version); "Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld",
transaction_version, transaction_finished_version);
} }
freeReplyObject(transaction_reply); freeReplyObject(transaction_reply);
@@ -1115,9 +1138,8 @@ error_out:
if (renew_num > 0 && renew_allowed != 1) { if (renew_num > 0 && renew_allowed != 1) {
for (i = 0; i < (unsigned int)serial_rule_num; i++) { for (i = 0; i < (unsigned int)serial_rule_num; i++) {
if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) { if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) {
fprintf(stdout, "%s %s %lu is not allowed due to lock contention\n", log_error(logger, MODULE_REDIS_MONITOR, "%s %s %lu is not allowed due to lock contention",
mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, s_rule[i].rule_id);
s_rule[i].rule_id);
} }
} }
@@ -1131,7 +1153,7 @@ error_out:
return success_cnt; return success_cnt;
} }
void cleanup_update_status(redisContext *c) void cleanup_update_status(redisContext *c, struct log_handle *logger)
{ {
long long version_upper_bound = 0; long long version_upper_bound = 0;
long long version_lower_bound = 0; long long version_lower_bound = 0;
@@ -1191,18 +1213,17 @@ void cleanup_update_status(redisContext *c)
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
fprintf(stdout, "Clean up update status from version %lld to %lld (%lld versions, %lld entries)\n", log_info(logger, MODULE_REDIS_MONITOR,
version_lower_bound, version_upper_bound, version_num, entry_num); "Clean up update status from version %lld to %lld (%lld versions, %lld entries)",
version_lower_bound, version_upper_bound, version_num, entry_num);
return; return;
error_out: error_out:
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
return;
} }
void check_maat_expiration(redisContext *c) void check_maat_expiration(redisContext *c, struct log_handle *logger)
{ {
UNUSED int ret = 0; UNUSED int ret = 0;
@@ -1229,14 +1250,14 @@ void check_maat_expiration(redisContext *c)
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply = NULL; data_reply = NULL;
int success_cnt = maat_cmd_write_rule(c, s_rule, s_rule_num, server_time); int success_cnt = maat_cmd_write_rule(c, s_rule, s_rule_num, server_time, logger);
if (success_cnt < 0) { if (success_cnt < 0) {
fprintf(stderr, "maat_cmd_write_rule failed.\n"); log_error(logger, MODULE_REDIS_MONITOR, "maat_cmd_write_rule failed.");
} else if (success_cnt == (int)s_rule_num) { } else if (success_cnt == (int)s_rule_num) {
fprintf(stdout, "Succesfully expired %zu rules in Redis\n", s_rule_num); log_info(logger, MODULE_REDIS_MONITOR, "Succesfully expired %zu rules in Redis", s_rule_num);
} else { } else {
fprintf(stderr, "Failed to expired %d of %zu rules in Redis, try later\n", log_error(logger, MODULE_REDIS_MONITOR, "Failed to expired %d of %zu rules in Redis, try later",
s_rule_num - success_cnt, s_rule_num); s_rule_num - success_cnt, s_rule_num);
} }
FREE(s_rule); FREE(s_rule);
@@ -1245,8 +1266,7 @@ void check_maat_expiration(redisContext *c)
void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
void (*start_fn)(long long, int, void *), void (*start_fn)(long long, int, void *),
int (*update_fn)(const char *, const char *, void *), int (*update_fn)(const char *, const char *, void *),
void (*finish_fn)(void *), void (*finish_fn)(void *), void *u_param)
void *u_param)
{ {
int i = 0; int i = 0;
int ret = 0; int ret = 0;
@@ -1258,13 +1278,14 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
enum table_type table_type; enum table_type table_type;
enum scan_type scan_type; enum scan_type scan_type;
struct table_schema *table_schema = NULL; struct table_schema *table_schema = NULL;
struct maat *maat_instance = (struct maat *)u_param;
//authorized to write //authorized to write
if (mr_ctx->write_ctx != NULL && mr_ctx->write_ctx->err == 0) { if (mr_ctx->write_ctx != NULL && mr_ctx->write_ctx->err == 0) {
//For thread safe, deliberately use redis_read_ctx but not redis_write_ctx. //For thread safe, deliberately use redis_read_ctx but not redis_write_ctx.
if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout_ms)) { if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout_ms)) {
check_maat_expiration(mr_ctx->read_ctx); check_maat_expiration(mr_ctx->read_ctx, maat_instance->logger);
cleanup_update_status(mr_ctx->read_ctx); cleanup_update_status(mr_ctx->read_ctx, maat_instance->logger);
redlock_unlock(mr_ctx->read_ctx, mr_expire_lock); redlock_unlock(mr_ctx->read_ctx, mr_expire_lock);
} }
} }
@@ -1278,9 +1299,10 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
if (mr_ctx->read_ctx != NULL) { if (mr_ctx->read_ctx != NULL) {
redisFree(mr_ctx->read_ctx); redisFree(mr_ctx->read_ctx);
} }
fprintf(stdout, "Reconnecting...\n"); log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Reconnecting...");
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db,
maat_instance->logger);
if (NULL == mr_ctx->read_ctx) { if (NULL == mr_ctx->read_ctx) {
return; return;
} else { } else {
@@ -1288,14 +1310,14 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
} }
} }
struct maat *maat_instance = (struct maat *)u_param;
struct serial_rule *rule_list = NULL; struct serial_rule *rule_list = NULL;
long long new_version = 0; long long new_version = 0;
int update_type = CM_UPDATE_TYPE_INC; int update_type = CM_UPDATE_TYPE_INC;
int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version, int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version,
&new_version, maat_instance->table_schema_mgr, &rule_list, &new_version, maat_instance->table_schema_mgr, &rule_list,
&update_type, maat_instance->cumulative_update_off); &update_type, maat_instance->cumulative_update_off,
maat_instance->logger);
//redis communication error //redis communication error
if (rule_num < 0) { if (rule_num < 0) {
redisFree(mr_ctx->read_ctx); redisFree(mr_ctx->read_ctx);
@@ -1310,12 +1332,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
} }
if (rule_num > 0) { if (rule_num > 0) {
ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, 0); ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger);
//redis communication error //redis communication error
if (ret < 0) { if (ret < 0) {
redisFree(mr_ctx->read_ctx); redisFree(mr_ctx->read_ctx);
mr_ctx->read_ctx = NULL; mr_ctx->read_ctx = NULL;
fprintf(stderr, "Get Redis value failed, abandon update and close connection\n"); log_error(maat_instance->logger, MODULE_REDIS_MONITOR,
"Get Redis value failed, abandon update and close connection");
goto clean_up; goto clean_up;
} }
@@ -1326,19 +1349,20 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
} }
if (empty_value_num == rule_num) { if (empty_value_num == rule_num) {
fprintf(stdout, "All %d rules are empty, abandon update\n", empty_value_num); log_info(maat_instance->logger, MODULE_REDIS_MONITOR,
"All %d rules are empty, abandon update", empty_value_num);
goto clean_up; goto clean_up;
} }
ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num, maat_instance, maat_instance->foreign_cont_dir); ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num, maat_instance, maat_instance->foreign_cont_dir);
if (ret > 0) { if (ret > 0) {
maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0); maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger);
} }
} }
start_fn(new_version, update_type, u_param); start_fn(new_version, update_type, u_param);
fprintf(stdout, "Start %s update: %lld -> %lld (%d entries)\n", log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Start %s update: %lld -> %lld (%d entries)",
update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num); update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num);
for (i = 0; i < rule_num; i++) { for (i = 0; i < rule_num; i++) {
if (NULL == rule_list[i].table_line) { if (NULL == rule_list[i].table_line) {
@@ -1360,7 +1384,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
valid_column = table_schema_get_valid_flag_column(table_schema); valid_column = table_schema_get_valid_flag_column(table_schema);
ret = invalidate_line(rule_list[i].table_line, table_type, valid_column); ret = invalidate_line(rule_list[i].table_line, table_type, valid_column);
if (ret < 0) { if (ret < 0) {
fprintf(stdout, "Invalidate line failed, invaid format %s\n", rule_list[i].table_line); log_error(maat_instance->logger, MODULE_REDIS_MONITOR,
"Invalidate line failed, invaid format %s", rule_list[i].table_line);
continue; continue;
} }
} }
@@ -1376,8 +1401,9 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
finish_fn(u_param); finish_fn(u_param);
if (call_update_num < rule_num) { if (call_update_num < rule_num) {
fprintf(stdout, "Load %d entries to match engine, no table: %d, empty value: %d\n", log_error(maat_instance->logger, MODULE_REDIS_MONITOR,
call_update_num, no_table_num, empty_value_num); "Load %d entries to match engine, no table: %d, empty value: %d",
call_update_num, no_table_num, empty_value_num);
} }
clean_up: clean_up:

60
src/maat_rule.cpp
View File

@@ -27,6 +27,8 @@
#include "maat_table_runtime.h" #include "maat_table_runtime.h"
#include "maat_table_schema.h" #include "maat_table_schema.h"
#define MODULE_MAAT_RULE module_name_str("maat.rule")
struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance) struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_instance)
{ {
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
@@ -105,13 +107,14 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
struct maat_runtime* maat_rt = NULL; struct maat_runtime* maat_rt = NULL;
int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name); int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name);
if (table_id < 0) { if (table_id < 0) {
fprintf(stderr, "update warning, unknown table name %s\n", table_name); log_error(maat_instance->logger, MODULE_MAAT_RULE, "update warning, unknown table name %s", table_name);
return -1; return -1;
} }
struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
if (NULL == table_schema) { if (NULL == table_schema) {
fprintf(stderr, "update warning, table name %s doesn't have table schema\n", table_name); log_error(maat_instance->logger, MODULE_MAAT_RULE,
"update warning, table name %s doesn't have table schema", table_name);
return -1; return -1;
} }
@@ -123,10 +126,10 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
maat_rt = maat_instance->maat_rt; maat_rt = maat_instance->maat_rt;
} }
struct table_item *table_item = table_schema_line_to_item(line, table_schema); struct table_item *table_item = table_schema_line_to_item(line, table_schema, maat_instance->logger);
if (table_item != NULL) { if (table_item != NULL) {
struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id);
table_runtime_update(table_rt, table_schema, line, table_item); table_runtime_update(table_rt, table_schema, line, table_item, maat_instance->logger);
FREE(table_item); FREE(table_item);
} }
@@ -157,16 +160,14 @@ void maat_finish_cb(void *u_param)
if (maat_instance->creating_maat_rt != NULL) { if (maat_instance->creating_maat_rt != NULL) {
maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt);
maat_runtime_commit(maat_instance->creating_maat_rt); maat_runtime_commit(maat_instance->creating_maat_rt);
fprintf(stdout, "Full config version %llu load %d entries complete\n", log_info(maat_instance->logger, MODULE_MAAT_RULE, "Full config version %llu load %d entries complete\n",
maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->rule_num);
maat_instance->creating_maat_rt->rule_num);
} else if (maat_instance->maat_rt != NULL) { } else if (maat_instance->maat_rt != NULL) {
maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt); maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt);
maat_instance->maat_rt->version = maat_instance->maat_version; maat_instance->maat_rt->version = maat_instance->maat_version;
maat_runtime_commit(maat_instance->maat_rt); maat_runtime_commit(maat_instance->maat_rt);
fprintf(stdout, "Inc config version %llu load %d entries complete\n", log_info(maat_instance->logger, MODULE_MAAT_RULE, "Inc config version %llu load %d entries complete\n",
maat_instance->maat_rt->version, maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num);
maat_instance->maat_rt->rule_num);
} }
} }
@@ -189,7 +190,7 @@ void *rule_monitor_loop(void *arg)
pthread_mutex_lock(&(maat_instance->background_update_mutex)); pthread_mutex_lock(&(maat_instance->background_update_mutex));
/* if deferred load on */ /* if deferred load on */
if (maat_instance->deferred_load != 0) { if (maat_instance->deferred_load != 0) {
fprintf(stdout, "Deferred Loading ON, updating in %s\n", __func__); log_info(maat_instance->logger, MODULE_MAAT_RULE, "Deferred Loading ON, updating in %s", __func__);
maat_read_full_config(maat_instance); maat_read_full_config(maat_instance);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -204,18 +205,14 @@ void *rule_monitor_loop(void *arg)
case DATA_SOURCE_REDIS: case DATA_SOURCE_REDIS:
redis_monitor_traverse(maat_instance->maat_version, redis_monitor_traverse(maat_instance->maat_version,
&(maat_instance->mr_ctx), &(maat_instance->mr_ctx),
maat_start_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_update_cb,
maat_finish_cb,
maat_instance); maat_instance);
break; break;
case DATA_SOURCE_IRIS_FILE: case DATA_SOURCE_IRIS_FILE:
config_monitor_traverse(maat_instance->maat_version, config_monitor_traverse(maat_instance->maat_version,
maat_instance->iris_ctx.inc_idx_dir, maat_instance->iris_ctx.inc_idx_dir,
maat_start_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_update_cb, maat_instance, maat_instance->logger);
maat_finish_cb,
maat_instance);
break; break;
case DATA_SOURCE_JSON_FILE: case DATA_SOURCE_JSON_FILE:
memset(md5_tmp, 0, sizeof(md5_tmp)); memset(md5_tmp, 0, sizeof(md5_tmp));
@@ -224,18 +221,19 @@ void *rule_monitor_loop(void *arg)
maat_instance->json_ctx.last_md5_time = attrib.st_ctim; maat_instance->json_ctx.last_md5_time = attrib.st_ctim;
md5_file(maat_instance->json_ctx.json_file, md5_tmp); md5_file(maat_instance->json_ctx.json_file, md5_tmp);
if (0 != strcmp(md5_tmp, maat_instance->json_ctx.effective_json_md5)) { if (0 != strcmp(md5_tmp, maat_instance->json_ctx.effective_json_md5)) {
ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file,
err_str, sizeof(err_str));
if (ret < 0) { if (ret < 0) {
fprintf(stdout, "Maat re-initiate with JSON file %s (md5=%s)failed: %s\n", log_error(maat_instance->logger, MODULE_MAAT_RULE,
maat_instance->json_ctx.json_file, md5_tmp, err_str); "Maat re-initiate with JSON file %s (md5=%s)failed: %s\n",
maat_instance->json_ctx.json_file, md5_tmp, err_str);
} else { } else {
config_monitor_traverse(0, maat_instance->json_ctx.iris_file, config_monitor_traverse(0, maat_instance->json_ctx.iris_file,
maat_start_cb, maat_start_cb, maat_update_cb, maat_finish_cb,
maat_update_cb, maat_instance, maat_instance->logger);
maat_finish_cb, log_info(maat_instance->logger, MODULE_MAAT_RULE,
maat_instance); "Maat re-initiate with JSON file %s success, md5: %s\n",
fprintf(stdout, "Maat re-initiate with JSON file %s success, md5: %s\n", maat_instance->json_ctx.json_file, md5_tmp);
maat_instance->json_ctx.json_file, md5_tmp);
} }
} }
} }
@@ -250,11 +248,11 @@ void *rule_monitor_loop(void *arg)
if (old_maat_rt != NULL) { if (old_maat_rt != NULL) {
if (maat_instance->maat_rt->version > old_maat_rt->version) { if (maat_instance->maat_rt->version > old_maat_rt->version) {
fprintf(stdout, "Maat version updated %lld -> %lld\n", log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version updated %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version); old_maat_rt->version, maat_instance->maat_rt->version);
} else { } else {
fprintf(stdout, "Maat version roll back %lld -> %lld\n", log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version roll back %lld -> %lld\n",
old_maat_rt->version, maat_instance->maat_rt->version); old_maat_rt->version, maat_instance->maat_rt->version);
} }
maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy); maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy);
} }

304
src/maat_table_runtime.cpp
View File

@@ -15,6 +15,7 @@
#include <hs/hs.h> #include <hs/hs.h>
#include "utils.h" #include "utils.h"
#include "log/log.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_table_runtime.h" #include "maat_table_runtime.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
@@ -23,15 +24,18 @@
#include "rcu_hash.h" #include "rcu_hash.h"
#include "IPMatcher.h" #include "IPMatcher.h"
#define MODULE_TABLE_RUNTIME module_name_str("maat.table_runtime")
#define MAAT_MAX_EXPR_ITEM_NUM 8 #define MAAT_MAX_EXPR_ITEM_NUM 8
struct plugin_ex_data { struct port_range {
void *data; uint16_t min_port;
void *user_data; uint16_t max_port;
}; };
struct plugin_user_ctx { struct ex_container_ctx {
int table_id; int table_id;
void (*custom_data_free)(void *custom_data);
struct ex_data_schema *ex_schema; struct ex_data_schema *ex_schema;
}; };
@@ -42,11 +46,6 @@ struct expr_runtime {
struct rcu_hash_table *htable; struct rcu_hash_table *htable;
}; };
struct ip_runtime {
struct ip_matcher *ip_matcher;
struct rcu_hash_table *htable;
};
struct plugin_runtime { struct plugin_runtime {
uint64_t acc_line_num; uint64_t acc_line_num;
struct ex_data_runtime *ex_data_rt; struct ex_data_runtime *ex_data_rt;
@@ -63,7 +62,8 @@ struct table_runtime {
enum table_type table_type; enum table_type table_type;
union { union {
struct expr_runtime expr_rt; struct expr_runtime expr_rt;
struct ip_runtime ip_rt; //struct ip_plugin_runtime ip_rt;
struct ip_plugin_runtime ip_plus_rt;
struct plugin_runtime plugin_rt; struct plugin_runtime plugin_rt;
struct ip_plugin_runtime ip_plugin_rt; struct ip_plugin_runtime ip_plugin_rt;
}; };
@@ -80,16 +80,26 @@ struct table_runtime_manager {
struct maat_garbage_bin *garbage_bin; struct maat_garbage_bin *garbage_bin;
}; };
void plugin_ex_data_free(void *user_ctx, void *data) void ex_data_container_free(void *ctx, void *data)
{ {
if (NULL == user_ctx || NULL == data) { if (NULL == ctx || NULL == data) {
return; return;
} }
struct plugin_user_ctx *ctx = (struct plugin_user_ctx *)user_ctx; struct ex_container_ctx *container_ctx = (struct ex_container_ctx *)ctx;
long argl = ctx->ex_schema->argl; long argl = container_ctx->ex_schema->argl;
void *argp = ctx->ex_schema->argp; void *argp = container_ctx->ex_schema->argp;
ctx->ex_schema->free_func(ctx->table_id, &data, argl, argp);
struct ex_data_container *ex_container = (struct ex_data_container *)data;
if (ex_container->ex_data != NULL && container_ctx->ex_schema->free_func != NULL) {
container_ctx->ex_schema->free_func(container_ctx->table_id, &(ex_container->ex_data), argl, argp);
}
if (ex_container->custom_data != NULL && container_ctx->custom_data_free != NULL) {
container_ctx->custom_data_free(ex_container->custom_data);
}
FREE(ex_container);
} }
void expr_rule_free(and_expr_t *expr_rule) void expr_rule_free(and_expr_t *expr_rule)
@@ -121,11 +131,14 @@ struct table_runtime *table_runtime_new(struct table_schema *table_schema, int m
table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free); table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free);
table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema); table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema);
break; break;
case TABLE_TYPE_IP_PLUS:
table_rt->ip_plus_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
break;
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, plugin_ex_data_free); table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, plugin_ex_data_free); table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
break; break;
default: default:
break; break;
@@ -263,9 +276,72 @@ void table_runtime_stream_close(struct table_runtime *table_rt)
} }
} }
struct ip_rule *ip_plugin_item_to_ip_rule(struct ip_plugin_item *ip_plugin_item) enum IP_TYPE ip_type_transform(enum ip_type type)
{ {
return NULL; if (type == IP_TYPE_V4) {
return IPv4;
} else {
return IPv6;
}
}
int table_runtime_scan_ip(struct table_runtime *table_rt, int thread_id, struct table_rt_2tuple *data,
int results[], size_t *n_result)
{
if (NULL == table_rt) {
return -1;
}
int i = 0;
int n_hit_result = 0;
struct scan_result scan_results[table_rt->rule_num] = {0};
struct ip_data ip;
ip.type = ip_type_transform(data->ip_type);
if (ip.type == IPv4) {
ip.ipv4 = data->ipv4;
} else {
memcpy(ip.ipv6, data->ipv6, sizeof(data->ipv6));
}
n_hit_result = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, table_rt->rule_num);
int index = 0;
/* filter rule_id by port */
for (i = 0; i < n_hit_result; i++) {
struct port_range *port_range = (struct port_range *)scan_results[i].tag;
if (data->port < port_range->min_port || data->port > port_range->max_port) {
continue;
}
results[index++] = scan_results[i].rule_id;
}
*n_result = index;
return 0;
}
void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule)
{
struct port_range *sport_range = ALLOC(struct port_range, 1);
if (4 == item->addr_type) {
rule->type = IPv4;
sport_range->min_port = item->ipv4.min_sport;
sport_range->max_port = item->ipv4.max_sport;
} else {
rule->type = IPv6;
sport_range->min_port = item->ipv6.min_sport;
sport_range->max_port = item->ipv6.max_sport;
}
rule->rule_id = item->item_id;
rule->user_tag = sport_range;
rule->ipv4_rule.start_ip = item->ipv4.min_sip;
rule->ipv4_rule.end_ip = item->ipv4.max_sip;
memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, sizeof(item->ipv6.min_sip));
memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, sizeof(item->ipv6.max_sip));
}
void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule)
{
} }
enum pattern_type expr_type2pattern_type(enum expr_type expr_type) enum pattern_type expr_type2pattern_type(enum expr_type expr_type)
@@ -286,7 +362,7 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type)
return pattern_type; return pattern_type;
} }
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item) and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger)
{ {
size_t i = 0; size_t i = 0;
size_t sub_expr_cnt = 0; size_t sub_expr_cnt = 0;
@@ -305,7 +381,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item)
} }
if (i >= MAAT_MAX_EXPR_ITEM_NUM) { if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
fprintf(stderr, "item_id:%d too many patterns\n", expr_item->item_id); log_error(logger, MODULE_TABLE_RUNTIME, "item_id:%d too many patterns",
expr_item->item_id);
return NULL; return NULL;
} }
@@ -340,7 +417,7 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item)
} }
void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len, void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len,
and_expr_t *expr_rule, int is_valid) and_expr_t *expr_rule, int is_valid, struct log_handle *logger)
{ {
void *data = NULL; void *data = NULL;
@@ -348,7 +425,7 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key
//delete //delete
data = rcu_hash_find(expr_rt->htable, key, key_len); data = rcu_hash_find(expr_rt->htable, key, key_len);
if (NULL == data) { if (NULL == data) {
fprintf(stderr, "the key:%s not exist, so can't be deleted\n", key); log_error(logger, MODULE_TABLE_RUNTIME, "the key:%s not exist, so can't be deleted", key);
return; return;
} }
rcu_hash_del(expr_rt->htable, key, key_len); rcu_hash_del(expr_rt->htable, key, key_len);
@@ -356,7 +433,7 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key
//add //add
data = rcu_hash_find(expr_rt->htable, key, key_len); data = rcu_hash_find(expr_rt->htable, key, key_len);
if (data != NULL) { if (data != NULL) {
fprintf(stderr, "the key:%s already exist, so can't be added\n", key); log_error(logger, MODULE_TABLE_RUNTIME, "the key:%s already exist, so can't be added", key);
return; return;
} }
and_expr_t *data = ALLOC(and_expr_t, 1); and_expr_t *data = ALLOC(and_expr_t, 1);
@@ -369,11 +446,33 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key
} }
} }
void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_schema *table_schema, void ip_plus_runtime_update_row(struct ip_plugin_runtime *ip_plus_rt, struct table_schema *table_schema,
const char *row, char *key, size_t key_len, int is_valid) const char *row, char *key, size_t key_len, struct ip_plus_item *ip_plus_item,
int is_valid, struct log_handle *logger)
{ {
struct plugin_ex_data *ex_data = ALLOC(struct plugin_ex_data, 1); struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
ex_data->data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt;
ex_container->ex_data = NULL;
ex_container->custom_data = ip_plus_item;
struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1);
ctx->custom_data_free = free;
ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx);
if (0 == is_valid) {
// delete
ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger);
} else {
// add
ex_data_runtime_add_ex_container(ip_plus_rt->ex_data_rt, key, key_len, ex_container);
}
}
void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_schema *table_schema,
const char *row, char *key, size_t key_len, int is_valid,
struct log_handle *logger)
{
struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
ex_container->ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len);
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema); int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema);
size_t cb_count = plugin_table_schema_callback_count(table_schema); size_t cb_count = plugin_table_schema_callback_count(table_schema);
@@ -381,10 +480,10 @@ void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_sc
if (1 == set_flag) { if (1 == set_flag) {
if (is_valid == 0) { if (is_valid == 0) {
// delete // delete
ex_data_runtime_del_ex_data(plugin_rt->ex_data_rt, key, key_len); ex_data_runtime_del_ex_container(plugin_rt->ex_data_rt, key, key_len, logger);
} else { } else {
// add // add
ex_data_runtime_add_ex_data(plugin_rt->ex_data_rt, key, key_len, (void *)ex_data); ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container);
} }
} }
@@ -401,57 +500,70 @@ void plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct table_sc
} }
void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct table_schema *table_schema, void ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, struct table_schema *table_schema,
const char *row, char *key, size_t key_len, const char *row, char *key, size_t key_len, struct ip_plugin_item *ip_plugin_item,
struct ip_rule *ip_rule, int is_valid) int is_valid, struct log_handle *logger)
{ {
struct plugin_ex_data *ex_data = ALLOC(struct plugin_ex_data, 1); struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1);
struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt; struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt;
ex_data->data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); ex_container->ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
ex_data->user_data = ip_rule; ex_container->custom_data = ip_plugin_item;
struct ex_container_ctx *ctx = ex_data_runtime_get_ex_container_ctx(ex_data_rt);
ctx->custom_data_free = free;
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema); int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema);
if (1 == set_flag) { if (1 == set_flag) {
if (0 == is_valid) { if (0 == is_valid) {
//delete //delete
ex_data_runtime_del_ex_data(ex_data_rt, key, key_len); ex_data_runtime_del_ex_container(ex_data_rt, key, key_len, logger);
} else { } else {
//add //add
ex_data_runtime_add_ex_data(ip_plugin_rt->ex_data_rt, key, key_len, ex_data); ex_data_runtime_add_ex_container(ip_plugin_rt->ex_data_rt, key, key_len, ex_container);
} }
} else { } else {
ex_data_runtime_cache_row_put(ip_plugin_rt->ex_data_rt, row); ex_data_runtime_cache_row_put(ip_plugin_rt->ex_data_rt, row);
} }
} }
void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema, const char *row, struct table_item *table_item) void table_runtime_update(struct table_runtime *table_rt, struct table_schema *table_schema,
const char *row, struct table_item *table_item, struct log_handle *logger)
{ {
int is_valid = -1; int is_valid = -1;
char *key = NULL; char *key = NULL;
size_t key_len = 0; size_t key_len = 0;
and_expr_t *expr_rule = NULL; and_expr_t *expr_rule = NULL;
struct ip_rule *ip_rule = NULL; struct ip_plus_item *ip_plus_item = NULL;
struct ip_plugin_item *ip_plugin_item = NULL;
switch (table_rt->table_type) { switch (table_rt->table_type) {
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
is_valid = table_item->expr_item.is_valid; is_valid = table_item->expr_item.is_valid;
expr_rule = expr_item_to_expr_rule(&table_item->expr_item); expr_rule = expr_item_to_expr_rule(&table_item->expr_item, logger);
key = (char *)&(table_item->expr_item.item_id); key = (char *)&(table_item->expr_item.item_id);
expr_runtime_update_row(&(table_rt->expr_rt), key, sizeof(int), expr_rule, is_valid); expr_runtime_update_row(&(table_rt->expr_rt), key, sizeof(int), expr_rule, is_valid, logger);
expr_rule_free(expr_rule); expr_rule_free(expr_rule);
break; break;
case TABLE_TYPE_IP_PLUS:
is_valid = table_item->ip_plus_item.is_valid;
ip_plus_item = ALLOC(struct ip_plus_item, 1);
memcpy(ip_plus_item, &(table_item->ip_plus_item), sizeof(table_item->ip_plus_item));
key = (char *)&(table_item->ip_plus_item.item_id);
ip_plus_runtime_update_row(&(table_rt->ip_plus_rt), table_schema, row, key, sizeof(int),
ip_plus_item, is_valid, logger);
break;
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
is_valid = table_item->plugin_item.is_valid; is_valid = table_item->plugin_item.is_valid;
key = table_item->plugin_item.key; key = table_item->plugin_item.key;
key_len = table_item->plugin_item.key_len; key_len = table_item->plugin_item.key_len;
plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, key, key_len, is_valid); plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, key, key_len, is_valid, logger);
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
is_valid = table_item->ip_plugin_item.is_valid; is_valid = table_item->ip_plugin_item.is_valid;
ip_rule = ip_plugin_item_to_ip_rule(&table_item->ip_plugin_item); ip_plugin_item = ALLOC(struct ip_plugin_item, 1);
memcpy(ip_plugin_item, &(table_item->ip_plugin_item), sizeof(table_item->ip_plugin_item));
key = (char *)&(table_item->ip_plugin_item.item_id); key = (char *)&(table_item->ip_plugin_item.item_id);
ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int), ip_rule, is_valid); ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, row, key, sizeof(int),
FREE(ip_rule); ip_plugin_item, is_valid, logger);
break; break;
default: default:
break; break;
@@ -503,27 +615,67 @@ int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread)
return ret; return ret;
} }
int ip_plus_runtime_commit(struct table_runtime *table_rt)
{
struct ex_data_container **ex_container = NULL;
struct ip_rule *rules = NULL;
size_t rule_cnt = 0;
int ret = 0;
struct ip_plugin_runtime *ip_plus_rt = &(table_rt->ip_plus_rt);
rule_cnt = ex_data_runtime_list_updating_ex_container(ip_plus_rt->ex_data_rt, &ex_container);
rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
struct ip_plus_item *item = (struct ip_plus_item *)ex_container[i]->custom_data;
ip_plus_item_to_ip_rule(item, &rules[i]);
}
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
size_t mem_used = 0;
if (rule_cnt > 0) {
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
ret = -1;
}
}
old_ip_matcher = ip_plus_rt->ip_matcher;
ip_plus_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ip_plus_rt->ex_data_rt);
table_rt->rule_num = ex_data_runtime_ex_container_count(ip_plus_rt->ex_data_rt);
FREE(rules);
FREE(ex_container);
return ret;
}
int plugin_runtime_commit(struct table_runtime *table_rt) int plugin_runtime_commit(struct table_runtime *table_rt)
{ {
ex_data_runtime_commit(table_rt->plugin_rt.ex_data_rt); ex_data_runtime_commit(table_rt->plugin_rt.ex_data_rt);
table_rt->rule_num = ex_data_runtime_ex_data_count(table_rt->plugin_rt.ex_data_rt); table_rt->rule_num = ex_data_runtime_ex_container_count(table_rt->plugin_rt.ex_data_rt);
return 0; return 0;
} }
int ip_plugin_runtime_commit(struct table_runtime *table_rt) int ip_plugin_runtime_commit(struct table_runtime *table_rt)
{ {
struct plugin_ex_data **ex_data_array = NULL; struct ex_data_container **ex_container = NULL;
struct ip_rule *rules = NULL; struct ip_rule *rules = NULL;
size_t rule_cnt = 0; size_t rule_cnt = 0;
int ret = 0; int ret = 0;
struct ip_plugin_runtime *ip_plugin_rt = &(table_rt->ip_plugin_rt); struct ip_plugin_runtime *ip_plugin_rt = &(table_rt->ip_plugin_rt);
rule_cnt = ex_data_runtime_list_updating_ex_data(ip_plugin_rt->ex_data_rt, (void ***)&ex_data_array); rule_cnt = ex_data_runtime_list_updating_ex_container(ip_plugin_rt->ex_data_rt, &ex_container);
rules = ALLOC(struct ip_rule, rule_cnt); rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) { for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct ip_rule *)ex_data_array[i]->user_data; struct ip_plugin_item *item = (struct ip_plugin_item *)ex_container[i]->custom_data;
ip_plugin_item_to_ip_rule(item, &rules[i]);
} }
struct ip_matcher *new_ip_matcher = NULL; struct ip_matcher *new_ip_matcher = NULL;
@@ -541,10 +693,10 @@ int ip_plugin_runtime_commit(struct table_runtime *table_rt)
ip_plugin_rt->ip_matcher = new_ip_matcher; ip_plugin_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ip_plugin_rt->ex_data_rt); ex_data_runtime_commit(ip_plugin_rt->ex_data_rt);
table_rt->rule_num = ex_data_runtime_ex_data_count(ip_plugin_rt->ex_data_rt); table_rt->rule_num = ex_data_runtime_ex_container_count(ip_plugin_rt->ex_data_rt);
FREE(rules); FREE(rules);
FREE(ex_data_array); FREE(ex_container);
return ret; return ret;
} }
@@ -591,8 +743,12 @@ void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_threa
case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR:
expr_runtime_commit(table_rt, nr_worker_thread); expr_runtime_commit(table_rt, nr_worker_thread);
break; break;
case TABLE_TYPE_IP_PLUS:
ip_plus_runtime_commit(table_rt);
break;
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
plugin_runtime_commit(table_rt); plugin_runtime_commit(table_rt);
break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_commit(table_rt); ip_plugin_runtime_commit(table_rt);
break; break;
@@ -635,35 +791,14 @@ const char* table_runtime_get_cached_row(struct table_runtime* table_rt, size_t
return line; return line;
} }
void *table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema,
const char *key, size_t key_len)
{
void *ex_data = NULL;
int set_flag = plugin_table_schema_ex_data_schema_flag(table_schema);
if (0 == set_flag) {
assert(0);
return NULL;
}
enum table_type table_type = table_schema_get_table_type(table_schema);
switch (table_type) {
case TABLE_TYPE_PLUGIN:
ex_data = ex_data_runtime_get_ex_data(table_rt->plugin_rt.ex_data_rt, key, key_len);
break;
case TABLE_TYPE_IP_PLUGIN:
break;
default:
break;
}
return ex_data;
}
struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt) struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table_rt)
{ {
struct ex_data_runtime *ex_data_rt = NULL; struct ex_data_runtime *ex_data_rt = NULL;
switch (table_rt->table_type) { switch (table_rt->table_type) {
case TABLE_TYPE_IP_PLUS:
ex_data_rt = table_rt->ip_plus_rt.ex_data_rt;
break;
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
ex_data_rt = table_rt->plugin_rt.ex_data_rt; ex_data_rt = table_rt->plugin_rt.ex_data_rt;
break; break;
@@ -677,26 +812,27 @@ struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table
return ex_data_rt; return ex_data_rt;
} }
void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema) void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema,
struct log_handle *logger)
{ {
struct ex_data_schema *ex_data_schema = plugin_table_schema_get_ex_data_schema(table_schema); struct ex_data_schema *ex_data_schema = plugin_table_schema_get_ex_data_schema(table_schema);
struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt); struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt);
ex_data_runtime_set_schema(ex_data_rt, ex_data_schema); ex_data_runtime_set_schema(ex_data_rt, ex_data_schema);
struct plugin_user_ctx *user_ctx = ALLOC(struct plugin_user_ctx, 1); struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1);
user_ctx->table_id = table_schema_get_table_id(table_schema); ctx->table_id = table_schema_get_table_id(table_schema);
user_ctx->ex_schema = ex_data_schema; ctx->ex_schema = ex_data_schema;
ex_data_runtime_set_user_ctx(ex_data_rt, user_ctx); ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx);
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt); size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
for (size_t i = 0; i < n_cached_row; i++) { for (size_t i = 0; i < n_cached_row; i++) {
const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i); const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i);
switch (table_rt->table_type) { switch (table_rt->table_type) {
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, NULL, 0, 1); plugin_runtime_update_row(&(table_rt->plugin_rt), table_schema, row, NULL, 0, 1, logger);
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, NULL, NULL, 0, NULL, 1); ip_plugin_runtime_update_row(&(table_rt->ip_plugin_rt), table_schema, NULL, NULL, 0, NULL, 1, logger);
break; break;
default: default:
break; break;

814
src/maat_table_schema.cpp
View File

File diff suppressed because it is too large Load Diff

141
src/maat_utils.cpp
View File

@@ -10,7 +10,9 @@
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <assert.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <sys/syscall.h>
#include <zlib.h> #include <zlib.h>
#include <openssl/md5.h> #include <openssl/md5.h>
#include <openssl/evp.h> #include <openssl/evp.h>
@@ -18,6 +20,19 @@
#include "utils.h" #include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
pid_t gettid()
{
return syscall(SYS_gettid);
}
const char *module_name_str(const char *name)
{
static __thread char module[64];
snprintf(module,sizeof(module),"%s(%d)", name, gettid());
return module;
}
char *maat_strdup(const char *s) char *maat_strdup(const char *s)
{ {
if (NULL == s) { if (NULL == s) {
@@ -378,3 +393,129 @@ size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_
return n_src; return n_src;
} }
enum maat_ip_format ip_format_str2int(const char *format)
{
if (0 == strcasecmp(format, "range")) {
return IP_FORMAT_RANGE;
} else if(0 == strcasecmp(format, "mask")) {
return IP_FORMAT_MASK;
} else if(0 == strcasecmp(format, "CIDR")) {
return IP_FORMAT_CIDR;
} else {
assert(0);
}
return IP_FORMAT_UNKNOWN;
}
int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, const char *ip2,
uint32_t range_begin[], uint32_t range_end[])
{
int cidr = 0;
int ret = 0;
if (ip_type != 4 && ip_type != 6) {
assert(0);
return -1;
}
if (ip_type == 4) {
uint32_t ipv4_addr = 0;
ret = inet_pton(AF_INET, ip1, &ipv4_addr);
if (ret <= 0) {
return -1;
}
ipv4_addr = ntohl(ipv4_addr);
uint32_t ipv4_range_end = 0;
uint32_t ipv4_mask = 0;
switch (format) {
case IP_FORMAT_RANGE:
range_begin[0] = ipv4_addr;
ret = inet_pton(AF_INET, ip2, &ipv4_range_end);
if (ret <= 0) {
return -1;
}
ipv4_range_end = ntohl(ipv4_range_end);
range_end[0] = ipv4_range_end;
break;
case IP_FORMAT_MASK:
ret = inet_pton(AF_INET, ip2, &ipv4_mask);
if (ret <= 0) {
return -1;
}
ipv4_mask = ntohl(ipv4_mask);
range_begin[0] = ipv4_addr & ipv4_mask;
range_end[0] = ipv4_addr | ~ipv4_mask;
break;
case IP_FORMAT_CIDR:
cidr = atoi(ip2);
if (cidr > 32 || cidr < 0) {
return -1;
}
ipv4_mask = (0xFFFFFFFFUL << (32 - cidr)) & 0xFFFFFFFFUL;
range_begin[0] = ipv4_addr & ipv4_mask;
range_end[0] = ipv4_addr | ~ipv4_mask;
break;
default:
assert(0);
}
} else {
//ipv6
int i = 0;
uint32_t ipv6_addr[4] = {0};
uint32_t ipv6_mask[4] = {0};
uint32_t ipv6_range_end[4] = {0};
ret = inet_pton(AF_INET6, ip1, ipv6_addr);
if (ret <= 0) {
return -1;
}
ipv6_ntoh(ipv6_addr);
switch (format) {
case IP_FORMAT_RANGE:
ret = inet_pton(AF_INET6, ip2, ipv6_range_end);
if (ret <= 0) {
return -1;
}
ipv6_ntoh(ipv6_range_end);
memcpy(range_begin, ipv6_addr, sizeof(ipv6_addr));
memcpy(range_end, ipv6_range_end, sizeof(ipv6_range_end));
break;
case IP_FORMAT_MASK:
ret = inet_pton(AF_INET6, ip2, ipv6_mask);
if (ret <= 0) {
return -1;
}
ipv6_ntoh(ipv6_mask);
for (i = 0; i < 4; i++) {
range_begin[i]=ipv6_addr[i] & ipv6_mask[i];
range_end[i] = ipv6_addr[i] | ~ipv6_mask[i];
}
break;
case IP_FORMAT_CIDR:
cidr = atoi(ip2);
if (cidr > 128 || cidr < 0) {
return -1;
}
for (i = 0; i < 4; i++) {
int bit32 = 128 - cidr - 32 * (3 - i);
if (bit32 < 0) {
bit32 = 0;
}
ipv6_mask[i] = (0xFFFFFFFFUL << bit32) & 0xFFFFFFFFUL;
range_begin[i] = ipv6_addr[i] & ipv6_mask[i];
range_end[i] = ipv6_addr[i] | ~ipv6_mask[i];
}
break;
default:
assert(0);
}
}
return 0;
}

5
src/rcu_hash.cpp
View File

@@ -142,6 +142,11 @@ void rcu_hash_set_user_ctx(struct rcu_hash_table *htable, void *user_ctx)
htable->user_ctx = user_ctx; htable->user_ctx = user_ctx;
} }
void *rcu_hash_get_user_ctx(struct rcu_hash_table *htable)
{
return htable->user_ctx;
}
void rcu_hash_commit_prepare(struct rcu_hash_table *htable) void rcu_hash_commit_prepare(struct rcu_hash_table *htable)
{ {
struct rcu_hash_node *node = NULL; struct rcu_hash_node *node = NULL;

19
test/maat_framework_gtest.cpp
View File

@@ -6,6 +6,7 @@
#include "maat_table_schema.h" #include "maat_table_schema.h"
#include "maat_table_runtime.h" #include "maat_table_runtime.h"
#include "maat_command.h" #include "maat_command.h"
#include "IPMatcher.h"
struct maat *g_maat_instance = NULL; struct maat *g_maat_instance = NULL;
const char *table_info_path = "./table_info.conf"; const char *table_info_path = "./table_info.conf";
@@ -206,6 +207,24 @@ TEST(maat_scan_string, hit_three_expr) {
EXPECT_EQ(result_array[2], 18); EXPECT_EQ(result_array[2], 18);
} }
TEST(maat_scan_ip, hit_ip) {
struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
int table_id = table_schema_manager_get_table_id(table_schema_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "192.168.50.24";
struct addr_4tuple addr;
addr.type = IP_TYPE_V4;
int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip);
EXPECT_EQ(ret, 1);
addr.ipv4.sport = htons(1);
int results[3] = {-1};
size_t n_result = 0;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, &n_result, nullptr);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(results[0], 4);
}
int main(int argc, char ** argv) int main(int argc, char ** argv)
{ {
int ret=0; int ret=0;

14
test/maat_input_mode_gtest.cpp
View File

@@ -57,7 +57,8 @@ TEST(iris_mode, maat_scan_string) {
int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, NULL, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL); ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, NULL,
tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
} }
@@ -141,7 +142,7 @@ TEST(redis_mode, maat_scan_string) {
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db); redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, NULL);
EXPECT_NE(c, nullptr); EXPECT_NE(c, nullptr);
redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb"); redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb");
@@ -155,25 +156,26 @@ TEST(redis_mode, maat_scan_string) {
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL); ret = json2iris(json_buff, json_filename, NULL, NULL, NULL, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, NULL);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
} }
size_t total_line_cnt = 0; size_t total_line_cnt = 0;
char tmp_iris_full_idx_path[128] = {0}; char tmp_iris_full_idx_path[128] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path); snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt); config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_cmd_redis_server_time_s(c); long long server_time = maat_cmd_redis_server_time_s(c);
EXPECT_NE(server_time, -1); EXPECT_NE(server_time, -1);
absolute_expire_time = server_time + 300; absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule); config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, NULL);
int success_cnt = 0; int success_cnt = 0;
do { do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time); success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, NULL);
} while (success_cnt < 0); } while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt); EXPECT_EQ(success_cnt, (int)total_line_cnt);

61
test/table_info.conf
View File

@@ -6,20 +6,20 @@
"scan_mode":"block", "scan_mode":"block",
"item_id":1, "item_id":1,
"group_id":2, "group_id":2,
"rule": { "custom": {
"keywords":3, "keywords":3,
"expr_type":4, "expr_type":4,
"match_method":5, "match_method":5,
"is_hexbin":6, "is_hexbin":6,
"is_valid":7 "is_valid":7
} }
}, },
{ {
"table_id":2, "table_id":2,
"table_name":"IP_PLUGIN_TABLE", "table_name":"IP_PLUGIN_TABLE",
"table_type":"ip_plugin", "table_type":"ip_plugin",
"item_id":1, "item_id":1,
"rule": { "custom": {
"ip_type":2, "ip_type":2,
"start_ip":3, "start_ip":3,
"end_ip":4, "end_ip":4,
@@ -31,11 +31,64 @@
"table_name":"PLUGIN_TABLE", "table_name":"PLUGIN_TABLE",
"table_type":"plugin", "table_type":"plugin",
"item_id":1, "item_id":1,
"rule": { "custom": {
"key":2, "key":2,
"tag":3, "tag":3,
"is_valid":4, "is_valid":4,
"foreign":[6,8,10] "foreign":[6,8,10]
} }
},
{
"table_id":4,
"table_name":"IP_PLUS_CONFIG",
"table_type":"ip_plus",
"item_id":1,
"group_id":2,
"custom": {
"addr_type":3,
"saddr_format":4,
"sip1":5,
"sip2":6,
"sport_format":7,
"sport1":8,
"sport2":9,
"daddr_format":10,
"dip1":11,
"dip2":12,
"dport_format":13,
"dport1":14,
"dport2":15,
"proto":16,
"direction":17,
"is_valid":18
}
},
{
"table_id":5,
"table_name":"COMPOSITION_IP_SOURCE",
"table_type":"virtual",
"physical_table":["IP_PLUS_CONFIG"]
},
{
"table_id":6,
"table_name":"COMPOSITION_IP_DEST",
"table_type":"virtual",
"physical_table":["IP_PLUS_CONFIG"]
},
{
"table_id":7,
"table_name":"COMPOSITION_IP_SESSION",
"table_type":"virtual",
"physical_table":["IP_PLUS_CONFIG"]
},
{
"table_id":8,
"table_name":"COMPOSITION_IP",
"table_type":"composition",
"composition_table": {
"source": "COMPOSITION_IP_SOURCE",
"dest": "COMPOSITION_IP_DEST",
"session": "COMPOSITION_IP_SESSION"
}
} }
] ]

27
tools/maat_redis_tool.cpp
View File

@@ -65,7 +65,7 @@ int set_file_rulenum(const char *path, int rule_num)
return 0; return 0;
} }
void read_rule_from_redis(redisContext *c, long long desire_version, const char* output_path) void read_rule_from_redis(redisContext *c, long long desire_version, const char *output_path, struct log_handle *logger)
{ {
int i = 0; int i = 0;
int ret = 0; int ret = 0;
@@ -80,7 +80,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char*
FILE *index_fp = NULL; FILE *index_fp = NULL;
struct serial_rule *rule_list = NULL; struct serial_rule *rule_list = NULL;
int rule_num = maat_cmd_get_rm_key_list(c, 0, desire_version, &version, NULL, &rule_list, &update_type, 0); int rule_num = maat_cmd_get_rm_key_list(c, 0, desire_version, &version, NULL, &rule_list, &update_type, 0, logger);
if (0 == rule_num) { if (0 == rule_num) {
if (desire_version != 0) { if (desire_version != 0) {
printf("Read desired version %lld failed.\n", desire_version); printf("Read desired version %lld failed.\n", desire_version);
@@ -102,7 +102,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char*
} }
printf("Reading value: \n"); printf("Reading value: \n");
ret = maat_cmd_get_redis_value(c, rule_list, rule_num, 1); ret = maat_cmd_get_redis_value(c, rule_list, rule_num, 1, logger);
if (ret < 0) { if (ret < 0) {
goto clean_up; goto clean_up;
} }
@@ -123,10 +123,10 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char*
} }
} }
ret = maat_cmd_get_foreign_keys_by_prefix(c, rule_list, rule_num, foreign_files_dir); ret = maat_cmd_get_foreign_keys_by_prefix(c, rule_list, rule_num, foreign_files_dir, logger);
if (ret > 0) { if (ret > 0) {
printf("%d lines has foreign content.\n", ret); printf("%d lines has foreign content.\n", ret);
maat_cmd_get_foreign_conts(c, rule_list, rule_num, 1); maat_cmd_get_foreign_conts(c, rule_list, rule_num, 1, NULL);
} }
snprintf(index_path,sizeof(index_path), "%s/full_config_index.%020lld", output_path, version); snprintf(index_path,sizeof(index_path), "%s/full_config_index.%020lld", output_path, version);
@@ -248,11 +248,13 @@ int main(int argc, char * argv[])
int redis_db = 0; int redis_db = 0;
char dump_dir[128] = {0}; char dump_dir[128] = {0};
char json_file[128] = {0}; char json_file[128] = {0};
char log_path[128] = "./maat_redis_tool.log";
int timeout = 0; int timeout = 0;
long long desired_version = 0; long long desired_version = 0;
strncpy(redis_ip, "127.0.0.1", sizeof(redis_ip)); strncpy(redis_ip, "127.0.0.1", sizeof(redis_ip));
strncpy(dump_dir, redis_dump_dir, sizeof(dump_dir)); strncpy(dump_dir, redis_dump_dir, sizeof(dump_dir));
struct log_handle *logger = log_handle_create(log_path, 0);
while ((oc = getopt(argc,argv,"h:p:n:d:v:f:j:t:")) != -1) { while ((oc = getopt(argc,argv,"h:p:n:d:v:f:j:t:")) != -1) {
switch (oc) { switch (oc) {
case 'h': case 'h':
@@ -289,7 +291,7 @@ int main(int argc, char * argv[])
} }
} }
redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db); redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger);
if (NULL == c) { if (NULL == c) {
return -1; return -1;
} }
@@ -297,7 +299,7 @@ int main(int argc, char * argv[])
char tmp_iris_path[128] = {0}; char tmp_iris_path[128] = {0};
if (model == WORK_MODE_DUMP) { if (model == WORK_MODE_DUMP) {
printf("Reading key list from %s:%d db%d.\n", redis_ip, redis_port, redis_db); printf("Reading key list from %s:%d db%d.\n", redis_ip, redis_port, redis_db);
read_rule_from_redis(c, desired_version, dump_dir); read_rule_from_redis(c, desired_version, dump_dir, logger);
} else if(model == WORK_MODE_JSON) { } else if(model == WORK_MODE_JSON) {
char *json_buff = NULL; char *json_buff = NULL;
size_t json_buff_sz = 0; size_t json_buff_sz = 0;
@@ -306,13 +308,14 @@ int main(int argc, char * argv[])
printf("open %s failed.\n", json_file); printf("open %s failed.\n", json_file);
} }
ret = json2iris(json_buff, json_file, NULL, NULL, NULL, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL); ret = json2iris(json_buff, json_file, NULL, NULL, NULL, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, NULL);
if (ret < 0) { if (ret < 0) {
printf("Invalid json format.\n"); printf("Invalid json format.\n");
} }
size_t total_line_cnt = 0; size_t total_line_cnt = 0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt); config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL);
printf("Serialize %s to %zu lines, write temp file to %s .\n", json_file, total_line_cnt, tmp_iris_path); printf("Serialize %s to %zu lines, write temp file to %s .\n", json_file, total_line_cnt, tmp_iris_path);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
@@ -325,13 +328,13 @@ int main(int argc, char * argv[])
absolute_expire_time = server_time + timeout; absolute_expire_time = server_time + timeout;
} }
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule); config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, NULL);
printf("Timeout = %lld\n", absolute_expire_time); printf("Timeout = %lld\n", absolute_expire_time);
ret = 0; ret = 0;
int success_cnt = 0; int success_cnt = 0;
do { do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time); success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, NULL);
} while(success_cnt < 0); } while(success_cnt < 0);
if (success_cnt != (int)total_line_cnt) { if (success_cnt != (int)total_line_cnt) {