diff --git a/deps/log/log.h b/deps/log/log.h index c1cd5d3..7b71a24 100644 --- a/deps/log/log.h +++ b/deps/log/log.h @@ -17,6 +17,7 @@ extern "C" #include #include #include +#include #define LOG_VERSION "0.1.0" diff --git a/src/inc_internal/maat_common.h b/src/inc_internal/maat_common.h index f05ae81..572bc95 100644 --- a/src/inc_internal/maat_common.h +++ b/src/inc_internal/maat_common.h @@ -25,7 +25,7 @@ struct maat_options { char instance_name[NAME_MAX]; size_t nr_worker_threads; - const char *accept_tags; + char *accept_tags; int rule_effect_interval_ms; int rule_update_checking_interval_ms; diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index ced29c2..2d19d2d 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -18,7 +18,7 @@ extern "C" #include "log/log.h" #include "cJSON/cJSON.h" -#include "maat.h" +#include "maat/maat.h" #include "maat_kv.h" #include "maat_rule.h" @@ -38,6 +38,7 @@ void compile_schema_free(void *compile_schema); void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void group2compile_schema_free(void *g2c_schema); +int group2compile_associated_compile_table_id(void *g2c_schema); int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id, maat_rule_ex_new_func_t *new_func, @@ -51,7 +52,7 @@ compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, siz size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema); /* compile runtime API */ -void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage_bin, +void *compile_runtime_new(void *compile_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void compile_runtime_free(void *compile_runtime); @@ -60,14 +61,15 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, const ch int compile_runtime_commit(void *compile_runtime); int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids, - int *compile_ids, size_t compile_ids_size, struct maat_state *state); + int vt_id, int *compile_ids, size_t compile_ids_size, struct maat_state *state); -size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct maat_group_topology *group_topo, - struct maat_compile_state *compile_state, +size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct group2group_runtime *g2g_rt, + struct maat_compile_state *compile_state, struct maat_hit_path *hit_paths, size_t hit_path_siz); /* group2compile runtime API */ -void *group2compile_runtime_new(void *g2c_schema, struct maat_garbage_bin *garbage_bin, +void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); +void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime); void group2compile_runtime_free(void *g2c_runtime); int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line, @@ -79,7 +81,7 @@ struct maat_compile_state *maat_compile_state_new(int thread_id); void maat_compile_state_free(struct maat_compile_state *compile_state); void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id, int Nth_scan, int Nth_item_result); -void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, struct maat_compile **compile_hash, +void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, void *compile_runtime, int group_id, int virtual_table_id); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); diff --git a/src/inc_internal/maat_ex_data.h b/src/inc_internal/maat_ex_data.h index 0a641dc..f751490 100644 --- a/src/inc_internal/maat_ex_data.h +++ b/src/inc_internal/maat_ex_data.h @@ -33,7 +33,8 @@ struct ex_container_ctx { struct ex_data_runtime; /* ex_data_runtime API */ -struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn); +struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, + struct log_handle *logger); void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt); diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index e6b0ba1..4670518 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -25,10 +25,11 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo void expr_schema_free(void *expr_schema); /* expr runtime API */ -void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); +void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); void expr_runtime_free(void *expr_runtime); -int expr_runtime_updating_flag(struct expr_runtime *expr_rt); +int expr_runtime_updating_flag(void *expr_runtime); int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column); int expr_runtime_commit(void *expr_runtime); @@ -38,7 +39,7 @@ int expr_runtime_commit(void *expr_runtime); * * @retval the num of hit group_id */ -int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, +int expr_runtime_scan_string(struct expr_runtime *expr_runtime, int thread_id, const char *data, size_t data_len, int group_ids[], size_t max_hit_num, int virtual_table_id, struct maat_state *state); diff --git a/src/inc_internal/maat_garbage_collection.h b/src/inc_internal/maat_garbage_collection.h index 1a22dd8..21aed45 100644 --- a/src/inc_internal/maat_garbage_collection.h +++ b/src/inc_internal/maat_garbage_collection.h @@ -20,17 +20,17 @@ extern "C" struct maat_garbage_bin; -struct maat_garbage_bin* maat_garbage_bin_new(int default_timeout); +struct maat_garbage_bin *maat_garbage_bin_new(int default_timeout); -void maat_garbage_bin_free(struct maat_garbage_bin* bin); +void maat_garbage_bin_free(struct maat_garbage_bin *bin); -void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void (* func)(void *)); +void maat_garbage_bagging(struct maat_garbage_bin *bin, void *garbage, void (* func)(void *)); -void maat_garbage_collect_routine(struct maat_garbage_bin* bin); +void maat_garbage_collect_routine(struct maat_garbage_bin *bin); -size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin); +size_t maat_garbage_bin_get_size(struct maat_garbage_bin *bin); -void maat_garbage_collect_by_force(struct maat_garbage_bin* bin); +void maat_garbage_collect_by_force(struct maat_garbage_bin *bin); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_group.h b/src/inc_internal/maat_group.h index cb7479c..62adc1b 100644 --- a/src/inc_internal/maat_group.h +++ b/src/inc_internal/maat_group.h @@ -37,22 +37,20 @@ struct maat_group { /* maat group topology API */ struct maat_group_topology; -struct maat_group_topology *maat_group_topology_new(struct log_handle *logger); -void maat_group_topology_free(struct maat_group_topology *group_topo); - -struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id); -void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group); /** * @retval if not found, return NULL */ -struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id); -int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id); -int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id); + +//struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id); +//struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id); +//void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group); +//int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id); +//int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id); /* build top groups */ -int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo); +//int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo); /* group2group schema API */ @@ -60,15 +58,24 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han void group2group_schema_free(void *g2g_schema); /* group2group runtime API */ -struct group2group_runtime * -group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, - struct log_handle *logger); +void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); void group2group_runtime_free(void *g2g_runtime); +struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id); +void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group); +struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id); + +int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id); +int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id); + +int group2group_runtime_build_top_groups(void *g2g_runtime); + + int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column); int group2group_runtime_commit(void *g2g_runtime); -int group2group_runtime_get_top_groups(struct group2group_runtime *g2g_rt, int *group_ids, size_t n_group_ids, +int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids, int *top_group_ids); #ifdef __cpluscplus diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h index 91651f8..ecaa8c0 100644 --- a/src/inc_internal/maat_ip.h +++ b/src/inc_internal/maat_ip.h @@ -22,18 +22,21 @@ void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle void ip_plus_schema_free(void *ip_plus_schema); /* ip plus runtime API */ -void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); +void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); void ip_plus_runtime_free(void *ip_plus_runtime); -int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line); +int ip_plus_runtime_updating_flag(void *ip_plus_runtime); +int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line, + int valid_column); int ip_plus_runtime_commit(void *ip_plus_runtime); struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt); /* ip runtime scan API */ -int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *data, - int *group_id_array, size_t n_group_id_array, int virtual_table_id, - struct maat_state *state); +int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr, + int *group_id_array, size_t n_group_id_array, int virtual_table_id, + struct maat_state *state); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_ip_plugin.h b/src/inc_internal/maat_ip_plugin.h index 39f9fc5..aa4e550 100644 --- a/src/inc_internal/maat_ip_plugin.h +++ b/src/inc_internal/maat_ip_plugin.h @@ -16,6 +16,10 @@ extern "C" { #endif +#include "log/log.h" +#include "maat/maat.h" +#include "cJSON/cJSON.h" + struct ip_plugin_runtime; /* ip plugin schema API */ @@ -29,15 +33,17 @@ int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp); + long argl, void *argp, + struct log_handle *logger); /* ip plugin runtime API */ -void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *garbage_bin, +void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void ip_plugin_runtime_free(void *ip_plugin_runtime); -int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt); -int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line); +int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime); +int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line, + int valid_column); int ip_plugin_runtime_commit(void *ip_plugin_runtime); struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime); diff --git a/src/inc_internal/maat_plugin.h b/src/inc_internal/maat_plugin.h index 3b00153..c73250c 100644 --- a/src/inc_internal/maat_plugin.h +++ b/src/inc_internal/maat_plugin.h @@ -16,6 +16,8 @@ extern "C" { #endif +#include "maat/maat.h" + #define MAX_FOREIGN_CLMN_NUM 8 /* plugin schema API */ @@ -40,19 +42,23 @@ int plugin_table_set_ex_data_schema(void *custom_schema, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp); + long argl, void *argp, + struct log_handle *logger); struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema); /* plugin runtime API */ -struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime); +void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void plugin_runtime_free(void *plugin_runtime); -int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt); +int plugin_runtime_updating_flag(void *plugin_runtime); -int plugin_runtime_update(struct plugin_runtime *plugin_rt, struct plugin_item *plugin_item, - struct plugin_schema *plugin_schema, const char *row, - struct log_handle *logger); +int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line, + int valid_column); int plugin_runtime_commit(void *plugin_runtime); +struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime); + #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index d866eed..b42bfd7 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -80,7 +80,6 @@ struct compile_rule { int compile_id; struct maat_rule_head head;// fix len of Maat_rule_t char *service_defined; - int is_valid; int declared_clause_num; double evaluation_order; struct compile_schema *ref_table; @@ -88,20 +87,9 @@ struct compile_rule { pthread_rwlock_t rwlock; }; -struct group2compile_rule { - int group_id; - int compile_id; - int is_valid; - int not_flag; - int vt_id; //virtual_table_id - int clause_index; - int associated_compile_table_id; -}; - struct group2group_rule { int group_id; int superior_group_id; - int is_valid; }; struct maat_runtime { @@ -114,7 +102,6 @@ struct maat_runtime { struct table_manager *ref_tbl_mgr; //share with maat instance size_t max_table_num; - size_t max_thread_num; uint32_t rule_num; struct maat_garbage_bin *ref_garbage_bin; @@ -254,9 +241,9 @@ struct maat_state { struct maat_compile_state *compile_mid; }; -int parse_accept_tag(const char *value, struct rule_tag **result, void *logger); +size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger); -int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag); +int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag); struct maat_item *maat_item_new(int item_id, int group_id, void *user_data); @@ -264,7 +251,7 @@ void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *) struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id); -void maat_item_inner_free(struct maat_item_inner *item); +void maat_item_inner_free(void *item_inner); void maat_start_cb(long long new_version, int update_type, void *u_para); @@ -297,7 +284,7 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri void maat_cmd_clear_rule_cache(struct serial_rule *s_rule); int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, - long long *new_version, struct table_schema_manager* table_schema_mgr, + long long *new_version, struct table_manager* tbl_mgr, struct serial_rule **list, int *update_type, int cumulative_off, struct log_handle *logger); diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index ba4938a..e57e61d 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -45,20 +45,23 @@ struct table_manager; struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags, struct log_handle *logger); -int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *garbage_bin); -int table_manager_deinit(struct table_manager *tbl_mgr); +int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num, + struct maat_garbage_bin *garbage_bin); +void table_manager_runtime_destroy(struct table_manager *tbl_mgr); void table_manager_destroy(struct table_manager *tbl_mgr); size_t table_manager_table_count(struct table_manager *tbl_mgr); int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name); enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id); +int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr); int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id); -int table_manager_accept_tags_match(const char *tags); +int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags); void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); -void table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line); +int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id); +int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line); void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id); #ifdef __cpluscplus diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h index b687df5..d731e78 100644 --- a/src/inc_internal/maat_virtual.h +++ b/src/inc_internal/maat_virtual.h @@ -31,6 +31,8 @@ enum scan_type { void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void virtual_schema_free(void *virtual_schema); + #ifdef __cpluscplus } #endif diff --git a/src/maat_api.cpp b/src/maat_api.cpp index 96ef3fd..5371bd5 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -26,6 +26,10 @@ #include "maat_redis_monitor.h" #include "maat_compile.h" #include "alignment.h" +#include "maat_garbage_collection.h" +#include "maat_group.h" +#include "maat_expr.h" +#include "maat_ip.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" @@ -34,16 +38,6 @@ #define DISTRICT_ANY -1 #define DISTRICT_UNKNOWN -2 -struct scan_item_hit_wrapper { - int Nth_scan; - struct maat_item_inner* wrapped_items[MAX_SCANNER_HIT_COMPILE_NUM]; - size_t n_wrapped_item; - - int *virtual_table_ids; - int virtual_table_id; - int is_last_item; -}; - inline int scan_state_should_compile_NOT(struct maat_state *mid) { if (mid && mid->is_last_scan==1 && mid->compile_mid && @@ -260,9 +254,8 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) if (NULL == maat_instance->tbl_mgr) { goto failed; } - //TODO: by luis - //maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr); - maat_instance->default_compile_table_id = -1; + + maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr); maat_instance->input_mode = opts->input_mode; switch (maat_instance->input_mode) { @@ -353,12 +346,12 @@ void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_he size_t generic_plugin_runtime_cached_row_count(void *custom_rt, enum table_type table_type) { - + return 0; } const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type, size_t row_id) { - + return NULL; } /* must be plugin table */ @@ -393,7 +386,7 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id, start(MAAT_RULE_UPDATE_TYPE_FULL, u_para); } - for (size_t i = 0; i < row_count; i++) { + for (size_t i = 0; i < row_cnt; i++) { const char *line = generic_plugin_runtime_get_cached_row(runtime, table_type, i); if (NULL == line) { break; @@ -416,18 +409,18 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp) + long argl, void *argp, struct log_handle *logger) { if (NULL == tbl_mgr || NULL == new_func || NULL == free_func || NULL == dup_func) { assert(0); - log_error(tbl_mgr->logger, MODULE_MAAT_API, + log_error(logger, MODULE_MAAT_API, "table(table_id:%d) %s failed: invalid parameter", __FUNCTION__); return -1; } void *schema = table_manager_get_schema(tbl_mgr, table_id); if (NULL == schema) { - log_error(tbl_mgr->logger, MODULE_MAAT_API, + log_error(logger, MODULE_MAAT_API, "Error: %s, table(table_id:%d) is not registered", __FUNCTION__, table_id); return -1; } @@ -438,24 +431,24 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t case TABLE_TYPE_PLUGIN: ex_schema = plugin_table_get_ex_data_schema(schema); if (NULL == ex_schema) { - log_error(tbl_mgr->logger, MODULE_MAAT_API, + log_error(logger, MODULE_MAAT_API, "Error: %s, table(table_id:%d) is not a valid plugin table", __FUNCTION__, table_id); return -1; } plugin_table_set_ex_data_schema(schema, new_func, free_func, - dup_func, argl, argp, tbl_mgr->logger); + dup_func, argl, argp, logger); break; case TABLE_TYPE_IP_PLUGIN: ex_schema = ip_plugin_table_get_ex_data_schema(schema); if (NULL == ex_schema) { - log_error(tbl_mgr->logger, MODULE_MAAT_API, + log_error(logger, MODULE_MAAT_API, "Error: %s, table(table_id:%d) is not a valid ip_plugin table", __FUNCTION__, table_id); return -1; } ip_plugin_table_set_ex_data_schema(schema, new_func, free_func, - dup_func, argl, argp); + dup_func, argl, argp, logger); break; default: break; @@ -464,8 +457,8 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t return 0; } -void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, enum table_type table_type, - int nr_worker_thread, long long version, struct log_handle *logger) +void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, int table_id, + enum table_type table_type, int valid_column) { struct ex_data_schema *ex_data_schema = NULL; struct ex_data_runtime *ex_data_rt = NULL; @@ -484,21 +477,20 @@ void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, enum t } ex_data_runtime_set_schema(ex_data_rt, ex_data_schema); - struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1); - - ctx->table_id = table_schema_get_table_id(custom_schema); - ctx->ex_schema = ex_data_schema; - ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx); + struct ex_container_ctx *ex_ctx = ALLOC(struct ex_container_ctx, 1); + ex_ctx->table_id = table_id; + ex_ctx->ex_schema = ex_data_schema; + ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_ctx); size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt); for (size_t i = 0; i < n_cached_row; i++) { const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i); - switch (table_rt->table_type) { + switch (table_type) { case TABLE_TYPE_PLUGIN: - plugin_runtime_update_row(custom_rt, custom_schema, row, NULL, 0, 1); + plugin_runtime_update(runtime, schema, row, valid_column); break; case TABLE_TYPE_IP_PLUGIN: - ip_plugin_runtime_update_row(custom_rt, custom_schema, NULL, NULL, 0, NULL, 1); + ip_plugin_runtime_update(runtime, schema, row, valid_column); break; default: break; @@ -508,10 +500,10 @@ void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, enum t switch (table_type) { case TABLE_TYPE_PLUGIN: - plugin_runtime_commit(custom_rt); + plugin_runtime_commit(runtime); break; case TABLE_TYPE_IP_PLUGIN: - ip_plugin_runtime_commit(custom_rt); + ip_plugin_runtime_commit(runtime); break; default: break; @@ -526,7 +518,8 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i { pthread_mutex_lock(&(maat_instance->background_update_mutex)); int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id, - new_func, free_func, dup_func, argl, argp); + new_func, free_func, dup_func, argl, argp, + maat_instance->logger); if (ret < 0) { pthread_mutex_unlock(&(maat_instance->background_update_mutex)); return -1; @@ -536,8 +529,8 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - generic_plugin_runtime_commit_ex_schema(runtime, schema, table_type, maat_instance->nr_worker_thread, - maat_instance->maat_rt->version, maat_instance->logger); + int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); + generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type, valid_column); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); @@ -620,26 +613,29 @@ struct maat_state *grab_state(struct maat_state **state, struct maat *maat_insta return mid; } -void scan_item_hit_wrapper_build(struct scan_item_hit_wrapper* wrapper, struct scan_result* results, size_t n_result, - int district_id, int is_last_item, int virtual_table_id, int Nth_scan) +int hit_group_to_compile(void *g2g_runtime, int *group_ids, int group_hit_cnt, int vt_id, + void *compile_runtime, int *compile_ids, size_t n_compile_id, + size_t *n_hit_compile_id, struct maat_state *mid) { - size_t i=0; - struct maat_item_inner *item = NULL; - wrapper->n_wrapped_item = 0; - wrapper->virtual_table_id = 0; - wrapper->virtual_table_ids = NULL; + int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + int n_top_group_ids = group2group_runtime_get_top_groups(g2g_runtime, group_ids, group_hit_cnt, top_group_ids); + size_t n_all_group_ids = 0; + if (n_top_group_ids > 0) { + n_all_group_ids = group_hit_cnt + n_top_group_ids; + } else { + n_all_group_ids = group_hit_cnt; + } - for (i = 0; i < n_result; i++) { - item = (struct maat_item_inner *)(results[i].tag); - if (item->district_id == district_id || district_id == DISTRICT_ANY) { - wrapper->wrapped_items[wrapper->n_wrapped_item] = item; - wrapper->n_wrapped_item++; - } - } - wrapper->is_last_item = is_last_item; - wrapper->virtual_table_id = virtual_table_id; - wrapper->Nth_scan = Nth_scan; - wrapper->virtual_table_ids = NULL; + int all_group_ids[n_all_group_ids] = {-1}; + // maat state find compile_table_id, if not found, maat_instance->default_compile_table_id + int n_compile_ids = compile_runtime_match((struct compile_runtime *)compile_runtime, all_group_ids, n_all_group_ids, + vt_id, compile_ids, n_compile_id, mid); + *n_hit_compile_id = n_compile_ids; + if (n_compile_ids > 0) { + return MAAT_HIT; + } else { + return MAAT_HALF_HIT; + } } int maat_scan_integer(struct maat *instance, int table_id, int thread_id, @@ -649,57 +645,12 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id, return 0; } -static int ip_scan_data_set(struct ip_addr *scan_data, const char *ip_addr) -{ - return 0; -} - -static int ip_composition_scan(int thread_id, const char *ip_addr, - int parent_table_id, int *virtual_table_id, - struct table_schema_manager *table_schema_mgr, - struct table_runtime_manager *table_rt_mgr, - int *group_id_array, size_t n_group_id_array, - struct maat_state *state) -{ - int child_table_id = 0; - - struct table_schema *real_table = table_schema_get_by_scan_type(table_schema_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id); - if (NULL == real_table) { - return 0; - } - - enum table_type table_type = table_schema_get_table_type(real_table); - if (table_type != TABLE_TYPE_IP_PLUS) { - return -1; - } - - int table_id = table_schema_get_table_id(real_table); - struct table_runtime *table_rt = table_manager_get_runtime(table_rt_mgr, table_id); - size_t rule_num = table_runtime_rule_count(table_rt); - if (0 == rule_num) { - return 0; - } - - struct ip_addr scan_data; - memset(&scan_data, 0, sizeof(struct ip_addr)); - - ip_scan_data_set(&scan_data, ip_addr); - size_t hit_group_cnt = 0; - int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, group_id_array, n_group_id_array, *virtual_table_id, state); - if (ret < 0) { - return -1; - } - - return hit_group_cnt; -} - int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, uint32_t ip_addr, int *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || - (thread_id < 0) || (NULL == ip_addr) || (NULL == results) || (0 == n_result) || - (NULL == state)) { + (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) { return -1; } @@ -708,52 +659,36 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, scan_count_inc(mid); int virtual_table_id = 0; - struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, - SCAN_TYPE_IP, &virtual_table_id); - if (NULL == table_schema) { - return MAAT_ERR; - } + // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, + // SCAN_TYPE_IP, &virtual_table_id); + // if (NULL == table_schema) { + // return MAAT_ERR; + // } - struct maat_runtime *maat_rt = maat_instance->maat_rt; - int group_result[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - int group_hit_cnt = 0; - int group_result_virtual_table_ids[MAX_SCANNER_HIT_GROUP_NUM]; + int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - int group_ret = -1; + assert(table_type == TABLE_TYPE_IP_PLUS); - group_ret = ip_composition_scan(thread_id, ip_addr, table_id, &virtual_table_id, - maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, - group_result + group_hit_cnt, MAX_SCANNER_HIT_COMPILE_NUM - group_hit_cnt, mid); - if (group_ret < 0) { - maat_instance->scan_err_cnt++; - } else if (0 == group_ret) { + // int table_id = table_schema_get_table_id(real_table); + void *ip_plus_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + // size_t rule_num = table_runtime_rule_count(table_rt); + // if (0 == rule_num) { + // return 0; + // } + + int group_hit_cnt = ip_plus_runtime_scan_ipv4((struct ip_plus_runtime *)ip_plus_rt, thread_id, ip_addr, + group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid); + if (group_hit_cnt < 0) { + return MAAT_ERR; + } else if (0 == group_hit_cnt) { return MAAT_OK; } else { - group_hit_cnt += group_ret; - } - - // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT - - struct table_runtime *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); - int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - int n_top_group_ids = group2group_runtime_get_top_groups(g2g_rt, group_result, group_hit_cnt, top_group_ids); - size_t n_all_group_ids = 0; - if (n_top_group_ids > 0) { - n_all_group_ids = group_hit_cnt + n_top_group_ids; - } else { - n_all_group_ids = group_hit_cnt; - } - - int all_group_ids[n_all_group_ids] = {-1}; - // maat state find compile_table_id, if not found, maat_instance->default_compile_table_id - struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id); - int n_compile_ids = compile_runtime_match(compile_table_rt, all_group_ids, n_all_group_ids, results, n_result, mid); - *n_hit_result = n_compile_ids; - - if (n_compile_ids > 0) { - return MAAT_HIT; - } else { - return MAAT_HALF_HIT; + // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT + void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); + void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id); + int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime, + results, n_result, n_hit_result, mid); + return ret; } } @@ -780,53 +715,29 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, scan_count_inc(mid); int virtual_table_id = 0; - struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, - SCAN_TYPE_STRING, &virtual_table_id); - if (NULL == table_schema) { - return MAAT_ERR; - } + //TODO: by luis get virtual_table_id + // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, + // SCAN_TYPE_STRING, &virtual_table_id); + // if (NULL == table_schema) { + // return MAAT_ERR; + // } - struct table_runtime *table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - int group_hit_cnt = table_runtime_scan_string(table_rt, thread_id, data, data_len, group_ids, MAX_SCANNER_HIT_GROUP_NUM, - virtual_table_id, mid); - if (group_hit_cnt <= 0) { + void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)runtime, thread_id, data, data_len, + group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid); + if (group_hit_cnt < 0) { + return MAAT_ERR; + } else if (0 == group_hit_cnt) { return MAAT_OK; - } - - // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT - - struct table_runtime *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); - int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - int n_top_group_ids = group2group_runtime_get_top_groups(g2g_rt, group_ids, group_hit_cnt, top_group_ids); - size_t n_all_group_ids = 0; - if (n_top_group_ids > 0) { - n_all_group_ids = group_hit_cnt + n_top_group_ids; } else { - n_all_group_ids = group_hit_cnt; + // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT + void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); + void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id); + int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime, + results, n_result, n_hit_result, mid); + return ret; } - - int all_group_ids[n_all_group_ids] = {-1}; - int i = 0, j = 0, k = 0; - for (i = 0; i < group_hit_cnt; i++) { - all_group_ids[i] = group_ids[i]; - } - for (j = i; j < n_all_group_ids; j++, k++) { - all_group_ids[j] = top_group_ids[k]; - } - - // maat state find compile_table_id, if not found, maat_instance->default_compile_table_id - struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id); - int n_compile_ids = compile_runtime_match(compile_table_rt, all_group_ids, n_all_group_ids, results, n_result, mid); - - *n_hit_result = n_compile_ids; - - if (n_compile_ids > 0) { - return MAAT_HIT; - } else { - return MAAT_HALF_HIT; - } } struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id) @@ -875,7 +786,8 @@ int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **sta return 0; } -int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, int compile_table_id) +int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, + int compile_table_id) { if (NULL == maat_instance->maat_rt) { return -1; @@ -887,7 +799,8 @@ int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_st return 0; } -size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid, struct maat_hit_path *paths, size_t n_path) +size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid, + struct maat_hit_path *paths, size_t n_path) { int compile_table_id = -1; @@ -897,7 +810,10 @@ size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid, st compile_table_id = mid->compile_table_id; } - compile_runtime_get_hit_paths(maat_instance->maat_rt->table_rt_mgr, compile_table_id, maat_instance->g2g_table_id, mid, paths, n_path); + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); + void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); + compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, + (struct group2group_runtime *)g2g_runtime, mid->compile_mid, paths, n_path); return 0; } @@ -907,10 +823,10 @@ size_t maat_get_hit_objects(struct maat_compile_state *compile_state, struct maa return 0; } -int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path) +int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **state, + struct maat_hit_path *paths, size_t n_path) { struct maat_state *mid = NULL; - struct maat_hit_path *paths; mid = grab_state(state, maat_instance, 0); if (NULL == mid->compile_mid || NULL == maat_instance->maat_rt) { @@ -922,7 +838,8 @@ int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **sta return n_read; } -int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj) +int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, + struct maat_hit_object *objs, size_t n_obj) { return 0; } diff --git a/src/maat_command.cpp b/src/maat_command.cpp index ba5a379..2432ce5 100644 --- a/src/maat_command.cpp +++ b/src/maat_command.cpp @@ -314,7 +314,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); if (valid_column < 0) { - log_error(maat_instance->logger, MODULE_MAAT_COMMAND, + log_error(maat_instance->logger, MODULE_MAAT_COMMAND, "Command set line id %d failed: table %s is not a plugin or ip_plugin table", line_rule->rule_id, line_rule->table_name); FREE(s_rule); diff --git a/src/maat_compile.cpp b/src/maat_compile.cpp index 7c48a5f..f98f781 100644 --- a/src/maat_compile.cpp +++ b/src/maat_compile.cpp @@ -26,6 +26,7 @@ #define MODULE_COMPILE module_name_str("maat.compile") #define MAX_TABLE_LINE_SIZE (1024 * 16) +#define MAX_COMPILE_EX_DATA_NUM 2 enum user_region_encode { USER_REGION_ENCODE_NONE=0, @@ -56,6 +57,7 @@ struct group2compile_schema { int virtual_table_name_column; int clause_index_column; char associated_compile_table_id; + struct table_manager *ref_tbl_mgr; int table_id;//ugly }; @@ -219,10 +221,10 @@ void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (NULL == item || item->type != cJSON_Number) { - goto error; + if (item != NULL && item->type == cJSON_Number) { + compile_schema->table_id = item->valueint; + read_cnt++; } - compile_schema->table_id = item->valueint; item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { @@ -237,8 +239,8 @@ void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle } custom_item = cJSON_GetObjectItem(item, "service_id"); - if (item != NULL && item->type == cJSON_Number) { - compile_schema->service_id_column = item->valueint; + if (custom_item != NULL && custom_item->type == cJSON_Number) { + compile_schema->service_id_column = custom_item->valueint; read_cnt++; } @@ -284,7 +286,7 @@ void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle read_cnt++; } - if (read_cnt < 9) { + if (read_cnt < 10) { goto error; } @@ -306,10 +308,16 @@ void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_h cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (NULL == item || item->type != cJSON_Number) { - goto error; + if (item != NULL && item->type == cJSON_Number) { + g2c_schema->table_id = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "associated_compile_table_id"); + if (item != NULL && item->type == cJSON_Number) { + g2c_schema->associated_compile_table_id = item->valueint; + read_cnt++; } - g2c_schema->table_id = item->valueint; item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { @@ -317,12 +325,6 @@ void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_h goto error; } - custom_item = cJSON_GetObjectItem(item, "associated_compile_table_id"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - g2c_schema->associated_compile_table_id = custom_item->valueint; - read_cnt++; - } - custom_item = cJSON_GetObjectItem(item, "group_id"); if (custom_item != NULL && custom_item->type == cJSON_Number) { g2c_schema->group_id_column = custom_item->valueint; @@ -353,7 +355,7 @@ void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_h read_cnt++; } - if (read_cnt < 6) { + if (read_cnt < 7) { goto error; } @@ -368,12 +370,19 @@ void group2compile_schema_free(void *g2c_schema) FREE(g2c_schema); } +int group2compile_associated_compile_table_id(void *g2c_schema) +{ + struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema; + + return schema->associated_compile_table_id; +} + struct compile_item * compile_item_new(const char *line, struct compile_schema *compile_schema, struct log_handle *logger) { size_t column_offset = 0; size_t column_len = 0; - + char tag_str[MAX_TABLE_LINE_SIZE] = {0}; struct compile_item *compile_item = ALLOC(struct compile_item, 1); int ret = get_column_pos(line, compile_schema->compile_id_column, &column_offset, &column_len); @@ -423,9 +432,8 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, struct goto error; } - char tag_str[MAX_TABLE_LINE_SIZE] = {0}; memcpy(tag_str, (line + column_offset), column_len); - +#if 0 if (n_accept_tag > 0 && strlen(tag_str) > 2) { str_unescape(tag_str); ret = compare_accept_tag(tag_str, accept_tags, n_accept_tag); @@ -442,7 +450,7 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, struct //table_schema->unmatched_tag_cnt++; } } - +#endif ret = get_column_pos(line, compile_schema->user_region_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_COMPILE, @@ -501,7 +509,7 @@ void compile_item_free(struct compile_item *compile_item) FREE(compile_item); } -void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage_bin, +void *compile_runtime_new(void *compile_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == compile_schema) { @@ -516,6 +524,34 @@ void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage return compile_rt; } +void maat_compile_free(struct maat_compile *compile) +{ + struct maat_clause_state *clause_state = NULL; + if (compile->user_data && compile->user_data_free) { + compile->user_data_free(compile->user_data); + } + + for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { + clause_state = compile->clause_states + i; + utarray_free(clause_state->literal_ids); + clause_state->literal_ids = NULL; + clause_state->in_use = 0; + } + compile->magic = 0; + free(compile); +} + +void maat_compile_hash_free(struct maat_compile **compile_hash) +{ + struct maat_compile *compile = NULL, *tmp_compile = NULL; + + HASH_ITER(hh, *compile_hash, compile, tmp_compile) { + HASH_DEL(*compile_hash, compile); + maat_compile_free(compile); + } + assert(*compile_hash == NULL); +} + void compile_runtime_free(void *compile_runtime) { if (NULL == compile_runtime) { @@ -534,17 +570,21 @@ void compile_runtime_free(void *compile_runtime) FREE(compile_rt); } -void *group2compile_runtime_new(void *g2c_schema, struct maat_garbage_bin *garbage_bin, +void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { - if (NULL == g2c_schema) { - return NULL; - } - struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1); return g2c_rt; } +void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime) +{ + struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; + + g2c_rt->ref_compile_rt = (struct compile_runtime *)compile_runtime; + g2c_rt->ref_g2g_rt = (struct group2group_runtime *)g2g_runtime; +} + void group2compile_runtime_free(void *g2c_runtime) { if (NULL == g2c_runtime) { @@ -576,6 +616,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema { size_t column_offset = 0; size_t column_len = 0; + char virtual_table_name[NAME_MAX] = {0}; struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1); int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len); @@ -619,11 +660,11 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema g2c_schema->table_id, line); goto error; } - char virtual_table_name[NAME_MAX] = {0}; + memcpy(virtual_table_name, (line + column_offset), column_len); if (is_valid_table_name(virtual_table_name)) { - g2c_item->vt_id = table_manager_get_table_id(tbl_mgr, virtual_table_name); + g2c_item->vt_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr, virtual_table_name); if (g2c_item->vt_id < 0) { log_error(logger, MODULE_COMPILE, "group2compile table(table_id:%d) line:%s unknown virtual table:%s", @@ -686,23 +727,6 @@ int maat_compile_set(struct maat_compile *compile, int declared_clause_num, void return 0; } -void maat_compile_free(struct maat_compile *compile) -{ - struct maat_clause_state *clause_state = NULL; - if (compile->user_data && compile->user_data_free) { - compile->user_data_free(compile->user_data); - } - - for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - clause_state = compile->clause_states + i; - utarray_free(clause_state->literal_ids); - clause_state->literal_ids = NULL; - clause_state->in_use = 0; - } - compile->magic = 0; - free(compile); -} - int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, struct maat_compile *compile) { int ret = 0; @@ -772,17 +796,6 @@ size_t maat_compile_hash_count(struct maat_compile *compile_hash) return HASH_COUNT(compile_hash); } -void maat_compile_hash_free(struct maat_compile **compile_hash) -{ - struct maat_compile *compile = NULL, *tmp_compile = NULL; - - HASH_ITER(hh, *compile_hash, compile, tmp_compile) { - HASH_DEL(*compile_hash, compile); - maat_compile_free(compile); - } - assert(*compile_hash == NULL); -} - int compare_literal_id(const void *pa, const void *pb) { struct maat_literal_id *la = (struct maat_literal_id *)pa; @@ -796,7 +809,8 @@ int compare_literal_id(const void *pa, const void *pb) return ret; } -int maat_compile_clause_add_literal(struct maat_compile *compile, struct maat_literal_id *literal_id, +int maat_compile_clause_add_literal(struct maat_compile *compile, + struct maat_literal_id *literal_id, int clause_index, int clause_not_flag) { struct maat_clause_state *clause_state = compile->clause_states + clause_index; @@ -807,7 +821,9 @@ int maat_compile_clause_add_literal(struct maat_compile *compile, struct maat_li compile->actual_clause_num++; } - struct maat_literal_id *tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); + struct maat_literal_id *tmp = NULL; + tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, + literal_id, compare_literal_id); if (tmp) { assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); return -1; @@ -819,10 +835,14 @@ int maat_compile_clause_add_literal(struct maat_compile *compile, struct maat_li return 0; } -int maat_compile_clause_remove_literal(struct maat_compile *compile, struct maat_literal_id *literal_id, int clause_index) +int maat_compile_clause_remove_literal(struct maat_compile *compile, + struct maat_literal_id *literal_id, + int clause_index) { struct maat_clause_state* clause_state = compile->clause_states + clause_index; - struct maat_literal_id *tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); + struct maat_literal_id *tmp = NULL; + tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, + literal_id, compare_literal_id); if (tmp) { assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); } else { @@ -841,7 +861,8 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile, struct maat } static const struct maat_clause * -maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, unsigned long long *clause_id_generator, +maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, + unsigned long long *clause_id_generator, struct maat_literal_id *literal_ids, size_t n_literal_id) { struct maat_clause *clause = NULL; @@ -873,7 +894,10 @@ static void maat_clause_hash_free(struct maat_clause *clause_hash) } } -struct bool_matcher *maat_compile_bool_matcher_new(struct maat_compile *compile_hash, unsigned long long *clause_id_generator, struct log_handle *logger) +struct bool_matcher * +maat_compile_bool_matcher_new(struct maat_compile *compile_hash, + unsigned long long *clause_id_generator, + struct log_handle *logger) { if (NULL == compile_hash || NULL == logger) { return NULL; @@ -986,7 +1010,8 @@ void maat_compile_bool_matcher_free(struct bool_matcher *bm) bool_matcher_free(bm); } -size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_compile_state *compile_state, void **user_data_array, size_t ud_array_size) +size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_compile_state *compile_state, + void **user_data_array, size_t ud_array_size) { struct maat_compile *compile = NULL; struct bool_expr_match *expr_match = ALLOC(struct bool_expr_match, MAX_SCANNER_HIT_COMPILE_NUM); @@ -1021,29 +1046,29 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_comp return ud_result_cnt; } -int maat_add_group_to_compile(struct maat_compile **compile_hash, struct maat_group_topology *group_topo, int group_id, int vt_id, - int clause_not_flag, int clause_index, int compile_id, struct log_handle *logger) +int maat_add_group_to_compile(struct maat_compile **compile_hash, void *g2g_runtime, + struct group2compile_item *g2c_item, struct log_handle *logger) { - struct maat_group *group = maat_group_topology_find_group(group_topo, group_id); + struct maat_group *group = group2group_runtime_find_group(g2g_runtime, g2c_item->group_id); if (!group) { - group = maat_group_topology_add_group(group_topo, group_id); + group = group2group_runtime_add_group(g2g_runtime, g2c_item->group_id); } int ret = -1; - struct maat_compile *compile = maat_compile_hash_find(compile_hash, compile_id); + struct maat_compile *compile = maat_compile_hash_find(compile_hash, g2c_item->compile_id); if (!compile) { - compile = maat_compile_new(compile_id); - ret = maat_compile_hash_add(compile_hash, compile_id, compile); + compile = maat_compile_new(g2c_item->compile_id); + ret = maat_compile_hash_add(compile_hash, g2c_item->compile_id, compile); if (ret < 0) { return -1; } } - struct maat_literal_id literal_id = {group_id, vt_id}; - ret = maat_compile_clause_add_literal(compile, &literal_id, clause_index, clause_not_flag); + struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vt_id}; + ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, g2c_item->not_flag); if (ret < 0) { log_error(logger, MODULE_COMPILE, "add literal_id{group_id:%d, vt_id:%d} to clause %d of compile %d failed", - group_id, vt_id, clause_index, compile_id); + g2c_item->group_id, g2c_item->vt_id, g2c_item->clause_index, g2c_item->compile_id); ret = -1; } else { group->ref_by_compile_cnt++; @@ -1053,31 +1078,34 @@ int maat_add_group_to_compile(struct maat_compile **compile_hash, struct maat_gr return ret; } -int maat_remove_group_from_compile(struct maat_compile **compile_hash, struct maat_group_topology *group_topo, int group_id, int vt_id, - int clause_not_flag, int clause_index, int compile_id, struct maat_garbage_bin *garbage_bin, +int maat_remove_group_from_compile(struct maat_compile **compile_hash, void *g2g_runtime, + struct group2compile_item *g2c_item, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { - struct maat_group *group = maat_group_topology_find_group(group_topo, group_id); + struct maat_group *group = group2group_runtime_find_group(g2g_runtime, g2c_item->group_id); if (!group) { - log_error(logger, MODULE_COMPILE, "Remove group %d from compile %d failed, group is not exisited.", - group_id, compile_id); + log_error(logger, MODULE_COMPILE, + "Remove group %d from compile %d failed, group is not exisited.", + g2c_item->group_id, g2c_item->compile_id); return -1; } struct maat_compile *compile = NULL; - HASH_FIND(hh, *compile_hash, &compile_id, sizeof(compile_id), compile); + HASH_FIND(hh, *compile_hash, &(g2c_item->compile_id), sizeof(g2c_item->compile_id), compile); if (!compile) { - log_error(logger, MODULE_COMPILE, "Remove group %d from compile %d failed, compile is not exisited.", - group_id, compile_id); + log_error(logger, MODULE_COMPILE, + "Remove group %d from compile %d failed, compile is not exisited.", + g2c_item->group_id, g2c_item->compile_id); return -1; } - struct maat_literal_id literal_id = {group_id, vt_id}; - int ret = maat_compile_clause_remove_literal(compile, &literal_id, clause_index); + struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vt_id}; + int ret = maat_compile_clause_remove_literal(compile, &literal_id, g2c_item->clause_index); if (ret < 0) { log_error(logger, MODULE_COMPILE, "Remove group %d vt_id %d from clause %d of compile %d failed, literal is not in compile.", - group_id, vt_id, clause_index, compile_id); + g2c_item->group_id, g2c_item->vt_id, g2c_item->clause_index, g2c_item->compile_id); return -1; } @@ -1170,7 +1198,8 @@ static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_path return 0; } -size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct maat_group_topology *group_topo, +size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, + struct group2group_runtime *g2g_rt, struct maat_compile_state *compile_state, struct maat_hit_path *hit_paths, size_t hit_path_size) { @@ -1181,7 +1210,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) { internal_path = (struct maat_internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i); - group = maat_group_topology_find_group(group_topo, internal_path->group_id); + group = group2group_runtime_find_group(g2g_rt, internal_path->group_id); if (0 == group->top_group_cnt && n_made_by_item < hit_path_size) { //group not referenced by compile @@ -1261,21 +1290,29 @@ void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state compile_state->this_scan_item_hit_cnt++; } -void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, struct maat_compile **compile_hash, - int group_id, int vt_id) +void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, + void *compile_runtime, int group_id, int vt_id) { + if (NULL == compile_state || NULL == compile_runtime) { + return; + } + + struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; + assert(compile_rt->compile_hash != NULL); + struct maat_compile *compile = NULL, *tmp_compile = NULL; struct maat_clause_state *clause_state = NULL; struct maat_literal_id literal_id = {group_id, vt_id}; + struct maat_literal_id *tmp = NULL; - HASH_ITER(hh, *compile_hash, compile, tmp_compile) { + HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) { for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { clause_state = compile->clause_states + i; if (!clause_state->in_use) { continue; } - struct maat_literal_id *tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, &literal_id, compare_literal_id); + tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, &literal_id, compare_literal_id); if (tmp) { //Deduplication if (utarray_find(compile_state->all_hit_clause_array, &(clause_state->clause_id), compare_clause_id)) { @@ -1293,138 +1330,8 @@ int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state) return compile_state->not_clause_hitted_flag; } -int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, - int valid_column) -{ - if (NULL == compile_runtime || NULL == compile_schema || NULL == line) { - return -1; - } - - int ret = -1; - struct maat_compile *compile = NULL; - struct compile_item *compile_item = NULL; - struct compile_schema *schema = (struct compile_schema *)compile_schema; - struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; - int is_valid = get_column_value(line, valid_column); - int compile_id = get_column_value(line, schema->compile_id_column); - if (is_valid < 0) { - return -1; - } else if (0 == is_valid) { - //delete - ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id, compile_rt->ref_garbage_bin); - if (ret < 0) { - log_error(logger, MODULE_COMPILE, - "remove compile table(table_id:%d) compile(compile_id:%d) from compile_hash failed", - schema->table_id, compile_id); - return -1; - } - } else { - //add - compile_item = compile_item_new(line, schema, compile_rt->logger); - struct compile_rule *compile_rule = ALLOC(struct compile_rule, 1); - compile_item_to_compile_rule(compile_item, schema, compile_rule); - compile_item_free(compile_item); - compile_item = NULL; - - compile = maat_compile_new(compile_rule->compile_id); - if (NULL == compile) { - destroy_compile_rule(compile_rule); - log_error(logger, MODULE_COMPILE, - "maat_compile_new failed, compile_table(table_id:%d) compile_id:%d", - schema->table_id, compile_item->compile_id); - return -1; - } - - maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule, (void (*)(void *))destroy_compile_rule); - ret = maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile); - if (ret < 0) { - maat_compile_free(compile); - log_error(logger, MODULE_TABLE_RUNTIME, - "add compile table(table_id:%d) compile(compile_id:%d) to compile_hash failed", - schema->table_id, compile_id); - return -1; - } - } - - return 0; -} - -int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line, - int valid_column) -{ - if (NULL == g2c_runtime || NULL == g2c_schema || NULL == line) { - return -1; - } - - struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema; - struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; - struct compile_runtime *compile_rt = g2c_rt->ref_compile_rt; - struct group2group_runtime *g2g_rt = g2c_rt->ref_g2g_rt; - int is_valid = get_column_value(line, valid_column); - if (is_valid < 0) { - return -1; - } - - int ret = -1; - struct group2compile_item *g2c_item = group2compile_item_new(line, g2c_schema, compile_rt->logger); - if (0 == is_valid) { - //delete - ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2g_rt->group_topo, g2c_item->group_id, - g2c_item->vt_id, g2c_item->not_flag, g2c_item->clause_index, - g2c_item->compile_id, compile_rt->ref_garbage_bin, compile_rt->logger); - if (0 == ret) { - if (g2c_item->not_flag) { - g2c_rt->not_flag_group--; - } - } - } else { - //add - ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2g_rt->group_topo, g2c_item->group_id, - g2c_item->vt_id, g2c_item->not_flag, g2c_item->clause_index, - g2c_item->compile_id, compile_rt->logger); - if (0 == ret) { - if (g2c_item->not_flag) { - g2c_rt->not_flag_group++; - } - } - } - - group2compile_item_free(g2c_item); - return ret; -} - -int compile_runtime_commit(void *compile_runtime) -{ - struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; - int ret = 0; - - size_t compile_cnt = maat_compile_hash_count(compile_rt->compile_hash); - if (0 == compile_cnt) { - return 0; - } - - struct bool_matcher *old_bool_matcher = NULL; - struct bool_matcher *new_bool_matcher = NULL; - - log_info(logger, MODULE_TABLE_RUNTIME, - "committing %zu compile rules for rebuilding compile bool_matcher engine", compile_cnt); - new_bool_matcher = maat_compile_bool_matcher_new(compile_rt->compile_hash, &compile_rt->clause_id_generator, logger); - if (NULL == new_bool_matcher) { - log_error(logger, MODULE_TABLE_RUNTIME, - "rebuild compile bool_matcher engine failed when update %zu compile rules", compile_cnt); - ret = -1; - } - - old_bool_matcher = compile_rt->bm; - compile_rt->bm = new_bool_matcher; - maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))maat_compile_bool_matcher_free); - - compile_rt->rule_num = compile_cnt; - - return ret; -} - -void *rule_ex_data_new(const struct maat_rule_head *rule_head, const char *srv_def, const struct compile_ex_data_schema *ex_schema) +void *rule_ex_data_new(const struct maat_rule_head *rule_head, const char *srv_def, + const struct compile_ex_data_schema *ex_schema) { void *ex_data = NULL; struct maat_rule rule; @@ -1472,45 +1379,169 @@ void compile_item_to_compile_rule(struct compile_item *compile_item, struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(compile_schema, i); compile_rule->ex_data[i] = rule_ex_data_new(&compile_rule->head, compile_rule->service_defined, ex_schema); } - compile_rule->is_valid = 1; compile_rule->compile_id = compile_item->compile_id; pthread_rwlock_init(&compile_rule->rwlock, NULL); } void destroy_compile_rule(struct compile_rule *compile_rule) { - struct table_schema *table_schema = compile_rule->ref_table; + struct compile_schema *schema = compile_rule->ref_table; assert(compile_rule->magic_num==COMPILE_RULE_MAGIC); - size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(table_schema); + size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(schema); for (size_t i = 0; i < n_rule_ex_schema; i++) { - struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(table_schema, i); + struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(schema, i); rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, ex_schema); compile_rule->ex_data[i] = NULL; } free(compile_rule->ex_data); - - compile_rule->is_valid = 0; compile_rule->declared_clause_num = -1; free(compile_rule->service_defined); compile_rule->service_defined = NULL; - free(compile_rule); } -struct group2compile_rule *g2c_item_to_g2c_rule(struct group2compile_item *g2c_item) +int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, + int valid_column) { - struct group2compile_rule *g2c_rule = ALLOC(struct group2compile_rule, 1); + if (NULL == compile_runtime || NULL == compile_schema || NULL == line) { + return -1; + } - g2c_rule->group_id = g2c_item->group_id; - g2c_rule->compile_id = g2c_item->compile_id; - g2c_rule->is_valid = g2c_item->is_valid; - g2c_rule->not_flag = g2c_item->not_flag; - g2c_rule->vt_id = g2c_item->vt_id; - g2c_rule->clause_index = g2c_item->clause_index; - g2c_rule->associated_compile_table_id = g2c_item->associated_compile_table_id; + int ret = -1; + struct maat_compile *compile = NULL; + struct compile_item *compile_item = NULL; + struct compile_schema *schema = (struct compile_schema *)compile_schema; + struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; + int is_valid = get_column_value(line, valid_column); + int compile_id = get_column_value(line, schema->compile_id_column); + if (is_valid < 0) { + return -1; + } else if (0 == is_valid) { + //delete + ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id, compile_rt->ref_garbage_bin); + if (ret < 0) { + log_error(compile_rt->logger, MODULE_COMPILE, + "remove compile table(table_id:%d) compile(compile_id:%d) from compile_hash failed", + schema->table_id, compile_id); + return -1; + } + } else { + //add + compile_item = compile_item_new(line, schema, compile_rt->logger); + if (NULL == compile_item) { + return -1; + } - return g2c_rule; + struct compile_rule *compile_rule = ALLOC(struct compile_rule, 1); + compile_item_to_compile_rule(compile_item, schema, compile_rule); + compile_item_free(compile_item); + compile_item = NULL; + + compile = maat_compile_new(compile_rule->compile_id); + if (NULL == compile) { + destroy_compile_rule(compile_rule); + log_error(compile_rt->logger, MODULE_COMPILE, + "maat_compile_new failed, compile_table(table_id:%d) compile_id:%d", + schema->table_id, compile_item->compile_id); + return -1; + } + + maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule, (void (*)(void *))destroy_compile_rule); + ret = maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile); + if (ret < 0) { + maat_compile_free(compile); + log_error(compile_rt->logger, MODULE_COMPILE, + "add compile table(table_id:%d) compile(compile_id:%d) to compile_hash failed", + schema->table_id, compile_id); + return -1; + } + } + + return 0; +} + +int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line, + int valid_column) +{ + if (NULL == g2c_runtime || NULL == g2c_schema || NULL == line) { + return -1; + } + + struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema; + struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; + struct compile_runtime *compile_rt = g2c_rt->ref_compile_rt; + struct group2group_runtime *g2g_rt = g2c_rt->ref_g2g_rt; + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } + + int ret = -1; + struct group2compile_item *g2c_item = group2compile_item_new(line, schema, compile_rt->logger); + if (NULL == g2c_item) { + return -1; + } + + if (0 == is_valid) { + //delete + ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item, + compile_rt->ref_garbage_bin, compile_rt->logger); + if (0 == ret) { + if (g2c_item->not_flag) { + g2c_rt->not_flag_group--; + } + } + } else { + //add + ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item, compile_rt->logger); + if (0 == ret) { + if (g2c_item->not_flag) { + g2c_rt->not_flag_group++; + } + } + } + + group2compile_item_free(g2c_item); + return ret; +} + +int compile_runtime_commit(void *compile_runtime) +{ + if (NULL == compile_runtime) { + return -1; + } + + struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; + size_t compile_cnt = maat_compile_hash_count(compile_rt->compile_hash); + if (0 == compile_cnt) { + return 0; + } + + struct bool_matcher *old_bool_matcher = NULL; + struct bool_matcher *new_bool_matcher = NULL; + + log_info(compile_rt->logger, MODULE_COMPILE, + "committing %zu compile rules for rebuilding compile bool_matcher engine", + compile_cnt); + + int ret = 0; + new_bool_matcher = maat_compile_bool_matcher_new(compile_rt->compile_hash, + &compile_rt->clause_id_generator, + compile_rt->logger); + if (NULL == new_bool_matcher) { + log_error(compile_rt->logger, MODULE_COMPILE, + "rebuild compile bool_matcher engine failed when update %zu compile rules", compile_cnt); + ret = -1; + } + + old_bool_matcher = compile_rt->bm; + compile_rt->bm = new_bool_matcher; + maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))maat_compile_bool_matcher_free); + + compile_rt->rule_num = compile_cnt; + + return ret; } static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b) @@ -1556,13 +1587,12 @@ static int compare_compile_rule(const void *a, const void *b) } int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids, - int *compile_ids, size_t compile_ids_size, struct maat_state *state) + int vt_id, int *compile_ids, size_t compile_ids_size, struct maat_state *state) { - if (NULL == compile_rt || table_rt->table_type != TABLE_TYPE_COMPILE) { + if (NULL == compile_rt) { return -1; } - struct compile_runtime *compile_rt = &(table_rt->compile_rt); struct maat_compile_state *compile_state = state->compile_mid; struct compile_rule *compile_rule_array[compile_ids_size]; diff --git a/src/maat_expr.cpp b/src/maat_expr.cpp index 413487b..1c2efd8 100644 --- a/src/maat_expr.cpp +++ b/src/maat_expr.cpp @@ -19,6 +19,7 @@ #include "maat_limits.h" #include "rcu_hash.h" #include "maat_rule.h" +#include "maat_compile.h" #include "maat_garbage_collection.h" #define MAX_DISTRICT_STR 128 @@ -34,6 +35,7 @@ struct expr_schema { int match_method_column; int is_hexbin_column; enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ + int table_id; //ugly }; enum expr_type { @@ -74,6 +76,7 @@ struct expr_runtime { struct maat_item *item_hash; void (*item_user_data_free)(void *); + int n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -140,12 +143,16 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len); if (ret < 0) { + log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no item_id", + expr_schema->table_id, line); goto error; } expr_item->item_id = atoi(line + column_offset); ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len); if (ret < 0) { + log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no group_id", + expr_schema->table_id, line); goto error; } expr_item->group_id = atoi(line + column_offset); @@ -169,19 +176,23 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem #endif ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); if (ret < 0) { + log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no keywords", + expr_schema->table_id, line); goto error; } if (column_len >= MAX_KEYWORDS_STR) { log_error(logger, MODULE_EXPR, - "update error: expr table[%s]:item_id[%d] keywords length too long", - table_name, expr_item->item_id); + "expr table(table_id:%d) line:%s keywords length too long", + expr_schema->table_id, line); goto error; } memcpy(expr_item->keywords, (line + column_offset), column_len); ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len); if (ret < 0) { + log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no expr_type", + expr_schema->table_id, line); goto error; } @@ -190,6 +201,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len); if (ret < 0) { + log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no match_method", + expr_schema->table_id, line); goto error; } @@ -198,6 +211,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len); if (ret < 0) { + log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no is_hexbin", + expr_schema->table_id, line); goto error; } db_hexbin = atoi(line + column_offset); @@ -217,8 +232,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem break; default: log_error(logger, MODULE_EXPR, - "update error: expr table[%s]:item_id[%d] invalid hexbin value:%d", - table_name, expr_item->item_id, db_hexbin); + "expr table(table_id:%d) line:%s has invalid hexbin value:%d", + expr_schema->table_id, line, db_hexbin); goto error; } @@ -244,7 +259,13 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo int ret = -1; cJSON *custom_item = NULL; - cJSON *item = cJSON_GetObjectItem(json, "custom"); + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (item != NULL && item->type == cJSON_Number) { + expr_schema->table_id = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { log_error(logger, MODULE_EXPR, "table %s has no custom column", table_name); goto error; @@ -319,7 +340,26 @@ void expr_schema_free(void *expr_schema) FREE(expr_schema); } -void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin, +void expr_rule_free(and_expr_t *expr_rule) +{ + if (NULL == expr_rule) { + return; + } + + for (size_t i = 0; i < expr_rule->n_patterns; i++) { + FREE(expr_rule->patterns[i].pat); + } + + FREE(expr_rule); +} + +void expr_ex_data_free(void *user_ctx, void *data) +{ + and_expr_t *expr_rule = (and_expr_t *)data; + expr_rule_free(expr_rule); +} + +void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == expr_schema) { @@ -332,6 +372,7 @@ void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin, expr_rt->htable = rcu_hash_new(expr_ex_data_free); expr_rt->scan_mode = schema->scan_mode; expr_rt->item_user_data_free = maat_item_inner_free; + expr_rt->n_worker_thread = max_thread_num; expr_rt->ref_garbage_bin = garbage_bin; expr_rt->logger = logger; @@ -401,6 +442,25 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_ return 0; } +enum pattern_type expr_type2pattern_type(enum expr_type expr_type) +{ + enum pattern_type pattern_type = PATTERN_TYPE_STR; + + switch (expr_type) { + case EXPR_TYPE_STRING: + case EXPR_TYPE_AND: + break; + case EXPR_TYPE_REGEX: + pattern_type = PATTERN_TYPE_REG; + break; + default: + break; + } + + return pattern_type; +} + +#define MAAT_MAX_EXPR_ITEM_NUM 8 and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger) { size_t i = 0; @@ -420,8 +480,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl } if (i >= MAAT_MAX_EXPR_ITEM_NUM) { - log_error(logger, MODULE_TABLE_RUNTIME, "expr item_id:%d too many patterns", - expr_item->item_id); + log_error(logger, MODULE_EXPR, + "expr item_id:%d too many patterns", expr_item->item_id); return NULL; } @@ -487,7 +547,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, } HASH_DELETE(hh, expr_rt->item_hash, item); - maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free); + maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, maat_item_inner_free); } else { //add HASH_FIND_INT(expr_rt->item_hash, &item_id, item); @@ -499,7 +559,6 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, struct expr_item *expr_item = expr_item_new(line, schema, expr_rt->logger); if (NULL == expr_item) { - log_error(expr_rt->logger, MODULE_EXPR, "expr line %s to item failed", line); return -1; } @@ -507,14 +566,14 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, //int district_id = get_district_id(maat_rt, expr_item->district); int district_id = -1; u_para = maat_item_inner_new(expr_item->group_id, item_id, district_id); - item = maat_item_new(item_id, group_id, u_para); + item = maat_item_new(item_id, expr_item->group_id, u_para); HASH_ADD_INT(expr_rt->item_hash, item_id, item); expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger); expr_item_free(expr_item); if (NULL == expr_rule) { - log_error(expr_rt->logger, MODULE_EXPR, "transform expr table:%s item to expr_rule failed, item_id:%d", - table_name, item_id); + log_error(expr_rt->logger, MODULE_EXPR, "transform expr table(table_id:%d) item to expr_rule failed, item_id:%d", + schema->table_id, item_id); return -1; } } @@ -564,7 +623,7 @@ int expr_runtime_commit(void *expr_runtime) log_info(expr_rt->logger, MODULE_EXPR, "committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt); - new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt, expr_rt->logger); + new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger); if (NULL == new_adapter_hs) { log_error(expr_rt->logger, MODULE_EXPR, "rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt); @@ -574,7 +633,7 @@ int expr_runtime_commit(void *expr_runtime) old_adapter_hs = expr_rt->hs; expr_rt->hs = new_adapter_hs; - maat_garbage_bagging(table_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy); + maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy); rcu_hash_commit(expr_rt->htable); expr_rt->rule_num = rcu_hash_count(expr_rt->htable); rule_cnt = rcu_hash_updating_count(expr_rt->htable); @@ -586,40 +645,22 @@ int expr_runtime_commit(void *expr_runtime) return ret; } -int expr_runtime_updating_flag(struct expr_runtime *expr_rt) +int expr_runtime_updating_flag(void *expr_runtime) { + struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; return rcu_hash_updating_flag(expr_rt->htable); } -void expr_rule_free(and_expr_t *expr_rule) -{ - if (NULL == expr_rule) { - return; - } - - for (size_t i = 0; i < expr_rule->n_patterns; i++) { - FREE(expr_rule->patterns[i].pat); - } - - FREE(expr_rule); -} - -void expr_ex_data_free(void *user_ctx, void *data) -{ - and_expr_t *expr_rule = (and_expr_t *)data; - expr_rule_free(expr_rule); -} - int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len, int group_id_array[], size_t n_group_id_array, int virtual_table_id, struct maat_state *state) { - if (NULL == table_rt) { + if (NULL == expr_rt) { return -1; } int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; size_t n_hit_item = 0; - int ret = adapter_hs_scan(table_rt->expr_rt.hs, thread_id, data, data_len, hit_item_ids, &n_hit_item); + int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, hit_item_ids, &n_hit_item); if (ret < 0) { return -1; } @@ -633,7 +674,7 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const size_t n_group_id = 0; size_t i = 0; for (i = 0; i < n_hit_item; i++) { - HASH_FIND_INT(table_rt->item_hash, &(hit_item_ids[i]), item); + HASH_FIND_INT(expr_rt->item_hash, &(hit_item_ids[i]), item); assert(item != NULL); if (!item) { // should not come here @@ -658,13 +699,14 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const } else { compile_table_id = state->compile_table_id; } - struct maat_runtime *maat_rt = state->maat_instance->maat_rt; - struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, compile_table_id); - assert(compile_table_rt->table_type == TABLE_TYPE_COMPILE); + + void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id); + enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id); + assert(table_type == TABLE_TYPE_COMPILE); // STEP 2: get the specified compile table's hit clause_id array by literal_id for (i = 0; i < n_group_id; i++) { - maat_compile_state_update_hit_clause(compile_state, &(compile_table_rt->compile_rt.compile_hash), group_id_array[i], virtual_table_id); + maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[i], virtual_table_id); } return n_group_id; @@ -672,28 +714,28 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) { - if (NULL == table_rt) { + if (NULL == expr_rt) { return; } - struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(table_rt->expr_rt.hs, thread_id); - table_rt->expr_rt.hs_stream = hs_stream; + struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(expr_rt->hs, thread_id); + expr_rt->hs_stream = hs_stream; } int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, int result[], size_t *n_result) { - if (NULL == table_rt) { + if (NULL == expr_rt) { return -1; } - return adapter_hs_scan_stream(table_rt->expr_rt.hs_stream, data, data_len, result, n_result); + return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, result, n_result); } void expr_runtime_stream_close(struct expr_runtime *expr_rt) { - if (table_rt != NULL) { - adapter_hs_stream_close(table_rt->expr_rt.hs_stream); - table_rt->expr_rt.hs_stream = NULL; + if (expr_rt != NULL) { + adapter_hs_stream_close(expr_rt->hs_stream); + expr_rt->hs_stream = NULL; } } diff --git a/src/maat_garbage_collection.cpp b/src/maat_garbage_collection.cpp index 99fcbba..2844518 100644 --- a/src/maat_garbage_collection.cpp +++ b/src/maat_garbage_collection.cpp @@ -31,7 +31,7 @@ struct maat_garbage_bin { int timeout_seconds; }; -struct maat_garbage_bin* maat_garbage_bin_new(int default_timeout) +struct maat_garbage_bin *maat_garbage_bin_new(int default_timeout) { struct maat_garbage_bin* bin = ALLOC(struct maat_garbage_bin, 1); diff --git a/src/maat_group.cpp b/src/maat_group.cpp index a5979e4..7d86298 100644 --- a/src/maat_group.cpp +++ b/src/maat_group.cpp @@ -60,10 +60,10 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (NULL == item || item->type != cJSON_Number) { - goto error; + if (item != NULL && item->type == cJSON_Number) { + g2g_schema->table_id = item->valueint; + read_cnt++; } - g2g_schema->table_id = item->valueint; item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { @@ -83,13 +83,13 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han read_cnt++; } - if (read_cnt < 2) { + if (read_cnt < 3) { goto error; } return g2g_schema; error: - FREE(g2c_schema); + FREE(g2g_schema); return NULL; } @@ -98,9 +98,27 @@ void group2group_schema_free(void *g2g_schema) FREE(g2g_schema); } -struct group2group_runtime * -group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, - struct log_handle *logger) +struct maat_group_topology *maat_group_topology_new(struct log_handle *logger) +{ + struct maat_group_topology *group_topo = ALLOC(struct maat_group_topology, 1); + UNUSED int ret = 0; + + group_topo->hash_group_by_id = NULL; + group_topo->hash_group_by_vertex = NULL; + + ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED); + assert(ret == IGRAPH_SUCCESS); + + ret = pthread_rwlock_init(&group_topo->rwlock, NULL); + assert(ret == 0); + + group_topo->logger = logger; + + return group_topo; +} + +void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) { struct group2group_runtime *g2g_rt = ALLOC(struct group2group_runtime, 1); @@ -111,6 +129,29 @@ group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_b return g2g_rt; } +void group_vertex_free(struct maat_group *group) +{ + free(group->top_group_ids); + free(group); +} + +void maat_group_topology_free(struct maat_group_topology *group_topo) +{ + struct maat_group *group = NULL, *tmp_group = NULL; + + HASH_CLEAR(hh_vertex_id, group_topo->hash_group_by_vertex);//No need group memory clean up. + HASH_ITER(hh_group_id, group_topo->hash_group_by_id, group, tmp_group) { + HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group); + group_vertex_free(group); + } + assert(group_topo->hash_group_by_id == NULL); + + igraph_destroy(&group_topo->group_graph); + + pthread_rwlock_unlock(&group_topo->rwlock); + pthread_rwlock_destroy(&group_topo->rwlock); +} + void group2group_runtime_free(void *g2g_runtime) { if (NULL == g2g_runtime) { @@ -163,48 +204,6 @@ void group2group_item_free(struct group2group_item *g2g_item) FREE(g2g_item); } -void group_vertex_free(struct maat_group *group) -{ - free(group->top_group_ids); - free(group); -} - -struct maat_group_topology *maat_group_topology_new(struct log_handle *logger) -{ - struct maat_group_topology *group_topo = ALLOC(struct maat_group_topology, 1); - UNUSED int ret = 0; - - group_topo->hash_group_by_id = NULL; - group_topo->hash_group_by_vertex = NULL; - - ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED); - assert(ret == IGRAPH_SUCCESS); - - ret = pthread_rwlock_init(&group_topo->rwlock, NULL); - assert(ret == 0); - - group_topo->logger = logger; - - return group_topo; -} - -void maat_group_topology_free(struct maat_group_topology *group_topo) -{ - struct maat_group *group = NULL, *tmp_group = NULL; - - HASH_CLEAR(hh_vertex_id, group_topo->hash_group_by_vertex);//No need group memory clean up. - HASH_ITER(hh_group_id, group_topo->hash_group_by_id, group, tmp_group) { - HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group); - group_vertex_free(group); - } - assert(group_topo->hash_group_by_id == NULL); - - igraph_destroy(&group_topo->group_graph); - - pthread_rwlock_unlock(&group_topo->rwlock); - pthread_rwlock_destroy(&group_topo->rwlock); -} - size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) { long int i; int printed = 0; @@ -216,8 +215,16 @@ size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) { return printed; } -struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id) +struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id) { + if (NULL == g2g_runtime) { + return NULL; + } + + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + struct maat_group_topology *group_topo = g2g_rt->group_topo; + assert(group_topo != NULL); + struct maat_group *group = ALLOC(struct maat_group, 1); group->group_id = group_id; @@ -232,10 +239,17 @@ struct maat_group *maat_group_topology_add_group(struct maat_group_topology *gro return group; } -void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group) +void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group) { + if (NULL == g2g_runtime || NULL == group) { + return; + } + igraph_vector_t v; char buff[4096] = {0}; + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + struct maat_group_topology *group_topo = g2g_rt->group_topo; + assert(group_topo != NULL); assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0); igraph_vector_init(&v, 8); @@ -258,36 +272,49 @@ void maat_group_topology_remove_group(struct maat_group_topology *group_topo, st group_vertex_free(group); } -struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id) +struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id) { - struct maat_group *group = NULL; + if (NULL == g2g_runtime) { + return NULL; + } + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + struct maat_group_topology *group_topo = g2g_rt->group_topo; + assert(group_topo != NULL); + + struct maat_group *group = NULL; HASH_FIND(hh_group_id, group_topo->hash_group_by_id, &group_id, sizeof(group_id), group); return group; } -int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id) +int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id) { + if (NULL == g2g_runtime) { + return -1; + } + int ret = 0; igraph_integer_t edge_id; - struct maat_group *group = NULL, *superior_group = NULL; + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + struct maat_group_topology *group_topo = g2g_rt->group_topo; + assert(group_topo != NULL); - group = maat_group_topology_find_group(group_topo, group_id); + struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_id); if (NULL == group) { - group = maat_group_topology_add_group(group_topo, group_id); + group = group2group_runtime_add_group(g2g_runtime, group_id); } - superior_group = maat_group_topology_find_group(group_topo, superior_group_id); + struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id); if (NULL == superior_group) { - superior_group = maat_group_topology_add_group(group_topo, superior_group_id); + superior_group = group2group_runtime_add_group(g2g_runtime, superior_group_id); } ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); //No duplicated edges between two groups. if (edge_id > 0) { - log_error(group_topo->logger, MODULE_GROUP, + log_error(g2g_rt->logger, MODULE_GROUP, "Add group %d to group %d failed, relation already exisited.", group->group_id, superior_group->group_id); ret = -1; @@ -301,23 +328,26 @@ int maat_group_topology_add_group_to_group(struct maat_group_topology *group_top return ret; } -int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id) +int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id) { - int ret = 0; - struct maat_group *group = NULL, *superior_group = NULL; + if (NULL == g2g_runtime) { + return -1; + } + + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; //No hash write operation, LOCK protection is unnecessary. - group = maat_group_topology_find_group(group_topo, group_id); + struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_id); if (NULL == group) { - log_error(group_topo->logger, MODULE_GROUP, + log_error(g2g_rt->logger, MODULE_GROUP, "Del group %d from group %d failed, group %d not exisited.", group_id, superior_group_id, group_id); return -1; } - superior_group = maat_group_topology_find_group(group_topo, superior_group_id); + struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id); if (NULL == superior_group) { - log_error(group_topo->logger, MODULE_GROUP, + log_error(g2g_rt->logger, MODULE_GROUP, "Del group %d from group %d failed, superior group %d not exisited.", group_id, superior_group_id, superior_group_id); return -1; @@ -325,6 +355,7 @@ int maat_group_topology_remove_group_from_group(struct maat_group_topology *grou igraph_es_t es; igraph_integer_t edge_num_before = 0, edge_num_after = 0; + struct maat_group_topology *group_topo = g2g_rt->group_topo; edge_num_before = igraph_ecount(&group_topo->group_graph); // The edges between the given pairs of vertices will be included in the edge selection. @@ -333,7 +364,7 @@ int maat_group_topology_remove_group_from_group(struct maat_group_topology *grou //first edge, the fifth is the first vertex of the second edge and so on. The last element //of the argument list must be -1 to denote the end of the argument list. //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small - ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); + int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); assert(ret==IGRAPH_SUCCESS); // ignore no such edge to abort(). igraph_set_error_handler(igraph_error_handler_ignore); @@ -367,18 +398,25 @@ static size_t effective_vertices_count(igraph_vector_t *vids) return i; } -int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo) +int group2group_runtime_build_top_groups(void *g2g_runtime) { + if (NULL == g2g_runtime) { + return -1; + } + struct maat_group *group = NULL, *tmp = NULL; struct maat_group *superior_group = NULL; int tmp_vid=0; - size_t i=0, top_group_cnt=0; + size_t top_group_cnt=0; int* temp_group_ids=NULL; - + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + struct maat_group_topology *group_topo = g2g_rt->group_topo; + assert(group_topo != NULL); + igraph_bool_t is_dag; igraph_is_dag(&(group_topo->group_graph), &is_dag); if (!is_dag) { - log_error(group_topo->logger, MODULE_GROUP, "Sub group cycle detected!"); + log_error(g2g_rt->logger, MODULE_GROUP, "Sub group cycle detected!"); return -1; } @@ -394,7 +432,7 @@ int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo) && 0 == group->ref_by_subordinate_group_cnt) { FREE(group->top_group_ids); - maat_group_topology_remove_group(group_topo, group); + group2group_runtime_remove_group(g2g_runtime, group); continue; } @@ -449,29 +487,27 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char * } int ret = -1; - struct group2group_item *g2g_item = NULL; struct group2group_schema *schema = (struct group2group_schema *)g2g_schema; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; - int item_id = get_column_value(line, schema->table_id); int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; - } else if (0 == is_valid) { - //delete - ret = maat_group_topology_remove_group_from_group(g2g_rt->group_topo, g2g_item->group_id, g2g_item->superior_group_id); - } else { - //add - g2g_item = group2group_item_new(line, schema, logger); } - if (1 == is_valid) { - //add - ret = maat_group_topology_add_group_to_group(g2g_rt->group_topo, g2g_item->group_id, g2g_item->superior_group_id); - } else { - //delete - + struct group2group_item *g2g_item = group2group_item_new(line, schema, g2g_rt->logger); + if (NULL == g2g_item) { + return -1; } + if (0 == is_valid) { + //delete + ret = group2group_runtime_remove_group_from_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id); + } else { + //add + ret = group2group_runtime_add_group_to_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id); + } + group2group_item_free(g2g_item); + return ret; } @@ -481,24 +517,19 @@ int group2group_runtime_commit(void *g2g_runtime) return -1; } - struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; - int ret = maat_group_topology_build_top_groups(g2g_rt->group_topo); - - return ret; + return group2group_runtime_build_top_groups(g2g_runtime); } -int group2group_runtime_get_top_groups(struct group2group_runtime *g2g_rt, int *group_ids, size_t n_group_ids, +int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids, int *top_group_ids) { - if (NULL == table_rt || NULL == group_ids || 0 == n_group_ids || - table_rt->table_type != TABLE_TYPE_GROUP2GROUP) { + if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) { return -1; } size_t top_group_index = 0; - struct group2group_runtime *g2g_rt = &(table_rt->g2g_rt); for (size_t i = 0; i < n_group_ids; i++) { - struct maat_group *group = maat_group_topology_find_group(g2g_rt->group_topo, group_ids[i]); + struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_ids[i]); if (!group) { continue; } diff --git a/src/maat_ip.cpp b/src/maat_ip.cpp index 75883db..ce047c9 100644 --- a/src/maat_ip.cpp +++ b/src/maat_ip.cpp @@ -9,6 +9,7 @@ */ #include +#include #include "utils.h" #include "log/log.h" @@ -19,6 +20,7 @@ #include "IPMatcher.h" #include "maat_ip.h" #include "maat_rule.h" +#include "maat_compile.h" #include "maat_garbage_collection.h" #define MODULE_IP module_name_str("maat.ip") @@ -88,10 +90,6 @@ struct ip_plus_runtime { struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; - // long long *scan_cnt; - // long long *hit_cnt; - // long long *not_grp_hit_cnt; - // long long *stream_num; }; void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) @@ -101,10 +99,10 @@ void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (NULL == item || item->type != cJSON_Number) { - goto error; + if (item != NULL && item->type == cJSON_Number) { + ip_plus_schema->table_id = item->valueint; + read_cnt++; } - ip_plus_schema->table_id = item->valueint; item = cJSON_GetObjectItem(json, "custom"); if (NULL == item || item->type != cJSON_Object) { @@ -136,85 +134,85 @@ void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "sip1"); + custom_item = cJSON_GetObjectItem(item, "sip1"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->sip1_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "sip2"); + custom_item = cJSON_GetObjectItem(item, "sip2"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->sip2_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "sport_format"); + custom_item = cJSON_GetObjectItem(item, "sport_format"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->sport_format_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "sport1"); + custom_item = cJSON_GetObjectItem(item, "sport1"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->sport1_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "sport2"); + custom_item = cJSON_GetObjectItem(item, "sport2"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->sport2_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "daddr_format"); + custom_item = cJSON_GetObjectItem(item, "daddr_format"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->daddr_format_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "dip1"); + custom_item = cJSON_GetObjectItem(item, "dip1"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->dip1_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "dip2"); + custom_item = cJSON_GetObjectItem(item, "dip2"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->dip2_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "dport_format"); + custom_item = cJSON_GetObjectItem(item, "dport_format"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->dport_format_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "dport1"); + custom_item = cJSON_GetObjectItem(item, "dport1"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->dport1_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "dport2"); + custom_item = cJSON_GetObjectItem(item, "dport2"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->dport2_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "proto"); + custom_item = cJSON_GetObjectItem(item, "proto"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->proto_column = custom_item->valueint; read_cnt++; } - custom_item = cJSON_GetObjectItem(json, "direction"); + custom_item = cJSON_GetObjectItem(item, "direction"); if (custom_item != NULL && custom_item->type == cJSON_Number) { ip_plus_schema->direction_column = custom_item->valueint; read_cnt++; } - if (read_cnt < 17) { + if (read_cnt < 18) { goto error; } @@ -229,7 +227,7 @@ void ip_plus_schema_free(void *ip_plus_schema) FREE(ip_plus_schema); } -void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, +void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == ip_plus_schema) { @@ -239,7 +237,7 @@ void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema; struct ip_plus_runtime *ip_plus_rt = ALLOC(struct ip_plus_runtime, 1); - ip_plus_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free); + ip_plus_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger); ip_plus_rt->item_user_data_free = maat_item_inner_free; ip_plus_rt->ref_garbage_bin = garbage_bin; ip_plus_rt->logger = logger; @@ -318,7 +316,9 @@ struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *i ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len); if (ret < 0) { - return -1; + log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no saddr_format", + ip_plus_schema->table_id, line); + goto error; } memcpy(saddr_format, (line + column_offset), column_len); if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) { @@ -519,6 +519,12 @@ int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key return 0; } +int ip_plus_runtime_updating_flag(void *ip_plus_runtime) +{ + struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; + return ex_data_runtime_updating_flag(ip_plus_rt->ex_data_rt); +} + int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line, int valid_column) { @@ -562,12 +568,11 @@ int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const ch ip_plus_item = ip_plus_item_new(line, schema, ip_plus_rt->logger); if (NULL == ip_plus_item) { - log_error(ip_plus_rt->logger, MODULE_IP, "ip_plus line:%s to item failed", line); return -1; } u_para = maat_item_inner_new(ip_plus_item->group_id, item_id, 0); - item = maat_item_new(item_id, group_id, u_para); + item = maat_item_new(item_id, ip_plus_item->group_id, u_para); HASH_ADD_INT(ip_plus_rt->item_hash, item_id, item); } @@ -619,11 +624,11 @@ int ip_plus_runtime_commit(void *ip_plus_runtime) size_t mem_used = 0; if (rule_cnt > 0) { - log_info(logger, MODULE_IP, + log_info(ip_plus_rt->logger, MODULE_IP, "committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_ip_matcher) { - log_error(logger, MODULE_IP, + log_error(ip_plus_rt->logger, MODULE_IP, "rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt); ret = -1; } @@ -631,7 +636,7 @@ int ip_plus_runtime_commit(void *ip_plus_runtime) old_ip_matcher = ip_plus_rt->ip_matcher; ip_plus_rt->ip_matcher = new_ip_matcher; - maat_garbage_bagging(garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); + maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); ex_data_runtime_commit(ex_data_rt); ip_plus_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt); @@ -641,11 +646,11 @@ int ip_plus_runtime_commit(void *ip_plus_runtime) return ret; } -int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *data, +int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr, int *group_id_array, size_t n_group_id_array, int virtual_table_id, struct maat_state *state) { - if (NULL == table_rt) { + if (NULL == ip_plus_rt) { return -1; } @@ -653,14 +658,10 @@ int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr * struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; struct ip_data ip; - ip.type = ip_type_transform(scan_data->ip_type); - if (ip.type == IPv4) { - ip.ipv4 = scan_data->ipv4; - } else { - memcpy(ip.ipv6, scan_data->ipv6, sizeof(scan_data->ipv6)); - } - - n_hit_item = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, MAX_SCANNER_HIT_ITEM_NUM); + ip.type = IPv4; + ip.ipv4 = ip_addr; + + n_hit_item = ip_matcher_match(ip_plus_rt->ip_matcher, &ip, scan_results, MAX_SCANNER_HIT_ITEM_NUM); if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; } @@ -671,7 +672,7 @@ int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr * size_t n_group_id = 0; int i = 0; for (i = 0; i < n_hit_item; i++) { - HASH_FIND_INT(table_rt->item_hash, &(scan_results[i].rule_id), item); + HASH_FIND_INT(ip_plus_rt->item_hash, &(scan_results[i].rule_id), item); assert(item != NULL); if (!item) { // should not come here @@ -686,7 +687,8 @@ int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr * } // update hit path - maat_compile_state_update_hit_path(compile_state, scan_results[i].rule_id, item->group_id, virtual_table_id, state->scan_cnt, i); + maat_compile_state_update_hit_path(compile_state, scan_results[i].rule_id, item->group_id, + virtual_table_id, state->scan_cnt, i); } // update hit clause: literal_id{group_id,vt_id} to clause_id @@ -696,12 +698,13 @@ int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr * } else { compile_table_id = state->compile_table_id; } - struct maat_runtime *maat_rt = state->maat_instance->maat_rt; - struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, compile_table_id); - assert(compile_table_rt->table_type == TABLE_TYPE_COMPILE); + + void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id); + enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id); + assert(table_type == TABLE_TYPE_COMPILE); for (size_t idx = 0; idx < n_group_id; idx++) { - maat_compile_state_update_hit_clause(compile_state, &(compile_table_rt->compile_rt.compile_hash), group_id_array[idx], virtual_table_id); + maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[idx], virtual_table_id); } return n_group_id; diff --git a/src/maat_ip_plugin.cpp b/src/maat_ip_plugin.cpp index e577a10..86247bd 100644 --- a/src/maat_ip_plugin.cpp +++ b/src/maat_ip_plugin.cpp @@ -8,14 +8,15 @@ *********************************************************************************************** */ +#include + #include "maat_ip_plugin.h" -#include "cJSON/cJSON.h" -#include "log/log.h" #include "utils.h" #include "maat_utils.h" #include "maat_ex_data.h" #include "IPMatcher.h" #include "maat_rule.h" +#include "maat_garbage_collection.h" #define MODULE_IP_PLUGIN module_name_str("maat.ip_plugin") #define MAX_IP_STR 128 @@ -44,16 +45,9 @@ struct ip_plugin_runtime { uint32_t rule_num; uint32_t updating_rule_num; - struct maat_item *item_hash; - void (*item_user_data_free)(void *); struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; - - // long long *scan_cnt; - // long long *hit_cnt; - // long long *not_grp_hit_cnt; - // long long *stream_num; }; void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) @@ -63,10 +57,10 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (NULL == item || item->type != cJSON_Number) { - goto error; + if (item != NULL && item->type == cJSON_Number) { + ip_plugin_schema->table_id = item->valueint; + read_cnt++; } - ip_plugin_schema->table_id = item->valueint; item = cJSON_GetObjectItem(json, "custom"); if (NULL == item || item->type != cJSON_Object) { @@ -98,7 +92,7 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl read_cnt++; } - if (read_cnt < 4) { + if (read_cnt < 5) { goto error; } @@ -193,42 +187,40 @@ void ip_plugin_item_free(struct ip_plugin_item *item) int ip_plugin_table_ex_data_schema_flag(struct ip_plugin_schema *ip_plugin_schema) { - + return 0; } int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp) + long argl, void *argp, + struct log_handle *logger) { - struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; - struct ex_data_schema *ex_schema = schema->ex_schema; - - if (ex_schema->set_flag) { - assert(0); - log_error(logger, MODULE_TABLE, "Error: %s, EX data schema already registed", - __FUNCTION__); + if (NULL == ip_plugin_schema) { return -1; } - ex_schema->new_func = new_func; - ex_schema->free_func = free_func; - ex_schema->dup_func = dup_func; - ex_schema->argl = argl; - ex_schema->argp = argp; - //ex_schema->set_flag = 1; + struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; + if (schema->ex_schema != NULL) { + assert(0); + log_error(logger, MODULE_IP_PLUGIN, + "Error: %s, EX data schema already registed", __FUNCTION__); + return -1; + } + + schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp); return 0; } int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_schema *schema, - const char *row, char *key, size_t key_len, struct ip_plugin_item *item, - int is_valid) + const char *row, char *key, size_t key_len, + struct ip_plugin_item *ip_plugin_item, int is_valid) { int ret = -1; struct ex_data_runtime *ex_data_rt = rt->ex_data_rt; - int set_flag = ip_plugin_table_schema_ex_data_schema_flag(schema); + int set_flag = ip_plugin_table_ex_data_schema_flag(schema); if (1 == set_flag) { if (0 == is_valid) { @@ -240,7 +232,7 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_ } else { //add void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); - struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)item); + struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)ip_plugin_item); ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); if (ret < 0) { return -1; @@ -253,7 +245,7 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_ return 0; } -void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *garbage_bin, +void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == ip_plugin_schema) { @@ -263,8 +255,7 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *gar struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1); - ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free); - ip_plugin_rt->item_user_data_free = maat_item_inner_free; + ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger); ip_plugin_rt->ref_garbage_bin = garbage_bin; ip_plugin_rt->logger = logger; @@ -280,21 +271,33 @@ void ip_plugin_runtime_free(void *ip_plugin_runtime) struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; if (ip_plugin_rt->ip_matcher != NULL) { ip_matcher_free(ip_plugin_rt->ip_matcher); + ip_plugin_rt->ip_matcher = NULL; } if (ip_plugin_rt->ex_data_rt != NULL) { ex_data_runtime_free(ip_plugin_rt->ex_data_rt); - } - - struct maat_item *item = NULL, *tmp_item = NULL; - HASH_ITER(hh, ip_plugin_rt->item_hash, item, tmp_item) { - HASH_DELETE(hh, ip_plugin_rt->item_hash, item); - maat_item_free(item, ip_plugin_rt->item_user_data_free); + ip_plugin_rt->ex_data_rt = NULL; } FREE(ip_plugin_rt); } +void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule) +{ + if (4 == item->ip_type) { + rule->type = IPv4; + ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, + &(rule->ipv4_rule.start_ip), &(rule->ipv4_rule.end_ip)); + } else { + rule->type = IPv6; + ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, + rule->ipv6_rule.start_ip, rule->ipv6_rule.end_ip); + } + + rule->rule_id = item->item_id; + rule->user_tag = NULL; +} + int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line, int valid_column) { @@ -302,57 +305,28 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, co return -1; } - struct maat_item *item = NULL; struct ip_plugin_item *ip_plugin_item = NULL; - struct maat_item_inner *u_para = NULL; struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; int item_id = get_column_value(line, schema->item_id_column); int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; - } else if (0 == is_valid) { - //delete - HASH_FIND_INT(ip_plugin_rt->item_hash, &item_id, item); - if (NULL == item) { - return -1; - } - - u_para = (struct maat_item_inner *)item->user_data; - item->user_data = NULL; - - if (NULL == u_para) { - return -1; - } - - HASH_DELETE(hh, ip_plugin_rt->item_hash, item); - maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free); - } else { + } + + if (1 == is_valid) { //add - HASH_FIND_INT(ip_plugin_rt->item_hash, &item_id, item); - if (item) { - log_error(ip_plus_rt->logger, MODULE_IP_PLUGIN, - "ip_plugin runtime add item %d to item_hash failed, already exist", item_id); - return -1; - } - ip_plugin_item = ip_plugin_item_new(line, schema, ip_plugin_rt->logger); if (NULL == ip_plugin_item) { - log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, "ip_plugin line:%s to item failed", line); return -1; } - - u_para = maat_item_inner_new(ip_plugin_item->group_id, item_id, 0); - item = maat_item_new(item_id, group_id, u_para); - HASH_ADD_INT(ip_plugin_rt->item_hash, item_id, item); } char *key = (char *)&item_id; - int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, row, key, sizeof(int), ip_plugin_item, is_valid); + int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key, sizeof(int), ip_plugin_item, is_valid); if (ret < 0) { if (ip_plugin_item != NULL) { - ip_plugin_item_free(ip_plugin_item); - ip_plugin_item = NULL; + FREE(ip_plugin_item); } return -1; } else { @@ -366,20 +340,6 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, co return 0; } -void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule) -{ - if (4 == item->ip_type) { - rule->type = IPv4; - ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, &(rule->ipv4_rule.start_ip), &(rule->ipv4_rule.end_ip)); - } else { - rule->type = IPv6; - ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, &(rule->ipv6_rule.start_ip), &(rule->ipv6_rule.end_ip)); - } - - rule->rule_id = item->item_id; - rule->user_tag = NULL; -} - int ip_plugin_runtime_commit(void *ip_plugin_runtime) { if (NULL == ip_plugin_runtime) { @@ -409,11 +369,11 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime) size_t mem_used = 0; if (rule_cnt > 0) { - log_info(ip_plugin_rt->logger, MODULE_TABLE_RUNTIME, + log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, "committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_ip_matcher) { - log_error(ip_plugin_rt->logger, MODULE_TABLE_RUNTIME, + log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, "rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt); ret = -1; } @@ -431,8 +391,9 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime) return ret; } -int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt) +int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime) { + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt); } diff --git a/src/maat_plugin.cpp b/src/maat_plugin.cpp index d525675..8f93288 100644 --- a/src/maat_plugin.cpp +++ b/src/maat_plugin.cpp @@ -14,7 +14,6 @@ #include "cJSON/cJSON.h" #include "utils.h" #include "maat_utils.h" -#include "maat.h" #include "maat_plugin.h" #include "maat_ex_data.h" #include "maat_limits.h" @@ -28,11 +27,6 @@ struct plugin_callback_schema { void *u_para; }; -struct plugin_item { - char key[MAX_KEYWORDS_STR]; - size_t key_len; -}; - struct plugin_runtime { uint64_t acc_line_num; struct ex_data_runtime *ex_data_rt; @@ -41,6 +35,7 @@ struct plugin_runtime { uint32_t updating_rule_num; struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; }; #define MAX_PLUGIN_PER_TABLE 32 @@ -80,10 +75,10 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle * cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); - if (NULL == item || item->type != cJSON_Number) { - goto error; + if (item != NULL && item->type == cJSON_Number) { + plugin_schema->table_id = item->valueint; + read_cnt++; } - plugin_schema->table_id = item->valueint; item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { @@ -92,7 +87,7 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle * } custom_item = cJSON_GetObjectItem(item, "item_id"); - if (custom_item == NULL || custom_item->type != cJSON_Object) { + if (custom_item != NULL && custom_item->type == cJSON_Number) { plugin_schema->item_id_column = custom_item->valueint; read_cnt++; } @@ -126,7 +121,7 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle * } } - if (read_cnt < 4) { + if (read_cnt < 5) { goto error; } @@ -197,41 +192,6 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema) } } -struct plugin_item *plugin_item_new(const char *line, struct plugin_schema *plugin_schema, - struct log_handle *logger) -{ - size_t column_offset = 0; - size_t column_len = 0; - struct plugin_item *plugin_item = ALLOC(struct plugin_item, 1); - - int ret = get_column_pos(line, plugin_schema->key_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_PLUGIN, - "plugin table(table_id:%d) line:%s has no key", - plugin_schema->table_id, line); - goto error; - } - - if (column_len > MAX_KEYWORDS_STR) { - log_error(logger, MODULE_PLUGIN, - "plugin table(table_id:%d): key:%s length:%zu too long, exceed %d", - plugin_schema->table_id, (line + column_offset), column_len, MAX_KEYWORDS_STR); - goto error; - } - memcpy(plugin_item->key, (line + column_offset), column_len); - plugin_item->key_len = column_len; - - return plugin_item; -error: - FREE(plugin_item); - return NULL; -} - -void plugin_item_free(struct plugin_item *plugin_item) -{ - FREE(plugin_item); -} - int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns) { if (NULL == plugin_schema) { @@ -279,6 +239,43 @@ struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema) return schema->ex_schema; } +void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == plugin_schema) { + return NULL; + } + + struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; + struct plugin_runtime *plugin_rt = ALLOC(struct plugin_runtime, 1); + plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger); + plugin_rt->ref_garbage_bin = garbage_bin; + plugin_rt->logger = logger; + + return plugin_rt; +} + +void plugin_runtime_free(void *plugin_runtime) +{ + if (NULL == plugin_runtime) { + return; + } + + struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; + if (plugin_rt->ex_data_rt != NULL) { + ex_data_runtime_free(plugin_rt->ex_data_rt); + plugin_rt->ex_data_rt = NULL; + } + + FREE(plugin_rt); +} + +int plugin_runtime_updating_flag(void *plugin_runtime) +{ + struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; + return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt); +} + int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_schema *plugin_schema, const char *row, char *key, size_t key_len, int is_valid) { @@ -359,11 +356,6 @@ int plugin_runtime_commit(void *plugin_runtime) return 0; } -int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt) -{ - return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt); -} - struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime) { if (NULL == plugin_runtime) { diff --git a/src/maat_redis_monitor.cpp b/src/maat_redis_monitor.cpp index 9d71d4d..25b4755 100644 --- a/src/maat_redis_monitor.cpp +++ b/src/maat_redis_monitor.cpp @@ -18,6 +18,7 @@ #include "maat_config_monitor.h" #include "maat_redis_monitor.h" #include "maat_plugin.h" +#include "maat_virtual.h" #define MODULE_REDIS_MONITOR module_name_str("maat.redis_monitor") @@ -97,7 +98,6 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_instance, const char *dir) { int rule_with_foreign_key = 0; - void *schema = NULL; for (int i = 0; i < rule_num; i++) { if (NULL == rule_list[i].table_line) { @@ -105,9 +105,9 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in } int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, rule_list[i].table_name); - schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - if (!table_schema || table_type != TABLE_TYPE_PLUGIN) { + if (!schema || table_type != TABLE_TYPE_PLUGIN) { continue; } @@ -607,7 +607,7 @@ FULL_UPDATE: continue; } - if (table_schema_mgr) { + if (tbl_mgr) { int table_id = table_manager_get_table_id(tbl_mgr, s_rule_array[full_idx].table_name); //Unrecognized table. if (table_id < 0) { @@ -1277,7 +1277,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, int valid_column = -1; enum table_type table_type; enum scan_type scan_type; - void *table_schema = NULL; + //void *table_schema = NULL; struct maat *maat_instance = (struct maat *)u_param; //authorized to write @@ -1315,7 +1315,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, int update_type = CM_UPDATE_TYPE_INC; int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version, - &new_version, maat_instance->table_schema_mgr, &rule_list, + &new_version, maat_instance->tbl_mgr, &rule_list, &update_type, maat_instance->cumulative_update_off, maat_instance->logger); //redis communication error diff --git a/src/maat_rule.cpp b/src/maat_rule.cpp index 32b0e9f..fb48996 100644 --- a/src/maat_rule.cpp +++ b/src/maat_rule.cpp @@ -19,6 +19,8 @@ #include "utils.h" #include "json2iris.h" +#include "log/log.h" +#include "cJSON/cJSON.h" #include "maat_utils.h" #include "maat_rule.h" #include "maat_config_monitor.h" @@ -27,6 +29,7 @@ #include "maat_compile.h" #include "maat_plugin.h" #include "alignment.h" +#include "maat_garbage_collection.h" #define MODULE_MAAT_RULE module_name_str("maat.rule") @@ -51,7 +54,7 @@ void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *) free(item); } -static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, int n_accept_tag) +static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, size_t n_accept_tag) { if (NULL == tag_obj || NULL == accept_tags) { return -1; @@ -70,7 +73,7 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, int name_matched = 0; int n_val = cJSON_GetArraySize(tag_vals_array); - for (int i = 0; i < n_accept_tag; i++) { + for (size_t i = 0; i < n_accept_tag; i++) { if (0 != strcmp(accept_tags[i].tag_name, tag_name)) { continue; } @@ -106,7 +109,7 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, } //@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}] -static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_tags, int n_accept_tag) +static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_tags, size_t n_accept_tag) { int matched = 0; @@ -138,9 +141,13 @@ error: } //@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]} -int parse_accept_tag(const char *value, struct rule_tag **result, void *logger) +size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger) { - cJSON *json = JSON_Parse(value); + if (NULL == value || NULL == result || NULL == logger) { + return 0; + } + + cJSON *json = cJSON_Parse(value); if (!json) { log_error(logger, MODULE_MAAT_RULE, "parse accept tag Error before: %-200.200s", cJSON_GetErrorPtr()); @@ -167,7 +174,7 @@ int parse_accept_tag(const char *value, struct rule_tag **result, void *logger) //@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]} //@return 1 on match, 0 on not match, -1 on error. -int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag) +int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag) { int ret = -1; int n_set = 0; @@ -191,7 +198,7 @@ int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, in goto error; } - ret = compare_each_tag_set(tag_set, accept_tags, n_tag); + ret = compare_each_tag_set(tag_set, accept_tags, n_accept_tag); //match or error occurs. if (ret != 0) { break; @@ -214,8 +221,13 @@ struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int distr return item; } -void maat_item_inner_free(struct maat_item_inner *item) +void maat_item_inner_free(void *item_inner) { + if (NULL == item_inner) { + return; + } + + struct maat_item_inner *item = (struct maat_item_inner *)item_inner; assert(item->magic_num == ITEM_RULE_MAGIC); assert(item->expr_id_cnt == 0 || item->expr_id_cnt == item->expr_id_ub - item->expr_id_lb + 1); item->magic_num = 0; @@ -228,15 +240,15 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); maat_rt->version = version; - int ret = table_manager_init(maat_instance->tbl_mgr, maat_instance->garbage_bin); + int ret = table_manager_runtime_create(maat_instance->tbl_mgr, maat_instance->nr_worker_thread, + maat_instance->garbage_bin); if (ret < 0) { FREE(maat_rt); return NULL; } maat_rt->ref_tbl_mgr = maat_instance->tbl_mgr; - maat_rt->max_table_num = table_manager_table_count(maat_instance->table_mgr); - maat_rt->max_thread_num = maat_instance->nr_worker_thread; + maat_rt->max_table_num = table_manager_table_count(maat_instance->tbl_mgr); maat_rt->logger = maat_instance->logger; maat_rt->ref_garbage_bin = maat_instance->garbage_bin; @@ -248,12 +260,7 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger) { for (size_t i = 0; i < maat_rt->max_table_num; i++) { - void *runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, i); - if (NULL == runtime) { - continue; - } - - table_manager_commit_runtime(runtime, maat_rt->version, maat_rt->max_thread_num, logger); + table_manager_commit_runtime(maat_rt->ref_tbl_mgr, i); } maat_rt->last_update_time = time(NULL); @@ -265,9 +272,9 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt) return; } - if (maat_rt->table_rt_mgr != NULL) { + if (maat_rt->ref_tbl_mgr != NULL) { table_manager_runtime_destroy(maat_rt->ref_tbl_mgr); - maat_rt->ref_table_mgr = NULL; + maat_rt->ref_tbl_mgr = NULL; } FREE(maat_rt); @@ -275,13 +282,10 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt) int maat_runtime_updating_flag(struct maat_runtime *maat_rt) { + int flag = -1; + for (size_t i = 0; i < maat_rt->max_table_num; i++) { - struct table_runtime *table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, i); - if (NULL == table_rt) { - continue; - } - - int flag = table_runtime_updating_flag(table_rt); + flag = table_manager_runtime_updating_flag(maat_rt->ref_tbl_mgr, i); if (1 == flag) { return 1; } @@ -300,11 +304,9 @@ void maat_start_cb(long long new_version, int update_type, void *u_param) maat_instance->maat_version = new_version; } - int table_id = -1; - enum table_type table_type = TABLE_TYPE_MAX; size_t table_cnt = table_manager_table_count(maat_instance->tbl_mgr); for (size_t i = 0; i < table_cnt; i++) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i); + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i); if (table_type != TABLE_TYPE_PLUGIN) { continue; } @@ -342,40 +344,49 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param) maat_rt = maat_instance->maat_rt; } - void *runtime = table_manager_get_runtime(maat_rt->tbl_mgr, table_id); - int ret = table_manager_update_runtime(runtime, schema, line); - if (ret < 0) { - log_error(maat_instance->logger, MODULE_MAAT_RULE, - "table manager update runtime error, table_name:%s", table_name); - return -1; - } + table_manager_update_runtime(maat_rt->ref_tbl_mgr, table_id, line); - //TODO: by luis - //int is_valid = table_manager_get_valid return 0; } uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt) { uint32_t total = 0; - struct table_runtime *table_rt = NULL; + void *runtime = NULL; for (size_t i = 0; i < maat_rt->max_table_num; i++) { - table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, i); - if (table_rt != NULL) { - total += table_runtime_rule_count(table_rt); + runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, i); + if (runtime != NULL) { + //TODO: by luis + //total += table_runtime_rule_count(runtime); } } return total; } +void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr) +{ + size_t table_cnt = table_manager_table_count(tbl_mgr); + enum table_type table_type = TABLE_TYPE_MAX; + + for (size_t i = 0; i < table_cnt; i++) { + table_type = table_manager_get_table_type(tbl_mgr, i); + if (table_type != TABLE_TYPE_PLUGIN) { + continue; + } + + void *plugin_schema = table_manager_get_schema(tbl_mgr, i); + plugin_table_all_callback_finish((struct plugin_schema *)plugin_schema); + } +} + void maat_finish_cb(void *u_param) { struct maat *maat_instance = (struct maat *)u_param; - //table_manager_all_plugin_cb_finish(maat_instance->table_schema_mgr); - plugin_table_all_callback_finish(maat_) + maat_plugin_table_all_callback_finish(maat_instance->tbl_mgr); + if (maat_instance->creating_maat_rt != NULL) { maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger); @@ -515,12 +526,6 @@ void *rule_monitor_loop(void *arg) } } - for (int i = 0; i < maat_instance->n_accept_tag; i++) { - FREE(maat_instance->accept_tags[i].tag_name); - FREE(maat_instance->accept_tags[i].tag_val); - } - FREE(maat_instance->accept_tags); - FREE(maat_instance); return NULL; diff --git a/src/maat_table.cpp b/src/maat_table.cpp index 5d2ac0e..73a7a81 100644 --- a/src/maat_table.cpp +++ b/src/maat_table.cpp @@ -12,6 +12,7 @@ #include #include "log/log.h" +#include "utils.h" #include "maat_utils.h" #include "maat_table.h" #include "maat_rule.h" @@ -23,6 +24,7 @@ #include "maat_group.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" +#include "maat_virtual.h" #define MODULE_TABLE module_name_str("maat.table") @@ -45,8 +47,9 @@ struct table_manager { size_t n_table; struct rule_tag *accept_tags; - int n_accept_tag; + size_t n_accept_tag; + int default_compile_table_id; struct maat_kv_store *tablename2id_map; struct log_handle *logger; }; @@ -56,11 +59,12 @@ struct table_operations { void *(*new_schema)(cJSON *json, const char *table_name, struct log_handle *logger); void (*free_schema)(void *schema); - void *(*new_runtime)(void *schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); + void *(*new_runtime)(void *schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void (*free_runtime)(void *runtime); int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column); int (*commit_runtime)(void *runtime); + int (*runtime_updating_flag)(void *runtime); }; struct table_operations table_ops[TABLE_TYPE_MAX] = { @@ -71,7 +75,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = expr_runtime_new, .free_runtime = expr_runtime_free, .update_runtime = expr_runtime_update, - .commit_runtime = expr_runtime_commit + .commit_runtime = expr_runtime_commit, + .runtime_updating_flag = expr_runtime_updating_flag }, { .type = TABLE_TYPE_EXPR_PLUS, @@ -80,7 +85,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = expr_runtime_new, .free_runtime = expr_runtime_free, .update_runtime = expr_runtime_update, - .commit_runtime = expr_runtime_commit + .commit_runtime = expr_runtime_commit, + .runtime_updating_flag = expr_runtime_updating_flag }, { .type = TABLE_TYPE_IP_PLUS, @@ -89,7 +95,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = ip_plus_runtime_new, .free_runtime = ip_plus_runtime_free, .update_runtime = ip_plus_runtime_update, - .commit_runtime = ip_plus_runtime_commit + .commit_runtime = ip_plus_runtime_commit, + .runtime_updating_flag = ip_plus_runtime_updating_flag }, { .type = TABLE_TYPE_INTERVAL, @@ -98,7 +105,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL + .commit_runtime = NULL, + .runtime_updating_flag = NULL }, { .type = TABLE_TYPE_INTERVAL_PLUS, @@ -107,7 +115,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL + .commit_runtime = NULL, + .runtime_updating_flag = NULL }, { .type = TABLE_TYPE_DIGEST, @@ -116,7 +125,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL + .commit_runtime = NULL, + .runtime_updating_flag = NULL }, { .type = TABLE_TYPE_SIMILARITY, @@ -125,7 +135,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL + .commit_runtime = NULL, + .runtime_updating_flag = NULL }, { .type = TABLE_TYPE_CONJUNCTION, @@ -134,12 +145,18 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL + .commit_runtime = NULL, + .runtime_updating_flag = NULL }, { .type = TABLE_TYPE_PLUGIN, .new_schema = plugin_schema_new, - + .free_schema = plugin_schema_free, + .new_runtime = plugin_runtime_new, + .free_runtime = plugin_runtime_free, + .update_runtime = plugin_runtime_update, + .commit_runtime = plugin_runtime_commit, + .runtime_updating_flag = plugin_runtime_updating_flag }, { .type = TABLE_TYPE_IP_PLUGIN, @@ -148,7 +165,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = ip_plugin_runtime_new, .free_runtime = ip_plugin_runtime_free, .update_runtime = ip_plugin_runtime_update, - .commit_runtime = ip_plugin_runtime_commit + .commit_runtime = ip_plugin_runtime_commit, + .runtime_updating_flag = ip_plugin_runtime_updating_flag }, { .type = TABLE_TYPE_FQDN_PLUGIN, @@ -170,8 +188,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { }, { .type = TABLE_TYPE_VIRTUAL, - .new_schema = NULL, - .free_schema = NULL, + .new_schema = virtual_schema_new, + .free_schema = virtual_schema_free, .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, @@ -184,16 +202,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = compile_runtime_new, .free_runtime = compile_runtime_free, .update_runtime = compile_runtime_update, - .commit_runtime = compile_runtime_commit - }, - { - .type = TABLE_TYPE_GROUP2COMPILE, - .new_schema = group2compile_schema_new, - .free_schema = group2compile_schema_free, - .new_runtime = NULL, - .free_runtime = NULL, - .update_runtime = group2compile_runtime_update, - .commit_runtime = NULL + .commit_runtime = compile_runtime_commit, + .runtime_updating_flag = NULL }, { .type = TABLE_TYPE_GROUP2GROUP, @@ -202,11 +212,23 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = group2group_runtime_new, .free_runtime = group2group_runtime_free, .update_runtime = group2group_runtime_update, - .commit_runtime = group2group_runtime_commit + .commit_runtime = group2group_runtime_commit, + .runtime_updating_flag = NULL + }, + { + .type = TABLE_TYPE_GROUP2COMPILE, + .new_schema = group2compile_schema_new, + .free_schema = group2compile_schema_free, + .new_runtime = group2compile_runtime_new, + .free_runtime = group2compile_runtime_free, + .update_runtime = group2compile_runtime_update, + .commit_runtime = NULL, + .runtime_updating_flag = NULL } }; -void *maat_table_schema_new(cJSON *json, const char *table_name, enum table_type table_type) +void *maat_table_schema_new(cJSON *json, const char *table_name, enum table_type table_type, + struct log_handle *logger) { void *schema = NULL; @@ -258,8 +280,9 @@ static void register_tablename2id(cJSON *json, struct maat_kv_store *tablename2i } if (strlen(item->valuestring) >= NAME_MAX) { - log_error(logger, MODULE_TABLE_SCHEMA, - "table(table_id:%d) name %s length too long", table_id, item->valuestring); + log_error(logger, MODULE_TABLE, + "table(table_id:%d) name %s length too long", + table_id, item->valuestring); return; } @@ -311,10 +334,13 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo } item = cJSON_GetObjectItem(json, "valid_column"); - if (NULL == item && item->type != cJSON_Number) { - goto error; + if (NULL == item || item->type != cJSON_Number) { + if (ptable->table_type != TABLE_TYPE_VIRTUAL) { + goto error; + } + } else { + ptable->valid_column = item->valueint; } - ptable->valid_column = item->valueint; return ptable; error: @@ -322,6 +348,17 @@ error: return NULL; } +void maat_table_runtime_free(void *runtime, enum table_type table_type) +{ + if (NULL == runtime) { + return; + } + + if (table_ops[table_type].free_runtime != NULL) { + table_ops[table_type].free_runtime(runtime); + } +} + void maat_table_free(struct maat_table *maat_tbl) { if (NULL == maat_tbl) { @@ -329,12 +366,12 @@ void maat_table_free(struct maat_table *maat_tbl) } if (maat_tbl->schema != NULL) { - maat_table_schema_free(maat_tbl->schema); + maat_table_schema_free(maat_tbl->schema, maat_tbl->table_type); maat_tbl->schema = NULL; } if (maat_tbl->runtime != NULL) { - maat_table_runtime_free(maat_tbl->runtime); + maat_table_runtime_free(maat_tbl->runtime, maat_tbl->table_type); maat_tbl->runtime = NULL; } @@ -400,9 +437,9 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch continue; } - maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type); + maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type, logger); if (NULL == maat_tbl->schema) { - log_error(logger, MODULE_TABLE, "Maat table schema new failed, table_name:%d", + log_error(logger, MODULE_TABLE, "Maat table schema new failed, table_name:%s", maat_tbl->table_name); maat_table_free(maat_tbl); continue; @@ -431,89 +468,20 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch } void *maat_table_runtime_new(void *schema, enum table_type table_type, - struct maat_garbage_bin *garbage_bin, + int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { void *runtime = NULL; if (table_ops[table_type].new_runtime != NULL) { - runtime = table_ops[table_type].new_runtime(schema, garbage_bin, logger); + runtime = table_ops[table_type].new_runtime(schema, max_thread_num, garbage_bin, logger); } return runtime; - - #if 0 - switch (table_rt->table_type) { - case TABLE_TYPE_COMPILE: - break; - case TABLE_TYPE_GROUP2COMPILE: - break; - case TABLE_TYPE_GROUP2GROUP: - table_rt->custom_rt = group2group_runtime_new(logger); - table_rt->g2g_rt.group_topo = maat_group_topology_new(logger); - break; - case TABLE_TYPE_EXPR: - table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free); - table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema); - break; - case TABLE_TYPE_IP_PLUS: - table_rt->ip_plus_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); - ex_container_ctx = ALLOC(struct ex_container_ctx, 1); - ex_container_ctx->custom_data_free = free; - //ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx); - break; - case TABLE_TYPE_PLUGIN: - table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); - break; - case TABLE_TYPE_IP_PLUGIN: - table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free); - ex_container_ctx = ALLOC(struct ex_container_ctx, 1); - ex_container_ctx->custom_data_free = free; - //ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx); - break; - default: - break; - } - #endif } - -void maat_table_runtime_free(void *runtime, enum table_type table_type) -{ - if (NULL == runtime) { - return; - } - - table_ops[table_type].free_runtime(runtime); -#if 0 - switch (table_rt->table_type) { - case TABLE_TYPE_COMPILE: - bool_matcher_free(table_rt->compile_rt.bm); - maat_compile_hash_free(&(table_rt->compile_rt.compile_hash)); - break; - case TABLE_TYPE_GROUP2COMPILE: - break; - case TABLE_TYPE_GROUP2GROUP: - maat_group_topology_free(table_rt->g2g_rt.group_topo); - break; - case TABLE_TYPE_EXPR: - adapter_hs_destroy(table_rt->expr_rt.hs); - rcu_hash_free(table_rt->expr_rt.htable); - break; - case TABLE_TYPE_PLUGIN: - ex_data_runtime_free(table_rt->plugin_rt.ex_data_rt); - break; - case TABLE_TYPE_IP_PLUGIN: - ip_matcher_free(table_rt->ip_plugin_rt.ip_matcher); - ex_data_runtime_free(table_rt->ip_plugin_rt.ex_data_rt); - break; - default: - break; - } -#endif -} - -int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *garbage_bin) +int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num, + struct maat_garbage_bin *garbage_bin) { if (NULL == tbl_mgr) { return -1; @@ -536,7 +504,9 @@ int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *g g2g_group_id = i; } - tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, garbage_bin, logger); + assert(NULL == tbl_mgr->tbl[i]->runtime); + tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, max_thread_num, + garbage_bin, tbl_mgr->logger); } assert(g2g_group_id != MAX_TABLE_NUM); @@ -544,7 +514,7 @@ int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *g /* group2compile runtime depends on associated compile runtime, must make sure associated compile runtime already exist */ for (i = 0; i < MAX_TABLE_NUM; i++) { - void *runtime = tbl_mgr->tbl[i]->runtime; + void *runtime = table_manager_get_runtime(tbl_mgr, i); if (NULL == runtime) { continue; } @@ -556,30 +526,30 @@ int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *g void *schema = table_manager_get_schema(tbl_mgr, i); - //int associated_compile_table_id = table_schema_get_associated_table_id(table_schema); - //TODO: by luis - int associated_compile_table_id = -1; + int associated_compile_table_id = group2compile_associated_compile_table_id(schema); void *compile_rt = table_manager_get_runtime(tbl_mgr, associated_compile_table_id); void *g2g_rt = table_manager_get_runtime(tbl_mgr, g2g_group_id); - - table_rt->g2c_rt.ref_compile_rt = &(compile_table_rt->compile_rt); - table_rt->g2c_rt.ref_g2g_rt = &(g2g_table_rt->g2g_rt); - assert(table_rt->g2c_rt.ref_compile_rt != NULL); - assert(table_rt->g2c_rt.ref_g2g_rt != NULL); + group2compile_runtime_init(runtime, compile_rt, g2g_rt); } return 0; } -void table_manager_deinit(struct table_manager *tbl_mgr) +void table_manager_runtime_destroy(struct table_manager *tbl_mgr) { if (NULL == tbl_mgr) { return; } for(size_t i = 0; i < MAX_TABLE_NUM; i++) { - maat_table_runtime_free(tbl_mgr->tbl[i]->runtime, tbl_mgr->tbl[i]->table_type); - tbl_mgr->tbl[i]->runtime = NULL; + void *runtime = table_manager_get_runtime(tbl_mgr, i); + if (NULL == runtime) { + continue; + } + + enum table_type table_type = table_manager_get_table_type(tbl_mgr, i); + maat_table_runtime_free(runtime, table_type); + tbl_mgr->tbl[i]->runtime = NULL; } } @@ -590,16 +560,25 @@ void table_manager_destroy(struct table_manager *tbl_mgr) } for (size_t i = 0; i < MAX_TABLE_NUM; i++) { - assert(NULL == tbl_mgr->tbl[i]->runtime); + void *runtime = table_manager_get_runtime(tbl_mgr, i); + assert(NULL == runtime); - if (NULL == tbl_mgr->tbl[i]->schema) { + void *schema = table_manager_get_schema(tbl_mgr, i); + if (NULL == schema) { continue; } - table_schema_free(tbl_mgr->tbl[i]->schema); - tbl_mgr->tbl[i]->schema = NULL; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, i); + maat_table_schema_free(schema, table_type); + tbl_mgr->tbl[i]->schema = NULL; } + for (size_t i = 0; i < tbl_mgr->n_accept_tag; i++) { + FREE(tbl_mgr->accept_tags[i].tag_name); + FREE(tbl_mgr->accept_tags[i].tag_val); + } + FREE(tbl_mgr->accept_tags); + maat_kv_store_free(tbl_mgr->tablename2id_map); FREE(tbl_mgr); } @@ -611,17 +590,36 @@ size_t table_manager_table_count(struct table_manager *tbl_mgr) int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) { + if (NULL == tbl_mgr || NULL == name) { + return -1; + } + int table_id = -1; + int ret = maat_kv_read(tbl_mgr->tablename2id_map, name, &table_id); + if (ret < 0) { + log_error(tbl_mgr->logger, MODULE_TABLE, "table:%s is not registered", name); + return -1; + } + + return table_id; } enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id) { + if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) { + return TABLE_TYPE_MAX; + } + if (NULL == tbl_mgr->tbl[table_id]) { + return TABLE_TYPE_MAX; + } + + return tbl_mgr->tbl[table_id]->table_type; } -int table_manager_get_logger(struct table_manager *tbl_mgr) +int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr) { - + return tbl_mgr->default_compile_table_id; } void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id) @@ -639,7 +637,7 @@ void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id) struct ex_data_schema *table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id) { - + return NULL; } int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id) @@ -657,7 +655,7 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id) int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags) { - + return 0; } void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id) @@ -666,8 +664,6 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id) return NULL; } - assert(table_id < (int)tbl_mgr->n_table); - if (NULL == tbl_mgr->tbl[table_id]) { return NULL; } @@ -675,6 +671,25 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id) return tbl_mgr->tbl[table_id]->runtime; } +int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id) +{ + if (NULL == tbl_mgr) { + return -1; + } + + if (NULL == tbl_mgr->tbl[table_id]) { + return -1; + } + + enum table_type table_type = tbl_mgr->tbl[table_id]->table_type; + void *runtime = table_manager_get_runtime(tbl_mgr, table_id); + if (table_ops[table_type].runtime_updating_flag != NULL) { + return table_ops[table_type].runtime_updating_flag(runtime); + } + + return -1; +} + int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line) { void *schema = table_manager_get_schema(tbl_mgr, table_id); @@ -702,45 +717,6 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, co } return table_ops[table_type].update_runtime(runtime, schema, line, valid_column); - - #if 0 - switch (table_rt->table_type) { - case TABLE_TYPE_COMPILE: - ret = compile_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, table_rt->ref_garbage_bin, logger); - break; - case TABLE_TYPE_GROUP2COMPILE: - ret = group2compile_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger); - break; - case TABLE_TYPE_GROUP2GROUP: - ret = group2group_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, logger); - break; - case TABLE_TYPE_EXPR: - ret = expr_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger); - break; - case TABLE_TYPE_IP_PLUS: - ret = ip_plus_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger); - break; - case TABLE_TYPE_PLUGIN: - ret = plugin_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, row, logger); - break; - case TABLE_TYPE_IP_PLUGIN: - ret = ip_plugin_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, row, logger); - break; - default: - break; - } - - if (ret < 0) { - return; - } - #endif -#if 0 - if (is_valid == 0) { - table_rt->rule_num--; - } else { - table_rt->rule_num++; - } -#endif } void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) @@ -755,5 +731,7 @@ void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) return; } - table_ops[table_type].commit_runtime(runtime); + if ( table_ops[table_type].commit_runtime != NULL) { + table_ops[table_type].commit_runtime(runtime); + } } \ No newline at end of file diff --git a/src/maat_virtual.cpp b/src/maat_virtual.cpp index 5941b6e..a9b9bf3 100644 --- a/src/maat_virtual.cpp +++ b/src/maat_virtual.cpp @@ -23,14 +23,16 @@ struct virtual_schema { void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) { - size_t read_cnt = 0; + //size_t read_cnt = 0; cJSON *item = cJSON_GetObjectItem(json, "physical_table"); if (NULL == item || item->type != cJSON_Array) { log_error(logger, MODULE_VIRTUAL, "virtual table %s has no physical_table column", table_name); return NULL; } - read_cnt++; + + struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); + return vt_schema; #if 0 struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); int cnt = cJSON_GetArraySize(item); @@ -57,3 +59,8 @@ void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle return vt_schema; #endif } + +void virtual_schema_free(void *virtual_schema) +{ + FREE(virtual_schema); +} \ No newline at end of file diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index caf0acc..99a2b06 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -75,14 +75,14 @@ TEST(maat_scan_ipv4, hit_ip_and_port) { int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 7); maat_state_free(&state); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 0); @@ -92,15 +92,14 @@ TEST(maat_scan_ipv4, hit_ip_and_port) { TEST(maat_scan_ipv4, hit_ip_and_port_range) { int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "192.168.50.24"; - struct addr_2tuple addr; - addr.type = IP_TYPE_V4; - int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip); + uint32_t sip; + int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 4); @@ -109,7 +108,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) { memset(results, 0, sizeof(results)); n_hit_result = 0; state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 4); @@ -118,7 +117,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) { memset(results, 0, sizeof(results)); n_hit_result = 0; state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 0); } @@ -128,33 +127,32 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { char ip_str1[32] = "10.0.1.20"; char ip_str2[32] = "10.0.1.25"; char ip_str3[32] = "10.0.1.26"; - struct addr_2tuple addr; - addr.type = IP_TYPE_V4; - int ret = inet_pton(AF_INET, ip_str1, &addr.ipv4.sip); + uint32_t sip; + int ret = inet_pton(AF_INET, ip_str1, &sip); EXPECT_EQ(ret, 1); int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 8); maat_state_free(&state); - ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); + ret = inet_pton(AF_INET, ip_str2, &sip); EXPECT_EQ(ret, 1); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 8); maat_state_free(&state); - ret = inet_pton(AF_INET, ip_str3, &addr.ipv4.sip); + ret = inet_pton(AF_INET, ip_str3, &sip); EXPECT_EQ(ret, 1); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); @@ -164,24 +162,23 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) { int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str1[32] = "192.168.0.1"; char ip_str2[32] = "192.168.0.0"; - struct addr_2tuple addr; - addr.type = IP_TYPE_V4; - int ret = inet_pton(AF_INET, ip_str1, &addr.ipv4.sip); + uint32_t sip; + int ret = inet_pton(AF_INET, ip_str1, &sip); EXPECT_EQ(ret, 1); int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 50); maat_state_free(&state); - ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); + ret = inet_pton(AF_INET, ip_str2, &sip); EXPECT_EQ(ret, 1); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); @@ -190,15 +187,14 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) { TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "192.168.40.10"; - struct addr_2tuple addr; - addr.type = IP_TYPE_V4; - int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip); + uint32_t sip; + int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 2); EXPECT_EQ(results[0], 63); @@ -206,7 +202,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { maat_state_free(&state); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); @@ -215,22 +211,21 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) { int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "1001:da8:205:1::101"; - struct addr_2tuple addr; - addr.type = IP_TYPE_V6; - int ret = inet_pton(AF_INET6, ip_str, &addr.ipv6.sip); + uint8_t sip6[128]; + int ret = inet_pton(AF_INET6, ip_str, sip6); EXPECT_EQ(ret, 1); int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 47); maat_state_free(&state); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); @@ -270,15 +265,14 @@ TEST(maat_scan_ip, dynamic_config) { int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "10.0.6.201"; - struct addr_2tuple addr; - addr.type = IP_TYPE_V4; - int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip); + uint32_t sip; + int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); @@ -294,7 +288,7 @@ TEST(maat_scan_ip, dynamic_config) { sleep(2); state = NULL; - ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, 0); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 9998); diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 456bc85..fab625f 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -3,6 +3,7 @@ #include "maat_utils.h" #include "maat_rule.h" #include "json2iris.h" +#include "maat_table.h" #include "maat_config_monitor.h" #include @@ -37,7 +38,7 @@ TEST(json_mode, maat_scan_string) { size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 191); @@ -80,7 +81,7 @@ TEST(iris_mode, maat_scan_string) { size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 191); @@ -193,14 +194,14 @@ TEST(redis_mode, maat_scan_string) { maat_options_set_logger(opts, logger); struct maat *maat_instance = maat_new(opts, table_info_path); - int table_id = table_manager_get_table_id(maat_instance->tbl_mgr "HTTP_URL"); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL"); char scan_data[128] = "hello"; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 191); diff --git a/test/table_info.conf b/test/table_info.conf index a653496..643631a 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -20,9 +20,9 @@ "table_id":2, "table_name":"GROUP2COMPILE", "table_type":"group2compile", + "associated_compile_table_id":1, "valid_column":3, "custom": { - "associated_compile_table_id":1, "group_id":1, "compile_id":2, "not_flag":4,