gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor hierarchy and maat_table

Browse Source
This commit is contained in:
liuwentan
2023-01-31 20:39:53 +08:00
parent 25f944a1d1
commit cca7d882e1
29 changed files with 1087 additions and 1107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
deps/log/log.h vendored
View File

@@ -17,6 +17,7 @@ extern "C"
#include <stdarg.h> #include <stdarg.h>
#include <stdbool.h> #include <stdbool.h>
#include <time.h> #include <time.h>
#include <pthread.h>
#define LOG_VERSION "0.1.0" #define LOG_VERSION "0.1.0"

2
src/inc_internal/maat_common.h
View File

@@ -25,7 +25,7 @@ struct maat_options {
char instance_name[NAME_MAX]; char instance_name[NAME_MAX];
size_t nr_worker_threads; size_t nr_worker_threads;
const char *accept_tags; char *accept_tags;
int rule_effect_interval_ms; int rule_effect_interval_ms;
int rule_update_checking_interval_ms; int rule_update_checking_interval_ms;

16
src/inc_internal/maat_compile.h
View File

@@ -18,7 +18,7 @@ extern "C"
#include "log/log.h" #include "log/log.h"
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
#include "maat.h" #include "maat/maat.h"
#include "maat_kv.h" #include "maat_kv.h"
#include "maat_rule.h" #include "maat_rule.h"
@@ -38,6 +38,7 @@ void compile_schema_free(void *compile_schema);
void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void group2compile_schema_free(void *g2c_schema); void group2compile_schema_free(void *g2c_schema);
int group2compile_associated_compile_table_id(void *g2c_schema);
int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id, int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id,
maat_rule_ex_new_func_t *new_func, maat_rule_ex_new_func_t *new_func,
@@ -51,7 +52,7 @@ compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, siz
size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema); size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema);
/* compile runtime API */ /* compile runtime API */
void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage_bin, void *compile_runtime_new(void *compile_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void compile_runtime_free(void *compile_runtime); void compile_runtime_free(void *compile_runtime);
@@ -60,14 +61,15 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, const ch
int compile_runtime_commit(void *compile_runtime); int compile_runtime_commit(void *compile_runtime);
int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids, int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids,
int *compile_ids, size_t compile_ids_size, struct maat_state *state); int vt_id, int *compile_ids, size_t compile_ids_size, struct maat_state *state);
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct maat_group_topology *group_topo, size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct group2group_runtime *g2g_rt,
struct maat_compile_state *compile_state, struct maat_compile_state *compile_state,
struct maat_hit_path *hit_paths, size_t hit_path_siz); struct maat_hit_path *hit_paths, size_t hit_path_siz);
/* group2compile runtime API */ /* group2compile runtime API */
void *group2compile_runtime_new(void *g2c_schema, struct maat_garbage_bin *garbage_bin, void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime);
void group2compile_runtime_free(void *g2c_runtime); void group2compile_runtime_free(void *g2c_runtime);
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line, int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line,
@@ -79,7 +81,7 @@ struct maat_compile_state *maat_compile_state_new(int thread_id);
void maat_compile_state_free(struct maat_compile_state *compile_state); void maat_compile_state_free(struct maat_compile_state *compile_state);
void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id, void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id,
int Nth_scan, int Nth_item_result); int Nth_scan, int Nth_item_result);
void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, struct maat_compile **compile_hash, void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, void *compile_runtime,
int group_id, int virtual_table_id); int group_id, int virtual_table_id);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);

3
src/inc_internal/maat_ex_data.h
View File

@@ -33,7 +33,8 @@ struct ex_container_ctx {
struct ex_data_runtime; struct ex_data_runtime;
/* ex_data_runtime API */ /* ex_data_runtime API */
struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn); struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn,
struct log_handle *logger);
void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt); void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt);

7
src/inc_internal/maat_expr.h
View File

@@ -25,10 +25,11 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo
void expr_schema_free(void *expr_schema); void expr_schema_free(void *expr_schema);
/* expr runtime API */ /* expr runtime API */
void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void expr_runtime_free(void *expr_runtime); void expr_runtime_free(void *expr_runtime);
int expr_runtime_updating_flag(struct expr_runtime *expr_rt); int expr_runtime_updating_flag(void *expr_runtime);
int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column); int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column);
int expr_runtime_commit(void *expr_runtime); int expr_runtime_commit(void *expr_runtime);
@@ -38,7 +39,7 @@ int expr_runtime_commit(void *expr_runtime);
* *
* @retval the num of hit group_id * @retval the num of hit group_id
*/ */
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, int expr_runtime_scan_string(struct expr_runtime *expr_runtime, int thread_id,
const char *data, size_t data_len, int group_ids[], const char *data, size_t data_len, int group_ids[],
size_t max_hit_num, int virtual_table_id, size_t max_hit_num, int virtual_table_id,
struct maat_state *state); struct maat_state *state);

12
src/inc_internal/maat_garbage_collection.h
View File

@@ -20,17 +20,17 @@ extern "C"
struct maat_garbage_bin; struct maat_garbage_bin;
struct maat_garbage_bin* maat_garbage_bin_new(int default_timeout); struct maat_garbage_bin *maat_garbage_bin_new(int default_timeout);
void maat_garbage_bin_free(struct maat_garbage_bin* bin); void maat_garbage_bin_free(struct maat_garbage_bin *bin);
void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void (* func)(void *)); void maat_garbage_bagging(struct maat_garbage_bin *bin, void *garbage, void (* func)(void *));
void maat_garbage_collect_routine(struct maat_garbage_bin* bin); void maat_garbage_collect_routine(struct maat_garbage_bin *bin);
size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin); size_t maat_garbage_bin_get_size(struct maat_garbage_bin *bin);
void maat_garbage_collect_by_force(struct maat_garbage_bin* bin); void maat_garbage_collect_by_force(struct maat_garbage_bin *bin);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

33
src/inc_internal/maat_group.h
View File

@@ -37,22 +37,20 @@ struct maat_group {
/* maat group topology API */ /* maat group topology API */
struct maat_group_topology; struct maat_group_topology;
struct maat_group_topology *maat_group_topology_new(struct log_handle *logger);
void maat_group_topology_free(struct maat_group_topology *group_topo);
struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id);
void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group);
/** /**
* @retval if not found, return NULL * @retval if not found, return NULL
*/ */
struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id);
int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id); //struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id);
//struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id);
//void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group);
//int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
//int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id);
/* build top groups */ /* build top groups */
int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo); //int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo);
/* group2group schema API */ /* group2group schema API */
@@ -60,15 +58,24 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han
void group2group_schema_free(void *g2g_schema); void group2group_schema_free(void *g2g_schema);
/* group2group runtime API */ /* group2group runtime API */
struct group2group_runtime * void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
struct log_handle *logger);
void group2group_runtime_free(void *g2g_runtime); void group2group_runtime_free(void *g2g_runtime);
struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id);
void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group);
struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id);
int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id);
int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id);
int group2group_runtime_build_top_groups(void *g2g_runtime);
int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column); int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column);
int group2group_runtime_commit(void *g2g_runtime); int group2group_runtime_commit(void *g2g_runtime);
int group2group_runtime_get_top_groups(struct group2group_runtime *g2g_rt, int *group_ids, size_t n_group_ids, int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids,
int *top_group_ids); int *top_group_ids);
#ifdef __cpluscplus #ifdef __cpluscplus

13
src/inc_internal/maat_ip.h
View File

@@ -22,18 +22,21 @@ void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle
void ip_plus_schema_free(void *ip_plus_schema); void ip_plus_schema_free(void *ip_plus_schema);
/* ip plus runtime API */ /* ip plus runtime API */
void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void ip_plus_runtime_free(void *ip_plus_runtime); void ip_plus_runtime_free(void *ip_plus_runtime);
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line); int ip_plus_runtime_updating_flag(void *ip_plus_runtime);
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line,
int valid_column);
int ip_plus_runtime_commit(void *ip_plus_runtime); int ip_plus_runtime_commit(void *ip_plus_runtime);
struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt); struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt);
/* ip runtime scan API */ /* ip runtime scan API */
int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *data, int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr,
int *group_id_array, size_t n_group_id_array, int virtual_table_id, int *group_id_array, size_t n_group_id_array, int virtual_table_id,
struct maat_state *state); struct maat_state *state);
#ifdef __cpluscplus #ifdef __cpluscplus
} }

14
src/inc_internal/maat_ip_plugin.h
View File

@@ -16,6 +16,10 @@ extern "C"
{ {
#endif #endif
#include "log/log.h"
#include "maat/maat.h"
#include "cJSON/cJSON.h"
struct ip_plugin_runtime; struct ip_plugin_runtime;
/* ip plugin schema API */ /* ip plugin schema API */
@@ -29,15 +33,17 @@ int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func, maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp); long argl, void *argp,
struct log_handle *logger);
/* ip plugin runtime API */ /* ip plugin runtime API */
void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *garbage_bin, void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger); struct log_handle *logger);
void ip_plugin_runtime_free(void *ip_plugin_runtime); void ip_plugin_runtime_free(void *ip_plugin_runtime);
int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt); int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime);
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line); int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line,
int valid_column);
int ip_plugin_runtime_commit(void *ip_plugin_runtime); int ip_plugin_runtime_commit(void *ip_plugin_runtime);
struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime); struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime);

18
src/inc_internal/maat_plugin.h
View File

@@ -16,6 +16,8 @@ extern "C"
{ {
#endif #endif
#include "maat/maat.h"
#define MAX_FOREIGN_CLMN_NUM 8 #define MAX_FOREIGN_CLMN_NUM 8
/* plugin schema API */ /* plugin schema API */
@@ -40,19 +42,23 @@ int plugin_table_set_ex_data_schema(void *custom_schema,
maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func, maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp); long argl, void *argp,
struct log_handle *logger);
struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema); struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema);
/* plugin runtime API */ /* plugin runtime API */
struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime); void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void plugin_runtime_free(void *plugin_runtime);
int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt); int plugin_runtime_updating_flag(void *plugin_runtime);
int plugin_runtime_update(struct plugin_runtime *plugin_rt, struct plugin_item *plugin_item, int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line,
struct plugin_schema *plugin_schema, const char *row, int valid_column);
struct log_handle *logger);
int plugin_runtime_commit(void *plugin_runtime); int plugin_runtime_commit(void *plugin_runtime);
struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime);
#ifdef __cpluscplus #ifdef __cpluscplus
} }
#endif #endif

21
src/inc_internal/maat_rule.h
View File

@@ -80,7 +80,6 @@ struct compile_rule {
int compile_id; int compile_id;
struct maat_rule_head head;// fix len of Maat_rule_t struct maat_rule_head head;// fix len of Maat_rule_t
char *service_defined; char *service_defined;
int is_valid;
int declared_clause_num; int declared_clause_num;
double evaluation_order; double evaluation_order;
struct compile_schema *ref_table; struct compile_schema *ref_table;
@@ -88,20 +87,9 @@ struct compile_rule {
pthread_rwlock_t rwlock; pthread_rwlock_t rwlock;
}; };
struct group2compile_rule {
int group_id;
int compile_id;
int is_valid;
int not_flag;
int vt_id; //virtual_table_id
int clause_index;
int associated_compile_table_id;
};
struct group2group_rule { struct group2group_rule {
int group_id; int group_id;
int superior_group_id; int superior_group_id;
int is_valid;
}; };
struct maat_runtime { struct maat_runtime {
@@ -114,7 +102,6 @@ struct maat_runtime {
struct table_manager *ref_tbl_mgr; //share with maat instance struct table_manager *ref_tbl_mgr; //share with maat instance
size_t max_table_num; size_t max_table_num;
size_t max_thread_num;
uint32_t rule_num; uint32_t rule_num;
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
@@ -254,9 +241,9 @@ struct maat_state {
struct maat_compile_state *compile_mid; struct maat_compile_state *compile_mid;
}; };
int parse_accept_tag(const char *value, struct rule_tag **result, void *logger); size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger);
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag); int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag);
struct maat_item *maat_item_new(int item_id, int group_id, void *user_data); struct maat_item *maat_item_new(int item_id, int group_id, void *user_data);
@@ -264,7 +251,7 @@ void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)
struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id); struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id);
void maat_item_inner_free(struct maat_item_inner *item); void maat_item_inner_free(void *item_inner);
void maat_start_cb(long long new_version, int update_type, void *u_para); void maat_start_cb(long long new_version, int update_type, void *u_para);
@@ -297,7 +284,7 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri
void maat_cmd_clear_rule_cache(struct serial_rule *s_rule); void maat_cmd_clear_rule_cache(struct serial_rule *s_rule);
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version,
long long *new_version, struct table_schema_manager* table_schema_mgr, long long *new_version, struct table_manager* tbl_mgr,
struct serial_rule **list, int *update_type, int cumulative_off, struct serial_rule **list, int *update_type, int cumulative_off,
struct log_handle *logger); struct log_handle *logger);

11
src/inc_internal/maat_table.h
View File

@@ -45,20 +45,23 @@ struct table_manager;
struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags, struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags,
struct log_handle *logger); struct log_handle *logger);
int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *garbage_bin); int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num,
int table_manager_deinit(struct table_manager *tbl_mgr); struct maat_garbage_bin *garbage_bin);
void table_manager_runtime_destroy(struct table_manager *tbl_mgr);
void table_manager_destroy(struct table_manager *tbl_mgr); void table_manager_destroy(struct table_manager *tbl_mgr);
size_t table_manager_table_count(struct table_manager *tbl_mgr); size_t table_manager_table_count(struct table_manager *tbl_mgr);
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name); int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name);
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id); enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id);
int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr);
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id); int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
int table_manager_accept_tags_match(const char *tags); int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags);
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);
void table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line); int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id);
int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line);
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id); void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id);
#ifdef __cpluscplus #ifdef __cpluscplus

2
src/inc_internal/maat_virtual.h
View File

@@ -31,6 +31,8 @@ enum scan_type {
void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger);
void virtual_schema_free(void *virtual_schema);
#ifdef __cpluscplus #ifdef __cpluscplus
} }
#endif #endif

299
src/maat_api.cpp
View File

@@ -26,6 +26,10 @@
#include "maat_redis_monitor.h" #include "maat_redis_monitor.h"
#include "maat_compile.h" #include "maat_compile.h"
#include "alignment.h" #include "alignment.h"
#include "maat_garbage_collection.h"
#include "maat_group.h"
#include "maat_expr.h"
#include "maat_ip.h"
#include "maat_plugin.h" #include "maat_plugin.h"
#include "maat_ip_plugin.h" #include "maat_ip_plugin.h"
@@ -34,16 +38,6 @@
#define DISTRICT_ANY -1 #define DISTRICT_ANY -1
#define DISTRICT_UNKNOWN -2 #define DISTRICT_UNKNOWN -2
struct scan_item_hit_wrapper {
int Nth_scan;
struct maat_item_inner* wrapped_items[MAX_SCANNER_HIT_COMPILE_NUM];
size_t n_wrapped_item;
int *virtual_table_ids;
int virtual_table_id;
int is_last_item;
};
inline int scan_state_should_compile_NOT(struct maat_state *mid) inline int scan_state_should_compile_NOT(struct maat_state *mid)
{ {
if (mid && mid->is_last_scan==1 && mid->compile_mid && if (mid && mid->is_last_scan==1 && mid->compile_mid &&
@@ -260,9 +254,8 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
if (NULL == maat_instance->tbl_mgr) { if (NULL == maat_instance->tbl_mgr) {
goto failed; goto failed;
} }
//TODO: by luis
//maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr); maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
maat_instance->default_compile_table_id = -1;
maat_instance->input_mode = opts->input_mode; maat_instance->input_mode = opts->input_mode;
switch (maat_instance->input_mode) { switch (maat_instance->input_mode) {
@@ -353,12 +346,12 @@ void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_he
size_t generic_plugin_runtime_cached_row_count(void *custom_rt, enum table_type table_type) size_t generic_plugin_runtime_cached_row_count(void *custom_rt, enum table_type table_type)
{ {
return 0;
} }
const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type, size_t row_id) const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type, size_t row_id)
{ {
return NULL;
} }
/* must be plugin table */ /* must be plugin table */
@@ -393,7 +386,7 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id,
start(MAAT_RULE_UPDATE_TYPE_FULL, u_para); start(MAAT_RULE_UPDATE_TYPE_FULL, u_para);
} }
for (size_t i = 0; i < row_count; i++) { for (size_t i = 0; i < row_cnt; i++) {
const char *line = generic_plugin_runtime_get_cached_row(runtime, table_type, i); const char *line = generic_plugin_runtime_get_cached_row(runtime, table_type, i);
if (NULL == line) { if (NULL == line) {
break; break;
@@ -416,18 +409,18 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t
maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func, maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp) long argl, void *argp, struct log_handle *logger)
{ {
if (NULL == tbl_mgr || NULL == new_func || NULL == free_func || NULL == dup_func) { if (NULL == tbl_mgr || NULL == new_func || NULL == free_func || NULL == dup_func) {
assert(0); assert(0);
log_error(tbl_mgr->logger, MODULE_MAAT_API, log_error(logger, MODULE_MAAT_API,
"table(table_id:%d) %s failed: invalid parameter", __FUNCTION__); "table(table_id:%d) %s failed: invalid parameter", __FUNCTION__);
return -1; return -1;
} }
void *schema = table_manager_get_schema(tbl_mgr, table_id); void *schema = table_manager_get_schema(tbl_mgr, table_id);
if (NULL == schema) { if (NULL == schema) {
log_error(tbl_mgr->logger, MODULE_MAAT_API, log_error(logger, MODULE_MAAT_API,
"Error: %s, table(table_id:%d) is not registered", __FUNCTION__, table_id); "Error: %s, table(table_id:%d) is not registered", __FUNCTION__, table_id);
return -1; return -1;
} }
@@ -438,24 +431,24 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
ex_schema = plugin_table_get_ex_data_schema(schema); ex_schema = plugin_table_get_ex_data_schema(schema);
if (NULL == ex_schema) { if (NULL == ex_schema) {
log_error(tbl_mgr->logger, MODULE_MAAT_API, log_error(logger, MODULE_MAAT_API,
"Error: %s, table(table_id:%d) is not a valid plugin table", "Error: %s, table(table_id:%d) is not a valid plugin table",
__FUNCTION__, table_id); __FUNCTION__, table_id);
return -1; return -1;
} }
plugin_table_set_ex_data_schema(schema, new_func, free_func, plugin_table_set_ex_data_schema(schema, new_func, free_func,
dup_func, argl, argp, tbl_mgr->logger); dup_func, argl, argp, logger);
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
ex_schema = ip_plugin_table_get_ex_data_schema(schema); ex_schema = ip_plugin_table_get_ex_data_schema(schema);
if (NULL == ex_schema) { if (NULL == ex_schema) {
log_error(tbl_mgr->logger, MODULE_MAAT_API, log_error(logger, MODULE_MAAT_API,
"Error: %s, table(table_id:%d) is not a valid ip_plugin table", "Error: %s, table(table_id:%d) is not a valid ip_plugin table",
__FUNCTION__, table_id); __FUNCTION__, table_id);
return -1; return -1;
} }
ip_plugin_table_set_ex_data_schema(schema, new_func, free_func, ip_plugin_table_set_ex_data_schema(schema, new_func, free_func,
dup_func, argl, argp); dup_func, argl, argp, logger);
break; break;
default: default:
break; break;
@@ -464,8 +457,8 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t
return 0; return 0;
} }
void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, enum table_type table_type, void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, int table_id,
int nr_worker_thread, long long version, struct log_handle *logger) enum table_type table_type, int valid_column)
{ {
struct ex_data_schema *ex_data_schema = NULL; struct ex_data_schema *ex_data_schema = NULL;
struct ex_data_runtime *ex_data_rt = NULL; struct ex_data_runtime *ex_data_rt = NULL;
@@ -484,21 +477,20 @@ void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, enum t
} }
ex_data_runtime_set_schema(ex_data_rt, ex_data_schema); ex_data_runtime_set_schema(ex_data_rt, ex_data_schema);
struct ex_container_ctx *ctx = ALLOC(struct ex_container_ctx, 1); struct ex_container_ctx *ex_ctx = ALLOC(struct ex_container_ctx, 1);
ex_ctx->table_id = table_id;
ctx->table_id = table_schema_get_table_id(custom_schema); ex_ctx->ex_schema = ex_data_schema;
ctx->ex_schema = ex_data_schema; ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_ctx);
ex_data_runtime_set_ex_container_ctx(ex_data_rt, ctx);
size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt); size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt);
for (size_t i = 0; i < n_cached_row; i++) { for (size_t i = 0; i < n_cached_row; i++) {
const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i); const char *row = ex_data_runtime_cached_row_get(ex_data_rt, i);
switch (table_rt->table_type) { switch (table_type) {
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
plugin_runtime_update_row(custom_rt, custom_schema, row, NULL, 0, 1); plugin_runtime_update(runtime, schema, row, valid_column);
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_update_row(custom_rt, custom_schema, NULL, NULL, 0, NULL, 1); ip_plugin_runtime_update(runtime, schema, row, valid_column);
break; break;
default: default:
break; break;
@@ -508,10 +500,10 @@ void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, enum t
switch (table_type) { switch (table_type) {
case TABLE_TYPE_PLUGIN: case TABLE_TYPE_PLUGIN:
plugin_runtime_commit(custom_rt); plugin_runtime_commit(runtime);
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
ip_plugin_runtime_commit(custom_rt); ip_plugin_runtime_commit(runtime);
break; break;
default: default:
break; break;
@@ -526,7 +518,8 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
{ {
pthread_mutex_lock(&(maat_instance->background_update_mutex)); pthread_mutex_lock(&(maat_instance->background_update_mutex));
int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id, int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id,
new_func, free_func, dup_func, argl, argp); new_func, free_func, dup_func, argl, argp,
maat_instance->logger);
if (ret < 0) { if (ret < 0) {
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
return -1; return -1;
@@ -536,8 +529,8 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
generic_plugin_runtime_commit_ex_schema(runtime, schema, table_type, maat_instance->nr_worker_thread, int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
maat_instance->maat_rt->version, maat_instance->logger); generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type, valid_column);
} }
pthread_mutex_unlock(&(maat_instance->background_update_mutex)); pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -620,26 +613,29 @@ struct maat_state *grab_state(struct maat_state **state, struct maat *maat_insta
return mid; return mid;
} }
void scan_item_hit_wrapper_build(struct scan_item_hit_wrapper* wrapper, struct scan_result* results, size_t n_result, int hit_group_to_compile(void *g2g_runtime, int *group_ids, int group_hit_cnt, int vt_id,
int district_id, int is_last_item, int virtual_table_id, int Nth_scan) void *compile_runtime, int *compile_ids, size_t n_compile_id,
size_t *n_hit_compile_id, struct maat_state *mid)
{ {
size_t i=0; int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
struct maat_item_inner *item = NULL; int n_top_group_ids = group2group_runtime_get_top_groups(g2g_runtime, group_ids, group_hit_cnt, top_group_ids);
wrapper->n_wrapped_item = 0; size_t n_all_group_ids = 0;
wrapper->virtual_table_id = 0; if (n_top_group_ids > 0) {
wrapper->virtual_table_ids = NULL; n_all_group_ids = group_hit_cnt + n_top_group_ids;
} else {
n_all_group_ids = group_hit_cnt;
}
for (i = 0; i < n_result; i++) { int all_group_ids[n_all_group_ids] = {-1};
item = (struct maat_item_inner *)(results[i].tag); // maat state find compile_table_id, if not found, maat_instance->default_compile_table_id
if (item->district_id == district_id || district_id == DISTRICT_ANY) { int n_compile_ids = compile_runtime_match((struct compile_runtime *)compile_runtime, all_group_ids, n_all_group_ids,
wrapper->wrapped_items[wrapper->n_wrapped_item] = item; vt_id, compile_ids, n_compile_id, mid);
wrapper->n_wrapped_item++; *n_hit_compile_id = n_compile_ids;
} if (n_compile_ids > 0) {
} return MAAT_HIT;
wrapper->is_last_item = is_last_item; } else {
wrapper->virtual_table_id = virtual_table_id; return MAAT_HALF_HIT;
wrapper->Nth_scan = Nth_scan; }
wrapper->virtual_table_ids = NULL;
} }
int maat_scan_integer(struct maat *instance, int table_id, int thread_id, int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
@@ -649,57 +645,12 @@ int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
return 0; return 0;
} }
static int ip_scan_data_set(struct ip_addr *scan_data, const char *ip_addr)
{
return 0;
}
static int ip_composition_scan(int thread_id, const char *ip_addr,
int parent_table_id, int *virtual_table_id,
struct table_schema_manager *table_schema_mgr,
struct table_runtime_manager *table_rt_mgr,
int *group_id_array, size_t n_group_id_array,
struct maat_state *state)
{
int child_table_id = 0;
struct table_schema *real_table = table_schema_get_by_scan_type(table_schema_mgr, child_table_id, SCAN_TYPE_IP, virtual_table_id);
if (NULL == real_table) {
return 0;
}
enum table_type table_type = table_schema_get_table_type(real_table);
if (table_type != TABLE_TYPE_IP_PLUS) {
return -1;
}
int table_id = table_schema_get_table_id(real_table);
struct table_runtime *table_rt = table_manager_get_runtime(table_rt_mgr, table_id);
size_t rule_num = table_runtime_rule_count(table_rt);
if (0 == rule_num) {
return 0;
}
struct ip_addr scan_data;
memset(&scan_data, 0, sizeof(struct ip_addr));
ip_scan_data_set(&scan_data, ip_addr);
size_t hit_group_cnt = 0;
int ret = table_runtime_scan_ip(table_rt, thread_id, &scan_data, group_id_array, n_group_id_array, *virtual_table_id, state);
if (ret < 0) {
return -1;
}
return hit_group_cnt;
}
int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
uint32_t ip_addr, int *results, size_t n_result, uint32_t ip_addr, int *results, size_t n_result,
size_t *n_hit_result, struct maat_state **state) size_t *n_hit_result, struct maat_state **state)
{ {
if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) ||
(thread_id < 0) || (NULL == ip_addr) || (NULL == results) || (0 == n_result) || (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) {
(NULL == state)) {
return -1; return -1;
} }
@@ -708,52 +659,36 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
scan_count_inc(mid); scan_count_inc(mid);
int virtual_table_id = 0; int virtual_table_id = 0;
struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
SCAN_TYPE_IP, &virtual_table_id); // SCAN_TYPE_IP, &virtual_table_id);
if (NULL == table_schema) { // if (NULL == table_schema) {
return MAAT_ERR; // return MAAT_ERR;
} // }
struct maat_runtime *maat_rt = maat_instance->maat_rt; int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int group_result[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int group_hit_cnt = 0;
int group_result_virtual_table_ids[MAX_SCANNER_HIT_GROUP_NUM];
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
int group_ret = -1; assert(table_type == TABLE_TYPE_IP_PLUS);
group_ret = ip_composition_scan(thread_id, ip_addr, table_id, &virtual_table_id, // int table_id = table_schema_get_table_id(real_table);
maat_instance->table_schema_mgr, maat_instance->maat_rt->table_rt_mgr, void *ip_plus_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
group_result + group_hit_cnt, MAX_SCANNER_HIT_COMPILE_NUM - group_hit_cnt, mid); // size_t rule_num = table_runtime_rule_count(table_rt);
if (group_ret < 0) { // if (0 == rule_num) {
maat_instance->scan_err_cnt++; // return 0;
} else if (0 == group_ret) { // }
int group_hit_cnt = ip_plus_runtime_scan_ipv4((struct ip_plus_runtime *)ip_plus_rt, thread_id, ip_addr,
group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid);
if (group_hit_cnt < 0) {
return MAAT_ERR;
} else if (0 == group_hit_cnt) {
return MAAT_OK; return MAAT_OK;
} else { } else {
group_hit_cnt += group_ret; // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT
} void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
// come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime,
results, n_result, n_hit_result, mid);
struct table_runtime *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); return ret;
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int n_top_group_ids = group2group_runtime_get_top_groups(g2g_rt, group_result, group_hit_cnt, top_group_ids);
size_t n_all_group_ids = 0;
if (n_top_group_ids > 0) {
n_all_group_ids = group_hit_cnt + n_top_group_ids;
} else {
n_all_group_ids = group_hit_cnt;
}
int all_group_ids[n_all_group_ids] = {-1};
// maat state find compile_table_id, if not found, maat_instance->default_compile_table_id
struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
int n_compile_ids = compile_runtime_match(compile_table_rt, all_group_ids, n_all_group_ids, results, n_result, mid);
*n_hit_result = n_compile_ids;
if (n_compile_ids > 0) {
return MAAT_HIT;
} else {
return MAAT_HALF_HIT;
} }
} }
@@ -780,53 +715,29 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
scan_count_inc(mid); scan_count_inc(mid);
int virtual_table_id = 0; int virtual_table_id = 0;
struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, //TODO: by luis get virtual_table_id
SCAN_TYPE_STRING, &virtual_table_id); // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
if (NULL == table_schema) { // SCAN_TYPE_STRING, &virtual_table_id);
return MAAT_ERR; // if (NULL == table_schema) {
} // return MAAT_ERR;
// }
struct table_runtime *table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int group_hit_cnt = table_runtime_scan_string(table_rt, thread_id, data, data_len, group_ids, MAX_SCANNER_HIT_GROUP_NUM, void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
virtual_table_id, mid); int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)runtime, thread_id, data, data_len,
if (group_hit_cnt <= 0) { group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid);
if (group_hit_cnt < 0) {
return MAAT_ERR;
} else if (0 == group_hit_cnt) {
return MAAT_OK; return MAAT_OK;
}
// come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT
struct table_runtime *g2g_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int n_top_group_ids = group2group_runtime_get_top_groups(g2g_rt, group_ids, group_hit_cnt, top_group_ids);
size_t n_all_group_ids = 0;
if (n_top_group_ids > 0) {
n_all_group_ids = group_hit_cnt + n_top_group_ids;
} else { } else {
n_all_group_ids = group_hit_cnt; // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime,
results, n_result, n_hit_result, mid);
return ret;
} }
int all_group_ids[n_all_group_ids] = {-1};
int i = 0, j = 0, k = 0;
for (i = 0; i < group_hit_cnt; i++) {
all_group_ids[i] = group_ids[i];
}
for (j = i; j < n_all_group_ids; j++, k++) {
all_group_ids[j] = top_group_ids[k];
}
// maat state find compile_table_id, if not found, maat_instance->default_compile_table_id
struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id);
int n_compile_ids = compile_runtime_match(compile_table_rt, all_group_ids, n_all_group_ids, results, n_result, mid);
*n_hit_result = n_compile_ids;
if (n_compile_ids > 0) {
return MAAT_HIT;
} else {
return MAAT_HALF_HIT;
}
} }
struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id) struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id)
@@ -875,7 +786,8 @@ int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **sta
return 0; return 0;
} }
int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, int compile_table_id) int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state,
int compile_table_id)
{ {
if (NULL == maat_instance->maat_rt) { if (NULL == maat_instance->maat_rt) {
return -1; return -1;
@@ -887,7 +799,8 @@ int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_st
return 0; return 0;
} }
size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid, struct maat_hit_path *paths, size_t n_path) size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid,
struct maat_hit_path *paths, size_t n_path)
{ {
int compile_table_id = -1; int compile_table_id = -1;
@@ -897,7 +810,10 @@ size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid, st
compile_table_id = mid->compile_table_id; compile_table_id = mid->compile_table_id;
} }
compile_runtime_get_hit_paths(maat_instance->maat_rt->table_rt_mgr, compile_table_id, maat_instance->g2g_table_id, mid, paths, n_path); void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt,
(struct group2group_runtime *)g2g_runtime, mid->compile_mid, paths, n_path);
return 0; return 0;
} }
@@ -907,10 +823,10 @@ size_t maat_get_hit_objects(struct maat_compile_state *compile_state, struct maa
return 0; return 0;
} }
int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path) int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **state,
struct maat_hit_path *paths, size_t n_path)
{ {
struct maat_state *mid = NULL; struct maat_state *mid = NULL;
struct maat_hit_path *paths;
mid = grab_state(state, maat_instance, 0); mid = grab_state(state, maat_instance, 0);
if (NULL == mid->compile_mid || NULL == maat_instance->maat_rt) { if (NULL == mid->compile_mid || NULL == maat_instance->maat_rt) {
@@ -922,7 +838,8 @@ int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **sta
return n_read; return n_read;
} }
int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj) int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state,
struct maat_hit_object *objs, size_t n_obj)
{ {
return 0; return 0;
} }

2
src/maat_command.cpp
View File

@@ -314,7 +314,7 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li
int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
if (valid_column < 0) { if (valid_column < 0) {
log_error(maat_instance->logger, MODULE_MAAT_COMMAND, log_error(maat_instance->logger, MODULE_MAAT_COMMAND,
"Command set line id %d failed: table %s is not a plugin or ip_plugin table", "Command set line id %d failed: table %s is not a plugin or ip_plugin table",
line_rule->rule_id, line_rule->table_name); line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);

514
src/maat_compile.cpp
View File

@@ -26,6 +26,7 @@
#define MODULE_COMPILE module_name_str("maat.compile") #define MODULE_COMPILE module_name_str("maat.compile")
#define MAX_TABLE_LINE_SIZE (1024 * 16) #define MAX_TABLE_LINE_SIZE (1024 * 16)
#define MAX_COMPILE_EX_DATA_NUM 2
enum user_region_encode { enum user_region_encode {
USER_REGION_ENCODE_NONE=0, USER_REGION_ENCODE_NONE=0,
@@ -56,6 +57,7 @@ struct group2compile_schema {
int virtual_table_name_column; int virtual_table_name_column;
int clause_index_column; int clause_index_column;
char associated_compile_table_id; char associated_compile_table_id;
struct table_manager *ref_tbl_mgr;
int table_id;//ugly int table_id;//ugly
}; };
@@ -219,10 +221,10 @@ void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle
cJSON *custom_item = NULL; cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) { if (item != NULL && item->type == cJSON_Number) {
goto error; compile_schema->table_id = item->valueint;
read_cnt++;
} }
compile_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
@@ -237,8 +239,8 @@ void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle
} }
custom_item = cJSON_GetObjectItem(item, "service_id"); custom_item = cJSON_GetObjectItem(item, "service_id");
if (item != NULL && item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
compile_schema->service_id_column = item->valueint; compile_schema->service_id_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
@@ -284,7 +286,7 @@ void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle
read_cnt++; read_cnt++;
} }
if (read_cnt < 9) { if (read_cnt < 10) {
goto error; goto error;
} }
@@ -306,10 +308,16 @@ void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_h
cJSON *custom_item = NULL; cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) { if (item != NULL && item->type == cJSON_Number) {
goto error; g2c_schema->table_id = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "associated_compile_table_id");
if (item != NULL && item->type == cJSON_Number) {
g2c_schema->associated_compile_table_id = item->valueint;
read_cnt++;
} }
g2c_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
@@ -317,12 +325,6 @@ void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_h
goto error; goto error;
} }
custom_item = cJSON_GetObjectItem(item, "associated_compile_table_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2c_schema->associated_compile_table_id = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "group_id"); custom_item = cJSON_GetObjectItem(item, "group_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
g2c_schema->group_id_column = custom_item->valueint; g2c_schema->group_id_column = custom_item->valueint;
@@ -353,7 +355,7 @@ void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_h
read_cnt++; read_cnt++;
} }
if (read_cnt < 6) { if (read_cnt < 7) {
goto error; goto error;
} }
@@ -368,12 +370,19 @@ void group2compile_schema_free(void *g2c_schema)
FREE(g2c_schema); FREE(g2c_schema);
} }
int group2compile_associated_compile_table_id(void *g2c_schema)
{
struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema;
return schema->associated_compile_table_id;
}
struct compile_item * struct compile_item *
compile_item_new(const char *line, struct compile_schema *compile_schema, struct log_handle *logger) compile_item_new(const char *line, struct compile_schema *compile_schema, struct log_handle *logger)
{ {
size_t column_offset = 0; size_t column_offset = 0;
size_t column_len = 0; size_t column_len = 0;
char tag_str[MAX_TABLE_LINE_SIZE] = {0};
struct compile_item *compile_item = ALLOC(struct compile_item, 1); struct compile_item *compile_item = ALLOC(struct compile_item, 1);
int ret = get_column_pos(line, compile_schema->compile_id_column, &column_offset, &column_len); int ret = get_column_pos(line, compile_schema->compile_id_column, &column_offset, &column_len);
@@ -423,9 +432,8 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, struct
goto error; goto error;
} }
char tag_str[MAX_TABLE_LINE_SIZE] = {0};
memcpy(tag_str, (line + column_offset), column_len); memcpy(tag_str, (line + column_offset), column_len);
#if 0
if (n_accept_tag > 0 && strlen(tag_str) > 2) { if (n_accept_tag > 0 && strlen(tag_str) > 2) {
str_unescape(tag_str); str_unescape(tag_str);
ret = compare_accept_tag(tag_str, accept_tags, n_accept_tag); ret = compare_accept_tag(tag_str, accept_tags, n_accept_tag);
@@ -442,7 +450,7 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, struct
//table_schema->unmatched_tag_cnt++; //table_schema->unmatched_tag_cnt++;
} }
} }
#endif
ret = get_column_pos(line, compile_schema->user_region_column, &column_offset, &column_len); ret = get_column_pos(line, compile_schema->user_region_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
@@ -501,7 +509,7 @@ void compile_item_free(struct compile_item *compile_item)
FREE(compile_item); FREE(compile_item);
} }
void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage_bin, void *compile_runtime_new(void *compile_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
if (NULL == compile_schema) { if (NULL == compile_schema) {
@@ -516,6 +524,34 @@ void *compile_runtime_new(void *compile_schema, struct maat_garbage_bin *garbage
return compile_rt; return compile_rt;
} }
void maat_compile_free(struct maat_compile *compile)
{
struct maat_clause_state *clause_state = NULL;
if (compile->user_data && compile->user_data_free) {
compile->user_data_free(compile->user_data);
}
for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
clause_state = compile->clause_states + i;
utarray_free(clause_state->literal_ids);
clause_state->literal_ids = NULL;
clause_state->in_use = 0;
}
compile->magic = 0;
free(compile);
}
void maat_compile_hash_free(struct maat_compile **compile_hash)
{
struct maat_compile *compile = NULL, *tmp_compile = NULL;
HASH_ITER(hh, *compile_hash, compile, tmp_compile) {
HASH_DEL(*compile_hash, compile);
maat_compile_free(compile);
}
assert(*compile_hash == NULL);
}
void compile_runtime_free(void *compile_runtime) void compile_runtime_free(void *compile_runtime)
{ {
if (NULL == compile_runtime) { if (NULL == compile_runtime) {
@@ -534,17 +570,21 @@ void compile_runtime_free(void *compile_runtime)
FREE(compile_rt); FREE(compile_rt);
} }
void *group2compile_runtime_new(void *g2c_schema, struct maat_garbage_bin *garbage_bin, void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
if (NULL == g2c_schema) {
return NULL;
}
struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1); struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1);
return g2c_rt; return g2c_rt;
} }
void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime)
{
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
g2c_rt->ref_compile_rt = (struct compile_runtime *)compile_runtime;
g2c_rt->ref_g2g_rt = (struct group2group_runtime *)g2g_runtime;
}
void group2compile_runtime_free(void *g2c_runtime) void group2compile_runtime_free(void *g2c_runtime)
{ {
if (NULL == g2c_runtime) { if (NULL == g2c_runtime) {
@@ -576,6 +616,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
{ {
size_t column_offset = 0; size_t column_offset = 0;
size_t column_len = 0; size_t column_len = 0;
char virtual_table_name[NAME_MAX] = {0};
struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1); struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1);
int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len); int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len);
@@ -619,11 +660,11 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
g2c_schema->table_id, line); g2c_schema->table_id, line);
goto error; goto error;
} }
char virtual_table_name[NAME_MAX] = {0};
memcpy(virtual_table_name, (line + column_offset), column_len); memcpy(virtual_table_name, (line + column_offset), column_len);
if (is_valid_table_name(virtual_table_name)) { if (is_valid_table_name(virtual_table_name)) {
g2c_item->vt_id = table_manager_get_table_id(tbl_mgr, virtual_table_name); g2c_item->vt_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr, virtual_table_name);
if (g2c_item->vt_id < 0) { if (g2c_item->vt_id < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"group2compile table(table_id:%d) line:%s unknown virtual table:%s", "group2compile table(table_id:%d) line:%s unknown virtual table:%s",
@@ -686,23 +727,6 @@ int maat_compile_set(struct maat_compile *compile, int declared_clause_num, void
return 0; return 0;
} }
void maat_compile_free(struct maat_compile *compile)
{
struct maat_clause_state *clause_state = NULL;
if (compile->user_data && compile->user_data_free) {
compile->user_data_free(compile->user_data);
}
for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
clause_state = compile->clause_states + i;
utarray_free(clause_state->literal_ids);
clause_state->literal_ids = NULL;
clause_state->in_use = 0;
}
compile->magic = 0;
free(compile);
}
int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, struct maat_compile *compile) int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, struct maat_compile *compile)
{ {
int ret = 0; int ret = 0;
@@ -772,17 +796,6 @@ size_t maat_compile_hash_count(struct maat_compile *compile_hash)
return HASH_COUNT(compile_hash); return HASH_COUNT(compile_hash);
} }
void maat_compile_hash_free(struct maat_compile **compile_hash)
{
struct maat_compile *compile = NULL, *tmp_compile = NULL;
HASH_ITER(hh, *compile_hash, compile, tmp_compile) {
HASH_DEL(*compile_hash, compile);
maat_compile_free(compile);
}
assert(*compile_hash == NULL);
}
int compare_literal_id(const void *pa, const void *pb) int compare_literal_id(const void *pa, const void *pb)
{ {
struct maat_literal_id *la = (struct maat_literal_id *)pa; struct maat_literal_id *la = (struct maat_literal_id *)pa;
@@ -796,7 +809,8 @@ int compare_literal_id(const void *pa, const void *pb)
return ret; return ret;
} }
int maat_compile_clause_add_literal(struct maat_compile *compile, struct maat_literal_id *literal_id, int maat_compile_clause_add_literal(struct maat_compile *compile,
struct maat_literal_id *literal_id,
int clause_index, int clause_not_flag) int clause_index, int clause_not_flag)
{ {
struct maat_clause_state *clause_state = compile->clause_states + clause_index; struct maat_clause_state *clause_state = compile->clause_states + clause_index;
@@ -807,7 +821,9 @@ int maat_compile_clause_add_literal(struct maat_compile *compile, struct maat_li
compile->actual_clause_num++; compile->actual_clause_num++;
} }
struct maat_literal_id *tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); struct maat_literal_id *tmp = NULL;
tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids,
literal_id, compare_literal_id);
if (tmp) { if (tmp) {
assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id));
return -1; return -1;
@@ -819,10 +835,14 @@ int maat_compile_clause_add_literal(struct maat_compile *compile, struct maat_li
return 0; return 0;
} }
int maat_compile_clause_remove_literal(struct maat_compile *compile, struct maat_literal_id *literal_id, int clause_index) int maat_compile_clause_remove_literal(struct maat_compile *compile,
struct maat_literal_id *literal_id,
int clause_index)
{ {
struct maat_clause_state* clause_state = compile->clause_states + clause_index; struct maat_clause_state* clause_state = compile->clause_states + clause_index;
struct maat_literal_id *tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); struct maat_literal_id *tmp = NULL;
tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids,
literal_id, compare_literal_id);
if (tmp) { if (tmp) {
assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id));
} else { } else {
@@ -841,7 +861,8 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile, struct maat
} }
static const struct maat_clause * static const struct maat_clause *
maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, unsigned long long *clause_id_generator, maat_clause_hash_fetch_clause(struct maat_clause **clause_hash,
unsigned long long *clause_id_generator,
struct maat_literal_id *literal_ids, size_t n_literal_id) struct maat_literal_id *literal_ids, size_t n_literal_id)
{ {
struct maat_clause *clause = NULL; struct maat_clause *clause = NULL;
@@ -873,7 +894,10 @@ static void maat_clause_hash_free(struct maat_clause *clause_hash)
} }
} }
struct bool_matcher *maat_compile_bool_matcher_new(struct maat_compile *compile_hash, unsigned long long *clause_id_generator, struct log_handle *logger) struct bool_matcher *
maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
unsigned long long *clause_id_generator,
struct log_handle *logger)
{ {
if (NULL == compile_hash || NULL == logger) { if (NULL == compile_hash || NULL == logger) {
return NULL; return NULL;
@@ -986,7 +1010,8 @@ void maat_compile_bool_matcher_free(struct bool_matcher *bm)
bool_matcher_free(bm); bool_matcher_free(bm);
} }
size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_compile_state *compile_state, void **user_data_array, size_t ud_array_size) size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_compile_state *compile_state,
void **user_data_array, size_t ud_array_size)
{ {
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
struct bool_expr_match *expr_match = ALLOC(struct bool_expr_match, MAX_SCANNER_HIT_COMPILE_NUM); struct bool_expr_match *expr_match = ALLOC(struct bool_expr_match, MAX_SCANNER_HIT_COMPILE_NUM);
@@ -1021,29 +1046,29 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_comp
return ud_result_cnt; return ud_result_cnt;
} }
int maat_add_group_to_compile(struct maat_compile **compile_hash, struct maat_group_topology *group_topo, int group_id, int vt_id, int maat_add_group_to_compile(struct maat_compile **compile_hash, void *g2g_runtime,
int clause_not_flag, int clause_index, int compile_id, struct log_handle *logger) struct group2compile_item *g2c_item, struct log_handle *logger)
{ {
struct maat_group *group = maat_group_topology_find_group(group_topo, group_id); struct maat_group *group = group2group_runtime_find_group(g2g_runtime, g2c_item->group_id);
if (!group) { if (!group) {
group = maat_group_topology_add_group(group_topo, group_id); group = group2group_runtime_add_group(g2g_runtime, g2c_item->group_id);
} }
int ret = -1; int ret = -1;
struct maat_compile *compile = maat_compile_hash_find(compile_hash, compile_id); struct maat_compile *compile = maat_compile_hash_find(compile_hash, g2c_item->compile_id);
if (!compile) { if (!compile) {
compile = maat_compile_new(compile_id); compile = maat_compile_new(g2c_item->compile_id);
ret = maat_compile_hash_add(compile_hash, compile_id, compile); ret = maat_compile_hash_add(compile_hash, g2c_item->compile_id, compile);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
} }
struct maat_literal_id literal_id = {group_id, vt_id}; struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vt_id};
ret = maat_compile_clause_add_literal(compile, &literal_id, clause_index, clause_not_flag); ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, g2c_item->not_flag);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, "add literal_id{group_id:%d, vt_id:%d} to clause %d of compile %d failed", log_error(logger, MODULE_COMPILE, "add literal_id{group_id:%d, vt_id:%d} to clause %d of compile %d failed",
group_id, vt_id, clause_index, compile_id); g2c_item->group_id, g2c_item->vt_id, g2c_item->clause_index, g2c_item->compile_id);
ret = -1; ret = -1;
} else { } else {
group->ref_by_compile_cnt++; group->ref_by_compile_cnt++;
@@ -1053,31 +1078,34 @@ int maat_add_group_to_compile(struct maat_compile **compile_hash, struct maat_gr
return ret; return ret;
} }
int maat_remove_group_from_compile(struct maat_compile **compile_hash, struct maat_group_topology *group_topo, int group_id, int vt_id, int maat_remove_group_from_compile(struct maat_compile **compile_hash, void *g2g_runtime,
int clause_not_flag, int clause_index, int compile_id, struct maat_garbage_bin *garbage_bin, struct group2compile_item *g2c_item,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
struct maat_group *group = maat_group_topology_find_group(group_topo, group_id); struct maat_group *group = group2group_runtime_find_group(g2g_runtime, g2c_item->group_id);
if (!group) { if (!group) {
log_error(logger, MODULE_COMPILE, "Remove group %d from compile %d failed, group is not exisited.", log_error(logger, MODULE_COMPILE,
group_id, compile_id); "Remove group %d from compile %d failed, group is not exisited.",
g2c_item->group_id, g2c_item->compile_id);
return -1; return -1;
} }
struct maat_compile *compile = NULL; struct maat_compile *compile = NULL;
HASH_FIND(hh, *compile_hash, &compile_id, sizeof(compile_id), compile); HASH_FIND(hh, *compile_hash, &(g2c_item->compile_id), sizeof(g2c_item->compile_id), compile);
if (!compile) { if (!compile) {
log_error(logger, MODULE_COMPILE, "Remove group %d from compile %d failed, compile is not exisited.", log_error(logger, MODULE_COMPILE,
group_id, compile_id); "Remove group %d from compile %d failed, compile is not exisited.",
g2c_item->group_id, g2c_item->compile_id);
return -1; return -1;
} }
struct maat_literal_id literal_id = {group_id, vt_id}; struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vt_id};
int ret = maat_compile_clause_remove_literal(compile, &literal_id, clause_index); int ret = maat_compile_clause_remove_literal(compile, &literal_id, g2c_item->clause_index);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_COMPILE, log_error(logger, MODULE_COMPILE,
"Remove group %d vt_id %d from clause %d of compile %d failed, literal is not in compile.", "Remove group %d vt_id %d from clause %d of compile %d failed, literal is not in compile.",
group_id, vt_id, clause_index, compile_id); g2c_item->group_id, g2c_item->vt_id, g2c_item->clause_index, g2c_item->compile_id);
return -1; return -1;
} }
@@ -1170,7 +1198,8 @@ static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_path
return 0; return 0;
} }
size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct maat_group_topology *group_topo, size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt,
struct maat_compile_state *compile_state, struct maat_compile_state *compile_state,
struct maat_hit_path *hit_paths, size_t hit_path_size) struct maat_hit_path *hit_paths, size_t hit_path_size)
{ {
@@ -1181,7 +1210,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct
for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) { for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) {
internal_path = (struct maat_internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i); internal_path = (struct maat_internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i);
group = maat_group_topology_find_group(group_topo, internal_path->group_id); group = group2group_runtime_find_group(g2g_rt, internal_path->group_id);
if (0 == group->top_group_cnt && n_made_by_item < hit_path_size) { if (0 == group->top_group_cnt && n_made_by_item < hit_path_size) {
//group not referenced by compile //group not referenced by compile
@@ -1261,21 +1290,29 @@ void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state
compile_state->this_scan_item_hit_cnt++; compile_state->this_scan_item_hit_cnt++;
} }
void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, struct maat_compile **compile_hash, void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state,
int group_id, int vt_id) void *compile_runtime, int group_id, int vt_id)
{ {
if (NULL == compile_state || NULL == compile_runtime) {
return;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
assert(compile_rt->compile_hash != NULL);
struct maat_compile *compile = NULL, *tmp_compile = NULL; struct maat_compile *compile = NULL, *tmp_compile = NULL;
struct maat_clause_state *clause_state = NULL; struct maat_clause_state *clause_state = NULL;
struct maat_literal_id literal_id = {group_id, vt_id}; struct maat_literal_id literal_id = {group_id, vt_id};
struct maat_literal_id *tmp = NULL;
HASH_ITER(hh, *compile_hash, compile, tmp_compile) { HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) {
for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
clause_state = compile->clause_states + i; clause_state = compile->clause_states + i;
if (!clause_state->in_use) { if (!clause_state->in_use) {
continue; continue;
} }
struct maat_literal_id *tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, &literal_id, compare_literal_id); tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, &literal_id, compare_literal_id);
if (tmp) { if (tmp) {
//Deduplication //Deduplication
if (utarray_find(compile_state->all_hit_clause_array, &(clause_state->clause_id), compare_clause_id)) { if (utarray_find(compile_state->all_hit_clause_array, &(clause_state->clause_id), compare_clause_id)) {
@@ -1293,138 +1330,8 @@ int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state)
return compile_state->not_clause_hitted_flag; return compile_state->not_clause_hitted_flag;
} }
int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, void *rule_ex_data_new(const struct maat_rule_head *rule_head, const char *srv_def,
int valid_column) const struct compile_ex_data_schema *ex_schema)
{
if (NULL == compile_runtime || NULL == compile_schema || NULL == line) {
return -1;
}
int ret = -1;
struct maat_compile *compile = NULL;
struct compile_item *compile_item = NULL;
struct compile_schema *schema = (struct compile_schema *)compile_schema;
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
int is_valid = get_column_value(line, valid_column);
int compile_id = get_column_value(line, schema->compile_id_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id, compile_rt->ref_garbage_bin);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"remove compile table(table_id:%d) compile(compile_id:%d) from compile_hash failed",
schema->table_id, compile_id);
return -1;
}
} else {
//add
compile_item = compile_item_new(line, schema, compile_rt->logger);
struct compile_rule *compile_rule = ALLOC(struct compile_rule, 1);
compile_item_to_compile_rule(compile_item, schema, compile_rule);
compile_item_free(compile_item);
compile_item = NULL;
compile = maat_compile_new(compile_rule->compile_id);
if (NULL == compile) {
destroy_compile_rule(compile_rule);
log_error(logger, MODULE_COMPILE,
"maat_compile_new failed, compile_table(table_id:%d) compile_id:%d",
schema->table_id, compile_item->compile_id);
return -1;
}
maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule, (void (*)(void *))destroy_compile_rule);
ret = maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile);
if (ret < 0) {
maat_compile_free(compile);
log_error(logger, MODULE_TABLE_RUNTIME,
"add compile table(table_id:%d) compile(compile_id:%d) to compile_hash failed",
schema->table_id, compile_id);
return -1;
}
}
return 0;
}
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line,
int valid_column)
{
if (NULL == g2c_runtime || NULL == g2c_schema || NULL == line) {
return -1;
}
struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema;
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
struct compile_runtime *compile_rt = g2c_rt->ref_compile_rt;
struct group2group_runtime *g2g_rt = g2c_rt->ref_g2g_rt;
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
}
int ret = -1;
struct group2compile_item *g2c_item = group2compile_item_new(line, g2c_schema, compile_rt->logger);
if (0 == is_valid) {
//delete
ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2g_rt->group_topo, g2c_item->group_id,
g2c_item->vt_id, g2c_item->not_flag, g2c_item->clause_index,
g2c_item->compile_id, compile_rt->ref_garbage_bin, compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
g2c_rt->not_flag_group--;
}
}
} else {
//add
ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2g_rt->group_topo, g2c_item->group_id,
g2c_item->vt_id, g2c_item->not_flag, g2c_item->clause_index,
g2c_item->compile_id, compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
g2c_rt->not_flag_group++;
}
}
}
group2compile_item_free(g2c_item);
return ret;
}
int compile_runtime_commit(void *compile_runtime)
{
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
int ret = 0;
size_t compile_cnt = maat_compile_hash_count(compile_rt->compile_hash);
if (0 == compile_cnt) {
return 0;
}
struct bool_matcher *old_bool_matcher = NULL;
struct bool_matcher *new_bool_matcher = NULL;
log_info(logger, MODULE_TABLE_RUNTIME,
"committing %zu compile rules for rebuilding compile bool_matcher engine", compile_cnt);
new_bool_matcher = maat_compile_bool_matcher_new(compile_rt->compile_hash, &compile_rt->clause_id_generator, logger);
if (NULL == new_bool_matcher) {
log_error(logger, MODULE_TABLE_RUNTIME,
"rebuild compile bool_matcher engine failed when update %zu compile rules", compile_cnt);
ret = -1;
}
old_bool_matcher = compile_rt->bm;
compile_rt->bm = new_bool_matcher;
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))maat_compile_bool_matcher_free);
compile_rt->rule_num = compile_cnt;
return ret;
}
void *rule_ex_data_new(const struct maat_rule_head *rule_head, const char *srv_def, const struct compile_ex_data_schema *ex_schema)
{ {
void *ex_data = NULL; void *ex_data = NULL;
struct maat_rule rule; struct maat_rule rule;
@@ -1472,45 +1379,169 @@ void compile_item_to_compile_rule(struct compile_item *compile_item,
struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(compile_schema, i); struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(compile_schema, i);
compile_rule->ex_data[i] = rule_ex_data_new(&compile_rule->head, compile_rule->service_defined, ex_schema); compile_rule->ex_data[i] = rule_ex_data_new(&compile_rule->head, compile_rule->service_defined, ex_schema);
} }
compile_rule->is_valid = 1;
compile_rule->compile_id = compile_item->compile_id; compile_rule->compile_id = compile_item->compile_id;
pthread_rwlock_init(&compile_rule->rwlock, NULL); pthread_rwlock_init(&compile_rule->rwlock, NULL);
} }
void destroy_compile_rule(struct compile_rule *compile_rule) void destroy_compile_rule(struct compile_rule *compile_rule)
{ {
struct table_schema *table_schema = compile_rule->ref_table; struct compile_schema *schema = compile_rule->ref_table;
assert(compile_rule->magic_num==COMPILE_RULE_MAGIC); assert(compile_rule->magic_num==COMPILE_RULE_MAGIC);
size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(table_schema); size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(schema);
for (size_t i = 0; i < n_rule_ex_schema; i++) { for (size_t i = 0; i < n_rule_ex_schema; i++) {
struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(table_schema, i); struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(schema, i);
rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, ex_schema); rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, ex_schema);
compile_rule->ex_data[i] = NULL; compile_rule->ex_data[i] = NULL;
} }
free(compile_rule->ex_data); free(compile_rule->ex_data);
compile_rule->is_valid = 0;
compile_rule->declared_clause_num = -1; compile_rule->declared_clause_num = -1;
free(compile_rule->service_defined); free(compile_rule->service_defined);
compile_rule->service_defined = NULL; compile_rule->service_defined = NULL;
free(compile_rule); free(compile_rule);
} }
struct group2compile_rule *g2c_item_to_g2c_rule(struct group2compile_item *g2c_item) int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line,
int valid_column)
{ {
struct group2compile_rule *g2c_rule = ALLOC(struct group2compile_rule, 1); if (NULL == compile_runtime || NULL == compile_schema || NULL == line) {
return -1;
}
g2c_rule->group_id = g2c_item->group_id; int ret = -1;
g2c_rule->compile_id = g2c_item->compile_id; struct maat_compile *compile = NULL;
g2c_rule->is_valid = g2c_item->is_valid; struct compile_item *compile_item = NULL;
g2c_rule->not_flag = g2c_item->not_flag; struct compile_schema *schema = (struct compile_schema *)compile_schema;
g2c_rule->vt_id = g2c_item->vt_id; struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
g2c_rule->clause_index = g2c_item->clause_index; int is_valid = get_column_value(line, valid_column);
g2c_rule->associated_compile_table_id = g2c_item->associated_compile_table_id; int compile_id = get_column_value(line, schema->compile_id_column);
if (is_valid < 0) {
return -1;
} else if (0 == is_valid) {
//delete
ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id, compile_rt->ref_garbage_bin);
if (ret < 0) {
log_error(compile_rt->logger, MODULE_COMPILE,
"remove compile table(table_id:%d) compile(compile_id:%d) from compile_hash failed",
schema->table_id, compile_id);
return -1;
}
} else {
//add
compile_item = compile_item_new(line, schema, compile_rt->logger);
if (NULL == compile_item) {
return -1;
}
return g2c_rule; struct compile_rule *compile_rule = ALLOC(struct compile_rule, 1);
compile_item_to_compile_rule(compile_item, schema, compile_rule);
compile_item_free(compile_item);
compile_item = NULL;
compile = maat_compile_new(compile_rule->compile_id);
if (NULL == compile) {
destroy_compile_rule(compile_rule);
log_error(compile_rt->logger, MODULE_COMPILE,
"maat_compile_new failed, compile_table(table_id:%d) compile_id:%d",
schema->table_id, compile_item->compile_id);
return -1;
}
maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule, (void (*)(void *))destroy_compile_rule);
ret = maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile);
if (ret < 0) {
maat_compile_free(compile);
log_error(compile_rt->logger, MODULE_COMPILE,
"add compile table(table_id:%d) compile(compile_id:%d) to compile_hash failed",
schema->table_id, compile_id);
return -1;
}
}
return 0;
}
int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line,
int valid_column)
{
if (NULL == g2c_runtime || NULL == g2c_schema || NULL == line) {
return -1;
}
struct group2compile_schema *schema = (struct group2compile_schema *)g2c_schema;
struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime;
struct compile_runtime *compile_rt = g2c_rt->ref_compile_rt;
struct group2group_runtime *g2g_rt = g2c_rt->ref_g2g_rt;
int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) {
return -1;
}
int ret = -1;
struct group2compile_item *g2c_item = group2compile_item_new(line, schema, compile_rt->logger);
if (NULL == g2c_item) {
return -1;
}
if (0 == is_valid) {
//delete
ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item,
compile_rt->ref_garbage_bin, compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
g2c_rt->not_flag_group--;
}
}
} else {
//add
ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item, compile_rt->logger);
if (0 == ret) {
if (g2c_item->not_flag) {
g2c_rt->not_flag_group++;
}
}
}
group2compile_item_free(g2c_item);
return ret;
}
int compile_runtime_commit(void *compile_runtime)
{
if (NULL == compile_runtime) {
return -1;
}
struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime;
size_t compile_cnt = maat_compile_hash_count(compile_rt->compile_hash);
if (0 == compile_cnt) {
return 0;
}
struct bool_matcher *old_bool_matcher = NULL;
struct bool_matcher *new_bool_matcher = NULL;
log_info(compile_rt->logger, MODULE_COMPILE,
"committing %zu compile rules for rebuilding compile bool_matcher engine",
compile_cnt);
int ret = 0;
new_bool_matcher = maat_compile_bool_matcher_new(compile_rt->compile_hash,
&compile_rt->clause_id_generator,
compile_rt->logger);
if (NULL == new_bool_matcher) {
log_error(compile_rt->logger, MODULE_COMPILE,
"rebuild compile bool_matcher engine failed when update %zu compile rules", compile_cnt);
ret = -1;
}
old_bool_matcher = compile_rt->bm;
compile_rt->bm = new_bool_matcher;
maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))maat_compile_bool_matcher_free);
compile_rt->rule_num = compile_cnt;
return ret;
} }
static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b) static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b)
@@ -1556,13 +1587,12 @@ static int compare_compile_rule(const void *a, const void *b)
} }
int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids, int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids,
int *compile_ids, size_t compile_ids_size, struct maat_state *state) int vt_id, int *compile_ids, size_t compile_ids_size, struct maat_state *state)
{ {
if (NULL == compile_rt || table_rt->table_type != TABLE_TYPE_COMPILE) { if (NULL == compile_rt) {
return -1; return -1;
} }
struct compile_runtime *compile_rt = &(table_rt->compile_rt);
struct maat_compile_state *compile_state = state->compile_mid; struct maat_compile_state *compile_state = state->compile_mid;
struct compile_rule *compile_rule_array[compile_ids_size]; struct compile_rule *compile_rule_array[compile_ids_size];

142
src/maat_expr.cpp
View File

@@ -19,6 +19,7 @@
#include "maat_limits.h" #include "maat_limits.h"
#include "rcu_hash.h" #include "rcu_hash.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_compile.h"
#include "maat_garbage_collection.h" #include "maat_garbage_collection.h"
#define MAX_DISTRICT_STR 128 #define MAX_DISTRICT_STR 128
@@ -34,6 +35,7 @@ struct expr_schema {
int match_method_column; int match_method_column;
int is_hexbin_column; int is_hexbin_column;
enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ enum hs_scan_mode scan_mode; /* adapter_hs scan mode */
int table_id; //ugly
}; };
enum expr_type { enum expr_type {
@@ -74,6 +76,7 @@ struct expr_runtime {
struct maat_item *item_hash; struct maat_item *item_hash;
void (*item_user_data_free)(void *); void (*item_user_data_free)(void *);
int n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger; struct log_handle *logger;
@@ -140,12 +143,16 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len); int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no item_id",
expr_schema->table_id, line);
goto error; goto error;
} }
expr_item->item_id = atoi(line + column_offset); expr_item->item_id = atoi(line + column_offset);
ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no group_id",
expr_schema->table_id, line);
goto error; goto error;
} }
expr_item->group_id = atoi(line + column_offset); expr_item->group_id = atoi(line + column_offset);
@@ -169,19 +176,23 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
#endif #endif
ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no keywords",
expr_schema->table_id, line);
goto error; goto error;
} }
if (column_len >= MAX_KEYWORDS_STR) { if (column_len >= MAX_KEYWORDS_STR) {
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"update error: expr table[%s]:item_id[%d] keywords length too long", "expr table(table_id:%d) line:%s keywords length too long",
table_name, expr_item->item_id); expr_schema->table_id, line);
goto error; goto error;
} }
memcpy(expr_item->keywords, (line + column_offset), column_len); memcpy(expr_item->keywords, (line + column_offset), column_len);
ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no expr_type",
expr_schema->table_id, line);
goto error; goto error;
} }
@@ -190,6 +201,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no match_method",
expr_schema->table_id, line);
goto error; goto error;
} }
@@ -198,6 +211,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len); ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no is_hexbin",
expr_schema->table_id, line);
goto error; goto error;
} }
db_hexbin = atoi(line + column_offset); db_hexbin = atoi(line + column_offset);
@@ -217,8 +232,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
break; break;
default: default:
log_error(logger, MODULE_EXPR, log_error(logger, MODULE_EXPR,
"update error: expr table[%s]:item_id[%d] invalid hexbin value:%d", "expr table(table_id:%d) line:%s has invalid hexbin value:%d",
table_name, expr_item->item_id, db_hexbin); expr_schema->table_id, line, db_hexbin);
goto error; goto error;
} }
@@ -244,7 +259,13 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo
int ret = -1; int ret = -1;
cJSON *custom_item = NULL; cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "custom"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (item != NULL && item->type == cJSON_Number) {
expr_schema->table_id = item->valueint;
read_cnt++;
}
item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
log_error(logger, MODULE_EXPR, "table %s has no custom column", table_name); log_error(logger, MODULE_EXPR, "table %s has no custom column", table_name);
goto error; goto error;
@@ -319,7 +340,26 @@ void expr_schema_free(void *expr_schema)
FREE(expr_schema); FREE(expr_schema);
} }
void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin, void expr_rule_free(and_expr_t *expr_rule)
{
if (NULL == expr_rule) {
return;
}
for (size_t i = 0; i < expr_rule->n_patterns; i++) {
FREE(expr_rule->patterns[i].pat);
}
FREE(expr_rule);
}
void expr_ex_data_free(void *user_ctx, void *data)
{
and_expr_t *expr_rule = (and_expr_t *)data;
expr_rule_free(expr_rule);
}
void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
if (NULL == expr_schema) { if (NULL == expr_schema) {
@@ -332,6 +372,7 @@ void *expr_runtime_new(void *expr_schema, struct maat_garbage_bin *garbage_bin,
expr_rt->htable = rcu_hash_new(expr_ex_data_free); expr_rt->htable = rcu_hash_new(expr_ex_data_free);
expr_rt->scan_mode = schema->scan_mode; expr_rt->scan_mode = schema->scan_mode;
expr_rt->item_user_data_free = maat_item_inner_free; expr_rt->item_user_data_free = maat_item_inner_free;
expr_rt->n_worker_thread = max_thread_num;
expr_rt->ref_garbage_bin = garbage_bin; expr_rt->ref_garbage_bin = garbage_bin;
expr_rt->logger = logger; expr_rt->logger = logger;
@@ -401,6 +442,25 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_
return 0; return 0;
} }
enum pattern_type expr_type2pattern_type(enum expr_type expr_type)
{
enum pattern_type pattern_type = PATTERN_TYPE_STR;
switch (expr_type) {
case EXPR_TYPE_STRING:
case EXPR_TYPE_AND:
break;
case EXPR_TYPE_REGEX:
pattern_type = PATTERN_TYPE_REG;
break;
default:
break;
}
return pattern_type;
}
#define MAAT_MAX_EXPR_ITEM_NUM 8
and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger) and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger)
{ {
size_t i = 0; size_t i = 0;
@@ -420,8 +480,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl
} }
if (i >= MAAT_MAX_EXPR_ITEM_NUM) { if (i >= MAAT_MAX_EXPR_ITEM_NUM) {
log_error(logger, MODULE_TABLE_RUNTIME, "expr item_id:%d too many patterns", log_error(logger, MODULE_EXPR,
expr_item->item_id); "expr item_id:%d too many patterns", expr_item->item_id);
return NULL; return NULL;
} }
@@ -487,7 +547,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line,
} }
HASH_DELETE(hh, expr_rt->item_hash, item); HASH_DELETE(hh, expr_rt->item_hash, item);
maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free); maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, maat_item_inner_free);
} else { } else {
//add //add
HASH_FIND_INT(expr_rt->item_hash, &item_id, item); HASH_FIND_INT(expr_rt->item_hash, &item_id, item);
@@ -499,7 +559,6 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line,
struct expr_item *expr_item = expr_item_new(line, schema, expr_rt->logger); struct expr_item *expr_item = expr_item_new(line, schema, expr_rt->logger);
if (NULL == expr_item) { if (NULL == expr_item) {
log_error(expr_rt->logger, MODULE_EXPR, "expr line %s to item failed", line);
return -1; return -1;
} }
@@ -507,14 +566,14 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line,
//int district_id = get_district_id(maat_rt, expr_item->district); //int district_id = get_district_id(maat_rt, expr_item->district);
int district_id = -1; int district_id = -1;
u_para = maat_item_inner_new(expr_item->group_id, item_id, district_id); u_para = maat_item_inner_new(expr_item->group_id, item_id, district_id);
item = maat_item_new(item_id, group_id, u_para); item = maat_item_new(item_id, expr_item->group_id, u_para);
HASH_ADD_INT(expr_rt->item_hash, item_id, item); HASH_ADD_INT(expr_rt->item_hash, item_id, item);
expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger); expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger);
expr_item_free(expr_item); expr_item_free(expr_item);
if (NULL == expr_rule) { if (NULL == expr_rule) {
log_error(expr_rt->logger, MODULE_EXPR, "transform expr table:%s item to expr_rule failed, item_id:%d", log_error(expr_rt->logger, MODULE_EXPR, "transform expr table(table_id:%d) item to expr_rule failed, item_id:%d",
table_name, item_id); schema->table_id, item_id);
return -1; return -1;
} }
} }
@@ -564,7 +623,7 @@ int expr_runtime_commit(void *expr_runtime)
log_info(expr_rt->logger, MODULE_EXPR, log_info(expr_rt->logger, MODULE_EXPR,
"committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt); "committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt);
new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, nr_worker_thread, rules, rule_cnt, expr_rt->logger); new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger);
if (NULL == new_adapter_hs) { if (NULL == new_adapter_hs) {
log_error(expr_rt->logger, MODULE_EXPR, log_error(expr_rt->logger, MODULE_EXPR,
"rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt); "rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt);
@@ -574,7 +633,7 @@ int expr_runtime_commit(void *expr_runtime)
old_adapter_hs = expr_rt->hs; old_adapter_hs = expr_rt->hs;
expr_rt->hs = new_adapter_hs; expr_rt->hs = new_adapter_hs;
maat_garbage_bagging(table_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy); maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy);
rcu_hash_commit(expr_rt->htable); rcu_hash_commit(expr_rt->htable);
expr_rt->rule_num = rcu_hash_count(expr_rt->htable); expr_rt->rule_num = rcu_hash_count(expr_rt->htable);
rule_cnt = rcu_hash_updating_count(expr_rt->htable); rule_cnt = rcu_hash_updating_count(expr_rt->htable);
@@ -586,40 +645,22 @@ int expr_runtime_commit(void *expr_runtime)
return ret; return ret;
} }
int expr_runtime_updating_flag(struct expr_runtime *expr_rt) int expr_runtime_updating_flag(void *expr_runtime)
{ {
struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
return rcu_hash_updating_flag(expr_rt->htable); return rcu_hash_updating_flag(expr_rt->htable);
} }
void expr_rule_free(and_expr_t *expr_rule)
{
if (NULL == expr_rule) {
return;
}
for (size_t i = 0; i < expr_rule->n_patterns; i++) {
FREE(expr_rule->patterns[i].pat);
}
FREE(expr_rule);
}
void expr_ex_data_free(void *user_ctx, void *data)
{
and_expr_t *expr_rule = (and_expr_t *)data;
expr_rule_free(expr_rule);
}
int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len, int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len,
int group_id_array[], size_t n_group_id_array, int virtual_table_id, struct maat_state *state) int group_id_array[], size_t n_group_id_array, int virtual_table_id, struct maat_state *state)
{ {
if (NULL == table_rt) { if (NULL == expr_rt) {
return -1; return -1;
} }
int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
size_t n_hit_item = 0; size_t n_hit_item = 0;
int ret = adapter_hs_scan(table_rt->expr_rt.hs, thread_id, data, data_len, hit_item_ids, &n_hit_item); int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, hit_item_ids, &n_hit_item);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
@@ -633,7 +674,7 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const
size_t n_group_id = 0; size_t n_group_id = 0;
size_t i = 0; size_t i = 0;
for (i = 0; i < n_hit_item; i++) { for (i = 0; i < n_hit_item; i++) {
HASH_FIND_INT(table_rt->item_hash, &(hit_item_ids[i]), item); HASH_FIND_INT(expr_rt->item_hash, &(hit_item_ids[i]), item);
assert(item != NULL); assert(item != NULL);
if (!item) { if (!item) {
// should not come here // should not come here
@@ -658,13 +699,14 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const
} else { } else {
compile_table_id = state->compile_table_id; compile_table_id = state->compile_table_id;
} }
struct maat_runtime *maat_rt = state->maat_instance->maat_rt;
struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, compile_table_id); void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id);
assert(compile_table_rt->table_type == TABLE_TYPE_COMPILE); enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id);
assert(table_type == TABLE_TYPE_COMPILE);
// STEP 2: get the specified compile table's hit clause_id array by literal_id // STEP 2: get the specified compile table's hit clause_id array by literal_id
for (i = 0; i < n_group_id; i++) { for (i = 0; i < n_group_id; i++) {
maat_compile_state_update_hit_clause(compile_state, &(compile_table_rt->compile_rt.compile_hash), group_id_array[i], virtual_table_id); maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[i], virtual_table_id);
} }
return n_group_id; return n_group_id;
@@ -672,28 +714,28 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
{ {
if (NULL == table_rt) { if (NULL == expr_rt) {
return; return;
} }
struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(table_rt->expr_rt.hs, thread_id); struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(expr_rt->hs, thread_id);
table_rt->expr_rt.hs_stream = hs_stream; expr_rt->hs_stream = hs_stream;
} }
int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len,
int result[], size_t *n_result) int result[], size_t *n_result)
{ {
if (NULL == table_rt) { if (NULL == expr_rt) {
return -1; return -1;
} }
return adapter_hs_scan_stream(table_rt->expr_rt.hs_stream, data, data_len, result, n_result); return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, result, n_result);
} }
void expr_runtime_stream_close(struct expr_runtime *expr_rt) void expr_runtime_stream_close(struct expr_runtime *expr_rt)
{ {
if (table_rt != NULL) { if (expr_rt != NULL) {
adapter_hs_stream_close(table_rt->expr_rt.hs_stream); adapter_hs_stream_close(expr_rt->hs_stream);
table_rt->expr_rt.hs_stream = NULL; expr_rt->hs_stream = NULL;
} }
} }

2
src/maat_garbage_collection.cpp
View File

@@ -31,7 +31,7 @@ struct maat_garbage_bin {
int timeout_seconds; int timeout_seconds;
}; };
struct maat_garbage_bin* maat_garbage_bin_new(int default_timeout) struct maat_garbage_bin *maat_garbage_bin_new(int default_timeout)
{ {
struct maat_garbage_bin* bin = ALLOC(struct maat_garbage_bin, 1); struct maat_garbage_bin* bin = ALLOC(struct maat_garbage_bin, 1);

225
src/maat_group.cpp
View File

@@ -60,10 +60,10 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han
cJSON *custom_item = NULL; cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) { if (item != NULL && item->type == cJSON_Number) {
goto error; g2g_schema->table_id = item->valueint;
read_cnt++;
} }
g2g_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
@@ -83,13 +83,13 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han
read_cnt++; read_cnt++;
} }
if (read_cnt < 2) { if (read_cnt < 3) {
goto error; goto error;
} }
return g2g_schema; return g2g_schema;
error: error:
FREE(g2c_schema); FREE(g2g_schema);
return NULL; return NULL;
} }
@@ -98,9 +98,27 @@ void group2group_schema_free(void *g2g_schema)
FREE(g2g_schema); FREE(g2g_schema);
} }
struct group2group_runtime * struct maat_group_topology *maat_group_topology_new(struct log_handle *logger)
group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, {
struct log_handle *logger) struct maat_group_topology *group_topo = ALLOC(struct maat_group_topology, 1);
UNUSED int ret = 0;
group_topo->hash_group_by_id = NULL;
group_topo->hash_group_by_vertex = NULL;
ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED);
assert(ret == IGRAPH_SUCCESS);
ret = pthread_rwlock_init(&group_topo->rwlock, NULL);
assert(ret == 0);
group_topo->logger = logger;
return group_topo;
}
void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{ {
struct group2group_runtime *g2g_rt = ALLOC(struct group2group_runtime, 1); struct group2group_runtime *g2g_rt = ALLOC(struct group2group_runtime, 1);
@@ -111,6 +129,29 @@ group2group_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_b
return g2g_rt; return g2g_rt;
} }
void group_vertex_free(struct maat_group *group)
{
free(group->top_group_ids);
free(group);
}
void maat_group_topology_free(struct maat_group_topology *group_topo)
{
struct maat_group *group = NULL, *tmp_group = NULL;
HASH_CLEAR(hh_vertex_id, group_topo->hash_group_by_vertex);//No need group memory clean up.
HASH_ITER(hh_group_id, group_topo->hash_group_by_id, group, tmp_group) {
HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group);
group_vertex_free(group);
}
assert(group_topo->hash_group_by_id == NULL);
igraph_destroy(&group_topo->group_graph);
pthread_rwlock_unlock(&group_topo->rwlock);
pthread_rwlock_destroy(&group_topo->rwlock);
}
void group2group_runtime_free(void *g2g_runtime) void group2group_runtime_free(void *g2g_runtime)
{ {
if (NULL == g2g_runtime) { if (NULL == g2g_runtime) {
@@ -163,48 +204,6 @@ void group2group_item_free(struct group2group_item *g2g_item)
FREE(g2g_item); FREE(g2g_item);
} }
void group_vertex_free(struct maat_group *group)
{
free(group->top_group_ids);
free(group);
}
struct maat_group_topology *maat_group_topology_new(struct log_handle *logger)
{
struct maat_group_topology *group_topo = ALLOC(struct maat_group_topology, 1);
UNUSED int ret = 0;
group_topo->hash_group_by_id = NULL;
group_topo->hash_group_by_vertex = NULL;
ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED);
assert(ret == IGRAPH_SUCCESS);
ret = pthread_rwlock_init(&group_topo->rwlock, NULL);
assert(ret == 0);
group_topo->logger = logger;
return group_topo;
}
void maat_group_topology_free(struct maat_group_topology *group_topo)
{
struct maat_group *group = NULL, *tmp_group = NULL;
HASH_CLEAR(hh_vertex_id, group_topo->hash_group_by_vertex);//No need group memory clean up.
HASH_ITER(hh_group_id, group_topo->hash_group_by_id, group, tmp_group) {
HASH_DELETE(hh_group_id, group_topo->hash_group_by_id, group);
group_vertex_free(group);
}
assert(group_topo->hash_group_by_id == NULL);
igraph_destroy(&group_topo->group_graph);
pthread_rwlock_unlock(&group_topo->rwlock);
pthread_rwlock_destroy(&group_topo->rwlock);
}
size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) { size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) {
long int i; long int i;
int printed = 0; int printed = 0;
@@ -216,8 +215,16 @@ size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) {
return printed; return printed;
} }
struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id) struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id)
{ {
if (NULL == g2g_runtime) {
return NULL;
}
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL);
struct maat_group *group = ALLOC(struct maat_group, 1); struct maat_group *group = ALLOC(struct maat_group, 1);
group->group_id = group_id; group->group_id = group_id;
@@ -232,10 +239,17 @@ struct maat_group *maat_group_topology_add_group(struct maat_group_topology *gro
return group; return group;
} }
void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group) void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group)
{ {
if (NULL == g2g_runtime || NULL == group) {
return;
}
igraph_vector_t v; igraph_vector_t v;
char buff[4096] = {0}; char buff[4096] = {0};
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL);
assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0); assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0);
igraph_vector_init(&v, 8); igraph_vector_init(&v, 8);
@@ -258,36 +272,49 @@ void maat_group_topology_remove_group(struct maat_group_topology *group_topo, st
group_vertex_free(group); group_vertex_free(group);
} }
struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id) struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id)
{ {
struct maat_group *group = NULL; if (NULL == g2g_runtime) {
return NULL;
}
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL);
struct maat_group *group = NULL;
HASH_FIND(hh_group_id, group_topo->hash_group_by_id, &group_id, sizeof(group_id), group); HASH_FIND(hh_group_id, group_topo->hash_group_by_id, &group_id, sizeof(group_id), group);
return group; return group;
} }
int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id) int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id)
{ {
if (NULL == g2g_runtime) {
return -1;
}
int ret = 0; int ret = 0;
igraph_integer_t edge_id; igraph_integer_t edge_id;
struct maat_group *group = NULL, *superior_group = NULL; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL);
group = maat_group_topology_find_group(group_topo, group_id); struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_id);
if (NULL == group) { if (NULL == group) {
group = maat_group_topology_add_group(group_topo, group_id); group = group2group_runtime_add_group(g2g_runtime, group_id);
} }
superior_group = maat_group_topology_find_group(group_topo, superior_group_id); struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id);
if (NULL == superior_group) { if (NULL == superior_group) {
superior_group = maat_group_topology_add_group(group_topo, superior_group_id); superior_group = group2group_runtime_add_group(g2g_runtime, superior_group_id);
} }
ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
//No duplicated edges between two groups. //No duplicated edges between two groups.
if (edge_id > 0) { if (edge_id > 0) {
log_error(group_topo->logger, MODULE_GROUP, log_error(g2g_rt->logger, MODULE_GROUP,
"Add group %d to group %d failed, relation already exisited.", "Add group %d to group %d failed, relation already exisited.",
group->group_id, superior_group->group_id); group->group_id, superior_group->group_id);
ret = -1; ret = -1;
@@ -301,23 +328,26 @@ int maat_group_topology_add_group_to_group(struct maat_group_topology *group_top
return ret; return ret;
} }
int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id) int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id)
{ {
int ret = 0; if (NULL == g2g_runtime) {
struct maat_group *group = NULL, *superior_group = NULL; return -1;
}
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
//No hash write operation, LOCK protection is unnecessary. //No hash write operation, LOCK protection is unnecessary.
group = maat_group_topology_find_group(group_topo, group_id); struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_id);
if (NULL == group) { if (NULL == group) {
log_error(group_topo->logger, MODULE_GROUP, log_error(g2g_rt->logger, MODULE_GROUP,
"Del group %d from group %d failed, group %d not exisited.", "Del group %d from group %d failed, group %d not exisited.",
group_id, superior_group_id, group_id); group_id, superior_group_id, group_id);
return -1; return -1;
} }
superior_group = maat_group_topology_find_group(group_topo, superior_group_id); struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id);
if (NULL == superior_group) { if (NULL == superior_group) {
log_error(group_topo->logger, MODULE_GROUP, log_error(g2g_rt->logger, MODULE_GROUP,
"Del group %d from group %d failed, superior group %d not exisited.", "Del group %d from group %d failed, superior group %d not exisited.",
group_id, superior_group_id, superior_group_id); group_id, superior_group_id, superior_group_id);
return -1; return -1;
@@ -325,6 +355,7 @@ int maat_group_topology_remove_group_from_group(struct maat_group_topology *grou
igraph_es_t es; igraph_es_t es;
igraph_integer_t edge_num_before = 0, edge_num_after = 0; igraph_integer_t edge_num_before = 0, edge_num_after = 0;
struct maat_group_topology *group_topo = g2g_rt->group_topo;
edge_num_before = igraph_ecount(&group_topo->group_graph); edge_num_before = igraph_ecount(&group_topo->group_graph);
// The edges between the given pairs of vertices will be included in the edge selection. // The edges between the given pairs of vertices will be included in the edge selection.
@@ -333,7 +364,7 @@ int maat_group_topology_remove_group_from_group(struct maat_group_topology *grou
//first edge, the fifth is the first vertex of the second edge and so on. The last element //first edge, the fifth is the first vertex of the second edge and so on. The last element
//of the argument list must be -1 to denote the end of the argument list. //of the argument list must be -1 to denote the end of the argument list.
//https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small
ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1);
assert(ret==IGRAPH_SUCCESS); assert(ret==IGRAPH_SUCCESS);
// ignore no such edge to abort(). // ignore no such edge to abort().
igraph_set_error_handler(igraph_error_handler_ignore); igraph_set_error_handler(igraph_error_handler_ignore);
@@ -367,18 +398,25 @@ static size_t effective_vertices_count(igraph_vector_t *vids)
return i; return i;
} }
int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo) int group2group_runtime_build_top_groups(void *g2g_runtime)
{ {
if (NULL == g2g_runtime) {
return -1;
}
struct maat_group *group = NULL, *tmp = NULL; struct maat_group *group = NULL, *tmp = NULL;
struct maat_group *superior_group = NULL; struct maat_group *superior_group = NULL;
int tmp_vid=0; int tmp_vid=0;
size_t i=0, top_group_cnt=0; size_t top_group_cnt=0;
int* temp_group_ids=NULL; int* temp_group_ids=NULL;
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
struct maat_group_topology *group_topo = g2g_rt->group_topo;
assert(group_topo != NULL);
igraph_bool_t is_dag; igraph_bool_t is_dag;
igraph_is_dag(&(group_topo->group_graph), &is_dag); igraph_is_dag(&(group_topo->group_graph), &is_dag);
if (!is_dag) { if (!is_dag) {
log_error(group_topo->logger, MODULE_GROUP, "Sub group cycle detected!"); log_error(g2g_rt->logger, MODULE_GROUP, "Sub group cycle detected!");
return -1; return -1;
} }
@@ -394,7 +432,7 @@ int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo)
&& 0 == group->ref_by_subordinate_group_cnt) { && 0 == group->ref_by_subordinate_group_cnt) {
FREE(group->top_group_ids); FREE(group->top_group_ids);
maat_group_topology_remove_group(group_topo, group); group2group_runtime_remove_group(g2g_runtime, group);
continue; continue;
} }
@@ -449,29 +487,27 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *
} }
int ret = -1; int ret = -1;
struct group2group_item *g2g_item = NULL;
struct group2group_schema *schema = (struct group2group_schema *)g2g_schema; struct group2group_schema *schema = (struct group2group_schema *)g2g_schema;
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime;
int item_id = get_column_value(line, schema->table_id);
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
} else if (0 == is_valid) {
//delete
ret = maat_group_topology_remove_group_from_group(g2g_rt->group_topo, g2g_item->group_id, g2g_item->superior_group_id);
} else {
//add
g2g_item = group2group_item_new(line, schema, logger);
} }
if (1 == is_valid) { struct group2group_item *g2g_item = group2group_item_new(line, schema, g2g_rt->logger);
//add if (NULL == g2g_item) {
ret = maat_group_topology_add_group_to_group(g2g_rt->group_topo, g2g_item->group_id, g2g_item->superior_group_id); return -1;
} else {
//delete
} }
if (0 == is_valid) {
//delete
ret = group2group_runtime_remove_group_from_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id);
} else {
//add
ret = group2group_runtime_add_group_to_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id);
}
group2group_item_free(g2g_item);
return ret; return ret;
} }
@@ -481,24 +517,19 @@ int group2group_runtime_commit(void *g2g_runtime)
return -1; return -1;
} }
struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; return group2group_runtime_build_top_groups(g2g_runtime);
int ret = maat_group_topology_build_top_groups(g2g_rt->group_topo);
return ret;
} }
int group2group_runtime_get_top_groups(struct group2group_runtime *g2g_rt, int *group_ids, size_t n_group_ids, int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids,
int *top_group_ids) int *top_group_ids)
{ {
if (NULL == table_rt || NULL == group_ids || 0 == n_group_ids || if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) {
table_rt->table_type != TABLE_TYPE_GROUP2GROUP) {
return -1; return -1;
} }
size_t top_group_index = 0; size_t top_group_index = 0;
struct group2group_runtime *g2g_rt = &(table_rt->g2g_rt);
for (size_t i = 0; i < n_group_ids; i++) { for (size_t i = 0; i < n_group_ids; i++) {
struct maat_group *group = maat_group_topology_find_group(g2g_rt->group_topo, group_ids[i]); struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_ids[i]);
if (!group) { if (!group) {
continue; continue;
} }

93
src/maat_ip.cpp
View File

@@ -9,6 +9,7 @@
*/ */
#include <stdint.h> #include <stdint.h>
#include <assert.h>
#include "utils.h" #include "utils.h"
#include "log/log.h" #include "log/log.h"
@@ -19,6 +20,7 @@
#include "IPMatcher.h" #include "IPMatcher.h"
#include "maat_ip.h" #include "maat_ip.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_compile.h"
#include "maat_garbage_collection.h" #include "maat_garbage_collection.h"
#define MODULE_IP module_name_str("maat.ip") #define MODULE_IP module_name_str("maat.ip")
@@ -88,10 +90,6 @@ struct ip_plus_runtime {
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger; struct log_handle *logger;
// long long *scan_cnt;
// long long *hit_cnt;
// long long *not_grp_hit_cnt;
// long long *stream_num;
}; };
void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
@@ -101,10 +99,10 @@ void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle
cJSON *custom_item = NULL; cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) { if (item != NULL && item->type == cJSON_Number) {
goto error; ip_plus_schema->table_id = item->valueint;
read_cnt++;
} }
ip_plus_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) { if (NULL == item || item->type != cJSON_Object) {
@@ -136,85 +134,85 @@ void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "sip1"); custom_item = cJSON_GetObjectItem(item, "sip1");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sip1_column = custom_item->valueint; ip_plus_schema->sip1_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "sip2"); custom_item = cJSON_GetObjectItem(item, "sip2");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sip2_column = custom_item->valueint; ip_plus_schema->sip2_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "sport_format"); custom_item = cJSON_GetObjectItem(item, "sport_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport_format_column = custom_item->valueint; ip_plus_schema->sport_format_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "sport1"); custom_item = cJSON_GetObjectItem(item, "sport1");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport1_column = custom_item->valueint; ip_plus_schema->sport1_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "sport2"); custom_item = cJSON_GetObjectItem(item, "sport2");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->sport2_column = custom_item->valueint; ip_plus_schema->sport2_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "daddr_format"); custom_item = cJSON_GetObjectItem(item, "daddr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->daddr_format_column = custom_item->valueint; ip_plus_schema->daddr_format_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "dip1"); custom_item = cJSON_GetObjectItem(item, "dip1");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dip1_column = custom_item->valueint; ip_plus_schema->dip1_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "dip2"); custom_item = cJSON_GetObjectItem(item, "dip2");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dip2_column = custom_item->valueint; ip_plus_schema->dip2_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "dport_format"); custom_item = cJSON_GetObjectItem(item, "dport_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport_format_column = custom_item->valueint; ip_plus_schema->dport_format_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "dport1"); custom_item = cJSON_GetObjectItem(item, "dport1");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport1_column = custom_item->valueint; ip_plus_schema->dport1_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "dport2"); custom_item = cJSON_GetObjectItem(item, "dport2");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->dport2_column = custom_item->valueint; ip_plus_schema->dport2_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "proto"); custom_item = cJSON_GetObjectItem(item, "proto");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->proto_column = custom_item->valueint; ip_plus_schema->proto_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
custom_item = cJSON_GetObjectItem(json, "direction"); custom_item = cJSON_GetObjectItem(item, "direction");
if (custom_item != NULL && custom_item->type == cJSON_Number) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
ip_plus_schema->direction_column = custom_item->valueint; ip_plus_schema->direction_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
if (read_cnt < 17) { if (read_cnt < 18) {
goto error; goto error;
} }
@@ -229,7 +227,7 @@ void ip_plus_schema_free(void *ip_plus_schema)
FREE(ip_plus_schema); FREE(ip_plus_schema);
} }
void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage_bin, void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
if (NULL == ip_plus_schema) { if (NULL == ip_plus_schema) {
@@ -239,7 +237,7 @@ void *ip_plus_runtime_new(void *ip_plus_schema, struct maat_garbage_bin *garbage
struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema; struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema;
struct ip_plus_runtime *ip_plus_rt = ALLOC(struct ip_plus_runtime, 1); struct ip_plus_runtime *ip_plus_rt = ALLOC(struct ip_plus_runtime, 1);
ip_plus_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free); ip_plus_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger);
ip_plus_rt->item_user_data_free = maat_item_inner_free; ip_plus_rt->item_user_data_free = maat_item_inner_free;
ip_plus_rt->ref_garbage_bin = garbage_bin; ip_plus_rt->ref_garbage_bin = garbage_bin;
ip_plus_rt->logger = logger; ip_plus_rt->logger = logger;
@@ -318,7 +316,9 @@ struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *i
ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len); ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len);
if (ret < 0) { if (ret < 0) {
return -1; log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no saddr_format",
ip_plus_schema->table_id, line);
goto error;
} }
memcpy(saddr_format, (line + column_offset), column_len); memcpy(saddr_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) { if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) {
@@ -519,6 +519,12 @@ int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key
return 0; return 0;
} }
int ip_plus_runtime_updating_flag(void *ip_plus_runtime)
{
struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime;
return ex_data_runtime_updating_flag(ip_plus_rt->ex_data_rt);
}
int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line, int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line,
int valid_column) int valid_column)
{ {
@@ -562,12 +568,11 @@ int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const ch
ip_plus_item = ip_plus_item_new(line, schema, ip_plus_rt->logger); ip_plus_item = ip_plus_item_new(line, schema, ip_plus_rt->logger);
if (NULL == ip_plus_item) { if (NULL == ip_plus_item) {
log_error(ip_plus_rt->logger, MODULE_IP, "ip_plus line:%s to item failed", line);
return -1; return -1;
} }
u_para = maat_item_inner_new(ip_plus_item->group_id, item_id, 0); u_para = maat_item_inner_new(ip_plus_item->group_id, item_id, 0);
item = maat_item_new(item_id, group_id, u_para); item = maat_item_new(item_id, ip_plus_item->group_id, u_para);
HASH_ADD_INT(ip_plus_rt->item_hash, item_id, item); HASH_ADD_INT(ip_plus_rt->item_hash, item_id, item);
} }
@@ -619,11 +624,11 @@ int ip_plus_runtime_commit(void *ip_plus_runtime)
size_t mem_used = 0; size_t mem_used = 0;
if (rule_cnt > 0) { if (rule_cnt > 0) {
log_info(logger, MODULE_IP, log_info(ip_plus_rt->logger, MODULE_IP,
"committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt); "committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) { if (NULL == new_ip_matcher) {
log_error(logger, MODULE_IP, log_error(ip_plus_rt->logger, MODULE_IP,
"rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt); "rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt);
ret = -1; ret = -1;
} }
@@ -631,7 +636,7 @@ int ip_plus_runtime_commit(void *ip_plus_runtime)
old_ip_matcher = ip_plus_rt->ip_matcher; old_ip_matcher = ip_plus_rt->ip_matcher;
ip_plus_rt->ip_matcher = new_ip_matcher; ip_plus_rt->ip_matcher = new_ip_matcher;
maat_garbage_bagging(garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
ex_data_runtime_commit(ex_data_rt); ex_data_runtime_commit(ex_data_rt);
ip_plus_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt); ip_plus_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt);
@@ -641,11 +646,11 @@ int ip_plus_runtime_commit(void *ip_plus_runtime)
return ret; return ret;
} }
int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *data, int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr,
int *group_id_array, size_t n_group_id_array, int virtual_table_id, int *group_id_array, size_t n_group_id_array, int virtual_table_id,
struct maat_state *state) struct maat_state *state)
{ {
if (NULL == table_rt) { if (NULL == ip_plus_rt) {
return -1; return -1;
} }
@@ -653,14 +658,10 @@ int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *
struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct ip_data ip; struct ip_data ip;
ip.type = ip_type_transform(scan_data->ip_type); ip.type = IPv4;
if (ip.type == IPv4) { ip.ipv4 = ip_addr;
ip.ipv4 = scan_data->ipv4;
} else { n_hit_item = ip_matcher_match(ip_plus_rt->ip_matcher, &ip, scan_results, MAX_SCANNER_HIT_ITEM_NUM);
memcpy(ip.ipv6, scan_data->ipv6, sizeof(scan_data->ipv6));
}
n_hit_item = ip_matcher_match(table_rt->ip_plus_rt.ip_matcher, &ip, scan_results, MAX_SCANNER_HIT_ITEM_NUM);
if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
} }
@@ -671,7 +672,7 @@ int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *
size_t n_group_id = 0; size_t n_group_id = 0;
int i = 0; int i = 0;
for (i = 0; i < n_hit_item; i++) { for (i = 0; i < n_hit_item; i++) {
HASH_FIND_INT(table_rt->item_hash, &(scan_results[i].rule_id), item); HASH_FIND_INT(ip_plus_rt->item_hash, &(scan_results[i].rule_id), item);
assert(item != NULL); assert(item != NULL);
if (!item) { if (!item) {
// should not come here // should not come here
@@ -686,7 +687,8 @@ int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *
} }
// update hit path // update hit path
maat_compile_state_update_hit_path(compile_state, scan_results[i].rule_id, item->group_id, virtual_table_id, state->scan_cnt, i); maat_compile_state_update_hit_path(compile_state, scan_results[i].rule_id, item->group_id,
virtual_table_id, state->scan_cnt, i);
} }
// update hit clause: literal_id{group_id,vt_id} to clause_id // update hit clause: literal_id{group_id,vt_id} to clause_id
@@ -696,12 +698,13 @@ int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, struct ip_addr *
} else { } else {
compile_table_id = state->compile_table_id; compile_table_id = state->compile_table_id;
} }
struct maat_runtime *maat_rt = state->maat_instance->maat_rt;
struct table_runtime *compile_table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, compile_table_id); void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id);
assert(compile_table_rt->table_type == TABLE_TYPE_COMPILE); enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id);
assert(table_type == TABLE_TYPE_COMPILE);
for (size_t idx = 0; idx < n_group_id; idx++) { for (size_t idx = 0; idx < n_group_id; idx++) {
maat_compile_state_update_hit_clause(compile_state, &(compile_table_rt->compile_rt.compile_hash), group_id_array[idx], virtual_table_id); maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[idx], virtual_table_id);
} }
return n_group_id; return n_group_id;

145
src/maat_ip_plugin.cpp
View File

@@ -8,14 +8,15 @@
*********************************************************************************************** ***********************************************************************************************
*/ */
#include <assert.h>
#include "maat_ip_plugin.h" #include "maat_ip_plugin.h"
#include "cJSON/cJSON.h"
#include "log/log.h"
#include "utils.h" #include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_ex_data.h" #include "maat_ex_data.h"
#include "IPMatcher.h" #include "IPMatcher.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_garbage_collection.h"
#define MODULE_IP_PLUGIN module_name_str("maat.ip_plugin") #define MODULE_IP_PLUGIN module_name_str("maat.ip_plugin")
#define MAX_IP_STR 128 #define MAX_IP_STR 128
@@ -44,16 +45,9 @@ struct ip_plugin_runtime {
uint32_t rule_num; uint32_t rule_num;
uint32_t updating_rule_num; uint32_t updating_rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger; struct log_handle *logger;
// long long *scan_cnt;
// long long *hit_cnt;
// long long *not_grp_hit_cnt;
// long long *stream_num;
}; };
void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
@@ -63,10 +57,10 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl
cJSON *custom_item = NULL; cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) { if (item != NULL && item->type == cJSON_Number) {
goto error; ip_plugin_schema->table_id = item->valueint;
read_cnt++;
} }
ip_plugin_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (NULL == item || item->type != cJSON_Object) { if (NULL == item || item->type != cJSON_Object) {
@@ -98,7 +92,7 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl
read_cnt++; read_cnt++;
} }
if (read_cnt < 4) { if (read_cnt < 5) {
goto error; goto error;
} }
@@ -193,42 +187,40 @@ void ip_plugin_item_free(struct ip_plugin_item *item)
int ip_plugin_table_ex_data_schema_flag(struct ip_plugin_schema *ip_plugin_schema) int ip_plugin_table_ex_data_schema_flag(struct ip_plugin_schema *ip_plugin_schema)
{ {
return 0;
} }
int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func, maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp) long argl, void *argp,
struct log_handle *logger)
{ {
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; if (NULL == ip_plugin_schema) {
struct ex_data_schema *ex_schema = schema->ex_schema;
if (ex_schema->set_flag) {
assert(0);
log_error(logger, MODULE_TABLE, "Error: %s, EX data schema already registed",
__FUNCTION__);
return -1; return -1;
} }
ex_schema->new_func = new_func; struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
ex_schema->free_func = free_func; if (schema->ex_schema != NULL) {
ex_schema->dup_func = dup_func; assert(0);
ex_schema->argl = argl; log_error(logger, MODULE_IP_PLUGIN,
ex_schema->argp = argp; "Error: %s, EX data schema already registed", __FUNCTION__);
//ex_schema->set_flag = 1; return -1;
}
schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp);
return 0; return 0;
} }
int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_schema *schema, int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_schema *schema,
const char *row, char *key, size_t key_len, struct ip_plugin_item *item, const char *row, char *key, size_t key_len,
int is_valid) struct ip_plugin_item *ip_plugin_item, int is_valid)
{ {
int ret = -1; int ret = -1;
struct ex_data_runtime *ex_data_rt = rt->ex_data_rt; struct ex_data_runtime *ex_data_rt = rt->ex_data_rt;
int set_flag = ip_plugin_table_schema_ex_data_schema_flag(schema); int set_flag = ip_plugin_table_ex_data_schema_flag(schema);
if (1 == set_flag) { if (1 == set_flag) {
if (0 == is_valid) { if (0 == is_valid) {
@@ -240,7 +232,7 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_
} else { } else {
//add //add
void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len);
struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)item); struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)ip_plugin_item);
ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
@@ -253,7 +245,7 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_
return 0; return 0;
} }
void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *garbage_bin, void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
if (NULL == ip_plugin_schema) { if (NULL == ip_plugin_schema) {
@@ -263,8 +255,7 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, struct maat_garbage_bin *gar
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1); struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1);
ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free); ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger);
ip_plugin_rt->item_user_data_free = maat_item_inner_free;
ip_plugin_rt->ref_garbage_bin = garbage_bin; ip_plugin_rt->ref_garbage_bin = garbage_bin;
ip_plugin_rt->logger = logger; ip_plugin_rt->logger = logger;
@@ -280,21 +271,33 @@ void ip_plugin_runtime_free(void *ip_plugin_runtime)
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
if (ip_plugin_rt->ip_matcher != NULL) { if (ip_plugin_rt->ip_matcher != NULL) {
ip_matcher_free(ip_plugin_rt->ip_matcher); ip_matcher_free(ip_plugin_rt->ip_matcher);
ip_plugin_rt->ip_matcher = NULL;
} }
if (ip_plugin_rt->ex_data_rt != NULL) { if (ip_plugin_rt->ex_data_rt != NULL) {
ex_data_runtime_free(ip_plugin_rt->ex_data_rt); ex_data_runtime_free(ip_plugin_rt->ex_data_rt);
} ip_plugin_rt->ex_data_rt = NULL;
struct maat_item *item = NULL, *tmp_item = NULL;
HASH_ITER(hh, ip_plugin_rt->item_hash, item, tmp_item) {
HASH_DELETE(hh, ip_plugin_rt->item_hash, item);
maat_item_free(item, ip_plugin_rt->item_user_data_free);
} }
FREE(ip_plugin_rt); FREE(ip_plugin_rt);
} }
void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule)
{
if (4 == item->ip_type) {
rule->type = IPv4;
ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip,
&(rule->ipv4_rule.start_ip), &(rule->ipv4_rule.end_ip));
} else {
rule->type = IPv6;
ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip,
rule->ipv6_rule.start_ip, rule->ipv6_rule.end_ip);
}
rule->rule_id = item->item_id;
rule->user_tag = NULL;
}
int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line, int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line,
int valid_column) int valid_column)
{ {
@@ -302,57 +305,28 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, co
return -1; return -1;
} }
struct maat_item *item = NULL;
struct ip_plugin_item *ip_plugin_item = NULL; struct ip_plugin_item *ip_plugin_item = NULL;
struct maat_item_inner *u_para = NULL;
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
int item_id = get_column_value(line, schema->item_id_column); int item_id = get_column_value(line, schema->item_id_column);
int is_valid = get_column_value(line, valid_column); int is_valid = get_column_value(line, valid_column);
if (is_valid < 0) { if (is_valid < 0) {
return -1; return -1;
} else if (0 == is_valid) { }
//delete
HASH_FIND_INT(ip_plugin_rt->item_hash, &item_id, item); if (1 == is_valid) {
if (NULL == item) {
return -1;
}
u_para = (struct maat_item_inner *)item->user_data;
item->user_data = NULL;
if (NULL == u_para) {
return -1;
}
HASH_DELETE(hh, ip_plugin_rt->item_hash, item);
maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free);
} else {
//add //add
HASH_FIND_INT(ip_plugin_rt->item_hash, &item_id, item);
if (item) {
log_error(ip_plus_rt->logger, MODULE_IP_PLUGIN,
"ip_plugin runtime add item %d to item_hash failed, already exist", item_id);
return -1;
}
ip_plugin_item = ip_plugin_item_new(line, schema, ip_plugin_rt->logger); ip_plugin_item = ip_plugin_item_new(line, schema, ip_plugin_rt->logger);
if (NULL == ip_plugin_item) { if (NULL == ip_plugin_item) {
log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, "ip_plugin line:%s to item failed", line);
return -1; return -1;
} }
u_para = maat_item_inner_new(ip_plugin_item->group_id, item_id, 0);
item = maat_item_new(item_id, group_id, u_para);
HASH_ADD_INT(ip_plugin_rt->item_hash, item_id, item);
} }
char *key = (char *)&item_id; char *key = (char *)&item_id;
int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, row, key, sizeof(int), ip_plugin_item, is_valid); int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key, sizeof(int), ip_plugin_item, is_valid);
if (ret < 0) { if (ret < 0) {
if (ip_plugin_item != NULL) { if (ip_plugin_item != NULL) {
ip_plugin_item_free(ip_plugin_item); FREE(ip_plugin_item);
ip_plugin_item = NULL;
} }
return -1; return -1;
} else { } else {
@@ -366,20 +340,6 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, co
return 0; return 0;
} }
void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule)
{
if (4 == item->ip_type) {
rule->type = IPv4;
ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, &(rule->ipv4_rule.start_ip), &(rule->ipv4_rule.end_ip));
} else {
rule->type = IPv6;
ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, &(rule->ipv6_rule.start_ip), &(rule->ipv6_rule.end_ip));
}
rule->rule_id = item->item_id;
rule->user_tag = NULL;
}
int ip_plugin_runtime_commit(void *ip_plugin_runtime) int ip_plugin_runtime_commit(void *ip_plugin_runtime)
{ {
if (NULL == ip_plugin_runtime) { if (NULL == ip_plugin_runtime) {
@@ -409,11 +369,11 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime)
size_t mem_used = 0; size_t mem_used = 0;
if (rule_cnt > 0) { if (rule_cnt > 0) {
log_info(ip_plugin_rt->logger, MODULE_TABLE_RUNTIME, log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt); "committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) { if (NULL == new_ip_matcher) {
log_error(ip_plugin_rt->logger, MODULE_TABLE_RUNTIME, log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt); "rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt);
ret = -1; ret = -1;
} }
@@ -431,8 +391,9 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime)
return ret; return ret;
} }
int ip_plugin_runtime_updating_flag(struct ip_plugin_runtime *ip_plugin_rt) int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime)
{ {
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt); return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt);
} }

94
src/maat_plugin.cpp
View File

@@ -14,7 +14,6 @@
#include "cJSON/cJSON.h" #include "cJSON/cJSON.h"
#include "utils.h" #include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat.h"
#include "maat_plugin.h" #include "maat_plugin.h"
#include "maat_ex_data.h" #include "maat_ex_data.h"
#include "maat_limits.h" #include "maat_limits.h"
@@ -28,11 +27,6 @@ struct plugin_callback_schema {
void *u_para; void *u_para;
}; };
struct plugin_item {
char key[MAX_KEYWORDS_STR];
size_t key_len;
};
struct plugin_runtime { struct plugin_runtime {
uint64_t acc_line_num; uint64_t acc_line_num;
struct ex_data_runtime *ex_data_rt; struct ex_data_runtime *ex_data_rt;
@@ -41,6 +35,7 @@ struct plugin_runtime {
uint32_t updating_rule_num; uint32_t updating_rule_num;
struct maat_garbage_bin *ref_garbage_bin; struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
}; };
#define MAX_PLUGIN_PER_TABLE 32 #define MAX_PLUGIN_PER_TABLE 32
@@ -80,10 +75,10 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *
cJSON *custom_item = NULL; cJSON *custom_item = NULL;
cJSON *item = cJSON_GetObjectItem(json, "table_id"); cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) { if (item != NULL && item->type == cJSON_Number) {
goto error; plugin_schema->table_id = item->valueint;
read_cnt++;
} }
plugin_schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "custom"); item = cJSON_GetObjectItem(json, "custom");
if (item == NULL || item->type != cJSON_Object) { if (item == NULL || item->type != cJSON_Object) {
@@ -92,7 +87,7 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *
} }
custom_item = cJSON_GetObjectItem(item, "item_id"); custom_item = cJSON_GetObjectItem(item, "item_id");
if (custom_item == NULL || custom_item->type != cJSON_Object) { if (custom_item != NULL && custom_item->type == cJSON_Number) {
plugin_schema->item_id_column = custom_item->valueint; plugin_schema->item_id_column = custom_item->valueint;
read_cnt++; read_cnt++;
} }
@@ -126,7 +121,7 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *
} }
} }
if (read_cnt < 4) { if (read_cnt < 5) {
goto error; goto error;
} }
@@ -197,41 +192,6 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema)
} }
} }
struct plugin_item *plugin_item_new(const char *line, struct plugin_schema *plugin_schema,
struct log_handle *logger)
{
size_t column_offset = 0;
size_t column_len = 0;
struct plugin_item *plugin_item = ALLOC(struct plugin_item, 1);
int ret = get_column_pos(line, plugin_schema->key_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_PLUGIN,
"plugin table(table_id:%d) line:%s has no key",
plugin_schema->table_id, line);
goto error;
}
if (column_len > MAX_KEYWORDS_STR) {
log_error(logger, MODULE_PLUGIN,
"plugin table(table_id:%d): key:%s length:%zu too long, exceed %d",
plugin_schema->table_id, (line + column_offset), column_len, MAX_KEYWORDS_STR);
goto error;
}
memcpy(plugin_item->key, (line + column_offset), column_len);
plugin_item->key_len = column_len;
return plugin_item;
error:
FREE(plugin_item);
return NULL;
}
void plugin_item_free(struct plugin_item *plugin_item)
{
FREE(plugin_item);
}
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns) int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns)
{ {
if (NULL == plugin_schema) { if (NULL == plugin_schema) {
@@ -279,6 +239,43 @@ struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema)
return schema->ex_schema; return schema->ex_schema;
} }
void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
if (NULL == plugin_schema) {
return NULL;
}
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
struct plugin_runtime *plugin_rt = ALLOC(struct plugin_runtime, 1);
plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger);
plugin_rt->ref_garbage_bin = garbage_bin;
plugin_rt->logger = logger;
return plugin_rt;
}
void plugin_runtime_free(void *plugin_runtime)
{
if (NULL == plugin_runtime) {
return;
}
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
if (plugin_rt->ex_data_rt != NULL) {
ex_data_runtime_free(plugin_rt->ex_data_rt);
plugin_rt->ex_data_rt = NULL;
}
FREE(plugin_rt);
}
int plugin_runtime_updating_flag(void *plugin_runtime)
{
struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt);
}
int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_schema *plugin_schema, int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_schema *plugin_schema,
const char *row, char *key, size_t key_len, int is_valid) const char *row, char *key, size_t key_len, int is_valid)
{ {
@@ -359,11 +356,6 @@ int plugin_runtime_commit(void *plugin_runtime)
return 0; return 0;
} }
int plugin_runtime_updating_flag(struct plugin_runtime *plugin_rt)
{
return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt);
}
struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime) struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime)
{ {
if (NULL == plugin_runtime) { if (NULL == plugin_runtime) {

12
src/maat_redis_monitor.cpp
View File

@@ -18,6 +18,7 @@
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
#include "maat_redis_monitor.h" #include "maat_redis_monitor.h"
#include "maat_plugin.h" #include "maat_plugin.h"
#include "maat_virtual.h"
#define MODULE_REDIS_MONITOR module_name_str("maat.redis_monitor") #define MODULE_REDIS_MONITOR module_name_str("maat.redis_monitor")
@@ -97,7 +98,6 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f
int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_instance, const char *dir) int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_instance, const char *dir)
{ {
int rule_with_foreign_key = 0; int rule_with_foreign_key = 0;
void *schema = NULL;
for (int i = 0; i < rule_num; i++) { for (int i = 0; i < rule_num; i++) {
if (NULL == rule_list[i].table_line) { if (NULL == rule_list[i].table_line) {
@@ -105,9 +105,9 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in
} }
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, rule_list[i].table_name); int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, rule_list[i].table_name);
schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
if (!table_schema || table_type != TABLE_TYPE_PLUGIN) { if (!schema || table_type != TABLE_TYPE_PLUGIN) {
continue; continue;
} }
@@ -607,7 +607,7 @@ FULL_UPDATE:
continue; continue;
} }
if (table_schema_mgr) { if (tbl_mgr) {
int table_id = table_manager_get_table_id(tbl_mgr, s_rule_array[full_idx].table_name); int table_id = table_manager_get_table_id(tbl_mgr, s_rule_array[full_idx].table_name);
//Unrecognized table. //Unrecognized table.
if (table_id < 0) { if (table_id < 0) {
@@ -1277,7 +1277,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
int valid_column = -1; int valid_column = -1;
enum table_type table_type; enum table_type table_type;
enum scan_type scan_type; enum scan_type scan_type;
void *table_schema = NULL; //void *table_schema = NULL;
struct maat *maat_instance = (struct maat *)u_param; struct maat *maat_instance = (struct maat *)u_param;
//authorized to write //authorized to write
@@ -1315,7 +1315,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
int update_type = CM_UPDATE_TYPE_INC; int update_type = CM_UPDATE_TYPE_INC;
int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version, int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version,
&new_version, maat_instance->table_schema_mgr, &rule_list, &new_version, maat_instance->tbl_mgr, &rule_list,
&update_type, maat_instance->cumulative_update_off, &update_type, maat_instance->cumulative_update_off,
maat_instance->logger); maat_instance->logger);
//redis communication error //redis communication error

103
src/maat_rule.cpp
View File

@@ -19,6 +19,8 @@
#include "utils.h" #include "utils.h"
#include "json2iris.h" #include "json2iris.h"
#include "log/log.h"
#include "cJSON/cJSON.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
@@ -27,6 +29,7 @@
#include "maat_compile.h" #include "maat_compile.h"
#include "maat_plugin.h" #include "maat_plugin.h"
#include "alignment.h" #include "alignment.h"
#include "maat_garbage_collection.h"
#define MODULE_MAAT_RULE module_name_str("maat.rule") #define MODULE_MAAT_RULE module_name_str("maat.rule")
@@ -51,7 +54,7 @@ void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)
free(item); free(item);
} }
static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, int n_accept_tag) static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, size_t n_accept_tag)
{ {
if (NULL == tag_obj || NULL == accept_tags) { if (NULL == tag_obj || NULL == accept_tags) {
return -1; return -1;
@@ -70,7 +73,7 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags,
int name_matched = 0; int name_matched = 0;
int n_val = cJSON_GetArraySize(tag_vals_array); int n_val = cJSON_GetArraySize(tag_vals_array);
for (int i = 0; i < n_accept_tag; i++) { for (size_t i = 0; i < n_accept_tag; i++) {
if (0 != strcmp(accept_tags[i].tag_name, tag_name)) { if (0 != strcmp(accept_tags[i].tag_name, tag_name)) {
continue; continue;
} }
@@ -106,7 +109,7 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags,
} }
//@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}] //@param tag_set likes [{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}]
static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_tags, int n_accept_tag) static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_tags, size_t n_accept_tag)
{ {
int matched = 0; int matched = 0;
@@ -138,9 +141,13 @@ error:
} }
//@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]} //@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]}
int parse_accept_tag(const char *value, struct rule_tag **result, void *logger) size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger)
{ {
cJSON *json = JSON_Parse(value); if (NULL == value || NULL == result || NULL == logger) {
return 0;
}
cJSON *json = cJSON_Parse(value);
if (!json) { if (!json) {
log_error(logger, MODULE_MAAT_RULE, log_error(logger, MODULE_MAAT_RULE,
"parse accept tag Error before: %-200.200s", cJSON_GetErrorPtr()); "parse accept tag Error before: %-200.200s", cJSON_GetErrorPtr());
@@ -167,7 +174,7 @@ int parse_accept_tag(const char *value, struct rule_tag **result, void *logger)
//@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]} //@param value {"tag_sets":[[{"tag":"location","value":["北京/朝阳/华严北里","上海/浦东/陆家嘴"]},{"tag":"isp","value":["电信","移动"]}],[{"tag":"location","value":["北京"]},{"tag":"isp","value":["联通"]}]]}
//@return 1 on match, 0 on not match, -1 on error. //@return 1 on match, 0 on not match, -1 on error.
int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, int n_tag) int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag)
{ {
int ret = -1; int ret = -1;
int n_set = 0; int n_set = 0;
@@ -191,7 +198,7 @@ int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, in
goto error; goto error;
} }
ret = compare_each_tag_set(tag_set, accept_tags, n_tag); ret = compare_each_tag_set(tag_set, accept_tags, n_accept_tag);
//match or error occurs. //match or error occurs.
if (ret != 0) { if (ret != 0) {
break; break;
@@ -214,8 +221,13 @@ struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int distr
return item; return item;
} }
void maat_item_inner_free(struct maat_item_inner *item) void maat_item_inner_free(void *item_inner)
{ {
if (NULL == item_inner) {
return;
}
struct maat_item_inner *item = (struct maat_item_inner *)item_inner;
assert(item->magic_num == ITEM_RULE_MAGIC); assert(item->magic_num == ITEM_RULE_MAGIC);
assert(item->expr_id_cnt == 0 || item->expr_id_cnt == item->expr_id_ub - item->expr_id_lb + 1); assert(item->expr_id_cnt == 0 || item->expr_id_cnt == item->expr_id_ub - item->expr_id_lb + 1);
item->magic_num = 0; item->magic_num = 0;
@@ -228,15 +240,15 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in
struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1); struct maat_runtime *maat_rt = ALLOC(struct maat_runtime, 1);
maat_rt->version = version; maat_rt->version = version;
int ret = table_manager_init(maat_instance->tbl_mgr, maat_instance->garbage_bin); int ret = table_manager_runtime_create(maat_instance->tbl_mgr, maat_instance->nr_worker_thread,
maat_instance->garbage_bin);
if (ret < 0) { if (ret < 0) {
FREE(maat_rt); FREE(maat_rt);
return NULL; return NULL;
} }
maat_rt->ref_tbl_mgr = maat_instance->tbl_mgr; maat_rt->ref_tbl_mgr = maat_instance->tbl_mgr;
maat_rt->max_table_num = table_manager_table_count(maat_instance->table_mgr); maat_rt->max_table_num = table_manager_table_count(maat_instance->tbl_mgr);
maat_rt->max_thread_num = maat_instance->nr_worker_thread;
maat_rt->logger = maat_instance->logger; maat_rt->logger = maat_instance->logger;
maat_rt->ref_garbage_bin = maat_instance->garbage_bin; maat_rt->ref_garbage_bin = maat_instance->garbage_bin;
@@ -248,12 +260,7 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in
void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger) void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger)
{ {
for (size_t i = 0; i < maat_rt->max_table_num; i++) { for (size_t i = 0; i < maat_rt->max_table_num; i++) {
void *runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, i); table_manager_commit_runtime(maat_rt->ref_tbl_mgr, i);
if (NULL == runtime) {
continue;
}
table_manager_commit_runtime(runtime, maat_rt->version, maat_rt->max_thread_num, logger);
} }
maat_rt->last_update_time = time(NULL); maat_rt->last_update_time = time(NULL);
@@ -265,9 +272,9 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt)
return; return;
} }
if (maat_rt->table_rt_mgr != NULL) { if (maat_rt->ref_tbl_mgr != NULL) {
table_manager_runtime_destroy(maat_rt->ref_tbl_mgr); table_manager_runtime_destroy(maat_rt->ref_tbl_mgr);
maat_rt->ref_table_mgr = NULL; maat_rt->ref_tbl_mgr = NULL;
} }
FREE(maat_rt); FREE(maat_rt);
@@ -275,13 +282,10 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt)
int maat_runtime_updating_flag(struct maat_runtime *maat_rt) int maat_runtime_updating_flag(struct maat_runtime *maat_rt)
{ {
int flag = -1;
for (size_t i = 0; i < maat_rt->max_table_num; i++) { for (size_t i = 0; i < maat_rt->max_table_num; i++) {
struct table_runtime *table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, i); flag = table_manager_runtime_updating_flag(maat_rt->ref_tbl_mgr, i);
if (NULL == table_rt) {
continue;
}
int flag = table_runtime_updating_flag(table_rt);
if (1 == flag) { if (1 == flag) {
return 1; return 1;
} }
@@ -300,11 +304,9 @@ void maat_start_cb(long long new_version, int update_type, void *u_param)
maat_instance->maat_version = new_version; maat_instance->maat_version = new_version;
} }
int table_id = -1;
enum table_type table_type = TABLE_TYPE_MAX;
size_t table_cnt = table_manager_table_count(maat_instance->tbl_mgr); size_t table_cnt = table_manager_table_count(maat_instance->tbl_mgr);
for (size_t i = 0; i < table_cnt; i++) { for (size_t i = 0; i < table_cnt; i++) {
table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i);
if (table_type != TABLE_TYPE_PLUGIN) { if (table_type != TABLE_TYPE_PLUGIN) {
continue; continue;
} }
@@ -342,40 +344,49 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
maat_rt = maat_instance->maat_rt; maat_rt = maat_instance->maat_rt;
} }
void *runtime = table_manager_get_runtime(maat_rt->tbl_mgr, table_id); table_manager_update_runtime(maat_rt->ref_tbl_mgr, table_id, line);
int ret = table_manager_update_runtime(runtime, schema, line);
if (ret < 0) {
log_error(maat_instance->logger, MODULE_MAAT_RULE,
"table manager update runtime error, table_name:%s", table_name);
return -1;
}
//TODO: by luis
//int is_valid = table_manager_get_valid
return 0; return 0;
} }
uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt) uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt)
{ {
uint32_t total = 0; uint32_t total = 0;
struct table_runtime *table_rt = NULL; void *runtime = NULL;
for (size_t i = 0; i < maat_rt->max_table_num; i++) { for (size_t i = 0; i < maat_rt->max_table_num; i++) {
table_rt = table_manager_get_runtime(maat_rt->tbl_mgr, i); runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, i);
if (table_rt != NULL) { if (runtime != NULL) {
total += table_runtime_rule_count(table_rt); //TODO: by luis
//total += table_runtime_rule_count(runtime);
} }
} }
return total; return total;
} }
void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr)
{
size_t table_cnt = table_manager_table_count(tbl_mgr);
enum table_type table_type = TABLE_TYPE_MAX;
for (size_t i = 0; i < table_cnt; i++) {
table_type = table_manager_get_table_type(tbl_mgr, i);
if (table_type != TABLE_TYPE_PLUGIN) {
continue;
}
void *plugin_schema = table_manager_get_schema(tbl_mgr, i);
plugin_table_all_callback_finish((struct plugin_schema *)plugin_schema);
}
}
void maat_finish_cb(void *u_param) void maat_finish_cb(void *u_param)
{ {
struct maat *maat_instance = (struct maat *)u_param; struct maat *maat_instance = (struct maat *)u_param;
//table_manager_all_plugin_cb_finish(maat_instance->table_schema_mgr); maat_plugin_table_all_callback_finish(maat_instance->tbl_mgr);
plugin_table_all_callback_finish(maat_)
if (maat_instance->creating_maat_rt != NULL) { if (maat_instance->creating_maat_rt != NULL) {
maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt);
maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger); maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger);
@@ -515,12 +526,6 @@ void *rule_monitor_loop(void *arg)
} }
} }
for (int i = 0; i < maat_instance->n_accept_tag; i++) {
FREE(maat_instance->accept_tags[i].tag_name);
FREE(maat_instance->accept_tags[i].tag_val);
}
FREE(maat_instance->accept_tags);
FREE(maat_instance); FREE(maat_instance);
return NULL; return NULL;

320
src/maat_table.cpp
View File

@@ -12,6 +12,7 @@
#include <assert.h> #include <assert.h>
#include "log/log.h" #include "log/log.h"
#include "utils.h"
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_table.h" #include "maat_table.h"
#include "maat_rule.h" #include "maat_rule.h"
@@ -23,6 +24,7 @@
#include "maat_group.h" #include "maat_group.h"
#include "maat_plugin.h" #include "maat_plugin.h"
#include "maat_ip_plugin.h" #include "maat_ip_plugin.h"
#include "maat_virtual.h"
#define MODULE_TABLE module_name_str("maat.table") #define MODULE_TABLE module_name_str("maat.table")
@@ -45,8 +47,9 @@ struct table_manager {
size_t n_table; size_t n_table;
struct rule_tag *accept_tags; struct rule_tag *accept_tags;
int n_accept_tag; size_t n_accept_tag;
int default_compile_table_id;
struct maat_kv_store *tablename2id_map; struct maat_kv_store *tablename2id_map;
struct log_handle *logger; struct log_handle *logger;
}; };
@@ -56,11 +59,12 @@ struct table_operations {
void *(*new_schema)(cJSON *json, const char *table_name, struct log_handle *logger); void *(*new_schema)(cJSON *json, const char *table_name, struct log_handle *logger);
void (*free_schema)(void *schema); void (*free_schema)(void *schema);
void *(*new_runtime)(void *schema, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void *(*new_runtime)(void *schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger);
void (*free_runtime)(void *runtime); void (*free_runtime)(void *runtime);
int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column); int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column);
int (*commit_runtime)(void *runtime); int (*commit_runtime)(void *runtime);
int (*runtime_updating_flag)(void *runtime);
}; };
struct table_operations table_ops[TABLE_TYPE_MAX] = { struct table_operations table_ops[TABLE_TYPE_MAX] = {
@@ -71,7 +75,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = expr_runtime_new, .new_runtime = expr_runtime_new,
.free_runtime = expr_runtime_free, .free_runtime = expr_runtime_free,
.update_runtime = expr_runtime_update, .update_runtime = expr_runtime_update,
.commit_runtime = expr_runtime_commit .commit_runtime = expr_runtime_commit,
.runtime_updating_flag = expr_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_EXPR_PLUS, .type = TABLE_TYPE_EXPR_PLUS,
@@ -80,7 +85,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = expr_runtime_new, .new_runtime = expr_runtime_new,
.free_runtime = expr_runtime_free, .free_runtime = expr_runtime_free,
.update_runtime = expr_runtime_update, .update_runtime = expr_runtime_update,
.commit_runtime = expr_runtime_commit .commit_runtime = expr_runtime_commit,
.runtime_updating_flag = expr_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_IP_PLUS, .type = TABLE_TYPE_IP_PLUS,
@@ -89,7 +95,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = ip_plus_runtime_new, .new_runtime = ip_plus_runtime_new,
.free_runtime = ip_plus_runtime_free, .free_runtime = ip_plus_runtime_free,
.update_runtime = ip_plus_runtime_update, .update_runtime = ip_plus_runtime_update,
.commit_runtime = ip_plus_runtime_commit .commit_runtime = ip_plus_runtime_commit,
.runtime_updating_flag = ip_plus_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_INTERVAL, .type = TABLE_TYPE_INTERVAL,
@@ -98,7 +105,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL .commit_runtime = NULL,
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_INTERVAL_PLUS, .type = TABLE_TYPE_INTERVAL_PLUS,
@@ -107,7 +115,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL .commit_runtime = NULL,
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_DIGEST, .type = TABLE_TYPE_DIGEST,
@@ -116,7 +125,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL .commit_runtime = NULL,
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_SIMILARITY, .type = TABLE_TYPE_SIMILARITY,
@@ -125,7 +135,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL .commit_runtime = NULL,
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_CONJUNCTION, .type = TABLE_TYPE_CONJUNCTION,
@@ -134,12 +145,18 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
.commit_runtime = NULL .commit_runtime = NULL,
.runtime_updating_flag = NULL
}, },
{ {
.type = TABLE_TYPE_PLUGIN, .type = TABLE_TYPE_PLUGIN,
.new_schema = plugin_schema_new, .new_schema = plugin_schema_new,
.free_schema = plugin_schema_free,
.new_runtime = plugin_runtime_new,
.free_runtime = plugin_runtime_free,
.update_runtime = plugin_runtime_update,
.commit_runtime = plugin_runtime_commit,
.runtime_updating_flag = plugin_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_IP_PLUGIN, .type = TABLE_TYPE_IP_PLUGIN,
@@ -148,7 +165,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = ip_plugin_runtime_new, .new_runtime = ip_plugin_runtime_new,
.free_runtime = ip_plugin_runtime_free, .free_runtime = ip_plugin_runtime_free,
.update_runtime = ip_plugin_runtime_update, .update_runtime = ip_plugin_runtime_update,
.commit_runtime = ip_plugin_runtime_commit .commit_runtime = ip_plugin_runtime_commit,
.runtime_updating_flag = ip_plugin_runtime_updating_flag
}, },
{ {
.type = TABLE_TYPE_FQDN_PLUGIN, .type = TABLE_TYPE_FQDN_PLUGIN,
@@ -170,8 +188,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
}, },
{ {
.type = TABLE_TYPE_VIRTUAL, .type = TABLE_TYPE_VIRTUAL,
.new_schema = NULL, .new_schema = virtual_schema_new,
.free_schema = NULL, .free_schema = virtual_schema_free,
.new_runtime = NULL, .new_runtime = NULL,
.free_runtime = NULL, .free_runtime = NULL,
.update_runtime = NULL, .update_runtime = NULL,
@@ -184,16 +202,8 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = compile_runtime_new, .new_runtime = compile_runtime_new,
.free_runtime = compile_runtime_free, .free_runtime = compile_runtime_free,
.update_runtime = compile_runtime_update, .update_runtime = compile_runtime_update,
.commit_runtime = compile_runtime_commit .commit_runtime = compile_runtime_commit,
}, .runtime_updating_flag = NULL
{
.type = TABLE_TYPE_GROUP2COMPILE,
.new_schema = group2compile_schema_new,
.free_schema = group2compile_schema_free,
.new_runtime = NULL,
.free_runtime = NULL,
.update_runtime = group2compile_runtime_update,
.commit_runtime = NULL
}, },
{ {
.type = TABLE_TYPE_GROUP2GROUP, .type = TABLE_TYPE_GROUP2GROUP,
@@ -202,11 +212,23 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
.new_runtime = group2group_runtime_new, .new_runtime = group2group_runtime_new,
.free_runtime = group2group_runtime_free, .free_runtime = group2group_runtime_free,
.update_runtime = group2group_runtime_update, .update_runtime = group2group_runtime_update,
.commit_runtime = group2group_runtime_commit .commit_runtime = group2group_runtime_commit,
.runtime_updating_flag = NULL
},
{
.type = TABLE_TYPE_GROUP2COMPILE,
.new_schema = group2compile_schema_new,
.free_schema = group2compile_schema_free,
.new_runtime = group2compile_runtime_new,
.free_runtime = group2compile_runtime_free,
.update_runtime = group2compile_runtime_update,
.commit_runtime = NULL,
.runtime_updating_flag = NULL
} }
}; };
void *maat_table_schema_new(cJSON *json, const char *table_name, enum table_type table_type) void *maat_table_schema_new(cJSON *json, const char *table_name, enum table_type table_type,
struct log_handle *logger)
{ {
void *schema = NULL; void *schema = NULL;
@@ -258,8 +280,9 @@ static void register_tablename2id(cJSON *json, struct maat_kv_store *tablename2i
} }
if (strlen(item->valuestring) >= NAME_MAX) { if (strlen(item->valuestring) >= NAME_MAX) {
log_error(logger, MODULE_TABLE_SCHEMA, log_error(logger, MODULE_TABLE,
"table(table_id:%d) name %s length too long", table_id, item->valuestring); "table(table_id:%d) name %s length too long",
table_id, item->valuestring);
return; return;
} }
@@ -311,10 +334,13 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo
} }
item = cJSON_GetObjectItem(json, "valid_column"); item = cJSON_GetObjectItem(json, "valid_column");
if (NULL == item && item->type != cJSON_Number) { if (NULL == item || item->type != cJSON_Number) {
goto error; if (ptable->table_type != TABLE_TYPE_VIRTUAL) {
goto error;
}
} else {
ptable->valid_column = item->valueint;
} }
ptable->valid_column = item->valueint;
return ptable; return ptable;
error: error:
@@ -322,6 +348,17 @@ error:
return NULL; return NULL;
} }
void maat_table_runtime_free(void *runtime, enum table_type table_type)
{
if (NULL == runtime) {
return;
}
if (table_ops[table_type].free_runtime != NULL) {
table_ops[table_type].free_runtime(runtime);
}
}
void maat_table_free(struct maat_table *maat_tbl) void maat_table_free(struct maat_table *maat_tbl)
{ {
if (NULL == maat_tbl) { if (NULL == maat_tbl) {
@@ -329,12 +366,12 @@ void maat_table_free(struct maat_table *maat_tbl)
} }
if (maat_tbl->schema != NULL) { if (maat_tbl->schema != NULL) {
maat_table_schema_free(maat_tbl->schema); maat_table_schema_free(maat_tbl->schema, maat_tbl->table_type);
maat_tbl->schema = NULL; maat_tbl->schema = NULL;
} }
if (maat_tbl->runtime != NULL) { if (maat_tbl->runtime != NULL) {
maat_table_runtime_free(maat_tbl->runtime); maat_table_runtime_free(maat_tbl->runtime, maat_tbl->table_type);
maat_tbl->runtime = NULL; maat_tbl->runtime = NULL;
} }
@@ -400,9 +437,9 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
continue; continue;
} }
maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type); maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type, logger);
if (NULL == maat_tbl->schema) { if (NULL == maat_tbl->schema) {
log_error(logger, MODULE_TABLE, "Maat table schema new failed, table_name:%d", log_error(logger, MODULE_TABLE, "Maat table schema new failed, table_name:%s",
maat_tbl->table_name); maat_tbl->table_name);
maat_table_free(maat_tbl); maat_table_free(maat_tbl);
continue; continue;
@@ -431,89 +468,20 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch
} }
void *maat_table_runtime_new(void *schema, enum table_type table_type, void *maat_table_runtime_new(void *schema, enum table_type table_type,
struct maat_garbage_bin *garbage_bin, int max_thread_num, struct maat_garbage_bin *garbage_bin,
struct log_handle *logger) struct log_handle *logger)
{ {
void *runtime = NULL; void *runtime = NULL;
if (table_ops[table_type].new_runtime != NULL) { if (table_ops[table_type].new_runtime != NULL) {
runtime = table_ops[table_type].new_runtime(schema, garbage_bin, logger); runtime = table_ops[table_type].new_runtime(schema, max_thread_num, garbage_bin, logger);
} }
return runtime; return runtime;
#if 0
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
break;
case TABLE_TYPE_GROUP2COMPILE:
break;
case TABLE_TYPE_GROUP2GROUP:
table_rt->custom_rt = group2group_runtime_new(logger);
table_rt->g2g_rt.group_topo = maat_group_topology_new(logger);
break;
case TABLE_TYPE_EXPR:
table_rt->expr_rt.htable = rcu_hash_new(expr_ex_data_free);
table_rt->expr_rt.scan_mode = expr_table_schema_get_scan_mode(table_schema);
break;
case TABLE_TYPE_IP_PLUS:
table_rt->ip_plus_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
ex_container_ctx = ALLOC(struct ex_container_ctx, 1);
ex_container_ctx->custom_data_free = free;
//ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx);
break;
case TABLE_TYPE_PLUGIN:
table_rt->plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
break;
case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin_rt.ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free);
ex_container_ctx = ALLOC(struct ex_container_ctx, 1);
ex_container_ctx->custom_data_free = free;
//ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_container_ctx);
break;
default:
break;
}
#endif
} }
int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num,
void maat_table_runtime_free(void *runtime, enum table_type table_type) struct maat_garbage_bin *garbage_bin)
{
if (NULL == runtime) {
return;
}
table_ops[table_type].free_runtime(runtime);
#if 0
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
bool_matcher_free(table_rt->compile_rt.bm);
maat_compile_hash_free(&(table_rt->compile_rt.compile_hash));
break;
case TABLE_TYPE_GROUP2COMPILE:
break;
case TABLE_TYPE_GROUP2GROUP:
maat_group_topology_free(table_rt->g2g_rt.group_topo);
break;
case TABLE_TYPE_EXPR:
adapter_hs_destroy(table_rt->expr_rt.hs);
rcu_hash_free(table_rt->expr_rt.htable);
break;
case TABLE_TYPE_PLUGIN:
ex_data_runtime_free(table_rt->plugin_rt.ex_data_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
ip_matcher_free(table_rt->ip_plugin_rt.ip_matcher);
ex_data_runtime_free(table_rt->ip_plugin_rt.ex_data_rt);
break;
default:
break;
}
#endif
}
int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *garbage_bin)
{ {
if (NULL == tbl_mgr) { if (NULL == tbl_mgr) {
return -1; return -1;
@@ -536,7 +504,9 @@ int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *g
g2g_group_id = i; g2g_group_id = i;
} }
tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, garbage_bin, logger); assert(NULL == tbl_mgr->tbl[i]->runtime);
tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, max_thread_num,
garbage_bin, tbl_mgr->logger);
} }
assert(g2g_group_id != MAX_TABLE_NUM); assert(g2g_group_id != MAX_TABLE_NUM);
@@ -544,7 +514,7 @@ int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *g
/* group2compile runtime depends on associated compile runtime, /* group2compile runtime depends on associated compile runtime,
must make sure associated compile runtime already exist */ must make sure associated compile runtime already exist */
for (i = 0; i < MAX_TABLE_NUM; i++) { for (i = 0; i < MAX_TABLE_NUM; i++) {
void *runtime = tbl_mgr->tbl[i]->runtime; void *runtime = table_manager_get_runtime(tbl_mgr, i);
if (NULL == runtime) { if (NULL == runtime) {
continue; continue;
} }
@@ -556,30 +526,30 @@ int table_manager_init(struct table_manager *tbl_mgr, struct maat_garbage_bin *g
void *schema = table_manager_get_schema(tbl_mgr, i); void *schema = table_manager_get_schema(tbl_mgr, i);
//int associated_compile_table_id = table_schema_get_associated_table_id(table_schema); int associated_compile_table_id = group2compile_associated_compile_table_id(schema);
//TODO: by luis
int associated_compile_table_id = -1;
void *compile_rt = table_manager_get_runtime(tbl_mgr, associated_compile_table_id); void *compile_rt = table_manager_get_runtime(tbl_mgr, associated_compile_table_id);
void *g2g_rt = table_manager_get_runtime(tbl_mgr, g2g_group_id); void *g2g_rt = table_manager_get_runtime(tbl_mgr, g2g_group_id);
group2compile_runtime_init(runtime, compile_rt, g2g_rt);
table_rt->g2c_rt.ref_compile_rt = &(compile_table_rt->compile_rt);
table_rt->g2c_rt.ref_g2g_rt = &(g2g_table_rt->g2g_rt);
assert(table_rt->g2c_rt.ref_compile_rt != NULL);
assert(table_rt->g2c_rt.ref_g2g_rt != NULL);
} }
return 0; return 0;
} }
void table_manager_deinit(struct table_manager *tbl_mgr) void table_manager_runtime_destroy(struct table_manager *tbl_mgr)
{ {
if (NULL == tbl_mgr) { if (NULL == tbl_mgr) {
return; return;
} }
for(size_t i = 0; i < MAX_TABLE_NUM; i++) { for(size_t i = 0; i < MAX_TABLE_NUM; i++) {
maat_table_runtime_free(tbl_mgr->tbl[i]->runtime, tbl_mgr->tbl[i]->table_type); void *runtime = table_manager_get_runtime(tbl_mgr, i);
tbl_mgr->tbl[i]->runtime = NULL; if (NULL == runtime) {
continue;
}
enum table_type table_type = table_manager_get_table_type(tbl_mgr, i);
maat_table_runtime_free(runtime, table_type);
tbl_mgr->tbl[i]->runtime = NULL;
} }
} }
@@ -590,16 +560,25 @@ void table_manager_destroy(struct table_manager *tbl_mgr)
} }
for (size_t i = 0; i < MAX_TABLE_NUM; i++) { for (size_t i = 0; i < MAX_TABLE_NUM; i++) {
assert(NULL == tbl_mgr->tbl[i]->runtime); void *runtime = table_manager_get_runtime(tbl_mgr, i);
assert(NULL == runtime);
if (NULL == tbl_mgr->tbl[i]->schema) { void *schema = table_manager_get_schema(tbl_mgr, i);
if (NULL == schema) {
continue; continue;
} }
table_schema_free(tbl_mgr->tbl[i]->schema); enum table_type table_type = table_manager_get_table_type(tbl_mgr, i);
tbl_mgr->tbl[i]->schema = NULL; maat_table_schema_free(schema, table_type);
tbl_mgr->tbl[i]->schema = NULL;
} }
for (size_t i = 0; i < tbl_mgr->n_accept_tag; i++) {
FREE(tbl_mgr->accept_tags[i].tag_name);
FREE(tbl_mgr->accept_tags[i].tag_val);
}
FREE(tbl_mgr->accept_tags);
maat_kv_store_free(tbl_mgr->tablename2id_map); maat_kv_store_free(tbl_mgr->tablename2id_map);
FREE(tbl_mgr); FREE(tbl_mgr);
} }
@@ -611,17 +590,36 @@ size_t table_manager_table_count(struct table_manager *tbl_mgr)
int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
{ {
if (NULL == tbl_mgr || NULL == name) {
return -1;
}
int table_id = -1;
int ret = maat_kv_read(tbl_mgr->tablename2id_map, name, &table_id);
if (ret < 0) {
log_error(tbl_mgr->logger, MODULE_TABLE, "table:%s is not registered", name);
return -1;
}
return table_id;
} }
enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id) enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id)
{ {
if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) {
return TABLE_TYPE_MAX;
}
if (NULL == tbl_mgr->tbl[table_id]) {
return TABLE_TYPE_MAX;
}
return tbl_mgr->tbl[table_id]->table_type;
} }
int table_manager_get_logger(struct table_manager *tbl_mgr) int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr)
{ {
return tbl_mgr->default_compile_table_id;
} }
void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id) void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id)
@@ -639,7 +637,7 @@ void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id)
struct ex_data_schema *table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id) struct ex_data_schema *table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id)
{ {
return NULL;
} }
int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id) int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id)
@@ -657,7 +655,7 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id)
int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags) int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags)
{ {
return 0;
} }
void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id) void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
@@ -666,8 +664,6 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
return NULL; return NULL;
} }
assert(table_id < (int)tbl_mgr->n_table);
if (NULL == tbl_mgr->tbl[table_id]) { if (NULL == tbl_mgr->tbl[table_id]) {
return NULL; return NULL;
} }
@@ -675,6 +671,25 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
return tbl_mgr->tbl[table_id]->runtime; return tbl_mgr->tbl[table_id]->runtime;
} }
int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id)
{
if (NULL == tbl_mgr) {
return -1;
}
if (NULL == tbl_mgr->tbl[table_id]) {
return -1;
}
enum table_type table_type = tbl_mgr->tbl[table_id]->table_type;
void *runtime = table_manager_get_runtime(tbl_mgr, table_id);
if (table_ops[table_type].runtime_updating_flag != NULL) {
return table_ops[table_type].runtime_updating_flag(runtime);
}
return -1;
}
int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line) int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line)
{ {
void *schema = table_manager_get_schema(tbl_mgr, table_id); void *schema = table_manager_get_schema(tbl_mgr, table_id);
@@ -702,45 +717,6 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, co
} }
return table_ops[table_type].update_runtime(runtime, schema, line, valid_column); return table_ops[table_type].update_runtime(runtime, schema, line, valid_column);
#if 0
switch (table_rt->table_type) {
case TABLE_TYPE_COMPILE:
ret = compile_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, table_rt->ref_garbage_bin, logger);
break;
case TABLE_TYPE_GROUP2COMPILE:
ret = group2compile_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger);
break;
case TABLE_TYPE_GROUP2GROUP:
ret = group2group_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, logger);
break;
case TABLE_TYPE_EXPR:
ret = expr_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger);
break;
case TABLE_TYPE_IP_PLUS:
ret = ip_plus_runtime_update(table_rt->custom_rt, table_item->custom_item, table_name, table_rt->ref_garbage_bin, logger);
break;
case TABLE_TYPE_PLUGIN:
ret = plugin_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, row, logger);
break;
case TABLE_TYPE_IP_PLUGIN:
ret = ip_plugin_runtime_update(table_rt->custom_rt, table_item->custom_item, table_schema, table_name, row, logger);
break;
default:
break;
}
if (ret < 0) {
return;
}
#endif
#if 0
if (is_valid == 0) {
table_rt->rule_num--;
} else {
table_rt->rule_num++;
}
#endif
} }
void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
@@ -755,5 +731,7 @@ void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
return; return;
} }
table_ops[table_type].commit_runtime(runtime); if ( table_ops[table_type].commit_runtime != NULL) {
table_ops[table_type].commit_runtime(runtime);
}
} }

11
src/maat_virtual.cpp
View File

@@ -23,14 +23,16 @@ struct virtual_schema {
void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger)
{ {
size_t read_cnt = 0; //size_t read_cnt = 0;
cJSON *item = cJSON_GetObjectItem(json, "physical_table"); cJSON *item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_Array) { if (NULL == item || item->type != cJSON_Array) {
log_error(logger, MODULE_VIRTUAL, "virtual table %s has no physical_table column", table_name); log_error(logger, MODULE_VIRTUAL, "virtual table %s has no physical_table column", table_name);
return NULL; return NULL;
} }
read_cnt++;
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
return vt_schema;
#if 0 #if 0
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
int cnt = cJSON_GetArraySize(item); int cnt = cJSON_GetArraySize(item);
@@ -57,3 +59,8 @@ void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle
return vt_schema; return vt_schema;
#endif #endif
} }
void virtual_schema_free(void *virtual_schema)
{
FREE(virtual_schema);
}

68
test/maat_framework_gtest.cpp
View File

@@ -75,14 +75,14 @@ TEST(maat_scan_ipv4, hit_ip_and_port) {
int results[3] = {-1}; int results[3] = {-1};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 7); EXPECT_EQ(results[0], 7);
maat_state_free(&state); maat_state_free(&state);
state = NULL; state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
@@ -92,15 +92,14 @@ TEST(maat_scan_ipv4, hit_ip_and_port) {
TEST(maat_scan_ipv4, hit_ip_and_port_range) { TEST(maat_scan_ipv4, hit_ip_and_port_range) {
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "192.168.50.24"; char ip_str[32] = "192.168.50.24";
struct addr_2tuple addr; uint32_t sip;
addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str, &sip);
int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int results[3] = {-1}; int results[3] = {-1};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 4); EXPECT_EQ(results[0], 4);
@@ -109,7 +108,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
memset(results, 0, sizeof(results)); memset(results, 0, sizeof(results));
n_hit_result = 0; n_hit_result = 0;
state = NULL; state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 4); EXPECT_EQ(results[0], 4);
@@ -118,7 +117,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) {
memset(results, 0, sizeof(results)); memset(results, 0, sizeof(results));
n_hit_result = 0; n_hit_result = 0;
state = NULL; state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
} }
@@ -128,33 +127,32 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
char ip_str1[32] = "10.0.1.20"; char ip_str1[32] = "10.0.1.20";
char ip_str2[32] = "10.0.1.25"; char ip_str2[32] = "10.0.1.25";
char ip_str3[32] = "10.0.1.26"; char ip_str3[32] = "10.0.1.26";
struct addr_2tuple addr; uint32_t sip;
addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str1, &sip);
int ret = inet_pton(AF_INET, ip_str1, &addr.ipv4.sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int results[3] = {-1}; int results[3] = {-1};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 8); EXPECT_EQ(results[0], 8);
maat_state_free(&state); maat_state_free(&state);
ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); ret = inet_pton(AF_INET, ip_str2, &sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
state = NULL; state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 8); EXPECT_EQ(results[0], 8);
maat_state_free(&state); maat_state_free(&state);
ret = inet_pton(AF_INET, ip_str3, &addr.ipv4.sip); ret = inet_pton(AF_INET, ip_str3, &sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
state = NULL; state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
@@ -164,24 +162,23 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str1[32] = "192.168.0.1"; char ip_str1[32] = "192.168.0.1";
char ip_str2[32] = "192.168.0.0"; char ip_str2[32] = "192.168.0.0";
struct addr_2tuple addr; uint32_t sip;
addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str1, &sip);
int ret = inet_pton(AF_INET, ip_str1, &addr.ipv4.sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int results[3] = {-1}; int results[3] = {-1};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 50); EXPECT_EQ(results[0], 50);
maat_state_free(&state); maat_state_free(&state);
ret = inet_pton(AF_INET, ip_str2, &addr.ipv4.sip); ret = inet_pton(AF_INET, ip_str2, &sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
state = NULL; state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
@@ -190,15 +187,14 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "192.168.40.10"; char ip_str[32] = "192.168.40.10";
struct addr_2tuple addr; uint32_t sip;
addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str, &sip);
int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int results[3] = {-1}; int results[3] = {-1};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 2); EXPECT_EQ(n_hit_result, 2);
EXPECT_EQ(results[0], 63); EXPECT_EQ(results[0], 63);
@@ -206,7 +202,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
maat_state_free(&state); maat_state_free(&state);
state = NULL; state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
@@ -215,22 +211,21 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) { TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) {
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "1001:da8:205:1::101"; char ip_str[32] = "1001:da8:205:1::101";
struct addr_2tuple addr; uint8_t sip6[128];
addr.type = IP_TYPE_V6; int ret = inet_pton(AF_INET6, ip_str, sip6);
int ret = inet_pton(AF_INET6, ip_str, &addr.ipv6.sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int results[3] = {-1}; int results[3] = {-1};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 47); EXPECT_EQ(results[0], 47);
maat_state_free(&state); maat_state_free(&state);
state = NULL; state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
@@ -270,15 +265,14 @@ TEST(maat_scan_ip, dynamic_config) {
int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG");
char ip_str[32] = "10.0.6.201"; char ip_str[32] = "10.0.6.201";
struct addr_2tuple addr; uint32_t sip;
addr.type = IP_TYPE_V4; int ret = inet_pton(AF_INET, ip_str, &sip);
int ret = inet_pton(AF_INET, ip_str, &addr.ipv4.sip);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
int results[3] = {-1}; int results[3] = {-1};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 0); EXPECT_EQ(n_hit_result, 0);
maat_state_free(&state); maat_state_free(&state);
@@ -294,7 +288,7 @@ TEST(maat_scan_ip, dynamic_config) {
sleep(2); sleep(2);
state = NULL; state = NULL;
ret = maat_scan_ip(g_maat_instance, table_id, 0, &addr, results, sizeof(results), &n_hit_result, &state); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 9998); EXPECT_EQ(results[0], 9998);

9
test/maat_input_mode_gtest.cpp
View File

@@ -3,6 +3,7 @@
#include "maat_utils.h" #include "maat_utils.h"
#include "maat_rule.h" #include "maat_rule.h"
#include "json2iris.h" #include "json2iris.h"
#include "maat_table.h"
#include "maat_config_monitor.h" #include "maat_config_monitor.h"
#include <gtest/gtest.h> #include <gtest/gtest.h>
@@ -37,7 +38,7 @@ TEST(json_mode, maat_scan_string) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 191);
@@ -80,7 +81,7 @@ TEST(iris_mode, maat_scan_string) {
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 191);
@@ -193,14 +194,14 @@ TEST(redis_mode, maat_scan_string) {
maat_options_set_logger(opts, logger); maat_options_set_logger(opts, logger);
struct maat *maat_instance = maat_new(opts, table_info_path); struct maat *maat_instance = maat_new(opts, table_info_path);
int table_id = table_manager_get_table_id(maat_instance->tbl_mgr "HTTP_URL"); int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL");
char scan_data[128] = "hello"; char scan_data[128] = "hello";
int results[5] = {0}; int results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; struct maat_state *state = NULL;
int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, MAAT_HIT);
EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 191); EXPECT_EQ(results[0], 191);

2
test/table_info.conf
View File

@@ -20,9 +20,9 @@
"table_id":2, "table_id":2,
"table_name":"GROUP2COMPILE", "table_name":"GROUP2COMPILE",
"table_type":"group2compile", "table_type":"group2compile",
"associated_compile_table_id":1,
"valid_column":3, "valid_column":3,
"custom": { "custom": {
"associated_compile_table_id":1,
"group_id":1, "group_id":1,
"compile_id":2, "compile_id":2,
"not_flag":4, "not_flag":4,