diff --git a/include/maat.h b/include/maat.h index 87ff79b..7facde1 100644 --- a/include/maat.h +++ b/include/maat.h @@ -51,15 +51,19 @@ enum maat_scan_status { #define MAX_SERVICE_DEFINE_LEN 128 struct maat_rule { int config_id; - int service_id; - uint8_t do_log; - uint8_t do_blacklist; - uint8_t action; uint8_t reserved; int serv_def_len; char service_defined[MAX_SERVICE_DEFINE_LEN]; }; +struct ip_addr { + int ip_type; //4: IPv4, 6: IPv6 + union { + unsigned int ipv4; //network order + unsigned int ipv6[4]; + }; +}; + typedef void maat_start_callback_t(int update_type, void *u_param); typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para); typedef void maat_finish_callback_t(void *u_para); @@ -69,10 +73,10 @@ typedef void maat_plugin_ex_free_func_t(int table_id, void **ad, long argl, void typedef void maat_plugin_ex_dup_func_t(int table_id, void **to, void **from, long argl, void *argp); typedef void maat_rule_ex_new_func_t(int idx, const struct maat_rule *rule, const char *srv_def_large, - void *ex_data, long argl, void *argp); + void **ex_data, long argl, void *argp); typedef void maat_rule_ex_free_func_t(int idx, const struct maat_rule *rule, const char *srv_def_large, - void *ex_data, long argl, void *argp); -typedef void maat_rule_ex_dup_func_t(int idx, void *to, void *from, long argl, void *argp); + void **ex_data, long argl, void *argp); +typedef void maat_rule_ex_dup_func_t(int idx, void **to, void **from, long argl, void *argp); /* maat_instance options API */ struct maat_options; @@ -106,7 +110,7 @@ int maat_table_callback_register(struct maat *instance, int table_id, void *u_para); /* maat compile table API */ -int maat_compile_table_ex_schema_register(struct maat *instance, int table_id, +int maat_compile_table_ex_schema_register(struct maat *instance, const char *table_name, maat_rule_ex_new_func_t *new_func, maat_rule_ex_free_func_t *free_func, maat_rule_ex_dup_func_t *dup_func, @@ -114,16 +118,16 @@ int maat_compile_table_ex_schema_register(struct maat *instance, int table_id, void *maat_compile_table_get_ex_data(struct maat *instance, int table_id, int compile_id, size_t idx); /* maat plugin table API */ -int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id, +int maat_plugin_table_ex_schema_register(struct maat *instance, const char *table_name, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, long argl, void *argp); /* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register, caller is responsible to free the data. */ -int maat_plugin_table_get_ex_data(struct maat *instance, int table_id, - const char *key, size_t key_len, - void **ex_data_array, size_t n_ex_data); +int maat_ip_plugin_table_get_ex_data(struct maat *instance, const char *table_name, + const struct ip_addr *ip, + void **ex_data_array, size_t n_ex_data); /* maat scan API */ struct maat_state; @@ -174,8 +178,8 @@ int maat_state_set_scan_district(struct maat *instance, struct maat_state **stat int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state); -int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, - int compile_table_id); +int maat_state_set_scan_compile_tables(struct maat *maat_instance, struct maat_state **state, + const char *compile_table[], size_t n_table); int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path); diff --git a/include/bool_matcher.h b/scanner/bool_matcher.h similarity index 100% rename from include/bool_matcher.h rename to scanner/bool_matcher.h diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index b9bf316..3669b0d 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -74,7 +74,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct group2group_runtime *g2g_rt, struct maat_compile_state *compile_state, struct maat_hit_path *hit_paths, - size_t n_hit_path); + size_t hit_path_index, size_t n_hit_path); /* group2compile runtime API */ void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, @@ -92,7 +92,7 @@ void maat_compile_state_free(struct maat_compile_state *compile_state); int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, int *hit_item_ids, size_t hit_item_cnt, - int *group_ids, size_t group_ids_size, + int *group_ids, int hit_group_index, size_t group_ids_size, size_t *n_hit_group_id, struct maat_state *state); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); diff --git a/src/inc_internal/maat_config_monitor.h b/src/inc_internal/maat_config_monitor.h index 77344b3..13bd837 100644 --- a/src/inc_internal/maat_config_monitor.h +++ b/src/inc_internal/maat_config_monitor.h @@ -17,6 +17,7 @@ extern "C" #endif #include +#include "log/log.h" void config_monitor_traverse(long long version, const char *idx_dir, void (*start_fn)(long long, int, void *), diff --git a/src/inc_internal/maat_ex_data.h b/src/inc_internal/maat_ex_data.h index cc8feac..8c277d3 100644 --- a/src/inc_internal/maat_ex_data.h +++ b/src/inc_internal/maat_ex_data.h @@ -82,9 +82,12 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container); -void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, - const char *key, size_t key_len); +void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len); +void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt, + struct ex_data_container *ex_container); + void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index b1c512e..159ea2e 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -44,14 +44,13 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name); * * @retval the num of hit group_id */ -int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, - const char *data, size_t data_len, - int *group_ids, size_t group_ids_size, - int vtable_ids, struct maat_state *state); +int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, + size_t data_len, int *group_ids, int hit_group_index, + size_t group_ids_size, int vtable_ids, struct maat_state *state); void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); -int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, - size_t data_len, int *group_ids, size_t group_ids_size, +int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, + int *group_ids, int hit_group_index, size_t group_ids_size, int vtable_id, struct maat_state *state); void expr_runtime_stream_close(struct expr_runtime *expr_rt); diff --git a/src/inc_internal/maat_flag.h b/src/inc_internal/maat_flag.h index 8deedba..6c41fe3 100644 --- a/src/inc_internal/maat_flag.h +++ b/src/inc_internal/maat_flag.h @@ -46,8 +46,8 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name); * * @retval the num of hit group_id */ -int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, - uint64_t flag, int *group_ids, size_t group_ids_size, +int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, uint64_t flag, + int *group_ids, int hit_group_index,size_t group_ids_size, int vtable_id, struct maat_state *state); void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id); diff --git a/src/inc_internal/maat_interval.h b/src/inc_internal/maat_interval.h index 344f5b7..663b979 100644 --- a/src/inc_internal/maat_interval.h +++ b/src/inc_internal/maat_interval.h @@ -47,8 +47,8 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name); * @retval the num of hit group_id */ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, - uint64_t integer, int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state); + uint64_t integer, int *group_ids, int hit_group_index, + size_t group_ids_size, int vtable_id, struct maat_state *state); void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id); long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread); diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h index 0d6c548..fe02619 100644 --- a/src/inc_internal/maat_ip.h +++ b/src/inc_internal/maat_ip.h @@ -41,8 +41,8 @@ struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt); /* ip runtime scan API */ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, - uint8_t *ip_addr, int *group_ids, size_t group_id_size, - int vtable_id, struct maat_state *state); + uint8_t *ip_addr, int *group_ids, int hit_group_index, + size_t group_id_size, int vtable_id, struct maat_state *state); void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id); long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread); diff --git a/src/inc_internal/maat_ip_plugin.h b/src/inc_internal/maat_ip_plugin.h index eec3c13..6af0a4d 100644 --- a/src/inc_internal/maat_ip_plugin.h +++ b/src/inc_internal/maat_ip_plugin.h @@ -32,7 +32,7 @@ void ip_plugin_schema_free(void *ip_plugin_schema); /* ip plugin table ex data API */ struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema); -int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, +void ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, @@ -51,6 +51,9 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name); struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime); +int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr *ip_addr, + void **ex_data_array, size_t n_ex_data_array); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_plugin.h b/src/inc_internal/maat_plugin.h index 61f9807..c64a4a9 100644 --- a/src/inc_internal/maat_plugin.h +++ b/src/inc_internal/maat_plugin.h @@ -40,7 +40,7 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema); int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns); /* plugin table ex data API */ -int plugin_table_set_ex_data_schema(void *plugin_schema, +void plugin_table_set_ex_data_schema(void *plugin_schema, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index b61dc1b..465f9d5 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -36,6 +36,8 @@ extern "C" #include "maat_virtual.h" #define MAX_TABLE_NUM 256 +#define MAX_COMPILE_TABLE_NUM 16 +#define MAX_PHYSICAL_TABLE_NUM 16 #define MAAT_UPDATE_TYPE_NONE 0 #define MAAT_UPDATE_TYPE_FULL 1 @@ -60,10 +62,6 @@ enum tag_match { struct maat_rule_head { int config_id; - int service_id; - char do_log; - char do_blacklist; - char action; char resevered; int serv_def_len; }; @@ -241,7 +239,8 @@ struct maat { struct maat_state { struct maat *maat_instance; int16_t thread_id; - int compile_table_id; //caller can select compile table to scan + size_t n_compile_table; + char compile_tables[MAX_COMPILE_TABLE_NUM][NAME_MAX]; //caller can select compile table to scan unsigned char is_set_district; unsigned char is_last_scan; int district_id; //-1: Any District; -2: Unkonwn District; diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h index 57956ab..c3ddbf3 100644 --- a/src/inc_internal/maat_virtual.h +++ b/src/inc_internal/maat_virtual.h @@ -40,7 +40,8 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, void virtual_schema_free(void *virtual_schema); -int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type); +int virtual_table_get_id(void *virtual_schema); +size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]); #ifdef __cplusplus } diff --git a/src/json2iris.c b/src/json2iris.c index 65c3e07..af95d40 100644 --- a/src/json2iris.c +++ b/src/json2iris.c @@ -1024,11 +1024,12 @@ void write_table_idx(struct iris_description *p_iris, struct iris_table *table) memcpy(buff, line_cnt_str, strlen(line_cnt_str)); memcpy(buff + strlen(line_cnt_str), table->buff, table->write_pos); + UNUSED int ret = 0; FILE *table_fp = fopen(table->table_path, "w"); if (p_iris->encrypt_key) { unsigned char *encrypt_buff = NULL; size_t encrypt_buff_sz = 0; - int ret = crypt_memory(buff, table_file_sz, &encrypt_buff, + ret = crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key, p_iris->encrypt_algo, 1, err_str, sizeof(err_str)); assert(ret == 0); diff --git a/src/maat_api.c b/src/maat_api.c index e786196..ac7fb59 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -468,18 +468,29 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id, return 0; } -int maat_compile_table_ex_schema_register(struct maat *maat_instance, int table_id, +int maat_compile_table_ex_schema_register(struct maat *maat_instance, const char *table_name, maat_rule_ex_new_func_t *new_func, maat_rule_ex_free_func_t *free_func, maat_rule_ex_dup_func_t *dup_func, long argl, void *argp) { - if (NULL == maat_instance || table_id < 0 || table_id > MAX_TABLE_NUM) { + if (NULL == maat_instance || NULL == table_name) { return -1; } + int table_id = maat_table_get_id(maat_instance, table_name); + if (table_id < 0) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "table:%s is not registered yet", table_name); + return -1; + } + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - assert(table_type == TABLE_TYPE_COMPILE); + if (table_type != TABLE_TYPE_COMPILE) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "table:%s is not compile table", table_name); + return -1; + } void *compile_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); assert(compile_schema != NULL); @@ -496,13 +507,15 @@ void *maat_compile_table_get_ex_data(struct maat *maat_instance, int table_id, i return compile_table_get_rule_ex_data(schema, compile_id, idx); } -int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int table_id, - maat_plugin_ex_new_func_t *new_func, - maat_plugin_ex_free_func_t *free_func, - maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp, struct log_handle *logger) +int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, const char *table_name, + int table_id, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp, struct log_handle *logger) { - if (NULL == tbl_mgr || NULL == new_func || NULL == free_func || NULL == dup_func) { + if (NULL == tbl_mgr || NULL == table_name || NULL == new_func || + NULL == free_func || NULL == dup_func) { assert(0); log_error(logger, MODULE_MAAT_API, "table(table_id:%d) %s failed: invalid parameter", __FUNCTION__); @@ -512,7 +525,8 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t void *schema = table_manager_get_schema(tbl_mgr, table_id); if (NULL == schema) { log_error(logger, MODULE_MAAT_API, - "Error: %s, table(table_id:%d) is not registered", __FUNCTION__, table_id); + "Error: %s, table:%s is not registered, can't register ex schema", + __FUNCTION__, table_name); return -1; } @@ -521,10 +535,10 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t switch (table_type) { case TABLE_TYPE_PLUGIN: ex_schema = plugin_table_get_ex_data_schema(schema); - if (NULL == ex_schema) { + if (ex_schema != NULL) { log_error(logger, MODULE_MAAT_API, - "Error: %s, table(table_id:%d) is not a valid plugin table", - __FUNCTION__, table_id); + "Error: %s, EX data schema already registed for plugin table:%s", + __FUNCTION__, table_name); return -1; } plugin_table_set_ex_data_schema(schema, new_func, free_func, @@ -532,10 +546,10 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t break; case TABLE_TYPE_IP_PLUGIN: ex_schema = ip_plugin_table_get_ex_data_schema(schema); - if (NULL == ex_schema) { + if (ex_schema != NULL) { log_error(logger, MODULE_MAAT_API, - "Error: %s, table(table_id:%d) is not a valid ip_plugin table", - __FUNCTION__, table_id); + "Error: %s, EX data schema already registed for ip_plugin table:%s", + __FUNCTION__, table_name); return -1; } ip_plugin_table_set_ex_data_schema(schema, new_func, free_func, @@ -548,7 +562,7 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t return 0; } -void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, +void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, const char *table_name, int table_id, enum table_type table_type, int valid_column) { @@ -592,24 +606,36 @@ void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, switch (table_type) { case TABLE_TYPE_PLUGIN: - plugin_runtime_commit(runtime, "false_plugin_table"); + plugin_runtime_commit(runtime, table_name); break; case TABLE_TYPE_IP_PLUGIN: - ip_plugin_runtime_commit(runtime, "false_plugin_table"); + ip_plugin_runtime_commit(runtime, table_name); break; default: break; } } -int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_id, +int maat_plugin_table_ex_schema_register(struct maat *maat_instance, const char *table_name, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, long argl, void *argp) { + if (NULL == maat_instance || NULL == table_name) { + return -1; + } + + int table_id = maat_table_get_id(maat_instance, table_name); + if (table_id < 0) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "table_name:%s not registered yet, can't register ex_schema.", table_name); + return -1; + } + pthread_mutex_lock(&(maat_instance->background_update_mutex)); - int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id, + int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_name, + table_id, new_func, free_func, dup_func, argl, argp, maat_instance->logger); if (ret < 0) { @@ -626,7 +652,7 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); - generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type, + generic_plugin_runtime_commit_ex_schema(runtime, schema, table_name, table_id, table_type, valid_column); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); @@ -634,39 +660,35 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i return 0; } -int maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, - const char *key, size_t key_len, - void **ex_data_array, size_t n_ex_data) +int maat_ip_plugin_table_get_ex_data(struct maat *maat_instance, const char *table_name, + const struct ip_addr *ip_addr, + void **ex_data_array, size_t n_ex_data) { + if (NULL == maat_instance || NULL == table_name || NULL == ip_addr || + NULL == ex_data_array || 0 == n_ex_data) { + return -1; + } + + int table_id = maat_table_get_id(maat_instance, table_name); + if (table_id < 0) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "table_name:%s not registered yet, can't register ex_schema.", table_name); + return -1; + } + struct maat_runtime *maat_rt = maat_instance->maat_rt; if (NULL == maat_rt) { return -1; } - enum table_type table_type = table_manager_get_table_type(maat_rt->ref_tbl_mgr, table_id); - void *runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, table_id); - if (NULL == runtime) { + void *ip_plugin_rt = table_manager_get_runtime(maat_rt->ref_tbl_mgr, table_id); + if (NULL == ip_plugin_rt) { return -1; } - struct ex_data_runtime *ex_data_rt = NULL; - switch (table_type) { - case TABLE_TYPE_PLUGIN: - ex_data_rt = plugin_runtime_get_ex_data_rt(runtime); - break; - case TABLE_TYPE_IP_PLUGIN: - ex_data_rt = ip_plugin_runtime_get_ex_data_rt(runtime); - break; - default: - break; - } - - if (NULL == ex_data_rt) { - return -1; - } - - *ex_data_array = ex_data_runtime_get_ex_data(ex_data_rt, key, key_len); - return 0; + int n_hit_ex_data = ip_plugin_runtime_get_ex_data(ip_plugin_rt, ip_addr, ex_data_array, n_ex_data); + + return n_hit_ex_data; } struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id) @@ -677,7 +699,7 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id) outer_state->maat_instance = maat_instance; outer_state->district_id = DISTRICT_ANY; outer_state->thread_id = (signed short)thread_id; - outer_state->compile_table_id = -1; //-1 means caller not specify compile table, use default compile table + outer_state->n_compile_table = 0; return outer_state; } @@ -740,19 +762,32 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; - int physical_table_id = 0; int vtable_id = 0; + size_t physical_table_cnt = 0; + int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; + memset(physical_table_ids, -1, sizeof(physical_table_ids)); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); if (table_type == TABLE_TYPE_VIRTUAL) { //find physical table id void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); - physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_FLAG); - if (physical_table_id < 0) { + vtable_id = virtual_table_get_id(virtual_schema); + if (vtable_id < 0) { + return MAAT_SCAN_ERR; + } + + physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); + if (physical_table_cnt <= 0) { return MAAT_SCAN_ERR; } } else { - physical_table_id = table_id; + //physical table type must be TABLE_TYPE_FLAG + if (table_type != TABLE_TYPE_FLAG) { + return MAAT_SCAN_ERR; + } + + physical_table_ids[0] = table_id; + physical_table_cnt = 1; } if (NULL == maat_instance->maat_rt) { @@ -764,38 +799,55 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - if (NULL == flag_rt) { - return MAAT_SCAN_ERR; - } + int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; + memset(group_ids, -1, sizeof(group_ids)); + int hit_group_index = 0; + for (size_t i = 0; i < physical_table_cnt; i++) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); + if (table_type != TABLE_TYPE_FLAG) { + continue; + } + + void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_ids[i]); + if (NULL == flag_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id, + flag, group_ids, hit_group_index, + MAX_SCANNER_HIT_GROUP_NUM, + vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } - int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, - thread_id, - flag, group_ids, - MAX_SCANNER_HIT_GROUP_NUM, - vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } - - size_t n_hit_compile = 0; - if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT if (group_hit_cnt > 0) { flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id); } + hit_group_index += group_hit_cnt; + } - int compile_table_id = -1; - if (mid->compile_table_id == -1) { - compile_table_id = maat_instance->default_compile_table_id; + size_t sum_hit_compile_cnt = 0; + if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0}; + size_t compile_table_cnt = 0; + if (0 == mid->n_compile_table) { + compile_table_id[0] = maat_instance->default_compile_table_id; + compile_table_cnt = 1; } else { - compile_table_id = mid->compile_table_id; + for (size_t i = 0; i < mid->n_compile_table; i++) { + compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); + } + compile_table_cnt = mid->n_compile_table; } - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - *n_hit_result = n_hit_compile; + for (size_t i = 0; i < compile_table_cnt; i++) { + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]); + size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + sum_hit_compile_cnt += n_hit_compile; + } + *n_hit_result = sum_hit_compile_cnt; assert(mid->is_last_scan < LAST_SCAN_FINISHED); if (LAST_SCAN_SET == mid->is_last_scan) { @@ -803,16 +855,16 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, } } - if (n_hit_compile > 0) { + if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == group_hit_cnt) { + if (0 == hit_group_index) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { - // n_hit_compile == 0 - if (group_hit_cnt > 0) { + // sum_hit_compile_cnt == 0 + if (hit_group_index > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -835,19 +887,32 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; - int physical_table_id = 0; int vtable_id = 0; + size_t physical_table_cnt = 0; + int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; + memset(physical_table_ids, -1, sizeof(physical_table_ids)); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); if (table_type == TABLE_TYPE_VIRTUAL) { //find physical table id void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); - physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_FLAG); - if (physical_table_id < 0) { + vtable_id = virtual_table_get_id(virtual_schema); + if (vtable_id < 0) { + return MAAT_SCAN_ERR; + } + + physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); + if (physical_table_cnt <= 0) { return MAAT_SCAN_ERR; } } else { - physical_table_id = table_id; + //physcial table type must be TABLE_TYPE_INTERVAL or TABLE_TYPE_INTERVAL_PLUS + if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { + return MAAT_SCAN_ERR; + } + + physical_table_ids[0] = table_id; + physical_table_cnt = 1; } if (NULL == maat_instance->maat_rt) { @@ -857,47 +922,63 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); - - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); - if ((table_type == TABLE_TYPE_INTERVAL_PLUS) && - (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { - maat_instance->scan_err_cnt++; - return MAAT_SCAN_ERR; - } - alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - void *interval_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - if (NULL == interval_rt) { - return MAAT_SCAN_ERR; - } + int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; + memset(group_ids, -1, sizeof(group_ids)); + int hit_group_index = 0; + for (size_t i = 0; i < physical_table_cnt; i++) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); + if ((table_type == TABLE_TYPE_INTERVAL_PLUS) && + (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { + maat_instance->scan_err_cnt++; + return MAAT_SCAN_ERR; + } - int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt, - thread_id, intval, group_ids, - MAX_SCANNER_HIT_GROUP_NUM, - vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } - - size_t n_hit_compile = 0; - if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { + continue; + } + + void *interval_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + if (NULL == interval_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt, + thread_id, intval, group_ids, hit_group_index, + MAX_SCANNER_HIT_GROUP_NUM, + vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + if (group_hit_cnt > 0) { interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id); } + hit_group_index += group_hit_cnt; + } - int compile_table_id = -1; - if (mid->compile_table_id == -1) { - compile_table_id = maat_instance->default_compile_table_id; + size_t sum_hit_compile_cnt = 0; + if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0}; + size_t compile_table_cnt = 0; + if (0 == mid->n_compile_table) { + compile_table_id[0] = maat_instance->default_compile_table_id; + compile_table_cnt = 1; } else { - compile_table_id = mid->compile_table_id; + for (size_t i = 0; i < mid->n_compile_table; i++) { + compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); + } + compile_table_cnt = mid->n_compile_table; } - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - *n_hit_result = n_hit_compile; + for (size_t i = 0; i < compile_table_cnt; i++) { + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]); + size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + sum_hit_compile_cnt += n_hit_compile; + } + *n_hit_result = sum_hit_compile_cnt; assert(mid->is_last_scan < LAST_SCAN_FINISHED); if (LAST_SCAN_SET == mid->is_last_scan) { @@ -905,16 +986,16 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, } } - if (n_hit_compile > 0) { + if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == group_hit_cnt) { + if (0 == hit_group_index) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { - // n_hit_compile == 0 - if (group_hit_cnt > 0) { + // sum_hit_compile_cnt == 0 + if (hit_group_index > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -940,50 +1021,90 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); int vtable_id = 0; - // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, - // SCAN_TYPE_IP, &virtual_table_id); - // if (NULL == table_schema) { - // return MAAT_SCAN_ERR; - // } - - int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - assert(table_type == TABLE_TYPE_IP_PLUS); - - // int table_id = table_schema_get_table_id(real_table); - void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - if (NULL == ip_rt) { - return MAAT_SCAN_ERR; - } - // size_t rule_num = table_runtime_rule_count(table_rt); - // if (0 == rule_num) { - // return 0; - // } - - int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4, - (uint8_t *)&ip_addr, group_ids, sizeof(group_ids), - vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } + size_t physical_table_cnt = 0; + int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; + memset(physical_table_ids, -1, sizeof(physical_table_ids)); - size_t n_hit_compile = 0; - if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + //find physical table id + void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + vtable_id = virtual_table_get_id(virtual_schema); + if (vtable_id < 0) { + return MAAT_SCAN_ERR; + } + + physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); + if (physical_table_cnt <= 0) { + return MAAT_SCAN_ERR; + } + } else { + //physcial table type must be TABLE_TYPE_IP_PLUS + if (table_type != TABLE_TYPE_IP_PLUS) { + return MAAT_SCAN_ERR; + } + + physical_table_ids[0] = table_id; + physical_table_cnt = 1; + } + + if (NULL == maat_instance->maat_rt) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "maat_scan_ipv4 error because of maat_runtime is NULL"); + return MAAT_SCAN_OK; + } + + maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); + alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); + + int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; + memset(group_ids, -1, sizeof(group_ids)); + int hit_group_index = 0; + for (size_t i = 0; i < physical_table_cnt; i++) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); + if (table_type != TABLE_TYPE_IP_PLUS) { + continue; + } + + void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + if (NULL == ip_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4, + (uint8_t *)&ip_addr, group_ids, hit_group_index, + MAX_SCANNER_HIT_GROUP_NUM, vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + if (group_hit_cnt > 0) { ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); } - - int compile_table_id = -1; - if (mid->compile_table_id == -1) { - compile_table_id = maat_instance->default_compile_table_id; + hit_group_index += group_hit_cnt; + } + + size_t sum_hit_compile_cnt = 0; + if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0}; + size_t compile_table_cnt = 0; + if (0 == mid->n_compile_table) { + compile_table_id[0] = maat_instance->default_compile_table_id; + compile_table_cnt = 1; } else { - compile_table_id = mid->compile_table_id; + for (size_t i = 0; i < mid->n_compile_table; i++) { + compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); + } + compile_table_cnt = mid->n_compile_table; } - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - *n_hit_result = n_hit_compile; + for (size_t i = 0; i < compile_table_cnt; i++) { + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]); + size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + sum_hit_compile_cnt += n_hit_compile; + } + *n_hit_result = sum_hit_compile_cnt; assert(mid->is_last_scan < LAST_SCAN_FINISHED); if (LAST_SCAN_SET == mid->is_last_scan) { @@ -991,16 +1112,16 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, } } - if (n_hit_compile > 0) { + if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == group_hit_cnt) { + if (0 == hit_group_index) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { // n_hit_compile == 0 - if (group_hit_cnt > 0) { + if (hit_group_index > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -1025,51 +1146,91 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; - maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); int vtable_id = 0; - // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, - // SCAN_TYPE_IP, &virtual_table_id); - // if (NULL == table_schema) { - // return MAAT_SCAN_ERR; - // } - - int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - assert(table_type == TABLE_TYPE_IP_PLUS); - - // int table_id = table_schema_get_table_id(real_table); - void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - if (NULL == ip_rt) { - return MAAT_SCAN_ERR; - } - // size_t rule_num = table_runtime_rule_count(table_rt); - // if (0 == rule_num) { - // return 0; - // } - - int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, ip_addr, - group_ids, sizeof(group_ids), vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } + size_t physical_table_cnt = 0; + int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; + memset(physical_table_ids, -1, sizeof(physical_table_ids)); - size_t n_hit_compile = 0; - if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + //find physical table id + void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + vtable_id = virtual_table_get_id(virtual_schema); + if (vtable_id < 0) { + return MAAT_SCAN_ERR; + } + + physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); + if (physical_table_cnt <= 0) { + return MAAT_SCAN_ERR; + } + } else { + //physcial table type must be TABLE_TYPE_IP_PLUS + if (table_type != TABLE_TYPE_IP_PLUS) { + return MAAT_SCAN_ERR; + } + + physical_table_ids[0] = table_id; + physical_table_cnt = 1; + } + + if (NULL == maat_instance->maat_rt) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "maat_scan_string error because of maat_runtime is NULL"); + return MAAT_SCAN_OK; + } + + maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); + alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); + + int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; + memset(group_ids, -1, sizeof(group_ids)); + int hit_group_index = 0; + for (size_t i = 0; i < physical_table_cnt; i++) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); + if (table_type != TABLE_TYPE_IP_PLUS) { + continue; + } + + void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + if (NULL == ip_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, + ip_addr, group_ids, hit_group_index, + MAX_SCANNER_HIT_GROUP_NUM, vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + if (group_hit_cnt > 0) { ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); } + hit_group_index += group_hit_cnt; + } - int compile_table_id = -1; - if (mid->compile_table_id == -1) { - compile_table_id = maat_instance->default_compile_table_id; + size_t sum_hit_compile_cnt = 0; + if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0}; + size_t compile_table_cnt = 0; + if (0 == mid->n_compile_table) { + compile_table_ids[0] = maat_instance->default_compile_table_id; + compile_table_cnt = 1; } else { - compile_table_id = mid->compile_table_id; + for (size_t i = 0; i < mid->n_compile_table; i++) { + compile_table_ids[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); + } + compile_table_cnt = mid->n_compile_table; } - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - *n_hit_result = n_hit_compile; + for (size_t i = 0; i < compile_table_cnt; i++) { + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]); + size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + sum_hit_compile_cnt += n_hit_compile; + } + *n_hit_result = sum_hit_compile_cnt; assert(mid->is_last_scan < LAST_SCAN_FINISHED); if (LAST_SCAN_SET == mid->is_last_scan) { @@ -1077,16 +1238,16 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, } } - if (n_hit_compile > 0) { + if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == group_hit_cnt) { + if (0 == hit_group_index) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { // n_hit_compile == 0 - if (group_hit_cnt > 0) { + if (hit_group_index > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -1110,19 +1271,34 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; - int physical_table_id = 0; int vtable_id = 0; + size_t physical_table_cnt = 0; + int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; + memset(physical_table_ids, -1, sizeof(physical_table_ids)); enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); if (table_type == TABLE_TYPE_VIRTUAL) { //find physical table id void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); - physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_STRING); - if (physical_table_id < 0) { + vtable_id = virtual_table_get_id(virtual_schema); + if (vtable_id < 0) { + return MAAT_SCAN_ERR; + } + + physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); + if (physical_table_cnt <= 0) { return MAAT_SCAN_ERR; } } else { - physical_table_id = table_id; + //physcial table type must be TABLE_TYPE_EXPR or TABLE_TYPE_EXPR_PLUS + if ((table_type == TABLE_TYPE_EXPR_PLUS) && + (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)){ + maat_instance->scan_err_cnt++; + return MAAT_SCAN_ERR; + } + + physical_table_ids[0] = table_id; + physical_table_cnt = 1; } if (NULL == maat_instance->maat_rt) { @@ -1132,48 +1308,59 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); - - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); - if ((table_type == TABLE_TYPE_EXPR_PLUS) && - (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { - maat_instance->scan_err_cnt++; - return MAAT_SCAN_ERR; - } - alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - if (NULL == expr_rt) { - return MAAT_SCAN_ERR; - } + int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; + memset(group_ids, -1, sizeof(group_ids)); + int hit_group_index = 0; + for (size_t i = 0; i < physical_table_cnt; i++) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); + if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { + continue; + } + + void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_ids[i]); + if (NULL == expr_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt, + thread_id, data, data_len, + group_ids, hit_group_index, + MAX_SCANNER_HIT_GROUP_NUM, + vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } - int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt, - thread_id, data, data_len, - group_ids, - MAX_SCANNER_HIT_GROUP_NUM, - vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } - - size_t n_hit_compile = 0; - if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { - // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT if (group_hit_cnt > 0) { expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); } + hit_group_index += group_hit_cnt; + } - int compile_table_id = -1; - if (mid->compile_table_id == -1) { - compile_table_id = maat_instance->default_compile_table_id; + size_t sum_hit_compile_cnt = 0; + if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + int compile_table_ids[MAX_COMPILE_TABLE_NUM]; + memset(compile_table_ids, -1, sizeof(compile_table_ids)); + size_t compile_table_cnt = 0; + if (0 == mid->n_compile_table) { + compile_table_ids[0] = maat_instance->default_compile_table_id; + compile_table_cnt = 1; } else { - compile_table_id = mid->compile_table_id; + for (size_t i = 0; i < mid->n_compile_table; i++) { + compile_table_ids[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); + } + compile_table_cnt = mid->n_compile_table; } - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - *n_hit_result = n_hit_compile; + for (size_t i = 0; i < compile_table_cnt; i++) { + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]); + size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + sum_hit_compile_cnt += n_hit_compile; + } + *n_hit_result = sum_hit_compile_cnt; assert(mid->is_last_scan < LAST_SCAN_FINISHED); if (LAST_SCAN_SET == mid->is_last_scan) { @@ -1181,16 +1368,16 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, } } - if (n_hit_compile > 0) { + if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == group_hit_cnt) { + if (0 == hit_group_index) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { // n_hit_compile == 0 - if (group_hit_cnt > 0) { + if (hit_group_index > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -1253,15 +1440,18 @@ int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **sta return 0; } -int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, - int compile_table_id) +int maat_state_set_scan_compile_tables(struct maat *maat_instance, struct maat_state **state, + const char *compile_tables[], size_t n_table) { if (NULL == maat_instance->maat_rt) { return -1; } struct maat_state *mid = grab_state(state, maat_instance, -1); - mid->compile_table_id = compile_table_id; + for (size_t i = 0; i < mid->n_compile_table; i++) { + + strncpy(mid->compile_tables[i], compile_tables[i], strlen(compile_tables[i])); + } return 0; } @@ -1269,20 +1459,33 @@ int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_st size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state, struct maat_hit_path *paths, size_t n_path) { - int compile_table_id = -1; + int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0}; + size_t compile_table_cnt = 0; - if (state->compile_table_id == -1) { - compile_table_id = maat_instance->default_compile_table_id; + if (0 == state->n_compile_table) { + compile_table_ids[0] = maat_instance->default_compile_table_id; + compile_table_cnt = 1; } else { - compile_table_id = state->compile_table_id; + for (size_t i = 0; i < state->n_compile_table; i++) { + compile_table_ids[i] = maat_table_get_id(maat_instance, state->compile_tables[i]); + } + compile_table_cnt = state->n_compile_table; } - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); - assert(NULL != compile_rt && NULL != g2g_runtime); - return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, - (struct group2group_runtime *)g2g_runtime, - state->compile_state, paths, n_path); + size_t hit_path_index = 0; + //size_t sum_hit_paths = 0; + for (size_t i = 0; i < compile_table_cnt; i++) { + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]); + void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); + assert(NULL != compile_rt && NULL != g2g_runtime); + size_t n_hit_path = compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, + (struct group2group_runtime *)g2g_runtime, + state->compile_state, paths, hit_path_index, + n_path); + hit_path_index += n_hit_path; + } + + return hit_path_index; } size_t maat_get_hit_objects(struct maat_compile_state *compile_state, diff --git a/src/maat_compile.c b/src/maat_compile.c index 50b52f4..4f043da 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -10,6 +10,7 @@ #include #include +#include #include "maat_utils.h" #include "log/log.h" @@ -35,10 +36,6 @@ enum user_region_encode { struct compile_schema { int compile_id_column; - int service_id_column; - int action_column; - int do_blacklist_column; - int do_log_column; int tags_column; int user_region_column; int clause_num_column; @@ -64,10 +61,6 @@ struct group2compile_schema { struct compile_item { int compile_id; - int service_id; - int action; - int do_blacklist; - int do_log; char user_region[MAX_TABLE_LINE_SIZE]; int clause_num; int evaluation_order; @@ -274,30 +267,6 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } - custom_item = cJSON_GetObjectItem(item, "service_id"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - compile_schema->service_id_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "action"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - compile_schema->action_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "do_blacklist"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - compile_schema->do_blacklist_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "do_log"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - compile_schema->do_log_column = custom_item->valueint; - read_cnt++; - } - custom_item = cJSON_GetObjectItem(item, "tags"); if (custom_item != NULL && custom_item->type == cJSON_Number) { compile_schema->tags_column = custom_item->valueint; @@ -324,7 +293,7 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, compile_schema->ref_tbl_mgr = tbl_mgr; - if (read_cnt < 10) { + if (read_cnt < 6) { goto error; } @@ -438,46 +407,6 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, } compile_item->compile_id = atoi(line + column_offset); - ret = get_column_pos(line, compile_schema->service_id_column, - &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_COMPILE, - "compile table(table_id:%d) line:%s has no service_id", - compile_schema->table_id, line); - goto error; - } - compile_item->service_id = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->action_column, - &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_COMPILE, - "compile table(table_id:%d) line:%s has no action", - compile_schema->table_id, line); - goto error; - } - compile_item->action = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->do_blacklist_column, - &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_COMPILE, - "compile table(table_id:%d) line:%s has no do_blacklist", - compile_schema->table_id, line); - goto error; - } - compile_item->do_blacklist = atoi(line + column_offset); - - ret = get_column_pos(line, compile_schema->do_log_column, - &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_COMPILE, - "compile table(table_id:%d) line:%s has no do_log", - compile_schema->table_id, line); - goto error; - } - compile_item->do_log = atoi(line + column_offset); - ret = get_column_pos(line, compile_schema->tags_column, &column_offset, &column_len); if (ret < 0) { @@ -1296,7 +1225,7 @@ static int maat_compile_has_literal(struct maat_compile* compile, return 0; } -static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths, +static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths, size_t n_path, const struct maat_hit_path *find) { for (size_t i = 0; i < n_path; i++) { @@ -1312,11 +1241,11 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct group2group_runtime *g2g_rt, struct maat_compile_state *compile_state, struct maat_hit_path *hit_paths, - size_t hit_path_size) + size_t hit_path_index, size_t hit_path_size) { - size_t i = 0, j = 0; + size_t i = 0, j = 0; struct maat_internal_hit_path *internal_path = NULL; - size_t hit_path_cnt = 0; + size_t hit_path_cnt = hit_path_index; size_t new_hit_path_cnt = 0; for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) { @@ -1502,10 +1431,6 @@ void compile_item_to_compile_rule(struct compile_item *compile_item, { struct maat_rule_head rule_head; rule_head.config_id = compile_item->compile_id; - rule_head.service_id = compile_item->service_id; - rule_head.action = compile_item->action; - rule_head.do_blacklist = compile_item->do_blacklist; - rule_head.do_log = compile_item->do_log; compile_rule->magic_num = COMPILE_RULE_MAGIC; compile_rule->head = rule_head; @@ -1792,11 +1717,11 @@ int compile_runtime_match(struct compile_runtime *compile_rt, int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, int *hit_item_ids, size_t hit_item_cnt, - int *group_ids, size_t group_ids_size, + int *group_ids, int hit_group_index, size_t group_ids_size, size_t *n_hit_group_id, struct maat_state *state) { struct maat_item *item = NULL; - size_t hit_group_cnt = 0; + size_t hit_group_cnt = hit_group_index; void *g2g_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, state->maat_instance->g2g_table_id); @@ -1827,31 +1752,29 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, *n_hit_group_id = hit_group_cnt; /* update hit clause */ - int compile_table_id = -1; - if (state->compile_table_id == -1) { - compile_table_id = state->maat_instance->default_compile_table_id; + int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0}; + size_t compile_table_cnt = 0; + if (0 == state->n_compile_table) { + compile_table_ids[0] = state->maat_instance->default_compile_table_id; + compile_table_cnt = 1; } else { - compile_table_id = state->compile_table_id; + for (size_t i = 0; i < state->n_compile_table; i++) { + compile_table_ids[i] = maat_table_get_id(state->maat_instance, state->compile_tables[i]); + } + compile_table_cnt = state->n_compile_table; } - void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, - compile_table_id); - enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, - compile_table_id); - assert(table_type == TABLE_TYPE_COMPILE); - - for (size_t i = 0; i < hit_group_cnt; i++) { - int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i], - 1, top_group_ids); - // if (0 == top_group_cnt) { - // maat_compile_state_update_hit_clause(state->compile_state, compile_rt, - // group_ids[i], vtable_id); - // } - - for (int j = 0; j < top_group_cnt; j++) { - maat_compile_state_update_hit_clause(state->compile_state, compile_rt, - top_group_ids[j], vtable_id); + for (size_t idx = 0; idx < compile_table_cnt; idx++) { + void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, + compile_table_ids[idx]); + for (size_t i = 0; i < hit_group_cnt; i++) { + int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i], + 1, top_group_ids); + for (int j = 0; j < top_group_cnt; j++) { + maat_compile_state_update_hit_clause(state->compile_state, compile_rt, + top_group_ids[j], vtable_id); + } } } diff --git a/src/maat_ex_data.c b/src/maat_ex_data.c index f2a2e60..a56f3af 100644 --- a/src/maat_ex_data.c +++ b/src/maat_ex_data.c @@ -250,8 +250,8 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, return 0; } -void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, - const char *key, size_t key_len) +void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len) { struct ex_data_container *ex_container = NULL; ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, @@ -268,6 +268,17 @@ void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, return dup_ex_data; } +void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt, + struct ex_data_container *ex_container) +{ + void *dup_ex_data = NULL; + ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, + &(ex_container->ex_data), + ex_data_rt->ex_schema->argl, + ex_data_rt->ex_schema->argp); + return dup_ex_data; +} + void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) { diff --git a/src/maat_expr.c b/src/maat_expr.c index da1e6fb..66ed732 100644 --- a/src/maat_expr.c +++ b/src/maat_expr.c @@ -840,10 +840,9 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name) return ret; } -int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, - const char *data, size_t data_len, - int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state) +int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, + size_t data_len, int *group_ids, int hit_group_index, + size_t group_ids_size, int vtable_ids, struct maat_state *state) { size_t n_hit_item = 0; struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; @@ -881,9 +880,9 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, } size_t group_hit_cnt = 0; - ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, - real_hit_item_cnt, group_ids, group_ids_size, - &group_hit_cnt, state); + ret = maat_compile_state_update(expr_rt->item_hash, vtable_ids, hit_item_ids, + real_hit_item_cnt, group_ids, hit_group_index, + group_ids_size, &group_hit_cnt, state); if (ret < 0) { return -1; } @@ -901,8 +900,8 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) expr_rt->hs_stream = hs_stream; } -int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, - size_t data_len, int *group_ids, size_t group_ids_size, +int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, + int *group_ids, int hit_group_index, size_t group_ids_size, int vtable_id, struct maat_state *state) { if (NULL == expr_rt) { @@ -936,8 +935,8 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t group_hit_cnt = 0; ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, - n_hit_item, group_ids, group_ids_size, - &group_hit_cnt, state); + n_hit_item, group_ids, hit_group_index, + group_ids_size, &group_hit_cnt, state); if (ret < 0) { return -1; } diff --git a/src/maat_flag.c b/src/maat_flag.c index 2dd1f2f..fbf5266 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -411,8 +411,8 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name) } int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, - uint64_t flag, int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state) + uint64_t flag, int *group_ids, int hit_group_index, + size_t group_ids_size, int vtable_id, struct maat_state *state) { struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; @@ -438,7 +438,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, size_t group_hit_cnt = 0; int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids, - n_hit_item, group_ids, group_ids_size, + n_hit_item, group_ids, hit_group_index, group_ids_size, &group_hit_cnt, state); if (ret < 0) { return -1; diff --git a/src/maat_interval.c b/src/maat_interval.c index 1586580..977b2be 100644 --- a/src/maat_interval.c +++ b/src/maat_interval.c @@ -440,9 +440,8 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name) } int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, - uint64_t integer, - int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state) + uint64_t integer, int *group_ids, int hit_group_index, + size_t group_ids_size, int vtable_id, struct maat_state *state) { struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; @@ -475,7 +474,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, size_t group_hit_cnt = 0; int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids, - n_hit_item, group_ids, group_ids_size, + n_hit_item, group_ids, hit_group_index, group_ids_size, &group_hit_cnt, state); if (ret < 0) { return -1; diff --git a/src/maat_ip.c b/src/maat_ip.c index 24ed568..8150feb 100644 --- a/src/maat_ip.c +++ b/src/maat_ip.c @@ -453,17 +453,15 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name) struct ip_matcher *old_ip_matcher = NULL; size_t mem_used = 0; - if (rule_cnt > 0) { - log_info(ip_rt->logger, MODULE_IP, - "table[%s] committing %zu ip rules for rebuilding ip_matcher engine", - table_name, rule_cnt); - new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); - if (NULL == new_ip_matcher) { - log_error(ip_rt->logger, MODULE_IP, - "table[%s] rebuild ip_matcher engine failed when update %zu ip rules", - table_name, rule_cnt); - ret = -1; - } + log_info(ip_rt->logger, MODULE_IP, + "table[%s] committing %zu ip rules for rebuilding ip_matcher engine", + table_name, rule_cnt); + new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); + if (NULL == new_ip_matcher) { + log_error(ip_rt->logger, MODULE_IP, + "table[%s] rebuild ip_matcher engine failed when update %zu ip rules", + table_name, rule_cnt); + ret = -1; } old_ip_matcher = ip_rt->ip_matcher; @@ -479,8 +477,8 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name) } int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, - uint8_t *ip_addr, int *group_ids, size_t group_id_size, - int vtable_id, struct maat_state *state) + uint8_t *ip_addr, int *group_ids, int hit_group_index, + size_t group_id_size, int vtable_id, struct maat_state *state) { int n_hit_item = 0; struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; @@ -518,7 +516,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, size_t group_hit_cnt = 0; int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item, - group_ids, group_id_size, &group_hit_cnt, state); + group_ids, hit_group_index, group_id_size, &group_hit_cnt, state); if (ret < 0) { return -1; } diff --git a/src/maat_ip_plugin.c b/src/maat_ip_plugin.c index 7cd5741..e198cee 100644 --- a/src/maat_ip_plugin.c +++ b/src/maat_ip_plugin.c @@ -16,16 +16,29 @@ #include "maat_ex_data.h" #include "IPMatcher.h" #include "maat_rule.h" +#include "maat.h" #include "maat_garbage_collection.h" #define MODULE_IP_PLUGIN module_name_str("maat.ip_plugin") #define MAX_IP_STR 128 +struct ipv4_item_rule { + uint32_t min_sip; /* 源地址下界；0表示忽略本字段 */ + uint32_t max_sip; /* 源地址上界；0表示固定IP=min_saddr */ +}; + +struct ipv6_item_rule { + uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */ + uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */ +}; + struct ip_plugin_item { int item_id; int ip_type; - char start_ip[MAX_IP_STR]; - char end_ip[MAX_IP_STR]; + union { + struct ipv4_item_rule ipv4; + struct ipv6_item_rule ipv6; + }; int rule_tag; }; @@ -34,6 +47,7 @@ struct ip_plugin_schema { int ip_type_column; int start_ip_column; int end_ip_column; + int addr_format_column; int rule_tag_column; struct ex_data_schema *ex_schema; int table_id; //ugly @@ -98,6 +112,12 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } + custom_item = cJSON_GetObjectItem(item, "addr_format"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->addr_format_column = custom_item->valueint; + read_cnt++; + } + // rule_tag is optional custom_item = cJSON_GetObjectItem(item, "rule_tag"); if (custom_item != NULL && custom_item->type == cJSON_Number) { @@ -106,7 +126,7 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, schema->ref_tbl_mgr = tbl_mgr; - if (read_cnt < 5) { + if (read_cnt < 6) { goto error; } @@ -193,7 +213,11 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema, size_t column_offset = 0; size_t column_len = 0; + char addr_format[16] = {0}; + char start_ip_str[40] = {0}; + char end_ip_str[40] = {0}; struct ip_plugin_item *ip_plugin_item = ALLOC(struct ip_plugin_item, 1); + ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_IP_PLUGIN, @@ -211,13 +235,29 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema, goto error; } ip_plugin_item->ip_type = atoi(line + column_offset); - if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) { + if (ip_plugin_item->ip_type != IPv4 && ip_plugin_item->ip_type != IPv6) { log_error(logger, MODULE_IP_PLUGIN, "ip_plugin table(table_id:%d) line:%s ip_type[%d] invalid", schema->table_id, line, ip_plugin_item->ip_type); goto error; } + ret = get_column_pos(line, schema->addr_format_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP_PLUGIN, + "ip_plugin table(table_id:%d) line:%s has no addr_format column", + schema->table_id, line); + goto error; + } + + memcpy(addr_format, (line + column_offset), column_len); + if (IP_FORMAT_UNKNOWN == ip_format_str2int(addr_format)) { + log_error(logger, MODULE_IP_PLUGIN, + "ip_plugin table(table_id:%d) line:%s has invalid addr_format, should be range/CIDR", + schema->table_id, line); + goto error; + } + ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_IP_PLUGIN, @@ -225,8 +265,7 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema, schema->table_id, line); goto error; } - strncpy(ip_plugin_item->start_ip, line + column_offset, - MIN(column_len, sizeof(ip_plugin_item->start_ip))); + strncpy(start_ip_str, line + column_offset, column_len); ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len); if (ret < 0) { @@ -235,8 +274,28 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema, schema->table_id, line); goto error; } - strncpy(ip_plugin_item->end_ip, line + column_offset, - MIN(column_len, sizeof(ip_plugin_item->end_ip))); + strncpy(end_ip_str, line + column_offset, column_len); + + if (IPv4 == ip_plugin_item->ip_type) { + ret = ip_format2range(ip_plugin_item->ip_type, ip_format_str2int(addr_format), start_ip_str, end_ip_str, + &ip_plugin_item->ipv4.min_sip, &ip_plugin_item->ipv4.max_sip); + if (ret < 0) { + log_error(logger, MODULE_IP_PLUGIN, + "ip_plugin table(table_id:%d) line:%s ip_format2range(ip4) failed", + schema->table_id, line); + goto error; + } + } else { + //ipv6 + ret = ip_format2range(ip_plugin_item->ip_type, ip_format_str2int(addr_format), start_ip_str, end_ip_str, + ip_plugin_item->ipv6.min_sip, ip_plugin_item->ipv6.max_sip); + if (ret < 0) { + log_error(logger, MODULE_IP_PLUGIN, + "ip_plugin table(table_id:%d) line:%s ip_format2range(ip6) failed", + schema->table_id, line); + goto error; + } + } return ip_plugin_item; error: @@ -249,33 +308,16 @@ void ip_plugin_item_free(struct ip_plugin_item *item) FREE(item); } -int ip_plugin_table_ex_data_schema_flag(struct ip_plugin_schema *ip_plugin_schema) -{ - return 0; -} - -int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, +void ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, long argl, void *argp, struct log_handle *logger) { - if (NULL == ip_plugin_schema) { - return -1; - } - struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; - if (schema->ex_schema != NULL) { - assert(0); - log_error(logger, MODULE_IP_PLUGIN, - "Error: %s, EX data schema already registed", __FUNCTION__); - return -1; - } schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp); - - return 0; } int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_schema *schema, @@ -284,9 +326,9 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_ { int ret = -1; struct ex_data_runtime *ex_data_rt = rt->ex_data_rt; - int set_flag = ip_plugin_table_ex_data_schema_flag(schema); + struct ex_data_schema *ex_schema = schema->ex_schema; - if (1 == set_flag) { + if (ex_schema != NULL) { if (0 == is_valid) { //delete ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len); @@ -350,14 +392,16 @@ void ip_plugin_runtime_free(void *ip_plugin_runtime) void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule) { - if (4 == item->ip_type) { + if (IPv4 == item->ip_type) { rule->type = IPv4; - ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, - &(rule->ipv4_rule.start_ip), &(rule->ipv4_rule.end_ip)); + rule->ipv4_rule.start_ip = item->ipv4.min_sip; + rule->ipv4_rule.end_ip = item->ipv4.max_sip; } else { rule->type = IPv6; - ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip, - rule->ipv6_rule.start_ip, rule->ipv6_rule.end_ip); + memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, + sizeof(item->ipv6.min_sip)); + memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, + sizeof(item->ipv6.max_sip)); } rule->rule_id = item->item_id; @@ -436,23 +480,22 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name) for (size_t i = 0; i < rule_cnt; i++) { struct ip_plugin_item *item = (struct ip_plugin_item *)ex_container[i]->custom_data; ip_plugin_item_to_ip_rule(item, &rules[i]); + rules[i].user_tag = ex_container[i]; } struct ip_matcher *new_ip_matcher = NULL; struct ip_matcher *old_ip_matcher = NULL; size_t mem_used = 0; - if (rule_cnt > 0) { - log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, - "table[%s] committing %zu ip_plugin rules for rebuilding ip_matcher engine", - table_name, rule_cnt); - new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); - if (NULL == new_ip_matcher) { - log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, - "table[%s] rebuild ip_matcher engine failed when update %zu ip_plugin rules", - table_name, rule_cnt); - ret = -1; - } + log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, + "table[%s] committing %zu ip_plugin rules for rebuilding ip_matcher engine", + table_name, rule_cnt); + new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); + if (NULL == new_ip_matcher) { + log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, + "table[%s] rebuild ip_matcher engine failed when update %zu ip_plugin rules", + table_name, rule_cnt); + ret = -1; } old_ip_matcher = ip_plugin_rt->ip_matcher; @@ -476,4 +519,36 @@ struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; return ip_plugin_rt->ex_data_rt; +} + +int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr *ip_addr, + void **ex_data_array, size_t n_ex_data) +{ + if (NULL == ip_plugin_runtime) { + return -1; + } + + struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; + if (NULL == ip_plugin_rt->ip_matcher) { + log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, + "ip_matcher is NULL, can't get ex data"); + return 0; + } + + struct scan_result results[n_ex_data]; + memset(results, 0, sizeof(results)); + + struct ip_data ip_data = *(const struct ip_data *)ip_addr; + if (ip_data.type == IPv4) { + ip_data.ipv4 = ntohl(ip_data.ipv4); + } else { + ipv6_ntoh(ip_data.ipv6); + } + + int n_result = ip_matcher_match(ip_plugin_rt->ip_matcher, &ip_data, results, n_ex_data); + for (int i = 0; i < n_result; i++) { + ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(ip_plugin_rt->ex_data_rt, + (struct ex_data_container *)results[i].tag); + } + return n_result; } \ No newline at end of file diff --git a/src/maat_plugin.c b/src/maat_plugin.c index 7c28c14..0975183 100644 --- a/src/maat_plugin.c +++ b/src/maat_plugin.c @@ -205,28 +205,16 @@ int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, return n_foreign; } -int plugin_table_set_ex_data_schema(void *plugin_schema, +void plugin_table_set_ex_data_schema(void *plugin_schema, maat_plugin_ex_new_func_t *new_func, maat_plugin_ex_free_func_t *free_func, maat_plugin_ex_dup_func_t *dup_func, long argl, void *argp, struct log_handle *logger) { - if (NULL == plugin_schema) { - return -1; - } - struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; - if (schema->ex_schema != NULL) { - assert(0); - log_error(logger, MODULE_PLUGIN, - "Error: %s, EX data schema already registed", - __FUNCTION__); - return -1; - } + schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp); - - return 0; } struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema) diff --git a/src/maat_rule.c b/src/maat_rule.c index a0b7a79..2d7db05 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -418,8 +418,6 @@ void *rule_monitor_loop(void *arg) while (maat_instance->is_running) { usleep(maat_instance->rule_update_checking_interval_ms * 1000); if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) { - log_info(maat_instance->logger, MODULE_MAAT_RULE, - "rule_monitor_loop.................%d", maat_instance->rule_update_checking_interval_ms * 1000); switch (maat_instance->input_mode) { case DATA_SOURCE_REDIS: redis_monitor_traverse(maat_instance->maat_version, diff --git a/src/maat_table.c b/src/maat_table.c index 3419800..a73ef0b 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -325,6 +325,7 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo item = cJSON_GetObjectItem(json, "table_type"); if (NULL == item || item->type != cJSON_String) { + log_error(logger, MODULE_TABLE, "table:%s has no table_type column", ptable->table_name); goto error; } @@ -337,6 +338,7 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo item = cJSON_GetObjectItem(json, "valid_column"); if (NULL == item || item->type != cJSON_Number) { if (ptable->table_type != TABLE_TYPE_VIRTUAL) { + log_error(logger, MODULE_TABLE, "table:%s has no valid column", ptable->table_name); goto error; } } else { diff --git a/src/maat_virtual.c b/src/maat_virtual.c index 097ed6a..5e3dbce 100644 --- a/src/maat_virtual.c +++ b/src/maat_virtual.c @@ -20,7 +20,8 @@ #define MODULE_VIRTUAL module_name_str("maat.virtual") struct virtual_schema { - int physical_table_id[SCAN_TYPE_MAX]; + char physical_tables[MAX_PHYSICAL_TABLE_NUM][NAME_MAX]; + size_t n_physical_table; int table_id; struct table_manager *ref_tbl_mgr; }; @@ -28,38 +29,38 @@ struct virtual_schema { void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger) { - cJSON *item = cJSON_GetObjectItem(json, "physical_table"); + struct virtual_schema *schema = ALLOC(struct virtual_schema, 1); + schema->ref_tbl_mgr = tbl_mgr; + + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (NULL == item || item->type != cJSON_Number) { + log_error(logger, MODULE_VIRTUAL, + "virtual table %s has no table_id column", table_name); + goto error; + } + schema->table_id = item->valueint; + + item = cJSON_GetObjectItem(json, "physical_table"); if (NULL == item || item->type != cJSON_Array) { log_error(logger, MODULE_VIRTUAL, "virtual table %s has no physical_table column", table_name); - return NULL; + goto error; } - struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); - vt_schema->ref_tbl_mgr = tbl_mgr; - int cnt = cJSON_GetArraySize(item); for (int i = 0; i < cnt; i++) { cJSON *tmp_item = cJSON_GetArrayItem(item, i); if (tmp_item != NULL && tmp_item->type == cJSON_String) { - int table_id = table_manager_get_table_id(tbl_mgr, tmp_item->valuestring); - /* physical table should already exist */ - if (table_id < 0) { - log_error(logger, MODULE_VIRTUAL, "table:%s is not registered", - tmp_item->valuestring); - FREE(vt_schema); - return NULL; - } - - enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); - assert(table_type != TABLE_TYPE_INVALID); - enum scan_type scan_type = maat_table_get_scan_type(table_type); - assert(scan_type != SCAN_TYPE_INVALID); - vt_schema->physical_table_id[scan_type]= table_id; + memcpy(schema->physical_tables[i], tmp_item->valuestring, + strlen(tmp_item->valuestring)); } } + schema->n_physical_table = cnt; - return vt_schema; + return schema; +error: + FREE(schema); + return NULL; } void virtual_schema_free(void *virtual_schema) @@ -67,12 +68,28 @@ void virtual_schema_free(void *virtual_schema) FREE(virtual_schema); } -int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type) +int virtual_table_get_id(void *virtual_schema) { if (NULL == virtual_schema) { return -1; } struct virtual_schema *schema = (struct virtual_schema *)virtual_schema; - return schema->physical_table_id[type]; + return schema->table_id; +} + +size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]) +{ + if (NULL == virtual_schema) { + return -1; + } + + struct virtual_schema *schema = (struct virtual_schema *)virtual_schema; + + for (size_t i = 0; i < schema->n_physical_table; i++) { + int table_id = table_manager_get_table_id(schema->ref_tbl_mgr, schema->physical_tables[i]); + physical_table_ids[i] = table_id; + } + + return schema->n_physical_table; } \ No newline at end of file diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index a1a60a3..09e0f45 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -476,7 +476,8 @@ TEST_F(MaatIPScan, IPv6) { int ret = inet_pton(AF_INET6, ip_str, &sip); EXPECT_EQ(ret, 1); - int results[ARRAY_SIZE] = {-1}; + int results[ARRAY_SIZE]; + memset(results, -1, sizeof(results)); size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, @@ -673,6 +674,47 @@ TEST_F(NOTLogic, ScanNotAtLast) { maat_state_free(&state); } +void maat_read_entry_start_cb(int update_type, void *u_para) +{ + +} + +void maat_read_entry_cb(int table_id, const char *table_line, void *u_para) +{ + char ip_str[16]={0}; + int entry_id=-1,seq=-1; + unsigned int ip_uint=0; + int is_valid=0; + unsigned int local_ip_nr=16820416;//192.168.0.1 + + sscanf(table_line, "%d\t%s\t%d\t%d", &seq,ip_str, &entry_id, &is_valid); + inet_pton(AF_INET,ip_str,&ip_uint); + if(local_ip_nr==ip_uint) + { + if(is_valid==1) + { + //printf("Load entry id %d success.\n",entry_id); + EXPECT_EQ(entry_id, 101); + } + else + { + //printf("Offload entry id %d success.\n",entry_id); + } + } +} + +void maat_read_entry_finish_cb(void *u_para) +{ + //Maat_feather_t feather=u_para; + // long long version=0; + // int ret=0,is_last_updating_table=0; + // ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version)); + // EXPECT_EQ(ret, 0); + // ret=Maat_read_state(feather,MAAT_STATE_LAST_UPDATING_TABLE, &is_last_updating_table, sizeof(is_last_updating_table)); + // EXPECT_EQ(ret, 0); + //printf("Maat Version %lld at plugin finish callback, is_last_update=%d.\n",version,is_last_updating_table); +} + class PluginTable : public testing::Test { protected: @@ -686,7 +728,245 @@ protected: }; TEST_F(PluginTable, Callback) { + int table_id = maat_table_get_id(g_maat_instance, "QD_ENTRY_INFO"); + int ret = maat_table_callback_register(g_maat_instance, table_id, + maat_read_entry_start_cb, + maat_read_entry_cb, + maat_read_entry_finish_cb, + g_maat_instance); + EXPECT_EQ(ret, 0); +} +class IPPluginTable : public testing::Test +{ +protected: + static void SetUpTestCase() { + + } + + static void TearDownTestCase() { + + } +}; + +struct ip_plugin_ud { + int rule_id; + char *buffer; + int ref_cnt; +}; +void ip_plugin_EX_new_cb(int table_id, const char *key, const char *table_line, + void **ad, long argl, void *argp) +{ + int *counter = (int *)argp; + size_t column_offset=0, column_len=0; + struct ip_plugin_ud *ud = ALLOC(struct ip_plugin_ud, 1); + + int ret = get_column_pos(table_line, 1, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + + ud->rule_id = atoi(table_line + column_offset); + + ret = get_column_pos(table_line, 5, &column_offset, &column_len); + EXPECT_EQ(ret, 0); + + ud->buffer = (char *)calloc(sizeof(char), column_len + 1); + strncpy(ud->buffer, table_line + column_offset, column_len); + ud->ref_cnt = 1; + *ad = ud; + (*counter)++; +} + +void ip_plugin_EX_free_cb(int table_id, void **ad, long argl, void *argp) +{ + struct ip_plugin_ud *ud = (struct ip_plugin_ud *)(*ad); + if ((__sync_sub_and_fetch(&ud->ref_cnt, 1) == 0)) { + free(ud->buffer); + free(ud); + *ad = NULL; + } +} + +void ip_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *argp) +{ + struct ip_plugin_ud *ud = (struct ip_plugin_ud *)(*from); + __sync_add_and_fetch(&(ud->ref_cnt), 1); + *to = ud; +} + +TEST_F(IPPluginTable, EX_DATA) { + int ip_plugin_ex_data_counter = 0; + const char *table_name = "TEST_IP_PLUGIN_WITH_EXDATA"; + + int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_name, + ip_plugin_EX_new_cb, + ip_plugin_EX_free_cb, + ip_plugin_EX_dup_cb, + 0, &ip_plugin_ex_data_counter); + EXPECT_EQ(ret, 0); + EXPECT_EQ(ip_plugin_ex_data_counter, 5); + + struct ip_addr ipv4; + ipv4.ip_type = IPv4; + ret = inet_pton(AF_INET, "192.168.30.100", &ipv4.ipv4); + EXPECT_EQ(ret, 1); + + struct ip_plugin_ud *results[ARRAY_SIZE]; + ret = maat_ip_plugin_table_get_ex_data(g_maat_instance, table_name, &ipv4, + (void **)results, ARRAY_SIZE); + EXPECT_EQ(ret, 2); + EXPECT_EQ(results[0]->rule_id, 101); + EXPECT_EQ(results[1]->rule_id, 102); + + int i = 0; + for (i = 0; i < ret; i++) { + ip_plugin_EX_free_cb(0, (void**)&(results[i]), 0, NULL); + } + + struct ip_addr ipv6; + ipv6.ip_type = IPv6; + inet_pton(AF_INET6, "2001:db8:1234::5210", &(ipv6.ipv6)); + memset(results, 0, sizeof(results)); + + ret = maat_ip_plugin_table_get_ex_data(g_maat_instance, table_name, &ipv6, + (void**)results, ARRAY_SIZE); + EXPECT_EQ(ret, 2); + EXPECT_EQ(results[0]->rule_id, 104); + EXPECT_EQ(results[1]->rule_id, 103); + + for (i = 0; i < ret; i++) { + ip_plugin_EX_free_cb(0, (void**)&(results[i]), 0, NULL); + } + + //Reproduce BugReport-Liumengyan-20210515 + inet_pton(AF_INET6, "240e:97c:4010:104::17", &(ipv6.ipv6)); + ret = maat_ip_plugin_table_get_ex_data(g_maat_instance, table_name, &ipv6, + (void**)results, ARRAY_SIZE); + EXPECT_EQ(ret, 0); +} + +class VirtualTable : public testing::Test +{ +protected: + static void SetUpTestCase() { + + } + + static void TearDownTestCase() { + + } +}; + +TEST_F(VirtualTable, basic) { + int results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + const char *table_name = "HTTP_RESPONSE_KEYWORDS"; + + int table_id = maat_table_get_id(g_maat_instance, table_name); + ASSERT_GT(table_id, 0); + + char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; + int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), + results, ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + EXPECT_EQ(n_hit_result, 0); + maat_state_free(&state); +} + +class CompileTable : public testing::Test +{ +protected: + static void SetUpTestCase() { + + } + + static void TearDownTestCase() { + + } +}; + +struct rule_ex_param { + int ref_cnt; + char name[NAME_MAX]; + int id; + pthread_mutex_t lock; +}; + +void compile_ex_param_new(int idx, const struct maat_rule *rule, const char *srv_def_large, + void **ad, long argl, void *argp) +{ + int *counter = (int *)argp; + *ad = NULL; + ASSERT_GT(rule->serv_def_len, 4); + + struct rule_ex_param *param = (struct rule_ex_param *)calloc(sizeof(struct rule_ex_param), 1); + + param->ref_cnt = 1; + pthread_mutex_init(&(param->lock), NULL); + + sscanf(srv_def_large, "%*[^:]:%[^,],%d", param->name, &(param->id)); + (*counter)++; + *ad = param; +} + +void compile_ex_param_free(int idx, const struct maat_rule *rule, const char *srv_def_large, + void **ad, long argl, void *argp) +{ + if (*ad == NULL) { + return; + } + + struct rule_ex_param *param = (struct rule_ex_param *)*ad; + pthread_mutex_lock(&(param->lock)); + param->ref_cnt--; + if (param->ref_cnt > 0) { + pthread_mutex_unlock(&(param->lock)); + return; + } + free(param); +} + +void compile_ex_param_dup(int idx, void **to, void **from, long argl, void *argp) +{ + struct rule_ex_param *from_param = *((struct rule_ex_param **)from); + pthread_mutex_lock(&(from_param->lock)); + from_param->ref_cnt++; + pthread_mutex_unlock(&(from_param->lock)); + *((struct rule_ex_param**)to) = from_param; +} + +TEST_F(CompileTable, CompileEXData) { + int results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; + const char *table_name = "HTTP_URL_LITERAL"; + const char *expect_name = "I have a name"; + + int table_id = maat_table_get_id(g_maat_instance, table_name); + ASSERT_GT(table_id, 0); + + int ex_data_counter = 0; + int ex_param_idx = maat_compile_table_ex_schema_register(g_maat_instance, "COMPILE_ALIAS", + compile_ex_param_new, + compile_ex_param_free, + compile_ex_param_dup, + 0, &ex_data_counter); + ASSERT_TRUE(ex_param_idx>=0); + EXPECT_EQ(ex_data_counter, 1); + + int ret = maat_scan_string(g_maat_instance, table_id, 0, url, strlen(url), + results, ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + + void *ex_data = maat_compile_table_get_ex_data(g_maat_instance, table_id, 0, ex_param_idx); + ASSERT_TRUE(ex_data!=NULL); + struct rule_ex_param *param = (struct rule_ex_param *)ex_data; + EXPECT_EQ(param->id, 7799); + EXPECT_EQ(strcmp(param->name, expect_name), 0); + compile_ex_param_free(0, NULL, NULL, &ex_data, 0, NULL); + + maat_state_free(&state); } int count_line_num_cb(const char *table_name, const char *line, void *u_para) diff --git a/test/maat_json.json b/test/maat_json.json index c7c77c0..dc5c275 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -2443,11 +2443,11 @@ { "table_name": "TEST_IP_PLUGIN_WITH_EXDATA", "table_content": [ - "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1", - "102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1", - "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1", - "104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1", - "105\t6\t2620:100:3000::\t2620:0100:30ff:ffff:ffff:ffff:ffff:ffff\tBugreport-liumengyan-20210517\t1" + "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1\trange", + "102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1\trange", + "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1\trange", + "104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1\trange", + "105\t6\t2620:100:3000::\t2620:0100:30ff:ffff:ffff:ffff:ffff:ffff\tBugreport-liumengyan-20210517\t1\trange" ] }, { diff --git a/test/table_info.conf b/test/table_info.conf index 1bf0200..5ab4b4c 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -6,10 +6,6 @@ "valid_column":8, "custom": { "compile_id":1, - "service_id":2, - "action":3, - "do_blacklist":4, - "do_log":5, "tags":6, "user_region":7, "clause_num":9, @@ -336,11 +332,13 @@ "table_id":29, "table_name":"TEST_IP_PLUGIN_WITH_EXDATA", "table_type":"ip_plugin", + "valid_column":6, "custom": { "item_id":1, "ip_type":2, "start_ip":3, - "end_ip":4 + "end_ip":4, + "addr_format":7 } }, { diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index bb1f6e3..045de09 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -1,5 +1,5 @@ # CMakeFiles for 3rd vendor library - +cmake_minimum_required(VERSION 3.5) include(ExternalProject) set(VENDOR_ROOT ${CMAKE_BINARY_DIR}/vendor) @@ -84,8 +84,13 @@ file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) add_library(hiredis-static STATIC IMPORTED GLOBAL) add_dependencies(hiredis-static hiredis) -set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a) + set_property(TARGET hiredis-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) +if(CMAKE_BUILD_TYPE STREQUAL "Debug") +set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a) +else() +set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredis.a) +endif() #igraph-0.7.1 ExternalProject_Add(igraph PREFIX igraph