diff --git a/include/maat.h b/include/maat.h index 71eaead..1c2d0d1 100644 --- a/include/maat.h +++ b/include/maat.h @@ -29,16 +29,16 @@ struct maat; struct maat_hit_path { int Nth_scan; - int item_id; - int sub_group_id; - int top_group_id; - int vtable_id; // 0 is not a virtual table. - int compile_id; + int vtable_id; // 0 is not a virtual table. + uint64_t item_id; + uint64_t sub_group_id; + uint64_t top_group_id; + uint64_t compile_id; }; struct maat_hit_object { int vtable_id; - int group_id; + uint64_t group_id; }; enum maat_scan_status { @@ -50,7 +50,7 @@ enum maat_scan_status { #define MAX_SERVICE_DEFINE_LEN 128 struct maat_rule { - int config_id; + uint64_t config_id; uint8_t reserved; int serv_def_len; char service_defined[MAX_SERVICE_DEFINE_LEN]; @@ -116,7 +116,7 @@ int maat_compile_table_ex_schema_register(struct maat *instance, int table_id, maat_rule_ex_dup_func_t *dup_func, long argl, void *argp); void *maat_compile_table_get_ex_data(struct maat *instance, int compile_table_id, - int compile_id, size_t idx); + uint64_t compile_id, size_t idx); /* maat plugin table API */ int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id, @@ -125,7 +125,9 @@ int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id, maat_plugin_ex_dup_func_t *dup_func, long argl, void *argp); /* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register, - caller is responsible to free the data. */ + caller is responsible to free the data. */ +void *maat_plugin_table_get_ex_data(struct maat *instance, int table_id, const char *key); + int maat_ip_plugin_table_get_ex_data(struct maat *instance, int table_id, const struct ip_addr *ip, void **ex_data_array, size_t n_ex_data); @@ -155,29 +157,30 @@ struct maat_state; * MAAT_SCAN_HIT */ int maat_scan_flag(struct maat *instance, int table_id, int thread_id, - uint64_t flag, int *results, size_t n_result, + uint64_t flag, uint64_t *results, size_t n_result, size_t *n_hit_result, struct maat_state **state); int maat_scan_integer(struct maat *instance, int table_id, int thread_id, - unsigned int intval, int *results, size_t n_result, + unsigned int intval, uint64_t *results, size_t n_result, size_t *n_hit_result, struct maat_state **state); int maat_scan_ipv4(struct maat *instance, int table_id, int thread_id, - uint32_t ip_addr, int *results, size_t n_result, + uint32_t ip_addr, uint64_t *results, size_t n_result, size_t *n_hit_result, struct maat_state **state); int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id, - uint8_t *ip_addr, int *results, size_t n_result, + uint8_t *ip_addr, uint64_t *results, size_t n_result, size_t *n_hit_result, struct maat_state **state); int maat_scan_string(struct maat *instance, int table_id, int thread_id, - const char *data, size_t data_len, int *results, size_t n_result, - size_t *n_hit_result, struct maat_state **state); + const char *data, size_t data_len, uint64_t *results, + size_t n_result, size_t *n_hit_result, + struct maat_state **state); struct maat_stream; struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id); -int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* data, int data_len, - int *results, size_t *n_result, struct maat_state **state); +int maat_scan_stream(struct maat_stream **stream, const char* data, int data_len, + uint64_t *results, size_t *n_result, struct maat_state **state); void maat_scan_stream_close(struct maat_stream **stream); diff --git a/scanner/adapter_hs.cpp b/scanner/adapter_hs.cpp index f2dad66..2ca6163 100644 --- a/scanner/adapter_hs.cpp +++ b/scanner/adapter_hs.cpp @@ -424,8 +424,7 @@ int matched_event_cb(unsigned int id, unsigned long long from, matched_pat->matched_l_offset = from; matched_pat->matched_r_offset = to; - HASH_ADD_INT(matched_pat_container->pat_hash, pattern_id, matched_pat); - + HASH_ADD(hh, matched_pat_container->pat_hash, pattern_id, sizeof(unsigned long long), matched_pat); return 0; } @@ -471,8 +470,8 @@ int hs_tag_validate(struct hs_tag *hs_tag, struct matched_pattern_container *mat /* check if real matched pattern, because pattern match_mode is different */ for (size_t i = 0; i < hs_tag->n_pat_attr; i++) { struct matched_pattern *tmp_matched_pat = NULL; - int pattern_id = hs_tag->pat_attr[i].pattern_id; - HASH_FIND_INT(matched_pat_container->pat_hash, &pattern_id, tmp_matched_pat); + unsigned long long pattern_id = hs_tag->pat_attr[i].pattern_id; + HASH_FIND(hh, matched_pat_container->pat_hash, &pattern_id, sizeof(unsigned long long), tmp_matched_pat); if (tmp_matched_pat) { int matched_ret = is_real_matched_pattern(tmp_matched_pat, hs_tag->pat_attr[i].match_mode, diff --git a/scanner/adapter_hs.h b/scanner/adapter_hs.h index 8a8fde4..2418ba7 100644 --- a/scanner/adapter_hs.h +++ b/scanner/adapter_hs.h @@ -54,7 +54,7 @@ enum hs_case_sensitive { }; struct hs_scan_result { - int item_id; + uint64_t item_id; void *user_tag; }; diff --git a/src/inc_internal/maat_command.h b/src/inc_internal/maat_command.h index 4539d35..e1a777e 100644 --- a/src/inc_internal/maat_command.h +++ b/src/inc_internal/maat_command.h @@ -28,7 +28,7 @@ struct maat_cmd_line { const char *table_name; const char *table_line; - int rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary. + uint64_t rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary. int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout. }; @@ -41,6 +41,8 @@ struct maat_cmd_line */ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule); +long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int increment); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index f5ba44c..4223b16 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -53,7 +53,7 @@ int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, maat_rule_ex_dup_func_t *dup_func, long argl, void *argp, struct log_handle *logger); -void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, int compile_id, size_t idx); +void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, uint64_t compile_id, size_t idx); void compile_table_rule_ex_data_iterate(struct compile_schema *compile_schema, int idx); size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema); @@ -68,7 +68,7 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, int valid_column); int compile_runtime_commit(void *compile_runtime, const char *table_name); -int compile_runtime_match(struct compile_runtime *compile_rt, int *compile_ids, +int compile_runtime_match(struct compile_runtime *compile_rt, uint64_t *compile_ids, size_t compile_ids_size, struct maat_state *state); size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, @@ -92,8 +92,7 @@ struct maat_compile_state *maat_compile_state_new(int thread_id); void maat_compile_state_free(struct maat_compile_state *compile_state); int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, - int *hit_item_ids, size_t hit_item_cnt, - int *group_ids, int hit_group_index, size_t group_ids_size, + uint64_t *hit_item_ids, size_t hit_item_cnt, size_t *n_hit_group_id, struct maat_state *state); int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index 159ea2e..c569a34 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -45,12 +45,10 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name); * @retval the num of hit group_id */ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, - size_t data_len, int *group_ids, int hit_group_index, - size_t group_ids_size, int vtable_ids, struct maat_state *state); + size_t data_len, int vtable_ids, struct maat_state *state); void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); -int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, - int *group_ids, int hit_group_index, size_t group_ids_size, +int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, int vtable_id, struct maat_state *state); void expr_runtime_stream_close(struct expr_runtime *expr_rt); diff --git a/src/inc_internal/maat_flag.h b/src/inc_internal/maat_flag.h index 6c41fe3..98084f3 100644 --- a/src/inc_internal/maat_flag.h +++ b/src/inc_internal/maat_flag.h @@ -46,8 +46,7 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name); * * @retval the num of hit group_id */ -int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, uint64_t flag, - int *group_ids, int hit_group_index,size_t group_ids_size, +int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, uint64_t flag, int vtable_id, struct maat_state *state); void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id); diff --git a/src/inc_internal/maat_group.h b/src/inc_internal/maat_group.h index 787cbc2..e914497 100644 --- a/src/inc_internal/maat_group.h +++ b/src/inc_internal/maat_group.h @@ -16,6 +16,8 @@ extern "C" { #endif +#include + #include "cJSON/cJSON.h" #include "maat_kv.h" @@ -38,13 +40,13 @@ void group2group_runtime_free(void *g2g_runtime); void maat_group_ref_inc(struct maat_group *group); void maat_group_ref_dec(struct maat_group *group); -struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id); +struct maat_group *group2group_runtime_add_group(void *g2g_runtime, uint64_t group_id); void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group); -struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id); +struct maat_group *group2group_runtime_find_group(void *g2g_runtime, uint64_t group_id); int group2group_runtime_build_top_groups(void *g2g_runtime); -int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, - size_t n_group_ids, int *top_group_ids); +int group2group_runtime_get_top_groups(void *g2g_runtime, uint64_t *group_ids, + size_t n_group_ids, uint64_t *top_group_ids); int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column); diff --git a/src/inc_internal/maat_interval.h b/src/inc_internal/maat_interval.h index 663b979..9ea07dc 100644 --- a/src/inc_internal/maat_interval.h +++ b/src/inc_internal/maat_interval.h @@ -47,8 +47,7 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name); * @retval the num of hit group_id */ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, - uint64_t integer, int *group_ids, int hit_group_index, - size_t group_ids_size, int vtable_id, struct maat_state *state); + uint64_t integer, int vtable_id, struct maat_state *state); void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id); long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread); diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h index fe02619..16aa950 100644 --- a/src/inc_internal/maat_ip.h +++ b/src/inc_internal/maat_ip.h @@ -41,8 +41,7 @@ struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt); /* ip runtime scan API */ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, - uint8_t *ip_addr, int *group_ids, int hit_group_index, - size_t group_id_size, int vtable_id, struct maat_state *state); + uint8_t *ip_addr, int vtable_id, struct maat_state *state); void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id); long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread); diff --git a/src/inc_internal/maat_plugin.h b/src/inc_internal/maat_plugin.h index ab4ef19..4e5539b 100644 --- a/src/inc_internal/maat_plugin.h +++ b/src/inc_internal/maat_plugin.h @@ -64,6 +64,8 @@ size_t plugin_runtime_cached_row_count(void *plugin_runtime); const char *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index); +void *plugin_runtime_get_ex_data(void *plugin_runtime, void *plugin_schema, const char *key); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index bd86e38..b620ad2 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -61,7 +61,7 @@ enum tag_match { }; struct maat_rule_head { - int config_id; + uint64_t config_id; char resevered; int serv_def_len; }; @@ -69,8 +69,8 @@ struct maat_rule_head { #define ITEM_RULE_MAGIC 0x4d3c2b1a struct maat_item_inner { long long magic_num; - int item_id; - int group_id; + uint64_t item_id; + uint64_t group_id; int district_id; int expr_id_cnt; int expr_id_lb; //low boundary @@ -78,8 +78,8 @@ struct maat_item_inner { }; struct maat_item { - int item_id; - int group_id; + uint64_t item_id; + uint64_t group_id; UT_hash_handle hh; void *user_data; }; @@ -87,7 +87,7 @@ struct maat_item { #define COMPILE_RULE_MAGIC 0x1a2b3c4d struct compile_rule { long long magic_num; - int compile_id; + uint64_t compile_id; struct maat_rule_head head;// fix len of Maat_rule_t char *service_defined; int declared_clause_num; @@ -98,8 +98,8 @@ struct compile_rule { }; struct group2group_rule { - int group_id; - int super_group_id; + uint64_t group_id; + uint64_t super_group_id; }; struct maat_runtime { @@ -254,11 +254,11 @@ size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_ int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag); -struct maat_item *maat_item_new(int item_id, int group_id, void *user_data); +struct maat_item *maat_item_new(uint64_t item_id, uint64_t group_id, void *user_data); void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)); -struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id); +struct maat_item_inner *maat_item_inner_new(uint64_t group_id, uint64_t item_id, int district_id); void maat_item_inner_free(void *item_inner); diff --git a/src/inc_internal/maat_utils.h b/src/inc_internal/maat_utils.h index 50d3ec5..145fd65 100644 --- a/src/inc_internal/maat_utils.h +++ b/src/inc_internal/maat_utils.h @@ -80,7 +80,7 @@ void ipv6_ntoh(unsigned int *v6_addr); int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len); /* the column value must be integer */ -int get_column_value(const char *line, int column_seq); +long long get_column_value(const char *line, int column_seq); int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *out_sz); diff --git a/src/maat_api.c b/src/maat_api.c index cd96016..1f5172b 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -492,7 +492,7 @@ int maat_compile_table_ex_schema_register(struct maat *maat_instance, int table_ } void *maat_compile_table_get_ex_data(struct maat *maat_instance, int compile_table_id, - int compile_id, size_t idx) + uint64_t compile_id, size_t idx) { struct compile_schema *schema = (struct compile_schema *)table_manager_get_schema(maat_instance->tbl_mgr, compile_table_id); @@ -684,6 +684,31 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i return 0; } +void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, const char *key) +{ + + if (NULL == maat_instance || table_id < 0 || table_id >= MAX_TABLE_NUM || NULL == key) { + return NULL; + } + + struct maat_runtime *maat_rt = maat_instance->maat_rt; + if (NULL == maat_rt) { + return NULL; + } + + void *plugin_rt = table_manager_get_runtime(maat_rt->ref_tbl_mgr, table_id); + if (NULL == plugin_rt) { + return NULL; + } + + void *plugin_schema = table_manager_get_schema(maat_rt->ref_tbl_mgr, table_id); + if (NULL == plugin_schema) { + return NULL; + } + + return plugin_runtime_get_ex_data(plugin_rt, plugin_schema, key); +} + int maat_ip_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, const struct ip_addr *ip_addr, void **ex_data_array, size_t n_ex_data) @@ -806,7 +831,7 @@ static inline int scan_status_should_compile_NOT(struct maat_state *state) return 0; } -size_t hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile_ids_size, +size_t hit_group_to_compile(void *compile_runtime, uint64_t *compile_ids, size_t compile_ids_size, struct maat_state *mid) { size_t n_hit_compile = compile_runtime_match((struct compile_runtime *)compile_runtime, @@ -814,8 +839,235 @@ size_t hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t comp return n_hit_compile; } +static int vtable_get_physical_table_ids(struct table_manager *tbl_mgr, int table_id, + int *physical_table_ids, size_t n_table_id_array, + int *vtable_id) +{ + size_t physical_table_cnt = 0; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + + if (table_type == TABLE_TYPE_VIRTUAL) { + //find physical table id + *vtable_id = table_id; + void *virtual_schema = table_manager_get_schema(tbl_mgr, table_id); + assert(virtual_schema != NULL); + physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); + return physical_table_cnt; + } else { + *vtable_id = 0; + physical_table_ids[0] = table_id; + physical_table_cnt = 1; + } + + return physical_table_cnt; +} + +int flag_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint64_t flag, + int physical_table_ids[], int physical_table_cnt, int vtable_id, + struct maat_state *mid) +{ + int sum_hit_group_cnt = 0; + + for (int i = 0; i < physical_table_cnt; i++) { + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); + if (table_type != TABLE_TYPE_FLAG) { + continue; + } + + void *flag_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); + if (NULL == flag_rt) { + return -1; + } + + int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id, + flag, vtable_id, mid); + if (group_hit_cnt < 0) { + return -1; + } + + if (group_hit_cnt > 0) { + flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id); + } + sum_hit_group_cnt += group_hit_cnt; + } + + return sum_hit_group_cnt; +} + +int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint64_t intval, + int physical_table_ids[], int physical_table_cnt, int vtable_id, + struct maat_state *mid) +{ + int sum_hit_group_cnt = 0; + + for (size_t i = 0; i < physical_table_cnt; i++) { + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); + if ((table_type == TABLE_TYPE_INTERVAL_PLUS) && + (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { + //maat_instance->scan_err_cnt++; + return MAAT_SCAN_ERR; + } + + if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { + continue; + } + + void *interval_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); + if (NULL == interval_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt, + thread_id, intval, vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + + if (group_hit_cnt > 0) { + interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id); + } + sum_hit_group_cnt += group_hit_cnt; + } + + return sum_hit_group_cnt; +} + +int ipv4_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint32_t ip_addr, + int physical_table_ids[], int physical_table_cnt, int vtable_id, + struct maat_state *mid) +{ + int sum_hit_group_cnt = 0; + + for (size_t i = 0; i < physical_table_cnt; i++) { + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); + if (table_type != TABLE_TYPE_IP_PLUS) { + continue; + } + + void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); + if (NULL == ip_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4, + (uint8_t *)&ip_addr, vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + + if (group_hit_cnt > 0) { + ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); + } + sum_hit_group_cnt += group_hit_cnt; + } + + return sum_hit_group_cnt; +} + +int ipv6_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint8_t *ip_addr, + int physical_table_ids[], int physical_table_cnt, int vtable_id, + struct maat_state *mid) +{ + int sum_hit_group_cnt = 0; + + for (size_t i = 0; i < physical_table_cnt; i++) { + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); + if (table_type != TABLE_TYPE_IP_PLUS) { + continue; + } + + void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); + if (NULL == ip_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, + ip_addr, vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + + if (group_hit_cnt > 0) { + ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); + } + sum_hit_group_cnt += group_hit_cnt; + } + + return sum_hit_group_cnt; +} + +int string_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, const char *data, + size_t data_len, int physical_table_ids[], int physical_table_cnt, + int vtable_id, struct maat_state *mid) +{ + int sum_hit_group_cnt = 0; + + for (size_t i = 0; i < physical_table_cnt; i++) { + enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); + if ((table_type == TABLE_TYPE_EXPR_PLUS) && + (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { + //maat_instance->scan_err_cnt++; + return -1; + } + + if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { + continue; + } + + void *expr_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); + if (NULL == expr_rt) { + return -1; + } + + int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt, + thread_id, data, data_len, + vtable_id, mid); + if (group_hit_cnt < 0) { + return -1; + } + + if (group_hit_cnt > 0) { + expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); + } + sum_hit_group_cnt += group_hit_cnt; + } + + return sum_hit_group_cnt; +} + +size_t group_to_compile(struct maat *maat_instance, uint64_t *results, size_t n_result, + struct maat_state *mid) +{ + int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0}; + size_t compile_table_cnt = 0; + size_t sum_hit_compile_cnt = 0; + + if (0 == mid->n_compile_table) { + compile_table_id[0] = maat_instance->default_compile_table_id; + compile_table_cnt = 1; + } else { + for (size_t i = 0; i < mid->n_compile_table; i++) { + compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); + } + compile_table_cnt = mid->n_compile_table; + } + + for (size_t i = 0; i < compile_table_cnt; i++) { + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]); + size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + sum_hit_compile_cnt += n_hit_compile; + } + + assert(mid->is_last_scan < LAST_SCAN_FINISHED); + if (LAST_SCAN_SET == mid->is_last_scan) { + mid->is_last_scan = LAST_SCAN_FINISHED; + } + + return sum_hit_compile_cnt; +} + int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, - uint64_t flag, int *results, size_t n_result, + uint64_t flag, uint64_t *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { if ((NULL == maat_instance) || table_id < 0 || table_id >= MAX_TABLE_NUM @@ -828,109 +1080,59 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; + if (NULL == maat_instance->maat_rt) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "maat_scan_flag error because of maat_runtime is NULL"); + return MAAT_SCAN_OK; + } + int vtable_id = 0; - size_t physical_table_cnt = 0; + int physical_table_cnt = 0; int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; memset(physical_table_ids, -1, sizeof(physical_table_ids)); - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - if (table_type == TABLE_TYPE_VIRTUAL) { - //find physical table id - void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); - vtable_id = virtual_table_get_id(virtual_schema); - if (vtable_id < 0) { - return MAAT_SCAN_ERR; - } + physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, + physical_table_ids, + MAX_PHYSICAL_TABLE_NUM, &vtable_id); + if (physical_table_cnt <= 0) { + return MAAT_SCAN_ERR; + } - physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); - if (physical_table_cnt <= 0) { - return MAAT_SCAN_ERR; - } - } else { - //physical table type must be TABLE_TYPE_FLAG + enum table_type table_type = TABLE_TYPE_INVALID; + if (0 == vtable_id) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); if (table_type != TABLE_TYPE_FLAG) { return MAAT_SCAN_ERR; } - - physical_table_ids[0] = table_id; - physical_table_cnt = 1; - } - - if (NULL == maat_instance->maat_rt) { - log_error(maat_instance->logger, MODULE_MAAT_API, - "maat_scan_string error because of maat_runtime is NULL"); - return MAAT_SCAN_OK; } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; - memset(group_ids, -1, sizeof(group_ids)); - int hit_group_index = 0; - for (size_t i = 0; i < physical_table_cnt; i++) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); - if (table_type != TABLE_TYPE_FLAG) { - continue; - } - - void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_ids[i]); - if (NULL == flag_rt) { - return MAAT_SCAN_ERR; - } - - int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id, - flag, group_ids, hit_group_index, - MAX_SCANNER_HIT_GROUP_NUM, - vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } - - if (group_hit_cnt > 0) { - flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id); - } - hit_group_index += group_hit_cnt; + int hit_group_cnt = flag_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, flag, + physical_table_ids, + physical_table_cnt, vtable_id, mid); + if (hit_group_cnt < 0) { + return MAAT_SCAN_ERR; } size_t sum_hit_compile_cnt = 0; - if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + if (hit_group_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT - int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0}; - size_t compile_table_cnt = 0; - if (0 == mid->n_compile_table) { - compile_table_id[0] = maat_instance->default_compile_table_id; - compile_table_cnt = 1; - } else { - for (size_t i = 0; i < mid->n_compile_table; i++) { - compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); - } - compile_table_cnt = mid->n_compile_table; - } - - for (size_t i = 0; i < compile_table_cnt; i++) { - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]); - size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - sum_hit_compile_cnt += n_hit_compile; - } + sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, mid); *n_hit_result = sum_hit_compile_cnt; - - assert(mid->is_last_scan < LAST_SCAN_FINISHED); - if (LAST_SCAN_SET == mid->is_last_scan) { - mid->is_last_scan = LAST_SCAN_FINISHED; - } } if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == hit_group_index) { + if (0 == hit_group_cnt) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { // sum_hit_compile_cnt == 0 - if (hit_group_index > 0) { + if (hit_group_cnt > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -941,7 +1143,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, } int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, - unsigned int intval, int *results, size_t n_result, + unsigned int intval, uint64_t *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { if ((NULL == maat_instance) || table_id < 0 || table_id >= MAX_TABLE_NUM @@ -954,115 +1156,59 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; + if (NULL == maat_instance->maat_rt) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "maat_scan_flag error because of maat_runtime is NULL"); + return MAAT_SCAN_OK; + } + int vtable_id = 0; - size_t physical_table_cnt = 0; + int physical_table_cnt = 0; int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; memset(physical_table_ids, -1, sizeof(physical_table_ids)); - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - if (table_type == TABLE_TYPE_VIRTUAL) { - //find physical table id - void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); - vtable_id = virtual_table_get_id(virtual_schema); - if (vtable_id < 0) { - return MAAT_SCAN_ERR; - } + physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, + physical_table_ids, + MAX_PHYSICAL_TABLE_NUM, &vtable_id); + if (physical_table_cnt <= 0) { + return MAAT_SCAN_ERR; + } - physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); - if (physical_table_cnt <= 0) { - return MAAT_SCAN_ERR; - } - } else { - //physcial table type must be TABLE_TYPE_INTERVAL or TABLE_TYPE_INTERVAL_PLUS + enum table_type table_type = TABLE_TYPE_INVALID; + if (0 == vtable_id) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { return MAAT_SCAN_ERR; } - - physical_table_ids[0] = table_id; - physical_table_cnt = 1; - } - - if (NULL == maat_instance->maat_rt) { - log_error(maat_instance->logger, MODULE_MAAT_API, - "maat_scan_string error because of maat_runtime is NULL"); - return MAAT_SCAN_OK; } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; - memset(group_ids, -1, sizeof(group_ids)); - int hit_group_index = 0; - for (size_t i = 0; i < physical_table_cnt; i++) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); - if ((table_type == TABLE_TYPE_INTERVAL_PLUS) && - (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { - maat_instance->scan_err_cnt++; - return MAAT_SCAN_ERR; - } - - if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { - continue; - } - - void *interval_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - if (NULL == interval_rt) { - return MAAT_SCAN_ERR; - } - - int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt, - thread_id, intval, group_ids, hit_group_index, - MAX_SCANNER_HIT_GROUP_NUM, - vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } - - if (group_hit_cnt > 0) { - interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id); - } - hit_group_index += group_hit_cnt; + int hit_group_cnt = interval_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, intval, + physical_table_ids, + physical_table_cnt, vtable_id, mid); + if (hit_group_cnt < 0) { + return MAAT_SCAN_ERR; } - + size_t sum_hit_compile_cnt = 0; - if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + if (hit_group_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT - int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0}; - size_t compile_table_cnt = 0; - if (0 == mid->n_compile_table) { - compile_table_id[0] = maat_instance->default_compile_table_id; - compile_table_cnt = 1; - } else { - for (size_t i = 0; i < mid->n_compile_table; i++) { - compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); - } - compile_table_cnt = mid->n_compile_table; - } - - for (size_t i = 0; i < compile_table_cnt; i++) { - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]); - size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - sum_hit_compile_cnt += n_hit_compile; - } + sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, mid); *n_hit_result = sum_hit_compile_cnt; - - assert(mid->is_last_scan < LAST_SCAN_FINISHED); - if (LAST_SCAN_SET == mid->is_last_scan) { - mid->is_last_scan = LAST_SCAN_FINISHED; - } } if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == hit_group_index) { + if (0 == hit_group_cnt) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { // sum_hit_compile_cnt == 0 - if (hit_group_index > 0) { + if (hit_group_cnt > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -1073,7 +1219,7 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, } int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, - uint32_t ip_addr, int *results, size_t n_result, + uint32_t ip_addr, uint64_t *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { if ((NULL == maat_instance) || table_id < 0 || table_id >= MAX_TABLE_NUM @@ -1086,110 +1232,59 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; - maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); + if (NULL == maat_instance->maat_rt) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "maat_scan_flag error because of maat_runtime is NULL"); + return MAAT_SCAN_OK; + } int vtable_id = 0; - size_t physical_table_cnt = 0; + int physical_table_cnt = 0; int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; memset(physical_table_ids, -1, sizeof(physical_table_ids)); - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - if (table_type == TABLE_TYPE_VIRTUAL) { - //find physical table id - void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); - vtable_id = virtual_table_get_id(virtual_schema); - if (vtable_id < 0) { - return MAAT_SCAN_ERR; - } + physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, + physical_table_ids, + MAX_PHYSICAL_TABLE_NUM, &vtable_id); + if (physical_table_cnt <= 0) { + return MAAT_SCAN_ERR; + } - physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); - if (physical_table_cnt <= 0) { - return MAAT_SCAN_ERR; - } - } else { - //physcial table type must be TABLE_TYPE_IP_PLUS + enum table_type table_type = TABLE_TYPE_INVALID; + if (0 == vtable_id) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); if (table_type != TABLE_TYPE_IP_PLUS) { return MAAT_SCAN_ERR; } - - physical_table_ids[0] = table_id; - physical_table_cnt = 1; - } - - if (NULL == maat_instance->maat_rt) { - log_error(maat_instance->logger, MODULE_MAAT_API, - "maat_scan_ipv4 error because of maat_runtime is NULL"); - return MAAT_SCAN_OK; } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; - memset(group_ids, -1, sizeof(group_ids)); - int hit_group_index = 0; - for (size_t i = 0; i < physical_table_cnt; i++) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); - if (table_type != TABLE_TYPE_IP_PLUS) { - continue; - } - - void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - if (NULL == ip_rt) { - return MAAT_SCAN_ERR; - } - - int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4, - (uint8_t *)&ip_addr, group_ids, hit_group_index, - MAX_SCANNER_HIT_GROUP_NUM, vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } - - if (group_hit_cnt > 0) { - ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); - } - hit_group_index += group_hit_cnt; + int hit_group_cnt = ipv4_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, ip_addr, + physical_table_ids, + physical_table_cnt, vtable_id, mid); + if (hit_group_cnt < 0) { + return MAAT_SCAN_ERR; } - size_t sum_hit_compile_cnt = 0; - if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + size_t sum_hit_compile_cnt = 0; + if (hit_group_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT - int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0}; - size_t compile_table_cnt = 0; - if (0 == mid->n_compile_table) { - compile_table_id[0] = maat_instance->default_compile_table_id; - compile_table_cnt = 1; - } else { - for (size_t i = 0; i < mid->n_compile_table; i++) { - compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); - } - compile_table_cnt = mid->n_compile_table; - } - - for (size_t i = 0; i < compile_table_cnt; i++) { - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]); - size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - sum_hit_compile_cnt += n_hit_compile; - } + sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, mid); *n_hit_result = sum_hit_compile_cnt; - - assert(mid->is_last_scan < LAST_SCAN_FINISHED); - if (LAST_SCAN_SET == mid->is_last_scan) { - mid->is_last_scan = LAST_SCAN_FINISHED; - } } if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == hit_group_index) { + if (0 == hit_group_cnt) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { // n_hit_compile == 0 - if (hit_group_index > 0) { + if (hit_group_cnt > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -1201,7 +1296,7 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, - uint8_t *ip_addr, int *results, size_t n_result, + uint8_t *ip_addr, uint64_t *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { if ((NULL == maat_instance) || table_id < 0 || table_id >= MAX_TABLE_NUM @@ -1214,108 +1309,59 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; + if (NULL == maat_instance->maat_rt) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "maat_scan_flag error because of maat_runtime is NULL"); + return MAAT_SCAN_OK; + } + int vtable_id = 0; - size_t physical_table_cnt = 0; + int physical_table_cnt = 0; int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; memset(physical_table_ids, -1, sizeof(physical_table_ids)); - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - if (table_type == TABLE_TYPE_VIRTUAL) { - //find physical table id - void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); - vtable_id = virtual_table_get_id(virtual_schema); - if (vtable_id < 0) { - return MAAT_SCAN_ERR; - } + physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, + physical_table_ids, + MAX_PHYSICAL_TABLE_NUM, &vtable_id); + if (physical_table_cnt <= 0) { + return MAAT_SCAN_ERR; + } - physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); - if (physical_table_cnt <= 0) { - return MAAT_SCAN_ERR; - } - } else { - //physcial table type must be TABLE_TYPE_IP_PLUS + enum table_type table_type = TABLE_TYPE_INVALID; + if (0 == vtable_id) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); if (table_type != TABLE_TYPE_IP_PLUS) { return MAAT_SCAN_ERR; } - - physical_table_ids[0] = table_id; - physical_table_cnt = 1; - } - - if (NULL == maat_instance->maat_rt) { - log_error(maat_instance->logger, MODULE_MAAT_API, - "maat_scan_string error because of maat_runtime is NULL"); - return MAAT_SCAN_OK; } maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; - memset(group_ids, -1, sizeof(group_ids)); - int hit_group_index = 0; - for (size_t i = 0; i < physical_table_cnt; i++) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); - if (table_type != TABLE_TYPE_IP_PLUS) { - continue; - } - - void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - if (NULL == ip_rt) { - return MAAT_SCAN_ERR; - } - - int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, - ip_addr, group_ids, hit_group_index, - MAX_SCANNER_HIT_GROUP_NUM, vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } - - if (group_hit_cnt > 0) { - ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); - } - hit_group_index += group_hit_cnt; + int hit_group_cnt = ipv6_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, ip_addr, + physical_table_ids, + physical_table_cnt, vtable_id, mid); + if (hit_group_cnt < 0) { + return MAAT_SCAN_ERR; } size_t sum_hit_compile_cnt = 0; - if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + if (hit_group_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT - int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0}; - size_t compile_table_cnt = 0; - if (0 == mid->n_compile_table) { - compile_table_ids[0] = maat_instance->default_compile_table_id; - compile_table_cnt = 1; - } else { - for (size_t i = 0; i < mid->n_compile_table; i++) { - compile_table_ids[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); - } - compile_table_cnt = mid->n_compile_table; - } - - for (size_t i = 0; i < compile_table_cnt; i++) { - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]); - size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - sum_hit_compile_cnt += n_hit_compile; - } + sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, mid); *n_hit_result = sum_hit_compile_cnt; - - assert(mid->is_last_scan < LAST_SCAN_FINISHED); - if (LAST_SCAN_SET == mid->is_last_scan) { - mid->is_last_scan = LAST_SCAN_FINISHED; - } } if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == hit_group_index) { + if (0 == hit_group_cnt) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { // n_hit_compile == 0 - if (hit_group_index > 0) { + if (hit_group_cnt > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -1326,7 +1372,7 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, } int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, - const char *data, size_t data_len, int *results, size_t n_result, + const char *data, size_t data_len, uint64_t *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { if ((NULL == maat_instance) || table_id < 0 || table_id >= MAX_TABLE_NUM @@ -1339,113 +1385,59 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; - int vtable_id = 0; - size_t physical_table_cnt = 0; - int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; - memset(physical_table_ids, -1, sizeof(physical_table_ids)); - - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - if (table_type == TABLE_TYPE_VIRTUAL) { - //find physical table id - void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); - vtable_id = virtual_table_get_id(virtual_schema); - if (vtable_id < 0) { - return MAAT_SCAN_ERR; - } - - physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids); - if (physical_table_cnt <= 0) { - return MAAT_SCAN_ERR; - } - } else { - //physcial table type must be TABLE_TYPE_EXPR or TABLE_TYPE_EXPR_PLUS - if ((table_type == TABLE_TYPE_EXPR_PLUS) && - (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)){ - maat_instance->scan_err_cnt++; - return MAAT_SCAN_ERR; - } - - physical_table_ids[0] = table_id; - physical_table_cnt = 1; - } - if (NULL == maat_instance->maat_rt) { log_error(maat_instance->logger, MODULE_MAAT_API, "maat_scan_string error because of maat_runtime is NULL"); return MAAT_SCAN_OK; } + int vtable_id = 0; + int physical_table_cnt = 0; + int physical_table_ids[MAX_PHYSICAL_TABLE_NUM]; + memset(physical_table_ids, -1, sizeof(physical_table_ids)); + + physical_table_cnt = vtable_get_physical_table_ids(maat_instance->tbl_mgr, table_id, + physical_table_ids, + MAX_PHYSICAL_TABLE_NUM, &vtable_id); + if (physical_table_cnt <= 0) { + return MAAT_SCAN_ERR; + } + + enum table_type table_type = TABLE_TYPE_INVALID; + if (0 == vtable_id) { + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); + if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { + return MAAT_SCAN_ERR; + } + } + maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int group_ids[MAX_SCANNER_HIT_GROUP_NUM]; - memset(group_ids, -1, sizeof(group_ids)); - int hit_group_index = 0; - for (size_t i = 0; i < physical_table_cnt; i++) { - table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]); - if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) { - continue; - } - - void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_ids[i]); - if (NULL == expr_rt) { - return MAAT_SCAN_ERR; - } - - int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt, - thread_id, data, data_len, - group_ids, hit_group_index, - MAX_SCANNER_HIT_GROUP_NUM, - vtable_id, mid); - if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; - } - - if (group_hit_cnt > 0) { - expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); - } - hit_group_index += group_hit_cnt; + int hit_group_cnt = string_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, data, + data_len, physical_table_ids, + physical_table_cnt, vtable_id, mid); + if (hit_group_cnt < 0) { + return MAAT_SCAN_ERR; } size_t sum_hit_compile_cnt = 0; - if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) { + if (hit_group_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT - int compile_table_ids[MAX_COMPILE_TABLE_NUM]; - memset(compile_table_ids, -1, sizeof(compile_table_ids)); - size_t compile_table_cnt = 0; - if (0 == mid->n_compile_table) { - compile_table_ids[0] = maat_instance->default_compile_table_id; - compile_table_cnt = 1; - } else { - for (size_t i = 0; i < mid->n_compile_table; i++) { - compile_table_ids[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); - } - compile_table_cnt = mid->n_compile_table; - } - - for (size_t i = 0; i < compile_table_cnt; i++) { - void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]); - size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); - sum_hit_compile_cnt += n_hit_compile; - } + sum_hit_compile_cnt = group_to_compile(maat_instance, results, n_result, mid); *n_hit_result = sum_hit_compile_cnt; - - assert(mid->is_last_scan < LAST_SCAN_FINISHED); - if (LAST_SCAN_SET == mid->is_last_scan) { - mid->is_last_scan = LAST_SCAN_FINISHED; - } } if (sum_hit_compile_cnt > 0) { alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); - if (0 == hit_group_index) { + if (0 == hit_group_cnt) { //hit NOT group alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); } return MAAT_SCAN_HIT; } else { // n_hit_compile == 0 - if (hit_group_index > 0) { + if (hit_group_cnt > 0) { return MAAT_SCAN_HALF_HIT; } } @@ -1460,9 +1452,8 @@ struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, i return NULL; } -int maat_scan_stream(struct maat_stream **stream, int thread_id, - const char *data, int data_len, int *results, - size_t *n_result, struct maat_state **state) +int maat_scan_stream(struct maat_stream **stream, const char *data, int data_len, + uint64_t *results, size_t *n_result, struct maat_state **state) { return 0; } diff --git a/src/maat_bool_plugin.c b/src/maat_bool_plugin.c index 52034f9..4015433 100644 --- a/src/maat_bool_plugin.c +++ b/src/maat_bool_plugin.c @@ -33,7 +33,7 @@ struct bool_plugin_schema { }; struct bool_plugin_item { - int item_id; + uint64_t item_id; size_t n_bool_item; unsigned long long bool_item_id[MAX_ITEMS_PER_BOOL_EXPR]; }; @@ -290,7 +290,7 @@ bool_plugin_item_new(const char *line, struct bool_plugin_schema *schema, schema->table_id, line); goto error; } - item->item_id = atoi(line + column_offset); + item->item_id = atoll(line + column_offset); ret = get_column_pos(line, schema->bool_expr_column, &column_offset, &column_len); if (ret < 0) { @@ -355,7 +355,7 @@ size_t ull_dedup(unsigned long long item_ids[], size_t n_item) return index + 1; } -struct bool_expr *bool_expr_new(int item_id, struct bool_plugin_item *item) +struct bool_expr *bool_expr_new(uint64_t item_id, struct bool_plugin_item *item) { struct bool_expr *expr = ALLOC(struct bool_expr, 1); @@ -383,7 +383,11 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche struct bool_expr *expr = NULL; struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema; struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; - int item_id = get_column_value(line, schema->item_id_column); + long long item_id = get_column_value(line, schema->item_id_column); + if (item_id < 0) { + return -1; + } + int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; @@ -404,7 +408,7 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche char *key = (char *)&item_id; int ret = bool_plugin_runtime_update_row(bool_plugin_rt, schema, line, key, - sizeof(int), expr, is_valid); + sizeof(uint64_t), expr, is_valid); if (ret < 0) { if (item != NULL) { FREE(item); diff --git a/src/maat_command.c b/src/maat_command.c index 0ee1441..a4f7ce8 100644 --- a/src/maat_command.c +++ b/src/maat_command.c @@ -352,4 +352,26 @@ error_out: FREE(s_rule); return ret; +} + +long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int increment) +{ + long long result = 0; + + redisContext *write_ctx = get_redis_ctx_for_write(maat_instance); + if (NULL == write_ctx) { + return -1; + } + + redisReply *data_reply = maat_cmd_wrap_redis_command(write_ctx, "INCRBY %s %d", key, increment); + if (data_reply->type == REDIS_REPLY_INTEGER) { + result = data_reply->integer; + } else { + result = -1; + } + + freeReplyObject(data_reply); + data_reply = NULL; + + return result; } \ No newline at end of file diff --git a/src/maat_compile.c b/src/maat_compile.c index b4f7c3c..67fc77f 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -29,7 +29,7 @@ #define MAX_COMPILE_EX_DATA_NUM 2 enum user_region_encode { - USER_REGION_ENCODE_NONE=0, + USER_REGION_ENCODE_NONE = 0, USER_REGION_ENCODE_ESCAPE, USER_REGION_ENCODE_BASE64 }; @@ -60,15 +60,15 @@ struct group2compile_schema { }; struct compile_item { - int compile_id; + uint64_t compile_id; char user_region[MAX_TABLE_LINE_SIZE]; int declared_clause_num; int evaluation_order; }; struct group2compile_item { - int group_id; - int compile_id; + uint64_t group_id; + uint64_t compile_id; int not_flag; int vtable_id; int clause_index; @@ -104,7 +104,7 @@ struct maat_clause_state { }; struct maat_literal_id { - int group_id; + uint64_t group_id; int vtable_id; }; @@ -118,14 +118,14 @@ struct maat_clause { struct compile_sort_para { double evaluation_order; int declared_clause_num; - int compile_id; + uint64_t compile_id; void *user; }; #define MAAT_COMPILE_MAGIC 0x4a5b6c7d struct maat_compile { unsigned int magic; - int compile_id; + uint64_t compile_id; int actual_clause_num; int declared_clause_num; int not_clause_cnt; @@ -138,8 +138,8 @@ struct maat_compile { struct maat_internal_hit_path { int Nth_scan; int Nth_hit_item; - int item_id; - int group_id; + uint64_t item_id; + uint64_t group_id; int vtable_id; }; @@ -184,13 +184,13 @@ int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, return idx; } -void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, int compile_id, int is_dettach) +void *compile_runtime_get_user_data(struct compile_runtime *compile_rt, uint64_t compile_id, int is_dettach) { struct maat_compile *compile = NULL; void *ret = NULL; pthread_rwlock_rdlock(&compile_rt->rwlock); - HASH_FIND_INT(compile_rt->compile_hash, &compile_id, compile); + HASH_FIND(hh, compile_rt->compile_hash, &compile_id, sizeof(uint64_t), compile); if (compile != NULL) { ret = compile->user_data; if (is_dettach) { @@ -278,7 +278,7 @@ void compile_table_rule_ex_data_iterate(struct compile_schema *compile_schema, i compile_runtime_user_data_iterate(compile_rt, rule_ex_data_new_cb, ex_schema); } -void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, int compile_id, size_t idx) +void *compile_table_get_rule_ex_data(struct compile_schema *compile_schema, uint64_t compile_id, size_t idx) { if (NULL == compile_schema) { return NULL; @@ -495,7 +495,7 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, compile_schema->table_id, line); goto error; } - compile_item->compile_id = atoi(line + column_offset); + compile_item->compile_id = atoll(line + column_offset); ret = get_column_pos(line, compile_schema->tags_column, &column_offset, &column_len); @@ -717,7 +717,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema g2c_schema->table_id, line); goto error; } - g2c_item->group_id = atoi(line + column_offset); + g2c_item->group_id = atoll(line + column_offset); ret = get_column_pos(line, g2c_schema->compile_id_column, &column_offset, &column_len); if (ret < 0) { @@ -726,7 +726,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema g2c_schema->table_id, line); goto error; } - g2c_item->compile_id = atoi(line + column_offset); + g2c_item->compile_id = atoll(line + column_offset); ret = get_column_pos(line, g2c_schema->not_flag_column, &column_offset, &column_len); if (ret < 0) { @@ -790,7 +790,7 @@ void group2compile_item_free(struct group2compile_item *g2c_item) } #define MAAT_HIER_COMPILE_MAGIC 0x4a5b6c7d -struct maat_compile *maat_compile_new(int compile_id) +struct maat_compile *maat_compile_new(uint64_t compile_id) { struct maat_compile *compile = ALLOC(struct maat_compile, 1); @@ -819,31 +819,31 @@ int maat_compile_set(struct maat_compile *compile, int declared_clause_num, return 0; } -int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, +int maat_compile_hash_add(struct maat_compile **compile_hash, uint64_t compile_id, struct maat_compile *compile) { int ret = 0; assert(compile->declared_clause_num >= 0); - HASH_ADD_INT(*compile_hash, compile_id, compile); - + HASH_ADD(hh, *compile_hash, compile_id, sizeof(uint64_t), compile); //TODO:mytest need to delete #if 0 size_t compile_cnt = HASH_COUNT(*compile_hash); struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL; HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) { - printf("compile->compile_id:%d, compile_cnt:%zu\n", compile1->compile_id, compile_cnt); + printf(" compile_id:%lu, compile_cnt:%zu\n", + compile1->compile_id, compile_cnt); } #endif return ret; } -void maat_compile_hash_set(struct maat_compile **compile_hash, int compile_id, +void maat_compile_hash_set(struct maat_compile **compile_hash, uint64_t compile_id, struct maat_compile *compile) { struct maat_compile *tmp_compile = NULL; - HASH_FIND_INT(*compile_hash, &compile_id, tmp_compile); + HASH_FIND(hh, *compile_hash, &compile_id, sizeof(uint64_t), tmp_compile); assert(tmp_compile != NULL); assert(tmp_compile->user_data == NULL); @@ -852,8 +852,8 @@ void maat_compile_hash_set(struct maat_compile **compile_hash, int compile_id, } -int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id, - struct maat_compile *compile, struct maat_garbage_bin *garbage_bin) +int maat_compile_hash_remove(struct maat_compile **compile_hash, struct maat_compile *compile, + struct maat_garbage_bin *garbage_bin) { if (0 == compile->actual_clause_num) { HASH_DEL(*compile_hash, compile); @@ -865,20 +865,20 @@ int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id, size_t compile_cnt = HASH_COUNT(*compile_hash); struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL; HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) { - printf("compile->compile_id:%d, compile_cnt:%zu\n", compile1->compile_id, compile_cnt); + printf(" compile_id:%lu, compile_cnt:%zu\n", + compile1->compile_id, compile_cnt); } #endif return 0; } -struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash, - int compile_id) +struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash, uint64_t compile_id) { struct maat_compile *compile = NULL; HASH_FIND(hh, *compile_hash, &compile_id, sizeof(compile_id), compile); - return compile; + return compile; } size_t maat_compile_in_use_count(struct maat_compile *compile_hash) @@ -930,7 +930,8 @@ int maat_compile_clause_add_literal(struct maat_compile *compile, tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); if (tmp) { - assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); + assert(tmp->group_id == literal_id->group_id); + assert(tmp->vtable_id == literal_id->vtable_id); return -1; } else { utarray_push_back(clause_state->literal_ids, literal_id); @@ -1055,7 +1056,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, #if 0 struct maat_literal_id *p = NULL; for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) { - printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n", + printf("compile_id:%lu, clause_id:%llu, literal{%lu: %d}\n", compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vtable_id); } #endif @@ -1066,6 +1067,8 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, } } + // printf("bool_matcher_new compile_id:%lu j:%zu, compile->declared_clause_num:%d\n", + // compile->compile_id, j, compile->declared_clause_num); //some compile may have zero groups, e.g. default policy. if (j == (size_t)compile->declared_clause_num && j > 0) { bool_expr_array[expr_cnt].expr_id = compile->compile_id; @@ -1163,14 +1166,9 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan return ud_result_cnt; } -int maat_add_group_to_compile(struct maat_compile **compile_hash, void *g2g_runtime, - struct group2compile_item *g2c_item, struct log_handle *logger) +int maat_add_group_to_compile(struct maat_compile **compile_hash, struct group2compile_item *g2c_item, + struct log_handle *logger) { - struct maat_group *group = group2group_runtime_find_group(g2g_runtime, g2c_item->group_id); - if (!group) { - group = group2group_runtime_add_group(g2g_runtime, g2c_item->group_id); - } - int ret = -1; struct maat_compile *compile = maat_compile_hash_find(compile_hash, g2c_item->compile_id); if (!compile) { @@ -1192,27 +1190,19 @@ int maat_add_group_to_compile(struct maat_compile **compile_hash, void *g2g_runt g2c_item->compile_id); ret = -1; } else { - maat_group_ref_inc(group); ret = 0; } + // printf("group2compile update compile_id:%lu, compile->declared_clause_num:%d\n", + // compile->compile_id, compile->declared_clause_num); return ret; } -int maat_remove_group_from_compile(struct maat_compile **compile_hash, - void *g2g_runtime, +int maat_remove_group_from_compile(struct maat_compile **compile_hash, struct group2compile_item *g2c_item, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { - struct maat_group *group = group2group_runtime_find_group(g2g_runtime, g2c_item->group_id); - if (!group) { - log_error(logger, MODULE_COMPILE, - "Remove group %d from compile %d failed, group is not exisited.", - g2c_item->group_id, g2c_item->compile_id); - return -1; - } - struct maat_compile *compile = NULL; HASH_FIND(hh, *compile_hash, &(g2c_item->compile_id), sizeof(g2c_item->compile_id), compile); if (!compile) { @@ -1230,7 +1220,6 @@ int maat_remove_group_from_compile(struct maat_compile **compile_hash, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, g2c_item->compile_id); return -1; } - maat_group_ref_dec(group); if (0 == compile->actual_clause_num && NULL == compile->user_data) { HASH_DEL(*compile_hash, compile); @@ -1274,7 +1263,7 @@ void maat_compile_state_free(struct maat_compile_state *compile_state) free(compile_state); } -static int maat_compile_hit_path_add(UT_array *hit_paths, int item_id, int group_id, +static int maat_compile_hit_path_add(UT_array *hit_paths, uint64_t item_id, uint64_t group_id, int vtable_id, int Nth_scan, int Nth_item_result) { struct maat_internal_hit_path new_path; @@ -1343,7 +1332,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, /* NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths */ - int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + uint64_t top_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; + memset(top_group_ids, 0, sizeof(top_group_ids)); int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &(internal_path->group_id), 1, top_group_ids); if (top_group_cnt <= 0) { @@ -1417,7 +1407,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, } void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, - int item_id, int group_id, int vtable_id, + uint64_t item_id, uint64_t group_id, int vtable_id, int Nth_scan, int Nth_item_result) { if (compile_state->Nth_scan != Nth_scan) { @@ -1434,7 +1424,7 @@ void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state } void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, - void *compile_runtime, int group_id, + void *compile_runtime, uint64_t group_id, int vtable_id) { if (NULL == compile_state || NULL == compile_runtime) { @@ -1448,7 +1438,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta unsigned long long *clause_id = 0; struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; - pthread_rwlock_wrlock(&compile_rt->rwlock); + pthread_rwlock_rdlock(&compile_rt->rwlock); assert(compile_rt->compile_hash != NULL); HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) { @@ -1554,11 +1544,15 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, struct compile_schema *schema = (struct compile_schema *)compile_schema; struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; int is_valid = get_column_value(line, valid_column); - int compile_id = get_column_value(line, schema->compile_id_column); if (is_valid < 0) { return -1; } + long long compile_id = get_column_value(line, schema->compile_id_column); + if (compile_id < 0) { + return -1; + } + if (0 == is_valid) { //delete pthread_rwlock_wrlock(&compile_rt->rwlock); @@ -1568,12 +1562,12 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, return -1; } - ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id, - compile, compile_rt->ref_garbage_bin); + ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile, + compile_rt->ref_garbage_bin); pthread_rwlock_unlock(&compile_rt->rwlock); if (ret < 0) { log_error(compile_rt->logger, MODULE_COMPILE, - "remove compile table(table_id:%d) compile(compile_id:%d) from compile_hash failed", + "remove compile table(table_id:%d) compile(compile_id:%lld) from compile_hash failed", schema->table_id, compile_id); return -1; } @@ -1604,16 +1598,15 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule, (void (*)(void *))destroy_compile_rule); - struct maat_compile *tmp_compile = maat_compile_hash_find(&(compile_rt->compile_hash), compile_id); if (tmp_compile != NULL) { maat_compile_hash_set(&(compile_rt->compile_hash), compile_id, compile); } else { maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile); } - if (compile->compile_id == 141) { - printf("compile->declared_clause_num:%d\n", compile->declared_clause_num); - } + // printf("compile_runtime_update compile_id:%lu, compile->declared_clause_num:%d\n", + // compile->compile_id, compile->declared_clause_num); + pthread_rwlock_unlock(&compile_rt->rwlock); } @@ -1642,27 +1635,45 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, return -1; } - pthread_rwlock_wrlock(&compile_rt->rwlock); + struct maat_group *group = NULL; if (0 == is_valid) { //delete - ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item, + group = group2group_runtime_find_group(g2g_rt, g2c_item->group_id); + if (!group) { + log_error(compile_rt->logger, MODULE_COMPILE, + "Remove group %d from compile %d failed, group is not exisited.", + g2c_item->group_id, g2c_item->compile_id); + return -1; + } + + pthread_rwlock_wrlock(&compile_rt->rwlock); + ret = maat_remove_group_from_compile(&(compile_rt->compile_hash), g2c_item, compile_rt->ref_garbage_bin, compile_rt->logger); + pthread_rwlock_unlock(&compile_rt->rwlock); if (0 == ret) { if (g2c_item->not_flag) { g2c_rt->not_flag_group--; } + maat_group_ref_dec(group); } } else { //add - ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item, - compile_rt->logger); + group = group2group_runtime_find_group(g2g_rt, g2c_item->group_id); + if (!group) { + group = group2group_runtime_add_group(g2g_rt, g2c_item->group_id); + } + + pthread_rwlock_wrlock(&compile_rt->rwlock); + ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2c_item, compile_rt->logger); + pthread_rwlock_unlock(&compile_rt->rwlock); if (0 == ret) { if (g2c_item->not_flag) { g2c_rt->not_flag_group++; } + maat_group_ref_inc(group); } } - pthread_rwlock_unlock(&compile_rt->rwlock); + group2compile_item_free(g2c_item); return ret; @@ -1758,7 +1769,7 @@ static int compare_compile_rule(const void *a, const void *b) } int compile_runtime_match(struct compile_runtime *compile_rt, - int *compile_ids, size_t compile_ids_size, + uint64_t *compile_ids, size_t compile_ids_size, struct maat_state *state) { struct maat_compile_state *compile_state = state->compile_state; @@ -1785,12 +1796,13 @@ int compile_runtime_match(struct compile_runtime *compile_rt, } int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, - int *hit_item_ids, size_t hit_item_cnt, - int *group_ids, int hit_group_index, size_t group_ids_size, + uint64_t *hit_item_ids, size_t hit_item_cnt, size_t *n_hit_group_id, struct maat_state *state) { struct maat_item *item = NULL; - size_t hit_group_cnt = hit_group_index; + uint64_t hit_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; + memset(hit_group_ids, 0, sizeof(hit_group_ids)); + size_t hit_group_cnt = 0; void *g2g_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, state->maat_instance->g2g_table_id); @@ -1799,18 +1811,18 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, } for (size_t i = 0; i < hit_item_cnt; i++) { - HASH_FIND_INT(item_hash, &(hit_item_ids[i]), item); + HASH_FIND(hh, item_hash, &(hit_item_ids[i]), sizeof(uint64_t), item); //assert(item != NULL); if (!item) { // item config has been deleted continue; } - if (hit_group_cnt >= group_ids_size) { - hit_group_cnt = group_ids_size; + if (hit_group_cnt >= MAX_SCANNER_HIT_GROUP_NUM) { + hit_group_cnt = MAX_SCANNER_HIT_GROUP_NUM; //Prevent group_id_array out of bounds } else { - group_ids[hit_group_cnt++] = item->group_id; + hit_group_ids[hit_group_cnt++] = item->group_id; } // update hit path @@ -1837,8 +1849,9 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_ids[idx]); for (size_t i = 0; i < hit_group_cnt; i++) { - int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i], + uint64_t top_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; + memset(top_group_ids, 0, sizeof(top_group_ids)); + int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &hit_group_ids[i], 1, top_group_ids); for (int j = 0; j < top_group_cnt; j++) { maat_compile_state_update_hit_clause(state->compile_state, compile_rt, diff --git a/src/maat_expr.c b/src/maat_expr.c index 66ed732..67c8e2c 100644 --- a/src/maat_expr.c +++ b/src/maat_expr.c @@ -58,8 +58,8 @@ enum match_method { }; struct expr_item { - int item_id; - int group_id; + uint64_t item_id; + uint64_t group_id; int district_id; char keywords[MAX_KEYWORDS_STR]; enum expr_type expr_type; @@ -156,7 +156,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem expr_schema->table_id, line); goto error; } - expr_item->item_id = atoi(line + column_offset); + expr_item->item_id = atoll(line + column_offset); ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len); if (ret < 0) { @@ -165,7 +165,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem expr_schema->table_id, line); goto error; } - expr_item->group_id = atoi(line + column_offset); + expr_item->group_id = atoll(line + column_offset); table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id); if (table_type == TABLE_TYPE_EXPR_PLUS) { @@ -485,7 +485,7 @@ void expr_runtime_free(void *expr_runtime) } int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, - size_t key_len, int item_id, and_expr_t *expr_rule, + size_t key_len, uint64_t item_id, and_expr_t *expr_rule, int is_valid, struct log_handle *logger) { void *data = NULL; @@ -495,7 +495,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, data = rcu_hash_find(expr_rt->htable, key, key_len); if (NULL == data) { log_error(logger, MODULE_EXPR, - "the key of expr rule not exist, can't be deleted, expr_id:%d", + "the key of expr rule not exist, can't be deleted, expr_id:%llu", item_id); return -1; } @@ -505,7 +505,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, data = rcu_hash_find(expr_rt->htable, key, key_len); if (data != NULL) { log_error(logger, MODULE_EXPR, - "the key of expr rule already exist, can't be added, expr_id:%d", + "the key of expr rule already exist, can't be added, expr_id:%llu", item_id); return -1; } @@ -719,13 +719,17 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, struct expr_schema *schema = (struct expr_schema *)expr_schema; struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; - int item_id = get_column_value(line, schema->item_id_column); + long long item_id = get_column_value(line, schema->item_id_column); + if (item_id < 0) { + return -1; + } + int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; } else if (0 == is_valid) { //delete - HASH_FIND_INT(expr_rt->item_hash, &item_id, item); + HASH_FIND(hh, expr_rt->item_hash, &item_id, sizeof(uint64_t), item); if (NULL == item) { return -1; } @@ -741,10 +745,10 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, maat_item_inner_free); } else { //add - HASH_FIND_INT(expr_rt->item_hash, &item_id, item); + HASH_FIND(hh, expr_rt->item_hash, &item_id, sizeof(uint64_t), item); if (item) { log_error(expr_rt->logger, MODULE_EXPR, - "expr runtime add item %d to item_hash failed, already exist", + "expr runtime add item %llu to item_hash failed, already exist", item_id); return -1; } @@ -756,20 +760,20 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, u_para = maat_item_inner_new(expr_item->group_id, item_id, expr_item->district_id); item = maat_item_new(item_id, expr_item->group_id, u_para); - HASH_ADD_INT(expr_rt->item_hash, item_id, item); + HASH_ADD(hh, expr_rt->item_hash, item_id, sizeof(uint64_t), item); expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger); expr_item_free(expr_item); if (NULL == expr_rule) { log_error(expr_rt->logger, MODULE_EXPR, - "transform expr table(table_id:%d) item to expr_rule failed, item_id:%d", + "transform expr table(table_id:%d) item to expr_rule failed, item_id:%llu", schema->table_id, item_id); return -1; } } char *key = (char *)&item_id; - ret = expr_runtime_update_row(expr_rt, key, sizeof(int), item_id, + ret = expr_runtime_update_row(expr_rt, key, sizeof(uint64_t), item_id, expr_rule, is_valid, expr_rt->logger); if (ret < 0) { if (expr_rule != NULL) { @@ -841,8 +845,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name) } int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, - size_t data_len, int *group_ids, int hit_group_index, - size_t group_ids_size, int vtable_ids, struct maat_state *state) + size_t data_len, int vtable_ids, struct maat_state *state) { size_t n_hit_item = 0; struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; @@ -865,12 +868,12 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; } - int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; + uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; struct maat_item_inner *item = NULL; int real_hit_item_cnt = 0; int district_id = state->district_id; - memset(hit_item_ids, -1, sizeof(hit_item_ids)); + memset(hit_item_ids, 0, sizeof(hit_item_ids)); for (size_t i = 0; i < n_hit_item; i++) { item = (struct maat_item_inner *)(hit_results[i].user_tag); @@ -881,8 +884,7 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d size_t group_hit_cnt = 0; ret = maat_compile_state_update(expr_rt->item_hash, vtable_ids, hit_item_ids, - real_hit_item_cnt, group_ids, hit_group_index, - group_ids_size, &group_hit_cnt, state); + real_hit_item_cnt, &group_hit_cnt, state); if (ret < 0) { return -1; } @@ -900,8 +902,7 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) expr_rt->hs_stream = hs_stream; } -int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, - int *group_ids, int hit_group_index, size_t group_ids_size, +int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, int vtable_id, struct maat_state *state) { if (NULL == expr_rt) { @@ -928,15 +929,16 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; } - int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; + uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; + memset(hit_item_ids, 0, sizeof(hit_item_ids)); + for (size_t i = 0; i < n_hit_item; i++) { hit_item_ids[i] = hit_results[i].item_id; } size_t group_hit_cnt = 0; ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, - n_hit_item, group_ids, hit_group_index, - group_ids_size, &group_hit_cnt, state); + n_hit_item, &group_hit_cnt, state); if (ret < 0) { return -1; } diff --git a/src/maat_flag.c b/src/maat_flag.c index fbf5266..9bb9775 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -34,8 +34,8 @@ struct flag_schema { }; struct flag_item { - int item_id; - int group_id; + uint64_t item_id; + uint64_t group_id; uint64_t flag; uint64_t flag_mask; }; @@ -176,7 +176,7 @@ void flag_runtime_free(void *flag_runtime) } int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len, - int item_id, struct flag_rule *rule, int is_valid) + uint64_t item_id, struct flag_rule *rule, int is_valid) { void *data = NULL; @@ -185,7 +185,7 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_ data = rcu_hash_find(flag_rt->htable, key, key_len); if (NULL == data) { log_error(flag_rt->logger, MODULE_FLAG, - "the key of flag rule not exist, can't be deleted, item_id:%d", + "the key of flag rule not exist, can't be deleted, item_id:%llu", item_id); return -1; } @@ -195,7 +195,7 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_ data = rcu_hash_find(flag_rt->htable, key, key_len); if (data != NULL) { log_error(flag_rt->logger, MODULE_FLAG, - "the key of flag rule already exist, can't be added, item_id:%d", + "the key of flag rule already exist, can't be added, item_id:%llu", item_id); return -1; } @@ -219,7 +219,7 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schem flag_schema->table_id, line); goto error; } - flag_item->item_id = atoi(line + column_offset); + flag_item->item_id = atoll(line + column_offset); ret = get_column_pos(line, flag_schema->group_id_column, &column_offset, &column_len); if (ret < 0) { @@ -228,7 +228,7 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schem flag_schema->table_id, line); goto error; } - flag_item->group_id = atoi(line + column_offset); + flag_item->group_id = atoll(line + column_offset); ret = get_column_pos(line, flag_schema->flag_column, &column_offset, &column_len); if (ret < 0) { @@ -291,13 +291,17 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, struct flag_schema *schema = (struct flag_schema *)flag_schema; struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; - int item_id = get_column_value(line, schema->item_id_column); + long long item_id = get_column_value(line, schema->item_id_column); + if (item_id < 0) { + return -1; + } + int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; } else if (0 == is_valid) { //delete - HASH_FIND_INT(flag_rt->item_hash, &item_id, item); + HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item); if (NULL == item) { return -1; } @@ -313,7 +317,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free); } else { //add - HASH_FIND_INT(flag_rt->item_hash, &item_id, item); + HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(uint64_t), item); if (item) { log_error(flag_rt->logger, MODULE_FLAG, "flag runtime add item %d to item_hash failed, already exist", @@ -328,7 +332,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, u_para = maat_item_inner_new(flag_item->group_id, item_id, 0); item = maat_item_new(item_id, flag_item->group_id, u_para); - HASH_ADD_INT(flag_rt->item_hash, item_id, item); + HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(uint64_t), item); flag_rule = flag_item_to_flag_rule(flag_item, u_para); flag_item_free(flag_item); @@ -341,7 +345,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, } char *key = (char *)&item_id; - int ret = flag_runtime_update_row(flag_rt, key, sizeof(int), item_id, flag_rule, is_valid); + int ret = flag_runtime_update_row(flag_rt, key, sizeof(uint64_t), item_id, flag_rule, is_valid); if (ret < 0) { if (flag_rule != NULL) { flag_rule_free(flag_rule); @@ -411,8 +415,7 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name) } int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, - uint64_t flag, int *group_ids, int hit_group_index, - size_t group_ids_size, int vtable_id, struct maat_state *state) + uint64_t flag, int vtable_id, struct maat_state *state) { struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; @@ -429,8 +432,8 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; } - int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; - memset(hit_item_ids, -1, sizeof(hit_item_ids)); + uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; + memset(hit_item_ids, 0, sizeof(hit_item_ids)); for (int i = 0; i < n_hit_item; i++) { hit_item_ids[i] = hit_results[i].rule_id; @@ -438,8 +441,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, size_t group_hit_cnt = 0; int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids, - n_hit_item, group_ids, hit_group_index, group_ids_size, - &group_hit_cnt, state); + n_hit_item, &group_hit_cnt, state); if (ret < 0) { return -1; } diff --git a/src/maat_fqdn_plugin.c b/src/maat_fqdn_plugin.c index 8590836..667732b 100644 --- a/src/maat_fqdn_plugin.c +++ b/src/maat_fqdn_plugin.c @@ -35,8 +35,9 @@ struct fqdn_plugin_schema { }; struct fqdn_plugin_item { - int item_id; + uint64_t item_id; int suffix_flag; + }; struct fqdn_plugin_runtime { @@ -257,7 +258,7 @@ fqdn_plugin_item_new(const char *line, struct fqdn_plugin_schema *schema, schema->table_id, line); goto error; } - item->item_id = atoi(line + column_offset); + item->item_id = atoll(line + column_offset); ret = get_column_pos(line, schema->suffix_flag_column, &column_offset, &column_len); if (ret < 0) { @@ -323,7 +324,11 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche struct FQDN_rule *rule = NULL; struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema; struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; - int item_id = get_column_value(line, schema->item_id_column); + long long item_id = get_column_value(line, schema->item_id_column); + if (item_id < 0) { + return -1; + } + int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; @@ -344,7 +349,7 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_sche char *key = (char *)&item_id; int ret = fqdn_plugin_runtime_update_row(fqdn_plugin_rt, schema, line, key, - sizeof(int), rule, is_valid); + sizeof(uint64_t), rule, is_valid); if (ret < 0) { if (item != NULL) { FREE(item); diff --git a/src/maat_group.c b/src/maat_group.c index 0c374b6..88b32c6 100644 --- a/src/maat_group.c +++ b/src/maat_group.c @@ -21,8 +21,8 @@ #define MODULE_GROUP module_name_str("maat.group") struct group2group_item { - int group_id; - int super_group_id; + uint64_t group_id; + uint64_t super_group_id; }; struct group2group_schema { @@ -34,13 +34,13 @@ struct group2group_schema { struct maat_group { igraph_integer_t vertex_id; - int group_id; + uint64_t group_id; int ref_by_compile_cnt; int ref_by_super_group_cnt; int ref_by_sub_group_cnt; size_t top_group_cnt; - int *top_group_ids; + uint64_t *top_group_ids; UT_hash_handle hh_group_id; UT_hash_handle hh_vertex_id; @@ -54,7 +54,6 @@ struct maat_group_topology { igraph_vector_t dfs_vids; igraph_integer_t grp_vertex_id_generator; - pthread_rwlock_t rwlock; struct log_handle *logger; }; @@ -64,6 +63,7 @@ struct group2group_runtime { uint32_t rule_num; uint32_t updating_rule_num; + pthread_rwlock_t rwlock; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; }; @@ -128,9 +128,6 @@ struct maat_group_topology *maat_group_topology_new(struct log_handle *logger) ret = igraph_empty(&group_topo->group_graph, 0, IGRAPH_DIRECTED); assert(ret == IGRAPH_SUCCESS); - ret = pthread_rwlock_init(&group_topo->rwlock, NULL); - assert(ret == 0); - group_topo->logger = logger; return group_topo; @@ -145,6 +142,7 @@ void *group2group_runtime_new(void *g2g_schema, int max_thread_num, g2g_rt->group_topo = maat_group_topology_new(logger); g2g_rt->ref_garbage_bin = garbage_bin; g2g_rt->logger = logger; + pthread_rwlock_init(&g2g_rt->rwlock, NULL); return g2g_rt; } @@ -167,9 +165,6 @@ void maat_group_topology_free(struct maat_group_topology *group_topo) assert(group_topo->hash_group_by_id == NULL); igraph_destroy(&group_topo->group_graph); - - pthread_rwlock_unlock(&group_topo->rwlock); - pthread_rwlock_destroy(&group_topo->rwlock); } void group2group_runtime_free(void *g2g_runtime) @@ -184,6 +179,8 @@ void group2group_runtime_free(void *g2g_runtime) maat_group_topology_free(g2g_rt->group_topo); } + //pthread_rwlock_unlock(&g2g_rt->rwlock); + pthread_rwlock_destroy(&g2g_rt->rwlock); FREE(g2g_rt); } @@ -212,7 +209,7 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema, g2g_schema->table_id, line); goto error; } - g2g_item->group_id = atoi(line + column_offset); + g2g_item->group_id = atoll(line + column_offset); ret = get_column_pos(line, g2g_schema->super_group_id_column, &column_offset, &column_len); @@ -222,7 +219,7 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema, g2g_schema->table_id, line); goto error; } - g2g_item->super_group_id = atoi(line + column_offset); + g2g_item->super_group_id = atoll(line + column_offset); return g2g_item; error: @@ -246,18 +243,18 @@ size_t print_igraph_vector(igraph_vector_t *v, char *buff, size_t sz) { return printed; } -struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id) +struct maat_group *_group2group_runtime_add_group(void *g2g_runtime, uint64_t group_id, int lock_flag) { - if (NULL == g2g_runtime) { - return NULL; - } - struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + + if (1 == lock_flag) { + pthread_rwlock_wrlock(&(g2g_rt->rwlock)); + } + struct maat_group_topology *group_topo = g2g_rt->group_topo; assert(group_topo != NULL); struct maat_group *group = ALLOC(struct maat_group, 1); - group->group_id = group_id; group->vertex_id = group_topo->grp_vertex_id_generator++; @@ -266,19 +263,32 @@ struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id HASH_ADD(hh_group_id, group_topo->hash_group_by_id, group_id, sizeof(group->group_id), group); HASH_ADD(hh_vertex_id, group_topo->hash_group_by_vertex, vertex_id, sizeof(group->vertex_id), group); - + if (1 == lock_flag) { + pthread_rwlock_unlock(&(g2g_rt->rwlock)); + } + return group; } -void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group) +struct maat_group *group2group_runtime_add_group(void *g2g_runtime, uint64_t group_id) { - if (NULL == g2g_runtime || NULL == group) { - return; + if (NULL == g2g_runtime) { + return NULL; } - igraph_vector_t v; + return _group2group_runtime_add_group(g2g_runtime, group_id, 1); +} + +void _group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group, int lock_flag) +{ + igraph_vector_t v; char buff[4096] = {0}; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + + if (1 == lock_flag) { + pthread_rwlock_wrlock(&(g2g_rt->rwlock)); + } + struct maat_group_topology *group_topo = g2g_rt->group_topo; assert(group_topo != NULL); @@ -302,26 +312,55 @@ void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *grou HASH_DELETE(hh_vertex_id, group_topo->hash_group_by_vertex, group); group_vertex_free(group); + if (1 == lock_flag) { + pthread_rwlock_unlock(&(g2g_rt->rwlock)); + } } -struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id) +void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group) +{ + if (NULL == g2g_runtime || NULL == group) { + return; + } + + _group2group_runtime_remove_group(g2g_runtime, group, 1); +} + +struct maat_group *_group2group_runtime_find_group(void *g2g_runtime, uint64_t group_id, int lock_flag) { if (NULL == g2g_runtime) { return NULL; } struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + if (1 == lock_flag) { + pthread_rwlock_rdlock(&(g2g_rt->rwlock)); + } + struct maat_group_topology *group_topo = g2g_rt->group_topo; assert(group_topo != NULL); struct maat_group *group = NULL; HASH_FIND(hh_group_id, group_topo->hash_group_by_id, &group_id, sizeof(group_id), group); + if (1 == lock_flag) { + pthread_rwlock_unlock(&(g2g_rt->rwlock)); + } + return group; } -int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, - int super_group_id) +struct maat_group *group2group_runtime_find_group(void *g2g_runtime, uint64_t group_id) +{ + if (NULL == g2g_runtime) { + return NULL; + } + + return _group2group_runtime_find_group(g2g_runtime, group_id, 1); +} + +int group2group_runtime_add_group_to_group(void *g2g_runtime, uint64_t group_id, + uint64_t super_group_id) { if (NULL == g2g_runtime) { return -1; @@ -330,18 +369,20 @@ int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int ret = 0; igraph_integer_t edge_id; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + + pthread_rwlock_wrlock(&(g2g_rt->rwlock)); struct maat_group_topology *group_topo = g2g_rt->group_topo; assert(group_topo != NULL); - struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_id); + struct maat_group *group = _group2group_runtime_find_group(g2g_runtime, group_id, 0); if (NULL == group) { - group = group2group_runtime_add_group(g2g_runtime, group_id); + group = _group2group_runtime_add_group(g2g_runtime, group_id, 0); } - struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime, - super_group_id); + struct maat_group *super_group = _group2group_runtime_find_group(g2g_runtime, + super_group_id, 0); if (NULL == super_group) { - super_group = group2group_runtime_add_group(g2g_runtime, super_group_id); + super_group = _group2group_runtime_add_group(g2g_runtime, super_group_id, 0); } ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, @@ -361,11 +402,12 @@ int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, ret = 0; } + pthread_rwlock_unlock(&(g2g_rt->rwlock)); return ret; } -int group2group_runtime_remove_group_from_group(void *g2g_runtime, - int group_id, int super_group_id) +int group2group_runtime_remove_group_from_group(void *g2g_runtime, uint64_t group_id, + uint64_t super_group_id) { if (NULL == g2g_runtime) { return -1; @@ -373,21 +415,24 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + pthread_rwlock_wrlock(&(g2g_rt->rwlock)); //No hash write operation, LOCK protection is unnecessary. - struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_id); + struct maat_group *group = _group2group_runtime_find_group(g2g_runtime, group_id, 0); if (NULL == group) { log_error(g2g_rt->logger, MODULE_GROUP, "Del group %d from group %d failed, group %d not exisited.", group_id, super_group_id, group_id); + pthread_rwlock_unlock(&(g2g_rt->rwlock)); return -1; } - struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime, - super_group_id); + struct maat_group *super_group = _group2group_runtime_find_group(g2g_runtime, + super_group_id, 0); if (NULL == super_group) { log_error(g2g_rt->logger, MODULE_GROUP, "Del group %d from group %d failed, superior group %d not exisited.", group_id, super_group_id, super_group_id); + pthread_rwlock_unlock(&(g2g_rt->rwlock)); return -1; } @@ -412,12 +457,14 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, igraph_es_destroy(&es); if (ret != IGRAPH_SUCCESS || edge_num_before - edge_num_after != 1) { + pthread_rwlock_unlock(&(g2g_rt->rwlock)); assert(0); return -1; } group->ref_by_super_group_cnt--; super_group->ref_by_sub_group_cnt--; + pthread_rwlock_unlock(&(g2g_rt->rwlock)); return 0; } @@ -447,8 +494,10 @@ int group2group_runtime_build_top_groups(void *g2g_runtime) struct maat_group *super_group = NULL; int tmp_vid=0; size_t top_group_cnt=0; - int* temp_group_ids=NULL; + int *temp_group_ids=NULL; struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + + pthread_rwlock_wrlock(&(g2g_rt->rwlock)); struct maat_group_topology *group_topo = g2g_rt->group_topo; assert(group_topo != NULL); @@ -456,6 +505,7 @@ int group2group_runtime_build_top_groups(void *g2g_runtime) igraph_is_dag(&(group_topo->group_graph), &is_dag); if (!is_dag) { log_error(g2g_rt->logger, MODULE_GROUP, "Sub group cycle detected!"); + pthread_rwlock_unlock(&(g2g_rt->rwlock)); return -1; } @@ -472,7 +522,7 @@ int group2group_runtime_build_top_groups(void *g2g_runtime) && 0 == group->ref_by_sub_group_cnt) { FREE(group->top_group_ids); - group2group_runtime_remove_group(g2g_runtime, group); + _group2group_runtime_remove_group(g2g_runtime, group, 0); continue; } @@ -510,12 +560,13 @@ int group2group_runtime_build_top_groups(void *g2g_runtime) free(group->top_group_ids); group->top_group_cnt = top_group_cnt; - group->top_group_ids = ALLOC(int, group->top_group_cnt); - memcpy(group->top_group_ids, temp_group_ids, sizeof(int)*group->top_group_cnt); + group->top_group_ids = ALLOC(uint64_t, group->top_group_cnt); + memcpy(group->top_group_ids, temp_group_ids, sizeof(uint64_t)*group->top_group_cnt); FREE(temp_group_ids); } igraph_vector_destroy(&group_topo->dfs_vids); + pthread_rwlock_unlock(&(g2g_rt->rwlock)); return 0; } @@ -571,16 +622,19 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name) return ret; } -int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, - size_t n_group_ids, int *top_group_ids) +int group2group_runtime_get_top_groups(void *g2g_runtime, uint64_t *group_ids, + size_t n_group_ids, uint64_t *top_group_ids) { if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) { return -1; } size_t top_group_index = 0; + struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + + pthread_rwlock_rdlock(&(g2g_rt->rwlock)); for (size_t i = 0; i < n_group_ids; i++) { - struct maat_group *group = group2group_runtime_find_group(g2g_runtime, group_ids[i]); + struct maat_group *group = _group2group_runtime_find_group(g2g_runtime, group_ids[i], 0); if (!group) { continue; } @@ -589,6 +643,7 @@ int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, top_group_ids[top_group_index++] = group->top_group_ids[j]; } } - + pthread_rwlock_unlock(&(g2g_rt->rwlock)); + return top_group_index; } diff --git a/src/maat_interval.c b/src/maat_interval.c index 977b2be..32e2ddc 100644 --- a/src/maat_interval.c +++ b/src/maat_interval.c @@ -34,8 +34,8 @@ struct interval_schema { }; struct interval_item { - int item_id; - int group_id; + uint64_t item_id; + uint64_t group_id; int district_id; int low_bound; int up_bound; @@ -197,7 +197,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema schema->table_id, line); goto error; } - item->item_id = atoi(line + column_offset); + item->item_id = atoll(line + column_offset); ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len); if (ret < 0) { @@ -206,7 +206,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema schema->table_id, line); goto error; } - item->group_id = atoi(line + column_offset); + item->group_id = atoll(line + column_offset); table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id); if (table_type == TABLE_TYPE_INTERVAL_PLUS) { @@ -276,7 +276,7 @@ void interval_rule_free(struct interval_rule *rule) } int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, size_t key_len, - int item_id, struct interval_rule *rule, int is_valid) + uint64_t item_id, struct interval_rule *rule, int is_valid) { void *data = NULL; @@ -285,7 +285,7 @@ int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, data = rcu_hash_find(interval_rt->htable, key, key_len); if (NULL == data) { log_error(interval_rt->logger, MODULE_INTERVAL, - "the key of interval rule not exist, can't be deleted, item_id:%d", + "the key of interval rule not exist, can't be deleted, item_id:%llu", item_id); return -1; } @@ -295,7 +295,7 @@ int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, data = rcu_hash_find(interval_rt->htable, key, key_len); if (data != NULL) { log_error(interval_rt->logger, MODULE_INTERVAL, - "the key of interval rule already exist, can't be added, item_id:%d", + "the key of interval rule already exist, can't be added, item_id:%llu", item_id); return -1; } @@ -319,13 +319,17 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, struct interval_schema *schema = (struct interval_schema *)interval_schema; struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; - int item_id = get_column_value(line, schema->item_id_column); + long long item_id = get_column_value(line, schema->item_id_column); + if (item_id < 0) { + return -1; + } + int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; } else if (0 == is_valid) { //delete - HASH_FIND_INT(interval_rt->item_hash, &item_id, item); + HASH_FIND(hh, interval_rt->item_hash, &item_id, sizeof(uint64_t), item); if (NULL == item) { return -1; } @@ -341,10 +345,10 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, maat_garbage_bagging(interval_rt->ref_garbage_bin, u_para, maat_item_inner_free); } else { //add - HASH_FIND_INT(interval_rt->item_hash, &item_id, item); + HASH_FIND(hh, interval_rt->item_hash, &item_id, sizeof(uint64_t), item); if (item) { log_error(interval_rt->logger, MODULE_INTERVAL, - "interval runtime add item %d to item_hash failed, already exist", + "interval runtime add item %llu to item_hash failed, already exist", item_id); return -1; } @@ -356,20 +360,20 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, u_para = maat_item_inner_new(interval_item->group_id, item_id, interval_item->district_id); item = maat_item_new(item_id, interval_item->group_id, u_para); - HASH_ADD_INT(interval_rt->item_hash, item_id, item); + HASH_ADD(hh, interval_rt->item_hash, item_id, sizeof(uint64_t), item); interval_rule = interval_item_to_interval_rule(interval_item, u_para); interval_item_free(interval_item); if (NULL == interval_rule) { log_error(interval_rt->logger, MODULE_INTERVAL, - "transform interval table(table_id:%d) item to interval_rule failed, item_id:%d", + "transform interval table(table_id:%d) item to interval_rule failed, item_id:%llu", schema->table_id, item_id); return -1; } } char *key = (char *)&item_id; - int ret = interval_runtime_update_row(interval_rt, key, sizeof(int), item_id, interval_rule, is_valid); + int ret = interval_runtime_update_row(interval_rt, key, sizeof(uint64_t), item_id, interval_rule, is_valid); if (ret < 0) { if (interval_rule != NULL) { interval_rule_free(interval_rule); @@ -440,8 +444,7 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name) } int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, - uint64_t integer, int *group_ids, int hit_group_index, - size_t group_ids_size, int vtable_id, struct maat_state *state) + uint64_t integer, int vtable_id, struct maat_state *state) { struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; @@ -458,12 +461,12 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; } - int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; + uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; struct maat_item_inner *item = NULL; int real_hit_item_cnt = 0; int district_id = state->district_id; - memset(hit_item_ids, -1, sizeof(hit_item_ids)); + memset(hit_item_ids, 0, sizeof(hit_item_ids)); for (int i = 0; i < n_hit_item; i++) { item = (struct maat_item_inner *)(hit_results[i].user_tag); @@ -474,8 +477,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, size_t group_hit_cnt = 0; int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids, - n_hit_item, group_ids, hit_group_index, group_ids_size, - &group_hit_cnt, state); + n_hit_item, &group_hit_cnt, state); if (ret < 0) { return -1; } diff --git a/src/maat_ip.c b/src/maat_ip.c index 8150feb..6d0d298 100644 --- a/src/maat_ip.c +++ b/src/maat_ip.c @@ -45,8 +45,8 @@ struct ipv6_item_rule { }; struct ip_item { - int item_id; - int group_id; + uint64_t item_id; + uint64_t group_id; int addr_type; union { struct ipv4_item_rule ipv4; @@ -218,7 +218,7 @@ struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema, ip_schema->table_id, line); goto error; } - ip_item->item_id = atoi(line + column_offset); + ip_item->item_id = atoll(line + column_offset); ret = get_column_pos(line, ip_schema->group_id_column, &column_offset, &column_len); if (ret < 0) { @@ -227,7 +227,7 @@ struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema, ip_schema->table_id, line); goto error; } - ip_item->group_id = atoi(line + column_offset); + ip_item->group_id = atoll(line + column_offset); ret = get_column_pos(line, ip_schema->addr_type_column, &column_offset, &column_len); if (ret < 0) { @@ -365,13 +365,17 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, struct maat_item_inner *u_para = NULL; struct ip_schema *schema = (struct ip_schema *)ip_schema; struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; - int item_id = get_column_value(line, schema->item_id_column); + long long item_id = get_column_value(line, schema->item_id_column); + if (item_id < 0) { + return -1; + } + int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; } else if (0 == is_valid) { //delete - HASH_FIND_INT(ip_rt->item_hash, &item_id, item); + HASH_FIND(hh, ip_rt->item_hash, &item_id, sizeof(uint64_t), item); if (NULL == item) { return -1; } @@ -388,10 +392,10 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, (void (*)(void *))maat_item_inner_free); } else { //add - HASH_FIND_INT(ip_rt->item_hash, &item_id, item); + HASH_FIND(hh, ip_rt->item_hash, &item_id, sizeof(uint64_t), item); if (item) { log_error(ip_rt->logger, MODULE_IP, - "ip runtime add item %d to item_hash failed, already exist", + "ip runtime add item %llu to item_hash failed, already exist", item_id); return -1; } @@ -403,11 +407,11 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, u_para = maat_item_inner_new(ip_item->group_id, item_id, 0); item = maat_item_new(item_id, ip_item->group_id, u_para); - HASH_ADD_INT(ip_rt->item_hash, item_id, item); + HASH_ADD(hh, ip_rt->item_hash, item_id, sizeof(uint64_t), item); } char *key = (char *)&item_id; - int ret = ip_runtime_update_row(ip_rt, key, sizeof(int), ip_item, is_valid); + int ret = ip_runtime_update_row(ip_rt, key, sizeof(uint64_t), ip_item, is_valid); if (ret < 0) { if (ip_item != NULL) { ip_item_free(ip_item); @@ -477,8 +481,7 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name) } int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, - uint8_t *ip_addr, int *group_ids, int hit_group_index, - size_t group_id_size, int vtable_id, struct maat_state *state) + uint8_t *ip_addr, int vtable_id, struct maat_state *state) { int n_hit_item = 0; struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; @@ -507,8 +510,8 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; } - int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; - memset(hit_item_ids, -1, sizeof(hit_item_ids)); + uint64_t hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; + memset(hit_item_ids, 0, sizeof(hit_item_ids)); for (int i = 0; i < n_hit_item; i++) { hit_item_ids[i] = scan_results[i].rule_id; @@ -516,7 +519,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, size_t group_hit_cnt = 0; int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item, - group_ids, hit_group_index, group_id_size, &group_hit_cnt, state); + &group_hit_cnt, state); if (ret < 0) { return -1; } diff --git a/src/maat_ip_plugin.c b/src/maat_ip_plugin.c index 58426f2..3c315e9 100644 --- a/src/maat_ip_plugin.c +++ b/src/maat_ip_plugin.c @@ -33,7 +33,7 @@ struct ipv6_item_rule { }; struct ip_plugin_item { - int item_id; + uint64_t item_id; int ip_type; union { struct ipv4_item_rule ipv4; @@ -226,7 +226,7 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema, schema->table_id, line); goto error; } - ip_plugin_item->item_id = atoi(line + column_offset); + ip_plugin_item->item_id = atoll(line + column_offset); ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len); if (ret < 0) { @@ -263,7 +263,7 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema, #endif const char *tmp_str = "range"; memcpy(addr_format, tmp_str, strlen(tmp_str)); - + ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_IP_PLUGIN, @@ -420,7 +420,11 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, struct ip_plugin_item *ip_plugin_item = NULL; struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; - int item_id = get_column_value(line, schema->item_id_column); + long long item_id = get_column_value(line, schema->item_id_column); + if (item_id < 0) { + return -1; + } + int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; @@ -436,7 +440,7 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, } char *key = (char *)&item_id; - int ret = ip_plugin_runtime_update_row(ip_plugin_rt, line, key, sizeof(int), + int ret = ip_plugin_runtime_update_row(ip_plugin_rt, line, key, sizeof(uint64_t), ip_plugin_item, is_valid); if (ret < 0) { if (ip_plugin_item != NULL) { diff --git a/src/maat_plugin.c b/src/maat_plugin.c index 92a429f..49cb4c1 100644 --- a/src/maat_plugin.c +++ b/src/maat_plugin.c @@ -362,9 +362,9 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, return -1; } - int item_id = get_column_value(line, schema->key_column); + long long item_id = get_column_value(line, schema->key_column); char *key = (char *)&item_id; - ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid); + ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(uint64_t), is_valid); if (ret < 0) { schema->update_err_cnt++; return -1; @@ -422,4 +422,19 @@ const char *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index) } return ex_data_runtime_cached_row_get(plugin_rt->ex_data_rt, index); +} + +void *plugin_runtime_get_ex_data(void *plugin_runtime, void *plugin_schema, const char *key) +{ + if (NULL == plugin_runtime || NULL == plugin_schema) { + return NULL; + } + + struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; + struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; + if (NULL == schema->ex_schema) { + return NULL; + } + + return ex_data_runtime_get_ex_data_by_key(plugin_rt->ex_data_rt, key, strlen(key)); } \ No newline at end of file diff --git a/src/maat_rule.c b/src/maat_rule.c index 8403d6b..bfbe4d1 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -32,7 +32,7 @@ #define MODULE_MAAT_RULE module_name_str("maat.rule") -struct maat_item *maat_item_new(int item_id, int group_id, void *user_data) +struct maat_item *maat_item_new(uint64_t item_id, uint64_t group_id, void *user_data) { struct maat_item *item = NULL; item = ALLOC(struct maat_item, 1); @@ -206,7 +206,7 @@ error: return ret; } -struct maat_item_inner *maat_item_inner_new(int group_id, int item_id, int district_id) +struct maat_item_inner *maat_item_inner_new(uint64_t group_id, uint64_t item_id, int district_id) { struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1); item->magic_num = ITEM_RULE_MAGIC; diff --git a/src/maat_utils.c b/src/maat_utils.c index 8d66ae3..2d51655 100644 --- a/src/maat_utils.c +++ b/src/maat_utils.c @@ -51,7 +51,7 @@ char *maat_strdup(const char *s) return d; } -int get_column_value(const char *line, int column_seq) +long long get_column_value(const char *line, int column_seq) { size_t column_offset = 0; size_t column_len = 0; @@ -61,7 +61,7 @@ int get_column_value(const char *line, int column_seq) return -1; } - return atoi(line + column_offset); + return atoll(line + column_offset); } int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len) diff --git a/src/maat_virtual.c b/src/maat_virtual.c index 5e3dbce..6bfeee7 100644 --- a/src/maat_virtual.c +++ b/src/maat_virtual.c @@ -81,7 +81,7 @@ int virtual_table_get_id(void *virtual_schema) size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]) { if (NULL == virtual_schema) { - return -1; + return 0; } struct virtual_schema *schema = (struct virtual_schema *)virtual_schema; diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 0d9e201..c442d92 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -40,7 +40,7 @@ TEST_F(MaatFlagScan, basic) { //compile_id:192 flag: 0000 0001 mask: 0000 0011 //scan_data: 0000 1001 or 0000 1101 should hit uint64_t scan_data = 9; - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data, results, @@ -83,7 +83,7 @@ TEST_F(MaatFlagScan, withExprRegion) { //compile_id:193 flag: 0000 0010 mask: 0000 0011 //scan_data: 0000 0010 or 0000 0100 should hit uint64_t flag_scan_data = 2; - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; @@ -114,7 +114,7 @@ TEST_F(MaatFlagScan, hitMultiCompile) { //compile_id:194 flag: 0001 0101 mask: 0001 1111 //scan_data: 0001 0101 should hit compile192 and compile194 uint64_t flag_scan_data = 21; - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; @@ -148,7 +148,7 @@ TEST_F(MaatStringScan, Expr8) { const char *table_name = "KEYWORDS_TABLE"; int table_id = maat_table_get_id(g_maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), @@ -166,7 +166,7 @@ TEST_F(MaatStringScan, Expr8) { TEST_F(MaatStringScan, Regex) { int ret = 0; - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *cookie = "Cookie: Txa123aheadBCAxd"; @@ -203,7 +203,7 @@ TEST_F(MaatStringScan, Regex) { } TEST_F(MaatStringScan, ExprPlus) { - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *region_name1 ="HTTP URL"; @@ -237,7 +237,7 @@ TEST_F(MaatStringScan, ExprPlus) { //TODO: #if 0 TEST_F(MaatStringScan, ShouldNotHitExprPlus) { - int results[ARRAY] = {0}; + uint64_t results[ARRAY] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *region_name = "tcp.payload"; @@ -268,7 +268,7 @@ TEST_F(MaatStringScan, ShouldNotHitExprPlus) { } TEST_F(MaatStringScan, ExprPlusWithHex) { - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *scan_data1 = "text/html; charset=UTF-8"; @@ -302,7 +302,7 @@ TEST_F(MaatStringScan, ExprPlusWithHex) { TEST_F(MaatStringScan, ExprPlusWithOffset) { - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *region_name = "Payload"; @@ -355,7 +355,7 @@ TEST_F(MaatStringScan, dynamic_config) { const char *table_name = "HTTP_URL_LITERAL"; int table_id = maat_table_get_id(g_maat_instance, table_name); char data[128] = "hello world"; - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, @@ -458,7 +458,7 @@ TEST_F(MaatIPScan, IPv4) { int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); - int results[ARRAY_SIZE] = {-1}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, @@ -478,8 +478,7 @@ TEST_F(MaatIPScan, IPv6) { int ret = inet_pton(AF_INET6, ip_str, &sip); EXPECT_EQ(ret, 1); - int results[ARRAY_SIZE]; - memset(results, -1, sizeof(results)); + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, @@ -499,7 +498,7 @@ TEST_F(MaatIPScan, dynamic_config) { int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); - int results[ARRAY_SIZE] = {-1}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, @@ -589,7 +588,7 @@ protected: }; TEST_F(MaatIntervalScan, Pure) { - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "CONTENT_SIZE"; @@ -610,7 +609,7 @@ TEST_F(MaatIntervalScan, Pure) { } TEST_F(MaatIntervalScan, IntervalPlus) { - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "INTERGER_PLUS"; @@ -645,7 +644,7 @@ TEST_F(NOTLogic, ScanNotAtLast) { const char *string_should_hit = "This string ONLY contains must-contained-string-of-rule-144."; const char *string_should_not_hit = "This string contains both must-contained-string-of-rule-144 and must-not-contained-string-of-rule-144."; - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *hit_table_name = "HTTP_URL_LITERAL"; @@ -674,23 +673,19 @@ void maat_read_entry_start_cb(int update_type, void *u_para) void maat_read_entry_cb(int table_id, const char *table_line, void *u_para) { - char ip_str[16]={0}; - int entry_id=-1,seq=-1; - unsigned int ip_uint=0; - int is_valid=0; - unsigned int local_ip_nr=16820416;//192.168.0.1 + char ip_str[16] = {0}; + int entry_id = -1, seq = -1; + unsigned int ip_uint = 0; + int is_valid = 0; + unsigned int local_ip_nr = 16820416;//192.168.0.1 sscanf(table_line, "%d\t%s\t%d\t%d", &seq,ip_str, &entry_id, &is_valid); - inet_pton(AF_INET,ip_str,&ip_uint); - if(local_ip_nr==ip_uint) - { - if(is_valid==1) - { + inet_pton(AF_INET, ip_str, &ip_uint); + if (local_ip_nr == ip_uint) { + if (is_valid == 1) { //printf("Load entry id %d success.\n",entry_id); EXPECT_EQ(entry_id, 101); - } - else - { + } else { //printf("Offload entry id %d success.\n",entry_id); } } @@ -745,7 +740,7 @@ protected: }; struct ip_plugin_ud { - int rule_id; + uint64_t rule_id; char *buffer; int ref_cnt; }; @@ -759,7 +754,7 @@ void ip_plugin_EX_new_cb(int table_id, const char *key, const char *table_line, int ret = get_column_pos(table_line, 1, &column_offset, &column_len); EXPECT_EQ(ret, 0); - ud->rule_id = atoi(table_line + column_offset); + ud->rule_id = atoll(table_line + column_offset); ret = get_column_pos(table_line, 5, &column_offset, &column_len); EXPECT_EQ(ret, 0); @@ -853,7 +848,7 @@ protected: }; TEST_F(VirtualTable, basic) { - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "HTTP_RESPONSE_KEYWORDS"; @@ -930,7 +925,7 @@ void compile_ex_param_dup(int idx, void **to, void **from, long argl, void *argp } TEST_F(CompileTable, CompileEXData) { - int results[ARRAY_SIZE] = {0}; + uint64_t results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; @@ -964,6 +959,129 @@ TEST_F(CompileTable, CompileEXData) { maat_state_free(&state); } +class MaatCmdTest : public testing::Test +{ +protected: + static void SetUpTestCase() { + + } + + static void TearDownTestCase() { + + } +}; + +struct user_info { + char name[256]; + char ip_addr[32]; + int id; + int ref_cnt; +}; +void plugin_ex_new_cb(int table_id, const char *key, const char *table_line, + void **ad, long argl, void *argp) +{ + int *counter = (int *)argp; + struct user_info *u = ALLOC(struct user_info, 1); + int valid = 0, tag = 0; + int ret = sscanf(table_line, "%d\t%s\t%s%d\t%d", &(u->id), u->ip_addr, u->name, &valid, &tag); + EXPECT_EQ(ret, 5); + u->ref_cnt = 1; + *ad = u; + (*counter)++; +} + +void plugin_ex_free_cb(int table_id, void **ad, long argl, void *argp) +{ + struct user_info *u = (struct user_info *)(*ad); + if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) { + free(u); + *ad = NULL; + } +} + +void plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *argp) +{ + struct user_info *u = (struct user_info *)(*from); + __sync_add_and_fetch(&(u->ref_cnt), 1); + *to = u; +} +#if 0 +TEST_F(MaatCmdTest, PluginEXData) { + const char *table_name = "TEST_PLUGIN_EXDATA_TABLE"; + const int TEST_CMD_LINE_NUM = 4; + struct maat_cmd_line line_rule; + const char *table_line[TEST_CMD_LINE_NUM] = {"1\t192.168.0.1\tmahuateng\t1\t0", + "2\t192.168.0.2\tliuqiangdong\t1\t0", + "3\t192.168.0.3\tmayun\t1\t0", + "4\t192.168.0.4\tliyanhong\t1\t0"}; + int table_id = maat_table_get_id(g_maat_instance, table_name); + ASSERT_GT(table_id, 0); + + /* 1st line */ + memset(&line_rule, 0, sizeof(line_rule)); + line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1); + line_rule.table_name = table_name; + line_rule.table_line = table_line[0]; + line_rule.expire_after = 0; + + int ret = maat_cmd_set_line(g_maat_instance, &line_rule); + EXPECT_GT(ret, 0); + + /* 2nd line */ + memset(&line_rule, 0, sizeof(line_rule)); + line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1); + line_rule.table_name = table_name; + line_rule.table_line = table_line[1]; + line_rule.expire_after = 0; + + ret = maat_cmd_set_line(g_maat_instance, &line_rule); + EXPECT_GT(ret, 0); + + /* 3rd line */ + memset(&line_rule, 0, sizeof(line_rule)); + line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1); + line_rule.table_name = table_name; + line_rule.table_line = table_line[2]; + line_rule.expire_after = 0; + + ret = maat_cmd_set_line(g_maat_instance, &line_rule); + EXPECT_GT(ret, 0); + + /* 4th line */ + memset(&line_rule, 0, sizeof(line_rule)); + line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1); + line_rule.table_name = table_name; + line_rule.table_line = table_line[3]; + line_rule.expire_after = 0; + + ret = maat_cmd_set_line(g_maat_instance, &line_rule); + EXPECT_GT(ret, 0); + + sleep(1); + int ex_data_counter = 0; + ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id, + plugin_ex_new_cb, + plugin_ex_free_cb, + plugin_ex_dup_cb, + 0, &ex_data_counter); + ASSERT_TRUE(ret >= 0); + EXPECT_EQ(ex_data_counter, TEST_CMD_LINE_NUM); + + struct user_info *uinfo = NULL; + uinfo = (struct user_info *)maat_plugin_table_get_ex_data(g_maat_instance, table_id, "192.168.0.2"); + ASSERT_TRUE(uinfo != NULL); + EXPECT_EQ(0, strcmp(uinfo->name, "liuqiangdong")); + EXPECT_EQ(uinfo->id, 2); + plugin_ex_free_cb(table_id, (void**)&uinfo, 0, NULL); + + ret = maat_cmd_set_line(g_maat_instance, &line_rule + 1); + EXPECT_GT(ret, 0); + + sleep(1); + uinfo = (struct user_info *)maat_plugin_table_get_ex_data(g_maat_instance, table_id, "192.168.0.2"); + ASSERT_TRUE(uinfo == NULL); +} +#endif int count_line_num_cb(const char *table_name, const char *line, void *u_para) { (*((unsigned int *)u_para))++; diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 3e2b341..8207d36 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -39,7 +39,7 @@ TEST(json_mode, maat_scan_string) { const char *table_name = "KEYWORDS_TABLE"; int table_id = maat_table_get_id(maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; - int results[5] = {0}; + uint64_t results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), @@ -84,7 +84,7 @@ TEST(iris_mode, maat_scan_string) { const char *table_name = "KEYWORDS_TABLE"; int table_id = maat_table_get_id(maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; - int results[5] = {0}; + uint64_t results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), @@ -206,7 +206,7 @@ TEST(redis_mode, maat_scan_string) { const char *table_name = "KEYWORDS_TABLE"; int table_id = maat_table_get_id(maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; - int results[5] = {0}; + uint64_t results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data),