diff --git a/include/maat/maat.h b/include/maat/maat.h index 623167c..1a83790 100644 --- a/include/maat/maat.h +++ b/include/maat/maat.h @@ -32,33 +32,22 @@ struct maat_hit_path { int item_id; int sub_group_id; int top_group_id; - int virtual_table_id; // 0 is not a virtual table. + int vtable_id; // 0 is not a virtual table. int compile_id; }; struct maat_hit_object { - int virtual_table_id; + int vtable_id; int group_id; }; -enum maat_scan_opt -{ - MAAT_SET_SCAN_DISTRICT = 1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default. - MAAT_SET_SCAN_LAST_ITEM, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination. - MAAT_SET_SCAN_COMPILE_TABLE_ID, //Caller can specify which compile table to use - MAAT_GET_SCAN_HIT_PATH, //VALUE is struct maat_hit_path*, an array of struct maat_hit_path, SIZE= sizeof(struct maat_hit_path)*N, - //Maat_get_scan_status returns actual got number. - MAAT_GET_SCAN_HIT_OBJECTS //VALUE is struct maat_hit_object*, an array of struct maat_hit_object, SIZE= sizeof(struct maat_hit_object)*N, +enum maat_scan_status { + MAAT_SCAN_ERR = -1, //scan error + MAAT_SCAN_OK, //scan but not hit(group or compile) + MAAT_SCAN_HALF_HIT, //half hit: hit group, not hit compile + MAAT_SCAN_HIT //scan hit compile }; -#define MAAT_OK 0 //scan but not hit(group or compile) -#define MAAT_ERR -1 //scan error -#define MAAT_HALF_HIT 1 //half hit: hit group, not hit compile -#define MAAT_HIT 2 //scan hit compile - -#define MAAT_RULE_UPDATE_TYPE_FULL 1 -#define MAAT_RULE_UPDATE_TYPE_INC 2 - typedef void maat_start_callback_t(int update_type, void *u_param); typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para); typedef void maat_finish_callback_t(void *u_para); @@ -125,8 +114,14 @@ struct maat_state; * @param n_hit_result: the number of hit compile id * @param state: scan mid status * - * @retval MAAT_ERR/MAAT_OK/MAAT_HALF_HIT + * @retval MAAT_SCAN_ERR + * MAAT_SCAN_OK + * MAAT_SCAN_HALF_HIT + * MAAT_SCAN_HIT */ +int maat_scan_flag(struct maat *instance, int table_id, int thread_id, + uint64_t flag, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state); int maat_scan_integer(struct maat *instance, int table_id, int thread_id, unsigned int intval, int *results, size_t n_result, size_t *n_hit_result, struct maat_state **state); @@ -152,15 +147,19 @@ int maat_scan_stream(struct maat_stream **stream, int thread_id, const char* dat void maat_scan_stream_close(struct maat_stream **stream); /* maat state API */ -int maat_state_set_scan_district(struct maat *instance, struct maat_state **state, const char *district, size_t district_len); +int maat_state_set_scan_district(struct maat *instance, struct maat_state **state, + const char *district, size_t district_len); int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state); -int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, int compile_table_id); +int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, + int compile_table_id); -int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state, struct maat_hit_path *paths, size_t n_path); +int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state, + struct maat_hit_path *paths, size_t n_path); -int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj); +int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, + struct maat_hit_object *objs, size_t n_obj); void maat_state_free(struct maat_state **state); diff --git a/include/utils.h b/include/utils.h deleted file mode 100644 index 9981a8f..0000000 --- a/include/utils.h +++ /dev/null @@ -1,36 +0,0 @@ -/* -********************************************************************************************** -* File: maat_utils.h -* Description: maat utils entry -* Authors: Liu WenTan -* Date: 2022-10-31 -* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. -*********************************************************************************************** -*/ - -#ifndef _UTILS_H_ -#define _UTILS_H_ - -#ifdef __cpluscplus -extern "C" -{ -#endif - -#include - -#define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) - -#define FREE(ptr) \ - { \ - if (ptr) \ - { \ - free(ptr); \ - ptr = NULL; \ - } \ - } - -#ifdef __cpluscplus -} -#endif - -#endif diff --git a/scanner/adapter_hs.cpp b/scanner/adapter_hs.cpp index fc3c8b9..02229b9 100644 --- a/scanner/adapter_hs.cpp +++ b/scanner/adapter_hs.cpp @@ -16,7 +16,6 @@ #include "adapter_hs.h" #include "uthash/utarray.h" #include "uthash/uthash.h" -#include "utils.h" #include "maat_utils.h" #include "bool_matcher.h" @@ -43,7 +42,7 @@ struct adapter_hs_runtime { /* adapter_hs instance */ struct adapter_hs { - size_t nr_worker_threads; + size_t n_worker_thread; size_t n_expr; size_t n_patterns; struct adapter_hs_runtime *hs_rt; @@ -59,25 +58,27 @@ struct adapter_hs_stream { UT_array *pattern_id_set; }; -static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t nr_worker_threads, int max_pattern_type, +static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, + size_t n_worker_thread, int pattern_type, struct log_handle *logger) { hs_database_t *database = NULL; - hs_rt->scratchs = ALLOC(hs_scratch_t *, nr_worker_threads); + hs_rt->scratchs = ALLOC(hs_scratch_t *, n_worker_thread); - if (max_pattern_type == PATTERN_TYPE_STR) { + if (pattern_type == PATTERN_TYPE_STR) { database = hs_rt->literal_db; } else { database = hs_rt->regex_db; } if (hs_alloc_scratch(database, &hs_rt->scratchs[0]) != HS_SUCCESS) { - log_error(logger, MODULE_ADAPTER_HS, "ERROR: Unable to allocate scratch space. Exiting."); + log_error(logger, MODULE_ADAPTER_HS, + "ERROR: Unable to allocate scratch space. Exiting."); hs_free_database(database); return -1; } - for (size_t i = 1; i < nr_worker_threads; i++) { + for (size_t i = 1; i < n_worker_thread; i++) { hs_error_t err = hs_clone_scratch(hs_rt->scratchs[0], &hs_rt->scratchs[i]); if (err != HS_SUCCESS) { log_error(logger, MODULE_ADAPTER_HS, "Unable to clone scratch prototype"); @@ -119,7 +120,8 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, scan_mode, NULL, &hs_rt->literal_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { - log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message); + log_error(logger, MODULE_ADAPTER_HS, + "%s compile error: %s", __func__, compile_err->message); } hs_free_compile_error(compile_err); @@ -133,7 +135,8 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, scan_mode, NULL, &hs_rt->regex_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { - log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message); + log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", + __func__, compile_err->message); } hs_free_compile_error(compile_err); goto error; @@ -186,12 +189,14 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt FREE(hs_cd); } -struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array, +struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t n_worker_thread, + and_expr_t *expr_array, size_t n_expr_array, struct log_handle *logger) { if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) || - 0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) { - log_error(logger, MODULE_ADAPTER_HS, "%s input parameters illegal!", __func__); + 0 == n_worker_thread || NULL == expr_array || 0 == n_expr_array) { + log_error(logger, MODULE_ADAPTER_HS, + "%s input parameters illegal!", __func__); return NULL; } @@ -202,7 +207,8 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads for (size_t i = 0; i < n_expr_array; i++) { if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) { log_error(logger, MODULE_ADAPTER_HS, - "the number of patterns in one expression should less than %d", MAX_EXPR_PATTERN_NUM); + "the number of patterns in one expression should less than %d", + MAX_EXPR_PATTERN_NUM); return NULL; } @@ -212,7 +218,8 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads } else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) { regex_pattern_num++; } else { - log_error(logger, MODULE_ADAPTER_HS, "unknown pattern type: %d", expr_array[i].patterns[j].type); + log_error(logger, MODULE_ADAPTER_HS, "unknown pattern type: %d", + expr_array[i].patterns[j].type); return NULL; } } @@ -283,7 +290,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads size_t mem_size = 0; struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1); - hs_instance->nr_worker_threads = nr_worker_threads; + hs_instance->n_worker_thread = n_worker_thread; hs_instance->n_patterns = pattern_id; hs_instance->n_expr = n_expr_array; hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1); @@ -321,7 +328,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads max_patterns_type = PATTERN_TYPE_REG; } - ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, nr_worker_threads, max_patterns_type, logger); + ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, max_patterns_type, logger); if (ret < 0) { goto error; } @@ -349,7 +356,7 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance) } if (hs_instance->hs_rt->scratchs != NULL) { - for (size_t i = 0; i < hs_instance->nr_worker_threads; i++) { + for (size_t i = 0; i < hs_instance->n_worker_thread; i++) { if (hs_instance->hs_rt->scratchs[i] != NULL) { hs_free_scratch(hs_instance->hs_rt->scratchs[i]); } @@ -399,10 +406,13 @@ int matched_event_cb(unsigned int id, unsigned long long from, return 0; } -int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *data, size_t data_len, - int results[], size_t *n_results) +int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, + const char *data, size_t data_len, + struct hs_scan_result *results, + size_t *n_results) { - if (NULL == hs_instance || NULL == data || (0 == data_len) || NULL == results || NULL == n_results) { + if (NULL == hs_instance || NULL == data || (0 == data_len) || + NULL == results || NULL == n_results) { return -1; } @@ -416,7 +426,8 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d int err_count = 0; if (hs_rt->literal_db != NULL) { - err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set); + err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, + matched_event_cb, pattern_id_set); if (err != HS_SUCCESS) { //log_error() err_count++; @@ -424,7 +435,8 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d } if (hs_rt->regex_db != NULL) { - err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set); + err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, + matched_event_cb, pattern_id_set); if (err != HS_SUCCESS) { //log_error() err_count++; @@ -442,18 +454,28 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d items[i] = *(unsigned long long *)utarray_eltptr(pattern_id_set, i); } - size_t matched_index = 0; - struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr); - size_t bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_instance->n_expr); + int ret = 0; + int matched_index = 0; + + struct bool_expr_match *bool_matcher_results = NULL; + bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr); + int bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, pattern_set_size, + bool_matcher_results, hs_instance->n_expr); + if (bool_matcher_ret < 0) { + ret = -1; + goto next; + } + for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { - results[matched_index] = bool_matcher_results[matched_index].expr_id; + results[matched_index].item_id = bool_matcher_results[matched_index].expr_id; + results[matched_index].user_tag = bool_matcher_results[matched_index].user_tag; } *n_results = bool_matcher_ret; - +next: FREE(bool_matcher_results); utarray_free(pattern_id_set); - return 0; + return ret; } struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id) @@ -487,14 +509,16 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, return hs_stream; } -int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len, +int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, + const char *data, size_t data_len, int results[], size_t *n_results) { hs_error_t err; int thread_id = hs_stream->thread_id; if (hs_stream->literal_stream != NULL) { - err = hs_scan_stream(hs_stream->literal_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id], + err = hs_scan_stream(hs_stream->literal_stream, data, data_len, + 0, hs_stream->hs_rt->scratchs[thread_id], matched_event_cb, hs_stream->pattern_id_set); if (err != HS_SUCCESS) { //log_error() @@ -503,7 +527,8 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data } if (hs_stream->regex_stream != NULL) { - err = hs_scan_stream(hs_stream->regex_stream, data, data_len, 0, hs_stream->hs_rt->scratchs[thread_id], + err = hs_scan_stream(hs_stream->regex_stream, data, data_len, + 0, hs_stream->hs_rt->scratchs[thread_id], matched_event_cb, hs_stream->pattern_id_set); if (err != HS_SUCCESS) { //log_error() @@ -519,8 +544,10 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data } size_t matched_index = 0; - struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); - size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_stream->n_expr); + struct bool_expr_match *bool_matcher_results = NULL; + bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); + size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, + bool_matcher_results, hs_stream->n_expr); for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { results[matched_index] = bool_matcher_results[matched_index].expr_id; } @@ -533,10 +560,19 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream) { + if (NULL == hs_stream) { + return; + } + int thread_id = hs_stream->thread_id; - hs_close_stream(hs_stream->literal_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL); - hs_close_stream(hs_stream->regex_stream, hs_stream->hs_rt->scratchs[thread_id], NULL, NULL); + if (hs_stream->hs_rt != NULL) { + hs_close_stream(hs_stream->literal_stream, + hs_stream->hs_rt->scratchs[thread_id], NULL, NULL); + hs_close_stream(hs_stream->regex_stream, + hs_stream->hs_rt->scratchs[thread_id], NULL, NULL); + } + utarray_free(hs_stream->pattern_id_set); /* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */ diff --git a/scanner/adapter_hs.h b/scanner/adapter_hs.h index a484263..fd0756d 100644 --- a/scanner/adapter_hs.h +++ b/scanner/adapter_hs.h @@ -38,6 +38,11 @@ enum pattern_type { PATTERN_TYPE_REG, }; +struct hs_scan_result { + int item_id; + void *user_tag; +}; + typedef struct { /* pattern type */ int type; @@ -53,6 +58,7 @@ typedef struct { uint32_t expr_id; size_t n_patterns; scan_pattern_t patterns[MAX_EXPR_PATTERN_NUM]; + void *user_tag; } and_expr_t; /** @@ -65,7 +71,8 @@ typedef struct { * * @retval the pointer to adapter_hs instance */ -struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array, struct log_handle *logger); +struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, + size_t n_expr_array, struct log_handle *logger); /** * @brief scan input data to match logic AND expression, return all matched expr_id @@ -78,7 +85,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads * @param n_results: number of elements in array of expr_id */ int adapter_hs_scan(struct adapter_hs *instance, int thread_id, const char *data, size_t data_len, - int results[], size_t *n_results); + struct hs_scan_result *results, size_t *n_results); /** * @brief destroy adapter_hs instance diff --git a/scanner/adapter_hs_gtest.cpp b/scanner/adapter_hs_gtest.cpp index 96e5c99..6b4cd26 100644 --- a/scanner/adapter_hs_gtest.cpp +++ b/scanner/adapter_hs_gtest.cpp @@ -1,6 +1,5 @@ #include -#include "utils.h" #include "adapter_hs.h" int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr) @@ -33,7 +32,7 @@ int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr) char *sub_pattern_token = strtok_r(pattern_token, ":", &save_pattern_ptr); expr[i].patterns[j].type = atoi(sub_pattern_token); size_t str_len = strlen(save_pattern_ptr); - expr[i].patterns[j].pat = ALLOC(char, str_len); + expr[i].patterns[j].pat = (char *)calloc(sizeof(char), str_len); memcpy(expr[i].patterns[j].pat, save_pattern_ptr, str_len); expr[i].patterns[j].pat_len = str_len; j++; @@ -50,7 +49,8 @@ void expr_array_free(and_expr_t expr_array[], size_t n_expr_array) { for (size_t i = 0; i < n_expr_array; i++) { for (size_t j = 0; j < expr_array[i].n_patterns; j++) { - FREE(expr_array[i].patterns[j].pat); + free(expr_array[i].patterns[j].pat); + expr_array[i].patterns[j].pat = NULL; } } } diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index f383b83..f5d566b 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -18,8 +18,9 @@ extern "C" #include "hiredis/hiredis.h" -int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx, - char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, +int json2iris(const char *json_buff, const char *json_filename, + redisContext *redis_write_ctx, char *iris_dir_buf, + int buf_len, char *encrypt_key, char *encrypt_algo, struct log_handle *logger); #ifdef __cpluscplus diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index 2d19d2d..c1ffb85 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -33,14 +33,17 @@ struct compile_ex_data_schema { }; /* compile schema API */ -void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); void compile_schema_free(void *compile_schema); -void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); void group2compile_schema_free(void *g2c_schema); int group2compile_associated_compile_table_id(void *g2c_schema); -int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id, +int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, + int table_id, maat_rule_ex_new_func_t *new_func, maat_rule_ex_free_func_t *free_func, maat_rule_ex_dup_func_t *dup_func, @@ -52,37 +55,43 @@ compile_table_get_rule_ex_data_schema(struct compile_schema *compile_schema, siz size_t compile_table_rule_ex_data_schema_count(struct compile_schema *compile_schema); /* compile runtime API */ -void *compile_runtime_new(void *compile_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *compile_runtime_new(void *compile_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void compile_runtime_free(void *compile_runtime); -int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, - int valid_column); +int compile_runtime_update(void *compile_runtime, void *compile_schema, + const char *line, int valid_column); int compile_runtime_commit(void *compile_runtime); -int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids, - int vt_id, int *compile_ids, size_t compile_ids_size, struct maat_state *state); +int compile_runtime_match(struct compile_runtime *compile_rt, int *compile_ids, + size_t compile_ids_size, struct maat_state *state); -size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct group2group_runtime *g2g_rt, +size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, + struct group2group_runtime *g2g_rt, struct maat_compile_state *compile_state, - struct maat_hit_path *hit_paths, size_t hit_path_siz); + struct maat_hit_path *hit_paths, + size_t n_hit_path); /* group2compile runtime API */ -void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime); void group2compile_runtime_free(void *g2c_runtime); -int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line, - int valid_column); +int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, + const char *line, int valid_column); /* maat compile state API */ struct maat_compile_state; struct maat_compile_state *maat_compile_state_new(int thread_id); void maat_compile_state_free(struct maat_compile_state *compile_state); -void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id, - int Nth_scan, int Nth_item_result); -void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, void *compile_runtime, - int group_id, int virtual_table_id); + +int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, + int *hit_item_ids, size_t hit_item_cnt, + int *group_ids, size_t group_ids_size, + size_t *n_hit_group_id, struct maat_state *state); + int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); #ifdef __cpluscplus diff --git a/src/inc_internal/maat_config_monitor.h b/src/inc_internal/maat_config_monitor.h index 5d17dbe..b42ea5e 100644 --- a/src/inc_internal/maat_config_monitor.h +++ b/src/inc_internal/maat_config_monitor.h @@ -18,17 +18,14 @@ extern "C" #include -#define CM_UPDATE_TYPE_NONE 0 -#define CM_UPDATE_TYPE_FULL 1 -#define CM_UPDATE_TYPE_INC 2 - void config_monitor_traverse(long long version, const char *idx_dir, void (*start_fn)(long long, int, void *), int (*update_fn)(const char *, const char *, void *), void (*finish_fn)(void *), void *u_param, struct log_handle *logger); -int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz); +int load_maat_json_file(struct maat *maat_instance, const char *json_filename, + char *err_str, size_t err_str_sz); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_ex_data.h b/src/inc_internal/maat_ex_data.h index f751490..82e8ddd 100644 --- a/src/inc_internal/maat_ex_data.h +++ b/src/inc_internal/maat_ex_data.h @@ -33,11 +33,10 @@ struct ex_container_ctx { struct ex_data_runtime; /* ex_data_runtime API */ -struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, - struct log_handle *logger); - +struct ex_data_runtime * +ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, + struct log_handle *logger); void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt); - void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt); /* ex_data_runtime cache row API */ @@ -56,19 +55,22 @@ struct ex_data_schema *ex_data_schema_new(maat_plugin_ex_new_func_t *new_func, long argl, void *argp); void ex_data_schema_free(struct ex_data_schema *ex_schema); -void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema); +void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, + struct ex_data_schema *schema); /* set user_ctx API */ -void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx); +void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, + struct ex_container_ctx *container_ctx); -struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt); +struct ex_container_ctx * +ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt); struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data); void ex_data_container_free(void *ctx, void *data); /* ex_data_runtime ex data API */ -void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, - const char *row, const char *key, size_t key_len); +void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, + const char *key, size_t key_len); int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, @@ -80,14 +82,14 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container); -void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); +void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len); -void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); +void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len); size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt); -int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt); - #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index 4670518..b2d0950 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -21,16 +21,18 @@ extern "C" struct expr_runtime; -void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); void expr_schema_free(void *expr_schema); /* expr runtime API */ -void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *expr_runtime_new(void *expr_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void expr_runtime_free(void *expr_runtime); -int expr_runtime_updating_flag(void *expr_runtime); -int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column); +int expr_runtime_update(void *expr_runtime, void *expr_schema, + const char *line, int valid_column); int expr_runtime_commit(void *expr_runtime); /* expr runtime scan API */ @@ -39,15 +41,19 @@ int expr_runtime_commit(void *expr_runtime); * * @retval the num of hit group_id */ -int expr_runtime_scan_string(struct expr_runtime *expr_runtime, int thread_id, - const char *data, size_t data_len, int group_ids[], - size_t max_hit_num, int virtual_table_id, - struct maat_state *state); +int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, + const char *data, size_t data_len, + int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state); void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); -int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, int results[], size_t *n_result); +int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, + size_t data_len, int result[], size_t *n_result); void expr_runtime_stream_close(struct expr_runtime *expr_rt); +void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id); +long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread); + #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_garbage_collection.h b/src/inc_internal/maat_garbage_collection.h index 21aed45..3ce3224 100644 --- a/src/inc_internal/maat_garbage_collection.h +++ b/src/inc_internal/maat_garbage_collection.h @@ -24,7 +24,8 @@ struct maat_garbage_bin *maat_garbage_bin_new(int default_timeout); void maat_garbage_bin_free(struct maat_garbage_bin *bin); -void maat_garbage_bagging(struct maat_garbage_bin *bin, void *garbage, void (* func)(void *)); +void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, + void (* func)(void *)); void maat_garbage_collect_routine(struct maat_garbage_bin *bin); diff --git a/src/inc_internal/maat_group.h b/src/inc_internal/maat_group.h index 62adc1b..45202b0 100644 --- a/src/inc_internal/maat_group.h +++ b/src/inc_internal/maat_group.h @@ -21,63 +21,34 @@ extern "C" #include "igraph/igraph.h" #include "maat_kv.h" -struct maat_group { - igraph_integer_t vertex_id; - int group_id; - int ref_by_compile_cnt; - int ref_by_superior_group_cnt; - int ref_by_subordinate_group_cnt; - - size_t top_group_cnt; - int *top_group_ids; - - UT_hash_handle hh_group_id; - UT_hash_handle hh_vertex_id; -}; - -/* maat group topology API */ -struct maat_group_topology; - -/** - * @retval if not found, return NULL -*/ - - -//struct maat_group *maat_group_topology_add_group(struct maat_group_topology *group_topo, int group_id); -//struct maat_group *maat_group_topology_find_group(struct maat_group_topology *group_topo, int group_id); -//void maat_group_topology_remove_group(struct maat_group_topology *group_topo, struct maat_group *group); -//int maat_group_topology_add_group_to_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id); -//int maat_group_topology_remove_group_from_group(struct maat_group_topology *group_topo, int group_id, int superior_group_id); - -/* build top groups */ -//int maat_group_topology_build_top_groups(struct maat_group_topology *group_topo); - +struct maat_group; /* group2group schema API */ -void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); void group2group_schema_free(void *g2g_schema); /* group2group runtime API */ -void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *group2group_runtime_new(void *g2g_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void group2group_runtime_free(void *g2g_runtime); +void maat_group_ref_inc(struct maat_group *group); +void maat_group_ref_dec(struct maat_group *group); + struct maat_group *group2group_runtime_add_group(void *g2g_runtime, int group_id); void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *group); struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_id); -int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id); -int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id); - int group2group_runtime_build_top_groups(void *g2g_runtime); +int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, + size_t n_group_ids, int *top_group_ids); - -int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column); +int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, + const char *line, int valid_column); int group2group_runtime_commit(void *g2g_runtime); -int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids, - int *top_group_ids); - #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h index ecaa8c0..4130fc7 100644 --- a/src/inc_internal/maat_ip.h +++ b/src/inc_internal/maat_ip.h @@ -16,30 +16,34 @@ extern "C" { #endif -struct ip_plus_runtime; +struct ip_runtime; -void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); -void ip_plus_schema_free(void *ip_plus_schema); +void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); +void ip_schema_free(void *ip_schema); -/* ip plus runtime API */ -void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, - struct log_handle *logger); -void ip_plus_runtime_free(void *ip_plus_runtime); +/* ip runtime API */ +void *ip_runtime_new(void *ip_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void ip_runtime_free(void *ip_runtime); -int ip_plus_runtime_updating_flag(void *ip_plus_runtime); -int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line, - int valid_column); -int ip_plus_runtime_commit(void *ip_plus_runtime); +int ip_runtime_update(void *ip_runtime, void *ip_schema, + const char *line, int valid_column); +int ip_runtime_commit(void *ip_runtime); -struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt); +struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt); /* ip runtime scan API */ -int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr, - int *group_id_array, size_t n_group_id_array, int virtual_table_id, - struct maat_state *state); +int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, int ip_type, + uint8_t *ip_addr, int *group_ids, size_t group_id_size, + int vtable_id, struct maat_state *state); + +void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id); +long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread); #ifdef __cpluscplus } #endif -#endif \ No newline at end of file +#endif diff --git a/src/inc_internal/maat_ip_plugin.h b/src/inc_internal/maat_ip_plugin.h index aa4e550..94ec278 100644 --- a/src/inc_internal/maat_ip_plugin.h +++ b/src/inc_internal/maat_ip_plugin.h @@ -23,7 +23,8 @@ extern "C" struct ip_plugin_runtime; /* ip plugin schema API */ -void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); void ip_plugin_schema_free(void *ip_plugin_schema); /* ip plugin table ex data API */ @@ -37,13 +38,13 @@ int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, struct log_handle *logger); /* ip plugin runtime API */ -void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void ip_plugin_runtime_free(void *ip_plugin_runtime); -int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime); -int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line, - int valid_column); +int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, + const char *line, int valid_column); int ip_plugin_runtime_commit(void *ip_plugin_runtime); struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime); diff --git a/src/inc_internal/maat_plugin.h b/src/inc_internal/maat_plugin.h index c73250c..a990184 100644 --- a/src/inc_internal/maat_plugin.h +++ b/src/inc_internal/maat_plugin.h @@ -21,7 +21,8 @@ extern "C" #define MAX_FOREIGN_CLMN_NUM 8 /* plugin schema API */ -void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); void plugin_schema_free(void *plugin_schema); struct plugin_schema; @@ -38,23 +39,22 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema); int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns); /* plugin table ex data API */ -int plugin_table_set_ex_data_schema(void *custom_schema, - maat_plugin_ex_new_func_t *new_func, - maat_plugin_ex_free_func_t *free_func, - maat_plugin_ex_dup_func_t *dup_func, - long argl, void *argp, - struct log_handle *logger); +int plugin_table_set_ex_data_schema(void *plugin_schema, + maat_plugin_ex_new_func_t *new_func, + maat_plugin_ex_free_func_t *free_func, + maat_plugin_ex_dup_func_t *dup_func, + long argl, void *argp, + struct log_handle *logger); struct ex_data_schema *plugin_table_get_ex_data_schema(void *custom_schema); /* plugin runtime API */ -void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *plugin_runtime_new(void *plugin_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void plugin_runtime_free(void *plugin_runtime); -int plugin_runtime_updating_flag(void *plugin_runtime); - -int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line, - int valid_column); +int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, + const char *line, int valid_column); int plugin_runtime_commit(void *plugin_runtime); struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime); diff --git a/src/inc_internal/maat_redis_monitor.h b/src/inc_internal/maat_redis_monitor.h index 971c86f..b4e7b01 100644 --- a/src/inc_internal/maat_redis_monitor.h +++ b/src/inc_internal/maat_redis_monitor.h @@ -20,7 +20,7 @@ extern "C" #include -void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, +void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, void (*start_fn)(long long, int, void *), int (*update_fn)(const char *, const char *, void *), void (*finish_fn)(void *), void *u_param); diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index b42bfd7..34593a6 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -34,6 +34,22 @@ extern "C" #define MAX_TABLE_NUM 256 +#define MAAT_UPDATE_TYPE_NONE 0 +#define MAAT_UPDATE_TYPE_FULL 1 +#define MAAT_UPDATE_TYPE_INC 2 + +enum last_scan_flag { + LAST_SCAN_UNSET, + LAST_SCAN_SET, + LAST_SCAN_FINISHED +}; + +enum tag_match { + TAG_MATCH_ERR = -1, + TAG_MATCH_UNMATCHED, + TAG_MATCH_MATCHED +}; + struct maat_rule_head { int config_id; int service_id; @@ -89,7 +105,7 @@ struct compile_rule { struct group2group_rule { int group_id; - int superior_group_id; + int super_group_id; }; struct maat_runtime { @@ -223,9 +239,11 @@ struct maat { /* statistics */ long long line_cmd_acc_num; - long long *outer_mid_cnt; - long long *compile_mid_cnt; + long long *outer_state_cnt; + long long *compile_state_cnt; long long *thread_call_cnt; + long long *hit_cnt; + long long *not_grp_hit_cnt; long long scan_err_cnt; }; @@ -238,7 +256,7 @@ struct maat_state { unsigned char is_last_scan; int district_id; //-1: Any District; -2: Unkonwn District; int scan_cnt; - struct maat_compile_state *compile_mid; + struct maat_compile_state *compile_state; }; size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger); @@ -264,7 +282,8 @@ void *rule_monitor_loop(void *arg); void maat_read_full_config(struct maat *maat_instance); /* maat command API for internal */ -redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger); +redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, + int redis_db, struct log_handle *logger); redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...); @@ -274,35 +293,39 @@ long long maat_cmd_redis_server_time_s(redisContext *c); long long maat_cmd_read_redis_integer(const redisReply *reply); -int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq); +int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, + int valid_column_seq); const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len); -int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, - long long server_time, struct log_handle *logger); +int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, + size_t serial_rule_num, long long server_time, + struct log_handle *logger); void maat_cmd_clear_rule_cache(struct serial_rule *s_rule); -int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, - long long *new_version, struct table_manager* tbl_mgr, - struct serial_rule **list, int *update_type, int cumulative_off, - struct log_handle *logger); +int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, + long long desired_version, long long *new_version, + struct table_manager *tbl_mgr, struct serial_rule **list, + int *update_type, int cumulative_off, struct log_handle *logger); -int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process, - struct log_handle *logger); +int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, + int rule_num, int print_process, struct log_handle *logger); -int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num, - const char *dir, struct log_handle *logger); +int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, + int rule_num, const char* dir, struct log_handle *logger); -void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn, - struct log_handle *logger); +void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, + int rule_num, int print_fn, struct log_handle *logger); void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule); -void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id, - const char *table_name, const char *line, long long timeout); +void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, + unsigned long rule_id, const char *table_name, + const char *line, long long timeout); -void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, const char *srv_def, int srv_def_len); +void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, + const char *srv_def, int srv_def_len); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index e57e61d..4b6916c 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -43,8 +43,9 @@ enum table_type { struct table_manager; -struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags, - struct log_handle *logger); +struct table_manager * +table_manager_create(const char *table_info_path, const char *accept_tags, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_num, struct maat_garbage_bin *garbage_bin); void table_manager_runtime_destroy(struct table_manager *tbl_mgr); @@ -53,14 +54,21 @@ void table_manager_destroy(struct table_manager *tbl_mgr); size_t table_manager_table_count(struct table_manager *tbl_mgr); int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name); enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id); + int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr); +int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr); + int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id); + +size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr); int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags); -void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); +int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str, + size_t district_str_len, int *district_id); +void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); -int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id); + int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line); void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id); diff --git a/src/inc_internal/maat_utils.h b/src/inc_internal/maat_utils.h index 8d2c58d..53d167d 100644 --- a/src/inc_internal/maat_utils.h +++ b/src/inc_internal/maat_utils.h @@ -31,6 +31,27 @@ extern "C" #define MIN(a, b) (((a) < (b)) ? (a) : (b)) #endif +#define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) + +#define FREE(ptr) \ + { \ + if (ptr) \ + { \ + free(ptr); \ + ptr = NULL; \ + } \ + } + +#ifndef offsetof +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif + +#ifndef container_of +#define container_of(ptr, type, member) ({ \ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) );}) +#endif + #define MAX_SCANNER_HIT_COMPILE_NUM 4096 #define MAX_SCANNER_HIT_GROUP_NUM 4096 #define MAX_SCANNER_HIT_ITEM_NUM 4096 diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h index d731e78..8b4214b 100644 --- a/src/inc_internal/maat_virtual.h +++ b/src/inc_internal/maat_virtual.h @@ -29,7 +29,8 @@ enum scan_type { SCAN_TYPE_MAX }; -void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger); +void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); void virtual_schema_free(void *virtual_schema); diff --git a/src/inc_internal/rcu_hash.h b/src/inc_internal/rcu_hash.h index 1ef32ed..44537d1 100644 --- a/src/inc_internal/rcu_hash.h +++ b/src/inc_internal/rcu_hash.h @@ -62,13 +62,6 @@ size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable); size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array); -/** - * @brief check if rcu hash table is updating - * - * @retval 1 means htable is updating, otherwise 0 -*/ -int rcu_hash_updating_flag(struct rcu_hash_table *htable); - #ifdef __cpluscplus } #endif diff --git a/src/json2iris.cpp b/src/json2iris.cpp index 2b1c3e0..5d87927 100644 --- a/src/json2iris.cpp +++ b/src/json2iris.cpp @@ -15,7 +15,6 @@ #include #include -#include "utils.h" #include "json2iris.h" #include "cJSON/cJSON.h" #include "maat_kv.h" @@ -23,7 +22,7 @@ #include "maat_rule.h" #include "uthash/uthash.h" -#define MODULE_JSON2IRIS module_name_str("maat.json2iris") +#define MODULE_JSON2IRIS module_name_str("maat.json2iris") #define MAX_COLUMN_NUM 32 @@ -89,7 +88,9 @@ struct translate_command { int default_int; }; -struct iris_table *query_table_info(struct iris_description *p_iris, const char *table_name, enum table_type table_type) +struct iris_table *query_table_info(struct iris_description *p_iris, + const char *table_name, + enum table_type table_type) { struct iris_table *table_info = NULL; HASH_FIND(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info); @@ -97,8 +98,10 @@ struct iris_table *query_table_info(struct iris_description *p_iris, const char table_info = ALLOC(struct iris_table, 1); table_info->line_count = 0; table_info->table_type = table_type; - memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name) - 1, strlen(table_name))); - snprintf(table_info->table_path, sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name); + memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name) - 1, + strlen(table_name))); + snprintf(table_info->table_path, sizeof(table_info->table_path), + "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name); HASH_ADD_KEYPTR(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info); } @@ -114,14 +117,18 @@ void free_iris_table_info(void *p) FREE(table); } -int set_iris_descriptor(const char *json_file, cJSON *json, const char *encrypt_key, const char *encrypt_algo, - const char *compile_tn, const char *group2compile_tn, const char* group2group_tn, - redisContext *redis_write_ctx, struct iris_description *iris_cfg) +int set_iris_descriptor(const char *json_file, cJSON *json, + const char *encrypt_key, const char *encrypt_algo, + const char *compile_tn, const char *group2compile_tn, + const char* group2group_tn, redisContext *redis_write_ctx, + struct iris_description *iris_cfg) { memset(iris_cfg, 0, sizeof(struct iris_description)); snprintf(iris_cfg->tmp_iris_dir, sizeof(iris_cfg->tmp_iris_dir), "%s_iris_tmp", json_file); - snprintf(iris_cfg->tmp_iris_index_dir, sizeof(iris_cfg->tmp_iris_index_dir), "%s_iris_tmp/index", json_file); - snprintf(iris_cfg->index_path, sizeof(iris_cfg->index_path), "%s/full_config_index.%010d", iris_cfg->tmp_iris_index_dir, json_version); + snprintf(iris_cfg->tmp_iris_index_dir, sizeof(iris_cfg->tmp_iris_index_dir), + "%s_iris_tmp/index", json_file); + snprintf(iris_cfg->index_path, sizeof(iris_cfg->index_path), "%s/full_config_index.%010d", + iris_cfg->tmp_iris_index_dir, json_version); iris_cfg->redis_write_ctx = redis_write_ctx; iris_cfg->str2int_map = maat_kv_store_new(); @@ -219,13 +226,15 @@ int create_tmp_dir(struct iris_description *p) return 0; } -int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_description *p_iris, +int write_plugin_line(cJSON *plug_table_json, int sequence, + struct iris_description *p_iris, struct log_handle *logger) { cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name"); if (NULL == item || item->type != cJSON_String) { log_error(logger, MODULE_JSON2IRIS, - "The %d plugin_table's table_name not defined or format error", sequence); + "The %d plugin_table's table_name not defined or format error", + sequence); return -1; } const char *table_name = item->valuestring; @@ -233,7 +242,8 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content"); if (NULL == table_content || table_content->type != cJSON_Array) { log_error(logger, MODULE_JSON2IRIS, - "%d plugin_table's table_content not defined or format error", sequence); + "%d plugin_table's table_content not defined or format error", + sequence); return -1; } int line_cnt = cJSON_GetArraySize(table_content); @@ -244,21 +254,25 @@ int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_descript for (int i = 0; i < line_cnt; i++) { each_line = cJSON_GetArrayItem(table_content, i); if (NULL == each_line || each_line->type != cJSON_String) { - log_error(logger, MODULE_JSON2IRIS, "plugin_table %s's line %d format error", - table_info->table_name, i + 1); + log_error(logger, MODULE_JSON2IRIS, + "plugin_table %s's line %d format error", + table_info->table_name, i + 1); continue; } line_content = each_line->valuestring; - table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), line_content, strlen(line_content)); - table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), "\n", 1); + table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, + &(table_info->buff_sz), line_content, strlen(line_content)); + table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, + &(table_info->buff_sz), "\n", 1); table_info->line_count++; } return 0; } -static struct group_info *group_info_read(struct group_info *group_name_map, const char *group_name) +static struct group_info *group_info_read(struct group_info *group_name_map, + const char *group_name) { struct group_info *node = NULL; HASH_FIND(hh, group_name_map, group_name, strlen(group_name), node); @@ -274,7 +288,8 @@ static int get_group_seq(struct iris_description *iris_cfg) if (NULL == iris_cfg->redis_write_ctx) { sequence = iris_cfg->group_cnt; } else { - data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, "INCRBY %s 1", mr_group_id_var); + data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, + "INCRBY %s 1", mr_group_id_var); sequence = (int)data_reply->integer - 1; freeReplyObject(data_reply); data_reply = NULL; @@ -284,7 +299,8 @@ static int get_group_seq(struct iris_description *iris_cfg) return sequence; } -static struct group_info *group_info_add_unsafe(struct iris_description *p_iris, const char *group_name) +static struct group_info *group_info_add_unsafe(struct iris_description *p_iris, + const char *group_name) { static struct group_info untitled_group; struct group_info *group_info = NULL; @@ -308,7 +324,8 @@ static int get_region_seq(struct iris_description *iris_cfg) if (NULL == iris_cfg->redis_write_ctx) { sequence = iris_cfg->region_cnt; } else { - redisReply *data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, "INCRBY %s 1", mr_region_id_var); + redisReply *data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, + "INCRBY %s 1", mr_region_id_var); sequence = (int)data_reply->integer - 1; freeReplyObject(data_reply); } @@ -317,7 +334,8 @@ static int get_region_seq(struct iris_description *iris_cfg) return sequence; } -int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct translate_command *cmd, int cmd_cnt, +int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, + struct translate_command *cmd, int cmd_cnt, struct iris_table *table, struct log_handle *logger) { int i = 0; @@ -335,7 +353,8 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla } if (NULL == item || item->type != cmd[i].json_type) { - log_error(logger, MODULE_JSON2IRIS, "%s not defined or wrong format", cmd[i].json_string); + log_error(logger, MODULE_JSON2IRIS, + "%s not defined or wrong format", cmd[i].json_string); ret = -1; goto error_out; } @@ -345,7 +364,8 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla char *p = item->valuestring; ret = maat_kv_read(str2int, p, &int_value); if (ret < 0) { - log_error(logger, MODULE_JSON2IRIS, "%s's value %s is not valid format", cmd[i].json_string, p); + log_error(logger, MODULE_JSON2IRIS, + "%s's value %s is not valid format", cmd[i].json_string, p); FREE(p); ret = -1; goto error_out; @@ -369,7 +389,8 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct transla } for (i = 0; i < cmd_cnt; i++) { - table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, cmd[i].json_value, strlen(cmd[i].json_value)); + table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, + cmd[i].json_value, strlen(cmd[i].json_value)); table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1); } table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1); @@ -384,8 +405,8 @@ error_out: return ret; } -int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, - struct log_handle *logger) +int write_expr_line(cJSON *region_json, struct iris_description *p_iris, + struct iris_table *table, struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -428,11 +449,12 @@ int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); + return direct_write_rule(region_json, p_iris->str2int_map, + json_cmd, cmd_cnt, table, logger); } -int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, - struct log_handle *logger) +int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, + struct iris_table *table, struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -540,11 +562,12 @@ int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, stru json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); + return direct_write_rule(region_json, p_iris->str2int_map, + json_cmd, cmd_cnt, table, logger); } -int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, - struct log_handle *logger) +int write_intval_line(cJSON *region_json, struct iris_description *p_iris, + struct iris_table *table, struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -576,11 +599,12 @@ int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struc json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); + return direct_write_rule(region_json, p_iris->str2int_map, + json_cmd, cmd_cnt, table, logger); } -int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, - struct log_handle *logger) +int write_digest_line(cJSON *region_json, struct iris_description *p_iris, + struct iris_table *table, struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -610,11 +634,12 @@ int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struc json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); + return direct_write_rule(region_json, p_iris->str2int_map, + json_cmd, cmd_cnt, table, logger); } -int write_similar_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table, - struct log_handle *logger) +int write_similar_line(cJSON *region_json, struct iris_description *p_iris, + struct iris_table *table, struct log_handle *logger) { struct translate_command json_cmd[MAX_COLUMN_NUM]; int cmd_cnt = 0; @@ -640,16 +665,18 @@ int write_similar_line(cJSON *region_json, struct iris_description *p_iris, stru json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; - return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger); + return direct_write_rule(region_json, p_iris->str2int_map, + json_cmd, cmd_cnt, table, logger); } -int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_description *p_iris, - struct log_handle *logger) +int write_region_rule(cJSON *region_json, int compile_id, int group_id, + struct iris_description *p_iris, struct log_handle *logger) { cJSON *item = cJSON_GetObjectItem(region_json, "table_name"); if (NULL == item || item->type != cJSON_String) { - log_error(logger, MODULE_JSON2IRIS, "compile rule %d's table_name not defined or format error", - compile_id); + log_error(logger, MODULE_JSON2IRIS, + "compile rule %d's table_name not defined or format error", + compile_id); return -1; } const char *table_name = item->valuestring; @@ -657,7 +684,8 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i item = cJSON_GetObjectItem(region_json, "table_type"); if (NULL == item || item->type != cJSON_String) { log_error(logger, MODULE_JSON2IRIS, - "compile rule %d's table name %s's table_type not defined or format error", compile_id, table_name); + "compile rule %d's table name %s's table_type not defined or format error", + compile_id, table_name); return -1; } @@ -665,15 +693,17 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i enum table_type table_type = TABLE_TYPE_EXPR; int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type)); if (ret != 1) { - log_error(logger, MODULE_JSON2IRIS, "compile rule %d table name %s's table_type %s invalid", - compile_id, table_name, table_type_str); + log_error(logger, MODULE_JSON2IRIS, + "compile rule %d table name %s's table_type %s invalid", + compile_id, table_name, table_type_str); return -1; } cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content"); if (NULL == table_content || table_content->type != cJSON_Object) { log_error(logger, MODULE_JSON2IRIS, - "compile rule %d table name %s's table_content not defined or format error", compile_id, table_name); + "compile rule %d table name %s's table_content not defined or format error", + compile_id, table_name); return -1; } @@ -709,33 +739,41 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct i return ret; } -int write_group2compile_line(int group_id, int compile_id, int group_not_flag, int clause_index, - const char *virtual_table, struct iris_description *p_iris) +int write_group2compile_line(int group_id, int compile_id, int group_not_flag, + int clause_index, const char *vtable, + struct iris_description *p_iris) { char buff[4096] = {0}; struct iris_table *table = p_iris->group2compile_table; - snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, group_not_flag, virtual_table, clause_index); - table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); + snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, + group_not_flag, vtable, clause_index); + table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), + buff, strlen(buff)); table->line_count++; return 0; } -int write_group2group_line(int group_id, int superior_group_id, struct iris_description *p_iris) +int write_group2group_line(int group_id, int super_group_id, + struct iris_description *p_iris) { char buff[4096] = {0}; struct iris_table *table = p_iris->group2group_table; - snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id, superior_group_id); - table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); + snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id, + super_group_id); + table->write_pos += memcat(&(table->buff), table->write_pos, + &(table->buff_sz), buff, strlen(buff)); table->line_count++; return 0; } -int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, - int Nth_group, struct iris_description *p_iris, struct log_handle *logger) +int write_group_rule(cJSON *group_json, int parent_id, + int parent_type, int tracking_compile_id, + int Nth_group, struct iris_description *p_iris, + struct log_handle *logger) { int ret = 0; int group_not_flag = 0; @@ -784,10 +822,11 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac if (region_json != NULL) { cJSON *region_rule = NULL; cJSON_ArrayForEach(region_rule, region_json) { - ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris, logger); + ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, + p_iris, logger); if (ret < 0) { - log_error(logger, MODULE_JSON2IRIS, "compile rule %d write region error", - tracking_compile_id); + log_error(logger, MODULE_JSON2IRIS, + "compile rule %d write region error", tracking_compile_id); return -1; } } @@ -799,7 +838,8 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac int i = 0; cJSON_ArrayForEach(item, sub_groups) { i++; - ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris, logger); + ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, + tracking_compile_id, i, p_iris, logger); if (ret < 0) { return -1; } @@ -814,20 +854,23 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac } if (parent_type == PARENT_TYPE_COMPILE) { - ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag, clause_index, virtual_table, p_iris); + ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag, + clause_index, virtual_table, p_iris); } else { ret = write_group2group_line(group_info->group_id, parent_id, p_iris); } if (ret < 0) { - log_error(logger, MODULE_JSON2IRIS, "%s rule %d write group error", str_parent_type[parent_type], parent_id); + log_error(logger, MODULE_JSON2IRIS, + "%s rule %d write group error", str_parent_type[parent_type], parent_id); return -1; } return 0; } -int write_compile_line(cJSON *compile, struct iris_description *p_iris, struct log_handle *logger) +int write_compile_line(cJSON *compile, struct iris_description *p_iris, + struct log_handle *logger) { cJSON *item=cJSON_GetObjectItem(compile, "compile_id"); if (item->type != cJSON_Number) { @@ -925,7 +968,8 @@ int write_compile_line(cJSON *compile, struct iris_description *p_iris, struct l table_info = query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE); } - int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, table_info, logger); + int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, + table_info, logger); if (ret < 0) { return -1; } @@ -949,15 +993,18 @@ void write_table_idx(struct iris_description *p_iris, struct iris_table *table) if (p_iris->encrypt_key) { unsigned char *encrypt_buff = NULL; size_t encrypt_buff_sz = 0; - int ret = crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key, + int ret = crypt_memory(buff, table_file_sz, &encrypt_buff, + &encrypt_buff_sz, p_iris->encrypt_key, p_iris->encrypt_algo, 1, err_str, sizeof(err_str)); assert(ret == 0); fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp); - fprintf(p_iris->idx_fp, "%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo); + fprintf(p_iris->idx_fp, "%s\t%d\t%s\t%s\n", table->table_name, table->line_count, + table->table_path, p_iris->encrypt_algo); FREE(encrypt_buff); } else { fwrite(buff, table_file_sz, 1, table_fp); - fprintf(p_iris->idx_fp, "%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); + fprintf(p_iris->idx_fp, "%s\t%d\t%s\n", table->table_name, table->line_count, + table->table_path); } fclose(table_fp); @@ -968,8 +1015,9 @@ int write_index_file(struct iris_description *p_iris, struct log_handle *logger) { p_iris->idx_fp = fopen(p_iris->index_path, "w"); if (NULL == p_iris->idx_fp) { - log_error(logger, MODULE_JSON2IRIS, "index file %s fopen error %s", - p_iris->index_path, strerror(errno)); + log_error(logger, MODULE_JSON2IRIS, + "index file %s fopen error %s", + p_iris->index_path, strerror(errno)); return -1; } @@ -985,7 +1033,8 @@ int write_index_file(struct iris_description *p_iris, struct log_handle *logger) return 0; } -int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle *logger) +int write_iris(cJSON *json, struct iris_description *p_iris, + struct log_handle *logger) { int i=0; int ret=0; @@ -1017,7 +1066,8 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle * parent_group = group_info_add_unsafe(p_iris, parent_group_name); } - ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris, logger); + ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, + 0, 0, p_iris, logger); if (ret < 0) { return -1; } @@ -1041,20 +1091,23 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle * group_array = cJSON_GetObjectItem(compile_obj, "groups"); if (NULL == group_array) { - log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no group", compile_id); + log_error(logger, MODULE_JSON2IRIS, + "compile rule %d have no group", compile_id); return -1; } int group_cnt = cJSON_GetArraySize(group_array); if (group_cnt <= 0) { - log_error(logger, MODULE_JSON2IRIS, "compile rule %d have no groups", compile_id); + log_error(logger, MODULE_JSON2IRIS, + "compile rule %d have no groups", compile_id); return -1; } i = 0; cJSON *group_obj = NULL; cJSON_ArrayForEach(group_obj, group_array) { - ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris, logger); + ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, + compile_id, i, p_iris, logger); if (ret < 0) { return -1; } @@ -1070,8 +1123,9 @@ int write_iris(cJSON *json, struct iris_description *p_iris, struct log_handle * return 0; } -int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx, - char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, +int json2iris(const char *json_buff, const char *json_filename, + redisContext *redis_write_ctx, char *iris_dir_buf, + int buf_len, char *encrypt_key, char *encrypt_algo, struct log_handle *logger) { int ret = -1; @@ -1085,7 +1139,8 @@ int json2iris(const char *json_buff, const char *json_filename, redisContext *re cJSON *json = cJSON_Parse(json_buff); if (!json) { - log_error(logger, MODULE_JSON2IRIS, "Error before: %-200.200s", cJSON_GetErrorPtr()); + log_error(logger, MODULE_JSON2IRIS, + "Error before: %-200.200s", cJSON_GetErrorPtr()); goto error_out; } @@ -1105,15 +1160,16 @@ int json2iris(const char *json_buff, const char *json_filename, redisContext *re } ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, - compile_tbl_name, group2compile_tbl_name, group2group_tbl_name, - redis_write_ctx, &iris_cfg); + compile_tbl_name, group2compile_tbl_name, + group2group_tbl_name, redis_write_ctx, &iris_cfg); if (ret < 0) { goto error_out; } ret = create_tmp_dir(&iris_cfg); if (ret < 0) { - log_error(logger, MODULE_JSON2IRIS, "create tmp folder %s error", iris_cfg.tmp_iris_dir); + log_error(logger, MODULE_JSON2IRIS, + "create tmp folder %s error", iris_cfg.tmp_iris_dir); goto error_out; } @@ -1121,7 +1177,8 @@ int json2iris(const char *json_buff, const char *json_filename, redisContext *re if (ret < 0) { goto error_out; } - memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir) + 1, (unsigned int)buf_len)); + memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, + MIN(strlen(iris_cfg.tmp_iris_index_dir) + 1, (unsigned int)buf_len)); cJSON_Delete(json); clear_iris_descriptor(&iris_cfg); diff --git a/src/maat_api.cpp b/src/maat_api.cpp index 5371bd5..b7e0ac7 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -12,7 +12,6 @@ #include #include -#include "utils.h" #include "maat_utils.h" #include "json2iris.h" #include "maat/maat.h" @@ -38,15 +37,10 @@ #define DISTRICT_ANY -1 #define DISTRICT_UNKNOWN -2 -inline int scan_state_should_compile_NOT(struct maat_state *mid) -{ - if (mid && mid->is_last_scan==1 && mid->compile_mid && - maat_compile_state_has_NOT_clause(mid->compile_mid)) { - return 1; - } else { - return 0; - } -} +enum district_set_flag { + DISTRICT_FLAG_UNSET, + DISTRICT_FLAG_SET +}; struct maat_options* maat_options_new(void) { @@ -111,7 +105,8 @@ int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms) return 0; } -int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len) +int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, + size_t name_len) { memcpy(opts->instance_name, instance_name, name_len); @@ -125,7 +120,8 @@ int maat_options_set_deferred_load_on(struct maat_options *opts) return 0; } -int maat_options_set_iris(struct maat_options *opts, const char *full_directory, const char *increment_directory) +int maat_options_set_iris(struct maat_options *opts, const char *full_directory, + const char *increment_directory) { if (strlen(full_directory) >= NAME_MAX || strlen(increment_directory) >= NAME_MAX) { return -1; @@ -146,7 +142,8 @@ int maat_options_set_json_file(struct maat_options *opts, const char *json_filen return 0; } -int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, uint16_t redis_port, int redis_db) +int maat_options_set_redis(struct maat_options *opts, const char *redis_ip, + uint16_t redis_port, int redis_db) { memcpy(opts->redis_ctx.redis_ip, redis_ip, strlen(redis_ip)); opts->redis_ctx.redis_port = redis_port; @@ -172,10 +169,13 @@ void maat_read_full_config(struct maat *maat_instance) switch (maat_instance->input_mode) { case DATA_SOURCE_REDIS: mr_ctx = &(maat_instance->mr_ctx); - log_info(maat_instance->logger, MODULE_MAAT_API, "Maat initiate from Redis %s:%hu db%d", + log_info(maat_instance->logger, MODULE_MAAT_API, + "Maat initiate from Redis %s:%hu db%d", mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); - mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, - mr_ctx->redis_db, maat_instance->logger); + mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, + mr_ctx->redis_port, + mr_ctx->redis_db, + maat_instance->logger); if (mr_ctx->read_ctx != NULL) { redis_monitor_traverse(maat_instance->maat_version, mr_ctx, maat_start_cb, maat_update_cb, maat_finish_cb, @@ -200,9 +200,11 @@ void maat_read_full_config(struct maat *maat_instance) } break; case DATA_SOURCE_JSON_FILE: - ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); + ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, + err_str, sizeof(err_str)); if (ret < 0) { - log_error(maat_instance->logger, MODULE_MAAT_API, "Maat re-initiate with JSON file %s failed: %s", + log_error(maat_instance->logger, MODULE_MAAT_API, + "Maat re-initiate with JSON file %s failed: %s", maat_instance->json_ctx.json_file, err_str); return; } @@ -212,7 +214,8 @@ void maat_read_full_config(struct maat *maat_instance) maat_start_cb, maat_update_cb, maat_finish_cb, maat_instance, maat_instance->logger); if (NULL == maat_instance->creating_maat_rt) { - log_error(maat_instance->logger, MODULE_MAAT_API, "At initiation: NO effective rule in %s", + log_error(maat_instance->logger, MODULE_MAAT_API, + "At initiation: NO effective rule in %s", maat_instance->json_ctx.iris_file); } break; @@ -243,36 +246,44 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) } else { char log_path[1024] = {0}; if (strlen(maat_instance->instance_name) > 0) { - snprintf(log_path, sizeof(log_path), "%s.log", maat_instance->instance_name); + snprintf(log_path, sizeof(log_path), "%s.log", + maat_instance->instance_name); } else { snprintf(log_path, sizeof(log_path), "maat.log"); } maat_instance->logger = log_handle_create(log_path, opts->log_level); } - maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags, maat_instance->logger); + maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags, + maat_instance->garbage_bin, maat_instance->logger); if (NULL == maat_instance->tbl_mgr) { goto failed; } maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr); - + maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr); maat_instance->input_mode = opts->input_mode; + switch (maat_instance->input_mode) { case DATA_SOURCE_REDIS: - memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip, strlen(opts->redis_ctx.redis_ip)); + memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip, + strlen(opts->redis_ctx.redis_ip)); maat_instance->mr_ctx.redis_port = opts->redis_ctx.redis_port; maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db; break; case DATA_SOURCE_IRIS_FILE: - memcpy(maat_instance->iris_ctx.full_idx_dir, opts->iris_ctx.full_idx_dir, strlen(opts->iris_ctx.full_idx_dir)); - memcpy(maat_instance->iris_ctx.inc_idx_dir, opts->iris_ctx.inc_idx_dir, strlen(opts->iris_ctx.inc_idx_dir)); + memcpy(maat_instance->iris_ctx.full_idx_dir, opts->iris_ctx.full_idx_dir, + strlen(opts->iris_ctx.full_idx_dir)); + memcpy(maat_instance->iris_ctx.inc_idx_dir, opts->iris_ctx.inc_idx_dir, + strlen(opts->iris_ctx.inc_idx_dir)); break; case DATA_SOURCE_JSON_FILE: - memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file)); + memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, + strlen(opts->json_ctx.json_file)); break; default: - log_error(maat_instance->logger, MODULE_MAAT_API, "data source unsupported:%d", maat_instance->input_mode); + log_error(maat_instance->logger, MODULE_MAAT_API, + "data source unsupported:%d", maat_instance->input_mode); goto failed; } @@ -287,9 +298,11 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) (maat_instance->gc_timeout_ms / 1000); maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s); - maat_instance->outer_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); - maat_instance->compile_mid_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); + maat_instance->outer_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); + maat_instance->compile_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); + maat_instance->hit_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); + maat_instance->not_grp_hit_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); pthread_mutex_init(&(maat_instance->background_update_mutex), NULL); @@ -349,7 +362,8 @@ size_t generic_plugin_runtime_cached_row_count(void *custom_rt, enum table_type return 0; } -const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type, size_t row_id) +const char *generic_plugin_runtime_get_cached_row(void *custom_rt, enum table_type table_type, + size_t row_id) { return NULL; } @@ -383,7 +397,7 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id, if (row_cnt > 0) { if (start != NULL) { - start(MAAT_RULE_UPDATE_TYPE_FULL, u_para); + start(MAAT_UPDATE_TYPE_FULL, u_para); } for (size_t i = 0; i < row_cnt; i++) { @@ -457,8 +471,9 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t return 0; } -void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, int table_id, - enum table_type table_type, int valid_column) +void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, + int table_id, enum table_type table_type, + int valid_column) { struct ex_data_schema *ex_data_schema = NULL; struct ex_data_runtime *ex_data_rt = NULL; @@ -518,26 +533,32 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i { pthread_mutex_lock(&(maat_instance->background_update_mutex)); int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id, - new_func, free_func, dup_func, argl, argp, - maat_instance->logger); + new_func, free_func, dup_func, + argl, argp, maat_instance->logger); if (ret < 0) { pthread_mutex_unlock(&(maat_instance->background_update_mutex)); return -1; } + enum table_type table_type = TABLE_TYPE_INVALID; + int valid_column = -1; if (maat_instance->maat_rt != NULL) { void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); void *schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); - int valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); - generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type, valid_column); + assert(runtime != NULL && schema != NULL); + + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id); + generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type, + valid_column); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); return 0; } -void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, const char *key, size_t key_len) +void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, + const char *key, size_t key_len) { struct maat_runtime *maat_rt = maat_instance->maat_rt; if (NULL == maat_rt) { @@ -569,11 +590,6 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, co return ex_data_runtime_get_ex_data(ex_data_rt, key, key_len); } -static void scan_count_inc(struct maat_state *mid) -{ - mid->scan_cnt++; -} - struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id) { struct maat_state *outer_state = NULL; @@ -587,7 +603,8 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id) return outer_state; } -struct maat_state *grab_state(struct maat_state **state, struct maat *maat_instance, int thread_id) +struct maat_state *grab_state(struct maat_state **state, struct maat *maat_instance, + int thread_id) { struct maat_state *mid = *state; @@ -596,46 +613,51 @@ struct maat_state *grab_state(struct maat_state **state, struct maat *maat_insta *state = mid; //Maat_set_scan_status calls grap_mid() with thread_num=-1. if (mid->thread_id >= 0) { - alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1); + alignment_int64_array_add(maat_instance->outer_state_cnt, thread_id, 1); } } if (mid->thread_id < 0 && thread_id >= 0) { mid->thread_id = thread_id; - alignment_int64_array_add(maat_instance->outer_mid_cnt, thread_id, 1); + alignment_int64_array_add(maat_instance->outer_state_cnt, thread_id, 1); } - if (NULL == mid->compile_mid) { - mid->compile_mid = maat_compile_state_new(thread_id); - alignment_int64_array_add(maat_instance->compile_mid_cnt, thread_id, 1); + if (NULL == mid->compile_state) { + mid->compile_state = maat_compile_state_new(thread_id); + alignment_int64_array_add(maat_instance->compile_state_cnt, thread_id, 1); } return mid; } -int hit_group_to_compile(void *g2g_runtime, int *group_ids, int group_hit_cnt, int vt_id, - void *compile_runtime, int *compile_ids, size_t n_compile_id, +inline int scan_status_should_compile_NOT(struct maat_state *state) +{ + if (state && (LAST_SCAN_SET == state->is_last_scan) && state->compile_state + && maat_compile_state_has_NOT_clause(state->compile_state)) { + return 1; + } + + return 0; +} + +int hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile_ids_size, size_t *n_hit_compile_id, struct maat_state *mid) { - int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - int n_top_group_ids = group2group_runtime_get_top_groups(g2g_runtime, group_ids, group_hit_cnt, top_group_ids); - size_t n_all_group_ids = 0; - if (n_top_group_ids > 0) { - n_all_group_ids = group_hit_cnt + n_top_group_ids; + int compile_id_cnt = compile_runtime_match((struct compile_runtime *)compile_runtime, + compile_ids, compile_ids_size, mid); + *n_hit_compile_id = compile_id_cnt; + if (compile_id_cnt > 0) { + return MAAT_SCAN_HIT; } else { - n_all_group_ids = group_hit_cnt; + return MAAT_SCAN_HALF_HIT; } +} - int all_group_ids[n_all_group_ids] = {-1}; - // maat state find compile_table_id, if not found, maat_instance->default_compile_table_id - int n_compile_ids = compile_runtime_match((struct compile_runtime *)compile_runtime, all_group_ids, n_all_group_ids, - vt_id, compile_ids, n_compile_id, mid); - *n_hit_compile_id = n_compile_ids; - if (n_compile_ids > 0) { - return MAAT_HIT; - } else { - return MAAT_HALF_HIT; - } +int maat_scan_flag(struct maat *instance, int table_id, int thread_id, + uint64_t flag, int *results, size_t n_result, + size_t *n_hit_result, struct maat_state **state) +{ + return 0; } int maat_scan_integer(struct maat *instance, int table_id, int thread_id, @@ -651,18 +673,20 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, { if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) { - return -1; + return MAAT_SCAN_ERR; } struct maat_state *mid = NULL; mid = grab_state(state, maat_instance, thread_id); - scan_count_inc(mid); + mid->scan_cnt++; - int virtual_table_id = 0; + maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); + + int vtable_id = 0; // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, // SCAN_TYPE_IP, &virtual_table_id); // if (NULL == table_schema) { - // return MAAT_ERR; + // return MAAT_SCAN_ERR; // } int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; @@ -670,74 +694,243 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, assert(table_type == TABLE_TYPE_IP_PLUS); // int table_id = table_schema_get_table_id(real_table); - void *ip_plus_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + if (NULL == ip_rt) { + return MAAT_SCAN_ERR; + } // size_t rule_num = table_runtime_rule_count(table_rt); // if (0 == rule_num) { // return 0; // } - int group_hit_cnt = ip_plus_runtime_scan_ipv4((struct ip_plus_runtime *)ip_plus_rt, thread_id, ip_addr, - group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid); + int group_hit_cnt = ip_runtime_scan_ip((struct ip_runtime *)ip_rt, thread_id, IPv4, + (uint8_t *)&ip_addr, group_ids, sizeof(group_ids), + vtable_id, mid); if (group_hit_cnt < 0) { - return MAAT_ERR; - } else if (0 == group_hit_cnt) { - return MAAT_OK; - } else { - // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT - void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); - void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id); - int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime, - results, n_result, n_hit_result, mid); - return ret; + return MAAT_SCAN_ERR; + } + + int compile_ret = 0; + if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + if (group_hit_cnt > 0) { + ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); + } + + int compile_table_id = -1; + if (mid->compile_table_id == -1) { + compile_table_id = maat_instance->default_compile_table_id; + } else { + compile_table_id = mid->compile_table_id; + } + + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); + compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid); + + assert(mid->is_last_scan < LAST_SCAN_FINISHED); + if (LAST_SCAN_SET == mid->is_last_scan) { + mid->is_last_scan = LAST_SCAN_FINISHED; + } } + + if (compile_ret > 0) { + alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); + } + + if (0 == group_hit_cnt && compile_ret > 0) { + // hit NOT group + alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + } + + maat_runtime_ref_dec(maat_instance->maat_rt, thread_id); + + if (0 == compile_ret && group_hit_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } + + return MAAT_SCAN_HIT; } -int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id, +int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, uint8_t *ip_addr, int *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { - return 0; + if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || + (thread_id < 0) || (NULL == ip_addr) || (NULL == results) || (0 == n_result) || + (NULL == state)) { + return MAAT_SCAN_ERR; + } + + struct maat_state *mid = NULL; + mid = grab_state(state, maat_instance, thread_id); + mid->scan_cnt++; + + maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); + int vtable_id = 0; + // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, + // SCAN_TYPE_IP, &virtual_table_id); + // if (NULL == table_schema) { + // return MAAT_SCAN_ERR; + // } + + int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + assert(table_type == TABLE_TYPE_IP_PLUS); + + // int table_id = table_schema_get_table_id(real_table); + void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + if (NULL == ip_rt) { + return MAAT_SCAN_ERR; + } + // size_t rule_num = table_runtime_rule_count(table_rt); + // if (0 == rule_num) { + // return 0; + // } + + int group_hit_cnt = ip_runtime_scan_ip((struct ip_runtime *)ip_rt, thread_id, IPv6, ip_addr, + group_ids, sizeof(group_ids), vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + + int compile_ret = 0; + if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + if (group_hit_cnt > 0) { + ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id); + } + + int compile_table_id = -1; + if (mid->compile_table_id == -1) { + compile_table_id = maat_instance->default_compile_table_id; + } else { + compile_table_id = mid->compile_table_id; + } + + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); + compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid); + + assert(mid->is_last_scan < LAST_SCAN_FINISHED); + if (LAST_SCAN_SET == mid->is_last_scan) { + mid->is_last_scan = LAST_SCAN_FINISHED; + } + } + + if (compile_ret > 0) { + alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); + } + + if (0 == group_hit_cnt && compile_ret > 0) { + // hit NOT group + alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + } + + maat_runtime_ref_dec(maat_instance->maat_rt, thread_id); + + if (0 == compile_ret && group_hit_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } + + return MAAT_SCAN_HIT; } int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, const char *data, size_t data_len, int *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { - if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || - (thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) || - (0 == n_result) || (NULL == state)) { - return MAAT_ERR; + if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) + || (thread_id < 0) || (NULL == data) || (0 == data_len) + || (NULL == results) || (0 == n_result) || (NULL == state)) { + return MAAT_SCAN_ERR; } struct maat_state *mid = NULL; mid = grab_state(state, maat_instance, thread_id); - scan_count_inc(mid); + mid->scan_cnt++; - int virtual_table_id = 0; + int vtable_id = 0; //TODO: by luis get virtual_table_id // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, // SCAN_TYPE_STRING, &virtual_table_id); // if (NULL == table_schema) { - // return MAAT_ERR; + // return MAAT_SCAN_ERR; // } - int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; - void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); - int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)runtime, thread_id, data, data_len, - group_ids, MAX_SCANNER_HIT_GROUP_NUM, virtual_table_id, mid); - if (group_hit_cnt < 0) { - return MAAT_ERR; - } else if (0 == group_hit_cnt) { - return MAAT_OK; - } else { - // come here means group_hit_cnt > 0, at least MAAT_HALF_HIT, or MAAT_HIT - void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); - void *compile_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->default_compile_table_id); - int ret = hit_group_to_compile(g2g_runtime, group_ids, group_hit_cnt, virtual_table_id, compile_runtime, - results, n_result, n_hit_result, mid); - return ret; + if (NULL == maat_instance->maat_rt) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "maat_scan_string error because of maat_runtime is NULL"); + return MAAT_SCAN_OK; } + + maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); + + //TODO: is TABLE_TYPE_EXPR_PLUS + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if ((table_type == TABLE_TYPE_EXPR_PLUS) && + (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { + maat_instance->scan_err_cnt++; + return MAAT_SCAN_ERR; + } + + alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); + + int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + if (NULL == expr_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, thread_id, + data, data_len, group_ids, sizeof(group_ids), + vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + + int compile_ret = 0; + if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + if (group_hit_cnt > 0) { + expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); + } + + if (group_hit_cnt > 0 && table_type == TABLE_TYPE_EXPR_PLUS) { + + } + + int compile_table_id = -1; + if (mid->compile_table_id == -1) { + compile_table_id = maat_instance->default_compile_table_id; + } else { + compile_table_id = mid->compile_table_id; + } + + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); + compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid); + + assert(mid->is_last_scan < LAST_SCAN_FINISHED); + if (LAST_SCAN_SET == mid->is_last_scan) { + mid->is_last_scan = LAST_SCAN_FINISHED; + } + } + + if (compile_ret > 0) { + alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); + } + + if (0 == group_hit_cnt && compile_ret > 0) { + // hit NOT group + alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + } + + maat_runtime_ref_dec(maat_instance->maat_rt, thread_id); + + if (0 == compile_ret && group_hit_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } + + return MAAT_SCAN_HIT; } struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, int thread_id) @@ -745,8 +938,9 @@ struct maat_stream *maat_scan_stream_open(struct maat *instance, int table_id, i return NULL; } -int maat_scan_stream(struct maat_stream **stream, int thread_id, const char *data, int data_len, - int results[], size_t *n_result, struct maat_state **state) +int maat_scan_stream(struct maat_stream **stream, int thread_id, + const char *data, int data_len, int results[], + size_t *n_result, struct maat_state **state) { return 0; } @@ -756,19 +950,27 @@ void maat_scan_stream_close(struct maat_stream **stream) } -int maat_state_set_scan_district(struct maat *maat_instance, struct maat_state **state, const char *district, size_t district_len) +int maat_state_set_scan_district(struct maat *maat_instance, + struct maat_state **state, + const char *district, + size_t district_len) { - if (NULL == maat_instance->maat_rt || NULL == district || district_len <= 0) { + if (NULL == maat_instance->maat_rt || NULL == district || + district_len <= 0) { return -1; } struct maat_state *mid = grab_state(state, maat_instance, -1); - int map_ret = maat_kv_read_unNull(maat_instance->maat_rt->district_map, district, district_len, &(mid->district_id)); - if (map_ret < 0) { + int ret = table_manager_set_scan_district(maat_instance->tbl_mgr, + district, district_len, + &(mid->district_id)); + // int map_ret = maat_kv_read_unNull(maat_instance->maat_rt->district_map, + // district, district_len, &(mid->district_id)); + if (ret < 0) { mid->district_id = DISTRICT_UNKNOWN; } - mid->is_set_district = 1; + mid->is_set_district = DISTRICT_FLAG_SET; return 0; } @@ -780,8 +982,8 @@ int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **sta } struct maat_state *mid = grab_state(state, maat_instance, -1); - assert(mid->is_last_scan == 0); - mid->is_last_scan = 1; + assert(mid->is_last_scan == LAST_SCAN_UNSET); + mid->is_last_scan = LAST_SCAN_SET; return 0; } @@ -799,26 +1001,27 @@ int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_st return 0; } -size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *mid, +size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state, struct maat_hit_path *paths, size_t n_path) { int compile_table_id = -1; - if (mid->compile_table_id == -1) { + if (state->compile_table_id == -1) { compile_table_id = maat_instance->default_compile_table_id; } else { - compile_table_id = mid->compile_table_id; + compile_table_id = state->compile_table_id; } void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); - compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, - (struct group2group_runtime *)g2g_runtime, mid->compile_mid, paths, n_path); - - return 0; + assert(NULL != compile_rt && NULL != g2g_runtime); + return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, + (struct group2group_runtime *)g2g_runtime, + state->compile_state, paths, n_path); } -size_t maat_get_hit_objects(struct maat_compile_state *compile_state, struct maat_hit_object *objs, size_t n_objs) +size_t maat_get_hit_objects(struct maat_compile_state *compile_state, + struct maat_hit_object *objs, size_t n_objs) { return 0; } @@ -829,7 +1032,7 @@ int maat_state_get_hit_paths(struct maat *maat_instance, struct maat_state **sta struct maat_state *mid = NULL; mid = grab_state(state, maat_instance, 0); - if (NULL == mid->compile_mid || NULL == maat_instance->maat_rt) { + if (NULL == mid->compile_state || NULL == maat_instance->maat_rt) { return 0; } @@ -853,13 +1056,13 @@ void maat_state_free(struct maat_state **state) mid = *state; if (mid->thread_id >= 0) { - alignment_int64_array_add(mid->maat_instance->outer_mid_cnt, mid->thread_id, -1); + alignment_int64_array_add(mid->maat_instance->outer_state_cnt, mid->thread_id, -1); } - if (mid->compile_mid != NULL) { - maat_compile_state_free(mid->compile_mid); - mid->compile_mid = NULL; - alignment_int64_array_add(mid->maat_instance->compile_mid_cnt, mid->thread_id, -1); + if (mid->compile_state != NULL) { + maat_compile_state_free(mid->compile_state); + mid->compile_state = NULL; + alignment_int64_array_add(mid->maat_instance->compile_state_cnt, mid->thread_id, -1); } mid->maat_instance = NULL; @@ -870,4 +1073,4 @@ void maat_state_free(struct maat_state **state) int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj) { return 0; -} \ No newline at end of file +} diff --git a/src/maat_command.cpp b/src/maat_command.cpp index 2432ce5..0ee1441 100644 --- a/src/maat_command.cpp +++ b/src/maat_command.cpp @@ -13,7 +13,6 @@ #include #include -#include "utils.h" #include "maat_utils.h" #include "maat_command.h" #include "maat_rule.h" @@ -52,7 +51,8 @@ redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ... return (redisReply *)reply; } -redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db, struct log_handle *logger) +redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, + int redis_db, struct log_handle *logger) { struct timeval connect_timeout; connect_timeout.tv_sec = 0; @@ -116,10 +116,12 @@ void maat_cmd_clear_rule_cache(struct serial_rule *s_rule) memset(s_rule, 0, sizeof(struct serial_rule)); } -int connect_redis_for_write(struct source_redis_ctx *mr_ctx, struct log_handle *logger) +int connect_redis_for_write(struct source_redis_ctx *mr_ctx, + struct log_handle *logger) { assert(mr_ctx->write_ctx == NULL); - mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, logger); + mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, + mr_ctx->redis_db, logger); if (NULL == mr_ctx->write_ctx) { return -1; } else { @@ -130,7 +132,8 @@ int connect_redis_for_write(struct source_redis_ctx *mr_ctx, struct log_handle * redisContext *get_redis_ctx_for_write(struct maat *maat_instance) { if (NULL == maat_instance->mr_ctx.write_ctx) { - int ret = connect_redis_for_write(&(maat_instance->mr_ctx), maat_instance->logger); + int ret = connect_redis_for_write(&(maat_instance->mr_ctx), + maat_instance->logger); if(ret!=0) { return NULL; @@ -139,8 +142,9 @@ redisContext *get_redis_ctx_for_write(struct maat *maat_instance) return maat_instance->mr_ctx.write_ctx; } -void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id, - const char *table_name, const char *line, long long timeout) +void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, + unsigned long rule_id, const char *table_name, + const char *line, long long timeout) { memset(rule, 0, sizeof(struct serial_rule)); rule->op = op; @@ -153,7 +157,8 @@ void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, } } -int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq) +int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, + int valid_column_seq) { int column_seq = 0; @@ -204,7 +209,8 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, size_t len = 0; int ret = get_column_pos(line, column_seq, &offset, &len); // 0 is also a valid value for some non-MAAT producer. - if (ret < 0 || offset >= strlen(line) || (line[offset] != '1' && line[offset] != '0')) { + if (ret < 0 || offset >= strlen(line) || (line[offset] != '1' && + line[offset] != '0')) { return -1; } @@ -306,7 +312,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, line_rule->table_name); if (table_id < 0) { - log_error(maat_instance->logger, MODULE_MAAT_COMMAND, "Command set line id %d failed: unknown table %s", + log_error(maat_instance->logger, MODULE_MAAT_COMMAND, + "Command set line id %d failed: unknown table %s", line_rule->rule_id, line_rule->table_name); FREE(s_rule); return -1; @@ -328,8 +335,8 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li absolute_expire_time = server_time + line_rule->expire_after; } - maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name, - line_rule->table_line, absolute_expire_time); + maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, + line_rule->table_name, line_rule->table_line, absolute_expire_time); int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_instance->logger); if (success_cnt != 1) { diff --git a/src/maat_compile.cpp b/src/maat_compile.cpp index f98f781..ce45f53 100644 --- a/src/maat_compile.cpp +++ b/src/maat_compile.cpp @@ -11,7 +11,6 @@ #include #include -#include "utils.h" #include "maat_utils.h" #include "log/log.h" #include "uthash/utarray.h" @@ -23,6 +22,7 @@ #include "maat_group.h" #include "maat/maat.h" #include "rcu_hash.h" +#include "maat_table.h" #define MODULE_COMPILE module_name_str("maat.compile") #define MAX_TABLE_LINE_SIZE (1024 * 16) @@ -48,17 +48,20 @@ struct compile_schema { size_t n_ex_schema; struct compile_ex_data_schema ex_schema[MAX_COMPILE_EX_DATA_NUM]; int table_id; //ugly + struct table_manager *tbl_mgr; + size_t unmatched_tag_cnt; }; struct group2compile_schema { int group_id_column; int compile_id_column; int not_flag_column; - int virtual_table_name_column; + int vtable_name_column; int clause_index_column; char associated_compile_table_id; struct table_manager *ref_tbl_mgr; int table_id;//ugly + struct table_manager *tbl_mgr; }; struct compile_item { @@ -76,7 +79,7 @@ struct group2compile_item { int group_id; int compile_id; int not_flag; - int vt_id; //virtual_table_id + int vtable_id; int clause_index; int associated_compile_table_id; }; @@ -90,6 +93,7 @@ struct compile_runtime { uint32_t rule_num; uint32_t updating_rule_num; + struct bool_expr_match *expr_match_buff; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; }; @@ -109,7 +113,7 @@ struct maat_clause_state { struct maat_literal_id { int group_id; - int vt_id; + int vtable_id; }; struct maat_clause { @@ -144,24 +148,25 @@ struct maat_internal_hit_path { int Nth_hit_item; int item_id; int group_id; - int virtual_table_id; + int vtable_id; }; struct maat_compile_state { int thread_id; int Nth_scan; time_t hier_ver; - size_t this_scan_item_hit_cnt; + size_t this_scan_hit_item_cnt; int not_clause_hitted_flag; int is_no_count_scan; size_t hit_path_cnt; UT_array *internal_hit_paths; - UT_array *all_hit_clause_array; - UT_array *this_scan_hit_clause_ids; + UT_array *all_hit_clauses; + UT_array *this_scan_hit_clauses; }; -int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, int table_id, +int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, + int table_id, maat_rule_ex_new_func_t *new_func, maat_rule_ex_free_func_t *free_func, maat_rule_ex_dup_func_t *dup_func, @@ -169,7 +174,8 @@ int compile_table_set_rule_ex_data_schema(struct compile_schema *compile_schema, struct log_handle *logger) { if (compile_schema->n_ex_schema == MAX_COMPILE_EX_DATA_NUM) { - log_error(logger, MODULE_COMPILE, "compile ex schema num reach maxium, can't set anymore"); + log_error(logger, MODULE_COMPILE, + "compile ex schema num reach maxium, can't set anymore"); return -1; } @@ -213,7 +219,9 @@ UT_icd ut_literal_id_icd = {sizeof(struct maat_literal_id), NULL, NULL, NULL}; UT_icd ut_clause_id_icd = {sizeof(uint64_t), NULL, NULL, NULL}; UT_icd ut_hit_path_icd = {sizeof(struct maat_internal_hit_path), NULL, NULL, NULL}; -void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, + struct log_handle *logger) { int read_cnt = 0; @@ -228,7 +236,8 @@ void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { - log_error(logger, MODULE_COMPILE, "table %s has no custom column", table_name); + log_error(logger, MODULE_COMPILE, + "table %s has no custom column", table_name); goto error; } @@ -286,6 +295,8 @@ void *compile_schema_new(cJSON *json, const char *table_name, struct log_handle read_cnt++; } + compile_schema->tbl_mgr = tbl_mgr; + if (read_cnt < 10) { goto error; } @@ -301,7 +312,8 @@ void compile_schema_free(void *compile_schema) FREE(compile_schema); } -void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +void *group2compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) { int read_cnt = 0; struct group2compile_schema *g2c_schema = ALLOC(struct group2compile_schema, 1); @@ -345,7 +357,7 @@ void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_h custom_item = cJSON_GetObjectItem(item, "virtual_table_name"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - g2c_schema->virtual_table_name_column = custom_item->valueint; + g2c_schema->vtable_name_column = custom_item->valueint; read_cnt++; } @@ -355,6 +367,8 @@ void *group2compile_schema_new(cJSON *json, const char *table_name, struct log_h read_cnt++; } + g2c_schema->tbl_mgr = tbl_mgr; + if (read_cnt < 7) { goto error; } @@ -378,80 +392,95 @@ int group2compile_associated_compile_table_id(void *g2c_schema) } struct compile_item * -compile_item_new(const char *line, struct compile_schema *compile_schema, struct log_handle *logger) +compile_item_new(const char *line, struct compile_schema *compile_schema, + struct log_handle *logger) { size_t column_offset = 0; size_t column_len = 0; char tag_str[MAX_TABLE_LINE_SIZE] = {0}; + size_t n_accept_tag = 0; struct compile_item *compile_item = ALLOC(struct compile_item, 1); - int ret = get_column_pos(line, compile_schema->compile_id_column, &column_offset, &column_len); + int ret = get_column_pos(line, compile_schema->compile_id_column, + &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no compile_id", + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s has no compile_id", compile_schema->table_id, line); goto error; } compile_item->compile_id = atoi(line + column_offset); - ret = get_column_pos(line, compile_schema->service_id_column, &column_offset, &column_len); + ret = get_column_pos(line, compile_schema->service_id_column, + &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no service_id", + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s has no service_id", compile_schema->table_id, line); goto error; } compile_item->service_id = atoi(line + column_offset); - ret = get_column_pos(line, compile_schema->action_column, &column_offset, &column_len); + ret = get_column_pos(line, compile_schema->action_column, + &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no action", + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s has no action", compile_schema->table_id, line); goto error; } compile_item->action = atoi(line + column_offset); - ret = get_column_pos(line, compile_schema->do_blacklist_column, &column_offset, &column_len); + ret = get_column_pos(line, compile_schema->do_blacklist_column, + &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no do_blacklist", + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s has no do_blacklist", compile_schema->table_id, line); goto error; } compile_item->do_blacklist = atoi(line + column_offset); - ret = get_column_pos(line, compile_schema->do_log_column, &column_offset, &column_len); + ret = get_column_pos(line, compile_schema->do_log_column, + &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no do_log", + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s has no do_log", compile_schema->table_id, line); goto error; } compile_item->do_log = atoi(line + column_offset); - ret = get_column_pos(line, compile_schema->tags_column, &column_offset, &column_len); + ret = get_column_pos(line, compile_schema->tags_column, + &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no tags", + log_error(logger, MODULE_COMPILE, + "compile table(table_id:%d) line:%s has no tags", compile_schema->table_id, line); goto error; } memcpy(tag_str, (line + column_offset), column_len); -#if 0 + n_accept_tag = table_manager_accept_tags_count(compile_schema->tbl_mgr); + if (n_accept_tag > 0 && strlen(tag_str) > 2) { str_unescape(tag_str); - ret = compare_accept_tag(tag_str, accept_tags, n_accept_tag); - ret = table_manager_accept_tag_match(tag_str); - if (ret < 0) { + ret = table_manager_accept_tags_match(compile_schema->tbl_mgr, tag_str); + if (TAG_MATCH_ERR == ret) { log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s is invalid tag", compile_schema->table_id, line); goto error; } - if (0 == ret) { //not matched - //TODO: by luis - //table_schema->unmatched_tag_cnt++; + if (TAG_MATCH_MATCHED == ret) { //not matched + compile_schema->unmatched_tag_cnt++; + goto error; } } -#endif - ret = get_column_pos(line, compile_schema->user_region_column, &column_offset, &column_len); + + ret = get_column_pos(line, compile_schema->user_region_column, + &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no user_region", @@ -476,7 +505,8 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, struct break; } - ret = get_column_pos(line, compile_schema->clause_num_column, &column_offset, &column_len); + ret = get_column_pos(line, compile_schema->clause_num_column, + &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no clause_num", @@ -485,7 +515,8 @@ compile_item_new(const char *line, struct compile_schema *compile_schema, struct } compile_item->clause_num = atoi(line + column_offset); - ret = get_column_pos(line, compile_schema->evaluation_order_column, &column_offset, &column_len); + ret = get_column_pos(line, compile_schema->evaluation_order_column, + &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_COMPILE, "compile table(table_id:%d) line:%s has no evaluation_order", @@ -509,7 +540,8 @@ void compile_item_free(struct compile_item *compile_item) FREE(compile_item); } -void *compile_runtime_new(void *compile_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *compile_runtime_new(void *compile_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == compile_schema) { @@ -518,6 +550,7 @@ void *compile_runtime_new(void *compile_schema, int max_thread_num, struct maat_ struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1); + compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM); compile_rt->logger = logger; compile_rt->ref_garbage_bin = garbage_bin; @@ -567,17 +600,23 @@ void compile_runtime_free(void *compile_runtime) maat_compile_hash_free(&(compile_rt->compile_hash)); } + if (compile_rt->expr_match_buff != NULL) { + FREE(compile_rt->expr_match_buff); + } + FREE(compile_rt); } -void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { struct group2compile_runtime *g2c_rt = ALLOC(struct group2compile_runtime, 1); return g2c_rt; } -void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, void *g2g_runtime) +void group2compile_runtime_init(void *g2c_runtime, void *compile_runtime, + void *g2g_runtime) { struct group2compile_runtime *g2c_rt = (struct group2compile_runtime *)g2c_runtime; @@ -603,7 +642,8 @@ int is_valid_table_name(const char *str) } } - if (strlen(str) == 0 || integer_cnt == strlen(str) || 0 == strcasecmp(str, "null")) { + if (strlen(str) == 0 || integer_cnt == strlen(str) || + 0 == strcasecmp(str, "null")) { return 0; } @@ -616,7 +656,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema { size_t column_offset = 0; size_t column_len = 0; - char virtual_table_name[NAME_MAX] = {0}; + char vtable_name[NAME_MAX] = {0}; struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1); int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset, &column_len); @@ -646,7 +686,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema } g2c_item->not_flag = atoi(line + column_offset); - ret = get_column_pos(line, g2c_schema->virtual_table_name_column, &column_offset, &column_len); + ret = get_column_pos(line, g2c_schema->vtable_name_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_COMPILE, "group2compile table(table_id:%d) line:%s has no virtual_table_name", @@ -661,14 +701,14 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema goto error; } - memcpy(virtual_table_name, (line + column_offset), column_len); + memcpy(vtable_name, (line + column_offset), column_len); - if (is_valid_table_name(virtual_table_name)) { - g2c_item->vt_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr, virtual_table_name); - if (g2c_item->vt_id < 0) { + if (is_valid_table_name(vtable_name)) { + g2c_item->vtable_id = table_manager_get_table_id(g2c_schema->ref_tbl_mgr, vtable_name); + if (g2c_item->vtable_id < 0) { log_error(logger, MODULE_COMPILE, "group2compile table(table_id:%d) line:%s unknown virtual table:%s", - g2c_schema->table_id, line, virtual_table_name); + g2c_schema->table_id, line, vtable_name); goto error; } } @@ -714,7 +754,8 @@ struct maat_compile *maat_compile_new(int compile_id) return compile; } -int maat_compile_set(struct maat_compile *compile, int declared_clause_num, void *user_data, void (*user_data_free)(void *)) +int maat_compile_set(struct maat_compile *compile, int declared_clause_num, + void *user_data, void (*user_data_free)(void *)) { if (user_data != NULL && NULL == user_data_free) { return -1; @@ -727,7 +768,8 @@ int maat_compile_set(struct maat_compile *compile, int declared_clause_num, void return 0; } -int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, struct maat_compile *compile) +int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, + struct maat_compile *compile) { int ret = 0; struct maat_compile *tmp_compile = NULL; @@ -742,7 +784,8 @@ int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, st if (tmp_compile->user_data != NULL) { ret = -1; } else { - maat_compile_set(tmp_compile, compile->declared_clause_num, compile->user_data, compile->user_data_free); + maat_compile_set(tmp_compile, compile->declared_clause_num, + compile->user_data, compile->user_data_free); } } @@ -757,7 +800,8 @@ int maat_compile_hash_add(struct maat_compile **compile_hash, int compile_id, st return ret; } -int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id, struct maat_garbage_bin *garbage_bin) +int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id, + struct maat_garbage_bin *garbage_bin) { struct maat_compile *compile = NULL; @@ -782,7 +826,8 @@ int maat_compile_hash_remove(struct maat_compile **compile_hash, int compile_id, return 0; } -struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash, int compile_id) +struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash, + int compile_id) { struct maat_compile *compile = NULL; @@ -801,7 +846,7 @@ int compare_literal_id(const void *pa, const void *pb) struct maat_literal_id *la = (struct maat_literal_id *)pa; struct maat_literal_id *lb = (struct maat_literal_id *)pb; - int ret = la->vt_id - lb->vt_id; + int ret = la->vtable_id - lb->vtable_id; if (0 == ret) { ret = la->group_id - lb->group_id; } @@ -863,7 +908,8 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile, static const struct maat_clause * maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, unsigned long long *clause_id_generator, - struct maat_literal_id *literal_ids, size_t n_literal_id) + struct maat_literal_id *literal_ids, + size_t n_literal_id) { struct maat_clause *clause = NULL; @@ -876,7 +922,8 @@ maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, memcpy(clause->literal_ids, literal_ids, n_literal_id * sizeof(struct maat_literal_id)); (*clause_id_generator)++; - HASH_ADD_KEYPTR(hh, *clause_hash, clause->literal_ids, n_literal_id * sizeof(struct maat_literal_id), clause); + HASH_ADD_KEYPTR(hh, *clause_hash, clause->literal_ids, + n_literal_id * sizeof(struct maat_literal_id), clause); } return clause; @@ -926,7 +973,8 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, has_clause_num++; literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->literal_ids, 0); n_literal_id = utarray_len(clause_state->literal_ids); - clause = maat_clause_hash_fetch_clause(&clause_hash, clause_id_generator, literal_ids, n_literal_id); + clause = maat_clause_hash_fetch_clause(&clause_hash, clause_id_generator, + literal_ids, n_literal_id); clause_state->clause_id = clause->clause_id; } assert(has_clause_num == compile->actual_clause_num); @@ -948,7 +996,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, struct maat_literal_id *p = NULL; for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) { printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n", - compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vt_id); + compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vtable_id); } #endif bool_expr_array[expr_cnt].items[j].item_id = compile->clause_states[i].clause_id; @@ -1010,7 +1058,8 @@ void maat_compile_bool_matcher_free(struct bool_matcher *bm) bool_matcher_free(bm); } -size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_compile_state *compile_state, +size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan, + struct maat_compile_state *compile_state, void **user_data_array, size_t ud_array_size) { struct maat_compile *compile = NULL; @@ -1026,8 +1075,10 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_comp } #endif - int bool_match_ret = bool_matcher_match(bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clause_array, 0), - utarray_len(compile_state->all_hit_clause_array), expr_match, MAX_SCANNER_HIT_COMPILE_NUM); + int bool_match_ret = bool_matcher_match(bm, + (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0), + utarray_len(compile_state->all_hit_clauses), + expr_match, MAX_SCANNER_HIT_COMPILE_NUM); for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) { compile = (struct maat_compile *)expr_match[i].user_tag; assert(compile->magic == MAAT_COMPILE_MAGIC); @@ -1036,6 +1087,10 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, struct maat_comp continue; } + if ((compile->not_clause_cnt > 0) && (LAST_SCAN_UNSET == is_last_scan)) { + compile_state->not_clause_hitted_flag = 1; + } + //TODO: not_clause if (compile->user_data) { user_data_array[ud_result_cnt] = compile->user_data; @@ -1064,21 +1119,25 @@ int maat_add_group_to_compile(struct maat_compile **compile_hash, void *g2g_runt } } - struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vt_id}; - ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, g2c_item->not_flag); + struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vtable_id}; + ret = maat_compile_clause_add_literal(compile, &literal_id, g2c_item->clause_index, + g2c_item->not_flag); if (ret < 0) { - log_error(logger, MODULE_COMPILE, "add literal_id{group_id:%d, vt_id:%d} to clause %d of compile %d failed", - g2c_item->group_id, g2c_item->vt_id, g2c_item->clause_index, g2c_item->compile_id); + log_error(logger, MODULE_COMPILE, + "add literal_id{group_id:%d, vtable_id:%d} to clause %d of compile %d failed", + g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, + g2c_item->compile_id); ret = -1; } else { - group->ref_by_compile_cnt++; + maat_group_ref_inc(group); ret = 0; } return ret; } -int maat_remove_group_from_compile(struct maat_compile **compile_hash, void *g2g_runtime, +int maat_remove_group_from_compile(struct maat_compile **compile_hash, + void *g2g_runtime, struct group2compile_item *g2c_item, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) @@ -1100,14 +1159,15 @@ int maat_remove_group_from_compile(struct maat_compile **compile_hash, void *g2g return -1; } - struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vt_id}; + struct maat_literal_id literal_id = {g2c_item->group_id, g2c_item->vtable_id}; int ret = maat_compile_clause_remove_literal(compile, &literal_id, g2c_item->clause_index); if (ret < 0) { log_error(logger, MODULE_COMPILE, - "Remove group %d vt_id %d from clause %d of compile %d failed, literal is not in compile.", - g2c_item->group_id, g2c_item->vt_id, g2c_item->clause_index, g2c_item->compile_id); + "Remove group %d vtable_id %d from clause %d of compile %d failed, literal is not in compile.", + g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, g2c_item->compile_id); return -1; } + maat_group_ref_dec(group); if (0 == compile->actual_clause_num && NULL == compile->user_data) { HASH_DEL(*compile_hash, compile); @@ -1151,7 +1211,8 @@ void maat_compile_state_free(struct maat_compile_state *compile_state) free(compile_state); } -static int maat_compile_hit_path_add(UT_array* hit_paths, int item_id, int group_id, int virtual_table_id, int Nth_scan, int Nth_item_result) +static int maat_compile_hit_path_add(UT_array *hit_paths, int item_id, int group_id, + int vtable_id, int Nth_scan, int Nth_item_result) { struct maat_internal_hit_path new_path; @@ -1159,13 +1220,15 @@ static int maat_compile_hit_path_add(UT_array* hit_paths, int item_id, int group new_path.Nth_hit_item = Nth_item_result; new_path.Nth_scan = Nth_scan; new_path.group_id = group_id; - new_path.virtual_table_id = virtual_table_id; + new_path.vtable_id = vtable_id; utarray_push_back(hit_paths, &new_path); + return 1; } -static int maat_compile_has_literal(struct maat_compile* compile, struct maat_literal_id *literal_id) +static int maat_compile_has_literal(struct maat_compile* compile, + struct maat_literal_id *literal_id) { int i = 0; struct maat_literal_id *tmp = NULL; @@ -1177,9 +1240,11 @@ static int maat_compile_has_literal(struct maat_compile* compile, struct maat_li continue; } - tmp = (struct maat_literal_id*)utarray_find(clause_state->literal_ids, literal_id, compare_literal_id); + tmp = (struct maat_literal_id*)utarray_find(clause_state->literal_ids, + literal_id, compare_literal_id); if (tmp) { - assert(tmp->group_id == literal_id->group_id && tmp->vt_id == literal_id->vt_id); + assert(tmp->group_id == literal_id->group_id && + tmp->vtable_id == literal_id->vtable_id); return 1; } } @@ -1187,7 +1252,8 @@ static int maat_compile_has_literal(struct maat_compile* compile, struct maat_li return 0; } -static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths, size_t n_path, const struct maat_hit_path *find) +static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths, + size_t n_path, const struct maat_hit_path *find) { for (size_t i = 0; i < n_path; i++) { if (0 == memcmp(hit_paths + i, find, sizeof(*find))) { @@ -1201,44 +1267,58 @@ static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_path size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct group2group_runtime *g2g_rt, struct maat_compile_state *compile_state, - struct maat_hit_path *hit_paths, size_t hit_path_size) + struct maat_hit_path *hit_paths, + size_t hit_path_size) { size_t i = 0, j = 0; struct maat_internal_hit_path *internal_path = NULL; - struct maat_group *group = NULL; - size_t n_made_by_item = 0, n_made_by_compile = 0; + size_t hit_path_cnt = 0; + size_t new_hit_path_cnt = 0; for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) { internal_path = (struct maat_internal_hit_path *)utarray_eltptr(compile_state->internal_hit_paths, i); - group = group2group_runtime_find_group(g2g_rt, internal_path->group_id); + /* + NOTE: maybe one item has been deleted, but it's item_id still exist in internal_hit_paths + */ + int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &(internal_path->group_id), + 1, top_group_ids); + if (top_group_cnt <= 0) { + /* + item->group_id has no top group, this group can only be referenced by compile + ------------------------------------------------------------------------------ + for example: + compile1 -> group1 -> group2 -> item1 + group3 -> item2 + + group1 and group3 has no top group + group1 is referenced by compile1, group3 is not referenced by any compile - if (0 == group->top_group_cnt && n_made_by_item < hit_path_size) { - //group not referenced by compile - hit_paths[n_made_by_item].Nth_scan = internal_path->Nth_scan; - hit_paths[n_made_by_item].item_id = internal_path->item_id; - hit_paths[n_made_by_item].sub_group_id = group->group_id; - hit_paths[n_made_by_item].top_group_id = -1; - hit_paths[n_made_by_item].virtual_table_id = internal_path->virtual_table_id; - hit_paths[n_made_by_item].compile_id = -1; - n_made_by_item++; - } else { - for (j = 0; j < group->top_group_cnt && n_made_by_item < hit_path_size; j++, n_made_by_item++) { - hit_paths[n_made_by_item].Nth_scan = internal_path->Nth_scan; - hit_paths[n_made_by_item].item_id = internal_path->item_id; - hit_paths[n_made_by_item].sub_group_id = group->group_id; - hit_paths[n_made_by_item].top_group_id = group->top_group_ids[j]; - hit_paths[n_made_by_item].virtual_table_id = internal_path->virtual_table_id; - hit_paths[n_made_by_item].compile_id = -1; - } - } + NOTE: Add the hit path as long as the item is hit + + */ + top_group_cnt = 1; //add one hit path which top_group_ids[0] = -1 + } + + for (int j = 0; j < top_group_cnt && hit_path_cnt < hit_path_size; j++, hit_path_cnt++) { + hit_paths[hit_path_cnt].Nth_scan = internal_path->Nth_scan; + hit_paths[hit_path_cnt].item_id = internal_path->item_id; + hit_paths[hit_path_cnt].sub_group_id = internal_path->group_id; + hit_paths[hit_path_cnt].top_group_id = top_group_ids[j]; //top_group_id may be -1 + hit_paths[hit_path_cnt].vtable_id = internal_path->vtable_id; + hit_paths[hit_path_cnt].compile_id = -1; + } } + /* assign hit_paths[].compile_id */ struct maat_compile *compile = NULL; struct maat_literal_id literal_id = {0, 0}; struct maat_hit_path tmp_path; - struct bool_expr_match *expr_match = ALLOC(struct bool_expr_match, MAX_SCANNER_HIT_COMPILE_NUM); - int bool_match_ret = bool_matcher_match(compile_rt->bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clause_array, 0), - utarray_len(compile_state->all_hit_clause_array), expr_match, MAX_SCANNER_HIT_COMPILE_NUM); + struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM; + int bool_match_ret = bool_matcher_match(compile_rt->bm, + (unsigned long long *)utarray_eltptr(compile_state->all_hit_clause_array, 0), + utarray_len(compile_state->all_hit_clause_array), expr_match, + MAX_SCANNER_HIT_COMPILE_NUM); for (int idx = 0; idx < bool_match_ret; idx++) { compile = (struct maat_compile *)expr_match[idx].user_tag; assert(compile->magic == MAAT_COMPILE_MAGIC); @@ -1247,51 +1327,52 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, continue; } - for (j = 0; j < n_made_by_item && (n_made_by_item+n_made_by_compile) < hit_path_size; j++) { - if (hit_paths[j].top_group_id<0) { + for (j = 0; j < hit_path_cnt && (hit_path_cnt + new_hit_path_cnt) < hit_path_size; j++) { + if (hit_paths[j].top_group_id < 0) { continue; } literal_id.group_id = hit_paths[j].top_group_id; - literal_id.vt_id = hit_paths[j].virtual_table_id; + literal_id.vtable_id = hit_paths[j].vtable_id; if (maat_compile_has_literal(compile, &literal_id)) { if (hit_paths[j].compile_id < 0) { hit_paths[j].compile_id = compile->compile_id; } else { + // means same literal_id hit more than one compile_id tmp_path = hit_paths[j]; tmp_path.compile_id = compile->compile_id; - if(maat_compile_is_hit_path_existed(hit_paths, n_made_by_item + n_made_by_compile, &tmp_path)) { - hit_paths[n_made_by_item + n_made_by_compile] = tmp_path; - n_made_by_compile++; + if(maat_compile_is_hit_path_existed(hit_paths, hit_path_cnt + new_hit_path_cnt, &tmp_path)) { + hit_paths[hit_path_cnt + new_hit_path_cnt] = tmp_path; + new_hit_path_cnt++; } } } } } - return n_made_by_item + n_made_by_compile; + return hit_path_cnt + new_hit_path_cnt; } -void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, int item_id, int group_id, int virtual_table_id, - int Nth_scan, int Nth_item_result) +void maat_compile_state_update_hit_path(struct maat_compile_state *compile_state, + int item_id, int group_id, int vtable_id, + int Nth_scan, int Nth_item_result) { if (compile_state->Nth_scan != Nth_scan) { - assert(compile_state->this_scan_item_hit_cnt == 0); + assert(compile_state->this_scan_hit_item_cnt == 0); compile_state->Nth_scan = Nth_scan; - utarray_clear(compile_state->this_scan_hit_clause_ids); + utarray_clear(compile_state->this_scan_hit_clauses); } - int ret = maat_compile_hit_path_add(compile_state->internal_hit_paths, item_id, group_id, virtual_table_id, Nth_scan, Nth_item_result); - if (!ret) { - return; - } + maat_compile_hit_path_add(compile_state->internal_hit_paths, item_id, group_id, + vtable_id, Nth_scan, Nth_item_result); compile_state->hit_path_cnt++; - compile_state->this_scan_item_hit_cnt++; + compile_state->this_scan_hit_item_cnt++; } void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_state, - void *compile_runtime, int group_id, int vt_id) + void *compile_runtime, int group_id, + int vtable_id) { if (NULL == compile_state || NULL == compile_runtime) { return; @@ -1302,9 +1383,12 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta struct maat_compile *compile = NULL, *tmp_compile = NULL; struct maat_clause_state *clause_state = NULL; - struct maat_literal_id literal_id = {group_id, vt_id}; + struct maat_literal_id literal_id = {group_id, vtable_id}; struct maat_literal_id *tmp = NULL; + unsigned long long *clause_id = 0; + size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses); + HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) { for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { clause_state = compile->clause_states + i; @@ -1312,16 +1396,29 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta continue; } - tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, &literal_id, compare_literal_id); + tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, + &literal_id, compare_literal_id); if (tmp) { //Deduplication - if (utarray_find(compile_state->all_hit_clause_array, &(clause_state->clause_id), compare_clause_id)) { + if (utarray_find(compile_state->all_hit_clauses, &(clause_state->clause_id), + compare_clause_id)) { continue; } - utarray_push_back(compile_state->all_hit_clause_array, &(clause_state->clause_id)); - utarray_sort(compile_state->all_hit_clause_array, compare_clause_id); + utarray_push_back(compile_state->this_scan_hit_clauses, &(clause_state->clause_id)); } - } + + //means this scan hit new clause + if ((utarray_len(compile_state->this_scan_hit_clauses) - new_clause_idx) > 0) { + utarray_reserve(compile_state->all_hit_clauses, + utarray_len(compile_state->this_scan_hit_clauses) - new_clause_idx); + + for (i = new_clause_idx; i < utarray_len(compile_state->this_scan_hit_clauses); i++) { + clause_id = (unsigned long long *)utarray_eltptr(compile_state->this_scan_hit_clauses, i); + utarray_push_back(compile_state->all_hit_clauses, clause_id); + } + utarray_sort(compile_state->all_hit_clauses, compare_clause_id); + } + } } } @@ -1342,14 +1439,15 @@ void *rule_ex_data_new(const struct maat_rule_head *rule_head, const char *srv_d return ex_data; } -void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, void *ex_data, - const struct compile_ex_data_schema *ex_schema) +void rule_ex_data_free(const struct maat_rule_head *rule_head, const char *srv_def, + void *ex_data, const struct compile_ex_data_schema *ex_schema) { struct maat_rule rule; memset(&rule, 0, sizeof(rule)); fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1); - ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, ex_schema->argl, ex_schema->argp); + ex_schema->free_func(ex_schema->idx, &rule, srv_def, ex_data, + ex_schema->argl, ex_schema->argp); } void compile_item_to_compile_rule(struct compile_item *compile_item, @@ -1371,13 +1469,15 @@ void compile_item_to_compile_rule(struct compile_item *compile_item, compile_rule->ref_table = compile_schema; compile_rule->head.serv_def_len = strlen(compile_item->user_region); compile_rule->service_defined = ALLOC(char, compile_rule->head.serv_def_len); - memcpy(compile_rule->service_defined, compile_item->user_region, compile_rule->head.serv_def_len); + memcpy(compile_rule->service_defined, compile_item->user_region, + compile_rule->head.serv_def_len); compile_rule->evaluation_order = compile_item->evaluation_order; size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(compile_schema); for (size_t i = 0; i < n_rule_ex_schema; i++) { struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(compile_schema, i); - compile_rule->ex_data[i] = rule_ex_data_new(&compile_rule->head, compile_rule->service_defined, ex_schema); + compile_rule->ex_data[i] = rule_ex_data_new(&compile_rule->head, + compile_rule->service_defined, ex_schema); } compile_rule->compile_id = compile_item->compile_id; pthread_rwlock_init(&compile_rule->rwlock, NULL); @@ -1391,7 +1491,8 @@ void destroy_compile_rule(struct compile_rule *compile_rule) size_t n_rule_ex_schema = compile_table_rule_ex_data_schema_count(schema); for (size_t i = 0; i < n_rule_ex_schema; i++) { struct compile_ex_data_schema *ex_schema = compile_table_get_rule_ex_data_schema(schema, i); - rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, compile_rule->ex_data+i, ex_schema); + rule_ex_data_free(&(compile_rule->head), compile_rule->service_defined, + compile_rule->ex_data+i, ex_schema); compile_rule->ex_data[i] = NULL; } free(compile_rule->ex_data); @@ -1401,10 +1502,11 @@ void destroy_compile_rule(struct compile_rule *compile_rule) free(compile_rule); } -int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, - int valid_column) +int compile_runtime_update(void *compile_runtime, void *compile_schema, + const char *line, int valid_column) { - if (NULL == compile_runtime || NULL == compile_schema || NULL == line) { + if (NULL == compile_runtime || NULL == compile_schema || + NULL == line) { return -1; } @@ -1419,7 +1521,8 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, const ch return -1; } else if (0 == is_valid) { //delete - ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id, compile_rt->ref_garbage_bin); + ret = maat_compile_hash_remove(&(compile_rt->compile_hash), compile_id, + compile_rt->ref_garbage_bin); if (ret < 0) { log_error(compile_rt->logger, MODULE_COMPILE, "remove compile table(table_id:%d) compile(compile_id:%d) from compile_hash failed", @@ -1447,7 +1550,8 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, const ch return -1; } - maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule, (void (*)(void *))destroy_compile_rule); + maat_compile_set(compile, compile_rule->declared_clause_num, compile_rule, + (void (*)(void *))destroy_compile_rule); ret = maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile); if (ret < 0) { maat_compile_free(compile); @@ -1461,8 +1565,8 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, const ch return 0; } -int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char *line, - int valid_column) +int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, + const char *line, int valid_column) { if (NULL == g2c_runtime || NULL == g2c_schema || NULL == line) { return -1; @@ -1494,7 +1598,8 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema, const char } } else { //add - ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item, compile_rt->logger); + ret = maat_add_group_to_compile(&(compile_rt->compile_hash), g2g_rt, g2c_item, + compile_rt->logger); if (0 == ret) { if (g2c_item->not_flag) { g2c_rt->not_flag_group++; @@ -1531,20 +1636,23 @@ int compile_runtime_commit(void *compile_runtime) compile_rt->logger); if (NULL == new_bool_matcher) { log_error(compile_rt->logger, MODULE_COMPILE, - "rebuild compile bool_matcher engine failed when update %zu compile rules", compile_cnt); + "rebuild compile bool_matcher engine failed when update %zu compile rules", + compile_cnt); ret = -1; } old_bool_matcher = compile_rt->bm; compile_rt->bm = new_bool_matcher; - maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, (void (*)(void*))maat_compile_bool_matcher_free); + maat_garbage_bagging(compile_rt->ref_garbage_bin, old_bool_matcher, + (void (*)(void*))maat_compile_bool_matcher_free); compile_rt->rule_num = compile_cnt; return ret; } -static int compile_sort_para_compare(const struct compile_sort_para *a, const struct compile_sort_para *b) +static int compile_sort_para_compare(const struct compile_sort_para *a, + const struct compile_sort_para *b) { //If both of compile rule's evaluation order are specified, compile rule with small evaluation order is priority. if (a->evaluation_order != 0 && b->evaluation_order != 0) { @@ -1566,7 +1674,9 @@ static int compile_sort_para_compare(const struct compile_sort_para *a, const st } } -static void compile_sort_para_set(struct compile_sort_para *para, const struct compile_rule *compile_relation, void *user) +static void compile_sort_para_set(struct compile_sort_para *para, + const struct compile_rule *compile_relation, + void *user) { para->compile_id = compile_relation->compile_id; para->evaluation_order = compile_relation->evaluation_order; @@ -1586,25 +1696,94 @@ static int compare_compile_rule(const void *a, const void *b) return compile_sort_para_compare(&sa, &sb); } -int compile_runtime_match(struct compile_runtime *compile_rt, int *group_ids, size_t n_group_ids, - int vt_id, int *compile_ids, size_t compile_ids_size, struct maat_state *state) +int compile_runtime_match(struct compile_runtime *compile_rt, + int *compile_ids, size_t compile_ids_size, + struct maat_state *state) { if (NULL == compile_rt) { return -1; } - struct maat_compile_state *compile_state = state->compile_mid; - struct compile_rule *compile_rule_array[compile_ids_size]; + struct maat_compile_state *compile_state = state->compile_state; + int is_last_scan = state->is_last_scan; + struct compile_rule *compile_rules[compile_ids_size]; // all hit clause_id -> compile_id - size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt->bm, compile_state, (void **)compile_rule_array, compile_ids_size); + size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt->bm, + is_last_scan, compile_state, + (void **)compile_rules, + compile_ids_size); if (bool_match_ret > 0) { - qsort(compile_rule_array, bool_match_ret, sizeof(struct compile_rule *), compare_compile_rule); + qsort(compile_rules, bool_match_ret, sizeof(struct compile_rule *), + compare_compile_rule); } for (size_t i = 0; i < bool_match_ret; i++) { - compile_ids[i] = compile_rule_array[i]->compile_id; + compile_ids[i] = compile_rules[i]->compile_id; } return MIN(bool_match_ret, compile_ids_size); +} + +int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, + int *hit_item_ids, size_t hit_item_cnt, + int *group_ids, size_t group_ids_size, + size_t *n_hit_group_id, struct maat_state *state) +{ + struct maat_item *item = NULL; + size_t hit_group_cnt = 0; + + void *g2g_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, + state->maat_instance->g2g_table_id); + if (NULL == g2g_rt) { + return -1; + } + + for (size_t i = 0; i < hit_item_cnt; i++) { + HASH_FIND_INT(item_hash, &(hit_item_ids[i]), item); + assert(item != NULL); + if (!item) { + // should not come here + continue; + } + + if (hit_group_cnt >= group_ids_size) { + hit_group_cnt = group_ids_size; + //Prevent group_id_array out of bounds + } else { + group_ids[hit_group_cnt++] = item->group_id; + } + + // update hit path + maat_compile_state_update_hit_path(state->compile_state, hit_item_ids[i], + item->group_id, vtable_id, state->scan_cnt, i); + } + + *n_hit_group_id = hit_group_cnt; + + /* update hit clause */ + int compile_table_id = -1; + if (state->compile_table_id == -1) { + compile_table_id = state->maat_instance->default_compile_table_id; + } else { + compile_table_id = state->compile_table_id; + } + + void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, + compile_table_id); + enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, + compile_table_id); + assert(table_type == TABLE_TYPE_COMPILE); + + for (size_t i = 0; i < hit_group_cnt; i++) { + int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i], + 1, top_group_ids); + for (int j = 0; j < top_group_cnt; j++) { + maat_compile_state_update_hit_clause(state->compile_state, compile_rt, + top_group_ids[j], vtable_id); + } + } + + return 0; } \ No newline at end of file diff --git a/src/maat_config_monitor.cpp b/src/maat_config_monitor.cpp index 0a44363..3c19e56 100644 --- a/src/maat_config_monitor.cpp +++ b/src/maat_config_monitor.cpp @@ -16,7 +16,6 @@ #include #include -#include "utils.h" #include "maat_utils.h" #include "maat_rule.h" #include "json2iris.h" @@ -84,7 +83,8 @@ const char *path2filename(const char *path) return path + i + 1; } -char *read_nxt_line_from_buff(const char *buff, size_t buff_size, size_t *offset, char *line, int line_size) +char *read_nxt_line_from_buff(const char *buff, size_t buff_size, + size_t *offset, char *line, int line_size) { int this_offset=0; const char* p; @@ -221,7 +221,7 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***idx_path, size_t *idx_num, struct log_handle *logger) { struct dirent **namelist = NULL; - int update_type = CM_UPDATE_TYPE_NONE; + int update_type = MAAT_UPDATE_TYPE_NONE; int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort); if (n < 0) { @@ -285,7 +285,7 @@ int get_new_idx_path(long long current_version, const char *file_dir, (*idx_path)[0] = (char *)malloc(path_len); snprintf((*idx_path)[0], path_len, "%s/%s", file_dir, namelist[full_file_idx]->d_name); *idx_num = 1; - update_type = CM_UPDATE_TYPE_FULL; + update_type = MAAT_UPDATE_TYPE_FULL; } else if (latest_inc_version > current_version) { //inc update,it's possible that do inc after full update in this function,but we'll process it at next loop. *idx_path = (char **)malloc(sizeof(char **) * inc_idx_num); @@ -295,9 +295,9 @@ int get_new_idx_path(long long current_version, const char *file_dir, snprintf((*idx_path)[i], path_len, "%s/%s", file_dir, namelist[inc_file_idx[i]]->d_name); } *idx_num = inc_idx_num; - update_type = CM_UPDATE_TYPE_INC; + update_type = MAAT_UPDATE_TYPE_INC; } else { - update_type = CM_UPDATE_TYPE_NONE; + update_type = MAAT_UPDATE_TYPE_NONE; } FREE(inc_file_idx); @@ -324,7 +324,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir, memset(table_array, 0, sizeof(table_array)); int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num, logger); - if (update_type != CM_UPDATE_TYPE_NONE) { + if (update_type != MAAT_UPDATE_TYPE_NONE) { for (i = 0; i < idx_path_num; i++) { log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]); int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM); @@ -370,12 +370,15 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, size_t uncompressed_buff_sz = 0; log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, - "Maat initial with JSON file %s, formating...", json_filename); + "Maat initial with JSON file %s, formating...", + json_filename); if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) { - ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo, - (unsigned char **)&decrypted_buff, &decrypted_buff_sz, - err_str, err_str_sz); + ret = decrypt_open(json_filename, maat_instance->decrypt_key, + maat_instance->decrypt_algo, + (unsigned char **)&decrypted_buff, + &decrypted_buff_sz, + err_str, err_str_sz); if (ret < 0) { log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, "Decrypt Maat JSON file %s failed", json_filename); @@ -387,7 +390,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, } if (maat_instance->maat_json_is_gzipped) { - ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz); + ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, + &uncompressed_buff_sz); FREE(json_buff); if (ret < 0) { log_error(maat_instance->logger, MODULE_CONFIG_MONITOR, @@ -410,7 +414,8 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, } ret = json2iris((const char*)json_buff, json_filename, NULL, - maat_instance->json_ctx.iris_file, sizeof(maat_instance->json_ctx.iris_file), + maat_instance->json_ctx.iris_file, + sizeof(maat_instance->json_ctx.iris_file), strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL, maat_instance->logger); @@ -422,17 +427,20 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, } if (!maat_instance->is_running) { - strncpy(maat_instance->json_ctx.json_file, json_filename, sizeof(maat_instance->json_ctx.json_file)); + strncpy(maat_instance->json_ctx.json_file, json_filename, + sizeof(maat_instance->json_ctx.json_file)); } ret = stat(json_filename, &fstat_buf); maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim; md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5); - log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, "JSON file %s md5: %s, generate index file %s OK", - maat_instance->json_ctx.json_file, - maat_instance->json_ctx.effective_json_md5, - maat_instance->json_ctx.iris_file); + log_info(maat_instance->logger, MODULE_CONFIG_MONITOR, + "JSON file %s md5: %s, generate index file %s OK", + maat_instance->json_ctx.json_file, + maat_instance->json_ctx.effective_json_md5, + maat_instance->json_ctx.iris_file); + maat_instance->input_mode = DATA_SOURCE_JSON_FILE; return 0; diff --git a/src/maat_ex_data.cpp b/src/maat_ex_data.cpp index 8b19f1c..0663c57 100644 --- a/src/maat_ex_data.cpp +++ b/src/maat_ex_data.cpp @@ -15,7 +15,6 @@ #include "uthash/utarray.h" #include "log/log.h" #include "rcu_hash.h" -#include "utils.h" #include "maat_utils.h" #include "maat_ex_data.h" @@ -48,8 +47,9 @@ void cache_row_free(void *p) UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free}; -struct ex_data_runtime *ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, - struct log_handle *logger) +struct ex_data_runtime * +ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, + struct log_handle *logger) { struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1); @@ -149,26 +149,31 @@ void ex_data_schema_free(struct ex_data_schema *ex_schema) FREE(ex_schema); } -void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema) +void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, + struct ex_data_schema *schema) { ex_data_rt->ex_schema = schema; } -void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, struct ex_container_ctx *container_ctx) +void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, + struct ex_container_ctx *container_ctx) { rcu_hash_set_user_ctx(ex_data_rt->htable, container_ctx); } -struct ex_container_ctx *ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt) +struct ex_container_ctx * +ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt) { return (struct ex_container_ctx *)rcu_hash_get_user_ctx(ex_data_rt->htable); } -void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, const char *key, size_t key_len) +void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, + const char *key, size_t key_len) { void *ex_data = NULL; struct ex_data_schema *ex_schema = ex_data_rt->ex_schema; - ex_schema->new_func(ex_data_rt->table_id, key, row, &ex_data, ex_schema->argl, ex_schema->argp); + ex_schema->new_func(ex_data_rt->table_id, key, row, &ex_data, + ex_schema->argl, ex_schema->argp); return ex_data; } @@ -194,11 +199,14 @@ void ex_data_container_free(void *ctx, void *data) void *argp = container_ctx->ex_schema->argp; struct ex_data_container *ex_container = (struct ex_data_container *)data; - if (ex_container->ex_data != NULL && container_ctx->ex_schema->free_func != NULL) { - container_ctx->ex_schema->free_func(container_ctx->table_id, &(ex_container->ex_data), argl, argp); + if (ex_container->ex_data != NULL + && container_ctx->ex_schema->free_func != NULL) { + container_ctx->ex_schema->free_func(container_ctx->table_id, + &(ex_container->ex_data), argl, argp); } - if (ex_container->custom_data != NULL && container_ctx->custom_data_free != NULL) { + if (ex_container->custom_data != NULL + && container_ctx->custom_data_free != NULL) { container_ctx->custom_data_free(ex_container->custom_data); } @@ -211,10 +219,12 @@ int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, { struct ex_data_container *tmp_container = NULL; - tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, + key, key_len); if (tmp_container != NULL) { log_error(ex_data_rt->logger, MODULE_EX_DATA, - "ex_data_runtime add ex container error: already exist same key:%s", key); + "ex_data_runtime add ex container error: already exist same key:%s", + key); return -1; } @@ -227,10 +237,12 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) { struct ex_data_container *tmp_container = NULL; - tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, + key, key_len); if (NULL == tmp_container) { log_error(ex_data_rt->logger, MODULE_EX_DATA, - "ex_data_runtime del ex container error: no such key:%s", key); + "ex_data_runtime del ex container error: no such key:%s", + key); return -1; } @@ -239,22 +251,30 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, return 0; } -void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) +void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len) { - struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + struct ex_data_container *ex_container = NULL; + ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, + key, key_len); if (NULL == ex_container) { return NULL; } void *dup_ex_data = NULL; - ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, &(ex_container->ex_data), - ex_data_rt->ex_schema->argl, ex_data_rt->ex_schema->argp); + ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, + &(ex_container->ex_data), + ex_data_rt->ex_schema->argl, + ex_data_rt->ex_schema->argp); return dup_ex_data; } -void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) +void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, + const char *key, size_t key_len) { - struct ex_data_container *ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); + struct ex_data_container *ex_container = NULL; + ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, + key, key_len); if (NULL == ex_container) { return NULL; } @@ -267,12 +287,8 @@ size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt) return rcu_hash_count(ex_data_rt->htable); } -size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, struct ex_data_container ***ex_container) +size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, + struct ex_data_container ***ex_container) { return rcu_hash_list_updating_data(ex_data_rt->htable, (void ***)ex_container); -} - -int ex_data_runtime_updating_flag(struct ex_data_runtime *ex_data_rt) -{ - return rcu_hash_updating_flag(ex_data_rt->htable); } \ No newline at end of file diff --git a/src/maat_expr.cpp b/src/maat_expr.cpp index 1c2efd8..20b9170 100644 --- a/src/maat_expr.cpp +++ b/src/maat_expr.cpp @@ -11,7 +11,6 @@ #include #include -#include "utils.h" #include "maat_expr.h" #include "adapter_hs.h" #include "maat_utils.h" @@ -20,6 +19,8 @@ #include "rcu_hash.h" #include "maat_rule.h" #include "maat_compile.h" +#include "maat_group.h" +#include "alignment.h" #include "maat_garbage_collection.h" #define MAX_DISTRICT_STR 128 @@ -36,6 +37,7 @@ struct expr_schema { int is_hexbin_column; enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ int table_id; //ugly + struct table_manager *ref_tbl_mgr; }; enum expr_type { @@ -56,7 +58,7 @@ enum match_method { struct expr_item { int item_id; int group_id; - char district[MAX_DISTRICT_STR]; + int district_id; char keywords[MAX_KEYWORDS_STR]; enum expr_type expr_type; enum match_method match_method; @@ -80,9 +82,8 @@ struct expr_runtime { struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; - // long long *scan_cnt; - // long long *hit_cnt; - // long long *not_grp_hit_cnt; + long long *scan_cnt; + long long *hit_cnt; // long long *stream_num; }; @@ -139,11 +140,13 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem int db_hexbin = -1; int expr_type = -1; int match_method_type = -1; + enum table_type table_type = TABLE_TYPE_INVALID; struct expr_item *expr_item = ALLOC(struct expr_item, 1); int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no item_id", + log_error(logger, MODULE_EXPR, + "expr table(table_id:%d) line:%s has no item_id", expr_schema->table_id, line); goto error; } @@ -151,32 +154,40 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no group_id", + log_error(logger, MODULE_EXPR, + "expr table(table_id:%d) line:%s has no group_id", expr_schema->table_id, line); goto error; } expr_item->group_id = atoi(line + column_offset); + table_type = table_manager_get_table_type(expr_schema->tbl_mgr, expr_schema->table_id); //TODO - #if 0 - if (table_item->table_type == TABLE_TYPE_EXPR_PLUS) { + #if 1 + if (table_type == TABLE_TYPE_EXPR_PLUS) { ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); if (ret < 0) { - return -1; + goto error; } if (column_len >= MAX_DISTRICT_STR) { log_error(logger, MODULE_EXPR, - "update error: expr table[%s]:item_id[%d] district length too long", - table_name, expr_item->item_id); - return -1; + "expr table(table_id:%d) line:%s district length too long", + expr_schema->table_id, line); + goto error; } - memcpy(expr_item->district, (line + column_offset), column_len); + + char district[MAX_DISTRICT_STR] = {0}; + memcpy(district, (line + column_offset), column_len); + assert(strlen(district) > 0); + str_unescape(district); + expr_item->district_id = get_district_id() } #endif ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no keywords", + log_error(logger, MODULE_EXPR, + "expr table(table_id:%d) line:%s has no keywords", expr_schema->table_id, line); goto error; } @@ -191,7 +202,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no expr_type", + log_error(logger, MODULE_EXPR, + "expr table(table_id:%d) line:%s has no expr_type", expr_schema->table_id, line); goto error; } @@ -201,7 +213,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no match_method", + log_error(logger, MODULE_EXPR, + "expr table(table_id:%d) line:%s has no match_method", expr_schema->table_id, line); goto error; } @@ -211,7 +224,8 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_EXPR, "expr table(table_id:%d) line:%s has no is_hexbin", + log_error(logger, MODULE_EXPR, + "expr table(table_id:%d) line:%s has no is_hexbin", expr_schema->table_id, line); goto error; } @@ -248,7 +262,8 @@ void expr_item_free(struct expr_item *expr_item) FREE(expr_item); } -void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) { int read_cnt = 0; struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1); @@ -267,15 +282,18 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { - log_error(logger, MODULE_EXPR, "table %s has no custom column", table_name); + log_error(logger, MODULE_EXPR, + "table %s has no custom column", table_name); goto error; } custom_item = cJSON_GetObjectItem(item, "scan_mode"); if (custom_item != NULL && custom_item->type == cJSON_String) { - ret = maat_kv_read(scan_mode_map, custom_item->valuestring, (int*)&(expr_schema->scan_mode)); + ret = maat_kv_read(scan_mode_map, custom_item->valuestring, + (int*)&(expr_schema->scan_mode)); if (ret < 0) { - log_error(logger, MODULE_EXPR, "scan_mode %s illegal", custom_item->valuestring); + log_error(logger, MODULE_EXPR, "scan_mode %s illegal", + custom_item->valuestring); goto error; } read_cnt++; @@ -323,6 +341,8 @@ void *expr_schema_new(cJSON *json, const char *table_name, struct log_handle *lo read_cnt++; } + expr_schema->tbl_mgr = tbl_mgr; + if (read_cnt < 8) { goto error; } @@ -359,7 +379,8 @@ void expr_ex_data_free(void *user_ctx, void *data) expr_rule_free(expr_rule); } -void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *expr_runtime_new(void *expr_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == expr_schema) { @@ -376,10 +397,8 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num, struct maat_garbag expr_rt->ref_garbage_bin = garbage_bin; expr_rt->logger = logger; - // expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); - // expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); - // expr_rt->not_grp_hit_cnt = alignment_int64_array_alloc(max_thread_num); - // expr_rt->stream_num = alignment_int64_array_alloc(max_thread_num); + expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); + expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); return expr_rt; } @@ -411,11 +430,23 @@ void expr_runtime_free(void *expr_runtime) HASH_DELETE(hh, expr_rt->item_hash, item); maat_item_free(item, expr_rt->item_user_data_free); } + + if (expr_rt->hit_cnt != NULL) { + alignment_int64_array_free(expr_rt->hit_cnt); + expr_rt->hit_cnt = NULL; + } + + if (expr_rt->scan_cnt != NULL) { + alignment_int64_array_free(expr_rt->scan_cnt); + expr_rt->scan_cnt = NULL; + } + FREE(expr_rt); } -int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len, - and_expr_t *expr_rule, int is_valid, struct log_handle *logger) +int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, + size_t key_len, and_expr_t *expr_rule, + int is_valid, struct log_handle *logger) { void *data = NULL; @@ -424,7 +455,8 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_ data = rcu_hash_find(expr_rt->htable, key, key_len); if (NULL == data) { log_error(logger, MODULE_EXPR, - "the key of expr rule not exist, can't be deleted, expr_id:%d", expr_rule->expr_id); + "the key of expr rule not exist, can't be deleted, expr_id:%d", + expr_rule->expr_id); return -1; } rcu_hash_del(expr_rt->htable, key, key_len); @@ -433,7 +465,8 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_ data = rcu_hash_find(expr_rt->htable, key, key_len); if (data != NULL) { log_error(logger, MODULE_EXPR, - "the key of expr rule already exist, can't be added, expr_id:%d", expr_rule->expr_id); + "the key of expr rule already exist, can't be added, expr_id:%d", + expr_rule->expr_id); return -1; } rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule); @@ -461,7 +494,8 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type) } #define MAAT_MAX_EXPR_ITEM_NUM 8 -and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handle *logger) +and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, + struct log_handle *logger) { size_t i = 0; size_t sub_expr_cnt = 0; @@ -481,7 +515,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl if (i >= MAAT_MAX_EXPR_ITEM_NUM) { log_error(logger, MODULE_EXPR, - "expr item_id:%d too many patterns", expr_item->item_id); + "expr item_id:%d too many patterns", + expr_item->item_id); return NULL; } @@ -515,7 +550,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, struct log_handl return expr_rule; } -int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column) +int expr_runtime_update(void *expr_runtime, void *expr_schema, + const char *line, int valid_column) { if (NULL == expr_runtime || NULL == expr_schema) { return -1; @@ -553,7 +589,8 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, HASH_FIND_INT(expr_rt->item_hash, &item_id, item); if (item) { log_error(expr_rt->logger, MODULE_EXPR, - "expr runtime add item %d to item_hash failed, already exist", item_id); + "expr runtime add item %d to item_hash failed, already exist", + item_id); return -1; } @@ -572,14 +609,16 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger); expr_item_free(expr_item); if (NULL == expr_rule) { - log_error(expr_rt->logger, MODULE_EXPR, "transform expr table(table_id:%d) item to expr_rule failed, item_id:%d", + log_error(expr_rt->logger, MODULE_EXPR, + "transform expr table(table_id:%d) item to expr_rule failed, item_id:%d", schema->table_id, item_id); return -1; } } char *key = (char *)&item_id; - ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, is_valid, expr_rt->logger); + ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, + is_valid, expr_rt->logger); if (ret < 0) { if (expr_rule != NULL) { expr_rule_free(expr_rule); @@ -622,20 +661,25 @@ int expr_runtime_commit(void *expr_runtime) struct adapter_hs *old_adapter_hs = NULL; log_info(expr_rt->logger, MODULE_EXPR, - "committing %zu expr rules for rebuilding adapter_hs engine", rule_cnt); - new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger); + "committing %zu expr rules for rebuilding adapter_hs engine", + rule_cnt); + new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, + expr_rt->n_worker_thread, + rules, rule_cnt, + expr_rt->logger); if (NULL == new_adapter_hs) { log_error(expr_rt->logger, MODULE_EXPR, - "rebuild adapter_hs engine failed when update %zu expr rules", rule_cnt); + "rebuild adapter_hs engine failed when update %zu expr rules", + rule_cnt); ret = -1; } old_adapter_hs = expr_rt->hs; expr_rt->hs = new_adapter_hs; - maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy); + maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, + (void (*)(void*))adapter_hs_destroy); rcu_hash_commit(expr_rt->htable); - expr_rt->rule_num = rcu_hash_count(expr_rt->htable); rule_cnt = rcu_hash_updating_count(expr_rt->htable); assert(rule_cnt == 0); @@ -645,71 +689,37 @@ int expr_runtime_commit(void *expr_runtime) return ret; } -int expr_runtime_updating_flag(void *expr_runtime) +int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, + const char *data, size_t data_len, + int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state) { - struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; - return rcu_hash_updating_flag(expr_rt->htable); -} - -int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len, - int group_id_array[], size_t n_group_id_array, int virtual_table_id, struct maat_state *state) -{ - if (NULL == expr_rt) { - return -1; - } - int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; size_t n_hit_item = 0; int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, hit_item_ids, &n_hit_item); if (ret < 0) { return -1; } + + if (0 == n_hit_item) { + return 0; + } + if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { + log_info(expr_rt->logger, MODULE_EXPR, + "hit expr item count:%d exceed maxium:%d", + n_hit_item, MAX_SCANNER_HIT_ITEM_NUM); n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; } - - struct maat_compile_state *compile_state = state->compile_mid; - //tranform item_id to group_id - struct maat_item *item = NULL; - size_t n_group_id = 0; - size_t i = 0; - for (i = 0; i < n_hit_item; i++) { - HASH_FIND_INT(expr_rt->item_hash, &(hit_item_ids[i]), item); - assert(item != NULL); - if (!item) { - // should not come here - continue; - } - - if (n_group_id >= n_group_id_array) { - n_group_id = n_group_id_array; - //Prevent group_id_array out of bounds - } else { - group_id_array[n_group_id++] = item->group_id; - } - - maat_compile_state_update_hit_path(compile_state, hit_item_ids[i], item->group_id, virtual_table_id, state->scan_cnt, i); - } - - // literal_id{group_id,vt_id} to clause_id - // STEP 1: get compile table runtime - int compile_table_id = -1; - if (state->compile_table_id == -1) { - compile_table_id = state->maat_instance->default_compile_table_id; - } else { - compile_table_id = state->compile_table_id; - } - - void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id); - enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id); - assert(table_type == TABLE_TYPE_COMPILE); - // STEP 2: get the specified compile table's hit clause_id array by literal_id - for (i = 0; i < n_group_id; i++) { - maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[i], virtual_table_id); + size_t group_hit_cnt = 0; + ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, n_hit_item, + group_ids, group_ids_size, &group_hit_cnt, state); + if (ret < 0) { + return -1; } - return n_group_id; + return group_hit_cnt; } void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) @@ -722,14 +732,15 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) expr_rt->hs_stream = hs_stream; } -int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, size_t data_len, - int result[], size_t *n_result) +int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, + size_t data_len, int result[], size_t *n_result) { if (NULL == expr_rt) { return -1; } - return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, result, n_result); + return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, + result, n_result); } void expr_runtime_stream_close(struct expr_runtime *expr_rt) @@ -739,3 +750,13 @@ void expr_runtime_stream_close(struct expr_runtime *expr_rt) expr_rt->hs_stream = NULL; } } + +void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id) +{ + alignment_int64_array_add(expr_rt->hit_cnt, thread_id, 1); +} + +long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread) +{ + return alignment_int64_array_sum(expr_rt->hit_cnt, n_thread); +} \ No newline at end of file diff --git a/src/maat_garbage_collection.cpp b/src/maat_garbage_collection.cpp index 2844518..d778a35 100644 --- a/src/maat_garbage_collection.cpp +++ b/src/maat_garbage_collection.cpp @@ -8,13 +8,13 @@ *********************************************************************************************** */ -#include "utils.h" - #include #include #include #include +#include "maat_utils.h" + struct maat_garbage_bag { time_t create_time; int timeout; @@ -60,7 +60,8 @@ size_t maat_garbage_bin_get_size(struct maat_garbage_bin* bin) return bin->bag_cnt; } -void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void (* func)(void *)) +void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, + void (* func)(void *)) { struct maat_garbage_bag *bag = ALLOC(struct maat_garbage_bag, 1); diff --git a/src/maat_group.cpp b/src/maat_group.cpp index 7d86298..f9b62fe 100644 --- a/src/maat_group.cpp +++ b/src/maat_group.cpp @@ -12,7 +12,6 @@ #include #include "maat_group.h" -#include "utils.h" #include "maat_utils.h" #include "uthash/uthash.h" #include "igraph/igraph.h" @@ -22,23 +21,28 @@ struct group2group_item { int group_id; - int superior_group_id; + int super_group_id; }; struct group2group_schema { int group_id_column; - int superior_group_id_column; + int super_group_id_column; int table_id;//ugly + struct table_manager *tbl_mgr; }; -struct group2group_runtime { - struct maat_group_topology *group_topo; +struct maat_group { + igraph_integer_t vertex_id; + int group_id; + int ref_by_compile_cnt; + int ref_by_super_group_cnt; + int ref_by_sub_group_cnt; - uint32_t rule_num; - uint32_t updating_rule_num; + size_t top_group_cnt; + int *top_group_ids; - struct maat_garbage_bin *ref_garbage_bin; - struct log_handle *logger; + UT_hash_handle hh_group_id; + UT_hash_handle hh_vertex_id; }; struct maat_group_topology { @@ -53,7 +57,18 @@ struct maat_group_topology { struct log_handle *logger; }; -void *group2group_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +struct group2group_runtime { + struct maat_group_topology *group_topo; + + uint32_t rule_num; + uint32_t updating_rule_num; + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; +}; + +void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) { int read_cnt = 0; struct group2group_schema *g2g_schema = ALLOC(struct group2group_schema, 1); @@ -67,7 +82,8 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { - log_error(logger, MODULE_GROUP, "table %s has no custom column", table_name); + log_error(logger, MODULE_GROUP, + "table %s has no custom column", table_name); goto error; } @@ -77,12 +93,14 @@ void *group2group_schema_new(cJSON *json, const char *table_name, struct log_han read_cnt++; } - custom_item = cJSON_GetObjectItem(item, "superior_group_id"); + custom_item = cJSON_GetObjectItem(item, "super_group_id"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - g2g_schema->superior_group_id_column = custom_item->valueint; + g2g_schema->super_group_id_column = custom_item->valueint; read_cnt++; } + g2g_schema->tbl_mgr = tbl_mgr; + if (read_cnt < 3) { goto error; } @@ -117,7 +135,8 @@ struct maat_group_topology *maat_group_topology_new(struct log_handle *logger) return group_topo; } -void *group2group_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *group2group_runtime_new(void *g2g_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { struct group2group_runtime *g2g_rt = ALLOC(struct group2group_runtime, 1); @@ -167,6 +186,16 @@ void group2group_runtime_free(void *g2g_runtime) FREE(g2g_rt); } +void maat_group_ref_inc(struct maat_group *group) +{ + group->ref_by_compile_cnt++; +} + +void maat_group_ref_dec(struct maat_group *group) +{ + group->ref_by_compile_cnt--; +} + struct group2group_item * group2group_item_new(const char *line, struct group2group_schema *g2g_schema, struct log_handle *logger) @@ -184,14 +213,15 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema, } g2g_item->group_id = atoi(line + column_offset); - ret = get_column_pos(line, g2g_schema->superior_group_id_column, &column_offset, &column_len); + ret = get_column_pos(line, g2g_schema->super_group_id_column, + &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_GROUP, - "group2group table(table_id:%d) line:%s has no superior_group_id", + "group2group table(table_id:%d) line:%s has no super_group_id", g2g_schema->table_id, line); goto error; } - g2g_item->superior_group_id = atoi(line + column_offset); + g2g_item->super_group_id = atoi(line + column_offset); return g2g_item; error: @@ -251,13 +281,14 @@ void group2group_runtime_remove_group(void *g2g_runtime, struct maat_group *grou struct maat_group_topology *group_topo = g2g_rt->group_topo; assert(group_topo != NULL); - assert(group->ref_by_compile_cnt == 0 && group->ref_by_superior_group_cnt == 0); + assert(group->ref_by_compile_cnt == 0 && group->ref_by_super_group_cnt == 0); igraph_vector_init(&v, 8); igraph_neighbors(&group_topo->group_graph, &v, group->vertex_id, IGRAPH_ALL); if (igraph_vector_size(&v) > 0) { print_igraph_vector(&v, buff, sizeof(buff)); - log_error(group_topo->logger, MODULE_GROUP, "Del group %d exception, still reached by %s.", - group->vertex_id, buff); + log_error(group_topo->logger, MODULE_GROUP, + "Del group %d exception, still reached by %s.", + group->vertex_id, buff); assert(0); } igraph_vector_destroy(&v); @@ -288,7 +319,8 @@ struct maat_group *group2group_runtime_find_group(void *g2g_runtime, int group_i return group; } -int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int superior_group_id) +int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, + int super_group_id) { if (NULL == g2g_runtime) { return -1; @@ -305,30 +337,34 @@ int group2group_runtime_add_group_to_group(void *g2g_runtime, int group_id, int group = group2group_runtime_add_group(g2g_runtime, group_id); } - struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id); - if (NULL == superior_group) { - superior_group = group2group_runtime_add_group(g2g_runtime, superior_group_id); + struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime, + super_group_id); + if (NULL == super_group) { + super_group = group2group_runtime_add_group(g2g_runtime, super_group_id); } - ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, superior_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); + ret = igraph_get_eid(&group_topo->group_graph, &edge_id, group->vertex_id, + super_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0); //No duplicated edges between two groups. if (edge_id > 0) { log_error(g2g_rt->logger, MODULE_GROUP, "Add group %d to group %d failed, relation already exisited.", - group->group_id, superior_group->group_id); + group->group_id, super_group->group_id); ret = -1; } else { - igraph_add_edge(&group_topo->group_graph, group->vertex_id, superior_group->vertex_id); - group->ref_by_superior_group_cnt++; - superior_group->ref_by_subordinate_group_cnt++; + igraph_add_edge(&group_topo->group_graph, group->vertex_id, + super_group->vertex_id); + group->ref_by_super_group_cnt++; + super_group->ref_by_sub_group_cnt++; ret = 0; } return ret; } -int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, int superior_group_id) +int group2group_runtime_remove_group_from_group(void *g2g_runtime, + int group_id, int super_group_id) { if (NULL == g2g_runtime) { return -1; @@ -341,15 +377,16 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, if (NULL == group) { log_error(g2g_rt->logger, MODULE_GROUP, "Del group %d from group %d failed, group %d not exisited.", - group_id, superior_group_id, group_id); + group_id, super_group_id, group_id); return -1; } - struct maat_group *superior_group = group2group_runtime_find_group(g2g_runtime, superior_group_id); - if (NULL == superior_group) { + struct maat_group *super_group = group2group_runtime_find_group(g2g_runtime, + super_group_id); + if (NULL == super_group) { log_error(g2g_rt->logger, MODULE_GROUP, "Del group %d from group %d failed, superior group %d not exisited.", - group_id, superior_group_id, superior_group_id); + group_id, super_group_id, super_group_id); return -1; } @@ -364,7 +401,8 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, //first edge, the fifth is the first vertex of the second edge and so on. The last element //of the argument list must be -1 to denote the end of the argument list. //https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small - int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, superior_group->vertex_id, -1); + int ret = igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group->vertex_id, + super_group->vertex_id, -1); assert(ret==IGRAPH_SUCCESS); // ignore no such edge to abort(). igraph_set_error_handler(igraph_error_handler_ignore); @@ -377,8 +415,8 @@ int group2group_runtime_remove_group_from_group(void *g2g_runtime, int group_id, return -1; } - group->ref_by_superior_group_cnt--; - superior_group->ref_by_subordinate_group_cnt--; + group->ref_by_super_group_cnt--; + super_group->ref_by_sub_group_cnt--; return 0; } @@ -405,7 +443,7 @@ int group2group_runtime_build_top_groups(void *g2g_runtime) } struct maat_group *group = NULL, *tmp = NULL; - struct maat_group *superior_group = NULL; + struct maat_group *super_group = NULL; int tmp_vid=0; size_t top_group_cnt=0; int* temp_group_ids=NULL; @@ -426,10 +464,11 @@ int group2group_runtime_build_top_groups(void *g2g_runtime) HASH_ITER (hh_group_id, group_topo->hash_group_by_id, group, tmp) { top_group_cnt = 0; temp_group_ids = NULL; + //Orphan, Not reference by any one, free it. if (0 == group->ref_by_compile_cnt - && 0 == group->ref_by_superior_group_cnt - && 0 == group->ref_by_subordinate_group_cnt) { + && 0 == group->ref_by_super_group_cnt + && 0 == group->ref_by_sub_group_cnt) { FREE(group->top_group_ids); group2group_runtime_remove_group(g2g_runtime, group); @@ -437,8 +476,8 @@ int group2group_runtime_build_top_groups(void *g2g_runtime) } //A group is need to build top groups when it has items and referenced by superior groups or compiles. - if (group->ref_by_compile_cnt > 0 || group->ref_by_superior_group_cnt > 0) { - if (0 == group->ref_by_superior_group_cnt) { + if (group->ref_by_compile_cnt > 0 || group->ref_by_super_group_cnt > 0) { + if (0 == group->ref_by_super_group_cnt) { //fast path, group is only referenced by compile rules. top_group_cnt = 1; temp_group_ids = ALLOC(int, top_group_cnt); @@ -456,11 +495,12 @@ int group2group_runtime_build_top_groups(void *g2g_runtime) break; } - HASH_FIND(hh_vertex_id, group_topo->hash_group_by_vertex, &tmp_vid, sizeof(tmp_vid), superior_group); + HASH_FIND(hh_vertex_id, group_topo->hash_group_by_vertex, &tmp_vid, + sizeof(tmp_vid), super_group); //including itself - if (superior_group->ref_by_compile_cnt > 0) { - temp_group_ids[top_group_cnt] = superior_group->group_id; + if (super_group->ref_by_compile_cnt > 0) { + temp_group_ids[top_group_cnt] = super_group->group_id; top_group_cnt++; } } @@ -479,10 +519,11 @@ int group2group_runtime_build_top_groups(void *g2g_runtime) return 0; } -int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, - int valid_column) +int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, + const char *line, int valid_column) { - if (NULL == g2g_runtime || NULL == g2g_schema || NULL == line) { + if (NULL == g2g_runtime || NULL == g2g_schema || + NULL == line) { return -1; } @@ -501,10 +542,12 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char * if (0 == is_valid) { //delete - ret = group2group_runtime_remove_group_from_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id); + ret = group2group_runtime_remove_group_from_group(g2g_runtime, g2g_item->group_id, + g2g_item->super_group_id); } else { //add - ret = group2group_runtime_add_group_to_group(g2g_runtime, g2g_item->group_id, g2g_item->superior_group_id); + ret = group2group_runtime_add_group_to_group(g2g_runtime, g2g_item->group_id, + g2g_item->super_group_id); } group2group_item_free(g2g_item); @@ -520,8 +563,8 @@ int group2group_runtime_commit(void *g2g_runtime) return group2group_runtime_build_top_groups(g2g_runtime); } -int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t n_group_ids, - int *top_group_ids) +int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, + size_t n_group_ids, int *top_group_ids) { if (NULL == g2g_runtime || NULL == group_ids || 0 == n_group_ids) { return -1; @@ -540,4 +583,4 @@ int group2group_runtime_get_top_groups(void *g2g_runtime, int *group_ids, size_t } return top_group_index; -} \ No newline at end of file +} diff --git a/src/maat_ip.cpp b/src/maat_ip.cpp index ce047c9..4fbceaf 100644 --- a/src/maat_ip.cpp +++ b/src/maat_ip.cpp @@ -11,16 +11,15 @@ #include #include -#include "utils.h" #include "log/log.h" #include "cJSON/cJSON.h" -#include "utils.h" #include "maat_utils.h" #include "maat_ex_data.h" #include "IPMatcher.h" #include "maat_ip.h" #include "maat_rule.h" #include "maat_compile.h" +#include "alignment.h" #include "maat_garbage_collection.h" #define MODULE_IP module_name_str("maat.ip") @@ -30,7 +29,7 @@ struct port_range { uint16_t max_port; }; -struct ip_plus_schema { +struct ip_schema { int item_id_column; int group_id_column; int addr_type_column; @@ -40,15 +39,10 @@ struct ip_plus_schema { int sport_format_column; int sport1_column; int sport2_column; - int daddr_format_column; - int dip1_column; - int dip2_column; - int dport_format_column; - int dport1_column; - int dport2_column; int proto_column; int direction_column; int table_id; //ugly + struct table_manager *tbl_mgr; }; struct ipv4_item_rule { @@ -69,7 +63,7 @@ struct ipv6_item_rule { uint16_t direction; /* 方向，0表示双向，1表示单向 */ }; -struct ip_plus_item { +struct ip_item { int item_id; int group_id; int addr_type; @@ -79,7 +73,7 @@ struct ip_plus_item { }; }; -struct ip_plus_runtime { +struct ip_runtime { struct ip_matcher* ip_matcher; struct ex_data_runtime* ex_data_rt; @@ -90,187 +84,174 @@ struct ip_plus_runtime { struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + + long long *scan_cnt; + long long *hit_cnt; }; -void *ip_plus_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) { size_t read_cnt = 0; - struct ip_plus_schema *ip_plus_schema = ALLOC(struct ip_plus_schema, 1); + struct ip_schema *ip_schema = ALLOC(struct ip_schema, 1); cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); if (item != NULL && item->type == cJSON_Number) { - ip_plus_schema->table_id = item->valueint; + ip_schema->table_id = item->valueint; read_cnt++; } item = cJSON_GetObjectItem(json, "custom"); if (NULL == item || item->type != cJSON_Object) { - log_error(logger, MODULE_IP, "ip_plus table %s has no custom column", table_name); + log_error(logger, MODULE_IP, + "ip table %s has no custom column", table_name); goto error; } custom_item = cJSON_GetObjectItem(item, "item_id"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->item_id_column = custom_item->valueint; + ip_schema->item_id_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "group_id"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->group_id_column = custom_item->valueint; + ip_schema->group_id_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "addr_type"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->addr_type_column = custom_item->valueint; + ip_schema->addr_type_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "saddr_format"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->saddr_format_column = custom_item->valueint; + ip_schema->saddr_format_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "sip1"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->sip1_column = custom_item->valueint; + ip_schema->sip1_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "sip2"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->sip2_column = custom_item->valueint; + ip_schema->sip2_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "sport_format"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->sport_format_column = custom_item->valueint; + ip_schema->sport_format_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "sport1"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->sport1_column = custom_item->valueint; + ip_schema->sport1_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "sport2"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->sport2_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "daddr_format"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->daddr_format_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "dip1"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->dip1_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "dip2"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->dip2_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "dport_format"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->dport_format_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "dport1"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->dport1_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "dport2"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->dport2_column = custom_item->valueint; + ip_schema->sport2_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "proto"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->proto_column = custom_item->valueint; + ip_schema->proto_column = custom_item->valueint; read_cnt++; } custom_item = cJSON_GetObjectItem(item, "direction"); if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_plus_schema->direction_column = custom_item->valueint; + ip_schema->direction_column = custom_item->valueint; read_cnt++; } - if (read_cnt < 18) { + ip_schema->tbl_mgr = tbl_mgr; + + if (read_cnt < 12) { goto error; } - return ip_plus_schema; + return ip_schema; error: - FREE(ip_plus_schema); + FREE(ip_schema); return NULL; } -void ip_plus_schema_free(void *ip_plus_schema) +void ip_schema_free(void *ip_schema) { - FREE(ip_plus_schema); + FREE(ip_schema); } -void *ip_plus_runtime_new(void *ip_plus_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, - struct log_handle *logger) +void *ip_runtime_new(void *ip_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) { - if (NULL == ip_plus_schema) { + if (NULL == ip_schema) { return NULL; } - struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema; - struct ip_plus_runtime *ip_plus_rt = ALLOC(struct ip_plus_runtime, 1); + struct ip_schema *schema = (struct ip_schema *)ip_schema; + struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1); - ip_plus_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger); - ip_plus_rt->item_user_data_free = maat_item_inner_free; - ip_plus_rt->ref_garbage_bin = garbage_bin; - ip_plus_rt->logger = logger; + ip_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, + ex_data_container_free, + logger); + ip_rt->item_user_data_free = maat_item_inner_free; + ip_rt->ref_garbage_bin = garbage_bin; + ip_rt->logger = logger; - return ip_plus_rt; + ip_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); + ip_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + + return ip_rt; } -void ip_plus_runtime_free(void *ip_plus_runtime) +void ip_runtime_free(void *ip_runtime) { - if (NULL == ip_plus_runtime) { + if (NULL == ip_runtime) { return; } - struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; - if (ip_plus_rt->ip_matcher != NULL) { - ip_matcher_free(ip_plus_rt->ip_matcher); + struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; + if (ip_rt->ip_matcher != NULL) { + ip_matcher_free(ip_rt->ip_matcher); } - if (ip_plus_rt->ex_data_rt != NULL) { - ex_data_runtime_free(ip_plus_rt->ex_data_rt); + if (ip_rt->ex_data_rt != NULL) { + ex_data_runtime_free(ip_rt->ex_data_rt); } struct maat_item *item = NULL, *tmp_item = NULL; - HASH_ITER(hh, ip_plus_rt->item_hash, item, tmp_item) { - HASH_DELETE(hh, ip_plus_rt->item_hash, item); - maat_item_free(item, ip_plus_rt->item_user_data_free); + HASH_ITER(hh, ip_rt->item_hash, item, tmp_item) { + HASH_DELETE(hh, ip_rt->item_hash, item); + maat_item_free(item, ip_rt->item_user_data_free); } - FREE(ip_plus_rt); + if (ip_rt->hit_cnt != NULL) { + alignment_int64_array_free(ip_rt->hit_cnt); + ip_rt->hit_cnt = NULL; + } + + if (ip_rt->scan_cnt != NULL) { + alignment_int64_array_free(ip_rt->scan_cnt); + ip_rt->scan_cnt = NULL; + } + + FREE(ip_rt); } -struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *ip_plus_schema, - struct log_handle *logger) +struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema, + struct log_handle *logger) { size_t column_offset = 0; size_t column_len = 0; @@ -282,198 +263,209 @@ struct ip_plus_item *ip_plus_item_new(const char *line, struct ip_plus_schema *i uint16_t sport2 = 0; uint16_t protocol = 0; uint16_t direction = 0; - struct ip_plus_item *ip_plus_item = ALLOC(struct ip_plus_item, 1); + struct ip_item *ip_item = ALLOC(struct ip_item, 1); - int ret = get_column_pos(line, ip_plus_schema->item_id_column, &column_offset, &column_len); + int ret = get_column_pos(line, ip_schema->item_id_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no item_id", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip plus table(table_id:%d) line:%s has no item_id", + ip_schema->table_id, line); goto error; } - ip_plus_item->item_id = atoi(line + column_offset); + ip_item->item_id = atoi(line + column_offset); - ret = get_column_pos(line, ip_plus_schema->group_id_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->group_id_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no group_id", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip plus table(table_id:%d) line:%s has no group_id", + ip_schema->table_id, line); goto error; } - ip_plus_item->group_id = atoi(line + column_offset); + ip_item->group_id = atoi(line + column_offset); - ret = get_column_pos(line, ip_plus_schema->addr_type_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->addr_type_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip plus table(table_id:%d) line:%s has no addr_type", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip plus table(table_id:%d) line:%s has no addr_type", + ip_schema->table_id, line); goto error; } - ip_plus_item->addr_type = atoi(line + column_offset); + ip_item->addr_type = atoi(line + column_offset); - if (ip_plus_item->addr_type != 4 && ip_plus_item->addr_type != 6) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has invalid addr type:%d", - ip_plus_schema->table_id, line, ip_plus_item->addr_type); + if (ip_item->addr_type != IPv4 && ip_item->addr_type != IPv6) { + log_error(logger, MODULE_IP, + "ip table(table_id:%d) line:%s has invalid addr type:%d", + ip_schema->table_id, line, ip_item->addr_type); goto error; } - ret = get_column_pos(line, ip_plus_schema->saddr_format_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->saddr_format_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no saddr_format", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip table(table_id:%d) line:%s has no saddr_format", + ip_schema->table_id, line); goto error; } memcpy(saddr_format, (line + column_offset), column_len); if (IP_FORMAT_UNKNOWN == ip_format_str2int(saddr_format)) { log_error(logger, MODULE_IP, - "ip_plus table(table_id:%d) line:%s has invalid saddr_format, should be range/mask/CIDR", - ip_plus_schema->table_id, line); + "ip table(table_id:%d) line:%s has invalid saddr_format, should be range/mask/CIDR", + ip_schema->table_id, line); goto error; } - ret = get_column_pos(line, ip_plus_schema->sip1_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->sip1_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip1", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sip1", + ip_schema->table_id, line); goto error; } memcpy(sip1_str, (line + column_offset), column_len); - ret = get_column_pos(line, ip_plus_schema->sip2_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->sip2_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sip2", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sip2", + ip_schema->table_id, line); goto error; } memcpy(sip2_str, (line + column_offset), column_len); - ret = get_column_pos(line, ip_plus_schema->sport_format_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->sport_format_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport_format", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip table(table_id:%d) line:%s has no sport_format", + ip_schema->table_id, line); goto error; } memcpy(sport_format, (line + column_offset), column_len); if (IP_FORMAT_UNKNOWN == ip_format_str2int(sport_format)) { log_error(logger, MODULE_IP, - "ip_plus table(table_id:%d) line:%s has invalid sport_format, should be range/mask/CIDR", - ip_plus_schema->table_id, line); + "ip table(table_id:%d) line:%s has invalid sport_format, should be range/mask/CIDR", + ip_schema->table_id, line); goto error; } - ret = get_column_pos(line, ip_plus_schema->sport1_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->sport1_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport1", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sport1", + ip_schema->table_id, line); goto error; } sport1 = atoi(line + column_offset); - ret = get_column_pos(line, ip_plus_schema->sport2_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->sport2_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no sport2", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip table(table_id:%d) line:%s has no sport2", + ip_schema->table_id, line); goto error; } sport2 = atoi(line + column_offset); - if (4 == ip_plus_item->addr_type) { - ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, - &ip_plus_item->ipv4.min_sip, &ip_plus_item->ipv4.max_sip); + if (IPv4 == ip_item->addr_type) { + ret = ip_format2range(ip_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, + &ip_item->ipv4.min_sip, &ip_item->ipv4.max_sip); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip4) failed", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip table(table_id:%d) line:%s ip_format2range(ip4) failed", + ip_schema->table_id, line); goto error; } if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { - ip_plus_item->ipv4.min_sport = sport1 & sport2; - ip_plus_item->ipv4.max_sport = sport1 | ~sport2; + ip_item->ipv4.min_sport = sport1 & sport2; + ip_item->ipv4.max_sport = sport1 | ~sport2; } else { - ip_plus_item->ipv4.min_sport = sport1; - ip_plus_item->ipv4.max_sport = sport2; + ip_item->ipv4.min_sport = sport1; + ip_item->ipv4.max_sport = sport2; } - ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->proto_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip table(table_id:%d) line:%s has no proto", + ip_schema->table_id, line); goto error; } - ip_plus_item->ipv4.proto = atoi(line + column_offset); - protocol = ip_plus_item->ipv4.proto; + ip_item->ipv4.proto = atoi(line + column_offset); + protocol = ip_item->ipv4.proto; - ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->direction_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip table(table_id:%d) line:%s has no direction", + ip_schema->table_id, line); goto error; } - ip_plus_item->ipv4.direction = atoi(line + column_offset); - direction = ip_plus_item->ipv4.direction; + ip_item->ipv4.direction = atoi(line + column_offset); + direction = ip_item->ipv4.direction; } else { //ipv6 - ret = ip_format2range(ip_plus_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, - ip_plus_item->ipv6.min_sip, ip_plus_item->ipv6.max_sip); + ret = ip_format2range(ip_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, + ip_item->ipv6.min_sip, ip_item->ipv6.max_sip); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s ip_format2range(ip6) failed", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, + "ip table(table_id:%d) line:%s ip_format2range(ip6) failed", + ip_schema->table_id, line); goto error; } if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { - ip_plus_item->ipv6.min_sport = sport1 & sport2; - ip_plus_item->ipv6.max_sport = sport1 | ~sport2; + ip_item->ipv6.min_sport = sport1 & sport2; + ip_item->ipv6.max_sport = sport1 | ~sport2; } else { - ip_plus_item->ipv6.min_sport = sport1; - ip_plus_item->ipv6.max_sport = sport2; + ip_item->ipv6.min_sport = sport1; + ip_item->ipv6.max_sport = sport2; } - ret = get_column_pos(line, ip_plus_schema->proto_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->proto_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no proto", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no proto", + ip_schema->table_id, line); goto error; } - ip_plus_item->ipv6.proto = atoi(line + column_offset); - protocol = ip_plus_item->ipv6.proto; + ip_item->ipv6.proto = atoi(line + column_offset); + protocol = ip_item->ipv6.proto; - ret = get_column_pos(line, ip_plus_schema->direction_column, &column_offset, &column_len); + ret = get_column_pos(line, ip_schema->direction_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP, "ip_plus table(table_id:%d) line:%s has no direction", - ip_plus_schema->table_id, line); + log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no direction", + ip_schema->table_id, line); goto error; } - ip_plus_item->ipv6.direction = atoi(line + column_offset); - direction = ip_plus_item->ipv6.direction; + ip_item->ipv6.direction = atoi(line + column_offset); + direction = ip_item->ipv6.direction; } if (protocol > 65535 || protocol < 0) { log_error(logger, MODULE_IP, - "ip_plus table(table_id:%d) line:%s has invalid proto:%d", - ip_plus_schema->table_id, line, protocol); + "ip table(table_id:%d) line:%s has invalid proto:%d", + ip_schema->table_id, line, protocol); goto error; } if (direction != 0 && direction != 1) { log_error(logger, MODULE_IP, - "ip_plus table(table_id:%d) line:%s has invalid direction:%d", - ip_plus_schema->table_id, line, direction); + "ip table(table_id:%d) line:%s has invalid direction:%d", + ip_schema->table_id, line, direction); goto error; } - return ip_plus_item; + return ip_item; error: - FREE(ip_plus_item); + FREE(ip_item); return NULL; } -void ip_plus_item_free(struct ip_plus_item *ip_plus_item) +void ip_item_free(struct ip_item *ip_item) { - FREE(ip_plus_item); + FREE(ip_item); } -void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule) +void ip_item_to_ip_rule(struct ip_item *item, struct ip_rule *rule) { struct port_range *sport_range = ALLOC(struct port_range, 1); - if (4 == item->addr_type) { + if (IPv4 == item->addr_type) { rule->type = IPv4; sport_range->min_port = item->ipv4.min_sport; sport_range->max_port = item->ipv4.max_sport; @@ -483,20 +475,22 @@ void ip_plus_item_to_ip_rule(struct ip_plus_item *item, struct ip_rule *rule) rule->type = IPv6; sport_range->min_port = item->ipv6.min_sport; sport_range->max_port = item->ipv6.max_sport; - memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, sizeof(item->ipv6.min_sip)); - memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, sizeof(item->ipv6.max_sip)); + memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, + sizeof(item->ipv6.min_sip)); + memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, + sizeof(item->ipv6.max_sip)); } rule->rule_id = item->item_id; rule->user_tag = sport_range; } -struct ex_data_runtime *ip_plus_runtime_get_ex_data_rt(struct ip_plus_runtime *ip_plus_rt) +struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt) { - return ip_plus_rt->ex_data_rt; + return ip_rt->ex_data_rt; } -int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key_len, - struct ip_plus_item *item, int is_valid) +int ip_runtime_update_row(struct ip_runtime *rt, char *key, size_t key_len, + struct ip_item *item, int is_valid) { int ret = -1; struct ex_data_runtime *ex_data_rt = rt->ex_data_rt; @@ -519,31 +513,25 @@ int ip_plus_runtime_update_row(struct ip_plus_runtime *rt, char *key, size_t key return 0; } -int ip_plus_runtime_updating_flag(void *ip_plus_runtime) +int ip_runtime_update(void *ip_runtime, void *ip_schema, + const char *line, int valid_column) { - struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; - return ex_data_runtime_updating_flag(ip_plus_rt->ex_data_rt); -} - -int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const char *line, - int valid_column) -{ - if (NULL == ip_plus_runtime || NULL == ip_plus_schema || NULL == line) { + if (NULL == ip_runtime || NULL == ip_schema || NULL == line) { return -1; } struct maat_item *item = NULL; - struct ip_plus_item *ip_plus_item = NULL; + struct ip_item *ip_item = NULL; struct maat_item_inner *u_para = NULL; - struct ip_plus_schema *schema = (struct ip_plus_schema *)ip_plus_schema; - struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; + struct ip_schema *schema = (struct ip_schema *)ip_schema; + struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; int item_id = get_column_value(line, schema->item_id_column); int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; } else if (0 == is_valid) { //delete - HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item); + HASH_FIND_INT(ip_rt->item_hash, &item_id, item); if (NULL == item) { return -1; } @@ -555,56 +543,58 @@ int ip_plus_runtime_update(void *ip_plus_runtime, void *ip_plus_schema, const ch return -1; } - HASH_DELETE(hh, ip_plus_rt->item_hash, item); - maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, u_para, (void (*)(void *))maat_item_inner_free); + HASH_DELETE(hh, ip_rt->item_hash, item); + maat_garbage_bagging(ip_rt->ref_garbage_bin, u_para, + (void (*)(void *))maat_item_inner_free); } else { //add - HASH_FIND_INT(ip_plus_rt->item_hash, &item_id, item); + HASH_FIND_INT(ip_rt->item_hash, &item_id, item); if (item) { - log_error(ip_plus_rt->logger, MODULE_IP, - "ip_plus runtime add item %d to item_hash failed, already exist", item_id); + log_error(ip_rt->logger, MODULE_IP, + "ip runtime add item %d to item_hash failed, already exist", + item_id); return -1; } - ip_plus_item = ip_plus_item_new(line, schema, ip_plus_rt->logger); - if (NULL == ip_plus_item) { + ip_item = ip_item_new(line, schema, ip_rt->logger); + if (NULL == ip_item) { return -1; } - u_para = maat_item_inner_new(ip_plus_item->group_id, item_id, 0); - item = maat_item_new(item_id, ip_plus_item->group_id, u_para); - HASH_ADD_INT(ip_plus_rt->item_hash, item_id, item); + u_para = maat_item_inner_new(ip_item->group_id, item_id, 0); + item = maat_item_new(item_id, ip_item->group_id, u_para); + HASH_ADD_INT(ip_rt->item_hash, item_id, item); } char *key = (char *)&item_id; - int ret = ip_plus_runtime_update_row(ip_plus_rt, key, sizeof(int), ip_plus_item, is_valid); + int ret = ip_runtime_update_row(ip_rt, key, sizeof(int), ip_item, is_valid); if (ret < 0) { - if (ip_plus_item != NULL) { - ip_plus_item_free(ip_plus_item); - ip_plus_item = NULL; + if (ip_item != NULL) { + ip_item_free(ip_item); + ip_item = NULL; } return -1; } else { if (0 == is_valid) { - ip_plus_rt->rule_num--; + ip_rt->rule_num--; } else { - ip_plus_rt->rule_num++; + ip_rt->rule_num++; } } return 0; } -int ip_plus_runtime_commit(void *ip_plus_runtime) +int ip_runtime_commit(void *ip_runtime) { - if (NULL == ip_plus_runtime) { + if (NULL == ip_runtime) { return -1; } int ret = 0; struct ex_data_container **ex_container = NULL; - struct ip_plus_runtime *ip_plus_rt = (struct ip_plus_runtime *)ip_plus_runtime; - struct ex_data_runtime *ex_data_rt = ip_plus_rt->ex_data_rt; + struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; + struct ex_data_runtime *ex_data_rt = ip_rt->ex_data_rt; size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); if (0 == rule_cnt) { @@ -615,8 +605,8 @@ int ip_plus_runtime_commit(void *ip_plus_runtime) struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt); for (size_t i = 0; i < rule_cnt; i++) { - struct ip_plus_item *item = (struct ip_plus_item *)ex_container[i]->custom_data; - ip_plus_item_to_ip_rule(item, &rules[i]); + struct ip_item *item = (struct ip_item *)ex_container[i]->custom_data; + ip_item_to_ip_rule(item, &rules[i]); } struct ip_matcher *new_ip_matcher = NULL; @@ -624,21 +614,23 @@ int ip_plus_runtime_commit(void *ip_plus_runtime) size_t mem_used = 0; if (rule_cnt > 0) { - log_info(ip_plus_rt->logger, MODULE_IP, - "committing %zu ip_plus rules for rebuilding ip_matcher engine", rule_cnt); + log_info(ip_rt->logger, MODULE_IP, + "committing %zu ip rules for rebuilding ip_matcher engine", + rule_cnt); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_ip_matcher) { - log_error(ip_plus_rt->logger, MODULE_IP, - "rebuild ip_matcher engine failed when update %zu ip_plus rules", rule_cnt); + log_error(ip_rt->logger, MODULE_IP, + "rebuild ip_matcher engine failed when update %zu ip rules", + rule_cnt); ret = -1; } } - old_ip_matcher = ip_plus_rt->ip_matcher; - ip_plus_rt->ip_matcher = new_ip_matcher; - maat_garbage_bagging(ip_plus_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); + old_ip_matcher = ip_rt->ip_matcher; + ip_rt->ip_matcher = new_ip_matcher; + maat_garbage_bagging(ip_rt->ref_garbage_bin, old_ip_matcher, + (void (*)(void*))ip_matcher_free); ex_data_runtime_commit(ex_data_rt); - ip_plus_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt); FREE(rules); FREE(ex_container); @@ -646,66 +638,57 @@ int ip_plus_runtime_commit(void *ip_plus_runtime) return ret; } -int ip_plus_runtime_scan_ipv4(struct ip_plus_runtime *ip_plus_rt, int thread_id, uint32_t ip_addr, - int *group_id_array, size_t n_group_id_array, int virtual_table_id, - struct maat_state *state) +int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, int ip_type, + uint8_t *ip_addr, int *group_ids, size_t group_id_size, + int vtable_id, struct maat_state *state) { - if (NULL == ip_plus_rt) { - return -1; - } - int n_hit_item = 0; struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; - struct ip_data ip; - ip.type = IPv4; - ip.ipv4 = ip_addr; - - n_hit_item = ip_matcher_match(ip_plus_rt->ip_matcher, &ip, scan_results, MAX_SCANNER_HIT_ITEM_NUM); + struct ip_data scan_data; + if (ip_type == IPv4) { + scan_data.type = IPv4; + scan_data.ipv4 = *(uint32_t *)ip_addr; + } else { + scan_data.type = IPv6; + for (int i = 0; i < 4; i++) { + scan_data.ipv6[i] = *((uint32_t *)ip_addr + i); + } + } + + n_hit_item = ip_matcher_match(ip_rt->ip_matcher, &scan_data, scan_results, sizeof(scan_result)); + if (n_hit_item <= 0) { + return n_hit_item; + } + if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { + log_info(ip_rt->logger, MODULE_IP, + "hit ip item count:%d exceed maxium:%d", + n_hit_item, MAX_SCANNER_HIT_ITEM_NUM); n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; } - struct maat_compile_state *compile_state = state->compile_mid; - //tranform item_id to group_id - struct maat_item *item = NULL; - size_t n_group_id = 0; - int i = 0; - for (i = 0; i < n_hit_item; i++) { - HASH_FIND_INT(ip_plus_rt->item_hash, &(scan_results[i].rule_id), item); - assert(item != NULL); - if (!item) { - // should not come here - continue; - } - - if (n_group_id >= n_group_id_array) { - n_group_id = n_group_id_array; - //Prevent group_id_array out of bounds - } else { - group_id_array[n_group_id++] = item->group_id; - } - - // update hit path - maat_compile_state_update_hit_path(compile_state, scan_results[i].rule_id, item->group_id, - virtual_table_id, state->scan_cnt, i); + int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; + for (int i = 0; i < n_hit_item; i++) { + hit_item_ids[i] = scan_results[i].rule_id; } - // update hit clause: literal_id{group_id,vt_id} to clause_id - int compile_table_id = -1; - if (state->compile_table_id == -1) { - compile_table_id = state->maat_instance->default_compile_table_id; - } else { - compile_table_id = state->compile_table_id; + size_t group_hit_cnt = 0; + int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item, + group_ids, group_id_size, &group_hit_cnt, state); + if (ret < 0) { + return -1; } - void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id); - enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, compile_table_id); - assert(table_type == TABLE_TYPE_COMPILE); - - for (size_t idx = 0; idx < n_group_id; idx++) { - maat_compile_state_update_hit_clause(compile_state, compile_rt, group_id_array[idx], virtual_table_id); - } + return group_hit_cnt; +} - return n_group_id; +void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id) +{ + alignment_int64_array_add(ip_rt->hit_cnt, thread_id, 1); +} + +long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread) +{ + return alignment_int64_array_sum(ip_rt->hit_cnt, n_thread); } \ No newline at end of file diff --git a/src/maat_ip_plugin.cpp b/src/maat_ip_plugin.cpp index 86247bd..1a00c1f 100644 --- a/src/maat_ip_plugin.cpp +++ b/src/maat_ip_plugin.cpp @@ -10,9 +10,8 @@ #include -#include "maat_ip_plugin.h" -#include "utils.h" #include "maat_utils.h" +#include "maat_ip_plugin.h" #include "maat_ex_data.h" #include "IPMatcher.h" #include "maat_rule.h" @@ -37,6 +36,10 @@ struct ip_plugin_schema { int rule_tag_column; struct ex_data_schema *ex_schema; int table_id; //ugly + struct table_manager *tbl_mgr; + + unsigned long long update_err_cnt; + unsigned long long unmatch_tag_cnt; }; struct ip_plugin_runtime { @@ -50,7 +53,8 @@ struct ip_plugin_runtime { struct log_handle *logger; }; -void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) { size_t read_cnt = 0; struct ip_plugin_schema *ip_plugin_schema = ALLOC(struct ip_plugin_schema, 1); @@ -64,7 +68,8 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl item = cJSON_GetObjectItem(json, "custom"); if (NULL == item || item->type != cJSON_Object) { - log_error(logger, MODULE_IP_PLUGIN, "table %s has no custom column", table_name); + log_error(logger, MODULE_IP_PLUGIN, + "table %s has no custom column", table_name); goto error; } @@ -92,6 +97,8 @@ void *ip_plugin_schema_new(cJSON *json, const char *table_name, struct log_handl read_cnt++; } + ip_plugin_schema->tbl_mgr = tbl_mgr; + if (read_cnt < 5) { goto error; } @@ -127,52 +134,102 @@ struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema return schema->ex_schema; } -struct ip_plugin_item *ip_plugin_item_new(const char *line, struct ip_plugin_schema *ip_plugin_schema, - struct log_handle *logger) +int ip_plugin_accept_tag_match(struct ip_plugin_schema *schema, const char *line, + struct log_handle *logger) { + size_t column_offset = 0; + size_t column_len = 0; + size_t n_tag = table_manager_accept_tags_count(schema->tbl_mgr); + + if (schema->rule_tag_column > 0 && n_tag > 0) { + int ret = get_column_pos(line, schema->rule_tag_column, + &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_IP_PLUGIN, + "ip_plugin table(table_id:%d) has no rule_tag, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (column_len > 2) { + char *tag_str = ALLOC(char, column_len + 1); + memcpy(tag_str, (line + column_offset), column_len); + ret = table_manager_accept_tags_match(schema->tbl_mgr, tag_str); + FREE(tag_str); + if (TAG_MATCH_ERR == ret) { + log_error(logger, MODULE_IP_PLUGIN, + "ip_plugin table(table_id:%d) has invalid tag format, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (TAG_MATCH_UNMATCHED == ret) { + schema->unmatch_tag_cnt++; + return TAG_MATCH_UNMATCHED; + } + } + } + + return TAG_MATCH_MATCHED; +} + +struct ip_plugin_item * +ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema, + struct log_handle *logger) +{ + int ret = ip_plugin_accept_tag_match(schema, line, logger); + if (ret == TAG_MATCH_UNMATCHED) { + return NULL; + } + size_t column_offset = 0; size_t column_len = 0; struct ip_plugin_item *ip_plugin_item = ALLOC(struct ip_plugin_item, 1); - - int ret = get_column_pos(line, ip_plugin_schema->item_id_column, &column_offset, &column_len); + ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no item_id", - ip_plugin_schema->table_id, line); + log_error(logger, MODULE_IP_PLUGIN, + "ip plugin table(table_id:%d) line:%s has no item_id", + schema->table_id, line); goto error; } ip_plugin_item->item_id = atoi(line + column_offset); - ret = get_column_pos(line, ip_plugin_schema->ip_type_column, &column_offset, &column_len); + ret = get_column_pos(line, schema->ip_type_column, &column_offset, &column_len); if (ret < 0) { - log_error(logger, MODULE_IP_PLUGIN, "ip plugin table(table_id:%d) line:%s has no ip_type", - ip_plugin_schema->table_id, line); + log_error(logger, MODULE_IP_PLUGIN, + "ip plugin table(table_id:%d) line:%s has no ip_type", + schema->table_id, line); goto error; } ip_plugin_item->ip_type = atoi(line + column_offset); if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) { log_error(logger, MODULE_IP_PLUGIN, "ip_plugin table(table_id:%d) line:%s ip_type[%d] invalid", - ip_plugin_schema->table_id, line, ip_plugin_item->ip_type); + schema->table_id, line, ip_plugin_item->ip_type); goto error; } - ret = get_column_pos(line, ip_plugin_schema->start_ip_column, &column_offset, &column_len); + ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_IP_PLUGIN, "ip_plugin table(table_id:%d) line:%s has no start_ip", - ip_plugin_schema->table_id, line); + schema->table_id, line); goto error; } - strncpy(ip_plugin_item->start_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->start_ip))); + strncpy(ip_plugin_item->start_ip, line + column_offset, + MIN(column_len, sizeof(ip_plugin_item->start_ip))); - ret = get_column_pos(line, ip_plugin_schema->end_ip_column, &column_offset, &column_len); + ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_IP_PLUGIN, "ip_plugin table(table_id:%d) line:%s has no end_ip", - ip_plugin_schema->table_id, line); + schema->table_id, line); goto error; } - strncpy(ip_plugin_item->end_ip, line + column_offset, MIN(column_len, sizeof(ip_plugin_item->end_ip))); + strncpy(ip_plugin_item->end_ip, line + column_offset, + MIN(column_len, sizeof(ip_plugin_item->end_ip))); return ip_plugin_item; error: @@ -245,7 +302,8 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_ return 0; } -void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == ip_plugin_schema) { @@ -255,7 +313,8 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct m struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1); - ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger); + ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, + ex_data_container_free, logger); ip_plugin_rt->ref_garbage_bin = garbage_bin; ip_plugin_rt->logger = logger; @@ -298,10 +357,11 @@ void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule rule->user_tag = NULL; } -int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, const char *line, - int valid_column) +int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, + const char *line, int valid_column) { - if (NULL == ip_plugin_runtime || NULL == ip_plugin_schema || NULL == line) { + if (NULL == ip_plugin_runtime || NULL == ip_plugin_schema || + NULL == line) { return -1; } @@ -323,7 +383,8 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, co } char *key = (char *)&item_id; - int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key, sizeof(int), ip_plugin_item, is_valid); + int ret = ip_plugin_runtime_update_row(ip_plugin_rt, schema, line, key, + sizeof(int), ip_plugin_item, is_valid); if (ret < 0) { if (ip_plugin_item != NULL) { FREE(ip_plugin_item); @@ -370,20 +431,22 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime) if (rule_cnt > 0) { log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, - "committing %zu ip_plugin rules for rebuilding ip_matcher engine", rule_cnt); + "committing %zu ip_plugin rules for rebuilding ip_matcher engine", + rule_cnt); new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_ip_matcher) { log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, - "rebuild ip_matcher engine failed when update %zu ip_plugin rules", rule_cnt); + "rebuild ip_matcher engine failed when update %zu ip_plugin rules", + rule_cnt); ret = -1; } } old_ip_matcher = ip_plugin_rt->ip_matcher; ip_plugin_rt->ip_matcher = new_ip_matcher; - maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); + maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, + (void (*)(void*))ip_matcher_free); ex_data_runtime_commit(ex_data_rt); - ip_plugin_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt); FREE(rules); FREE(ex_container); @@ -391,12 +454,6 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime) return ret; } -int ip_plugin_runtime_updating_flag(void *ip_plugin_runtime) -{ - struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; - return ex_data_runtime_updating_flag(ip_plugin_rt->ex_data_rt); -} - struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime) { if (NULL == ip_plugin_runtime) { diff --git a/src/maat_kv.cpp b/src/maat_kv.cpp index 055b427..2b19e17 100644 --- a/src/maat_kv.cpp +++ b/src/maat_kv.cpp @@ -12,7 +12,7 @@ #include #include "uthash/uthash.h" -#include "utils.h" +#include "maat_utils.h" #define MAAT_KV_MAX_KEY_LEN 512 diff --git a/src/maat_plugin.cpp b/src/maat_plugin.cpp index 8f93288..945fe58 100644 --- a/src/maat_plugin.cpp +++ b/src/maat_plugin.cpp @@ -12,11 +12,12 @@ #include "log/log.h" #include "cJSON/cJSON.h" -#include "utils.h" #include "maat_utils.h" +#include "maat_rule.h" #include "maat_plugin.h" #include "maat_ex_data.h" #include "maat_limits.h" +#include "maat_table.h" #define MODULE_PLUGIN module_name_str("maat.plugin") @@ -49,6 +50,10 @@ struct plugin_schema { struct plugin_callback_schema cb[MAX_PLUGIN_PER_TABLE]; struct ex_data_schema *ex_schema; int table_id; //ugly + struct table_manager *tbl_mgr; + + unsigned long long update_err_cnt; + unsigned long long unmatch_tag_cnt; }; static int read_integer_array(char *string, int *array, int size) @@ -68,7 +73,8 @@ static int read_integer_array(char *string, int *array, int size) return i; } -void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) { size_t read_cnt = 0; struct plugin_schema *plugin_schema = ALLOC(struct plugin_schema, 1); @@ -82,7 +88,8 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle * item = cJSON_GetObjectItem(json, "custom"); if (item == NULL || item->type != cJSON_Object) { - log_error(logger, MODULE_PLUGIN, "table %s has no custom column", table_name); + log_error(logger, MODULE_PLUGIN, + "table %s has no custom column", table_name); goto error; } @@ -121,6 +128,8 @@ void *plugin_schema_new(cJSON *json, const char *table_name, struct log_handle * } } + plugin_schema->tbl_mgr = tbl_mgr; + if (read_cnt < 5) { goto error; } @@ -160,8 +169,9 @@ int plugin_table_add_callback(void *plugin_schema, int table_id, size_t idx = schema->cb_cnt; if (idx == MAX_PLUGIN_PER_TABLE) { - log_error(logger, MODULE_PLUGIN, "the plugin number of table_id: %d exceed maxium:%d", - table_id, MAX_PLUGIN_PER_TABLE); + log_error(logger, MODULE_PLUGIN, + "the plugin number of table_id: %d exceed maxium:%d", + table_id, MAX_PLUGIN_PER_TABLE); return -1; } @@ -192,7 +202,8 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema) } } -int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns) +int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, + int *foreign_columns) { if (NULL == plugin_schema) { return -1; @@ -221,7 +232,8 @@ int plugin_table_set_ex_data_schema(void *plugin_schema, if (schema->ex_schema != NULL) { assert(0); log_error(logger, MODULE_PLUGIN, - "Error: %s, EX data schema already registed", __FUNCTION__); + "Error: %s, EX data schema already registed", + __FUNCTION__); return -1; } schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp); @@ -239,7 +251,8 @@ struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema) return schema->ex_schema; } -void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, +void *plugin_runtime_new(void *plugin_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == plugin_schema) { @@ -248,7 +261,8 @@ void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct maat_ga struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; struct plugin_runtime *plugin_rt = ALLOC(struct plugin_runtime, 1); - plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, logger); + plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, + ex_data_container_free, logger); plugin_rt->ref_garbage_bin = garbage_bin; plugin_rt->logger = logger; @@ -270,18 +284,12 @@ void plugin_runtime_free(void *plugin_runtime) FREE(plugin_rt); } -int plugin_runtime_updating_flag(void *plugin_runtime) -{ - struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; - return ex_data_runtime_updating_flag(plugin_rt->ex_data_rt); -} - -int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_schema *plugin_schema, - const char *row, char *key, size_t key_len, int is_valid) +int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, + struct plugin_schema *plugin_schema, + const char *row, char *key, size_t key_len, + int is_valid) { int ret = -1; - void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); - struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL); struct ex_data_schema *ex_schema = plugin_schema->ex_schema; /* already set plugin_table_schema's ex_data_schema */ @@ -294,6 +302,8 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_sc } } else { // add + void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); + struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL); ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container); if (ret < 0) { return -1; @@ -313,33 +323,77 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, struct plugin_sc ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row); } - plugin_rt->acc_line_num++; - return 0; } -int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line, - int valid_column) +int plugin_accept_tag_match(struct plugin_schema *schema, const char *line, + struct log_handle *logger) { - if (NULL == plugin_runtime || NULL == plugin_runtime) { + size_t column_offset = 0; + size_t column_len = 0; + size_t n_tag = table_manager_accept_tags_count(schema->tbl_mgr); + + if (schema->rule_tag_column > 0 && n_tag > 0) { + int ret = get_column_pos(line, schema->rule_tag_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_PLUGIN, + "plugin table(table_id:%d) has no rule_tag, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (column_len > 2) { + char *tag_str = ALLOC(char, column_len + 1); + memcpy(tag_str, (line + column_offset), column_len); + ret = table_manager_accept_tags_match(schema->tbl_mgr, tag_str); + FREE(tag_str); + if (TAG_MATCH_ERR == ret) { + log_error(logger, MODULE_PLUGIN, + "plugin table(table_id:%d) has invalid tag format, line:%s", + schema->table_id, line); + schema->update_err_cnt++; + return TAG_MATCH_ERR; + } + + if (TAG_MATCH_UNMATCHED == ret) { + schema->unmatch_tag_cnt++; + return TAG_MATCH_UNMATCHED; + } + } + } + + return TAG_MATCH_MATCHED; +} + +int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, + const char *line, int valid_column) +{ + if (NULL == plugin_runtime || NULL == plugin_schema) { return -1; } struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; - int item_id = get_column_value(line, schema->item_id_column); int is_valid = get_column_value(line, valid_column); if (is_valid < 0) { return -1; } - char *key = (char *)&item_id; - int ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid); - if (ret < 0) { - - } else { - + int ret = plugin_accept_tag_match(schema, line, plugin_rt->logger); + if (ret == TAG_MATCH_UNMATCHED) { + return -1; } + + int item_id = get_column_value(line, schema->item_id_column); + char *key = (char *)&item_id; + ret = plugin_runtime_update_row(plugin_rt, schema, line, key, sizeof(int), is_valid); + if (ret < 0) { + schema->update_err_cnt++; + return -1; + } + plugin_rt->acc_line_num++; + return 0; } @@ -351,7 +405,6 @@ int plugin_runtime_commit(void *plugin_runtime) struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; ex_data_runtime_commit(plugin_rt->ex_data_rt); - //table_rt->rule_num = ex_data_runtime_ex_container_count(table_rt->plugin_rt.ex_data_rt); return 0; } diff --git a/src/maat_redis_monitor.cpp b/src/maat_redis_monitor.cpp index 25b4755..305205d 100644 --- a/src/maat_redis_monitor.cpp +++ b/src/maat_redis_monitor.cpp @@ -11,8 +11,9 @@ #include #include #include +#include -#include "utils.h" +#include "maat/maat.h" #include "maat_utils.h" #include "maat_command.h" #include "maat_config_monitor.h" @@ -39,28 +40,34 @@ const char *foreign_key_prefix = "__FILE_"; const char *mr_op_str[] = {"DEL", "ADD", "RENEW_TIMEOUT"}; -char *get_foreign_cont_filename(const char *table_name, int rule_id, const char *foreign_key, const char *dir) +char *get_foreign_cont_filename(const char *table_name, int rule_id, + const char *foreign_key, const char *dir) { char buffer[512] = {0}; - snprintf(buffer, sizeof(buffer),"%s/%s-%d-%s", dir, table_name, rule_id, foreign_key); + snprintf(buffer, sizeof(buffer),"%s/%s-%d-%s", dir, + table_name, rule_id, foreign_key); char *filename = ALLOC(char, strlen(buffer) + 1); memcpy(filename, buffer, strlen(buffer)); return filename; } -void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_foreign, - const char *dir, struct log_handle *logger) +void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, + int n_foreign, const char *dir, struct log_handle *logger) { int foreign_key_size = 0; p_rule->f_keys = ALLOC(struct foreign_key, n_foreign); for (int i = 0; i < n_foreign; i++) { - const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size); + const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, + foreign_columns[i], + &foreign_key_size); if (NULL == p_foreign) { - log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign keys failed: No %dth column", - p_rule->table_name, p_rule->rule_id, foreign_columns[i]); + log_error(logger, MODULE_REDIS_MONITOR, + "Get %s,%lu foreign keys failed: No %dth column", + p_rule->table_name, p_rule->rule_id, + foreign_columns[i]); continue; } @@ -70,7 +77,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f } if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) { - log_error(logger, MODULE_REDIS_MONITOR, "Get %s,%lu foreign key failed: Invalid source prefix %s", + log_error(logger, MODULE_REDIS_MONITOR, + "Get %s,%lu foreign key failed: Invalid source prefix %s", p_rule->table_name, p_rule->rule_id, p_foreign); continue; } @@ -80,13 +88,17 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f p_foreign += strlen(foreign_source_prefix); if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) { - log_info(logger, MODULE_REDIS_MONITOR, "%s, %lu foreign key prefix %s is not recommended", + log_info(logger, MODULE_REDIS_MONITOR, + "%s, %lu foreign key prefix %s is not recommended", p_rule->table_name, p_rule->rule_id, p_foreign); } p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size+1); memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign, foreign_key_size); - p_rule->f_keys[p_rule->n_foreign].filename = get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[p_rule->n_foreign].key, dir); + p_rule->f_keys[p_rule->n_foreign].filename = get_foreign_cont_filename(p_rule->table_name, + p_rule->rule_id, + p_rule->f_keys[p_rule->n_foreign].key, + dir); p_rule->n_foreign++; } @@ -95,7 +107,8 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_f } } -int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_instance, const char *dir) +int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, + int rule_num, struct maat *maat_instance, const char *dir) { int rule_with_foreign_key = 0; @@ -112,7 +125,8 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, in } int foreign_columns[8]; - int n_foreign_column = plugin_table_get_foreign_column((struct plugin_schema *)schema, foreign_columns); + int n_foreign_column = plugin_table_get_foreign_column((struct plugin_schema *)schema, + foreign_columns); if (0 == n_foreign_column) { continue; } @@ -175,9 +189,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru redisReply* reply = NULL; for (i = 0; i < rule_num; i++) { - snprintf(redis_cmd, sizeof(redis_cmd), "GET %s:%s,%lu", mr_key_prefix[rule_list[i].op], - rule_list[i].table_name, - rule_list[i].rule_id); + snprintf(redis_cmd, sizeof(redis_cmd), + "GET %s:%s,%lu", mr_key_prefix[rule_list[i].op], + rule_list[i].table_name, + rule_list[i].rule_id); ret = redisAppendCommand(c, redis_cmd); assert(ret == REDIS_OK); } @@ -185,9 +200,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru for (i = 0; i < rule_num; i++) { ret = maat_cmd_wrap_redis_get_reply(c, &reply); if (ret == REDIS_ERR) { - log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed, redis server error", - mr_key_prefix[rule_list[i].op], - rule_list[i].table_name, rule_list[i].rule_id); + log_error(logger, MODULE_REDIS_MONITOR, + "Redis GET %s:%s,%lu failed, redis server error", + mr_key_prefix[rule_list[i].op], + rule_list[i].table_name, rule_list[i].rule_id); error_happened = 1; break; } @@ -199,9 +215,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru retry_ids[failed_cnt] = i; failed_cnt++; } else { - log_error(logger, MODULE_REDIS_MONITOR, "Redis GET %s:%s,%lu failed", - mr_key_prefix[rule_list[i].op], - rule_list[i].table_name, rule_list[i].rule_id); + log_error(logger, MODULE_REDIS_MONITOR, + "Redis GET %s:%s,%lu failed", + mr_key_prefix[rule_list[i].op], + rule_list[i].table_name, rule_list[i].rule_id); error_happened = 1; } } @@ -218,9 +235,10 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru int idx = 0; for (i = 0; i < failed_cnt; i++) { idx = retry_ids[i]; - snprintf(redis_cmd, sizeof(redis_cmd), "GET %s:%s,%lu", mr_key_prefix[MAAT_OP_DEL], - rule_list[idx].table_name, - rule_list[idx].rule_id); + snprintf(redis_cmd, sizeof(redis_cmd), + "GET %s:%s,%lu", mr_key_prefix[MAAT_OP_DEL], + rule_list[idx].table_name, + rule_list[idx].rule_id); ret = redisAppendCommand(c, redis_cmd); } @@ -228,7 +246,9 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru idx = retry_ids[i]; ret = maat_cmd_wrap_redis_get_reply(c, &reply); if (ret == REDIS_ERR) { - log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, redis server error", redis_cmd); + log_error(logger, MODULE_REDIS_MONITOR, + "redis command %s failed, redis server error", + redis_cmd); FREE(retry_ids); return -1; } @@ -237,12 +257,14 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru rule_list[idx].table_line = maat_strdup(reply->str); } else if(reply->type==REDIS_REPLY_ERROR) { //Deal with Redis response: "Loading Redis is loading the database in memory" - log_error(logger, MODULE_REDIS_MONITOR, "redis command %s error, reply type=%d, error str=%s", - redis_cmd, reply->type, reply->str); + log_error(logger, MODULE_REDIS_MONITOR, + "redis command %s error, reply type=%d, error str=%s", + redis_cmd, reply->type, reply->str); } else { //Handle type "nil" - log_error(logger, MODULE_REDIS_MONITOR, "redis command %s failed, reply type=%d", - redis_cmd, reply->type); + log_error(logger, MODULE_REDIS_MONITOR, + "redis command %s failed, reply type=%d", + redis_cmd, reply->type); } freeReplyObject(reply); @@ -254,8 +276,8 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru return 0; } -int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, - int print_process, struct log_handle *logger) +int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, + int rule_num, int print_process, struct log_handle *logger) { int max_redis_batch = 4096; int success_cnt = 0; @@ -286,14 +308,17 @@ int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int } int get_inc_key_list(long long instance_version, long long target_version, - redisContext *c, struct serial_rule **list, struct log_handle *logger) + redisContext *c, struct serial_rule **list, + struct log_handle *logger) { //Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version. //The elements are considered to be ordered from low to high scores(instance_version). - redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", mr_status_sset, - instance_version,target_version); + redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", + mr_status_sset, instance_version, + target_version); if (NULL == reply) { - log_error(logger, MODULE_REDIS_MONITOR, "GET %s failed with a NULL reply, error: %s", + log_error(logger, MODULE_REDIS_MONITOR, + "GET %s failed with a NULL reply, error: %s", mr_status_sset, c->errstr); return -1; } @@ -307,10 +332,14 @@ int get_inc_key_list(long long instance_version, long long target_version, return 0; } - redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", mr_status_sset, reply->element[0]->str); + redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", + mr_status_sset, + reply->element[0]->str); if (tmp_reply->type != REDIS_REPLY_STRING) { - log_error(logger, MODULE_REDIS_MONITOR, "ZSCORE %s %s failed Version: %lld->%lld", - mr_status_sset, reply->element[0]->str, instance_version, target_version); + log_error(logger, MODULE_REDIS_MONITOR, + "ZSCORE %s %s failed Version: %lld->%lld", + mr_status_sset, reply->element[0]->str, + instance_version, target_version); freeReplyObject(tmp_reply); tmp_reply = NULL; freeReplyObject(reply); @@ -327,7 +356,8 @@ int get_inc_key_list(long long instance_version, long long target_version, } if (nearest_rule_version != instance_version + 1) { - log_info(logger, MODULE_REDIS_MONITOR, "Noncontinuous VERSION Redis: %lld MAAT: %lld", + log_info(logger, MODULE_REDIS_MONITOR, + "Noncontinuous VERSION Redis: %lld MAAT: %lld", nearest_rule_version, instance_version); } @@ -338,10 +368,11 @@ int get_inc_key_list(long long instance_version, long long target_version, for (i = 0, j = 0; i < (int)reply->elements; i++) { assert(reply->element[i]->type == REDIS_REPLY_STRING); - int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", op_str, - s_rule[j].table_name, &(s_rule[j].rule_id)); + int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", + op_str, s_rule[j].table_name, &(s_rule[j].rule_id)); if (ret != 3 || s_rule[i].rule_id < 0) { - log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str); + log_error(logger, MODULE_REDIS_MONITOR, + "Invalid Redis Key: %s", reply->element[i]->str); continue; } @@ -350,7 +381,8 @@ int get_inc_key_list(long long instance_version, long long target_version, } else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) { s_rule[j].op = MAAT_OP_DEL; } else { - log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key: %s", reply->element[i]->str); + log_error(logger, MODULE_REDIS_MONITOR, + "Invalid Redis Key: %s", reply->element[i]->str); continue; } j++; @@ -398,7 +430,8 @@ struct serial_rule *serial_rule_clone(const struct serial_rule *s_rule) new_rule->f_keys[j].key = ALLOC(char, s_rule->f_keys[j].key_len); memcpy(new_rule->f_keys[j].key, s_rule->f_keys[j].key, s_rule->f_keys[j].key_len); new_rule->f_keys[j].filename = ALLOC(char, strlen(s_rule->f_keys[j].filename)); - memcpy(new_rule->f_keys[j].filename, s_rule->f_keys[j].filename, strlen(s_rule->f_keys[j].filename)); + memcpy(new_rule->f_keys[j].filename, s_rule->f_keys[j].filename, + strlen(s_rule->f_keys[j].filename)); } return new_rule; @@ -466,10 +499,10 @@ int recovery_history_version(const struct serial_rule *current, int current_num, return ret; } -int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, - long long *new_version, struct table_manager *tbl_mgr, - struct serial_rule **list, int *update_type, int cumulative_off, - struct log_handle *logger) +int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, + long long desired_version, long long *new_version, + struct table_manager *tbl_mgr, struct serial_rule **list, + int *update_type, int cumulative_off, struct log_handle *logger) { int rule_num = 0; long long target_version = 0; @@ -478,20 +511,23 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION"); if (reply != NULL) { if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { - log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed, maybe Redis is busy"); + log_error(logger, MODULE_REDIS_MONITOR, + "GET MAAT_VERSION failed, maybe Redis is busy"); freeReplyObject(reply); reply = NULL; return -1; } } else { - log_error(logger, MODULE_REDIS_MONITOR, "GET MAAT_VERSION failed with NULL reply, error: %s", c->errstr); + log_error(logger, MODULE_REDIS_MONITOR, + "GET MAAT_VERSION failed with NULL reply, error: %s", c->errstr); return -1; } long long redis_version = maat_cmd_read_redis_integer(reply); if (redis_version < 0) { if (reply->type == REDIS_REPLY_ERROR) { - log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", reply->str); + log_error(logger, MODULE_REDIS_MONITOR, + "Redis Communication error: %s", reply->str); } return -1; } @@ -508,7 +544,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l } if (redis_version < instance_version) { - log_error(logger, MODULE_REDIS_MONITOR, "VERSION roll back MAAT: %lld -> Redis: %lld", + log_error(logger, MODULE_REDIS_MONITOR, + "VERSION roll back MAAT: %lld -> Redis: %lld", instance_version, redis_version); goto FULL_UPDATE; } @@ -521,7 +558,8 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l do { target_version++; - rule_num = get_inc_key_list(instance_version, target_version, c, &s_rule_array, logger); + rule_num = get_inc_key_list(instance_version, target_version, + c, &s_rule_array, logger); if (rule_num > 0) { break; } else if (rule_num < 0) { @@ -533,21 +571,25 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long l } while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off); if (0 == rule_num) { - log_info(logger, MODULE_REDIS_MONITOR, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s", - mr_status_sset, instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON"); + log_info(logger, MODULE_REDIS_MONITOR, + "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s", + mr_status_sset, instance_version, target_version-1, + cumulative_off == 1 ? "OFF" : "ON"); return 0; } - log_info(logger, MODULE_REDIS_MONITOR, "Inc Update from instance_version %lld to %lld (%d entries)", + log_info(logger, MODULE_REDIS_MONITOR, + "Inc Update from instance_version %lld to %lld (%d entries)", instance_version, target_version, rule_num); *list = s_rule_array; - *update_type = CM_UPDATE_TYPE_INC; + *update_type = MAAT_UPDATE_TYPE_INC; *new_version = target_version; return rule_num; FULL_UPDATE: - log_info(logger, MODULE_REDIS_MONITOR, "Initiate full update from instance_version %lld to %lld", + log_info(logger, MODULE_REDIS_MONITOR, + "Initiate full update from instance_version %lld to %lld", instance_version, desired_version == 0 ? redis_version : desired_version); size_t append_cmd_cnt = 0; int ret = redisAppendCommand(c, "MULTI"); @@ -569,12 +611,14 @@ FULL_UPDATE: reply = maat_cmd_wrap_redis_command(c, "EXEC"); if (NULL == reply) { - log_error(logger, MODULE_REDIS_MONITOR, "Redis Communication error: %s", c->errstr); + log_error(logger, MODULE_REDIS_MONITOR, + "Redis Communication error: %s", c->errstr); return -1; } if (reply->type != REDIS_REPLY_ARRAY) { - log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", reply->type); + log_error(logger, MODULE_REDIS_MONITOR, + "Invalid Redis Key List type %d", reply->type); freeReplyObject(reply); reply = NULL; return -1; @@ -583,7 +627,8 @@ FULL_UPDATE: *new_version = maat_cmd_read_redis_integer(reply->element[0]); redisReply *sub_reply = reply->element[1]; if (sub_reply->type != REDIS_REPLY_ARRAY) { - log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key List type %d", sub_reply->type); + log_error(logger, MODULE_REDIS_MONITOR, + "Invalid Redis Key List type %d", sub_reply->type); freeReplyObject(reply); reply = NULL; return -1; @@ -593,17 +638,21 @@ FULL_UPDATE: s_rule_array = ALLOC(struct serial_rule, sub_reply->elements); for (i = 0, full_idx = 0; i < sub_reply->elements; i++) { if (sub_reply->element[i]->type != REDIS_REPLY_STRING) { - log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Type: %d", sub_reply->element[i]->type); + log_error(logger, MODULE_REDIS_MONITOR, + "Invalid Redis Key Type: %d", sub_reply->element[i]->type); continue; } ret = sscanf(sub_reply->element[i]->str, "%*[^:]:%[^,],%ld", - s_rule_array[full_idx].table_name, - &(s_rule_array[full_idx].rule_id)); + s_rule_array[full_idx].table_name, + &(s_rule_array[full_idx].rule_id)); s_rule_array[full_idx].op = MAAT_OP_ADD; - if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || strlen(s_rule_array[full_idx].table_name) == 0) { - log_error(logger, MODULE_REDIS_MONITOR, "Invalid Redis Key Format: %s", sub_reply->element[i]->str); + if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || + strlen(s_rule_array[full_idx].table_name) == 0) { + log_error(logger, MODULE_REDIS_MONITOR, + "Invalid Redis Key Format: %s", + sub_reply->element[i]->str); continue; } @@ -623,22 +672,27 @@ FULL_UPDATE: if (desired_version != 0) { struct serial_rule *changed_rule_array = NULL; - int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array, logger); + int changed_rule_num = get_inc_key_list(desired_version, redis_version, + c, &changed_rule_array, logger); if (changed_rule_num < 0) { - log_error(logger, MODULE_REDIS_MONITOR, "Recover history version %lld faild where as redis version is %lld", + log_error(logger, MODULE_REDIS_MONITOR, + "Recover history version %lld faild where as redis version is %lld", desired_version, redis_version); } else if(0 == changed_rule_num) { - log_error(logger, MODULE_REDIS_MONITOR, "Nothing to recover from history version %lld to redis version is %lld", + log_error(logger, MODULE_REDIS_MONITOR, + "Nothing to recover from history version %lld to redis version is %lld", desired_version, redis_version); } else { struct serial_rule *history_rule_array = NULL; - ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array); + ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, + changed_rule_num, &history_rule_array); if (ret > 0) { FREE(s_rule_array); s_rule_array = history_rule_array; rule_num = ret; *new_version = desired_version; - log_info(logger, MODULE_REDIS_MONITOR, "Successfully recovered from history version %lld to redis version is %lld", + log_info(logger, MODULE_REDIS_MONITOR, + "Successfully recovered from history version %lld to redis version is %lld", desired_version, redis_version); } } @@ -646,14 +700,15 @@ FULL_UPDATE: } *list = s_rule_array; - *update_type = CM_UPDATE_TYPE_FULL; - log_info(logger, MODULE_REDIS_MONITOR, "Full update %d keys of version %lld", rule_num, *new_version); + *update_type = MAAT_UPDATE_TYPE_FULL; + log_info(logger, MODULE_REDIS_MONITOR, + "Full update %d keys of version %lld", rule_num, *new_version); return rule_num ; } -void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn, - struct log_handle *logger) +void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, + int rule_num, int print_fn, struct log_handle *logger) { int i = 0; int j = 0; @@ -676,8 +731,9 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule ret = remove(rule_list[i].f_keys[j].filename); if (ret == -1) { - log_error(logger, MODULE_REDIS_MONITOR, "Foreign content file %s remove failed", - rule_list[i].f_keys[j].filename); + log_error(logger, MODULE_REDIS_MONITOR, + "Foreign content file %s remove failed", + rule_list[i].f_keys[j].filename); } } } else { @@ -734,7 +790,8 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule fclose(fp); fp = NULL; if (1 == print_fn) { - printf("Written foreign content %s\n", s_rule->f_keys[track[i].foreign_idx].filename); + printf("Written foreign content %s\n", + s_rule->f_keys[track[i].foreign_idx].filename); } } } @@ -747,8 +804,8 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule return; } -void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn, - struct log_handle *logger) +void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, + int rule_num, int print_fn, struct log_handle *logger) { int max_redis_batch = 4096; int success_cnt = 0; @@ -797,12 +854,14 @@ void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule) pos_rewrite_line += strlen(s_rule->f_keys[i].filename); } - strncat(pos_rewrite_line, pos_origin_line, strlen(s_rule->table_line) - (pos_origin_line - s_rule->table_line)); + strncat(pos_rewrite_line, pos_origin_line, + strlen(s_rule->table_line) - (pos_origin_line - s_rule->table_line)); FREE(s_rule->table_line); s_rule->table_line = rewrite_line; } -void expected_reply_add(struct expected_reply* expected, int s_rule_seq, int type, long long integer) +void expected_reply_add(struct expected_reply* expected, int s_rule_seq, + int type, long long integer) { int i = expected->possible_reply_num; assert(i < POSSIBLE_REDIS_REPLY_SIZE); @@ -816,7 +875,8 @@ int redlock_try_lock(redisContext *c, const char *lock_name, long long expire) { int ret = 0; - redisReply *reply = maat_cmd_wrap_redis_command(c, "SET %s locked NX PX %lld", lock_name, expire); + redisReply *reply = maat_cmd_wrap_redis_command(c, "SET %s locked NX PX %lld", + lock_name, expire); if (reply->type == REDIS_REPLY_NIL) { ret = 0; } else { @@ -829,8 +889,9 @@ int redlock_try_lock(redisContext *c, const char *lock_name, long long expire) return ret; } -long long exec_serial_rule_begin(redisContext* c, size_t rule_num, size_t renew_rule_num, - int *renew_allowed, long long *transaction_version) +long long exec_serial_rule_begin(redisContext* c, size_t rule_num, + size_t renew_rule_num, int *renew_allowed, + long long *transaction_version) { int ret = -1; redisReply *data_reply = NULL; @@ -878,8 +939,9 @@ const char* lua_exec_done= "redis.call(\'del\', KEYS[4]);" "redis.call(\'zadd\', KEYS[3], ARGV[1], maat_version);" "return maat_version;"; -redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list, long long server_time, - int renew_allowed, struct expected_reply *expect_reply, size_t *cnt) +redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list, + long long server_time, int renew_allowed, + struct expected_reply *expect_reply, size_t *cnt) { redisReply *data_reply = NULL; @@ -907,8 +969,10 @@ redisReply* exec_serial_rule_end(redisContext *c, const char *transaction_list, return data_reply; } -void exec_serial_rule(redisContext *c, const char *transaction_list, struct serial_rule *s_rule, size_t rule_num, - struct expected_reply *expect_reply, size_t *cnt, size_t offset, int renew_allowed) +void exec_serial_rule(redisContext *c, const char *transaction_list, + struct serial_rule *s_rule, size_t rule_num, + struct expected_reply *expect_reply, size_t *cnt, + size_t offset, int renew_allowed) { size_t i = 0; size_t append_cmd_cnt = 0; @@ -1053,8 +1117,9 @@ int mr_operation_success(redisReply *actual_reply, struct expected_reply *expect return 0; } -int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, - long long server_time, struct log_handle *logger) +int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, + size_t serial_rule_num, long long server_time, + struct log_handle *logger) { size_t i = 0; size_t rule_seq = 0; @@ -1079,7 +1144,8 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri } } - int ret = exec_serial_rule_begin(c, serial_rule_num, renew_num, &renew_allowed, &transaction_version); + int ret = exec_serial_rule_begin(c, serial_rule_num, renew_num, &renew_allowed, + &transaction_version); //Preconditions for transaction are not satisfied. if (ret != 0) { success_cnt = -1; @@ -1087,18 +1153,20 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, size_t seri } if (transaction_version > 0) { - snprintf(transaction_list, sizeof(transaction_list), "MAAT_TRANSACTION_%lld", transaction_version); + snprintf(transaction_list, sizeof(transaction_list), "MAAT_TRANSACTION_%lld", + transaction_version); } while (success_cnt < serial_rule_num) { size_t batch_cnt = MIN(serial_rule_num - success_cnt, max_redis_batch); - exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt, expected_reply, &multi_cmd_cnt, - success_cnt, renew_allowed); + exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt, + expected_reply, &multi_cmd_cnt, success_cnt, renew_allowed); assert(multi_cmd_cntelements == multi_cmd_cnt); for (i = 0; i < multi_cmd_cnt; i++) { @@ -1138,8 +1206,10 @@ error_out: if (renew_num > 0 && renew_allowed != 1) { for (i = 0; i < (unsigned int)serial_rule_num; i++) { if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) { - log_error(logger, MODULE_REDIS_MONITOR, "%s %s %lu is not allowed due to lock contention", - mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, s_rule[i].rule_id); + log_error(logger, MODULE_REDIS_MONITOR, + "%s %s %lu is not allowed due to lock contention", + mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, + s_rule[i].rule_id); } } @@ -1171,11 +1241,11 @@ void cleanup_update_status(redisContext *c, struct log_handle *logger) int append_cmd_cnt = 0; redisAppendCommand(c, "ZRANGEBYSCORE %s -inf %lld", - mr_version_sset, server_time - MAAT_REDIS_SYNC_TIME); + mr_version_sset, server_time - MAAT_REDIS_SYNC_TIME); append_cmd_cnt++; redisAppendCommand(c, "ZREMRANGEBYSCORE %s -inf %lld", - mr_version_sset,server_time - MAAT_REDIS_SYNC_TIME); + mr_version_sset,server_time - MAAT_REDIS_SYNC_TIME); append_cmd_cnt++; //consume reply "OK" and "QUEUED". @@ -1207,8 +1277,9 @@ void cleanup_update_status(redisContext *c, struct log_handle *logger) reply = NULL; //To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally. - reply = maat_cmd_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld", mr_status_sset, - version_lower_bound, version_upper_bound); + reply = maat_cmd_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld", + mr_status_sset, version_lower_bound, + version_upper_bound); entry_num = maat_cmd_read_redis_integer(reply); freeReplyObject(reply); reply = NULL; @@ -1232,7 +1303,8 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger) return; } - redisReply *data_reply= maat_cmd_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld", mr_expire_sset, server_time); + redisReply *data_reply= maat_cmd_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld", + mr_expire_sset, server_time); if (data_reply->type != REDIS_REPLY_ARRAY || 0 == data_reply->elements) { freeReplyObject(data_reply); data_reply = NULL; @@ -1244,7 +1316,8 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger) for (size_t i = 0; i < s_rule_num; i++) { s_rule[i].op = MAAT_OP_DEL; - ret = sscanf(data_reply->element[i]->str, "%[^,],%ld", s_rule[i].table_name, &(s_rule[i].rule_id)); + ret = sscanf(data_reply->element[i]->str, "%[^,],%ld", + s_rule[i].table_name, &(s_rule[i].rule_id)); assert(ret == 2); } freeReplyObject(data_reply); @@ -1254,9 +1327,11 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger) if (success_cnt < 0) { log_error(logger, MODULE_REDIS_MONITOR, "maat_cmd_write_rule failed."); } else if (success_cnt == (int)s_rule_num) { - log_info(logger, MODULE_REDIS_MONITOR, "Succesfully expired %zu rules in Redis", s_rule_num); + log_info(logger, MODULE_REDIS_MONITOR, + "Succesfully expired %zu rules in Redis", s_rule_num); } else { - log_error(logger, MODULE_REDIS_MONITOR, "Failed to expired %d of %zu rules in Redis, try later", + log_error(logger, MODULE_REDIS_MONITOR, + "Failed to expired %d of %zu rules in Redis, try later", s_rule_num - success_cnt, s_rule_num); } @@ -1301,7 +1376,9 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, } log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Reconnecting..."); - mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db, + mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, + mr_ctx->redis_port, + mr_ctx->redis_db, maat_instance->logger); if (NULL == mr_ctx->read_ctx) { return; @@ -1312,11 +1389,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, struct serial_rule *rule_list = NULL; long long new_version = 0; - int update_type = CM_UPDATE_TYPE_INC; + int update_type = MAAT_UPDATE_TYPE_INC; - int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version, - &new_version, maat_instance->tbl_mgr, &rule_list, - &update_type, maat_instance->cumulative_update_off, + int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, + maat_instance->load_specific_version, + &new_version, maat_instance->tbl_mgr, + &rule_list, &update_type, + maat_instance->cumulative_update_off, maat_instance->logger); //redis communication error if (rule_num < 0) { @@ -1327,12 +1406,13 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, maat_instance->load_specific_version = 0;//only valid for one time. //error or nothing changed - if (0 == rule_num && update_type == CM_UPDATE_TYPE_INC) { + if (0 == rule_num && update_type == MAAT_UPDATE_TYPE_INC) { return; } if (rule_num > 0) { - ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger); + ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, + 0, maat_instance->logger); //redis communication error if (ret < 0) { redisFree(mr_ctx->read_ctx); @@ -1354,15 +1434,19 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, goto clean_up; } - ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num, maat_instance, maat_instance->foreign_cont_dir); + ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num, + maat_instance, maat_instance->foreign_cont_dir); if (ret > 0) { - maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, maat_instance->logger); + maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0, + maat_instance->logger); } } start_fn(new_version, update_type, u_param); - log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Start %s update: %lld -> %lld (%d entries)", - update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num); + log_info(maat_instance->logger, MODULE_REDIS_MONITOR, + "Start %s update: %lld -> %lld (%d entries)", + update_type == MAAT_UPDATE_TYPE_INC ? "INC" : "FULL", + version, new_version, rule_num); for (i = 0; i < rule_num; i++) { if (NULL == rule_list[i].table_line) { @@ -1385,7 +1469,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, ret = invalidate_line(rule_list[i].table_line, table_type, valid_column); if (ret < 0) { log_error(maat_instance->logger, MODULE_REDIS_MONITOR, - "Invalidate line failed, invaid format %s", rule_list[i].table_line); + "Invalidate line failed, invaid format %s", + rule_list[i].table_line); continue; } } diff --git a/src/maat_rule.cpp b/src/maat_rule.cpp index fb48996..67304db 100644 --- a/src/maat_rule.cpp +++ b/src/maat_rule.cpp @@ -17,7 +17,6 @@ #include #include -#include "utils.h" #include "json2iris.h" #include "log/log.h" #include "cJSON/cJSON.h" @@ -57,18 +56,18 @@ void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *) static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, size_t n_accept_tag) { if (NULL == tag_obj || NULL == accept_tags) { - return -1; + return TAG_MATCH_ERR; } cJSON *tab_name_obj = cJSON_GetObjectItem(tag_obj, "tag"); if (NULL == tab_name_obj || tab_name_obj->type != cJSON_String) { - return -1; + return TAG_MATCH_ERR; } const char *tag_name = tab_name_obj->valuestring; cJSON *tag_vals_array = cJSON_GetObjectItem(tag_obj, "value"); if (NULL == tag_vals_array || tag_vals_array->type != cJSON_Array) { - return -1; + return TAG_MATCH_ERR; } int name_matched = 0; @@ -82,7 +81,7 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, for (int j = 0; j < n_val; j++) { cJSON *tag_val_obj = cJSON_GetArrayItem(tag_vals_array, j); if (NULL == tag_val_obj || tag_val_obj->type != cJSON_String) { - return -1; + return TAG_MATCH_ERR; } const char *tag_val = tag_val_obj->valuestring; @@ -95,16 +94,16 @@ static int compare_each_tag(cJSON *tag_obj, const struct rule_tag *accept_tags, //make sure the overlap is ended with a '/' if (0 == strncmp(accept_tags[i].tag_val, tag_val, strlen(tag_val)) && (strlen(accept_tags[i].tag_val) == strlen(tag_val) || accept_tags[i].tag_val[strlen(tag_val)] == '/')) { - return 1; + return TAG_MATCH_MATCHED; } } } //no matched name is considered as a if (name_matched > 0) { - return 0; + return TAG_MATCH_UNMATCHED; } else { - return 1; + return TAG_MATCH_MATCHED; } } @@ -117,12 +116,12 @@ static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_ta for (int i = 0; i < n_tag; i++) { cJSON *tag_obj = cJSON_GetArrayItem(tag_set, i); if (NULL == tag_obj || tag_obj->type != cJSON_Object) { - goto error; + return TAG_MATCH_ERR; } int ret = compare_each_tag(tag_obj, accept_tags, n_accept_tag); if (ret < 0) { - return -1; + return TAG_MATCH_ERR; } if(1 == ret) { @@ -131,13 +130,10 @@ static int compare_each_tag_set(cJSON *tag_set, const struct rule_tag *accept_ta } if (matched == n_tag) { - return 1; + return TAG_MATCH_MATCHED; } else { - return 0; + return TAG_MATCH_UNMATCHED; } - -error: - return -1; } //@param value is a JSON, like {"tags":[{"tag":"location","value":"北京/朝阳/华严北里/甲22号},{"tag":"isp","value":"电信"}]} @@ -176,7 +172,7 @@ size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_ //@return 1 on match, 0 on not match, -1 on error. int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag) { - int ret = -1; + int ret = TAG_MATCH_ERR; int n_set = 0; cJSON *tag_set = NULL; cJSON *tag_set_array = NULL; @@ -200,7 +196,7 @@ int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, si ret = compare_each_tag_set(tag_set, accept_tags, n_accept_tag); //match or error occurs. - if (ret != 0) { + if (ret != TAG_MATCH_UNMATCHED) { break; } } @@ -280,25 +276,11 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt) FREE(maat_rt); } -int maat_runtime_updating_flag(struct maat_runtime *maat_rt) -{ - int flag = -1; - - for (size_t i = 0; i < maat_rt->max_table_num; i++) { - flag = table_manager_runtime_updating_flag(maat_rt->ref_tbl_mgr, i); - if (1 == flag) { - return 1; - } - } - - return 0; -} - void maat_start_cb(long long new_version, int update_type, void *u_param) { struct maat *maat_instance = (struct maat *)u_param; - if (update_type == CM_UPDATE_TYPE_FULL) { + if (update_type == MAAT_UPDATE_TYPE_FULL) { maat_instance->creating_maat_rt = maat_runtime_create(new_version, maat_instance); } else { maat_instance->maat_version = new_version; @@ -390,14 +372,18 @@ void maat_finish_cb(void *u_param) if (maat_instance->creating_maat_rt != NULL) { maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_runtime_commit(maat_instance->creating_maat_rt, maat_instance->logger); - log_info(maat_instance->logger, MODULE_MAAT_RULE, "Full config version %llu load %d entries complete\n", - maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->rule_num); + log_info(maat_instance->logger, MODULE_MAAT_RULE, + "Full config version %llu load %d entries complete", + maat_instance->creating_maat_rt->version, + maat_instance->creating_maat_rt->rule_num); } else if (maat_instance->maat_rt != NULL) { maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt); maat_instance->maat_rt->version = maat_instance->maat_version; maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger); - log_info(maat_instance->logger, MODULE_MAAT_RULE, "Inc config version %llu load %d entries complete\n", - maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); + log_info(maat_instance->logger, MODULE_MAAT_RULE, + "Inc config version %llu load %d entries complete", + maat_instance->maat_rt->version, + maat_instance->maat_rt->rule_num); } } @@ -420,7 +406,8 @@ void *rule_monitor_loop(void *arg) pthread_mutex_lock(&(maat_instance->background_update_mutex)); /* if deferred load on */ if (maat_instance->deferred_load != 0) { - log_info(maat_instance->logger, MODULE_MAAT_RULE, "Deferred Loading ON, updating in %s", __func__); + log_info(maat_instance->logger, MODULE_MAAT_RULE, + "Deferred Loading ON, updating in %s", __func__); maat_read_full_config(maat_instance); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); @@ -478,13 +465,16 @@ void *rule_monitor_loop(void *arg) if (old_maat_rt != NULL) { if (maat_instance->maat_rt->version > old_maat_rt->version) { - log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version updated %lld -> %lld\n", + log_info(maat_instance->logger, MODULE_MAAT_RULE, + "Maat version updated %lld -> %lld\n", old_maat_rt->version, maat_instance->maat_rt->version); } else { - log_info(maat_instance->logger, MODULE_MAAT_RULE, "Maat version roll back %lld -> %lld\n", + log_info(maat_instance->logger, MODULE_MAAT_RULE, + "Maat version roll back %lld -> %lld\n", old_maat_rt->version, maat_instance->maat_rt->version); } - maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy); + maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, + (void (*)(void*))maat_runtime_destroy); } maat_instance->creating_maat_rt = NULL; @@ -493,12 +483,11 @@ void *rule_monitor_loop(void *arg) } if (maat_instance->maat_rt != NULL) { - int updating_flag = maat_runtime_updating_flag(maat_instance->maat_rt); time_t time_window = time(NULL) - maat_instance->maat_rt->last_update_time; - if ((updating_flag > 0) && (time_window >= maat_instance->rule_effect_interval_ms / 1000)) { + if (time_window >= maat_instance->rule_effect_interval_ms / 1000) { maat_runtime_commit(maat_instance->maat_rt, maat_instance->logger); - log_info(maat_instance->logger,MODULE_MAAT_RULE, + log_info(maat_instance->logger, MODULE_MAAT_RULE, "Actual update config version %u, %d entries load to rulescan after postpone.", maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); } @@ -514,6 +503,12 @@ void *rule_monitor_loop(void *arg) maat_garbage_bin_free(maat_instance->garbage_bin); table_manager_destroy(maat_instance->tbl_mgr); + alignment_int64_array_free(maat_instance->outer_state_cnt); + alignment_int64_array_free(maat_instance->compile_state_cnt); + alignment_int64_array_free(maat_instance->thread_call_cnt); + alignment_int64_array_free(maat_instance->hit_cnt); + alignment_int64_array_free(maat_instance->not_grp_hit_cnt); + if (maat_instance->input_mode == DATA_SOURCE_REDIS) { if (maat_instance->mr_ctx.read_ctx != NULL) { redisFree(maat_instance->mr_ctx.read_ctx); diff --git a/src/maat_table.cpp b/src/maat_table.cpp index 73a7a81..9e14640 100644 --- a/src/maat_table.cpp +++ b/src/maat_table.cpp @@ -12,7 +12,6 @@ #include #include "log/log.h" -#include "utils.h" #include "maat_utils.h" #include "maat_table.h" #include "maat_rule.h" @@ -26,7 +25,7 @@ #include "maat_ip_plugin.h" #include "maat_virtual.h" -#define MODULE_TABLE module_name_str("maat.table") +#define MODULE_TABLE module_name_str("maat.table") struct table_item { enum table_type table_type; @@ -50,21 +49,28 @@ struct table_manager { size_t n_accept_tag; int default_compile_table_id; + int g2g_table_id; struct maat_kv_store *tablename2id_map; + + struct maat_kv_store *district_map; + struct maat_kv_store *tmp_district_map; + + struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; }; struct table_operations { enum table_type type; - void *(*new_schema)(cJSON *json, const char *table_name, struct log_handle *logger); + void *(*new_schema)(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, + struct log_handle *logger); void (*free_schema)(void *schema); - void *(*new_runtime)(void *schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger); + void *(*new_runtime)(void *schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); void (*free_runtime)(void *runtime); int (*update_runtime)(void *runtime, void *schema, const char *line, int valid_column); int (*commit_runtime)(void *runtime); - int (*runtime_updating_flag)(void *runtime); }; struct table_operations table_ops[TABLE_TYPE_MAX] = { @@ -75,8 +81,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = expr_runtime_new, .free_runtime = expr_runtime_free, .update_runtime = expr_runtime_update, - .commit_runtime = expr_runtime_commit, - .runtime_updating_flag = expr_runtime_updating_flag + .commit_runtime = expr_runtime_commit }, { .type = TABLE_TYPE_EXPR_PLUS, @@ -85,18 +90,16 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = expr_runtime_new, .free_runtime = expr_runtime_free, .update_runtime = expr_runtime_update, - .commit_runtime = expr_runtime_commit, - .runtime_updating_flag = expr_runtime_updating_flag + .commit_runtime = expr_runtime_commit }, { .type = TABLE_TYPE_IP_PLUS, - .new_schema = ip_plus_schema_new, - .free_schema = ip_plus_schema_free, - .new_runtime = ip_plus_runtime_new, - .free_runtime = ip_plus_runtime_free, - .update_runtime = ip_plus_runtime_update, - .commit_runtime = ip_plus_runtime_commit, - .runtime_updating_flag = ip_plus_runtime_updating_flag + .new_schema = ip_schema_new, + .free_schema = ip_schema_free, + .new_runtime = ip_runtime_new, + .free_runtime = ip_runtime_free, + .update_runtime = ip_runtime_update, + .commit_runtime = ip_runtime_commit }, { .type = TABLE_TYPE_INTERVAL, @@ -105,8 +108,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL, - .runtime_updating_flag = NULL + .commit_runtime = NULL }, { .type = TABLE_TYPE_INTERVAL_PLUS, @@ -115,8 +117,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL, - .runtime_updating_flag = NULL + .commit_runtime = NULL }, { .type = TABLE_TYPE_DIGEST, @@ -125,8 +126,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL, - .runtime_updating_flag = NULL + .commit_runtime = NULL }, { .type = TABLE_TYPE_SIMILARITY, @@ -135,8 +135,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL, - .runtime_updating_flag = NULL + .commit_runtime = NULL }, { .type = TABLE_TYPE_CONJUNCTION, @@ -145,8 +144,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = NULL, .free_runtime = NULL, .update_runtime = NULL, - .commit_runtime = NULL, - .runtime_updating_flag = NULL + .commit_runtime = NULL }, { .type = TABLE_TYPE_PLUGIN, @@ -155,8 +153,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = plugin_runtime_new, .free_runtime = plugin_runtime_free, .update_runtime = plugin_runtime_update, - .commit_runtime = plugin_runtime_commit, - .runtime_updating_flag = plugin_runtime_updating_flag + .commit_runtime = plugin_runtime_commit }, { .type = TABLE_TYPE_IP_PLUGIN, @@ -165,8 +162,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = ip_plugin_runtime_new, .free_runtime = ip_plugin_runtime_free, .update_runtime = ip_plugin_runtime_update, - .commit_runtime = ip_plugin_runtime_commit, - .runtime_updating_flag = ip_plugin_runtime_updating_flag + .commit_runtime = ip_plugin_runtime_commit }, { .type = TABLE_TYPE_FQDN_PLUGIN, @@ -202,8 +198,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = compile_runtime_new, .free_runtime = compile_runtime_free, .update_runtime = compile_runtime_update, - .commit_runtime = compile_runtime_commit, - .runtime_updating_flag = NULL + .commit_runtime = compile_runtime_commit }, { .type = TABLE_TYPE_GROUP2GROUP, @@ -212,8 +207,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = group2group_runtime_new, .free_runtime = group2group_runtime_free, .update_runtime = group2group_runtime_update, - .commit_runtime = group2group_runtime_commit, - .runtime_updating_flag = NULL + .commit_runtime = group2group_runtime_commit }, { .type = TABLE_TYPE_GROUP2COMPILE, @@ -222,18 +216,19 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .new_runtime = group2compile_runtime_new, .free_runtime = group2compile_runtime_free, .update_runtime = group2compile_runtime_update, - .commit_runtime = NULL, - .runtime_updating_flag = NULL + .commit_runtime = NULL } }; -void *maat_table_schema_new(cJSON *json, const char *table_name, enum table_type table_type, +void *maat_table_schema_new(cJSON *json, const char *table_name, + enum table_type table_type, + struct table_manager *tbl_mgr, struct log_handle *logger) { void *schema = NULL; if (table_ops[table_type].new_schema != NULL) { - schema = table_ops[table_type].new_schema(json, table_name, logger); + schema = table_ops[table_type].new_schema(json, tbl_mgr, table_name, logger); } return schema; @@ -378,8 +373,9 @@ void maat_table_free(struct maat_table *maat_tbl) FREE(maat_tbl); } -struct table_manager *table_manager_create(const char *table_info_path, const char *accept_tags, - struct log_handle *logger) +struct table_manager * +table_manager_create(const char *table_info_path, const char *accept_tags, + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { if (NULL == table_info_path) { return NULL; @@ -389,7 +385,8 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch size_t json_buff_sz = 0; int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz); if (ret < 0) { - log_error(logger, MODULE_TABLE, "Maat read table info %s error.", table_info_path); + log_error(logger, MODULE_TABLE, + "Maat read table info %s error.", table_info_path); return NULL; } @@ -397,14 +394,16 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch cJSON *json = NULL; root = cJSON_Parse((const char *)json_buff); if (!root) { - log_error(logger, MODULE_TABLE, "Error before: %-200.200s", cJSON_GetErrorPtr()); + log_error(logger, MODULE_TABLE, + "Error before: %-200.200s", cJSON_GetErrorPtr()); FREE(json_buff); return NULL; } int json_array_size = cJSON_GetArraySize(root); if (json_array_size <= 0) { - log_error(logger, MODULE_TABLE, "invalid json content in %s", table_info_path); + log_error(logger, MODULE_TABLE, + "invalid json content in %s", table_info_path); free(json_buff); return NULL; } @@ -416,8 +415,11 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger); tbl_mgr->logger = logger; tbl_mgr->tablename2id_map = maat_kv_store_new(); + tbl_mgr->district_map = maat_kv_store_new(); + tbl_mgr->ref_garbage_bin = garbage_bin; int default_compile_table_id = MAX_TABLE_NUM; + int g2g_table_id = MAX_TABLE_NUM; for (int i = 0; i < json_array_size; i++) { json = cJSON_GetArrayItem(root, i); @@ -437,10 +439,11 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch continue; } - maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->table_type, logger); + maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, + maat_tbl->table_type, tbl_mgr, logger); if (NULL == maat_tbl->schema) { - log_error(logger, MODULE_TABLE, "Maat table schema new failed, table_name:%s", - maat_tbl->table_name); + log_error(logger, MODULE_TABLE, + "Maat table schema new failed, table_name:%s", maat_tbl->table_name); maat_table_free(maat_tbl); continue; } @@ -451,14 +454,23 @@ struct table_manager *table_manager_create(const char *table_info_path, const ch } } + if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) { + g2g_table_id = maat_tbl->table_id; + } + tbl_mgr->tbl[maat_tbl->table_id] = maat_tbl; tbl_mgr->n_table++; } } assert(default_compile_table_id != MAX_TABLE_NUM); + assert(g2g_table_id != MAX_TABLE_NUM); + tbl_mgr->default_compile_table_id = default_compile_table_id; + tbl_mgr->g2g_table_id = g2g_table_id; + log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id); + log_info(logger, MODULE_TABLE, "group2group table id: %d", g2g_table_id); maat_kv_store_free(reserved_word_map); cJSON_Delete(root); @@ -490,7 +502,6 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n assert(tbl_mgr->n_table != 0); size_t i = 0; - int g2g_group_id = MAX_TABLE_NUM; enum table_type table_type = TABLE_TYPE_MAX; for (i = 0; i < MAX_TABLE_NUM; i++) { @@ -500,17 +511,11 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n } table_type = table_manager_get_table_type(tbl_mgr, i); - if (table_type == TABLE_TYPE_GROUP2GROUP) { - g2g_group_id = i; - } - assert(NULL == tbl_mgr->tbl[i]->runtime); tbl_mgr->tbl[i]->runtime = maat_table_runtime_new(schema, table_type, max_thread_num, garbage_bin, tbl_mgr->logger); } - assert(g2g_group_id != MAX_TABLE_NUM); - /* group2compile runtime depends on associated compile runtime, must make sure associated compile runtime already exist */ for (i = 0; i < MAX_TABLE_NUM; i++) { @@ -528,10 +533,14 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n void *schema = table_manager_get_schema(tbl_mgr, i); int associated_compile_table_id = group2compile_associated_compile_table_id(schema); void *compile_rt = table_manager_get_runtime(tbl_mgr, associated_compile_table_id); + int g2g_group_id = table_manager_get_group2group_table_id(tbl_mgr); void *g2g_rt = table_manager_get_runtime(tbl_mgr, g2g_group_id); group2compile_runtime_init(runtime, compile_rt, g2g_rt); } + /* new district map */ + tbl_mgr->district_map = maat_kv_store_new(); + return 0; } @@ -551,6 +560,9 @@ void table_manager_runtime_destroy(struct table_manager *tbl_mgr) maat_table_runtime_free(runtime, table_type); tbl_mgr->tbl[i]->runtime = NULL; } + + /* free district map */ + maat_kv_store_free(tbl_mgr->district_map); } void table_manager_destroy(struct table_manager *tbl_mgr) @@ -580,6 +592,7 @@ void table_manager_destroy(struct table_manager *tbl_mgr) FREE(tbl_mgr->accept_tags); maat_kv_store_free(tbl_mgr->tablename2id_map); + FREE(tbl_mgr); } @@ -622,6 +635,11 @@ int table_manager_get_defaut_compile_table_id(struct table_manager *tbl_mgr) return tbl_mgr->default_compile_table_id; } +int table_manager_get_group2group_table_id(struct table_manager *tbl_mgr) +{ + return tbl_mgr->g2g_table_id; +} + void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id) { if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) { @@ -635,7 +653,8 @@ void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id) return tbl_mgr->tbl[table_id]->schema; } -struct ex_data_schema *table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id) +struct ex_data_schema * +table_manager_get_table_ex_data_schema(struct table_manager *tbl_mgr, int table_id) { return NULL; } @@ -653,9 +672,21 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id) return tbl_mgr->tbl[table_id]->valid_column; } +size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr) +{ + return tbl_mgr->n_accept_tag; +} + int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags) { - return 0; + return compare_accept_tag(tags, tbl_mgr->accept_tags, tbl_mgr->n_accept_tag); +} + +int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str, + size_t district_str_len, int *district_id) +{ + return maat_kv_read_unNull(tbl_mgr->district_map, + district_str, district_str_len, district_id); } void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id) @@ -671,26 +702,8 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id) return tbl_mgr->tbl[table_id]->runtime; } -int table_manager_runtime_updating_flag(struct table_manager *tbl_mgr, int table_id) -{ - if (NULL == tbl_mgr) { - return -1; - } - - if (NULL == tbl_mgr->tbl[table_id]) { - return -1; - } - - enum table_type table_type = tbl_mgr->tbl[table_id]->table_type; - void *runtime = table_manager_get_runtime(tbl_mgr, table_id); - if (table_ops[table_type].runtime_updating_flag != NULL) { - return table_ops[table_type].runtime_updating_flag(runtime); - } - - return -1; -} - -int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line) +int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, + const char *line) { void *schema = table_manager_get_schema(tbl_mgr, table_id); if (NULL == schema) { @@ -716,7 +729,16 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, co return -1; } - return table_ops[table_type].update_runtime(runtime, schema, line, valid_column); + int ret = table_ops[table_type].update_runtime(runtime, schema, line, valid_column); + + if (tbl_mgr->tmp_district_map != NULL) { + struct maat_kv_store *tmp_map = tbl_mgr->district_map; + tbl_mgr->district_map = tbl_mgr->tmp_district_map; + tbl_mgr->tmp_district_map = NULL; + maat_garbage_bagging(tbl_mgr->ref_garbage_bin, tmp_map, (void (*)(void *))maat_kv_store_free); + } + + return ret; } void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) diff --git a/src/maat_utils.cpp b/src/maat_utils.cpp index e44854e..d23e7ff 100644 --- a/src/maat_utils.cpp +++ b/src/maat_utils.cpp @@ -17,7 +17,6 @@ #include #include -#include "utils.h" #include "maat_utils.h" pid_t gettid() @@ -257,8 +256,8 @@ char *md5_file(const char *filename, char *md5string) return md5string; } -int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz, - const char *key, const char *algorithm, int do_encrypt, +int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, + size_t *out_sz, const char *key, const char *algorithm, int do_encrypt, char *err_str, size_t err_str_sz) { OpenSSL_add_all_algorithms(); @@ -325,7 +324,8 @@ error_out: } int decrypt_open(const char* file_name, const char* key, const char* algorithm, - unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz) + unsigned char**pp_out, size_t *out_sz, + char* err_str, size_t err_str_sz) { size_t file_sz = 0; unsigned char *file_buff = NULL; @@ -334,7 +334,8 @@ int decrypt_open(const char* file_name, const char* key, const char* algorithm, return -1; } - ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz); + ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, + 0, err_str, err_str_sz); FREE(file_buff); return ret; diff --git a/src/maat_virtual.cpp b/src/maat_virtual.cpp index a9b9bf3..5786a59 100644 --- a/src/maat_virtual.cpp +++ b/src/maat_virtual.cpp @@ -10,7 +10,6 @@ #include "cJSON/cJSON.h" #include "maat_kv.h" -#include "utils.h" #include "maat_utils.h" #include "log/log.h" #include "maat_virtual.h" @@ -19,19 +18,25 @@ struct virtual_schema { int physical_table_id[SCAN_TYPE_MAX]; + int table_id; + struct table_manager *tbl_mgr; }; -void *virtual_schema_new(cJSON *json, const char *table_name, struct log_handle *logger) +void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) { //size_t read_cnt = 0; cJSON *item = cJSON_GetObjectItem(json, "physical_table"); if (NULL == item || item->type != cJSON_Array) { - log_error(logger, MODULE_VIRTUAL, "virtual table %s has no physical_table column", table_name); + log_error(logger, MODULE_VIRTUAL, + "virtual table %s has no physical_table column", table_name); return NULL; } struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); + + vt_schema->tbl_mgr = tbl_mgr; return vt_schema; #if 0 struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); diff --git a/src/rcu_hash.cpp b/src/rcu_hash.cpp index 93d3382..5df0a43 100644 --- a/src/rcu_hash.cpp +++ b/src/rcu_hash.cpp @@ -14,7 +14,7 @@ #include #include "rcu_hash.h" -#include "utils.h" +#include "maat_utils.h" struct rcu_hash_garbage_bag { void *garbage; @@ -68,7 +68,8 @@ size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable) return htable->garbage_q_len; } -void rcu_hash_garbage_bagging(struct rcu_hash_garbage_q* garbage_q, void* garbage, void (* func)(void *)) +void rcu_hash_garbage_bagging(struct rcu_hash_garbage_q* garbage_q, + void* garbage, void (* func)(void *)) { struct rcu_hash_garbage_bag *bag = ALLOC(struct rcu_hash_garbage_bag, 1); @@ -171,7 +172,8 @@ void rcu_hash_commit_prepare(struct rcu_hash_table *htable) htable->is_updating = 1; } -void rcu_hash_add(struct rcu_hash_table *htable, const char *key, size_t key_len, void *data) +void rcu_hash_add(struct rcu_hash_table *htable, const char *key, + size_t key_len, void *data) { if (NULL == htable || NULL == key || 0 == key_len) { return; @@ -227,7 +229,8 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len } if (node != NULL) { - rcu_hash_garbage_bagging(&(htable->garbage_q), node, (void (*)(void*))rcu_hash_node_free); + rcu_hash_garbage_bagging(&(htable->garbage_q), node, + (void (*)(void*))rcu_hash_node_free); htable->garbage_q_len++; } } @@ -344,9 +347,4 @@ size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_a } return node_cnt; -} - -int rcu_hash_updating_flag(struct rcu_hash_table *htable) -{ - return htable->is_updating; } \ No newline at end of file diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 99a2b06..6006506 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -1,6 +1,5 @@ #include -#include "utils.h" #include "maat/maat.h" #include "maat_rule.h" #include "maat_utils.h" @@ -9,64 +8,95 @@ #include "json2iris.h" #include "maat_config_monitor.h" -struct maat *g_maat_instance = NULL; +#define MODULE_FRAMEWORK_GTEST module_name_str("maat.framework_gtest") + const char *table_info_path = "./table_info.conf"; const char *json_path="./maat_json.json"; const char *json_filename = "maat_json.json"; +struct maat *g_maat_instance = NULL; -TEST(maat_scan_string, hit_one_expr) { - int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); - +class MaatStringScan : public testing::Test +{ +protected: + static void SetUpTestCase() { + table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); + } + + static void TearDownTestCase() { + + } + + static int table_id; +}; +int MaatStringScan::table_id; + +TEST_F(MaatStringScan, hit_one_expr) { + int table_id = MaatStringScan::table_id; char scan_data[128] = "hello"; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_HIT); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 191); struct maat_hit_path hit_path[128] = {0}; int n_read = 0; n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, sizeof(hit_path)); + EXPECT_NE(n_read, 0); maat_state_free(&state); } -TEST(maat_scan_string, hit_two_expr) { - int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); - +TEST_F(MaatStringScan, hit_two_expr) { + int table_id = MaatStringScan::table_id; char data[128] = "should hit aaa bbb"; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 2); - EXPECT_EQ(results[0], 28); - EXPECT_EQ(results[1], 27); + EXPECT_EQ(results[0], 139); + EXPECT_EQ(results[1], 138); maat_state_free(&state); } -TEST(maat_scan_string, hit_three_expr) { - int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); +TEST_F(MaatStringScan, hit_three_expr) { + int table_id = MaatStringScan::table_id; char data[128] = "should hit aaa bbb C#中国"; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 3); - EXPECT_EQ(results[0], 28); - EXPECT_EQ(results[1], 27); - EXPECT_EQ(results[2], 18); + EXPECT_EQ(results[0], 139); + EXPECT_EQ(results[1], 138); + EXPECT_EQ(results[2], 129); maat_state_free(&state); } -TEST(maat_scan_ipv4, hit_ip_and_port) { - int table_id = maat_table_get_id(g_maat_instance, "IP_PLUS_CONFIG"); +class MaatIPScan : public testing::Test +{ +protected: + static void SetUpTestCase() { + table_id = maat_table_get_id(g_maat_instance, "IP_PLUS_CONFIG"); + } + + static void TearDownTestCase() { + + } + + static int table_id; +}; +int MaatIPScan::table_id; + +TEST_F(MaatIPScan, hit_ip_and_port) { + int table_id = MaatIPScan::table_id; char ip_str[32] = "192.168.58.19"; uint32_t sip; int ret = inet_pton(AF_INET, ip_str, &sip); @@ -76,20 +106,20 @@ TEST(maat_scan_ipv4, hit_ip_and_port) { size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 7); maat_state_free(&state); state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } - -TEST(maat_scan_ipv4, hit_ip_and_port_range) { +#if 0 +TEST_F(MaatStringScan, hit_ip_and_port_range) { int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); char ip_str[32] = "192.168.50.24"; uint32_t sip; @@ -100,7 +130,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) { size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 4); maat_state_free(&state); @@ -109,7 +139,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) { n_hit_result = 0; state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 4); maat_state_free(&state); @@ -118,7 +148,7 @@ TEST(maat_scan_ipv4, hit_ip_and_port_range) { n_hit_result = 0; state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(n_hit_result, 0); } @@ -135,7 +165,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 8); maat_state_free(&state); @@ -144,7 +174,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { EXPECT_EQ(ret, 1); state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 8); maat_state_free(&state); @@ -153,7 +183,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { EXPECT_EQ(ret, 1); state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } @@ -170,7 +200,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) { size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 50); maat_state_free(&state); @@ -179,7 +209,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) { EXPECT_EQ(ret, 1); state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } @@ -195,7 +225,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 2); EXPECT_EQ(results[0], 63); EXPECT_EQ(results[1], 67); @@ -203,7 +233,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } @@ -219,14 +249,14 @@ TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) { size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 47); maat_state_free(&state); state = NULL; ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } @@ -239,7 +269,7 @@ TEST(maat_scan_string, dynamic_config) { size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); @@ -255,7 +285,7 @@ TEST(maat_scan_string, dynamic_config) { sleep(2); state = NULL; ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 9999); maat_state_free(&state); @@ -273,7 +303,7 @@ TEST(maat_scan_ip, dynamic_config) { size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); @@ -289,11 +319,13 @@ TEST(maat_scan_ip, dynamic_config) { sleep(2); state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, 0); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 9998); maat_state_free(&state); } +#endif + int count_line_num_cb(const char *table_name, const char *line, void *u_para) { @@ -341,25 +373,22 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para) return 0; } -int main(int argc, char ** argv) +int write_config_to_redis(char *redis_ip, int redis_port, int redis_db, + struct log_handle *logger) { - int ret=0; - ::testing::InitGoogleTest(&argc, argv); - char json_iris_path[128] = {0}; - char redis_ip[64] = "127.0.0.1"; - int redis_port = 6379; - int redis_db = 0; - - struct log_handle *logger = log_handle_create("./tmp.log", 0); snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger); - EXPECT_NE(c, nullptr); + if (nullptr == c) { + return -1; + } redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb"); - EXPECT_NE(reply, nullptr); + if (nullptr == reply) { + return -1; + } if ((access(json_iris_path, F_OK)) < 0) { char tmp_iris_path[128] = {0}; @@ -367,11 +396,15 @@ int main(int argc, char ** argv) size_t json_buff_sz = 0; int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); - EXPECT_NE(ret, -1); + if (ret < 0) { + return -1; + } ret = json2iris(json_buff, json_filename, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger); - EXPECT_NE(ret, -1); + if (ret < 0) { + return -1; + } } size_t total_line_cnt = 0; @@ -381,7 +414,9 @@ int main(int argc, char ** argv) struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); long long server_time = maat_cmd_redis_server_time_s(c); - EXPECT_NE(server_time, -1); + if (server_time < 0) { + return -1; + } absolute_expire_time = server_time + 300; config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger); @@ -399,17 +434,47 @@ int main(int argc, char ** argv) FREE(s_rule); redisFree(c); + return 0; +} + +int main(int argc, char ** argv) +{ + int ret=0; + ::testing::InitGoogleTest(&argc, argv); + + struct log_handle *logger = log_handle_create("./maat_framework_gtest.log", 0); + if (NULL == logger) { + printf("create log handle failed.\n"); + return -1; + } + + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger); + if (ret < 0) { + log_error(logger, MODULE_FRAMEWORK_GTEST, "write config to redis failed."); + log_handle_destroy(logger); + return -1; + } + struct maat_options *opts = maat_options_new(); maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_logger(opts, logger); g_maat_instance = maat_new(opts, table_info_path); maat_options_free(opts); - - ret=RUN_ALL_TESTS(); - - log_handle_destroy(g_maat_instance->logger); + if (NULL == g_maat_instance) { + log_error(logger, MODULE_FRAMEWORK_GTEST, "create maat instance in MaatStringScan failed."); + log_handle_destroy(logger); + return -1; + } + + ret=RUN_ALL_TESTS(); + maat_free(g_maat_instance); + log_handle_destroy(logger); return ret; -} \ No newline at end of file +} diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index fab625f..391aa41 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -1,4 +1,3 @@ -#include "utils.h" #include "maat/maat.h" #include "maat_utils.h" #include "maat_rule.h" @@ -11,10 +10,6 @@ const char *table_info_path = "./table_info.conf"; const char *json_filename = "maat_json.json"; -TEST(EQ_Test, Always_True) { - EXPECT_EQ(1, 1); -} - TEST(json_mode, maat_scan_string) { char json_iris_path[128] = {0}; snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); @@ -38,7 +33,7 @@ TEST(json_mode, maat_scan_string) { size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_HIT); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 191); @@ -81,7 +76,7 @@ TEST(iris_mode, maat_scan_string) { size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_HIT); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 191); @@ -201,7 +196,7 @@ TEST(redis_mode, maat_scan_string) { size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_HIT); + EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 191); @@ -217,4 +212,4 @@ int main(int argc, char ** argv) ::testing::InitGoogleTest(&argc, argv); ret=RUN_ALL_TESTS(); return ret; -} \ No newline at end of file +} diff --git a/test/rcu_hash_gtest.cpp b/test/rcu_hash_gtest.cpp index 9504268..c97814c 100644 --- a/test/rcu_hash_gtest.cpp +++ b/test/rcu_hash_gtest.cpp @@ -1,5 +1,5 @@ -#include "../src/inc_internal/rcu_hash.h" -#include "../include/utils.h" +#include "rcu_hash.h" +#include "maat_utils.h" #include @@ -47,9 +47,6 @@ TEST(rcu_hash_add_one_node, single_thread) { ret = rcu_hash_list_updating_data(htable, &data_array); EXPECT_EQ(ret, 1); - ret = rcu_hash_updating_flag(htable); - EXPECT_EQ(ret, 1); - rcu_hash_commit(htable); /* find in hash after commit */ @@ -69,9 +66,6 @@ TEST(rcu_hash_add_one_node, single_thread) { ret = rcu_hash_list_updating_data(htable, &data_array); EXPECT_EQ(ret, 0); - ret = rcu_hash_updating_flag(htable); - EXPECT_EQ(ret, 0); - rcu_hash_free(htable); } @@ -112,9 +106,6 @@ TEST(rcu_hash_add_multi_node, single_thread) { ret = rcu_hash_list_updating_data(htable, &data_array); EXPECT_EQ(ret, 2); - ret = rcu_hash_updating_flag(htable); - EXPECT_EQ(ret, 1); - rcu_hash_commit(htable); /* find in hash after commit */ @@ -141,9 +132,6 @@ TEST(rcu_hash_add_multi_node, single_thread) { ret = rcu_hash_list_updating_data(htable, &data_array); EXPECT_EQ(ret, 0); - ret = rcu_hash_updating_flag(htable); - EXPECT_EQ(ret, 0); - rcu_hash_free(htable); } @@ -181,9 +169,6 @@ TEST(rcu_hash_del_one_node, single_thread) { ret = rcu_hash_list_updating_data(htable, &data_array); EXPECT_EQ(ret, 0); - ret = rcu_hash_updating_flag(htable); - EXPECT_EQ(ret, 1); - rcu_hash_commit(htable); /* find in hash after commit */ diff --git a/test/table_info.conf b/test/table_info.conf index 643631a..5aab16f 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -68,7 +68,7 @@ "valid_column":3, "custom": { "group_id":1, - "superior_group_id":2 + "super_group_id":2 } }, { @@ -125,12 +125,6 @@ "sport_format":7, "sport1":8, "sport2":9, - "daddr_format":10, - "dip1":11, - "dip2":12, - "dport_format":13, - "dport1":14, - "dport2":15, "proto":16, "direction":17 } diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 68306c6..17b5ece 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -5,7 +5,6 @@ #include #include -#include "utils.h" #include "maat_rule.h" #include "maat_utils.h" #include "maat_command.h" @@ -70,7 +69,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char int i = 0; int ret = 0; int line_count = 0; - int update_type = CM_UPDATE_TYPE_INC; + int update_type = MAAT_UPDATE_TYPE_INC; long long version = 0; const char *cur_table = NULL; char foreign_files_dir[256] = {0}; @@ -95,7 +94,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char return; } - assert(update_type == CM_UPDATE_TYPE_FULL); + assert(update_type == MAAT_UPDATE_TYPE_FULL); printf("MAAT Version: %lld, key number: %d\n", version, rule_num); if (0 == rule_num) { goto clean_up;