From 84a271144b4e1edfe8855f4bdd3f696763cf7b6c Mon Sep 17 00:00:00 2001 From: liuwentan Date: Tue, 29 Nov 2022 14:12:40 +0800 Subject: [PATCH] fix memory leak and add framework test --- scanner/adapter_hs.cpp | 6 +- scanner/adapter_hs.h | 2 +- src/CMakeLists.txt | 6 - src/inc_internal/maat_table_runtime.h | 11 +- src/inc_internal/maat_table_schema.h | 8 +- src/inc_internal/maat_utils.h | 3 + src/inc_internal/rcu_hash.h | 4 +- src/maat_api.cpp | 14 +- src/maat_config_monitor.cpp | 10 +- src/maat_rule.cpp | 11 +- src/maat_table_runtime.cpp | 87 ++++++++-- src/maat_table_schema.cpp | 6 +- src/maat_utils.cpp | 8 + src/rcu_hash.cpp | 1 - test/maat_framework_gtest.cpp | 151 +++++++++++++++++- test/rcu_hash_gtest.cpp | 38 ++++- test/rule/full/2022-11-24/HTTP_URL.000001 | 2 - test/rule/full/HTTP_URL.000001 | 7 + test/rule/full/index/full_config_index.000001 | 2 +- 19 files changed, 321 insertions(+), 56 deletions(-) delete mode 100644 test/rule/full/2022-11-24/HTTP_URL.000001 create mode 100644 test/rule/full/HTTP_URL.000001 diff --git a/scanner/adapter_hs.cpp b/scanner/adapter_hs.cpp index 46f0896..2ff4a5f 100644 --- a/scanner/adapter_hs.cpp +++ b/scanner/adapter_hs.cpp @@ -285,7 +285,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads /* create bool matcher */ hs_instance->hs_rt->bm = bool_matcher_new(exprs, n_expr_array, &mem_size); if (hs_instance->hs_rt->bm != NULL) { - fprintf(stdout, "Adapter_hs module: build bool matcher of %u expressions with %u bytes memory.", + fprintf(stdout, "Adapter_hs module: build bool matcher of %zu expressions with %zu bytes memory.", n_expr_array, mem_size); } else { fprintf(stderr, "Adapter_hs module: build bool matcher failed."); @@ -395,6 +395,10 @@ int matched_event_cb(unsigned int id, unsigned long long from, int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *data, size_t data_len, int results[], size_t *n_results) { + if (NULL == hs_instance || NULL == data || (0 == data_len) || NULL == results || NULL == n_results) { + return -1; + } + struct adapter_hs_runtime *hs_rt = hs_instance->hs_rt; hs_scratch_t *scratch = hs_rt->scratchs[thread_id]; UT_array *pattern_id_set; diff --git a/scanner/adapter_hs.h b/scanner/adapter_hs.h index f395a47..3d6f9f3 100644 --- a/scanner/adapter_hs.h +++ b/scanner/adapter_hs.h @@ -32,7 +32,7 @@ enum scan_mode { /* pattern type: PATTERN_TYPE_STR(pure literal string) or PATTERN_TYPE_REG(regex expression) */ enum pattern_type { - PATTERN_TYPE_STR = 1, + PATTERN_TYPE_STR = 0, PATTERN_TYPE_REG, }; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 29aa90f..1e1b481 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,9 +11,6 @@ add_definitions(-fPIC) set(MAAT_SRC maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp) -set(MAAT_SRC_M main.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp - maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp) - set(LIB_SOURCE_FILES ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c) @@ -30,9 +27,6 @@ set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) target_link_libraries(maat_frame_static adapter-static pthread) -add_executable(main ${MAAT_SRC_M}) -target_link_libraries(main maat_frame_static) - # Shared Library Output #add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) #set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX) diff --git a/src/inc_internal/maat_table_runtime.h b/src/inc_internal/maat_table_runtime.h index f665af1..bc6db8c 100644 --- a/src/inc_internal/maat_table_runtime.h +++ b/src/inc_internal/maat_table_runtime.h @@ -56,14 +56,15 @@ int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, int result[], size_t n_result); void table_runtime_stream_close(struct table_runtime *table_rt); -/* plugin table runtime API */ -size_t plugin_table_runtime_cached_row_count(struct table_runtime *table_rt); +/* table runtime cached row API */ +size_t table_runtime_cached_row_count(struct table_runtime *table_rt); -const char* plugin_table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq); +const char* table_runtime_get_cached_row(struct table_runtime *table_rt, size_t row_seq); -void *plugin_table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema, const char *key, size_t key_len); +/* table runtime ex data API */ +void *table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema, const char *key, size_t key_len); -void plugin_table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema); +void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema); #ifdef __cpluscplus } diff --git a/src/inc_internal/maat_table_schema.h b/src/inc_internal/maat_table_schema.h index 440067f..570b591 100644 --- a/src/inc_internal/maat_table_schema.h +++ b/src/inc_internal/maat_table_schema.h @@ -21,8 +21,9 @@ extern "C" #include "maat/maat.h" #include "adapter_hs.h" +#define MAX_TABLE_NUM 256 #define MAX_DISTRICT_STR 128 -#define MAX_IP_STR 128 +#define MAX_IP_STR 128 #define MAX_KEYWORDS_STR 1024 enum table_type { @@ -35,10 +36,9 @@ enum table_type { }; enum expr_type { - EXPR_TYPE_STRING = 1, - EXPR_TYPE_REGEX, + EXPR_TYPE_STRING = 0, EXPR_TYPE_AND, - EXPR_TYPE_OFFSET, + EXPR_TYPE_REGEX, EXPR_TYPE_MAX }; diff --git a/src/inc_internal/maat_utils.h b/src/inc_internal/maat_utils.h index 4ef044b..3d26770 100644 --- a/src/inc_internal/maat_utils.h +++ b/src/inc_internal/maat_utils.h @@ -42,6 +42,9 @@ char *str_unescape_and(char *s); char *str_unescape(char *s); +/* system cmd wrapper */ +int system_cmd_mkdir(const char* path); + #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/rcu_hash.h b/src/inc_internal/rcu_hash.h index ee22a6f..dfb8255 100644 --- a/src/inc_internal/rcu_hash.h +++ b/src/inc_internal/rcu_hash.h @@ -1,7 +1,7 @@ /* ********************************************************************************************** -* File: maat_rhash.h -* Description: maat rcu hashtable +* File: rcu_hash.h +* Description: rcu hashtable * Authors: Liu WenTan * Date: 2022-10-31 * Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. diff --git a/src/maat_api.cpp b/src/maat_api.cpp index 9dcb0ce..544bebe 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -207,14 +207,14 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id, } struct table_runtime *table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id); - size_t row_count = plugin_table_runtime_cached_row_count(table_rt); + size_t row_count = table_runtime_cached_row_count(table_rt); if (row_count > 0) { if (start != NULL) { start(MAAT_RULE_UPDATE_TYPE_FULL, u_para); } for (size_t i = 0; i < row_count; i++) { - const char *line = plugin_table_runtime_get_cached_row(table_rt, i); + const char *line = table_runtime_get_cached_row(table_rt, i); if (NULL == line) { break; } @@ -249,7 +249,7 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i struct table_runtime *table_rt = NULL; if (maat_instance->maat_rt != NULL) { table_rt = table_runtime_get(maat_instance->maat_rt->table_rt_mgr, table_id); - plugin_table_runtime_commit_ex_data_schema(table_rt, table_schema); + table_runtime_commit_ex_data_schema(table_rt, table_schema); } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); @@ -267,7 +267,7 @@ void *maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id, struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); struct table_runtime *table_rt = table_runtime_get(maat_rt->table_rt_mgr, table_id); - return plugin_table_runtime_get_ex_data(table_rt, table_schema, key, key_len); + return table_runtime_get_ex_data(table_rt, table_schema, key, key_len); } int maat_scan_integer(struct maat *instance, int table_id, int thread_id, @@ -288,6 +288,12 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, const char *data, size_t data_len, int results[], size_t *n_results, struct maat_state *state) { + if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || + (thread_id < 0) || (NULL == data) || (0 == data_len) || (NULL == results) || + (NULL == n_results)) { + return -1; + } + struct table_runtime_manager *table_rt_mgr = maat_instance->maat_rt->table_rt_mgr; struct table_runtime *table_rt = table_runtime_get(table_rt_mgr, table_id); diff --git a/src/maat_config_monitor.cpp b/src/maat_config_monitor.cpp index 69c5862..9a9ab5d 100644 --- a/src/maat_config_monitor.cpp +++ b/src/maat_config_monitor.cpp @@ -238,13 +238,15 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id } if (strlen(namelist[i]->d_name) > 42) { - //log_error("config file %s filename too long,should like full_config_index.00000000000000000001",namelist[i]->d_name); + fprintf(stderr, "config file %s filename too long,should like full_config_index.00000000000000000001", + namelist[i]->d_name); continue; } int ret = sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld", update_str, &config_seq); if (ret != 2) { - //log_error("config file %s filename error,should like full_config_index.00000000000000000001",namelist[i]->d_name); + fprintf(stderr, "config file %s filename error,should like full_config_index.00000000000000000001", + namelist[i]->d_name); continue; } @@ -262,7 +264,7 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***id } } } else { - //log_error("config file %s,not full or inc config",namelist[i]->d_name); + fprintf(stderr, "config file %s,not full or inc config", namelist[i]->d_name); } } @@ -317,7 +319,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir, for (i = 0; i < idx_path_num; i++) { int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, CM_MAX_TABLE_NUM); if (table_num < 0) { - //log_error luis + fprintf(stderr, "load %s failed, abandon udpate.", idx_path_array[i]); break; } diff --git a/src/maat_rule.cpp b/src/maat_rule.cpp index fed4857..45e57b9 100644 --- a/src/maat_rule.cpp +++ b/src/maat_rule.cpp @@ -140,14 +140,14 @@ void maat_finish_cb(void *u_param) if (maat_instance->creating_maat_rt != NULL) { maat_instance->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_instance->creating_maat_rt); maat_runtime_commit(maat_instance->creating_maat_rt); - fprintf(stdout, "Full config version %llu load %d entries complete", + fprintf(stdout, "Full config version %llu load %d entries complete\n", maat_instance->creating_maat_rt->version, maat_instance->creating_maat_rt->rule_num); } else if (maat_instance->maat_rt != NULL) { maat_instance->maat_rt->rule_num = maat_runtime_rule_num(maat_instance->maat_rt); maat_instance->maat_rt->version = maat_instance->maat_version; maat_runtime_commit(maat_instance->maat_rt); - fprintf(stdout, "Inc config version %llu load %d entries complete", + fprintf(stdout, "Inc config version %llu load %d entries complete\n", maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); } @@ -198,6 +198,13 @@ void *rule_monitor_loop(void *arg) maat_instance->maat_rt = maat_instance->creating_maat_rt; if (old_maat_rt != NULL) { + if (maat_instance->maat_rt->version > old_maat_rt->version) { + fprintf(stdout, "Maat version updated %lld -> %lld.", + old_maat_rt->version, maat_instance->maat_rt->version); + } else { + fprintf(stdout, "Maat version roll back %lld -> %lld.", + old_maat_rt->version, maat_instance->maat_rt->version); + } maat_garbage_bagging(maat_instance->garbage_bin, old_maat_rt, (void (*)(void*))maat_runtime_destroy); } diff --git a/src/maat_table_runtime.cpp b/src/maat_table_runtime.cpp index 3ebe716..a9eaa98 100644 --- a/src/maat_table_runtime.cpp +++ b/src/maat_table_runtime.cpp @@ -88,8 +88,19 @@ void plugin_ex_data_free(void *user_ctx, void *data) ctx->ex_schema->free_func(ctx->table_id, &data, argl, argp); } +void expr_rule_free(and_expr_t *expr_rule) +{ + if (expr_rule != NULL) { + for (size_t i = 0; i < expr_rule->n_patterns; i++) { + free(expr_rule->patterns[i].pat); + expr_rule->patterns[i].pat = NULL; + } + } +} void expr_ex_data_free(void *user_ctx, void *data) { + and_expr_t *expr_rule = (and_expr_t *)data; + expr_rule_free(expr_rule); free(data); } @@ -164,6 +175,10 @@ table_runtime_manager_create(struct table_schema_manager *table_schema_mgr, int void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr) { + if (NULL == table_rt_mgr) { + return; + } + for(size_t i = 0; i < table_rt_mgr->n_table_rt; i++) { table_runtime_free(table_rt_mgr->table_rt[i]); table_rt_mgr->table_rt[i] = NULL; @@ -176,28 +191,48 @@ void table_runtime_manager_destroy(struct table_runtime_manager *table_rt_mgr) struct table_runtime *table_runtime_get(struct table_runtime_manager *table_rt_mgr, int table_id) { + if (NULL == table_rt_mgr || (table_id < 0) || (table_id >= MAX_TABLE_NUM)) { + return NULL; + } + assert(table_id < (int)table_rt_mgr->n_table_rt); return table_rt_mgr->table_rt[table_id]; } size_t table_runtime_rule_count(struct table_runtime *table_rt) { + if (NULL == table_rt) { + return 0; + } + return table_rt->rule_num; } enum table_type table_runtime_get_type(struct table_runtime* table_rt) { + if (NULL == table_rt) { + return TABLE_TYPE_MAX; + } + return table_rt->table_type; } int table_runtime_scan_string(struct table_runtime* table_rt, int thread_id, const char *data, size_t data_len, int result[], size_t *n_result) { + if (NULL == table_rt) { + return -1; + } + return adapter_hs_scan(table_rt->expr_rt.hs, thread_id, data, data_len, result, n_result); } void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id) { + if (NULL == table_rt) { + return; + } + struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(table_rt->expr_rt.hs, thread_id); table_rt->expr_rt.hs_stream = hs_stream; } @@ -205,13 +240,19 @@ void table_runtime_stream_open(struct table_runtime *table_rt, int thread_id) int table_runtime_scan_stream(struct table_runtime *table_rt, const char *data, size_t data_len, int result[], size_t *n_result) { + if (NULL == table_rt) { + return -1; + } + return adapter_hs_scan_stream(table_rt->expr_rt.hs_stream, data, data_len, result, n_result); } void table_runtime_stream_close(struct table_runtime *table_rt) { - adapter_hs_stream_close(table_rt->expr_rt.hs_stream); - table_rt->expr_rt.hs_stream = NULL; + if (table_rt != NULL) { + adapter_hs_stream_close(table_rt->expr_rt.hs_stream); + table_rt->expr_rt.hs_stream = NULL; + } } struct ip_rule *ip_plugin_item_to_ip_rule(struct ip_plugin_item *ip_plugin_item) @@ -219,6 +260,24 @@ struct ip_rule *ip_plugin_item_to_ip_rule(struct ip_plugin_item *ip_plugin_item) return NULL; } +enum pattern_type expr_type2pattern_type(enum expr_type expr_type) +{ + enum pattern_type pattern_type = PATTERN_TYPE_STR; + + switch (expr_type) { + case EXPR_TYPE_STRING: + case EXPR_TYPE_AND: + break; + case EXPR_TYPE_REGEX: + pattern_type = PATTERN_TYPE_REG; + break; + default: + break; + } + + return pattern_type; +} + and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item) { size_t i = 0; @@ -265,7 +324,7 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item) expr_rule->patterns[i].pat = ALLOC(char, strlen(sub_key_array[i])); memcpy(expr_rule->patterns[i].pat, sub_key_array[i], strlen(sub_key_array[i])); expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]); - expr_rule->patterns[i].type = expr_item->expr_type; + expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); } expr_rule->n_patterns = sub_expr_cnt; @@ -281,7 +340,7 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key //delete data = rcu_hash_find(expr_rt->htable, key, key_len); if (NULL == data) { - //log_error("the key:%s not exist, so can't be deleted."); + fprintf(stderr, "the key:%s not exist, so can't be deleted.", key); return; } rcu_hash_del(expr_rt->htable, key, key_len); @@ -289,11 +348,15 @@ void expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key //add data = rcu_hash_find(expr_rt->htable, key, key_len); if (data != NULL) { - //log_error("the key:%s already exist, so can't be added."); + fprintf(stderr, "the key:%s already exist, so can't be added.", key); return; } and_expr_t *data = ALLOC(and_expr_t, 1); memcpy(data, expr_rule, sizeof(and_expr_t)); + for (size_t i = 0; i < expr_rule->n_patterns; i++) { + data->patterns[i].pat = ALLOC(char, expr_rule->patterns[i].pat_len); + memcpy(data->patterns[i].pat, expr_rule->patterns[i].pat, expr_rule->patterns[i].pat_len); + } rcu_hash_add(expr_rt->htable, key, key_len, (void *)data); } } @@ -367,7 +430,7 @@ void table_runtime_update(struct table_runtime *table_rt, struct table_schema *t expr_rule = expr_item_to_expr_rule(&table_item->expr_item); key = (char *)&(table_item->expr_item.item_id); expr_runtime_update_row(&(table_rt->expr_rt), key, sizeof(int), expr_rule, is_valid); - free(expr_rule); + expr_rule_free(expr_rule); break; case TABLE_TYPE_PLUGIN: is_valid = table_item->plugin_item.is_valid; @@ -423,7 +486,9 @@ int expr_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_thread) maat_garbage_bagging(table_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy); rcu_hash_commit(expr_rt->htable); table_rt->rule_num = rcu_hash_count(expr_rt->htable); - + rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array); + assert(rule_cnt == 0); + free(rules); free(ex_data_array); @@ -528,7 +593,7 @@ void table_runtime_commit(struct table_runtime *table_rt, size_t nr_worker_threa } } -size_t plugin_table_runtime_cached_row_count(struct table_runtime* table_rt) +size_t table_runtime_cached_row_count(struct table_runtime* table_rt) { size_t row_count = 0; struct ex_data_runtime *ex_data_rt = NULL; @@ -545,7 +610,7 @@ size_t plugin_table_runtime_cached_row_count(struct table_runtime* table_rt) return row_count; } -const char* plugin_table_runtime_get_cached_row(struct table_runtime* table_rt, size_t row_seq) +const char* table_runtime_get_cached_row(struct table_runtime* table_rt, size_t row_seq) { const char *line = NULL; struct ex_data_runtime *ex_data_rt = NULL; @@ -562,7 +627,7 @@ const char* plugin_table_runtime_get_cached_row(struct table_runtime* table_rt, return line; } -void *plugin_table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema, +void *table_runtime_get_ex_data(struct table_runtime *table_rt, struct table_schema *table_schema, const char *key, size_t key_len) { void *ex_data = NULL; @@ -604,7 +669,7 @@ struct ex_data_runtime *table_runtime_get_ex_data_rt(struct table_runtime *table return ex_data_rt; } -void plugin_table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema) +void table_runtime_commit_ex_data_schema(struct table_runtime *table_rt, struct table_schema *table_schema) { struct ex_data_schema *ex_data_schema = plugin_table_schema_get_ex_data_schema(table_schema); struct ex_data_runtime *ex_data_rt = table_runtime_get_ex_data_rt(table_rt); diff --git a/src/maat_table_schema.cpp b/src/maat_table_schema.cpp index bc3ed15..48d44eb 100644 --- a/src/maat_table_schema.cpp +++ b/src/maat_table_schema.cpp @@ -21,7 +21,6 @@ #include "adapter_hs.h" #include "maat_ex_data.h" -#define MAX_TABLE_NUM 256 #define MAX_TABLE_LINE_SIZE (1024 * 16) #define MAX_FOREIGN_CLMN_NUM 8 @@ -574,9 +573,6 @@ enum expr_type int_to_expr_type(int expr_type) case 2: type = EXPR_TYPE_REGEX; break; - case 3: - type = EXPR_TYPE_OFFSET; - break; default: break; } @@ -700,7 +696,7 @@ int populate_expr_table_item(const char *line, struct expr_table_schema *expr_sc expr_item->is_case_sensitive = TRUE; break; default: - //fprintf(stderr, "invalid hexbin value of expr table"); + fprintf(stderr, "invalid hexbin value of expr table"); return -1; } ret = get_column_pos(line, expr_schema->is_valid_column, &column_offset, &column_len); diff --git a/src/maat_utils.cpp b/src/maat_utils.cpp index 016418e..444704f 100644 --- a/src/maat_utils.cpp +++ b/src/maat_utils.cpp @@ -178,4 +178,12 @@ char *str_unescape(char *s) } s[j] = '\0'; return s; +} + +#define MAX_SYSTEM_CMD_LEN 512 +int system_cmd_mkdir(const char *path) +{ + char cmd[MAX_SYSTEM_CMD_LEN] = {0}; + snprintf(cmd, sizeof(cmd), "mkdir -p %s", path); + return system(cmd); } \ No newline at end of file diff --git a/src/rcu_hash.cpp b/src/rcu_hash.cpp index dfb330a..624d7e8 100644 --- a/src/rcu_hash.cpp +++ b/src/rcu_hash.cpp @@ -305,7 +305,6 @@ size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_a size_t node_cnt = 0; struct rcu_hash_node *node = NULL, *tmp = NULL; - assert(htable->is_updating == 1); if (htable->effective_hash == 'a') { node_cnt = HASH_CNT(hh_b, htable->hashmap_b); *data_array = ALLOC(void *, node_cnt); diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 74a7d5f..714f1dc 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -2,23 +2,161 @@ #include "maat/maat.h" #include "maat_rule.h" +#include "maat_utils.h" #include "maat_table_schema.h" #include "maat_table_runtime.h" struct maat *g_maat_instance = NULL; -const char *table_info_path = "/home/liuwentan/project/maat-v4/test/table_info.conf"; -const char *rule_path = "/home/liuwentan/project/maat-v4/test/rule/full/index"; +const char *table_info_path = "./table_info.conf"; +const char *rule_full_path = "./rule/full/index"; +const char *rule_inc_path = "./rule/inc/index"; -TEST(maat_scan_string, literal) { +TEST(maat_scan_string, hit_one_expr) { struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); - char data[64] = "www.baidu.com"; + char data[64] = ""; + int result_array[5] = {0}; + size_t n_result_array = 0; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, -1); + + char data1[64] = "baidu.com"; + ret = maat_scan_string(g_maat_instance, table_id, 0, data1, strlen(data1), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 0); + + char data2[64] = "hello"; + ret = maat_scan_string(g_maat_instance, table_id, 0, data2, strlen(data2), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 1); + EXPECT_EQ(result_array[0], 101); + + memset(result_array, 0, sizeof(result_array)); + char data3[64] = "maat"; + ret = maat_scan_string(g_maat_instance, table_id, 0, data3, strlen(data3), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 1); + EXPECT_EQ(result_array[0], 102); + + memset(result_array, 0, sizeof(result_array)); + char data4[64] = "world"; + ret = maat_scan_string(g_maat_instance, table_id, 0, data4, strlen(data4), result_array, &n_result_array, NULL); + EXPECT_EQ(n_result_array, 1); + EXPECT_EQ(result_array[0], 103); +} + +TEST(maat_scan_string, hit_two_expr) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + char data[64] = "hello maat"; int result_array[5] = {0}; size_t n_result_array = 0; int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 2); + EXPECT_EQ(result_array[0], 102); + EXPECT_EQ(result_array[1], 101); + + memset(result_array, 0, sizeof(result_array)); + char data1[64] = "maat world"; + ret = maat_scan_string(g_maat_instance, table_id, 0, data1, strlen(data1), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 2); + EXPECT_EQ(result_array[0], 103); + EXPECT_EQ(result_array[1], 102); +} + +TEST(maat_scan_string, hit_three_expr) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + char data[64] = "hello world"; + int result_array[5] = {0}; + size_t n_result_array = 0; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 3); + EXPECT_EQ(result_array[0], 104); + EXPECT_EQ(result_array[1], 103); + EXPECT_EQ(result_array[2], 101); +} + +TEST(maat_scan_string, hit_four_expr) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + char data[64] = "9hello world"; + int result_array[5] = {0}; + size_t n_result_array = 0; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 4); + EXPECT_EQ(result_array[0], 105); + EXPECT_EQ(result_array[1], 104); + EXPECT_EQ(result_array[2], 103); + EXPECT_EQ(result_array[3], 101); +} + +TEST(maat_scan_string, hit_five_expr) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + char data[64] = "9hello 8world"; + int result_array[5] = {0}; + size_t n_result_array = 0; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 5); + EXPECT_EQ(result_array[0], 106); + EXPECT_EQ(result_array[1], 105); + EXPECT_EQ(result_array[2], 104); + EXPECT_EQ(result_array[3], 103); + EXPECT_EQ(result_array[4], 101); +} + +TEST(maat_scan_string, config_dynamic_update) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + + char data[128] = "www.baidu.com"; + int result_array[5] = {0}; + size_t n_result_array = 0; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 0); + + /* generate inc update config */ + system_cmd_mkdir("./rule/inc/index"); + char new_rule[128] = "207 7 baidu 0 0 0 1"; + + FILE *fp = fopen("./rule/inc/HTTP_URL.000002", "w+"); + EXPECT_NE(fp, nullptr); + fprintf(fp, "%s\n", "0000000001"); + fprintf(fp, "%s\n", new_rule); + fclose(fp); + + char new_rule_idx[128] = "HTTP_URL 1 ./rule/inc/HTTP_URL.000002"; + fp = fopen("./rule/inc/index/inc_config_index.000002", "w+"); + EXPECT_NE(fp, nullptr); + fprintf(fp, "%s\n", new_rule_idx); + fclose(fp); + + /* if updated rule already valid */ + memset(result_array, 0, sizeof(result_array)); + sleep(2); + ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 101); + EXPECT_EQ(result_array[0], 207); + + /* if the old rules are still valid */ + memset(result_array, 0, sizeof(result_array)); + char data1[64] = "9hello 8world"; + ret = maat_scan_string(g_maat_instance, table_id, 0, data1, strlen(data1), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 5); + EXPECT_EQ(result_array[0], 106); + EXPECT_EQ(result_array[1], 105); + EXPECT_EQ(result_array[2], 104); + EXPECT_EQ(result_array[3], 103); + EXPECT_EQ(result_array[4], 101); } int main(int argc, char ** argv) @@ -27,7 +165,8 @@ int main(int argc, char ** argv) ::testing::InitGoogleTest(&argc, argv); struct maat_options *opts = maat_options_new(); - maat_options_set_iris_full_dir(opts, rule_path); + maat_options_set_iris_full_dir(opts, rule_full_path); + maat_options_set_iris_inc_dir(opts, rule_inc_path); g_maat_instance = maat_new(opts, table_info_path); EXPECT_NE(g_maat_instance, nullptr); diff --git a/test/rcu_hash_gtest.cpp b/test/rcu_hash_gtest.cpp index 204023d..4198518 100644 --- a/test/rcu_hash_gtest.cpp +++ b/test/rcu_hash_gtest.cpp @@ -43,6 +43,13 @@ TEST(rcu_hash_add_one_node, single_thread) { ret = rcu_hash_garbage_queue_len(htable); EXPECT_EQ(ret, 0); + void **data_array = NULL; + ret = rcu_hash_list_updating_data(htable, &data_array); + EXPECT_EQ(ret, 1); + + ret = rcu_hash_updating_flag(htable); + EXPECT_EQ(ret, 1); + rcu_hash_commit(htable); /* find in hash after commit */ @@ -59,6 +66,12 @@ TEST(rcu_hash_add_one_node, single_thread) { ret = rcu_hash_garbage_queue_len(htable); EXPECT_EQ(ret, 0); + ret = rcu_hash_list_updating_data(htable, &data_array); + EXPECT_EQ(ret, 0); + + ret = rcu_hash_updating_flag(htable); + EXPECT_EQ(ret, 0); + rcu_hash_free(htable); } @@ -95,6 +108,13 @@ TEST(rcu_hash_add_multi_node, single_thread) { ret = rcu_hash_garbage_queue_len(htable); EXPECT_EQ(ret, 0); + void **data_array = NULL; + ret = rcu_hash_list_updating_data(htable, &data_array); + EXPECT_EQ(ret, 2); + + ret = rcu_hash_updating_flag(htable); + EXPECT_EQ(ret, 1); + rcu_hash_commit(htable); /* find in hash after commit */ @@ -118,6 +138,12 @@ TEST(rcu_hash_add_multi_node, single_thread) { ret = rcu_hash_garbage_queue_len(htable); EXPECT_EQ(ret, 0); + ret = rcu_hash_list_updating_data(htable, &data_array); + EXPECT_EQ(ret, 0); + + ret = rcu_hash_updating_flag(htable); + EXPECT_EQ(ret, 0); + rcu_hash_free(htable); } @@ -136,11 +162,15 @@ TEST(rcu_hash_del_one_node, single_thread) { /* add to hash */ rcu_hash_add(htable, key, key_len, (void *)data); + void **data_array = NULL; + int ret = rcu_hash_list_updating_data(htable, &data_array); + EXPECT_EQ(ret, 1); + /* find in hash before commit */ void *res = rcu_hash_find(htable, key, key_len); EXPECT_EQ(res, nullptr); - int ret = rcu_hash_count(htable); + ret = rcu_hash_count(htable); EXPECT_EQ(ret, 0); ret = rcu_hash_garbage_queue_len(htable); @@ -148,6 +178,12 @@ TEST(rcu_hash_del_one_node, single_thread) { rcu_hash_del(htable, key, key_len); + ret = rcu_hash_list_updating_data(htable, &data_array); + EXPECT_EQ(ret, 0); + + ret = rcu_hash_updating_flag(htable); + EXPECT_EQ(ret, 1); + rcu_hash_commit(htable); /* find in hash after commit */ diff --git a/test/rule/full/2022-11-24/HTTP_URL.000001 b/test/rule/full/2022-11-24/HTTP_URL.000001 deleted file mode 100644 index 57a9e99..0000000 --- a/test/rule/full/2022-11-24/HTTP_URL.000001 +++ /dev/null @@ -1,2 +0,0 @@ -0000000001 -101 1 www.baidu.com 0 0 0 1 \ No newline at end of file diff --git a/test/rule/full/HTTP_URL.000001 b/test/rule/full/HTTP_URL.000001 new file mode 100644 index 0000000..71524fe --- /dev/null +++ b/test/rule/full/HTTP_URL.000001 @@ -0,0 +1,7 @@ +0000000006 +101 1 hello 0 0 0 1 +102 2 maat 0 0 0 1 +103 3 [W|w]orld 2 0 0 1 +104 4 hello&world 1 0 0 1 +105 5 [0-9]hello&world 2 0 0 1 +106 6 [0-9]hello&[0-9]world 2 0 0 1 \ No newline at end of file diff --git a/test/rule/full/index/full_config_index.000001 b/test/rule/full/index/full_config_index.000001 index 0087cff..f16a255 100644 --- a/test/rule/full/index/full_config_index.000001 +++ b/test/rule/full/index/full_config_index.000001 @@ -1 +1 @@ -HTTP_URL 1 /home/liuwentan/project/maat-v4/test/rule/full/2022-11-24/HTTP_URL.000001 \ No newline at end of file +HTTP_URL 6 /home/liuwentan/project/maat-v4/test/rule/full/HTTP_URL.000001 \ No newline at end of file