diff --git a/src/inc_internal/maat_ex_data.h b/src/inc_internal/maat_ex_data.h index cde4729..6aa35c9 100644 --- a/src/inc_internal/maat_ex_data.h +++ b/src/inc_internal/maat_ex_data.h @@ -77,8 +77,8 @@ int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); -size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt, - struct ex_container ***ex_container); +size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, + struct ex_container ***ex_container); size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt); diff --git a/src/maat_bool_plugin.c b/src/maat_bool_plugin.c index 1fa1c56..f8a5fe8 100644 --- a/src/maat_bool_plugin.c +++ b/src/maat_bool_plugin.c @@ -33,11 +33,11 @@ struct bool_plugin_schema { struct bool_plugin_runtime { struct bool_matcher *matcher; struct ex_data_runtime *ex_data_rt; - long long rule_num; size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + long long rule_num; long long update_err_cnt; }; @@ -449,11 +449,9 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name return 0; } - ex_data_runtime_commit(ex_data_rt); - struct bool_expr *rules = NULL; struct ex_container **ex_container = NULL; - size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container); + size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); if (rule_cnt > 0) { rules = ALLOC(struct bool_expr, rule_cnt); for (size_t i = 0; i < rule_cnt; i++) { @@ -485,6 +483,8 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name old_bool_matcher = bool_plugin_rt->matcher; bool_plugin_rt->matcher = new_bool_matcher; + ex_data_runtime_commit(ex_data_rt); + if (old_bool_matcher != NULL) { maat_garbage_bagging(bool_plugin_rt->ref_garbage_bin, old_bool_matcher, NULL, garbage_bool_matcher_free); diff --git a/src/maat_ex_data.c b/src/maat_ex_data.c index da9be96..af7d4fc 100644 --- a/src/maat_ex_data.c +++ b/src/maat_ex_data.c @@ -280,10 +280,10 @@ int ex_data_runtime_is_updating(struct ex_data_runtime *ex_data_rt) return rcu_hash_is_updating(ex_data_rt->htable); } -size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt, - struct ex_container ***ex_container) +size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, + struct ex_container ***ex_container) { - return rcu_hash_list(ex_data_rt->htable, (void ***)ex_container); + return rcu_updating_hash_list(ex_data_rt->htable, (void ***)ex_container); } void ex_data_runtime_garbage_collect_routine(struct ex_data_runtime *ex_data_rt) diff --git a/src/maat_fqdn_plugin.c b/src/maat_fqdn_plugin.c index 58409b0..fd62759 100644 --- a/src/maat_fqdn_plugin.c +++ b/src/maat_fqdn_plugin.c @@ -34,11 +34,12 @@ struct fqdn_plugin_schema { struct fqdn_plugin_runtime { struct FQDN_engine *engine; struct ex_data_runtime *ex_data_rt; - long long rule_num; - long long update_err_cnt; size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + + long long rule_num; + long long update_err_cnt; }; void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -455,11 +456,9 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name return 0; } - ex_data_runtime_commit(ex_data_rt); - struct FQDN_rule *rules = NULL; struct ex_container **ex_container = NULL; - size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container); + size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); if (rule_cnt > 0) { rules = ALLOC(struct FQDN_rule, rule_cnt); for (size_t i = 0; i < rule_cnt; i++) { @@ -490,6 +489,8 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name old_fqdn_engine = fqdn_plugin_rt->engine; fqdn_plugin_rt->engine = new_fqdn_engine; + ex_data_runtime_commit(ex_data_rt); + if (old_fqdn_engine != NULL) { maat_garbage_bagging(fqdn_plugin_rt->ref_garbage_bin, old_fqdn_engine, NULL, garbage_fqdn_engine_free); diff --git a/src/maat_ip.c b/src/maat_ip.c index 0ce331e..471d4f1 100644 --- a/src/maat_ip.c +++ b/src/maat_ip.c @@ -66,7 +66,6 @@ struct ip_item { struct ip_runtime { struct ip_matcher *ip_matcher; - struct interval_matcher *intval_matcher; struct rcu_hash_table *item_hash; // long long rule_num; @@ -428,11 +427,6 @@ void ip_runtime_free(void *ip_runtime) ip_rt->ip_matcher = NULL; } - if (ip_rt->intval_matcher != NULL) { - interval_matcher_free(ip_rt->intval_matcher); - ip_rt->intval_matcher = NULL; - } - if (ip_rt->item_hash != NULL) { rcu_hash_free(ip_rt->item_hash); ip_rt->item_hash = NULL; @@ -601,9 +595,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, struct ip_matcher *new_ip_matcher = NULL; struct ip_matcher *old_ip_matcher = NULL; - struct interval_matcher *new_intval_matcher = NULL; - struct interval_matcher *old_intval_matcher = NULL; - if (rule_cnt > 0) { new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_ip_matcher) { @@ -617,15 +608,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, "table[%s] commit %zu ip rules and rebuild ip_matcher completed" ", version:%lld", table_name, rule_cnt, maat_rt_version); } - - new_intval_matcher = interval_matcher_new(intval_rules, rule_cnt); - if (NULL == new_intval_matcher) { - log_error(ip_rt->logger, MODULE_IP, - "[%s:%d] table[%s] rebuild interval_matcher engine " - "failed when update %zu ip rules", __FUNCTION__, __LINE__, - table_name, rule_cnt); - ret = -1; - } } old_ip_matcher = ip_rt->ip_matcher; @@ -637,13 +619,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name, garbage_ip_matcher_free); } - old_intval_matcher = ip_rt->intval_matcher; - ip_rt->intval_matcher = new_intval_matcher; - if (old_intval_matcher != NULL) { - maat_garbage_bagging(ip_rt->ref_garbage_bin, old_intval_matcher, NULL, - garbage_interval_matcher_free); - } - ip_rt->rule_num = rule_cnt; if (rules != NULL) { diff --git a/src/maat_ip_plugin.c b/src/maat_ip_plugin.c index e84e921..57a7fad 100644 --- a/src/maat_ip_plugin.c +++ b/src/maat_ip_plugin.c @@ -38,11 +38,12 @@ struct ip_plugin_schema { struct ip_plugin_runtime { struct ip_matcher *ip_matcher; struct ex_data_runtime *ex_data_rt; - long long rule_num; - long long update_err_cnt; size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + + long long rule_num; + long long update_err_cnt; }; void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -494,11 +495,9 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name, return 0; } - ex_data_runtime_commit(ex_data_rt); - struct ip_rule *rules = NULL; struct ex_container **ex_container = NULL; - size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container); + size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); if (rule_cnt > 0) { rules = ALLOC(struct ip_rule, rule_cnt); for (size_t i = 0; i < rule_cnt; i++) { @@ -529,6 +528,8 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name, old_ip_matcher = ip_plugin_rt->ip_matcher; ip_plugin_rt->ip_matcher = new_ip_matcher; + ex_data_runtime_commit(ex_data_rt); + if (old_ip_matcher != NULL) { maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, NULL, garbage_ip_matcher_free); diff --git a/src/maat_ipport_plugin.c b/src/maat_ipport_plugin.c index c96d1bc..e9979a8 100644 --- a/src/maat_ipport_plugin.c +++ b/src/maat_ipport_plugin.c @@ -57,15 +57,14 @@ struct ipport_item { struct ipport_plugin_runtime { struct ip_matcher *ip_matcher; - struct interval_matcher *intval_matcher; struct rcu_hash_table *item_hash; // - - long long rule_num; - long long update_err_cnt; struct ex_data_runtime *ex_data_rt; size_t n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + + long long rule_num; + long long update_err_cnt; }; void *ipport_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -239,11 +238,6 @@ void ipport_plugin_runtime_free(void *ipport_plugin_runtime) ipport_plugin_rt->ip_matcher = NULL; } - if (ipport_plugin_rt->intval_matcher != NULL) { - interval_matcher_free(ipport_plugin_rt->intval_matcher); - ipport_plugin_rt->intval_matcher = NULL; - } - if (ipport_plugin_rt->ex_data_rt != NULL) { ex_data_runtime_free(ipport_plugin_rt->ex_data_rt); ipport_plugin_rt->ex_data_rt = NULL; @@ -477,11 +471,9 @@ int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_ return 0; } - ex_data_runtime_commit(ex_data_rt); - struct ip_rule *rules = NULL; struct ex_container **ex_container = NULL; - size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container); + size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); if (rule_cnt > 0) { rules = ALLOC(struct ip_rule, rule_cnt); for (size_t i = 0; i < rule_cnt; i++) { @@ -512,6 +504,8 @@ int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_ old_ip_matcher = ipport_plugin_rt->ip_matcher; ipport_plugin_rt->ip_matcher = new_ip_matcher; + ex_data_runtime_commit(ex_data_rt); + if (old_ip_matcher != NULL) { maat_garbage_bagging(ipport_plugin_rt->ref_garbage_bin, old_ip_matcher, NULL, garbage_ip_matcher_free); diff --git a/src/maat_plugin.c b/src/maat_plugin.c index 66e8f39..ea4b520 100644 --- a/src/maat_plugin.c +++ b/src/maat_plugin.c @@ -32,12 +32,13 @@ struct plugin_callback_schema { }; struct plugin_runtime { - long long acc_line_num; struct ex_data_runtime *ex_data_rt; - long long rule_num; - long long update_err_cnt; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; + + long long rule_num; + long long acc_line_num; + long long update_err_cnt; }; enum plugin_key_type { diff --git a/src/maat_utils.c b/src/maat_utils.c index 71becc5..f6b8b93 100644 --- a/src/maat_utils.c +++ b/src/maat_utils.c @@ -318,7 +318,7 @@ int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_ou int ret = EVP_BytesToKey(cipher, dgst, salt, (unsigned char *)key, strlen((const char *)key), 1, cipher_key, cipher_iv); if(0 == ret) { - snprintf(err_str, err_str_sz, "Key and IV generatioin failed."); + snprintf(err_str, err_str_sz, "Key and IV generation failed."); return 0; } diff --git a/test/ipport_plugin/ipport_plugin_gtest.cpp b/test/ipport_plugin/ipport_plugin_gtest.cpp index d3f7754..365af8c 100644 --- a/test/ipport_plugin/ipport_plugin_gtest.cpp +++ b/test/ipport_plugin/ipport_plugin_gtest.cpp @@ -15,36 +15,22 @@ const char *table_info_path = "./ipport_plugin_table_info.conf"; const char *log_file = "./ipport_plugin_gtest.log"; +const char *g_ip_str = "192.0.2.0"; class IPPortPluginTable : public testing::Test { protected: static void SetUpTestCase() { logger = log_handle_create(log_file, 0); - - // char iris_idx_path[PATH_MAX] = {0}; - // snprintf(iris_idx_path, sizeof(iris_idx_path), "test_data/index"); - - // struct maat_options *opts = maat_options_new(); - // maat_options_set_iris(opts, iris_idx_path, iris_idx_path); - // maat_options_set_logger(opts, log_file, LOG_LEVEL_INFO); - // maat_options_set_stat_file(opts, "./stat.log"); - - // struct maat *_shared_maat_inst = maat_new(opts, table_info_path); - // assert(_shared_maat_inst); - // maat_options_free(opts); } static void TearDownTestCase() { log_handle_destroy(logger); - // maat_free(_shared_maat_inst); } static struct log_handle *logger; - // static struct maat *_shared_maat_inst; }; -// struct maat *IPPortPluginTable::_shared_maat_inst; struct log_handle *IPPortPluginTable::logger; int count_line_num_cb(const char *table_name, const char *line, void *u_para) @@ -207,7 +193,7 @@ void *ipport_plugin_scan_thread(void *arg) struct ip_addr ipv4; ipv4.ip_type = 4; - int ret = inet_pton(AF_INET, "192.0.2.0", &ipv4.ipv4); + int ret = inet_pton(AF_INET, g_ip_str, &ipv4.ipv4); EXPECT_EQ(ret, 1); uint16_t port = htons(200); @@ -236,7 +222,7 @@ void *ipport_plugin_scan_thread(void *arg) } static void test_add_ipport_plugin_command(struct maat *maat_inst, const char *table_name, - const char *ip_str, long long item_id) + long long item_id, const char *ip_str, int port1, int port2) { int table_id = maat_get_table_id(maat_inst, table_name); assert(table_id >= 0); @@ -246,7 +232,29 @@ static void test_add_ipport_plugin_command(struct maat *maat_inst, const char *t char table_line[1024] = {0}; - sprintf(table_line, "%lld\t4\t%s\t100\t200\t1", item_id, ip_str); + sprintf(table_line, "%lld\t4\t%s\t%d\t%d\t1", item_id, ip_str, port1, port2); + + struct maat_cmd_line line_rule; + line_rule.rule_id = item_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = 0; + + maat_cmd_set_line(maat_inst, &line_rule); +} + +static void test_del_ipport_plugin_command(struct maat *maat_inst, const char *table_name, + long long item_id, const char *ip_str, int port1, int port2) +{ + int table_id = maat_get_table_id(maat_inst, table_name); + assert(table_id >= 0); + + enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id); + assert(table_type == TABLE_TYPE_IPPORT_PLUGIN); + + + char table_line[1024] = {0}; + sprintf(table_line, "%lld\t4\t%s\t%d\t%d\t0", item_id, ip_str, port1, port2); struct maat_cmd_line line_rule; line_rule.rule_id = item_id; @@ -262,17 +270,22 @@ void *ipport_plugin_update_thread(void *arg) struct thread_param *param = (struct thread_param *)arg; struct maat *maat_inst = param->maat_inst; const char *table_name = param->table_name; - const int CMD_EXPR_NUM = 100; - char ip_buf[128] = {0}; + const int CMD_EXPR_NUM = 1024; long long item_id = 9000000; for (int i = 0; i < CMD_EXPR_NUM; i++) { - sprintf(ip_buf, "172.16.1.%d", i); - test_add_ipport_plugin_command(maat_inst, table_name, ip_buf, item_id); + test_add_ipport_plugin_command(maat_inst, table_name, item_id, g_ip_str, i+201, i+201); item_id++; usleep(100 * 1000); } + item_id = 9000000; + for (int i = 0; i < CMD_EXPR_NUM; i++) { + test_del_ipport_plugin_command(maat_inst, table_name, item_id, g_ip_str, i+201, i+201); + usleep(100 * 1000); + item_id++; + } + int *is_all_hit = ALLOC(int, 1); *is_all_hit = 1; diff --git a/test/ipport_plugin/ipport_plugin_table_info.conf b/test/ipport_plugin/ipport_plugin_table_info.conf index d5dc24d..05a7aae 100644 --- a/test/ipport_plugin/ipport_plugin_table_info.conf +++ b/test/ipport_plugin/ipport_plugin_table_info.conf @@ -5,7 +5,6 @@ "table_type":"ipport_plugin", "valid_column":6, "custom": { - "gc_timeout_s": 3, "item_id":1, "ip_type":2, "ip_addr":3, diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 37412e6..8c8841b 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -322,79 +322,6 @@ static void random_keyword_generate(char *keyword_buf, size_t sz) keyword_buf[i]='\0'; return; } -#if 0 -class MaatIris : public testing::Test -{ -protected: - static void SetUpTestCase() { - logger = log_handle_create("./maat_framework_gtest.log", 0); - - struct maat_options *opts = maat_options_new(); - maat_options_set_iris(opts, "./redis_dump", "./redis_dump"); - maat_options_set_stat_file(opts, "./stat.log"); - maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); - - const char *table_info_path = "./verify_table_info.conf"; - _shared_maat_inst = maat_new(opts, table_info_path); - maat_options_free(opts); - if (NULL == _shared_maat_inst) { - log_error(logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatIris failed.", - __FUNCTION__, __LINE__); - } - } - - static void TearDownTestCase() { - maat_free(_shared_maat_inst); - log_handle_destroy(logger); - } - - static struct log_handle *logger; - static struct maat *_shared_maat_inst; -}; - -struct maat *MaatIris::_shared_maat_inst; -struct log_handle *MaatIris::logger; - -TEST_F(MaatIris, basic) { - struct maat *maat_inst = MaatIris::_shared_maat_inst; - struct log_handle *logger = MaatIris::logger; - - long long results[ARRAY_SIZE] = {0}; - size_t n_hit_result = 0; - int thread_id = 0; - struct maat_state *state = maat_state_new(maat_inst, thread_id); - - uint32_t sip_addr; - inet_pton(AF_INET, "100.64.1.1", &sip_addr); - uint16_t sport = htons(80); - - - int table_id = maat_get_table_id(maat_inst, "TSG_SECURITY_SOURCE_ADDR"); - ASSERT_GT(table_id, 0); - int ret = maat_scan_ipv4(maat_inst, table_id, sip_addr, sport, 6, - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - table_id = maat_get_table_id(maat_inst, "TSG_OBJ_APP_ID"); - ASSERT_GT(table_id, 0); - - ret = maat_scan_integer(maat_inst, table_id, 105, results, ARRAY_SIZE, - &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - - table_id = maat_get_table_id(maat_inst, "TSG_FIELD_HTTP_URL"); - ASSERT_GT(table_id, 0); - - const char *scan_data = "www.luis.com"; - ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), - results, ARRAY_SIZE, &n_hit_result, state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 1267067); - maat_state_free(state); -} -#endif const char* watched_json="./json_update/maat.json"; const char* old_json="./json_update/old.json";