[BUGFIX]fix xx_plugin user_tag lifecycle

src/inc_internal/maat_ex_data.h

@@ -77,8 +77,8 @@ int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
 int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
                                       const char *key, size_t key_len);
 
-size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt,
+size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
-                                         struct ex_container ***ex_container);
+                                                  struct ex_container ***ex_container);
 
 size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt);
 

src/maat_bool_plugin.c

@@ -33,11 +33,11 @@ struct bool_plugin_schema {
 struct bool_plugin_runtime {
     struct bool_matcher *matcher;
     struct ex_data_runtime *ex_data_rt;
-    long long rule_num;
     size_t n_worker_thread;
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
 
+    long long rule_num;
     long long update_err_cnt;
 };
 
@@ -449,11 +449,9 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
         return 0;
     }
 
-    ex_data_runtime_commit(ex_data_rt);
-
     struct bool_expr *rules = NULL;
     struct ex_container **ex_container = NULL;
-    size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
+    size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
     if (rule_cnt > 0) {
         rules = ALLOC(struct bool_expr, rule_cnt);
         for (size_t i = 0; i < rule_cnt; i++) {
@@ -485,6 +483,8 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
 
     old_bool_matcher = bool_plugin_rt->matcher;
     bool_plugin_rt->matcher = new_bool_matcher;
+    ex_data_runtime_commit(ex_data_rt);
+
     if (old_bool_matcher != NULL) {
         maat_garbage_bagging(bool_plugin_rt->ref_garbage_bin, old_bool_matcher, NULL,
                              garbage_bool_matcher_free);

src/maat_ex_data.c

@@ -280,10 +280,10 @@ int ex_data_runtime_is_updating(struct ex_data_runtime *ex_data_rt)
     return rcu_hash_is_updating(ex_data_rt->htable);
 }
 
-size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt, 
+size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, 
-                                         struct ex_container ***ex_container)
+                                                  struct ex_container ***ex_container)
 {
-    return rcu_hash_list(ex_data_rt->htable, (void ***)ex_container);
+    return rcu_updating_hash_list(ex_data_rt->htable, (void ***)ex_container);
 }
 
 void ex_data_runtime_garbage_collect_routine(struct ex_data_runtime *ex_data_rt)

src/maat_fqdn_plugin.c

@@ -34,11 +34,12 @@ struct fqdn_plugin_schema {
 struct fqdn_plugin_runtime {
     struct FQDN_engine *engine;
     struct ex_data_runtime *ex_data_rt;
-    long long rule_num;
-    long long update_err_cnt;
     size_t n_worker_thread;
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
+
+    long long rule_num;
+    long long update_err_cnt;
 };
 
 void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
@@ -455,11 +456,9 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
         return 0;
     }
 
-    ex_data_runtime_commit(ex_data_rt);
-
     struct FQDN_rule *rules = NULL;
     struct ex_container **ex_container = NULL;
-    size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
+    size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
     if (rule_cnt > 0) {
         rules = ALLOC(struct FQDN_rule, rule_cnt);
         for (size_t i = 0; i < rule_cnt; i++) {
@@ -490,6 +489,8 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
 
     old_fqdn_engine = fqdn_plugin_rt->engine;
     fqdn_plugin_rt->engine = new_fqdn_engine;
+    ex_data_runtime_commit(ex_data_rt);
+
     if (old_fqdn_engine != NULL) {
         maat_garbage_bagging(fqdn_plugin_rt->ref_garbage_bin, old_fqdn_engine, NULL,
                              garbage_fqdn_engine_free);

src/maat_ip.c

@@ -66,7 +66,6 @@ struct ip_item {
 
 struct ip_runtime {
 	struct ip_matcher *ip_matcher;
-    struct interval_matcher *intval_matcher;
     struct rcu_hash_table *item_hash;  // <item_id, struct ip_item>
 
     long long rule_num;
@@ -428,11 +427,6 @@ void ip_runtime_free(void *ip_runtime)
         ip_rt->ip_matcher = NULL;
     }
 
-    if (ip_rt->intval_matcher != NULL) {
-        interval_matcher_free(ip_rt->intval_matcher);
-        ip_rt->intval_matcher = NULL;
-    }
-
     if (ip_rt->item_hash != NULL) {
         rcu_hash_free(ip_rt->item_hash);
         ip_rt->item_hash = NULL;
@@ -601,9 +595,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
     struct ip_matcher *new_ip_matcher = NULL;
     struct ip_matcher *old_ip_matcher = NULL;
 
-    struct interval_matcher *new_intval_matcher = NULL;
-    struct interval_matcher *old_intval_matcher = NULL;
-
     if (rule_cnt > 0) {
         new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
         if (NULL == new_ip_matcher) {
@@ -617,15 +608,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
                      "table[%s] commit %zu ip rules and rebuild ip_matcher completed"
                      ", version:%lld", table_name, rule_cnt, maat_rt_version);
         }
-
-        new_intval_matcher = interval_matcher_new(intval_rules, rule_cnt);
-        if (NULL == new_intval_matcher) {
-            log_error(ip_rt->logger, MODULE_IP,
-                      "[%s:%d] table[%s] rebuild interval_matcher engine "
-                      "failed when update %zu ip rules", __FUNCTION__, __LINE__,
-                      table_name, rule_cnt);
-            ret = -1;
-        }
     }
 
     old_ip_matcher = ip_rt->ip_matcher;
@@ -637,13 +619,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
                              garbage_ip_matcher_free);
     }
 
-    old_intval_matcher = ip_rt->intval_matcher;
-    ip_rt->intval_matcher = new_intval_matcher;
-    if (old_intval_matcher != NULL) {
-        maat_garbage_bagging(ip_rt->ref_garbage_bin, old_intval_matcher, NULL,
-                             garbage_interval_matcher_free);
-    }
-
     ip_rt->rule_num = rule_cnt;
     
     if (rules != NULL) {

src/maat_ip_plugin.c

@@ -38,11 +38,12 @@ struct ip_plugin_schema {
 struct ip_plugin_runtime {
 	struct ip_matcher *ip_matcher;
     struct ex_data_runtime *ex_data_rt;
-    long long rule_num;
-    long long update_err_cnt;
     size_t n_worker_thread;
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
+
+    long long rule_num;
+    long long update_err_cnt;
 };
 
 void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
@@ -494,11 +495,9 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
         return 0;
     }
 
-    ex_data_runtime_commit(ex_data_rt);
-
     struct ip_rule *rules = NULL;
     struct ex_container **ex_container = NULL;
-    size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
+    size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
     if (rule_cnt > 0) {
         rules = ALLOC(struct ip_rule, rule_cnt);
         for (size_t i = 0; i < rule_cnt; i++) {
@@ -529,6 +528,8 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
 
     old_ip_matcher = ip_plugin_rt->ip_matcher;
     ip_plugin_rt->ip_matcher = new_ip_matcher;
+    ex_data_runtime_commit(ex_data_rt);
+
     if (old_ip_matcher != NULL) {
         maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, NULL,
                              garbage_ip_matcher_free);

src/maat_ipport_plugin.c

@@ -57,15 +57,14 @@ struct ipport_item {
 
 struct ipport_plugin_runtime {
     struct ip_matcher *ip_matcher;
-    struct interval_matcher *intval_matcher;
     struct rcu_hash_table *item_hash;  //<item_id, struct ipport_item>
-
-    long long rule_num;
-    long long update_err_cnt;
     struct ex_data_runtime *ex_data_rt;
     size_t n_worker_thread;
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
+
+    long long rule_num;
+    long long update_err_cnt;
 };
 
 void *ipport_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
@@ -239,11 +238,6 @@ void ipport_plugin_runtime_free(void *ipport_plugin_runtime)
         ipport_plugin_rt->ip_matcher = NULL;
     }
 
-    if (ipport_plugin_rt->intval_matcher != NULL) {
-        interval_matcher_free(ipport_plugin_rt->intval_matcher);
-        ipport_plugin_rt->intval_matcher = NULL;
-    }
-
     if (ipport_plugin_rt->ex_data_rt != NULL) {
         ex_data_runtime_free(ipport_plugin_rt->ex_data_rt);
         ipport_plugin_rt->ex_data_rt = NULL;
@@ -477,11 +471,9 @@ int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_
         return 0;
     }
 
-    ex_data_runtime_commit(ex_data_rt);
-
     struct ip_rule *rules = NULL;
     struct ex_container **ex_container = NULL;
-    size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
+    size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
     if (rule_cnt > 0) {
         rules = ALLOC(struct ip_rule, rule_cnt);
         for (size_t i = 0; i < rule_cnt; i++) {
@@ -512,6 +504,8 @@ int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_
 
     old_ip_matcher = ipport_plugin_rt->ip_matcher;
     ipport_plugin_rt->ip_matcher = new_ip_matcher;
+    ex_data_runtime_commit(ex_data_rt);
+
     if (old_ip_matcher != NULL) {
         maat_garbage_bagging(ipport_plugin_rt->ref_garbage_bin, old_ip_matcher, NULL,
                              garbage_ip_matcher_free);

src/maat_plugin.c

@@ -32,12 +32,13 @@ struct plugin_callback_schema {
 };
 
 struct plugin_runtime {
-    long long acc_line_num;
     struct ex_data_runtime *ex_data_rt;
-    long long rule_num;
-    long long update_err_cnt;
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
+
+    long long rule_num;
+    long long acc_line_num;
+    long long update_err_cnt;
 };
 
 enum plugin_key_type {

src/maat_utils.c

@@ -318,7 +318,7 @@ int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_ou
 	int ret = EVP_BytesToKey(cipher, dgst, salt, (unsigned char *)key, 
                              strlen((const char *)key), 1, cipher_key, cipher_iv);
 	if(0 == ret) {
-		snprintf(err_str, err_str_sz, "Key and IV generatioin failed.");
+		snprintf(err_str, err_str_sz, "Key and IV generation failed.");
 		return 0;
 	}
 

test/ipport_plugin/ipport_plugin_gtest.cpp

@@ -15,36 +15,22 @@
 
 const char *table_info_path = "./ipport_plugin_table_info.conf";
 const char *log_file = "./ipport_plugin_gtest.log";
+const char *g_ip_str = "192.0.2.0";
 
 class IPPortPluginTable : public testing::Test
 {
 protected:
     static void SetUpTestCase() {
         logger = log_handle_create(log_file, 0);
-        
-        // char iris_idx_path[PATH_MAX] = {0};
-        // snprintf(iris_idx_path, sizeof(iris_idx_path), "test_data/index");
-       
-        // struct maat_options *opts = maat_options_new();
-        // maat_options_set_iris(opts, iris_idx_path, iris_idx_path);
-        // maat_options_set_logger(opts, log_file, LOG_LEVEL_INFO);
-        // maat_options_set_stat_file(opts, "./stat.log");
-
-        // struct maat *_shared_maat_inst = maat_new(opts, table_info_path);
-        // assert(_shared_maat_inst);
-        // maat_options_free(opts);
     }
 
     static void TearDownTestCase() {
         log_handle_destroy(logger);
-        // maat_free(_shared_maat_inst);
     }
 
     static struct log_handle *logger;
-    // static struct maat *_shared_maat_inst;
 };
 
-// struct maat *IPPortPluginTable::_shared_maat_inst;
 struct log_handle *IPPortPluginTable::logger;
 
 int count_line_num_cb(const char *table_name, const char *line, void *u_para)
@@ -207,7 +193,7 @@ void *ipport_plugin_scan_thread(void *arg)
 
     struct ip_addr ipv4;
     ipv4.ip_type = 4;
-    int ret = inet_pton(AF_INET, "192.0.2.0", &ipv4.ipv4);
+    int ret = inet_pton(AF_INET, g_ip_str, &ipv4.ipv4);
     EXPECT_EQ(ret, 1);
 
     uint16_t port = htons(200);
@@ -236,7 +222,7 @@ void *ipport_plugin_scan_thread(void *arg)
 }
 
 static void test_add_ipport_plugin_command(struct maat *maat_inst, const char *table_name,
-                                           const char *ip_str, long long item_id)
+                                           long long item_id, const char *ip_str, int port1, int port2)
 {
     int table_id = maat_get_table_id(maat_inst, table_name);
     assert(table_id >= 0);
@@ -246,7 +232,29 @@ static void test_add_ipport_plugin_command(struct maat *maat_inst, const char *t
 
 
     char table_line[1024] = {0};
-    sprintf(table_line, "%lld\t4\t%s\t100\t200\t1", item_id, ip_str);
+    sprintf(table_line, "%lld\t4\t%s\t%d\t%d\t1", item_id, ip_str, port1, port2);
+
+    struct maat_cmd_line line_rule;
+    line_rule.rule_id = item_id;
+    line_rule.table_line = table_line;
+    line_rule.table_name = table_name;
+    line_rule.expire_after = 0;
+
+    maat_cmd_set_line(maat_inst, &line_rule);
+}
+
+static void test_del_ipport_plugin_command(struct maat *maat_inst, const char *table_name,
+                                           long long item_id, const char *ip_str, int port1, int port2)
+{
+    int table_id = maat_get_table_id(maat_inst, table_name);
+    assert(table_id >= 0);
+    
+    enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
+    assert(table_type == TABLE_TYPE_IPPORT_PLUGIN);
+
+
+    char table_line[1024] = {0};
+    sprintf(table_line, "%lld\t4\t%s\t%d\t%d\t0", item_id, ip_str, port1, port2);
 
     struct maat_cmd_line line_rule;
     line_rule.rule_id = item_id;
@@ -262,17 +270,22 @@ void *ipport_plugin_update_thread(void *arg)
     struct thread_param *param = (struct thread_param *)arg;
     struct maat *maat_inst = param->maat_inst;
     const char *table_name = param->table_name;
-    const int CMD_EXPR_NUM = 100;
+    const int CMD_EXPR_NUM = 1024;
-    char ip_buf[128] = {0};
     long long item_id = 9000000;
 
     for (int i = 0; i < CMD_EXPR_NUM; i++) {
-        sprintf(ip_buf, "172.16.1.%d", i);
+        test_add_ipport_plugin_command(maat_inst, table_name, item_id, g_ip_str, i+201, i+201);
-        test_add_ipport_plugin_command(maat_inst, table_name, ip_buf, item_id);
         item_id++;
         usleep(100 * 1000);
     }
 
+    item_id = 9000000; 
+    for (int i = 0; i < CMD_EXPR_NUM; i++) {
+        test_del_ipport_plugin_command(maat_inst, table_name, item_id, g_ip_str, i+201, i+201);
+        usleep(100 * 1000);
+        item_id++;
+    }
+
     int *is_all_hit = ALLOC(int, 1);
     *is_all_hit = 1;
     

test/ipport_plugin/ipport_plugin_table_info.conf

@@ -5,7 +5,6 @@
         "table_type":"ipport_plugin",
         "valid_column":6,
         "custom": {
-            "gc_timeout_s": 3,
             "item_id":1,
             "ip_type":2,
             "ip_addr":3,

test/maat_framework_gtest.cpp

@@ -322,79 +322,6 @@ static void random_keyword_generate(char *keyword_buf, size_t sz)
 	keyword_buf[i]='\0';
 	return;
 }
-#if 0
-class MaatIris : public testing::Test
-{
-protected:
-    static void SetUpTestCase() {
-        logger = log_handle_create("./maat_framework_gtest.log", 0);        
-
-        struct maat_options *opts = maat_options_new();
-        maat_options_set_iris(opts, "./redis_dump", "./redis_dump");
-        maat_options_set_stat_file(opts, "./stat.log");
-        maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
-
-        const char *table_info_path = "./verify_table_info.conf";
-        _shared_maat_inst = maat_new(opts, table_info_path);
-        maat_options_free(opts);
-        if (NULL == _shared_maat_inst) {
-            log_error(logger, MODULE_FRAMEWORK_GTEST,
-                      "[%s:%d] create maat instance in MaatIris failed.",
-                      __FUNCTION__, __LINE__);
-        }
-    }
-
-    static void TearDownTestCase() {
-        maat_free(_shared_maat_inst);
-        log_handle_destroy(logger);
-    }
-
-    static struct log_handle *logger;
-    static struct maat *_shared_maat_inst;
-};
-
-struct maat *MaatIris::_shared_maat_inst;
-struct log_handle *MaatIris::logger;
-
-TEST_F(MaatIris, basic) {
-    struct maat *maat_inst = MaatIris::_shared_maat_inst;
-    struct log_handle *logger = MaatIris::logger;
-
-    long long results[ARRAY_SIZE] = {0};
-    size_t n_hit_result = 0;
-    int thread_id = 0;
-    struct maat_state *state = maat_state_new(maat_inst, thread_id);
-
-    uint32_t sip_addr;
-    inet_pton(AF_INET, "100.64.1.1", &sip_addr);
-    uint16_t sport = htons(80);
-  
-
-    int table_id = maat_get_table_id(maat_inst, "TSG_SECURITY_SOURCE_ADDR");
-    ASSERT_GT(table_id, 0);
-    int ret = maat_scan_ipv4(maat_inst, table_id, sip_addr, sport, 6,
-                             results, ARRAY_SIZE, &n_hit_result, state);
-    EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
-
-    table_id = maat_get_table_id(maat_inst, "TSG_OBJ_APP_ID");
-    ASSERT_GT(table_id, 0);
-
-    ret = maat_scan_integer(maat_inst, table_id, 105, results, ARRAY_SIZE,
-                            &n_hit_result, state);
-    EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
-    
-    table_id = maat_get_table_id(maat_inst, "TSG_FIELD_HTTP_URL");
-    ASSERT_GT(table_id, 0);
-
-    const char *scan_data = "www.luis.com";
-    ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
-                           results, ARRAY_SIZE, &n_hit_result, state);
-    EXPECT_EQ(ret, MAAT_SCAN_HIT);
-    EXPECT_EQ(n_hit_result, 1);
-    EXPECT_EQ(results[0], 1267067);
-    maat_state_free(state);
-}
-#endif
 
 const char* watched_json="./json_update/maat.json";
 const char* old_json="./json_update/old.json";