optimize district & support virtual table conjunction

include/maat.h

@@ -229,8 +229,8 @@ void maat_state_reset(struct maat_state *state);
 
 void maat_state_free(struct maat_state *state);
 
-int maat_state_set_scan_district(struct maat_state *state, const char *district, 
+int maat_state_set_scan_district(struct maat_state *state, int table_id, 
-                                 size_t district_len);
+                                 const char *district, size_t district_len);
 
 int maat_state_set_last_scan(struct maat_state *state);
 

src/inc_internal/maat_ex_data.h

@@ -36,6 +36,7 @@ struct ex_data_schema {
 struct ex_container_schema {
     int table_id;
     struct ex_data_schema *ex_schema;
+    void (*user_data_free)(void *);
 };
 
 struct ex_data_runtime;

src/inc_internal/maat_expr.h

@@ -52,6 +52,9 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_str
                              int vtable_id, struct maat_state *state);
 void expr_runtime_stream_close(struct adapter_hs_stream *s_handle);
 
+int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district, 
+                                   size_t district_len, long long *district_id);
+
 void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id);
 long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread);
 

src/inc_internal/maat_flag.h

@@ -48,6 +48,9 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name);
 int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, long long flag,
                       int vtable_id, struct maat_state *state);
 
+int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *district, 
+                                    size_t district_len, long long *district_id);
+
 void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id);
 long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread);
 

src/inc_internal/maat_interval.h

@@ -46,6 +46,9 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name);
 int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, 
                           long long integer, int vtable_id, struct maat_state *state);
 
+int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, const char *district, 
+                                       size_t district_len, long long *district_id);
+                                       
 void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id);
 long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread);
 

src/inc_internal/maat_rule.h

@@ -321,6 +321,7 @@ void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
 void garbage_ip_matcher_free(void *ip_matcher, void *arg);
 void garbage_interval_matcher_free(void *ip_matcher, void *arg);
 void garbage_bool_matcher_free(void *bool_matcher, void *arg);
+void garbage_maat_kv_store_free(void *kv_store, void *arg);
 
 #ifdef __cplusplus
 }

src/inc_internal/maat_table.h

@@ -66,10 +66,6 @@ int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
 size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr);
 int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags);
 
-int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str, 
-                                    size_t district_str_len, long long *district_id);
-long long table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district);
-
 void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
 void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);
 

src/inc_internal/maat_virtual.h

@@ -38,7 +38,6 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 
 void virtual_schema_free(void *virtual_schema);
 
-int virtual_table_get_id(void *virtual_schema);
 int virtual_table_get_physical_table_id(void *virtual_schema);
 
 #ifdef __cplusplus

src/maat_api.c

@@ -1719,8 +1719,8 @@ void maat_state_free(struct maat_state *state)
     free(state);
 }
 
-int maat_state_set_scan_district(struct maat_state *state, const char *district, 
+int maat_state_set_scan_district(struct maat_state *state, int vtable_id, 
-                                 size_t district_len)
+                                 const char *district, size_t district_len)
 {
     if (NULL == state || NULL == district || 0 == district_len) {
         return -1;
@@ -1730,12 +1730,49 @@ int maat_state_set_scan_district(struct maat_state *state, const char *district,
     assert(maat_instance != NULL);
 
     if (NULL == maat_instance->maat_rt) {
+        log_error(maat_instance->logger, MODULE_MAAT_API, 
+                  "maat runtime is NULL, can't set district");
         return -1;
     }
 
-    int ret = table_manager_set_scan_district(maat_instance->tbl_mgr, 
+    enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, vtable_id);
-                                              district, district_len,
+    if (table_type != TABLE_TYPE_FLAG_PLUS && table_type != TABLE_TYPE_EXPR_PLUS &&
-                                              &(state->district_id));
+        table_type != TABLE_TYPE_INTERVAL_PLUS && table_type != TABLE_TYPE_VIRTUAL) {
+        log_error(maat_instance->logger, MODULE_MAAT_API, 
+                 "table_type:%d is invalid, can't set district", table_type);
+        return -1;
+    }
+
+    int physical_table_id = vtable_id;
+    if (table_type == TABLE_TYPE_VIRTUAL) {
+        physical_table_id = vtable_get_physical_table_id(maat_instance->tbl_mgr, vtable_id);
+    }
+
+    int ret = -1;
+    table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
+    void *runtime = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_id);
+    assert(runtime != NULL);
+    
+    switch (table_type) {
+        case TABLE_TYPE_FLAG_PLUS:
+            ret = flag_runtime_set_scan_district((struct flag_runtime *)runtime, district, 
+                                                 district_len, &(state->district_id));
+            break;
+        case TABLE_TYPE_EXPR_PLUS:
+            ret = expr_runtime_set_scan_district((struct expr_runtime *)runtime, district, 
+                                                 district_len, &(state->district_id));
+            break;
+        case TABLE_TYPE_INTERVAL_PLUS:
+            ret = interval_runtime_set_scan_district((struct interval_runtime *)runtime, district, 
+                                                     district_len, &(state->district_id));
+            break;
+        default:
+            log_error(maat_instance->logger, MODULE_MAAT_API, 
+                      "[%s:%d] table_type:%d can't set district", 
+                      __FUNCTION__, __LINE__, table_type);
+            break;
+    }
+
     if (ret < 0) {
         state->district_id = DISTRICT_UNKNOWN;
     }

src/maat_bool_plugin.c

@@ -189,7 +189,7 @@ void *bool_plugin_runtime_new(void *bool_plugin_schema, int max_thread_num,
 
     bool_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, 
                                                      ex_container_free, 
-                                                    logger);
+                                                     logger);
     bool_plugin_rt->ref_garbage_bin = garbage_bin;
     bool_plugin_rt->logger = logger;
 

src/maat_compile.c

@@ -586,7 +586,7 @@ void compile_runtime_free(void *compile_runtime)
     }
     
     if (compile_rt->clause_by_literals_hash != NULL) {
-        maat_clause_hash_free(&compile_rt->clause_by_literals_hash);
+        maat_clause_hash_free(&(compile_rt->clause_by_literals_hash));
         compile_rt->clause_by_literals_hash = NULL;
     }
 

src/maat_ex_data.c

@@ -205,16 +205,24 @@ void ex_container_free(void *schema, void *data)
     }
 
     struct ex_container *ex_container = (struct ex_container *)data;
+    struct ex_container_schema *container_schema = (struct ex_container_schema *)schema;
+
+    /* free ex_container->custom_data */
     if (ex_container->custom_data != NULL) {
-        FREE(ex_container->custom_data);
+        if (container_schema != NULL && container_schema->user_data_free != NULL) {
+            container_schema->user_data_free(ex_container->custom_data);
+        } else {
+             FREE(ex_container->custom_data);
+        }
     }
 
-    struct ex_container_schema *container_schema = (struct ex_container_schema *)schema;
+    /* free ex_container->ex_data */
-    if (container_schema != NULL && container_schema->ex_schema != NULL) {
+    if (ex_container->ex_data != NULL) {
-        long argl = container_schema->ex_schema->argl;
+        if (container_schema != NULL && container_schema->ex_schema != NULL &&
-        void *argp = container_schema->ex_schema->argp;
+            container_schema->ex_schema->free_func != NULL) {
+            long argl = container_schema->ex_schema->argl;
+            void *argp = container_schema->ex_schema->argp;
 
-        if (ex_container->ex_data != NULL && container_schema->ex_schema->free_func != NULL) {
             container_schema->ex_schema->free_func(container_schema->table_id,
                                                    &(ex_container->ex_data), argl, argp);
         }

src/maat_expr.c

@@ -75,6 +75,9 @@ struct expr_runtime {
     int n_worker_thread;
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
+    int district_num;
+    struct maat_kv_store *district_map;
+    struct maat_kv_store *tmp_district_map;
 
     long long *scan_cnt;
     long long *hit_cnt;
@@ -129,8 +132,39 @@ enum hs_match_mode int_to_match_mode(int match_method)
     return mode;
 }
 
+long long expr_runtime_get_district_id(struct expr_runtime *expr_rt, const char *district)
+{
+    long long district_id = DISTRICT_ANY;
+
+	int map_ret = maat_kv_read(expr_rt->district_map, district, &district_id);
+	if (map_ret < 0) {
+        if (NULL == expr_rt->tmp_district_map) {
+            expr_rt->tmp_district_map = maat_kv_store_duplicate(expr_rt->district_map);
+        }
+        
+        map_ret = maat_kv_read(expr_rt->tmp_district_map, district, &district_id);
+        if (map_ret < 0) {
+            district_id = expr_rt->district_num;
+		    maat_kv_register(expr_rt->tmp_district_map, district, district_id);
+		    expr_rt->district_num++;
+        }
+	}
+
+	return district_id;
+}
+
+int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *district, 
+                                    size_t district_len, long long *district_id)
+{
+    if (NULL == expr_rt || NULL == district || 0 == district_len) {
+        return -1;
+    }
+
+    return maat_kv_read_unNull(expr_rt->district_map, district, district_len, district_id);
+}
+
 struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schema,
-                                struct log_handle *logger)
+                                struct expr_runtime *expr_rt)
 {
     size_t column_offset = 0;
     size_t column_len = 0;
@@ -142,7 +176,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
 
     int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_EXPR, 
+        log_error(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table(table_id:%d) line:%s has no item_id",
                   __FUNCTION__, __LINE__, expr_schema->table_id, line);
         goto error;
@@ -151,7 +185,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
 
     ret = get_column_pos(line, expr_schema->group_id_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_EXPR, 
+        log_error(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table(table_id:%d) line:%s has no group_id",
                   __FUNCTION__, __LINE__, expr_schema->table_id, line);
         goto error;
@@ -166,7 +200,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
         }
 
         if (column_len >= MAX_DISTRICT_STR) {
-            log_error(logger, MODULE_EXPR, 
+            log_error(expr_rt->logger, MODULE_EXPR, 
                       "[%s:%d] expr table(table_id:%d) line:%s district length too long",
                       __FUNCTION__, __LINE__, expr_schema->table_id, line);
             goto error;
@@ -176,14 +210,14 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
         memcpy(district, (line + column_offset), column_len);
         assert(strlen(district) > 0);
         str_unescape(district);
-        expr_item->district_id = table_manager_get_district_id(expr_schema->ref_tbl_mgr, district);
+        expr_item->district_id = expr_runtime_get_district_id(expr_rt, district);
     } else {
         expr_item->district_id = DISTRICT_ANY;
     }
 
     ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_EXPR, 
+        log_error(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table(table_id:%d) line:%s has no expr_type",
                   __FUNCTION__, __LINE__, expr_schema->table_id, line);
         goto error;
@@ -192,7 +226,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
     expr_type = atoi(line + column_offset);
     expr_item->expr_type = int_to_expr_type(expr_type);
     if (expr_item->expr_type == EXPR_TYPE_INVALID) {
-        log_error(logger, MODULE_EXPR, 
+        log_error(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table(table_id:%d) line:%s has invalid expr_type",
                  __FUNCTION__, __LINE__, expr_schema->table_id, line);
         goto error;
@@ -200,7 +234,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
 
     ret = get_column_pos(line, expr_schema->match_method_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_EXPR, 
+        log_error(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table(table_id:%d) line:%s has no match_method",
                   __FUNCTION__, __LINE__, expr_schema->table_id, line);
         goto error;
@@ -209,7 +243,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
     match_method_type = atoi(line + column_offset);
     expr_item->match_mode = int_to_match_mode(match_method_type);
     if (expr_item->match_mode == HS_MATCH_MODE_INVALID) {
-        log_error(logger, MODULE_EXPR, 
+        log_error(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table(table_id:%d) line:%s has invalid match_method",
                   __FUNCTION__, __LINE__, expr_schema->table_id, line);
         goto error;
@@ -217,7 +251,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
 
     ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_EXPR, 
+        log_error(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table(table_id:%d) line:%s has no is_hexbin",
                   __FUNCTION__, __LINE__, expr_schema->table_id, line);
         goto error;
@@ -238,7 +272,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
             expr_item->is_case_sensitive = TRUE;
             break;
         default:
-            log_error(logger, MODULE_EXPR, 
+            log_error(expr_rt->logger, MODULE_EXPR, 
                       "[%s:%d] expr table(table_id:%d) line:%s has invalid hexbin value:%d",
                       __FUNCTION__, __LINE__, expr_schema->table_id, line, db_hexbin);
             goto error;
@@ -246,14 +280,14 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
     
     ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_EXPR, 
+        log_error(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table(table_id:%d) line:%s has no keywords",
                   __FUNCTION__, __LINE__, expr_schema->table_id, line);
         goto error;
     }
 
     if (column_len >= MAX_KEYWORDS_STR) {
-        log_error(logger, MODULE_EXPR, 
+        log_error(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table(table_id:%d) line:%s keywords length too long",
                   __FUNCTION__, __LINE__, expr_schema->table_id, line);
         goto error;
@@ -426,6 +460,7 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num,
     expr_rt->n_worker_thread = max_thread_num;
     expr_rt->ref_garbage_bin = garbage_bin;
     expr_rt->logger = logger;
+    expr_rt->district_map = maat_kv_store_new();
 
     expr_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
     expr_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
@@ -445,11 +480,6 @@ void expr_runtime_free(void *expr_runtime)
         expr_rt->hs = NULL;
     }
 
-    // if (expr_rt->hs_stream != NULL) {
-    //     adapter_hs_stream_close(expr_rt->hs_stream);
-    //     expr_rt->hs_stream = NULL;
-    // }
-
     if (expr_rt->htable != NULL) {
         rcu_hash_free(expr_rt->htable);
         expr_rt->htable = NULL;
@@ -460,6 +490,13 @@ void expr_runtime_free(void *expr_runtime)
         expr_rt->item_htable = NULL;
     }
 
+    assert(expr_rt->tmp_district_map == NULL);
+
+    if (expr_rt->district_map != NULL) {
+        maat_kv_store_free(expr_rt->district_map);
+        expr_rt->district_map = NULL;
+    }
+
     if (expr_rt->hit_cnt != NULL) {
         alignment_int64_array_free(expr_rt->hit_cnt);
         expr_rt->hit_cnt = NULL;
@@ -720,7 +757,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
         rcu_hash_del(expr_rt->item_htable, (char *)&item_id, sizeof(item_id));
     } else {
         //add
-        struct expr_item *expr_item = expr_item_new(line, schema, expr_rt->logger);
+        struct expr_item *expr_item = expr_item_new(line, schema, expr_rt);
         if (NULL == expr_item) {
             return -1;
         }
@@ -781,6 +818,14 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
 
     rcu_hash_commit(expr_rt->htable);
 
+    if (expr_rt->tmp_district_map != NULL) {
+        struct maat_kv_store *tmp_map = expr_rt->district_map;
+        expr_rt->district_map = expr_rt->tmp_district_map;
+        expr_rt->tmp_district_map = NULL;
+        maat_garbage_bagging(expr_rt->ref_garbage_bin, tmp_map, NULL, 
+                             garbage_maat_kv_store_free);
+    }
+
     struct hs_expr *rules = NULL;
     void **ex_data_array = NULL;
     size_t rule_cnt = rcu_hash_list(expr_rt->htable, &ex_data_array);
@@ -904,7 +949,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_str
     size_t n_hit_item = 0;
     struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
     memset(hit_results, 0, sizeof(hit_results));
-    
+
     int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
     if (ret < 0) {
         return -1;

src/maat_flag.c

@@ -49,6 +49,9 @@ struct flag_runtime {
     uint32_t rule_num;
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
+    int district_num;
+    struct maat_kv_store *district_map;
+    struct maat_kv_store *tmp_district_map;
 
     long long *scan_cnt;
     long long *hit_cnt;
@@ -174,6 +177,7 @@ void *flag_runtime_new(void *flag_schema, int max_thread_num,
     flag_rt->item_htable = rcu_hash_new(flag_maat_item_free);
     flag_rt->ref_garbage_bin = garbage_bin;
     flag_rt->logger = logger;
+    flag_rt->district_map = maat_kv_store_new();
 
     flag_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
     flag_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
@@ -203,6 +207,13 @@ void flag_runtime_free(void *flag_runtime)
         flag_rt->matcher = NULL;
     }
 
+    assert(flag_rt->tmp_district_map == NULL);
+
+    if (flag_rt->district_map != NULL) {
+        maat_kv_store_free(flag_rt->district_map);
+        flag_rt->district_map = NULL;
+    }
+
     if (flag_rt->hit_cnt != NULL) {
         alignment_int64_array_free(flag_rt->hit_cnt);
         flag_rt->hit_cnt = NULL;
@@ -238,8 +249,39 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_
     return 0;
 }
 
+long long flag_runtime_get_district_id(struct flag_runtime *flag_rt, const char *district)
+{
+    long long district_id = DISTRICT_ANY;
+
+	int map_ret = maat_kv_read(flag_rt->district_map, district, &district_id);
+	if (map_ret < 0) {
+        if (NULL == flag_rt->tmp_district_map) {
+            flag_rt->tmp_district_map = maat_kv_store_duplicate(flag_rt->district_map);
+        }
+        
+        map_ret = maat_kv_read(flag_rt->tmp_district_map, district, &district_id);
+        if (map_ret < 0) {
+            district_id = flag_rt->district_num;
+		    maat_kv_register(flag_rt->tmp_district_map, district, district_id);
+		    flag_rt->district_num++;
+        }
+	}
+
+	return district_id;
+}
+
+int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *district, 
+                                    size_t district_len, long long *district_id)
+{
+    if (NULL == flag_rt || NULL == district || 0 == district_len) {
+        return -1;
+    }
+    
+    return maat_kv_read_unNull(flag_rt->district_map, district, district_len, district_id);
+}
+
 struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
-                                struct log_handle *logger)
+                                struct flag_runtime *flag_rt)
 {
     size_t column_offset = 0;
     size_t column_len = 0;
@@ -248,7 +290,7 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
 
     int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_FLAG, 
+        log_error(flag_rt->logger, MODULE_FLAG, 
                   "[%s:%d] flag table(table_id:%d) line:%s has no item_id",
                   __FUNCTION__, __LINE__, schema->table_id, line);
         goto error;
@@ -257,7 +299,7 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
 
     ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_FLAG, 
+        log_error(flag_rt->logger, MODULE_FLAG, 
                   "[%s:%d] flag table(table_id:%d) line:%s has no group_id",
                   __FUNCTION__, __LINE__, schema->table_id, line);
         goto error;
@@ -272,7 +314,7 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
         }
 
         if (column_len >= MAX_DISTRICT_STR) {
-            log_error(logger, MODULE_FLAG, 
+            log_error(flag_rt->logger, MODULE_FLAG, 
                       "[%s:%d] flag_plus table(table_id:%d) line:%s district length too long",
                       __FUNCTION__, __LINE__, schema->table_id, line);
             goto error;
@@ -282,14 +324,14 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
         memcpy(district, (line + column_offset), column_len);
         assert(strlen(district) > 0);
         str_unescape(district);
-        item->district_id = table_manager_get_district_id(schema->ref_tbl_mgr, district);
+        item->district_id = flag_runtime_get_district_id(flag_rt, district);
     } else {
         item->district_id = DISTRICT_ANY;
     }
 
     ret = get_column_pos(line, schema->flag_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_FLAG, 
+        log_error(flag_rt->logger, MODULE_FLAG, 
                   "[%s:%d] flag table(table_id:%d) line:%s has no flag",
                   __FUNCTION__, __LINE__, schema->table_id, line);
         goto error;
@@ -299,7 +341,7 @@ struct flag_item *flag_item_new(const char *line, struct flag_schema *schema,
 
      ret = get_column_pos(line, schema->flag_mask_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_FLAG, 
+        log_error(flag_rt->logger, MODULE_FLAG, 
                   "[%s:%d] flag table(table_id:%d) line:%s has no flag_mask",
                   __FUNCTION__, __LINE__, schema->table_id, line);
         goto error;
@@ -363,7 +405,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
        rcu_hash_del(flag_rt->item_htable, (char *)&item_id, sizeof(item_id));
     } else {
         //add
-        struct flag_item *flag_item = flag_item_new(line, schema, flag_rt->logger);
+        struct flag_item *flag_item = flag_item_new(line, schema, flag_rt);
         if (NULL == flag_item) {
             return -1;
         }
@@ -424,6 +466,14 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name)
 
     rcu_hash_commit(flag_rt->htable);
 
+    if (flag_rt->tmp_district_map != NULL) {
+        struct maat_kv_store *tmp_map = flag_rt->district_map;
+        flag_rt->district_map = flag_rt->tmp_district_map;
+        flag_rt->tmp_district_map = NULL;
+        maat_garbage_bagging(flag_rt->ref_garbage_bin, tmp_map, NULL, 
+                             garbage_maat_kv_store_free);
+    }
+
     struct flag_rule *rules = NULL;
     void **ex_data_array = NULL;
     size_t rule_cnt = rcu_hash_list(flag_rt->htable, &ex_data_array);

src/maat_fqdn_plugin.c

@@ -161,6 +161,22 @@ int fqdn_plugin_table_set_ex_data_schema(void *fqdn_plugin_schema,
     return 0;
 }
 
+void fqdn_rule_free(struct FQDN_rule *fqdn_rule)
+{
+	assert(fqdn_rule->user_tag==NULL);
+	free(fqdn_rule->FQDN);
+	fqdn_rule->FQDN=NULL;
+	free(fqdn_rule);
+	return;
+}
+
+void fqdn_ex_container_free(void *schema, void *data)
+{
+    struct ex_container_schema *container_schema = (struct ex_container_schema *)schema;
+    container_schema->user_data_free = (void (*)(void *))fqdn_rule_free;
+    ex_container_free(container_schema, data);
+}
+
 void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num,
                               struct maat_garbage_bin *garbage_bin,
                               struct log_handle *logger)
@@ -172,7 +188,7 @@ void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num,
     struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema;
     struct fqdn_plugin_runtime *fqdn_plugin_rt = ALLOC(struct fqdn_plugin_runtime, 1);
 
-    fqdn_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_container_free, 
+    fqdn_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, fqdn_ex_container_free,
                                                      logger);
     fqdn_plugin_rt->ref_garbage_bin = garbage_bin;
     fqdn_plugin_rt->logger = logger;

src/maat_interval.c

@@ -46,6 +46,9 @@ struct interval_runtime {
     uint32_t rule_num;
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
+    int district_num;
+    struct maat_kv_store *district_map;
+    struct maat_kv_store *tmp_district_map;
 
     long long *scan_cnt;
     long long *hit_cnt;
@@ -171,6 +174,7 @@ void *interval_runtime_new(void *interval_schema, int max_thread_num,
     interval_rt->item_htable = rcu_hash_new(interval_maat_item_free);
     interval_rt->ref_garbage_bin = garbage_bin;
     interval_rt->logger = logger;
+    interval_rt->district_map = maat_kv_store_new();
 
     interval_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
     interval_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
@@ -200,6 +204,13 @@ void interval_runtime_free(void *interval_runtime)
         interval_rt->matcher = NULL;
     }
 
+    assert(interval_rt->tmp_district_map == NULL);
+
+    if (interval_rt->district_map != NULL) {
+        maat_kv_store_free(interval_rt->district_map);
+        interval_rt->district_map = NULL;
+    }
+
     if (interval_rt->hit_cnt != NULL) {
         alignment_int64_array_free(interval_rt->hit_cnt);
         interval_rt->hit_cnt = NULL;
@@ -213,8 +224,39 @@ void interval_runtime_free(void *interval_runtime)
     FREE(interval_rt);
 }
 
+long long interval_runtime_get_district_id(struct interval_runtime *interval_rt, const char *district)
+{
+    long long district_id = DISTRICT_ANY;
+
+	int map_ret = maat_kv_read(interval_rt->district_map, district, &district_id);
+	if (map_ret < 0) {
+        if (NULL == interval_rt->tmp_district_map) {
+            interval_rt->tmp_district_map = maat_kv_store_duplicate(interval_rt->district_map);
+        }
+        
+        map_ret = maat_kv_read(interval_rt->tmp_district_map, district, &district_id);
+        if (map_ret < 0) {
+            district_id = interval_rt->district_num;
+		    maat_kv_register(interval_rt->tmp_district_map, district, district_id);
+		    interval_rt->district_num++;
+        }
+	}
+
+	return district_id;
+}
+
+int interval_runtime_set_scan_district(struct interval_runtime *interval_rt, const char *district, 
+                                       size_t district_len, long long *district_id)
+{
+    if (NULL == interval_rt || NULL == district || 0 == district_len) {
+        return -1;
+    }
+
+    return maat_kv_read_unNull(interval_rt->district_map, district, district_len, district_id);
+}
+
 struct interval_item *interval_item_new(const char *line, struct interval_schema *schema,
-                                        struct log_handle *logger)
+                                        struct interval_runtime *interval_rt)
 {
     size_t column_offset = 0;
     size_t column_len = 0;
@@ -223,7 +265,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
 
     int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_INTERVAL, 
+        log_error(interval_rt->logger, MODULE_INTERVAL, 
                   "[%s:%d] interval table(table_id:%d) line:%s has no item_id",
                   __FUNCTION__, __LINE__, schema->table_id, line);
         goto error;
@@ -232,7 +274,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
 
     ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_INTERVAL, 
+        log_error(interval_rt->logger, MODULE_INTERVAL, 
                   "[%s:%d] interval table(table_id:%d) line:%s has no group_id",
                   __FUNCTION__, __LINE__, schema->table_id, line);
         goto error;
@@ -247,7 +289,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
         }
 
         if (column_len >= MAX_DISTRICT_STR) {
-            log_error(logger, MODULE_INTERVAL, 
+            log_error(interval_rt->logger, MODULE_INTERVAL, 
                       "[%s:%d] interval_plus table(table_id:%d) line:%s district length too long",
                       __FUNCTION__, __LINE__, schema->table_id, line);
             goto error;
@@ -257,14 +299,14 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
         memcpy(district, (line + column_offset), column_len);
         assert(strlen(district) > 0);
         str_unescape(district);
-        item->district_id = table_manager_get_district_id(schema->ref_tbl_mgr, district);
+        item->district_id = interval_runtime_get_district_id(interval_rt, district);
     } else {
         item->district_id = DISTRICT_ANY;
     }
 
     ret = get_column_pos(line, schema->low_bound_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_INTERVAL, 
+        log_error(interval_rt->logger, MODULE_INTERVAL, 
                   "[%s:%d] interval table(table_id:%d) line:%s has no low_bound",
                   __FUNCTION__, __LINE__, schema->table_id, line);
         goto error;
@@ -273,7 +315,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema
 
     ret = get_column_pos(line, schema->up_bound_column, &column_offset, &column_len);
     if (ret < 0) {
-        log_error(logger, MODULE_INTERVAL, 
+        log_error(interval_rt->logger, MODULE_INTERVAL, 
                   "[%s:%d] interval table(table_id:%d) line:%s has no up_bound",
                   __FUNCTION__, __LINE__, schema->table_id, line);
         goto error;
@@ -359,7 +401,7 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
         rcu_hash_del(interval_rt->item_htable, (char *)&item_id, sizeof(item_id));
     } else {
         //add
-        struct interval_item *interval_item = interval_item_new(line, schema, interval_rt->logger);
+        struct interval_item *interval_item = interval_item_new(line, schema, interval_rt);
         if (NULL == interval_item) {
             return -1;
         }
@@ -420,6 +462,14 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name)
 
     rcu_hash_commit(interval_rt->htable);
 
+    if (interval_rt->tmp_district_map != NULL) {
+        struct maat_kv_store *tmp_map = interval_rt->district_map;
+        interval_rt->district_map = interval_rt->tmp_district_map;
+        interval_rt->tmp_district_map = NULL;
+        maat_garbage_bagging(interval_rt->ref_garbage_bin, tmp_map, NULL, 
+                             garbage_maat_kv_store_free);
+    }
+
     void **ex_data_array = NULL;
     struct interval_rule *rules = NULL;
     size_t rule_cnt = rcu_hash_list(interval_rt->htable, &ex_data_array);

src/maat_plugin.c

@@ -265,9 +265,7 @@ void *plugin_runtime_new(void *plugin_schema, int max_thread_num,
 
     struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
     struct plugin_runtime *plugin_rt = ALLOC(struct plugin_runtime, 1);
-    plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, 
+    plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_container_free, logger);
-                                                ex_container_free, 
-                                                logger);
     plugin_rt->ref_garbage_bin = garbage_bin;
     plugin_rt->logger = logger;
 

src/maat_redis_monitor.c

@@ -1378,6 +1378,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
 		mr_ctx->last_reconnect_time = time(NULL);
 		if (mr_ctx->read_ctx != NULL) {
 			redisFree(mr_ctx->read_ctx);
+            mr_ctx->read_ctx = NULL;
 		}
 		log_info(maat_instance->logger, MODULE_REDIS_MONITOR, "Reconnecting...");
 

src/maat_rule.c

@@ -310,6 +310,12 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt)
     FREE(maat_rt);
 }
 
+void garbage_maat_kv_store_free(void *kv_store, void *arg)
+{
+    struct maat_kv_store *store = (struct maat_kv_store *)kv_store;
+    maat_kv_store_free(store);
+}
+
 void garbage_maat_runtime_destroy(void *maat_runtime, void *arg)
 {
     struct maat_runtime *maat_rt = (struct maat_runtime *)maat_runtime;
@@ -562,8 +568,8 @@ void *rule_monitor_loop(void *arg)
     }
 
     maat_runtime_destroy(maat_instance->maat_rt);
-	table_manager_destroy(maat_instance->tbl_mgr);
     maat_garbage_bin_free(maat_instance->garbage_bin);
+    table_manager_destroy(maat_instance->tbl_mgr); //table manager MUST be freed at last.
 
     alignment_int64_array_free(maat_instance->thread_call_cnt);
     alignment_int64_array_free(maat_instance->hit_cnt);

src/maat_table.c

@@ -54,9 +54,6 @@ struct table_manager {
     int default_compile_table_id;
     int g2g_table_id;
     struct maat_kv_store *tablename2id_map;
-    struct maat_kv_store *district_map;
-    struct maat_kv_store *tmp_district_map;
-    uint32_t district_num;
 
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
@@ -620,9 +617,6 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
         group2compile_runtime_init(runtime, compile_rt, g2g_rt);
     }
 
-    /* new district map */
-    tbl_mgr->district_map = maat_kv_store_new();
-
     return 0;
 }
 
@@ -641,17 +635,6 @@ void table_manager_runtime_destroy(struct table_manager *tbl_mgr)
             tbl_mgr->tbl[i]->runtime = NULL;
         }
 	}
-
-    /* free district map */
-    if (tbl_mgr->district_map != NULL) {
-        maat_kv_store_free(tbl_mgr->district_map);
-        tbl_mgr->district_map = NULL;
-    }
-    
-    if (tbl_mgr->tmp_district_map != NULL) {
-        maat_kv_store_free(tbl_mgr->tmp_district_map);
-        tbl_mgr->tmp_district_map = NULL;
-    }
 }
 
 void table_manager_destroy(struct table_manager *tbl_mgr)
@@ -680,8 +663,11 @@ void table_manager_destroy(struct table_manager *tbl_mgr)
         FREE(tbl_mgr->accept_tags);
     }
 
-    maat_kv_store_free(tbl_mgr->tablename2id_map);
+    if (tbl_mgr->tablename2id_map != NULL) {
-    tbl_mgr->tablename2id_map = NULL;
+        maat_kv_store_free(tbl_mgr->tablename2id_map);
+        tbl_mgr->tablename2id_map = NULL;
+    }
+
 	FREE(tbl_mgr);
 }
 
@@ -702,7 +688,7 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
         log_error(tbl_mgr->logger, MODULE_TABLE, 
                   "[%s:%d] table:%s is not registered", 
                   __FUNCTION__, __LINE__, name);
-		return -1;		
+		return -1;
 	}
 
     return (int)table_id;
@@ -773,33 +759,6 @@ int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *t
     return compare_accept_tag(tags, tbl_mgr->accept_tags, tbl_mgr->n_accept_tag);
 }
 
-int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district, 
-                                    size_t district_len, long long *district_id)
-{
-    return maat_kv_read_unNull(tbl_mgr->district_map, district, district_len, district_id);
-}
-
-long long table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district)
-{
-    long long district_id = DISTRICT_ANY;
-
-	int map_ret = maat_kv_read(tbl_mgr->district_map, district, &district_id);
-	if (map_ret < 0) {
-		if (NULL == tbl_mgr->tmp_district_map) {
-			tbl_mgr->tmp_district_map = maat_kv_store_duplicate(tbl_mgr->district_map);
-		}
-
-		map_ret = maat_kv_read(tbl_mgr->tmp_district_map, district, &district_id);
-		if (map_ret < 0) {
-			district_id = tbl_mgr->district_num;
-			maat_kv_register(tbl_mgr->tmp_district_map, district, district_id);
-			tbl_mgr->district_num++;
-		}
-	}
-
-	return district_id;
-}
-
 void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
 {
     if (NULL == tbl_mgr || (table_id < 0) || (table_id >= MAX_TABLE_NUM)) {
@@ -813,12 +772,6 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id)
 	return tbl_mgr->tbl[table_id]->runtime;
 }
 
-void garbage_maat_kv_store_free(void *maat_kv_store, void *arg)
-{
-    struct maat_kv_store *kv_store = (struct maat_kv_store *)maat_kv_store;
-    maat_kv_store_free(kv_store);
-}
-
 int table_manager_update_runtime(struct table_manager *tbl_mgr, const char *table_name,
                                  int table_id, const char *line)
 {
@@ -834,40 +787,43 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, const char *tabl
 
     int valid_column = table_manager_get_valid_column(tbl_mgr, table_id);
     if (valid_column < 0) {
+        log_error(tbl_mgr->logger, MODULE_TABLE, 
+                  "[%s:%d] table:%s has no valid column, can't update runtime",
+                  __FUNCTION__, __LINE__, table_name);
         return -1;
     }
 
     enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
     if (table_type == TABLE_TYPE_INVALID) {
+        log_error(tbl_mgr->logger, MODULE_TABLE, 
+                  "[%s:%d] table:%s table_type is invalid, can't update runtime",
+                  __FUNCTION__, __LINE__, table_name);
         return -1;
     }
 
     if (NULL == table_ops[table_type].update_runtime) {
+        log_error(tbl_mgr->logger, MODULE_TABLE, 
+                  "[%s:%d] table:%s has no update_runtime function, can't update runtime",
+                  __FUNCTION__, __LINE__, table_name);
         return -1;
     }
     
-    int ret = table_ops[table_type].update_runtime(runtime, schema, table_name, 
+    return table_ops[table_type].update_runtime(runtime, schema, table_name, 
-                                                   line, valid_column);
+                                                line, valid_column);
-
-    if (tbl_mgr->tmp_district_map != NULL) {
-        struct maat_kv_store *tmp_map = tbl_mgr->district_map;
-        tbl_mgr->district_map = tbl_mgr->tmp_district_map;
-        tbl_mgr->tmp_district_map = NULL;
-        maat_garbage_bagging(tbl_mgr->ref_garbage_bin, tmp_map, NULL, garbage_maat_kv_store_free);
-    }
-
-    return ret;
 }
 
 void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
 {
-    enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
+    void *runtime = table_manager_get_runtime(tbl_mgr, table_id);
-    if (table_type == TABLE_TYPE_INVALID) {
+    if (NULL == runtime) {
         return;
     }
 
-    void *runtime = table_manager_get_runtime(tbl_mgr, table_id);
+    enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
-    if (NULL == runtime) {
+    if (table_type == TABLE_TYPE_INVALID) {
+        log_error(tbl_mgr->logger, MODULE_TABLE, 
+                  "[%s:%d] table(table_id:%d) table_type is invalid, can't commit runtime",
+                  __FUNCTION__, __LINE__, table_id);
         return;
     }
 

src/maat_virtual.c

@@ -61,16 +61,6 @@ void virtual_schema_free(void *virtual_schema)
     FREE(virtual_schema);
 }
 
-int virtual_table_get_id(void *virtual_schema)
-{
-    if (NULL == virtual_schema) {
-        return -1;
-    }
-
-    struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
-    return schema->table_id;
-}
-
 int virtual_table_get_physical_table_id(void *virtual_schema)
 {
     if (NULL == virtual_schema) {

test/maat_framework_gtest.cpp

@@ -289,7 +289,7 @@ int ip_table_set_line(struct maat *maat_instance, const char *table_name, enum m
 
     return maat_cmd_set_line(maat_instance, &line_rule);
 }
-
+#if 1
 class MaatFlagScan : public testing::Test
 {
 protected:
@@ -511,7 +511,7 @@ TEST_F(MaatFlagScan, FlagPlus) {
                              ARRAY_SIZE, &n_hit_result, state);
     EXPECT_EQ(ret, MAAT_SCAN_ERR);
  
-    ret = maat_state_set_scan_district(state, region_name, strlen(region_name));
+    ret = maat_state_set_scan_district(state, flag_table_id, region_name, strlen(region_name));
     ASSERT_EQ(ret, 0);
 
     ret = maat_scan_flag(maat_instance, flag_table_id, scan_data1, results,
@@ -675,7 +675,7 @@ TEST_F(MaatStringScan, ExprPlus) {
 						        results, ARRAY_SIZE, &n_hit_result, state);
 	EXPECT_EQ(ret, MAAT_SCAN_ERR);//Should return error for district not setting.
 
-	ret = maat_state_set_scan_district(state, region_name1, strlen(region_name1));
+	ret = maat_state_set_scan_district(state, table_id, region_name1, strlen(region_name1));
 	ASSERT_EQ(ret, 0);
 	ret = maat_scan_string(maat_instance, table_id, scan_data1, strlen(scan_data1), 
 						   results, ARRAY_SIZE, &n_hit_result, state);
@@ -683,7 +683,7 @@ TEST_F(MaatStringScan, ExprPlus) {
 	EXPECT_EQ(results[0], 128);
 	maat_state_reset(state);
 
-	ret = maat_state_set_scan_district(state, region_name2, strlen(region_name2));
+	ret = maat_state_set_scan_district(state, table_id, region_name2, strlen(region_name2));
 	ASSERT_EQ(ret, 0);
 	ret = maat_scan_string(maat_instance, table_id, scan_data2, strlen(scan_data2), 
 						   results, ARRAY_SIZE, &n_hit_result, state);
@@ -731,7 +731,7 @@ TEST_F(MaatStringScan, ExprPlusWithOffset)
 	int table_id = maat_get_table_id(maat_instance, "APP_PAYLOAD");
 	ASSERT_GT(table_id, 0);
 
-	int ret = maat_state_set_scan_district(state, region_name, strlen(region_name));
+	int ret = maat_state_set_scan_district(state, table_id, region_name, strlen(region_name));
 	EXPECT_EQ(ret, 0);
 
 	ret = maat_scan_string(maat_instance, table_id, (char*)udp_payload_not_hit, sizeof(udp_payload_not_hit), 
@@ -761,14 +761,14 @@ TEST_F(MaatStringScan, ExprPlusWithHex) {
 	int table_id = maat_get_table_id(maat_instance, "HTTP_SIGNATURE");
 	ASSERT_GT(table_id, 0);
 
-	int ret = maat_state_set_scan_district(state, region_name1, strlen(region_name1));
+	int ret = maat_state_set_scan_district(state, table_id, region_name1, strlen(region_name1));
 	ASSERT_EQ(ret, 0);
 	ret = maat_scan_string(maat_instance, table_id, scan_data1, strlen(scan_data1), 
 						   results, ARRAY_SIZE, &n_hit_result, state);
 	EXPECT_EQ(ret, MAAT_SCAN_HIT);
     EXPECT_EQ(results[0], 156);
 
-    ret = maat_state_set_scan_district(state, region_name2, strlen(region_name2));
+    ret = maat_state_set_scan_district(state, table_id, region_name2, strlen(region_name2));
 	ASSERT_EQ(ret, 0);
 	ret = maat_scan_string(maat_instance, table_id, scan_data1, strlen(scan_data1), 
 						   results, ARRAY_SIZE, &n_hit_result, state);
@@ -801,7 +801,7 @@ TEST_F(MaatStringScan, ExprAndExprPlus) {
                                 results, ARRAY_SIZE, &n_hit_result, state);
     EXPECT_EQ(ret, MAAT_SCAN_ERR);
 
-    ret = maat_state_set_scan_district(state, region_name, strlen(region_name));
+    ret = maat_state_set_scan_district(state, expr_plus_table_id, region_name, strlen(region_name));
     ASSERT_EQ(ret, 0);
     ret = maat_scan_string(maat_instance, expr_plus_table_id, scan_data, strlen(scan_data),
                             results, ARRAY_SIZE, &n_hit_result, state);
@@ -839,7 +839,7 @@ TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
 	int table_id = maat_get_table_id(maat_instance, "APP_PAYLOAD");
 	ASSERT_GT(table_id, 0);
 
-	int ret = maat_state_set_scan_district(state, region_name, strlen(region_name));
+	int ret = maat_state_set_scan_district(state, table_id, region_name, strlen(region_name));
 	ASSERT_EQ(ret, 0);
     
 	ret = maat_scan_string(maat_instance, table_id, (char *)udp_payload_not_hit, sizeof(udp_payload_not_hit), 
@@ -1518,7 +1518,7 @@ TEST_F(MaatIntervalScan, IntervalPlus) {
     int table_id = maat_get_table_id(maat_instance, table_name);
     const char *district_str = "interval.plus";
 
-    int ret = maat_state_set_scan_district(state, district_str, strlen(district_str));
+    int ret = maat_state_set_scan_district(state, table_id, district_str, strlen(district_str));
     EXPECT_EQ(ret, 0);
 
     unsigned int scan_data1 = 2020;
@@ -1841,7 +1841,6 @@ protected:
 
         struct maat_options *opts = maat_options_new();
         maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
-        maat_options_set_deferred_load_on(opts);
         maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
         maat_options_set_accept_tags(opts, accept_tags);
 
@@ -2186,7 +2185,7 @@ void fqdn_plugin_ex_new_cb(const char *table_name, int table_id, const char *key
 	EXPECT_EQ(ret, 0);
 
 	sscanf(table_line + column_offset, "catid=%d", &ud->catid);
-	ud->ref_cnt = 0;
+	ud->ref_cnt = 1;
 	*ad = ud;
 	(*counter)++;
 }
@@ -2268,7 +2267,7 @@ void bool_plugin_ex_new_cb(const char *table_name, int table_id, const char *key
 
 	ud->name = (char *)malloc(column_len+1);
 	memcpy(ud->name, table_line+column_offset, column_len);
-	ud->ref_cnt = 0;
+	ud->ref_cnt = 1;
 	*ad = ud;
 	(*counter)++;
 }
@@ -2949,6 +2948,7 @@ protected:
         maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
 
         _shared_maat_instance = maat_new(opts, table_info);
+        maat_options_free(opts);
         EXPECT_TRUE(_shared_maat_instance != NULL);
     }
 
@@ -3025,7 +3025,7 @@ TEST_F(MaatFileTest, StreamFiles) {
 
     free(name_list);
 }
-
+#endif
 class MaatCmdTest : public testing::Test
 {
 protected:
@@ -3638,7 +3638,7 @@ TEST_F(MaatCmdTest, VirtualTable) {
     int table_id = maat_get_table_id(maat_instance, "HTTP_REQUEST_HEADER");
     ASSERT_GT(table_id, 0);
 
-    ret = maat_state_set_scan_district(state, "User-Agent", strlen("User-Agent"));
+    ret = maat_state_set_scan_district(state, table_id, "User-Agent", strlen("User-Agent"));
     EXPECT_EQ(ret, 0);
 
     ret = maat_scan_string(maat_instance, table_id, http_req_hdr_ua, strlen(http_req_hdr_ua),
@@ -3648,7 +3648,7 @@ TEST_F(MaatCmdTest, VirtualTable) {
     table_id = maat_get_table_id(maat_instance, "HTTP_RESPONSE_HEADER");
     ASSERT_GT(table_id, 0);
 
-    ret = maat_state_set_scan_district(state, "Cookie", strlen("Cookie"));
+    ret = maat_state_set_scan_district(state, table_id, "Cookie", strlen("Cookie"));
     EXPECT_EQ(ret, 0);
 
     ret = maat_scan_string(maat_instance, table_id, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie),
@@ -3673,7 +3673,7 @@ TEST_F(MaatCmdTest, VirtualTable) {
     table_id = maat_get_table_id(maat_instance, "HTTP_RESPONSE_HEADER");
     ASSERT_GT(table_id, 0);
 
-    ret = maat_state_set_scan_district(state, "Cookie", strlen("Cookie"));
+    ret = maat_state_set_scan_district(state, table_id, "Cookie", strlen("Cookie"));
     EXPECT_EQ(ret, 0);
 
     ret = maat_scan_string(maat_instance, table_id, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie),
@@ -3946,7 +3946,7 @@ void plugin_ex_new_cb(const char *table_name, int table_id, const char *key,
 	int valid = 0, tag = 0;
 	int ret = sscanf(table_line, "%d\t%s\t%s%d\t%d", &(u->id), u->ip_addr, u->name, &valid, &tag);
 	EXPECT_EQ(ret, 5);
-	u->ref_cnt = 0;
+	u->ref_cnt = 1;
 	*ad = u;
 	(*counter)++;
 }
@@ -4466,7 +4466,7 @@ TEST_F(MaatCmdTest, HitPath) {
     int table_id = maat_get_table_id(maat_instance, "HTTP_REQUEST_HEADER");
     ASSERT_GT(table_id, 0);
 
-    ret = maat_state_set_scan_district(state, "URL", strlen("URL"));
+    ret = maat_state_set_scan_district(state, table_id, "URL", strlen("URL"));
     EXPECT_EQ(ret, 0);
 
     int Nth_scan = 0;
@@ -4494,7 +4494,7 @@ TEST_F(MaatCmdTest, HitPath) {
     table_id = maat_get_table_id(maat_instance, "HTTP_RESPONSE_HEADER");
 	ASSERT_GT(table_id, 0);
 
-    ret = maat_state_set_scan_district(state, "Cookie", strlen("Cookie"));
+    ret = maat_state_set_scan_district(state, table_id, "Cookie", strlen("Cookie"));
     EXPECT_EQ(ret, 0);
 
     Nth_scan++;

test/table_info.conf

@@ -127,8 +127,7 @@
         "valid_column":4,
          "custom": {
             "key_type":"integer",
-            "key":1,
+            "key":1
-            "tag":5
         }
     },
     {
@@ -288,23 +287,12 @@
     {
         "table_id":23,
         "table_name":"VIRTUAL_IP_PLUS_TABLE",
+        "db_tables":["VIRTUAL_IP_PLUS_SOURCE", "VIRTUAL_IP_PLUS_DESTINATION"],
         "table_type":"virtual",
         "physical_table": "IP_PLUS_CONFIG"
     },
     {
         "table_id":24,
-        "table_name":"VIRTUAL_IP_PLUS_SOURCE",
-        "table_type":"virtual",
-        "physical_table": "IP_PLUS_CONFIG"
-    },
-    {
-        "table_id":25,
-        "table_name":"VIRTUAL_IP_PLUS_DESTINATION",
-        "table_type":"virtual",
-        "physical_table": "IP_PLUS_CONFIG"
-    },
-    {
-        "table_id":26,
         "table_name":"TEST_IP_PLUGIN_WITH_EXDATA",
         "table_type":"ip_plugin",
         "valid_column":6,
@@ -317,7 +305,7 @@
         }
     },
     {
-        "table_id":27,
+        "table_id":25,
         "table_name":"AS_NUMBER",
         "table_type":"expr",
         "valid_column":7,
@@ -331,19 +319,19 @@
         }
     },
     {
-        "table_id":28,
+        "table_id":26,
         "table_name":"SOURCE_IP_ASN",
         "table_type":"virtual",
         "physical_table":"AS_NUMBER"
     },
     {
-        "table_id":29,
+        "table_id":27,
         "table_name":"DESTINATION_IP_ASN",
         "table_type":"virtual",
         "physical_table":"AS_NUMBER"
     },
     {
-        "table_id":30,
+        "table_id":28,
         "table_name":"GeoLocation",
         "table_type":"expr",
         "valid_column":7,
@@ -357,13 +345,13 @@
         }
     },
     {
-        "table_id":31,
+        "table_id":29,
         "table_name":"SOURCE_IP_GEO",
         "table_type":"virtual",
         "physical_table":"GeoLocation"
     },
     {
-        "table_id":32,
+        "table_id":30,
         "table_name":"INTERGER_PLUS",
         "table_type":"intval_plus",
         "valid_column":6,
@@ -376,7 +364,7 @@
         }
     },
     {
-        "table_id":33,
+        "table_id":31,
         "table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA",
         "table_type":"fqdn_plugin",
         "valid_column":5,
@@ -387,7 +375,7 @@
         }
     },
     {
-        "table_id":34,
+        "table_id":32,
         "table_name":"APP_ID",
         "table_type":"intval",
         "valid_column":5,
@@ -399,7 +387,7 @@
         }
     },
     {
-        "table_id":35,
+        "table_id":33,
         "table_name":"EMPTY_KEYWORD",
         "table_type":"expr",
         "valid_column":7,
@@ -413,7 +401,7 @@
         }
     },
     {
-        "table_id":36,
+        "table_id":34,
         "table_name":"EMPTY_INTERGER",
         "table_type":"intval",
         "valid_column":5,
@@ -425,7 +413,7 @@
         }
     },
     {
-        "table_id":37,
+        "table_id":35,
         "table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA",
         "table_type":"bool_plugin",
         "valid_column":4,
@@ -435,7 +423,7 @@
         }
     },
     {
-        "table_id":38,
+        "table_id":36,
         "table_name":"FLAG_CONFIG",
         "table_type":"flag",
         "valid_column":5,
@@ -447,7 +435,7 @@
         }
     },
     {
-        "table_id":39,
+        "table_id":37,
         "table_name":"FLAG_PLUS_CONFIG",
         "table_type":"flag_plus",
         "valid_column":6,
@@ -460,7 +448,7 @@
         }
     },
     {
-        "table_id":40,
+        "table_id":38,
         "table_name":"TEST_PLUGIN_KEY_TYPE_TABLE",
         "table_type":"plugin",
         "valid_column":4,