compile table support conjunction, ip_plugin support cidr

include/maat.h

@@ -51,15 +51,19 @@ enum maat_scan_status {
 #define	MAX_SERVICE_DEFINE_LEN 128
 struct maat_rule {
 	int	config_id;
-	int	service_id;
-	uint8_t	do_log;
-	uint8_t do_blacklist;
-	uint8_t	action;
 	uint8_t	reserved;
 	int	serv_def_len;
 	char service_defined[MAX_SERVICE_DEFINE_LEN];
 };
 
+struct ip_addr {
+    int ip_type;             //4: IPv4, 6: IPv6
+    union {
+        unsigned int ipv4;   //network order
+        unsigned int ipv6[4];
+    };
+};
+
 typedef void maat_start_callback_t(int update_type, void *u_param);
 typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para);
 typedef void maat_finish_callback_t(void *u_para);
@@ -69,10 +73,10 @@ typedef void maat_plugin_ex_free_func_t(int table_id, void **ad, long argl, void
 typedef void maat_plugin_ex_dup_func_t(int table_id, void **to, void **from, long argl, void *argp);
 
 typedef void maat_rule_ex_new_func_t(int idx, const struct maat_rule *rule, const char *srv_def_large,
-									 void *ex_data, long argl, void *argp);
+									 void **ex_data, long argl, void *argp);
 typedef void maat_rule_ex_free_func_t(int idx, const struct maat_rule *rule, const char *srv_def_large,
-									  void *ex_data, long argl, void *argp);
+									  void **ex_data, long argl, void *argp);
-typedef void maat_rule_ex_dup_func_t(int idx, void *to, void *from, long argl, void *argp);
+typedef void maat_rule_ex_dup_func_t(int idx, void **to, void **from, long argl, void *argp);
 
 /* maat_instance options API */
 struct maat_options;
@@ -106,7 +110,7 @@ int maat_table_callback_register(struct maat *instance, int table_id,
 								 void *u_para);
 
 /* maat compile table API */
-int maat_compile_table_ex_schema_register(struct maat *instance, int table_id,
+int maat_compile_table_ex_schema_register(struct maat *instance, const char *table_name,
                                           maat_rule_ex_new_func_t *new_func,
                                           maat_rule_ex_free_func_t *free_func,
                                           maat_rule_ex_dup_func_t *dup_func,
@@ -114,15 +118,15 @@ int maat_compile_table_ex_schema_register(struct maat *instance, int table_id,
 void *maat_compile_table_get_ex_data(struct maat *instance, int table_id, int compile_id, size_t idx);
 
 /* maat plugin table API */
-int maat_plugin_table_ex_schema_register(struct maat *instance, int table_id,
+int maat_plugin_table_ex_schema_register(struct maat *instance, const char *table_name,
                                          maat_plugin_ex_new_func_t *new_func,
                                          maat_plugin_ex_free_func_t *free_func,
                                          maat_plugin_ex_dup_func_t *dup_func,
                                          long argl, void *argp);
 /* returned data is duplicated by dup_func of maat_plugin_table_ex_schema_register, 
    caller is responsible to free the data. */                                     
-int maat_plugin_table_get_ex_data(struct maat *instance, int table_id,
+int maat_ip_plugin_table_get_ex_data(struct maat *instance, const char *table_name,
-                                    const char *key, size_t key_len,
+                                     const struct ip_addr *ip,
                                      void **ex_data_array, size_t n_ex_data);
 /* maat scan API */
 struct maat_state;
@@ -174,8 +178,8 @@ int maat_state_set_scan_district(struct maat *instance, struct maat_state **stat
 
 int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **state);
 
-int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state, 
+int maat_state_set_scan_compile_tables(struct maat *maat_instance, struct maat_state **state, 
-                                      int compile_table_id);
+                                       const char *compile_table[], size_t n_table);
 
 int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state, 
                              struct maat_hit_path *paths, size_t n_path);

src/inc_internal/maat_compile.h

@@ -74,7 +74,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
                                      struct group2group_runtime *g2g_rt, 
                                      struct maat_compile_state *compile_state,
                                      struct maat_hit_path *hit_paths,
-                                     size_t n_hit_path);
+                                     size_t hit_path_index, size_t n_hit_path);
 /* group2compile runtime API */
 void *group2compile_runtime_new(void *g2c_schema, int max_thread_num,
                                 struct maat_garbage_bin *garbage_bin,
@@ -92,7 +92,7 @@ void maat_compile_state_free(struct maat_compile_state *compile_state);
 
 int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
                               int *hit_item_ids, size_t hit_item_cnt,
-                              int *group_ids, size_t group_ids_size,
+                              int *group_ids, int hit_group_index, size_t group_ids_size,
                               size_t *n_hit_group_id, struct maat_state *state);
                               
 int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);

src/inc_internal/maat_config_monitor.h

@@ -17,6 +17,7 @@ extern "C"
 #endif
 
 #include <stdint.h>
+#include "log/log.h"
 
 void config_monitor_traverse(long long version, const char *idx_dir,
 							void (*start_fn)(long long, int, void *),

src/inc_internal/maat_ex_data.h

@@ -82,9 +82,12 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
 size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
                                                   struct ex_data_container ***ex_container);
 
-void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, 
+void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt, 
                                          const char *key, size_t key_len);
 
+void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt,
+                                               struct ex_data_container *ex_container);
+                                               
 void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, 
                                       const char *key, size_t key_len);
 

src/inc_internal/maat_expr.h

@@ -44,14 +44,13 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name);
  * 
  * @retval the num of hit group_id
 */
-int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, 
+int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, 
-                      const char *data, size_t data_len,
+                      size_t data_len, int *group_ids, int hit_group_index,
-                      int *group_ids, size_t group_ids_size, 
+                      size_t group_ids_size, int vtable_ids, struct maat_state *state);
-                      int vtable_ids, struct maat_state *state);
 
 void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
-int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data,
+int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, 
-                             size_t data_len, int *group_ids, size_t group_ids_size, 
+                             int *group_ids, int hit_group_index, size_t group_ids_size, 
                              int vtable_id, struct maat_state *state);
 void expr_runtime_stream_close(struct expr_runtime *expr_rt);
 

src/inc_internal/maat_flag.h

@@ -46,8 +46,8 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name);
  * 
  * @retval the num of hit group_id
 */
-int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, 
+int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, uint64_t flag, 
-                      uint64_t flag, int *group_ids, size_t group_ids_size, 
+                      int *group_ids, int hit_group_index,size_t group_ids_size, 
                       int vtable_id, struct maat_state *state);
 
 void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id);

src/inc_internal/maat_interval.h

@@ -47,8 +47,8 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name);
  * @retval the num of hit group_id
 */
 int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, 
-                          uint64_t integer, int *group_ids, size_t group_ids_size, 
+                          uint64_t integer, int *group_ids, int hit_group_index, 
-                          int vtable_id, struct maat_state *state);
+                          size_t group_ids_size, int vtable_id, struct maat_state *state);
 
 void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id);
 long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread);

src/inc_internal/maat_ip.h

@@ -41,8 +41,8 @@ struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt);
 
 /* ip runtime scan API */
 int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, 
-                    uint8_t *ip_addr, int *group_ids, size_t group_id_size, 
+                    uint8_t *ip_addr, int *group_ids, int hit_group_index,
-                    int vtable_id, struct maat_state *state);
+                    size_t group_id_size, int vtable_id, struct maat_state *state);
 
 void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id);
 long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread);

src/inc_internal/maat_ip_plugin.h

@@ -32,7 +32,7 @@ void ip_plugin_schema_free(void *ip_plugin_schema);
 /* ip plugin table ex data API */
 struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema);
 
-int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, 
+void ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, 
                                        maat_plugin_ex_new_func_t *new_func,
 							           maat_plugin_ex_free_func_t *free_func,
 							           maat_plugin_ex_dup_func_t *dup_func,
@@ -51,6 +51,9 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name);
 
 struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime);
 
+int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr *ip_addr, 
+                                  void **ex_data_array, size_t n_ex_data_array);
+
 #ifdef __cplusplus
 }
 #endif

src/inc_internal/maat_plugin.h

@@ -40,7 +40,7 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema);
 int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns);
 
 /* plugin table ex data API */
-int plugin_table_set_ex_data_schema(void *plugin_schema, 
+void plugin_table_set_ex_data_schema(void *plugin_schema, 
                                     maat_plugin_ex_new_func_t *new_func,
 							        maat_plugin_ex_free_func_t *free_func,
 							        maat_plugin_ex_dup_func_t *dup_func,

src/inc_internal/maat_rule.h

@@ -36,6 +36,8 @@ extern "C"
 #include "maat_virtual.h"
 
 #define MAX_TABLE_NUM	256
+#define MAX_COMPILE_TABLE_NUM 16
+#define MAX_PHYSICAL_TABLE_NUM  16
 
 #define MAAT_UPDATE_TYPE_NONE 0
 #define	MAAT_UPDATE_TYPE_FULL 1
@@ -60,10 +62,6 @@ enum tag_match {
 
 struct maat_rule_head {
 	int config_id;
-	int	service_id;
-	char do_log;
-	char do_blacklist;
-	char action;
 	char resevered;
 	int	serv_def_len;
 };
@@ -241,7 +239,8 @@ struct maat {
 struct maat_state {
 	struct maat *maat_instance;
 	int16_t thread_id;
-    int compile_table_id; //caller can select compile table to scan
+    size_t n_compile_table;
+    char compile_tables[MAX_COMPILE_TABLE_NUM][NAME_MAX]; //caller can select compile table to scan
 	unsigned char is_set_district;
 	unsigned char is_last_scan;
 	int district_id; //-1: Any District; -2: Unkonwn District;

src/inc_internal/maat_virtual.h

@@ -40,7 +40,8 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 
 void virtual_schema_free(void *virtual_schema);
 
-int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type);
+int virtual_table_get_id(void *virtual_schema);
+size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]);
 
 #ifdef __cplusplus
 }

src/json2iris.c

@@ -1024,11 +1024,12 @@ void write_table_idx(struct iris_description *p_iris, struct iris_table *table)
 	memcpy(buff, line_cnt_str, strlen(line_cnt_str));
 	memcpy(buff + strlen(line_cnt_str), table->buff, table->write_pos);
 
+    UNUSED int ret = 0;
 	FILE *table_fp = fopen(table->table_path, "w");
 	if (p_iris->encrypt_key) {
         unsigned char *encrypt_buff = NULL;
         size_t encrypt_buff_sz = 0;
-		int ret = crypt_memory(buff, table_file_sz, &encrypt_buff, 
+		ret = crypt_memory(buff, table_file_sz, &encrypt_buff, 
                                 &encrypt_buff_sz, p_iris->encrypt_key, 
                                 p_iris->encrypt_algo, 1, err_str, sizeof(err_str));
 		assert(ret == 0);

src/maat_api.c

@@ -468,18 +468,29 @@ int maat_table_callback_register(struct maat *maat_instance, int table_id,
     return 0;
 }
 
-int maat_compile_table_ex_schema_register(struct maat *maat_instance, int table_id,
+int maat_compile_table_ex_schema_register(struct maat *maat_instance, const char *table_name,
                                           maat_rule_ex_new_func_t *new_func,
                                           maat_rule_ex_free_func_t *free_func,
                                           maat_rule_ex_dup_func_t *dup_func,
                                           long argl, void *argp)
 {
-    if (NULL == maat_instance || table_id < 0 || table_id > MAX_TABLE_NUM) {
+    if (NULL == maat_instance || NULL == table_name) {
+        return -1;
+    }
+    
+    int table_id = maat_table_get_id(maat_instance, table_name);
+    if (table_id < 0) {
+        log_error(maat_instance->logger, MODULE_MAAT_API, 
+                  "table:%s is not registered yet", table_name);
         return -1;
     }
 
     enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
-    assert(table_type == TABLE_TYPE_COMPILE);
+    if (table_type != TABLE_TYPE_COMPILE) {
+        log_error(maat_instance->logger, MODULE_MAAT_API, 
+                  "table:%s is not compile table", table_name);
+        return -1;
+    }
 
     void *compile_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
     assert(compile_schema != NULL);
@@ -496,13 +507,15 @@ void *maat_compile_table_get_ex_data(struct maat *maat_instance, int table_id, i
     return compile_table_get_rule_ex_data(schema, compile_id, idx);
 }
 
-int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int table_id,
+int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, const char *table_name,
+                                            int table_id,
 									        maat_plugin_ex_new_func_t *new_func,
 									        maat_plugin_ex_free_func_t *free_func,
 									        maat_plugin_ex_dup_func_t *dup_func,
 									        long argl, void *argp, struct log_handle *logger)
 {
-    if (NULL == tbl_mgr || NULL == new_func || NULL == free_func || NULL == dup_func) {
+    if (NULL == tbl_mgr || NULL == table_name || NULL == new_func || 
+        NULL == free_func || NULL == dup_func) {
         assert(0);
         log_error(logger, MODULE_MAAT_API, 
                   "table(table_id:%d) %s failed: invalid parameter", __FUNCTION__);
@@ -512,7 +525,8 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t
     void *schema = table_manager_get_schema(tbl_mgr, table_id);
     if (NULL == schema) {
         log_error(logger, MODULE_MAAT_API, 
-                  "Error: %s, table(table_id:%d) is not registered", __FUNCTION__, table_id);
+                  "Error: %s, table:%s is not registered, can't register ex schema", 
+                  __FUNCTION__, table_name);
         return -1;
     }
 
@@ -521,10 +535,10 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t
     switch (table_type) {
         case TABLE_TYPE_PLUGIN:
             ex_schema = plugin_table_get_ex_data_schema(schema);
-            if (NULL == ex_schema) {
+            if (ex_schema != NULL) {
                 log_error(logger, MODULE_MAAT_API,
-                          "Error: %s, table(table_id:%d) is not a valid plugin table",
+                          "Error: %s, EX data schema already registed for plugin table:%s",
-                          __FUNCTION__, table_id);
+                          __FUNCTION__, table_name);
                 return -1;
             }
             plugin_table_set_ex_data_schema(schema, new_func, free_func,
@@ -532,10 +546,10 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t
             break;
         case TABLE_TYPE_IP_PLUGIN:
             ex_schema = ip_plugin_table_get_ex_data_schema(schema);
-            if (NULL == ex_schema) {
+            if (ex_schema != NULL) {
                 log_error(logger, MODULE_MAAT_API,
-                          "Error: %s, table(table_id:%d) is not a valid ip_plugin table",
+                          "Error: %s, EX data schema already registed for ip_plugin table:%s",
-                          __FUNCTION__, table_id);
+                          __FUNCTION__, table_name);
                 return -1;
             }
             ip_plugin_table_set_ex_data_schema(schema, new_func, free_func,
@@ -548,7 +562,7 @@ int generic_plugin_table_ex_schema_register(struct table_manager *tbl_mgr, int t
     return 0;
 }
 
-void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema,
+void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, const char *table_name,
                                              int table_id, enum table_type table_type,
                                              int valid_column)
 {
@@ -592,24 +606,36 @@ void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema,
 
     switch (table_type) {
         case TABLE_TYPE_PLUGIN:
-            plugin_runtime_commit(runtime, "false_plugin_table");
+            plugin_runtime_commit(runtime, table_name);
             break;
         case TABLE_TYPE_IP_PLUGIN:
-            ip_plugin_runtime_commit(runtime, "false_plugin_table");
+            ip_plugin_runtime_commit(runtime, table_name);
             break;
         default:
             break;
     }
 }
 
-int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_id,
+int maat_plugin_table_ex_schema_register(struct maat *maat_instance, const char *table_name,
                                          maat_plugin_ex_new_func_t *new_func,
                                          maat_plugin_ex_free_func_t *free_func,
                                          maat_plugin_ex_dup_func_t *dup_func,
                                          long argl, void *argp)
 {
+    if (NULL == maat_instance || NULL == table_name) {
+        return -1;
+    }
+
+    int table_id = maat_table_get_id(maat_instance, table_name);
+    if (table_id < 0) {
+        log_error(maat_instance->logger, MODULE_MAAT_API, 
+                  "table_name:%s not registered yet, can't register ex_schema.", table_name);
+        return -1;
+    }
+
     pthread_mutex_lock(&(maat_instance->background_update_mutex));
-    int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_id,
+    int ret = generic_plugin_table_ex_schema_register(maat_instance->tbl_mgr, table_name,
+                                                      table_id,
                                                       new_func, free_func, dup_func,
                                                       argl, argp, maat_instance->logger);
     if (ret < 0) {
@@ -626,7 +652,7 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
 
         table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
         valid_column = table_manager_get_valid_column(maat_instance->tbl_mgr, table_id);
-        generic_plugin_runtime_commit_ex_schema(runtime, schema, table_id, table_type,
+        generic_plugin_runtime_commit_ex_schema(runtime, schema, table_name, table_id, table_type,
                                                 valid_column);
     }
     pthread_mutex_unlock(&(maat_instance->background_update_mutex));
@@ -634,39 +660,35 @@ int maat_plugin_table_ex_schema_register(struct maat *maat_instance, int table_i
     return 0;
 }
 
-int maat_plugin_table_get_ex_data(struct maat *maat_instance, int table_id,
+int maat_ip_plugin_table_get_ex_data(struct maat *maat_instance, const char *table_name,
-                                  const char *key, size_t key_len,
+                                     const struct ip_addr *ip_addr,
                                      void **ex_data_array, size_t n_ex_data)
 {
+    if (NULL == maat_instance || NULL == table_name || NULL == ip_addr ||
+        NULL == ex_data_array || 0 == n_ex_data) {
+        return -1;
+    }
+    
+    int table_id = maat_table_get_id(maat_instance, table_name);
+    if (table_id < 0) {
+        log_error(maat_instance->logger, MODULE_MAAT_API, 
+                  "table_name:%s not registered yet, can't register ex_schema.", table_name);
+        return -1;
+    }
+
     struct maat_runtime *maat_rt = maat_instance->maat_rt;
     if (NULL == maat_rt) {
         return -1;
     }
 
-    enum table_type table_type = table_manager_get_table_type(maat_rt->ref_tbl_mgr, table_id);
+    void *ip_plugin_rt = table_manager_get_runtime(maat_rt->ref_tbl_mgr, table_id);
-    void *runtime = table_manager_get_runtime(maat_rt->ref_tbl_mgr, table_id);
+    if (NULL == ip_plugin_rt) {
-    if (NULL == runtime) {
         return -1;
     }
 
-    struct ex_data_runtime *ex_data_rt = NULL;
+    int n_hit_ex_data = ip_plugin_runtime_get_ex_data(ip_plugin_rt, ip_addr, ex_data_array, n_ex_data);
-    switch (table_type) {
-        case TABLE_TYPE_PLUGIN:
-            ex_data_rt = plugin_runtime_get_ex_data_rt(runtime);
-            break;
-        case TABLE_TYPE_IP_PLUGIN:
-            ex_data_rt = ip_plugin_runtime_get_ex_data_rt(runtime);
-            break;
-        default:
-            break;
-    }
     
-    if (NULL == ex_data_rt) {
+    return n_hit_ex_data;
-        return -1;
-    }
-
-    *ex_data_array = ex_data_runtime_get_ex_data(ex_data_rt, key, key_len);
-    return 0;
 }
 
 struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
@@ -677,7 +699,7 @@ struct maat_state *make_outer_state(struct maat *maat_instance, int thread_id)
 	outer_state->maat_instance = maat_instance;
 	outer_state->district_id = DISTRICT_ANY;
 	outer_state->thread_id = (signed short)thread_id;
-    outer_state->compile_table_id = -1; //-1 means caller not specify compile table, use default compile table
+    outer_state->n_compile_table = 0;
 
 	return outer_state;
 }
@@ -740,19 +762,32 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
     mid = grab_state(state, maat_instance, thread_id);
     mid->scan_cnt++;
 
-    int physical_table_id = 0;
     int vtable_id = 0;
+    size_t physical_table_cnt = 0;
+    int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
+    memset(physical_table_ids, -1, sizeof(physical_table_ids));
     
     enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
     if (table_type == TABLE_TYPE_VIRTUAL) {
         //find physical table id
         void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
-        physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_FLAG);
+        vtable_id = virtual_table_get_id(virtual_schema);
-        if (physical_table_id < 0) {
+        if (vtable_id < 0) {
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids);
+        if (physical_table_cnt <= 0) {
             return MAAT_SCAN_ERR;
         }
     } else {
-        physical_table_id = table_id;
+        //physical table type must be TABLE_TYPE_FLAG
+        if (table_type != TABLE_TYPE_FLAG) {
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_ids[0] = table_id;
+        physical_table_cnt = 1;
     }
 
     if (NULL == maat_instance->maat_rt) {
@@ -764,38 +799,55 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
     maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
     alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
 
-    int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
+    int group_ids[MAX_SCANNER_HIT_GROUP_NUM];
-    void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
+    memset(group_ids, -1, sizeof(group_ids));
+    int hit_group_index = 0;
+    for (size_t i = 0; i < physical_table_cnt; i++) {
+        table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]);
+        if (table_type != TABLE_TYPE_FLAG) {
+            continue;
+        }
+
+        void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_ids[i]);
         if (NULL == flag_rt) {
             return MAAT_SCAN_ERR;
         }
 
-    int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, 
+        int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id,
-                                                 thread_id,
+                                              flag, group_ids, hit_group_index,
-                                                 flag, group_ids, 
                                               MAX_SCANNER_HIT_GROUP_NUM,
                                               vtable_id, mid);
         if (group_hit_cnt < 0) {
             return MAAT_SCAN_ERR;
         }
 
-    size_t n_hit_compile = 0;
-    if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
-        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
         if (group_hit_cnt > 0) {
             flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id);
         }
-
+        hit_group_index += group_hit_cnt;
-        int compile_table_id = -1;
-        if (mid->compile_table_id == -1) {
-            compile_table_id = maat_instance->default_compile_table_id;
-        } else {
-            compile_table_id = mid->compile_table_id;
     }
 
-        void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
+    size_t sum_hit_compile_cnt = 0;
-        n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+    if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) {
-        *n_hit_result = n_hit_compile;
+        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
+        int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0};
+        size_t compile_table_cnt = 0;
+        if (0 == mid->n_compile_table) {
+            compile_table_id[0] = maat_instance->default_compile_table_id;
+            compile_table_cnt = 1;
+        } else {
+            for (size_t i = 0; i < mid->n_compile_table; i++) {
+                compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]);
+            }
+            compile_table_cnt = mid->n_compile_table;
+        }
+
+        for (size_t i = 0; i < compile_table_cnt; i++) {
+            void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]);
+            size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+            sum_hit_compile_cnt += n_hit_compile;
+        }
+        *n_hit_result = sum_hit_compile_cnt;
 
         assert(mid->is_last_scan < LAST_SCAN_FINISHED);
         if (LAST_SCAN_SET == mid->is_last_scan) {
@@ -803,16 +855,16 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
         }
     }
 
-    if (n_hit_compile > 0) {
+    if (sum_hit_compile_cnt > 0) {
         alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
-        if (0 == group_hit_cnt) {
+        if (0 == hit_group_index) {
             //hit NOT group
              alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
         }
         return MAAT_SCAN_HIT;
     } else {
-        // n_hit_compile == 0
+        // sum_hit_compile_cnt == 0
-        if (group_hit_cnt > 0) {
+        if (hit_group_index > 0) {
             return MAAT_SCAN_HALF_HIT;
         }
     }
@@ -835,19 +887,32 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id,
     mid = grab_state(state, maat_instance, thread_id);
     mid->scan_cnt++;
 
-    int physical_table_id = 0;
     int vtable_id = 0;
+    size_t physical_table_cnt = 0;
+    int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
+    memset(physical_table_ids, -1, sizeof(physical_table_ids));
     
     enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
     if (table_type == TABLE_TYPE_VIRTUAL) {
         //find physical table id
         void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
-        physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_FLAG);
+        vtable_id = virtual_table_get_id(virtual_schema);
-        if (physical_table_id < 0) {
+        if (vtable_id < 0) {
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids);
+        if (physical_table_cnt <= 0) {
             return MAAT_SCAN_ERR;
         }
     } else {
-        physical_table_id = table_id;
+        //physcial table type must be TABLE_TYPE_INTERVAL or TABLE_TYPE_INTERVAL_PLUS
+        if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_ids[0] = table_id;
+        physical_table_cnt = 1;
     }
 
     if (NULL == maat_instance->maat_rt) {
@@ -857,47 +922,63 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id,
     }
 
     maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
+    alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
 
-    table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
+    int group_ids[MAX_SCANNER_HIT_GROUP_NUM];
+    memset(group_ids, -1, sizeof(group_ids));
+    int hit_group_index = 0;
+    for (size_t i = 0; i < physical_table_cnt; i++) {
+        table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]);
         if ((table_type == TABLE_TYPE_INTERVAL_PLUS) &&
             (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
             maat_instance->scan_err_cnt++;
             return MAAT_SCAN_ERR;
         }
 
-    alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
+        if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) {
+            continue;
+        }
 
-    int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
         void *interval_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
         if (NULL == interval_rt) {
             return MAAT_SCAN_ERR;
         }
 
         int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt,
-                                              thread_id, intval, group_ids, 
+                                                  thread_id, intval, group_ids, hit_group_index,
                                                   MAX_SCANNER_HIT_GROUP_NUM,
                                                   vtable_id, mid);
         if (group_hit_cnt < 0) {
             return MAAT_SCAN_ERR;
         }
         
-    size_t n_hit_compile = 0;
-    if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
-        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
         if (group_hit_cnt > 0) {
             interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id);
         }
-
+        hit_group_index += group_hit_cnt;
-        int compile_table_id = -1;
-        if (mid->compile_table_id == -1) {
-            compile_table_id = maat_instance->default_compile_table_id;
-        } else {
-            compile_table_id = mid->compile_table_id;
     }
 
-        void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
+    size_t sum_hit_compile_cnt = 0;
-        n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+    if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) {
-        *n_hit_result = n_hit_compile;
+        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
+        int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0};
+        size_t compile_table_cnt = 0;
+        if (0 == mid->n_compile_table) {
+            compile_table_id[0] = maat_instance->default_compile_table_id;
+            compile_table_cnt = 1;
+        } else {
+            for (size_t i = 0; i < mid->n_compile_table; i++) {
+                compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]);
+            }
+            compile_table_cnt = mid->n_compile_table;
+        }
+
+        for (size_t i = 0; i < compile_table_cnt; i++) {
+            void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]);
+            size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+            sum_hit_compile_cnt += n_hit_compile;
+        }
+        *n_hit_result = sum_hit_compile_cnt;
 
         assert(mid->is_last_scan < LAST_SCAN_FINISHED);
         if (LAST_SCAN_SET == mid->is_last_scan) {
@@ -905,16 +986,16 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id,
         }
     }
 
-    if (n_hit_compile > 0) {
+    if (sum_hit_compile_cnt > 0) {
         alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
-        if (0 == group_hit_cnt) {
+        if (0 == hit_group_index) {
             //hit NOT group
              alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
         }
         return MAAT_SCAN_HIT;
     } else {
-        // n_hit_compile == 0
+        // sum_hit_compile_cnt == 0
-        if (group_hit_cnt > 0) {
+        if (hit_group_index > 0) {
             return MAAT_SCAN_HALF_HIT;
         }
     }
@@ -940,50 +1021,90 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
     maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
 
     int vtable_id = 0;
-    // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
+    size_t physical_table_cnt = 0;
-    //                                                                   SCAN_TYPE_IP, &virtual_table_id);
+    int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
-    // if (NULL == table_schema) {
+    memset(physical_table_ids, -1, sizeof(physical_table_ids));
-    //     return MAAT_SCAN_ERR;
-    // }
     
-    int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
     enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
-    assert(table_type == TABLE_TYPE_IP_PLUS);
+    if (table_type == TABLE_TYPE_VIRTUAL) {
+        //find physical table id
+        void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
+        vtable_id = virtual_table_get_id(virtual_schema);
+        if (vtable_id < 0) {
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids);
+        if (physical_table_cnt <= 0) {
+            return MAAT_SCAN_ERR;
+        }
+    } else {
+        //physcial table type must be TABLE_TYPE_IP_PLUS 
+        if (table_type != TABLE_TYPE_IP_PLUS) {
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_ids[0] = table_id;
+        physical_table_cnt = 1;
+    }
+
+    if (NULL == maat_instance->maat_rt) {
+        log_error(maat_instance->logger, MODULE_MAAT_API, 
+                 "maat_scan_ipv4 error because of maat_runtime is NULL");
+        return MAAT_SCAN_OK;
+    }
+
+    maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
+    alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
+
+    int group_ids[MAX_SCANNER_HIT_GROUP_NUM];
+    memset(group_ids, -1, sizeof(group_ids));
+    int hit_group_index = 0;
+    for (size_t i = 0; i < physical_table_cnt; i++) {
+        table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]);
+        if (table_type != TABLE_TYPE_IP_PLUS) {
+            continue;
+        }
 
-    // int table_id = table_schema_get_table_id(real_table);
         void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
         if (NULL == ip_rt) {
             return MAAT_SCAN_ERR;
         }
-    // size_t rule_num = table_runtime_rule_count(table_rt);
-    // if (0 == rule_num) {
-    //     return 0;
-    // }
 
         int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4,
-                                        (uint8_t *)&ip_addr, group_ids, sizeof(group_ids),
+                                            (uint8_t *)&ip_addr, group_ids, hit_group_index,
-                                         vtable_id, mid);
+                                            MAX_SCANNER_HIT_GROUP_NUM, vtable_id, mid);
         if (group_hit_cnt < 0) {
             return MAAT_SCAN_ERR;
         }
         
-     size_t n_hit_compile = 0;
-    if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
-        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
         if (group_hit_cnt > 0) {
             ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
         }
-
+        hit_group_index += group_hit_cnt;
-        int compile_table_id = -1;
-        if (mid->compile_table_id == -1) {
-            compile_table_id = maat_instance->default_compile_table_id;
-        } else {
-            compile_table_id = mid->compile_table_id;
     }
     
-        void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
+     size_t sum_hit_compile_cnt = 0;
-        n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+    if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) {
-        *n_hit_result = n_hit_compile;
+        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
+        int compile_table_id[MAX_COMPILE_TABLE_NUM] = {0};
+        size_t compile_table_cnt = 0;
+        if (0 == mid->n_compile_table) {
+            compile_table_id[0] = maat_instance->default_compile_table_id;
+            compile_table_cnt = 1;
+        } else {
+            for (size_t i = 0; i < mid->n_compile_table; i++) {
+                compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]);
+            }
+            compile_table_cnt = mid->n_compile_table;
+        }
+
+        for (size_t i = 0; i < compile_table_cnt; i++) {
+            void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id[i]);
+            size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+            sum_hit_compile_cnt += n_hit_compile;
+        }
+        *n_hit_result = sum_hit_compile_cnt;
 
         assert(mid->is_last_scan < LAST_SCAN_FINISHED);
         if (LAST_SCAN_SET == mid->is_last_scan) {
@@ -991,16 +1112,16 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id,
         }
     }
 
-    if (n_hit_compile > 0) {
+    if (sum_hit_compile_cnt > 0) {
         alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
-        if (0 == group_hit_cnt) {
+        if (0 == hit_group_index) {
             //hit NOT group
              alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
         }
         return MAAT_SCAN_HIT;
     } else {
         // n_hit_compile == 0
-        if (group_hit_cnt > 0) {
+        if (hit_group_index > 0) {
             return MAAT_SCAN_HALF_HIT;
         }
     }
@@ -1025,51 +1146,91 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id,
     mid = grab_state(state, maat_instance, thread_id);
     mid->scan_cnt++;
 
-    maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
     int vtable_id = 0;
-    // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id,
+    size_t physical_table_cnt = 0;
-    //                                                                   SCAN_TYPE_IP, &virtual_table_id);
+    int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
-    // if (NULL == table_schema) {
+    memset(physical_table_ids, -1, sizeof(physical_table_ids));
-    //     return MAAT_SCAN_ERR;
-    // }
     
-    int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
     enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
-    assert(table_type == TABLE_TYPE_IP_PLUS);
+    if (table_type == TABLE_TYPE_VIRTUAL) {
+        //find physical table id
+        void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
+        vtable_id = virtual_table_get_id(virtual_schema);
+        if (vtable_id < 0) {
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids);
+        if (physical_table_cnt <= 0) {
+            return MAAT_SCAN_ERR;
+        }
+    } else {
+        //physcial table type must be TABLE_TYPE_IP_PLUS
+        if (table_type != TABLE_TYPE_IP_PLUS) {
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_ids[0] = table_id;
+        physical_table_cnt = 1;
+    }
+
+    if (NULL == maat_instance->maat_rt) {
+        log_error(maat_instance->logger, MODULE_MAAT_API, 
+                 "maat_scan_string error because of maat_runtime is NULL");
+        return MAAT_SCAN_OK;
+    }
+
+    maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
+    alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
+
+    int group_ids[MAX_SCANNER_HIT_GROUP_NUM];
+    memset(group_ids, -1, sizeof(group_ids));
+    int hit_group_index = 0;
+    for (size_t i = 0; i < physical_table_cnt; i++) {
+        table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]);
+        if (table_type != TABLE_TYPE_IP_PLUS) {
+            continue;
+        }
 
-    // int table_id = table_schema_get_table_id(real_table);
         void *ip_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
         if (NULL == ip_rt) {
             return MAAT_SCAN_ERR;
         }
-    // size_t rule_num = table_runtime_rule_count(table_rt);
-    // if (0 == rule_num) {
-    //     return 0;
-    // }
 
-    int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, ip_addr, 
+        int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6,
-                                         group_ids, sizeof(group_ids), vtable_id, mid);
+                                            ip_addr, group_ids, hit_group_index,
+                                            MAX_SCANNER_HIT_GROUP_NUM, vtable_id, mid);
         if (group_hit_cnt < 0) {
             return MAAT_SCAN_ERR;
         }
         
-    size_t n_hit_compile = 0;
-    if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
-        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
         if (group_hit_cnt > 0) {
             ip_runtime_scan_hit_inc((struct ip_runtime *)ip_rt, thread_id);
         }
-
+        hit_group_index += group_hit_cnt;
-        int compile_table_id = -1;
-        if (mid->compile_table_id == -1) {
-            compile_table_id = maat_instance->default_compile_table_id;
-        } else {
-            compile_table_id = mid->compile_table_id;
     }
 
-        void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
+    size_t sum_hit_compile_cnt = 0;
-        n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+    if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) {
-        *n_hit_result = n_hit_compile;
+        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
+        int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0};
+        size_t compile_table_cnt = 0;
+        if (0 == mid->n_compile_table) {
+            compile_table_ids[0] = maat_instance->default_compile_table_id;
+            compile_table_cnt = 1;
+        } else {
+            for (size_t i = 0; i < mid->n_compile_table; i++) {
+                compile_table_ids[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]);
+            }
+            compile_table_cnt = mid->n_compile_table;
+        }
+
+        for (size_t i = 0; i < compile_table_cnt; i++) {
+            void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]);
+            size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+            sum_hit_compile_cnt += n_hit_compile;
+        }
+        *n_hit_result = sum_hit_compile_cnt;
 
         assert(mid->is_last_scan < LAST_SCAN_FINISHED);
         if (LAST_SCAN_SET == mid->is_last_scan) {
@@ -1077,16 +1238,16 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id,
         }
     }
 
-    if (n_hit_compile > 0) {
+    if (sum_hit_compile_cnt > 0) {
         alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
-        if (0 == group_hit_cnt) {
+        if (0 == hit_group_index) {
             //hit NOT group
              alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
         }
         return MAAT_SCAN_HIT;
     } else {
         // n_hit_compile == 0
-        if (group_hit_cnt > 0) {
+        if (hit_group_index > 0) {
             return MAAT_SCAN_HALF_HIT;
         }
     }
@@ -1110,19 +1271,34 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
     mid = grab_state(state, maat_instance, thread_id);
     mid->scan_cnt++;
 
-    int physical_table_id = 0;
     int vtable_id = 0;
+    size_t physical_table_cnt = 0;
+    int physical_table_ids[MAX_PHYSICAL_TABLE_NUM];
+    memset(physical_table_ids, -1, sizeof(physical_table_ids));
     
     enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
     if (table_type == TABLE_TYPE_VIRTUAL) {
         //find physical table id
         void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
-        physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_STRING);
+        vtable_id = virtual_table_get_id(virtual_schema);
-        if (physical_table_id < 0) {
+        if (vtable_id < 0) {
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_cnt = virtual_table_get_physical_table_id(virtual_schema, physical_table_ids);
+        if (physical_table_cnt <= 0) {
             return MAAT_SCAN_ERR;
         }
     } else {
-        physical_table_id = table_id;
+        //physcial table type must be TABLE_TYPE_EXPR or TABLE_TYPE_EXPR_PLUS
+        if ((table_type == TABLE_TYPE_EXPR_PLUS) &&
+            (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)){
+            maat_instance->scan_err_cnt++;
+            return MAAT_SCAN_ERR;
+        }
+
+        physical_table_ids[0] = table_id;
+        physical_table_cnt = 1;
     }
 
     if (NULL == maat_instance->maat_rt) {
@@ -1132,48 +1308,59 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
     }
 
     maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
-
-    table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
-    if ((table_type == TABLE_TYPE_EXPR_PLUS) && 
-        (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
-        maat_instance->scan_err_cnt++;
-        return MAAT_SCAN_ERR;
-    }
-
     alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
 
-    int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
+    int group_ids[MAX_SCANNER_HIT_GROUP_NUM];
-    void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
+    memset(group_ids, -1, sizeof(group_ids));
+    int hit_group_index = 0;
+    for (size_t i = 0; i < physical_table_cnt; i++) {
+        table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[i]);
+        if (table_type != TABLE_TYPE_EXPR && table_type != TABLE_TYPE_EXPR_PLUS) {
+            continue;
+        }
+
+        void *expr_rt = table_manager_get_runtime(maat_instance->tbl_mgr, physical_table_ids[i]);
         if (NULL == expr_rt) {
             return MAAT_SCAN_ERR;
         }
 
         int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt,
                                               thread_id, data, data_len,
-                                           group_ids, 
+                                              group_ids, hit_group_index,
                                               MAX_SCANNER_HIT_GROUP_NUM,
                                               vtable_id, mid);
         if (group_hit_cnt < 0) {
             return MAAT_SCAN_ERR;
         }
 
-     size_t n_hit_compile = 0;
-    if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
-        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
         if (group_hit_cnt > 0) {
             expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
         }
-
+        hit_group_index += group_hit_cnt;
-        int compile_table_id = -1;
-        if (mid->compile_table_id == -1) {
-            compile_table_id = maat_instance->default_compile_table_id;
-        } else {
-            compile_table_id = mid->compile_table_id;
     }
 
-        void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
+     size_t sum_hit_compile_cnt = 0;
-        n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+    if (hit_group_index > 0 || scan_status_should_compile_NOT(mid)) {
-        *n_hit_result = n_hit_compile;
+        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
+        int compile_table_ids[MAX_COMPILE_TABLE_NUM];
+        memset(compile_table_ids, -1, sizeof(compile_table_ids));
+        size_t compile_table_cnt = 0;
+        if (0 == mid->n_compile_table) {
+            compile_table_ids[0] = maat_instance->default_compile_table_id;
+            compile_table_cnt = 1;
+        } else {
+            for (size_t i = 0; i < mid->n_compile_table; i++) {
+                compile_table_ids[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]);
+            }
+            compile_table_cnt = mid->n_compile_table;
+        }
+
+        for (size_t i = 0; i < compile_table_cnt; i++) {
+            void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]);
+            size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid);
+            sum_hit_compile_cnt += n_hit_compile;
+        }
+        *n_hit_result = sum_hit_compile_cnt;
 
         assert(mid->is_last_scan < LAST_SCAN_FINISHED);
         if (LAST_SCAN_SET == mid->is_last_scan) {
@@ -1181,16 +1368,16 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
         }
     }
 
-    if (n_hit_compile > 0) {
+    if (sum_hit_compile_cnt > 0) {
         alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
-        if (0 == group_hit_cnt) {
+        if (0 == hit_group_index) {
             //hit NOT group
              alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
         }
         return MAAT_SCAN_HIT;
     } else {
         // n_hit_compile == 0
-        if (group_hit_cnt > 0) {
+        if (hit_group_index > 0) {
             return MAAT_SCAN_HALF_HIT;
         }
     }
@@ -1253,15 +1440,18 @@ int maat_state_set_last_scan(struct maat *maat_instance, struct maat_state **sta
     return 0;
 }
 
-int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_state **state,
+int maat_state_set_scan_compile_tables(struct maat *maat_instance, struct maat_state **state,
-                                      int compile_table_id)
+                                       const char *compile_tables[], size_t n_table)
 {
     if (NULL == maat_instance->maat_rt) {
         return -1;
     }
 
     struct maat_state *mid = grab_state(state, maat_instance, -1);
-    mid->compile_table_id = compile_table_id;
+    for (size_t i = 0; i < mid->n_compile_table; i++) {
+        
+        strncpy(mid->compile_tables[i], compile_tables[i], strlen(compile_tables[i]));
+    }
 
     return 0;
 }
@@ -1269,20 +1459,33 @@ int maat_state_set_scan_compile_table(struct maat *maat_instance, struct maat_st
 size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state,
                           struct maat_hit_path *paths, size_t n_path)
 {
-    int compile_table_id = -1;
+    int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0};
+    size_t compile_table_cnt = 0;
 
-    if (state->compile_table_id == -1) {
+    if (0 == state->n_compile_table) {
-        compile_table_id = maat_instance->default_compile_table_id;
+        compile_table_ids[0] = maat_instance->default_compile_table_id;
+        compile_table_cnt = 1;
     } else {
-        compile_table_id = state->compile_table_id;
+        for (size_t i = 0; i < state->n_compile_table; i++) {
+            compile_table_ids[i] = maat_table_get_id(maat_instance, state->compile_tables[i]);
+        }
+        compile_table_cnt = state->n_compile_table;
     }
 
-    void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
+    size_t hit_path_index = 0;
+    //size_t sum_hit_paths = 0;
+    for (size_t i = 0; i < compile_table_cnt; i++) {
+        void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_ids[i]);
         void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
         assert(NULL != compile_rt && NULL != g2g_runtime);
-    return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, 
+        size_t n_hit_path = compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, 
                                                           (struct group2group_runtime *)g2g_runtime,
-                                         state->compile_state, paths, n_path);
+                                                          state->compile_state, paths, hit_path_index, 
+                                                          n_path);
+        hit_path_index += n_hit_path;
+    }
+   
+    return hit_path_index;
 }
 
 size_t maat_get_hit_objects(struct maat_compile_state *compile_state,

src/maat_compile.c

@@ -10,6 +10,7 @@
 
 #include <assert.h>
 #include <pthread.h>
+#include <limits.h>
 
 #include "maat_utils.h"
 #include "log/log.h"
@@ -35,10 +36,6 @@ enum user_region_encode {
 
 struct compile_schema {
     int compile_id_column;
-    int service_id_column;
-    int action_column;
-    int do_blacklist_column;
-    int do_log_column;
     int tags_column;
     int user_region_column;
     int clause_num_column;
@@ -64,10 +61,6 @@ struct group2compile_schema {
 
 struct compile_item {
     int compile_id;
-    int service_id;
-    int action;
-    int do_blacklist;
-    int do_log;
     char user_region[MAX_TABLE_LINE_SIZE];
     int clause_num;
     int evaluation_order;
@@ -274,30 +267,6 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         read_cnt++;
     }
 
-    custom_item = cJSON_GetObjectItem(item, "service_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        compile_schema->service_id_column = custom_item->valueint;
-        read_cnt++;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "action");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        compile_schema->action_column = custom_item->valueint;
-        read_cnt++;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "do_blacklist");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        compile_schema->do_blacklist_column = custom_item->valueint;
-        read_cnt++;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "do_log");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        compile_schema->do_log_column = custom_item->valueint;
-        read_cnt++;
-    }
-
     custom_item = cJSON_GetObjectItem(item, "tags");
     if (custom_item != NULL && custom_item->type == cJSON_Number) {
         compile_schema->tags_column = custom_item->valueint;
@@ -324,7 +293,7 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 
     compile_schema->ref_tbl_mgr = tbl_mgr;
 
-    if (read_cnt < 10) {
+    if (read_cnt < 6) {
 		goto error;
     }
 
@@ -438,46 +407,6 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
     }
     compile_item->compile_id = atoi(line + column_offset);
 
-    ret = get_column_pos(line, compile_schema->service_id_column, 
-                         &column_offset, &column_len);
-    if (ret < 0) {
-        log_error(logger, MODULE_COMPILE, 
-                  "compile table(table_id:%d) line:%s has no service_id",
-                  compile_schema->table_id, line);
-        goto error;
-    }
-    compile_item->service_id = atoi(line + column_offset);
-
-    ret = get_column_pos(line, compile_schema->action_column, 
-                        &column_offset, &column_len);
-    if (ret < 0) {
-        log_error(logger, MODULE_COMPILE, 
-                  "compile table(table_id:%d) line:%s has no action",
-                  compile_schema->table_id, line);
-        goto error;
-    }
-    compile_item->action = atoi(line + column_offset);
-
-    ret = get_column_pos(line, compile_schema->do_blacklist_column, 
-                         &column_offset, &column_len);
-    if (ret < 0) {
-        log_error(logger, MODULE_COMPILE, 
-                  "compile table(table_id:%d) line:%s has no do_blacklist",
-                  compile_schema->table_id, line);
-        goto error;
-    }
-    compile_item->do_blacklist = atoi(line + column_offset);
-
-    ret = get_column_pos(line, compile_schema->do_log_column, 
-                         &column_offset, &column_len);
-    if (ret < 0) {
-        log_error(logger, MODULE_COMPILE, 
-                 "compile table(table_id:%d) line:%s has no do_log",
-                  compile_schema->table_id, line);
-        goto error;
-    }
-    compile_item->do_log = atoi(line + column_offset);
-
     ret = get_column_pos(line, compile_schema->tags_column,
                          &column_offset, &column_len);
     if (ret < 0) {
@@ -1312,11 +1241,11 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
                                      struct group2group_runtime *g2g_rt, 
 									 struct maat_compile_state *compile_state, 
                                   	 struct maat_hit_path *hit_paths, 
-                                     size_t hit_path_size)
+                                     size_t hit_path_index, size_t hit_path_size)
 {
 	size_t i = 0, j = 0;
 	struct maat_internal_hit_path *internal_path = NULL;
-    size_t hit_path_cnt = 0;
+    size_t hit_path_cnt = hit_path_index;
     size_t new_hit_path_cnt = 0;
 
 	for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) {
@@ -1502,10 +1431,6 @@ void compile_item_to_compile_rule(struct compile_item *compile_item,
 {
     struct maat_rule_head rule_head;
     rule_head.config_id = compile_item->compile_id;
-    rule_head.service_id = compile_item->service_id;
-    rule_head.action = compile_item->action;
-    rule_head.do_blacklist = compile_item->do_blacklist;
-    rule_head.do_log = compile_item->do_log;
     
     compile_rule->magic_num = COMPILE_RULE_MAGIC;
     compile_rule->head = rule_head;
@@ -1792,11 +1717,11 @@ int compile_runtime_match(struct compile_runtime *compile_rt,
 
 int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
                               int *hit_item_ids, size_t hit_item_cnt,
-                              int *group_ids, size_t group_ids_size,
+                              int *group_ids, int hit_group_index, size_t group_ids_size,
                               size_t *n_hit_group_id, struct maat_state *state)
 {
     struct maat_item *item = NULL;
-    size_t hit_group_cnt = 0;
+    size_t hit_group_cnt = hit_group_index;
 
     void *g2g_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
                                              state->maat_instance->g2g_table_id);
@@ -1827,33 +1752,31 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
     *n_hit_group_id = hit_group_cnt;
 
     /* update hit clause */
-    int compile_table_id = -1;
+    int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0};
-    if (state->compile_table_id == -1) {
+    size_t compile_table_cnt = 0;
-        compile_table_id = state->maat_instance->default_compile_table_id;
+    if (0 == state->n_compile_table) {
+        compile_table_ids[0] = state->maat_instance->default_compile_table_id;
+        compile_table_cnt = 1;
     } else {
-        compile_table_id = state->compile_table_id;
+        for (size_t i = 0; i < state->n_compile_table; i++) {
+            compile_table_ids[i] = maat_table_get_id(state->maat_instance, state->compile_tables[i]);
+        }
+        compile_table_cnt = state->n_compile_table;
     }
 
+    for (size_t idx = 0; idx < compile_table_cnt; idx++) {
         void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, 
-                                                 compile_table_id);
+                                                     compile_table_ids[idx]);
-    enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr, 
-                                                              compile_table_id);
-    assert(table_type == TABLE_TYPE_COMPILE);
-
         for (size_t i = 0; i < hit_group_cnt; i++) {
             int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
             int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i],
                                                                    1, top_group_ids);
-        // if (0 == top_group_cnt) {
-        //     maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
-        //                                          group_ids[i], vtable_id);
-        // }
-        
             for (int j = 0; j < top_group_cnt; j++) {
                 maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
                                                      top_group_ids[j], vtable_id);
             }
         }
+    }
 
     return 0;
 }

src/maat_ex_data.c

@@ -250,7 +250,7 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
     return 0;
 }
 
-void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt, 
+void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt, 
                                          const char *key, size_t key_len)
 {
     struct ex_data_container *ex_container = NULL;
@@ -268,6 +268,17 @@ void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt,
     return dup_ex_data;
 }
 
+void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt, 
+                                               struct ex_data_container *ex_container)
+{
+    void *dup_ex_data = NULL;
+    ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, 
+                                    &(ex_container->ex_data), 
+                                    ex_data_rt->ex_schema->argl, 
+                                    ex_data_rt->ex_schema->argp);
+    return dup_ex_data;
+}
+
 void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, 
                                       const char *key, size_t key_len)
 {

src/maat_expr.c

@@ -840,10 +840,9 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
     return ret;
 }
 
-int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, 
+int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, 
-                             const char *data, size_t data_len,
+                      size_t data_len, int *group_ids, int hit_group_index,
-                             int *group_ids, size_t group_ids_size, 
+                      size_t group_ids_size, int vtable_ids, struct maat_state *state)
-                             int vtable_id, struct maat_state *state)
 {
     size_t n_hit_item = 0;
     struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -881,9 +880,9 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
     }
 
     size_t group_hit_cnt = 0;
-    ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, 
+    ret = maat_compile_state_update(expr_rt->item_hash, vtable_ids, hit_item_ids, 
-                                    real_hit_item_cnt, group_ids, group_ids_size, 
+                                    real_hit_item_cnt, group_ids, hit_group_index,
-                                    &group_hit_cnt, state);
+                                    group_ids_size, &group_hit_cnt, state);
     if (ret < 0) {
         return -1;
     }
@@ -901,8 +900,8 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
     expr_rt->hs_stream = hs_stream;
 }
 
-int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, 
+int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, 
-                             size_t data_len, int *group_ids, size_t group_ids_size, 
+                             int *group_ids, int hit_group_index, size_t group_ids_size, 
                              int vtable_id, struct maat_state *state)
 {
     if (NULL == expr_rt) {
@@ -936,8 +935,8 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data,
 
     size_t group_hit_cnt = 0;
     ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, 
-                                    n_hit_item, group_ids, group_ids_size, 
+                                    n_hit_item, group_ids, hit_group_index,
-                                    &group_hit_cnt, state);
+                                    group_ids_size, &group_hit_cnt, state);
     if (ret < 0) {
         return -1;
     }

src/maat_flag.c

@@ -411,8 +411,8 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name)
 }
 
 int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, 
-                           uint64_t flag, int *group_ids, size_t group_ids_size, 
+                      uint64_t flag, int *group_ids, int hit_group_index, 
-                           int vtable_id, struct maat_state *state)
+                      size_t group_ids_size, int vtable_id, struct maat_state *state)
 {
     struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
     
@@ -438,7 +438,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
 
     size_t group_hit_cnt = 0;
     int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids, 
-                                        n_hit_item, group_ids, group_ids_size, 
+                                        n_hit_item, group_ids, hit_group_index, group_ids_size, 
                                         &group_hit_cnt, state);
     if (ret < 0) {
         return -1;

src/maat_interval.c

@@ -440,9 +440,8 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name)
 }
 
 int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, 
-                                 uint64_t integer, 
+                          uint64_t integer, int *group_ids, int hit_group_index, 
-                                 int *group_ids, size_t group_ids_size, 
+                          size_t group_ids_size, int vtable_id, struct maat_state *state)
-                                 int vtable_id, struct maat_state *state)
 {
     struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
     
@@ -475,7 +474,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
 
     size_t group_hit_cnt = 0;
     int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids, 
-                                        n_hit_item, group_ids, group_ids_size, 
+                                        n_hit_item, group_ids, hit_group_index, group_ids_size, 
                                         &group_hit_cnt, state);
     if (ret < 0) {
         return -1;

src/maat_ip.c

@@ -453,7 +453,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name)
     struct ip_matcher *old_ip_matcher = NULL;
     size_t mem_used = 0;
 
-    if (rule_cnt > 0) {
     log_info(ip_rt->logger, MODULE_IP,
              "table[%s] committing %zu ip rules for rebuilding ip_matcher engine",
              table_name, rule_cnt);
@@ -464,7 +463,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name)
                   table_name, rule_cnt);
         ret = -1;
     }
-    }
 
     old_ip_matcher = ip_rt->ip_matcher;
     ip_rt->ip_matcher = new_ip_matcher;
@@ -479,8 +477,8 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name)
 }
 
 int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, 
-                    uint8_t *ip_addr, int *group_ids, size_t group_id_size, 
+                    uint8_t *ip_addr, int *group_ids, int hit_group_index, 
-                    int vtable_id, struct maat_state *state)
+                    size_t group_id_size, int vtable_id, struct maat_state *state)
 {
     int n_hit_item = 0;
     struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -518,7 +516,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
 
     size_t group_hit_cnt = 0;
     int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item,
-                                        group_ids, group_id_size, &group_hit_cnt, state);
+                                        group_ids, hit_group_index, group_id_size, &group_hit_cnt, state);
     if (ret < 0) {
         return -1;
     }

src/maat_ip_plugin.c

@@ -16,16 +16,29 @@
 #include "maat_ex_data.h"
 #include "IPMatcher.h"
 #include "maat_rule.h"
+#include "maat.h"
 #include "maat_garbage_collection.h"
 
 #define MODULE_IP_PLUGIN	 module_name_str("maat.ip_plugin")
 #define MAX_IP_STR    128
 
+struct ipv4_item_rule {
+    uint32_t min_sip;   /* 源地址下界；0表示忽略本字段 */
+    uint32_t max_sip;   /* 源地址上界；0表示固定IP=min_saddr */
+};
+
+struct ipv6_item_rule {
+    uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */
+    uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */
+};
+
 struct ip_plugin_item {
     int item_id;
 	int ip_type;
-	char start_ip[MAX_IP_STR];
+	union {
-	char end_ip[MAX_IP_STR];
+        struct ipv4_item_rule ipv4;
+        struct ipv6_item_rule ipv6;
+    };
 	int rule_tag;
 };
 
@@ -34,6 +47,7 @@ struct ip_plugin_schema {
 	int ip_type_column;
 	int start_ip_column;
 	int end_ip_column;
+    int addr_format_column;
 	int rule_tag_column;
 	struct ex_data_schema *ex_schema;
     int table_id; //ugly
@@ -98,6 +112,12 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         read_cnt++;
     }
 
+    custom_item = cJSON_GetObjectItem(item, "addr_format");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->addr_format_column = custom_item->valueint;
+        read_cnt++;
+    }
+
     // rule_tag is optional
     custom_item = cJSON_GetObjectItem(item, "rule_tag");
     if (custom_item != NULL && custom_item->type == cJSON_Number) {
@@ -106,7 +126,7 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 
     schema->ref_tbl_mgr = tbl_mgr;
 
-    if (read_cnt < 5) {
+    if (read_cnt < 6) {
         goto error;
     }
     
@@ -193,7 +213,11 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
 
     size_t column_offset = 0;
     size_t column_len = 0;
+    char addr_format[16] = {0};
+    char start_ip_str[40] = {0};
+    char end_ip_str[40] = {0};
     struct ip_plugin_item *ip_plugin_item = ALLOC(struct ip_plugin_item, 1);
+
     ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
     if (ret < 0) {
         log_error(logger, MODULE_IP_PLUGIN, 
@@ -211,13 +235,29 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
         goto error;
     }
     ip_plugin_item->ip_type = atoi(line + column_offset);
-    if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) {
+    if (ip_plugin_item->ip_type != IPv4 && ip_plugin_item->ip_type != IPv6) {
         log_error(logger, MODULE_IP_PLUGIN, 
                   "ip_plugin table(table_id:%d) line:%s ip_type[%d] invalid",
                   schema->table_id, line, ip_plugin_item->ip_type);
         goto error;
     }
 
+    ret = get_column_pos(line, schema->addr_format_column, &column_offset, &column_len);
+    if (ret < 0) {
+        log_error(logger, MODULE_IP_PLUGIN, 
+                  "ip_plugin table(table_id:%d) line:%s has no addr_format column",
+                  schema->table_id, line);
+        goto error;
+    }
+
+    memcpy(addr_format, (line + column_offset), column_len);
+    if (IP_FORMAT_UNKNOWN == ip_format_str2int(addr_format)) {
+        log_error(logger, MODULE_IP_PLUGIN, 
+                  "ip_plugin table(table_id:%d) line:%s has invalid addr_format, should be range/CIDR",
+                  schema->table_id, line);
+        goto error;
+    }
+
     ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len);
     if (ret < 0) {
         log_error(logger, MODULE_IP_PLUGIN, 
@@ -225,8 +265,7 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
                   schema->table_id, line);
         goto error;
     }
-    strncpy(ip_plugin_item->start_ip, line + column_offset, 
+    strncpy(start_ip_str, line + column_offset, column_len);
-            MIN(column_len, sizeof(ip_plugin_item->start_ip)));
     
     ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len);
     if (ret < 0) {
@@ -235,8 +274,28 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
                   schema->table_id, line);
         goto error;
     }
-    strncpy(ip_plugin_item->end_ip, line + column_offset, 
+    strncpy(end_ip_str, line + column_offset, column_len);
-            MIN(column_len, sizeof(ip_plugin_item->end_ip)));
+
+    if (IPv4 == ip_plugin_item->ip_type) {
+        ret = ip_format2range(ip_plugin_item->ip_type, ip_format_str2int(addr_format), start_ip_str, end_ip_str, 
+                              &ip_plugin_item->ipv4.min_sip, &ip_plugin_item->ipv4.max_sip);
+        if (ret < 0) {
+            log_error(logger, MODULE_IP_PLUGIN, 
+                      "ip_plugin table(table_id:%d) line:%s ip_format2range(ip4) failed",
+                      schema->table_id, line);
+            goto error;
+        }
+    } else {
+        //ipv6
+        ret = ip_format2range(ip_plugin_item->ip_type, ip_format_str2int(addr_format), start_ip_str, end_ip_str, 
+                              ip_plugin_item->ipv6.min_sip, ip_plugin_item->ipv6.max_sip);
+        if (ret < 0) {
+            log_error(logger, MODULE_IP_PLUGIN, 
+                      "ip_plugin table(table_id:%d) line:%s ip_format2range(ip6) failed",
+                      schema->table_id, line);
+            goto error;
+        }
+    }
 
     return ip_plugin_item;
 error:
@@ -249,33 +308,16 @@ void ip_plugin_item_free(struct ip_plugin_item *item)
     FREE(item);
 }
 
-int ip_plugin_table_ex_data_schema_flag(struct ip_plugin_schema *ip_plugin_schema)
+void ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, 
-{
-    return 0;
-}
-
-int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema, 
                                        maat_plugin_ex_new_func_t *new_func,
 							           maat_plugin_ex_free_func_t *free_func,
 							           maat_plugin_ex_dup_func_t *dup_func,
 							           long argl, void *argp,
                                        struct log_handle *logger)
 {
-    if (NULL == ip_plugin_schema) {
-        return -1;
-    }
-
     struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
-    if (schema->ex_schema != NULL) {
-        assert(0);
-        log_error(logger, MODULE_IP_PLUGIN,
-                  "Error: %s, EX data schema already registed", __FUNCTION__);
-        return -1;
-    }
 
     schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp);
-
-    return 0;
 }
 
 int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_schema *schema,
@@ -284,9 +326,9 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_
 {
     int ret = -1;
     struct ex_data_runtime *ex_data_rt = rt->ex_data_rt;
-    int set_flag = ip_plugin_table_ex_data_schema_flag(schema);
+    struct ex_data_schema *ex_schema = schema->ex_schema;
 
-    if (1 == set_flag) {
+    if (ex_schema != NULL) {
         if (0 == is_valid) {
             //delete
             ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len);
@@ -350,14 +392,16 @@ void ip_plugin_runtime_free(void *ip_plugin_runtime)
 
 void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule)
 {
-    if (4 == item->ip_type) {
+    if (IPv4 == item->ip_type) {
         rule->type = IPv4;
-        ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip,
+        rule->ipv4_rule.start_ip = item->ipv4.min_sip;
-                        &(rule->ipv4_rule.start_ip), &(rule->ipv4_rule.end_ip));
+        rule->ipv4_rule.end_ip = item->ipv4.max_sip;
     } else {
         rule->type = IPv6;
-        ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip,
+        memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, 
-                        rule->ipv6_rule.start_ip, rule->ipv6_rule.end_ip);
+               sizeof(item->ipv6.min_sip));
+        memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, 
+               sizeof(item->ipv6.max_sip));
     }
     
     rule->rule_id = item->item_id;
@@ -436,13 +480,13 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name)
     for (size_t i = 0; i < rule_cnt; i++) {
         struct ip_plugin_item *item = (struct ip_plugin_item *)ex_container[i]->custom_data;
         ip_plugin_item_to_ip_rule(item, &rules[i]);
+        rules[i].user_tag = ex_container[i];
     }
 
     struct ip_matcher *new_ip_matcher = NULL;
     struct ip_matcher *old_ip_matcher = NULL;
     size_t mem_used = 0;
 
-    if (rule_cnt > 0) {
     log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
              "table[%s] committing %zu ip_plugin rules for rebuilding ip_matcher engine",
              table_name, rule_cnt);
@@ -453,7 +497,6 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name)
                   table_name, rule_cnt);
         ret = -1;
     }
-    }
 
     old_ip_matcher = ip_plugin_rt->ip_matcher;
     ip_plugin_rt->ip_matcher = new_ip_matcher;
@@ -477,3 +520,35 @@ struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime
 
     return ip_plugin_rt->ex_data_rt;
 }
+
+int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr *ip_addr, 
+                                  void **ex_data_array, size_t n_ex_data)
+{
+    if (NULL == ip_plugin_runtime) {
+        return -1;
+    }
+
+    struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
+    if (NULL == ip_plugin_rt->ip_matcher) {
+        log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, 
+                 "ip_matcher is NULL, can't get ex data");
+        return 0;
+    }
+
+    struct scan_result results[n_ex_data];
+	memset(results, 0, sizeof(results));
+
+    struct ip_data ip_data = *(const struct ip_data *)ip_addr;
+	if (ip_data.type == IPv4) {
+		ip_data.ipv4 = ntohl(ip_data.ipv4);
+	} else {
+		ipv6_ntoh(ip_data.ipv6);
+	}
+
+	int n_result = ip_matcher_match(ip_plugin_rt->ip_matcher, &ip_data, results, n_ex_data);
+	for (int i = 0; i < n_result; i++) {
+		ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(ip_plugin_rt->ex_data_rt, 
+                                                                    (struct ex_data_container *)results[i].tag);
+	}
+	return n_result;
+}

src/maat_plugin.c

@@ -205,28 +205,16 @@ int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema,
     return n_foreign;
 }
 
-int plugin_table_set_ex_data_schema(void *plugin_schema, 
+void plugin_table_set_ex_data_schema(void *plugin_schema, 
                                     maat_plugin_ex_new_func_t *new_func,
 							        maat_plugin_ex_free_func_t *free_func,
 							        maat_plugin_ex_dup_func_t *dup_func,
 							        long argl, void *argp,
                                     struct log_handle *logger)
 {
-    if (NULL == plugin_schema) {
-        return -1;
-    }
-
     struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
-    if (schema->ex_schema != NULL) {
-        assert(0);
-        log_error(logger, MODULE_PLUGIN, 
-                  "Error: %s, EX data schema already registed", 
-                  __FUNCTION__);
-        return -1;
-    }
-    schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp);
     
-    return 0;
+    schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp);
 }
 
 struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema)

src/maat_rule.c

@@ -418,8 +418,6 @@ void *rule_monitor_loop(void *arg)
     while (maat_instance->is_running) {
         usleep(maat_instance->rule_update_checking_interval_ms * 1000);
 		if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
-            log_info(maat_instance->logger, MODULE_MAAT_RULE,
-                     "rule_monitor_loop.................%d", maat_instance->rule_update_checking_interval_ms * 1000);
 			switch (maat_instance->input_mode) {
                 case DATA_SOURCE_REDIS:
                     redis_monitor_traverse(maat_instance->maat_version,

src/maat_table.c

@@ -325,6 +325,7 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo
 
     item = cJSON_GetObjectItem(json, "table_type");
     if (NULL == item || item->type != cJSON_String) {
+        log_error(logger, MODULE_TABLE, "table:%s has no table_type column", ptable->table_name);
         goto error;
     }
 
@@ -337,6 +338,7 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo
     item = cJSON_GetObjectItem(json, "valid_column");
     if (NULL == item || item->type != cJSON_Number) {
         if (ptable->table_type != TABLE_TYPE_VIRTUAL) {
+            log_error(logger, MODULE_TABLE, "table:%s has no valid column", ptable->table_name);
             goto error;
         }
     } else {

src/maat_virtual.c

@@ -20,7 +20,8 @@
 #define MODULE_VIRTUAL	 module_name_str("maat.virtual")
 
 struct virtual_schema {
-	int physical_table_id[SCAN_TYPE_MAX];
+    char physical_tables[MAX_PHYSICAL_TABLE_NUM][NAME_MAX];
+    size_t n_physical_table;
     int table_id;
     struct table_manager *ref_tbl_mgr;
 };
@@ -28,51 +29,67 @@ struct virtual_schema {
 void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
                          const char *table_name, struct log_handle *logger)
 {
-    cJSON *item = cJSON_GetObjectItem(json, "physical_table");
+    struct virtual_schema *schema = ALLOC(struct virtual_schema, 1);
+    schema->ref_tbl_mgr = tbl_mgr;
+   
+    cJSON *item = cJSON_GetObjectItem(json, "table_id");
+    if (NULL == item || item->type != cJSON_Number) {
+        log_error(logger, MODULE_VIRTUAL, 
+                  "virtual table %s has no table_id column", table_name);
+        goto error;
+    }
+    schema->table_id = item->valueint;
+
+    item = cJSON_GetObjectItem(json, "physical_table");
     if (NULL == item || item->type != cJSON_Array) {
         log_error(logger, MODULE_VIRTUAL, 
                   "virtual table %s has no physical_table column", table_name);
-        return NULL;
+        goto error;
     }
 
-    struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
-    vt_schema->ref_tbl_mgr = tbl_mgr;
-   
     int cnt = cJSON_GetArraySize(item);
     for (int i = 0; i < cnt; i++) {
         cJSON *tmp_item = cJSON_GetArrayItem(item, i);
         if (tmp_item != NULL && tmp_item->type == cJSON_String) {
-            int table_id = table_manager_get_table_id(tbl_mgr, tmp_item->valuestring);
+           memcpy(schema->physical_tables[i], tmp_item->valuestring, 
-            /* physical table should already exist */
+                  strlen(tmp_item->valuestring));
-            if (table_id < 0) {
+        }
-                log_error(logger, MODULE_VIRTUAL, "table:%s is not registered",
+    }
-                            tmp_item->valuestring);
+    schema->n_physical_table = cnt;
-                FREE(vt_schema);
+
+    return schema;
+error:
+    FREE(schema);
     return NULL;
 }
 
-            enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
-            assert(table_type != TABLE_TYPE_INVALID);
-            enum scan_type scan_type = maat_table_get_scan_type(table_type);
-            assert(scan_type != SCAN_TYPE_INVALID);
-            vt_schema->physical_table_id[scan_type]= table_id;
-        }
-    }
-
-    return vt_schema;
-}
-
 void virtual_schema_free(void *virtual_schema)
 {
     FREE(virtual_schema);
 }
 
-int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type)
+int virtual_table_get_id(void *virtual_schema)
 {
     if (NULL == virtual_schema) {
         return -1;
     }
 
     struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
-    return schema->physical_table_id[type];
+    return schema->table_id;
+}
+
+size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[])
+{
+    if (NULL == virtual_schema) {
+        return -1;
+    }
+
+    struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
+
+    for (size_t i = 0; i < schema->n_physical_table; i++) {
+        int table_id = table_manager_get_table_id(schema->ref_tbl_mgr, schema->physical_tables[i]);
+        physical_table_ids[i] = table_id;
+    }
+    
+    return schema->n_physical_table;
 }

test/maat_framework_gtest.cpp

@@ -476,7 +476,8 @@ TEST_F(MaatIPScan, IPv6) {
     int ret = inet_pton(AF_INET6, ip_str, &sip);
     EXPECT_EQ(ret, 1);
 
-    int results[ARRAY_SIZE] = {-1};
+    int results[ARRAY_SIZE];
+    memset(results, -1, sizeof(results));
     size_t n_hit_result = 0;
     struct maat_state *state = NULL;
     ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE,
@@ -673,6 +674,47 @@ TEST_F(NOTLogic, ScanNotAtLast) {
 	maat_state_free(&state);
 }
 
+void maat_read_entry_start_cb(int update_type, void *u_para)
+{
+
+}
+
+void maat_read_entry_cb(int table_id, const char *table_line, void *u_para)
+{
+	char ip_str[16]={0};
+	int entry_id=-1,seq=-1;
+	unsigned int ip_uint=0;
+	int is_valid=0;
+	unsigned int local_ip_nr=16820416;//192.168.0.1
+
+	sscanf(table_line, "%d\t%s\t%d\t%d", &seq,ip_str, &entry_id, &is_valid);
+	inet_pton(AF_INET,ip_str,&ip_uint);
+	if(local_ip_nr==ip_uint)
+	{
+		if(is_valid==1)
+		{
+			//printf("Load entry id %d success.\n",entry_id);
+			EXPECT_EQ(entry_id, 101);
+		}
+		else
+		{
+			//printf("Offload entry id %d success.\n",entry_id);
+		}
+	}
+}
+
+void maat_read_entry_finish_cb(void *u_para)
+{
+	//Maat_feather_t feather=u_para;
+	// long long version=0;
+	// int ret=0,is_last_updating_table=0;
+	// ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
+	// EXPECT_EQ(ret, 0);
+	// ret=Maat_read_state(feather,MAAT_STATE_LAST_UPDATING_TABLE, &is_last_updating_table, sizeof(is_last_updating_table));
+	// EXPECT_EQ(ret, 0);
+	//printf("Maat Version %lld at plugin finish callback, is_last_update=%d.\n",version,is_last_updating_table);
+}
+
 class PluginTable : public testing::Test
 {
 protected:
@@ -686,7 +728,245 @@ protected:
 };
 
 TEST_F(PluginTable, Callback) {
+    int table_id = maat_table_get_id(g_maat_instance, "QD_ENTRY_INFO");
+    int ret = maat_table_callback_register(g_maat_instance, table_id, 
+                                           maat_read_entry_start_cb,
+                                           maat_read_entry_cb, 
+                                           maat_read_entry_finish_cb,
+                                           g_maat_instance);
+    EXPECT_EQ(ret, 0);
+}
 
+class IPPluginTable : public testing::Test
+{
+protected:
+    static void SetUpTestCase() {
+        
+    }
+
+    static void TearDownTestCase() {
+       
+    }
+};
+
+struct ip_plugin_ud {
+	int rule_id;
+	char *buffer;
+	int ref_cnt;
+};
+void ip_plugin_EX_new_cb(int table_id, const char *key, const char *table_line, 
+                         void **ad, long argl, void *argp)
+{
+	int *counter = (int *)argp;
+	size_t column_offset=0, column_len=0;
+	struct ip_plugin_ud *ud = ALLOC(struct ip_plugin_ud, 1);
+
+	int ret = get_column_pos(table_line, 1, &column_offset, &column_len);
+	EXPECT_EQ(ret, 0);
+
+	ud->rule_id = atoi(table_line + column_offset);
+
+	ret = get_column_pos(table_line, 5, &column_offset, &column_len);	
+	EXPECT_EQ(ret, 0);
+
+	ud->buffer = (char *)calloc(sizeof(char), column_len + 1);
+	strncpy(ud->buffer, table_line + column_offset, column_len);
+	ud->ref_cnt = 1;
+	*ad = ud;
+	(*counter)++;
+}
+
+void ip_plugin_EX_free_cb(int table_id, void **ad, long argl, void *argp)
+{
+	struct ip_plugin_ud *ud = (struct ip_plugin_ud *)(*ad);
+	if ((__sync_sub_and_fetch(&ud->ref_cnt, 1) == 0)) {
+		free(ud->buffer);
+		free(ud);
+		*ad = NULL;
+	}
+}
+
+void ip_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *argp)
+{
+	struct ip_plugin_ud *ud = (struct ip_plugin_ud *)(*from);
+	__sync_add_and_fetch(&(ud->ref_cnt), 1);
+	*to = ud;
+}
+
+TEST_F(IPPluginTable, EX_DATA) {
+    int ip_plugin_ex_data_counter = 0;
+    const char *table_name = "TEST_IP_PLUGIN_WITH_EXDATA";
+
+    int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_name,
+                                                   ip_plugin_EX_new_cb,
+                                                   ip_plugin_EX_free_cb,
+                                                   ip_plugin_EX_dup_cb,
+                                                   0, &ip_plugin_ex_data_counter);
+    EXPECT_EQ(ret, 0);
+    EXPECT_EQ(ip_plugin_ex_data_counter, 5);
+
+    struct ip_addr ipv4;
+    ipv4.ip_type = IPv4;
+    ret = inet_pton(AF_INET, "192.168.30.100", &ipv4.ipv4);
+    EXPECT_EQ(ret, 1);
+
+    struct ip_plugin_ud *results[ARRAY_SIZE];
+    ret = maat_ip_plugin_table_get_ex_data(g_maat_instance, table_name, &ipv4, 
+                                           (void **)results, ARRAY_SIZE);
+    EXPECT_EQ(ret, 2);
+    EXPECT_EQ(results[0]->rule_id, 101);
+    EXPECT_EQ(results[1]->rule_id, 102);
+
+    int i = 0;
+    for (i = 0; i < ret; i++) {
+		ip_plugin_EX_free_cb(0, (void**)&(results[i]), 0, NULL);
+	}
+
+    struct ip_addr ipv6;
+    ipv6.ip_type = IPv6;
+    inet_pton(AF_INET6, "2001:db8:1234::5210", &(ipv6.ipv6));
+    memset(results, 0, sizeof(results));
+
+	ret = maat_ip_plugin_table_get_ex_data(g_maat_instance, table_name, &ipv6, 
+                                           (void**)results, ARRAY_SIZE);
+	EXPECT_EQ(ret, 2);
+	EXPECT_EQ(results[0]->rule_id, 104);
+	EXPECT_EQ(results[1]->rule_id, 103);
+
+	for (i = 0; i < ret; i++) {
+		ip_plugin_EX_free_cb(0, (void**)&(results[i]), 0, NULL);
+	}
+
+	//Reproduce BugReport-Liumengyan-20210515
+	inet_pton(AF_INET6, "240e:97c:4010:104::17", &(ipv6.ipv6));
+	ret = maat_ip_plugin_table_get_ex_data(g_maat_instance, table_name, &ipv6, 
+                                           (void**)results, ARRAY_SIZE);
+	EXPECT_EQ(ret, 0);
+}
+
+class VirtualTable : public testing::Test
+{
+protected:
+    static void SetUpTestCase() {
+        
+    }
+
+    static void TearDownTestCase() {
+       
+    }
+};
+
+TEST_F(VirtualTable, basic) {
+    int results[ARRAY_SIZE] = {0};
+    size_t n_hit_result = 0;
+    struct maat_state *state = NULL;
+    const char *table_name = "HTTP_RESPONSE_KEYWORDS";
+    
+	int table_id = maat_table_get_id(g_maat_instance, table_name);
+	ASSERT_GT(table_id, 0);
+
+    char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8";
+    int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data),
+                                results, ARRAY_SIZE, &n_hit_result, &state);
+    EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
+    EXPECT_EQ(n_hit_result, 0);
+    maat_state_free(&state);
+}
+
+class CompileTable : public testing::Test
+{
+protected:
+    static void SetUpTestCase() {
+        
+    }
+
+    static void TearDownTestCase() {
+       
+    }    
+};
+
+struct rule_ex_param {
+	int ref_cnt;
+	char name[NAME_MAX];
+	int id;
+	pthread_mutex_t lock;
+};
+
+void compile_ex_param_new(int idx, const struct maat_rule *rule, const char *srv_def_large, 
+						  void **ad, long argl, void *argp)
+{
+	int *counter = (int *)argp;
+	*ad = NULL;
+	ASSERT_GT(rule->serv_def_len, 4);
+
+	struct rule_ex_param *param = (struct rule_ex_param *)calloc(sizeof(struct rule_ex_param), 1);
+	
+	param->ref_cnt = 1;
+	pthread_mutex_init(&(param->lock), NULL);
+	
+	sscanf(srv_def_large, "%*[^:]:%[^,],%d", param->name, &(param->id));
+	(*counter)++;
+	*ad = param;
+}
+
+void compile_ex_param_free(int idx, const struct maat_rule *rule, const char *srv_def_large, 
+                           void **ad, long argl, void *argp)
+{
+	if (*ad == NULL) {
+		return;
+	}
+
+	struct rule_ex_param *param = (struct rule_ex_param *)*ad;
+	pthread_mutex_lock(&(param->lock));
+	param->ref_cnt--;
+	if (param->ref_cnt > 0) {	
+		pthread_mutex_unlock(&(param->lock));
+		return;
+	}
+	free(param);
+}
+
+void compile_ex_param_dup(int idx, void **to, void **from, long argl, void *argp)
+{
+	struct rule_ex_param *from_param = *((struct rule_ex_param **)from);
+	pthread_mutex_lock(&(from_param->lock));
+	from_param->ref_cnt++;
+	pthread_mutex_unlock(&(from_param->lock));
+	*((struct rule_ex_param**)to) = from_param;
+}
+
+TEST_F(CompileTable, CompileEXData) {
+    int results[ARRAY_SIZE] = {0};
+    size_t n_hit_result = 0;
+    struct maat_state *state = NULL;
+    const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
+	const char *table_name = "HTTP_URL_LITERAL";
+	const char *expect_name = "I have a name";
+
+    int table_id =  maat_table_get_id(g_maat_instance, table_name);
+    ASSERT_GT(table_id, 0);
+
+    int ex_data_counter = 0;
+    int ex_param_idx = maat_compile_table_ex_schema_register(g_maat_instance, "COMPILE_ALIAS",
+                                                             compile_ex_param_new,
+                                                             compile_ex_param_free,
+                                                             compile_ex_param_dup,
+                                                             0, &ex_data_counter);
+    ASSERT_TRUE(ex_param_idx>=0);
+	EXPECT_EQ(ex_data_counter, 1);
+
+    int ret = maat_scan_string(g_maat_instance, table_id, 0, url, strlen(url),
+                                results, ARRAY_SIZE, &n_hit_result, &state);
+    EXPECT_EQ(ret, MAAT_SCAN_HIT);
+
+    void *ex_data = maat_compile_table_get_ex_data(g_maat_instance, table_id, 0, ex_param_idx);
+    ASSERT_TRUE(ex_data!=NULL);
+    struct rule_ex_param *param = (struct rule_ex_param *)ex_data;
+	EXPECT_EQ(param->id, 7799);
+	EXPECT_EQ(strcmp(param->name, expect_name), 0);
+	compile_ex_param_free(0, NULL, NULL, &ex_data, 0, NULL);
+
+	maat_state_free(&state);
 }
 
 int count_line_num_cb(const char *table_name, const char *line, void *u_para)

test/maat_json.json

@@ -2443,11 +2443,11 @@
         {
             "table_name": "TEST_IP_PLUGIN_WITH_EXDATA",
             "table_content": [
-				"101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1",
+				"101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1\trange",
-				"102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1",
+				"102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1\trange",
-				"103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1",
+				"103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1\trange",
-				"104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1",
+				"104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1\trange",
-				"105\t6\t2620:100:3000::\t2620:0100:30ff:ffff:ffff:ffff:ffff:ffff\tBugreport-liumengyan-20210517\t1"
+				"105\t6\t2620:100:3000::\t2620:0100:30ff:ffff:ffff:ffff:ffff:ffff\tBugreport-liumengyan-20210517\t1\trange"
             ]
         },
         {

test/table_info.conf

@@ -6,10 +6,6 @@
         "valid_column":8,
         "custom": {
             "compile_id":1,
-            "service_id":2,
-            "action":3,
-            "do_blacklist":4,
-            "do_log":5,
             "tags":6,
             "user_region":7,
             "clause_num":9,
@@ -336,11 +332,13 @@
         "table_id":29,
         "table_name":"TEST_IP_PLUGIN_WITH_EXDATA",
         "table_type":"ip_plugin",
+        "valid_column":6,
         "custom": {
             "item_id":1,
             "ip_type":2,
             "start_ip":3,
-            "end_ip":4
+            "end_ip":4,
+            "addr_format":7
         }
     },
     {

vendor/CMakeLists.txt

@@ -1,5 +1,5 @@
 # CMakeFiles for 3rd vendor library
-
+cmake_minimum_required(VERSION 3.5)
 include(ExternalProject)
 
 set(VENDOR_ROOT ${CMAKE_BINARY_DIR}/vendor)
@@ -84,8 +84,13 @@ file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
 
 add_library(hiredis-static STATIC IMPORTED GLOBAL)
 add_dependencies(hiredis-static hiredis)
-set_property(TARGET hiredis-static  PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a)
+
 set_property(TARGET hiredis-static  PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+set_property(TARGET hiredis-static  PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a)
+else()
+set_property(TARGET hiredis-static  PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredis.a)
+endif()
 
 #igraph-0.7.1
 ExternalProject_Add(igraph PREFIX igraph