fix flag_matcher and interval_matcher compile error

scanner/adapter_hs.cpp

@@ -414,7 +414,7 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
                     size_t n_result, size_t *n_hit_result)
 {
     if (NULL == hs_instance || NULL == data || (0 == data_len) || 
-        NULL == results || NULL == n_results) {
+        NULL == results || 0 == n_result || NULL == n_hit_result) {
         return -1;
     }
 
@@ -515,9 +515,8 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
     return hs_stream;
 }
 
-int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream,
+int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len,
-                           const char *data, size_t data_len,
+                           struct hs_scan_result *results, size_t n_result, size_t *n_hit_result)
-                           int results[], size_t *n_results)
 {
     hs_error_t err;
 
@@ -549,19 +548,30 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream,
         items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->pattern_id_set, i);
     }
 
-    size_t matched_index = 0;
+    int ret = 0;
+    int matched_index = 0;
     struct bool_expr_match *bool_matcher_results = NULL;
+
     bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
-    size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size,
+    int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size,
                                                  bool_matcher_results, hs_stream->n_expr);
-    for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
+    if (bool_matcher_ret < 0) {
-        results[matched_index] = bool_matcher_results[matched_index].expr_id;
+        ret = -1;
+        goto next;
     }
-    *n_results = bool_matcher_ret;
 
+    if (bool_matcher_ret > n_result) {
+        bool_matcher_ret = n_result;
+    }
+
+    for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) {
+        results[matched_index].item_id = bool_matcher_results[matched_index].expr_id;
+        results[matched_index].user_tag = bool_matcher_results[matched_index].user_tag;
+    }
+    *n_hit_result = bool_matcher_ret;
+next:
     FREE(bool_matcher_results);
-
+    return ret;
-    return 0;
 }
 
 void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)

scanner/adapter_hs.h

@@ -106,7 +106,7 @@ struct adapter_hs_stream;
 struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id);
 
 int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, size_t data_len,
-                           int results[], size_t *n_results);
+                           struct hs_scan_result *results, size_t n_result, size_t *n_hit_result);
 
 void adapter_hs_stream_close(struct adapter_hs_stream *stream);
 

src/CMakeLists.txt

@@ -10,7 +10,7 @@ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
 add_definitions(-fPIC)
 set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp 
     maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp 
-    maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_intval.cpp maat_flag.cpp
+    maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_flag.cpp maat_interval.cpp
     maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_virtual.cpp)
 
 set(LIB_SOURCE_FILES
@@ -27,7 +27,7 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX)
 set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe)
 set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
 
-target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z)
+target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static flagmatcher-static intervalmatcher-static pthread crypto z)
 
 # Shared Library Output
 #add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})

src/inc_internal/maat_expr.h

@@ -44,11 +44,12 @@ int expr_runtime_commit(void *expr_runtime);
 int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, 
                              const char *data, size_t data_len,
                              int *group_ids, size_t group_ids_size, 
-                             int vtable_id, struct maat_state *state);
+                             int vtable_ids, struct maat_state *state);
 
 void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
 int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data,
-                             size_t data_len, int result[], size_t *n_result);
+                             size_t data_len, int *group_ids, size_t group_ids_size, 
+                             int vtable_id, struct maat_state *state);
 void expr_runtime_stream_close(struct expr_runtime *expr_rt);
 
 void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id);

src/inc_internal/maat_flag.h

@@ -16,6 +16,8 @@ extern "C"
 {
 #endif
 
+#include <stdint.h>
+
 #include "log/log.h"
 #include "cJSON/cJSON.h"
 
@@ -42,9 +44,8 @@ int flag_runtime_commit(void *flag_runtime);
  * @retval the num of hit group_id
 */
 int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, 
-                             const char *data, size_t data_len, 
+                           uint64_t flag, int *group_ids, size_t group_ids_size, 
-                             int *group_ids, size_t group_ids_size, 
+                           int vtable_id, struct maat_state *state);
-                             int vtable_id, struct maat_state *state);
 
 void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id);
 long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread);

src/inc_internal/maat_interval.h

@@ -0,0 +1,57 @@
+/*
+**********************************************************************************************
+*	File: maat_interval.h
+*   Description: 
+*	Authors: Liu WenTan <liuwentan@geedgenetworks.com>
+*	Date:    2022-10-31
+*   Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
+***********************************************************************************************
+*/
+
+#ifndef _MAAT_INTERVAL_H_
+#define _MAAT_INTERVAL_H_
+
+#ifdef __cpluscplus
+extern "C"
+{
+#endif
+
+#include <stdint.h>
+
+#include "log/log.h"
+#include "cJSON/cJSON.h"
+
+struct interval_runtime;
+
+void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
+                          const char *table_name, struct log_handle *logger);
+void interval_schema_free(void *interval_schema);
+
+/* interval runtime API */
+void *interval_runtime_new(void *interval_schema, int max_thread_num, 
+                           struct maat_garbage_bin *garbage_bin,
+                           struct log_handle *logger);
+void interval_runtime_free(void *interval_runtime);
+
+int interval_runtime_update(void *interval_runtime, void *interval_schema, 
+                            const char *line, int valid_column);
+int interval_runtime_commit(void *interval_runtime);
+
+/* interval runtime scan API */
+/**
+ * @brief scan intval to get hit group_ids
+ * 
+ * @retval the num of hit group_id
+*/
+int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id, 
+                                 uint64_t integer, int *group_ids, size_t group_ids_size, 
+                                 int vtable_id, struct maat_state *state);
+
+void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id);
+long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread);
+
+#ifdef __cpluscplus
+}
+#endif
+
+#endif

src/inc_internal/maat_intval.h

@@ -1,56 +0,0 @@
-/*
-**********************************************************************************************
-*	File: maat_intval.h
-*   Description: 
-*	Authors: Liu WenTan <liuwentan@geedgenetworks.com>
-*	Date:    2022-10-31
-*   Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
-***********************************************************************************************
-*/
-
-#ifndef _MAAT_INTVAL_H_
-#define _MAAT_INTVAL_H_
-
-#ifdef __cpluscplus
-extern "C"
-{
-#endif
-
-#include "log/log.h"
-#include "cJSON/cJSON.h"
-
-struct intval_runtime;
-
-void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
-                      const char *table_name, struct log_handle *logger);
-void _schema_free(void *intval_schema);
-
-/* intval runtime API */
-void *intval_runtime_new(void *intval_schema, int max_thread_num, 
-                       struct maat_garbage_bin *garbage_bin,
-                       struct log_handle *logger);
-void intval_runtime_free(void *intval_runtime);
-
-int intval_runtime_update(void *intval_runtime, void *intval_schema, 
-                        const char *line, int valid_column);
-int intval_runtime_commit(void *intval_runtime);
-
-/* intval runtime scan API */
-/**
- * @brief scan intval to get hit group_ids
- * 
- * @retval the num of hit group_id
-*/
-int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id, 
-                             const char *data, size_t data_len, 
-                             int *group_ids, size_t group_ids_size, 
-                             int vtable_id, struct maat_state *state);
-
-void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id);
-long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread);
-
-#ifdef __cpluscplus
-}
-#endif
-
-#endif

src/inc_internal/maat_rule.h

@@ -31,6 +31,7 @@ extern "C"
 #include "IPMatcher.h"
 #include "maat_kv.h"
 #include "maat_table.h"
+#include "maat_virtual.h"
 
 #define MAX_TABLE_NUM	256
 
@@ -38,6 +39,11 @@ extern "C"
 #define	MAAT_UPDATE_TYPE_FULL 1
 #define	MAAT_UPDATE_TYPE_INC 2
 
+#define	DISTRICT_ANY		-1
+#define	DISTRICT_UNKNOWN	-2
+
+#define MAX_DISTRICT_STR     128
+
 enum last_scan_flag {
     LAST_SCAN_UNSET,
     LAST_SCAN_SET,
@@ -253,6 +259,7 @@ struct maat_state {
 	struct maat_compile_state *compile_state;
 };
 
+enum scan_type maat_table_get_scan_type(enum table_type table_type);
 
 size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger);
 

src/inc_internal/maat_table.h

@@ -67,6 +67,7 @@ int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *t
 int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str, 
                                     size_t district_str_len, int *district_id);
 int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district);
+
 void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id);
 void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);
 

src/inc_internal/maat_virtual.h

@@ -26,6 +26,7 @@ enum scan_type {
 	SCAN_TYPE_IP,
 	SCAN_TYPE_INTERVAL,
 	SCAN_TYPE_STRING,
+    SCAN_TYPE_FLAG,
 	SCAN_TYPE_MAX
 };
 
@@ -34,6 +35,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 
 void virtual_schema_free(void *virtual_schema);
 
+int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type);
 
 #ifdef __cpluscplus
 }

src/maat_api.cpp

@@ -28,6 +28,7 @@
 #include "maat_garbage_collection.h"
 #include "maat_group.h"
 #include "maat_expr.h"
+#include "maat_flag.h"
 #include "maat_ip.h"
 #include "maat_plugin.h"
 #include "maat_ip_plugin.h"
@@ -35,9 +36,6 @@
 
 #define MODULE_MAAT_API	 module_name_str("maat.api")
 
-#define	DISTRICT_ANY		-1
-#define	DISTRICT_UNKNOWN	-2
-
 enum district_set_flag {
     DISTRICT_FLAG_UNSET,
     DISTRICT_FLAG_SET
@@ -53,17 +51,13 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type)
         break;
     case TABLE_TYPE_EXPR:
     case TABLE_TYPE_EXPR_PLUS:
-    case TABLE_TYPE_SIMILARITY:
-    case TABLE_TYPE_DIGEST:
         ret = SCAN_TYPE_STRING;
         break;
     case TABLE_TYPE_INTERVAL:
     case TABLE_TYPE_INTERVAL_PLUS:
         ret = SCAN_TYPE_INTERVAL;
         break;
-    case TABLE_TYPE_IP:
     case TABLE_TYPE_IP_PLUS:
-    case TABLE_TYPE_COMPOSITION:
         ret = SCAN_TYPE_IP;
         break;
     case TABLE_TYPE_PLUGIN:
@@ -84,6 +78,7 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type)
     default:
         break;
     }
+
     return ret;
 }
 
@@ -299,14 +294,6 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
         maat_instance->logger = log_handle_create(log_path, opts->log_level);
     }
 
-    maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags,
-                                                  maat_instance->garbage_bin, maat_instance->logger);
-    if (NULL == maat_instance->tbl_mgr) {
-        goto failed;
-    }
-
-    maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
-    maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
     maat_instance->input_mode = opts->input_mode;
 
     switch (maat_instance->input_mode) {
@@ -343,6 +330,15 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)
                                 (maat_instance->gc_timeout_ms / 1000);
     maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s);
 
+    maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags,
+                                                  maat_instance->garbage_bin, maat_instance->logger);
+    if (NULL == maat_instance->tbl_mgr) {
+        goto failed;
+    }
+
+    maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr);
+    maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr);
+
     maat_instance->outer_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
     maat_instance->compile_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
     maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads);
@@ -698,11 +694,97 @@ int hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile
     }
 }
 
-int maat_scan_flag(struct maat *instance, int table_id, int thread_id,
+int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id,
                    uint64_t flag, int *results, size_t n_result,
                    size_t *n_hit_result, struct maat_state **state)
 {
-    return 0;
+     if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) 
+        || (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) {
+        return MAAT_SCAN_ERR;
+    }
+    
+    struct maat_state *mid = NULL;
+    mid = grab_state(state, maat_instance, thread_id);
+    mid->scan_cnt++;
+
+    int physical_table_id = 0;
+    int vtable_id = 0;
+    
+    enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
+    if (table_type == TABLE_TYPE_VIRTUAL) {
+        //find physical table id
+        void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
+        physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_FLAG);
+        if (physical_table_id < 0) {
+            return MAAT_SCAN_ERR;
+        }
+    } else {
+        physical_table_id = table_id;
+    }
+
+    if (NULL == maat_instance->maat_rt) {
+        log_error(maat_instance->logger, MODULE_MAAT_API, 
+                 "maat_scan_string error because of maat_runtime is NULL");
+        return MAAT_SCAN_OK;
+    }
+
+    maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
+    alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1);
+
+    int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
+    void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id);
+    if (NULL == flag_rt) {
+        return MAAT_SCAN_ERR;
+    }
+
+    int group_hit_cnt = flag_runtime_scan_flag((struct flag_runtime *)flag_rt, 
+                                                 thread_id,
+                                                 flag, group_ids, 
+                                                 MAX_SCANNER_HIT_GROUP_NUM,
+                                                 vtable_id, mid);
+    if (group_hit_cnt < 0) {
+        return MAAT_SCAN_ERR;
+    } 
+    
+    int compile_ret = 0;
+    if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
+        // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
+        if (group_hit_cnt > 0) {
+            flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id);
+        }
+
+        int compile_table_id = -1;
+        if (mid->compile_table_id == -1) {
+            compile_table_id = maat_instance->default_compile_table_id;
+        } else {
+            compile_table_id = mid->compile_table_id;
+        }
+
+        void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
+        compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid);
+
+        assert(mid->is_last_scan < LAST_SCAN_FINISHED);
+        if (LAST_SCAN_SET == mid->is_last_scan) {
+            mid->is_last_scan = LAST_SCAN_FINISHED;
+        }
+    }
+
+    if (compile_ret > 0) {
+        alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1);
+    }
+
+    if (0 == group_hit_cnt && compile_ret > 0) {
+        // hit NOT group
+        alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1);
+    }
+
+    maat_runtime_ref_dec(maat_instance->maat_rt, thread_id);
+
+    if (0 == compile_ret && group_hit_cnt > 0) {
+        return MAAT_SCAN_HALF_HIT;
+    }
+
+    return MAAT_SCAN_HIT;
 }
 
 int maat_scan_integer(struct maat *instance, int table_id, int thread_id,
@@ -894,11 +976,14 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
     mid = grab_state(state, maat_instance, thread_id);
     mid->scan_cnt++;
 
-    int physical_table_id = -1;
+    int physical_table_id = 0;
+    int vtable_id = 0;
+    
     enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
     if (table_type == TABLE_TYPE_VIRTUAL) {
         //find physical table id
-        physical_table_id = virtual_table_get_physical_table_id(table_id, SCAN_TYPE_STRING);
+        void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id);
+        physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_STRING);
         if (physical_table_id < 0) {
             return MAAT_SCAN_ERR;
         }
@@ -914,8 +999,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
 
     maat_runtime_ref_inc(maat_instance->maat_rt, thread_id);
 
-    //TODO: is TABLE_TYPE_EXPR_PLUS
+    table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
-    enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id);
     if ((table_type == TABLE_TYPE_EXPR_PLUS) && 
         (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) {
         maat_instance->scan_err_cnt++;
@@ -930,7 +1014,8 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
         return MAAT_SCAN_ERR;
     }
 
-    int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, thread_id,
+    int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, 
+                                                 thread_id,
                                                  data, data_len, group_ids, 
                                                  MAX_SCANNER_HIT_GROUP_NUM,
                                                  vtable_id, mid);
@@ -939,17 +1024,12 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id,
     } 
     
     int compile_ret = 0;
-    int district_id = DISTRICT_ANY;
     if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) {
         // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT
         if (group_hit_cnt > 0) {
             expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id);
         }
 
-        if (group_hit_cnt > 0 && table_type == TABLE_TYPE_EXPR_PLUS) {
-            district_id = mid->district_id;
-        }
-
         int compile_table_id = -1;
         if (mid->compile_table_id == -1) {
             compile_table_id = maat_instance->default_compile_table_id;

src/maat_compile.cpp

@@ -992,7 +992,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash,
 				}
 
 				//TODO:mytest need to delete
-				#if 0
+				#if 1
 				struct maat_literal_id *p = NULL;
 				for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) {
     				printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n", 
@@ -1067,16 +1067,16 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan
 	size_t ud_result_cnt = 0;
 
 	//TODO:mytest need to delete
-	#if 0
+	#if 1
 	unsigned long long *p;
-	for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clause_array); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clause_array, p))
+    printf("utarray_len:%u\n", utarray_len(compile_state->all_hit_clauses));
+	for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clauses); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clauses, p))
 	{
-		printf(" before bool_matcher_match compile_state clause_id:%llu\n",  *p);
+		printf("before bool_matcher_match compile_state clause_id:%llu\n",  *p);
 	}
 	#endif
 	
-	int bool_match_ret = bool_matcher_match(bm, 
+	int bool_match_ret = bool_matcher_match(bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
-                                            (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
 											utarray_len(compile_state->all_hit_clauses), 
                                             expr_match, MAX_SCANNER_HIT_COMPILE_NUM);
 	for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) {
@@ -1197,8 +1197,8 @@ struct maat_compile_state *maat_compile_state_new(int thread_id)
 	compile_state->thread_id = thread_id;
 	//compile_state->hier_ver = hier->version;
 	utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd);
-	utarray_new(compile_state->all_hit_clause_array, &ut_clause_id_icd);
+	utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd);
-	utarray_new(compile_state->this_scan_hit_clause_ids, &ut_clause_id_icd);
+	utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd);
 
 	return compile_state;
 }
@@ -1206,8 +1206,8 @@ struct maat_compile_state *maat_compile_state_new(int thread_id)
 void maat_compile_state_free(struct maat_compile_state *compile_state)
 {
     utarray_free(compile_state->internal_hit_paths);
-	utarray_free(compile_state->all_hit_clause_array);
+	utarray_free(compile_state->all_hit_clauses);
-	utarray_free(compile_state->this_scan_hit_clause_ids);
+	utarray_free(compile_state->this_scan_hit_clauses);
 	free(compile_state);
 }
 
@@ -1316,8 +1316,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
     struct maat_hit_path tmp_path;
     struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM;
     int bool_match_ret = bool_matcher_match(compile_rt->bm, 
-                                            (unsigned long long *)utarray_eltptr(compile_state->all_hit_clause_array, 0),
+                                            (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0),
-											utarray_len(compile_state->all_hit_clause_array), expr_match, 
+											utarray_len(compile_state->all_hit_clauses), expr_match, 
                                             MAX_SCANNER_HIT_COMPILE_NUM);
 	for (int idx = 0; idx < bool_match_ret; idx++) {
         compile = (struct maat_compile *)expr_match[idx].user_tag;
@@ -1387,7 +1387,6 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
     struct maat_literal_id *tmp = NULL;
 
     unsigned long long *clause_id = 0;
-    size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
 
 	HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) {
 		for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
@@ -1396,6 +1395,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
 				continue;
 			}
 
+            size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
 			tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, 
                                                          &literal_id, compare_literal_id);
 			if (tmp) {
@@ -1404,6 +1404,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
                                 compare_clause_id)) {
 					continue;
 				}
+
                 utarray_push_back(compile_state->this_scan_hit_clauses, &(clause_state->clause_id));
 			}
 
@@ -1779,6 +1780,11 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
         int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
         int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i], 
                                                                1, top_group_ids);
+        if (0 == top_group_cnt) {
+            maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
+                                                 group_ids[i], vtable_id);
+        }
+        
         for (int j = 0; j < top_group_cnt; j++) {
             maat_compile_state_update_hit_clause(state->compile_state, compile_rt, 
                                                  top_group_ids[j], vtable_id);

src/maat_expr.cpp

@@ -23,8 +23,6 @@
 #include "alignment.h"
 #include "maat_garbage_collection.h"
 
-#define MAX_DISTRICT_STR     128
-
 #define MODULE_EXPR	 module_name_str("maat.expr")
 
 struct expr_schema {
@@ -161,7 +159,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem
     }
     expr_item->group_id = atoi(line + column_offset);
 
-    table_type = table_manager_get_table_type(expr_schema->tbl_mgr, expr_schema->table_id);
+    table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id);
     if (table_type == TABLE_TYPE_EXPR_PLUS) {
         ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len);
         if (ret < 0) {
@@ -339,7 +337,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         read_cnt++;
     }
 
-    expr_schema->tbl_mgr = tbl_mgr;
+    expr_schema->ref_tbl_mgr = tbl_mgr;
 
     if (read_cnt < 8) {
 		goto error;
@@ -443,7 +441,7 @@ void expr_runtime_free(void *expr_runtime)
 }
 
 int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, 
-                            size_t key_len, and_expr_t *expr_rule, 
+                            size_t key_len, int item_id, and_expr_t *expr_rule, 
                             int is_valid, struct log_handle *logger)
 {
     void *data = NULL;
@@ -454,7 +452,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
         if (NULL == data) {
             log_error(logger, MODULE_EXPR, 
                       "the key of expr rule not exist, can't be deleted, expr_id:%d",
-                      expr_rule->expr_id);
+                      item_id);
             return -1;
         }
         rcu_hash_del(expr_rt->htable, key, key_len);
@@ -464,7 +462,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
         if (data != NULL) {
             log_error(logger, MODULE_EXPR,
                       "the key of expr rule already exist, can't be added, expr_id:%d", 
-                      expr_rule->expr_id);
+                      item_id);
             return -1;
         }
         rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule);
@@ -492,7 +490,7 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type)
 }
 
 #define MAAT_MAX_EXPR_ITEM_NUM	8
-and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, 
+and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data,
                                    struct log_handle *logger)
 {
     size_t i = 0;
@@ -543,9 +541,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item,
         expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]);
         expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type);
     }
-    int *district_tag = ALLOC(int, 1);
+   
-    *district_tag = expr_item->district_id;
+    expr_rule->user_tag = user_data;
-    expr_rule->user_tag = district_tag;
     expr_rule->n_patterns = sub_expr_cnt;
     
     return expr_rule;
@@ -554,7 +551,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item,
 int expr_runtime_update(void *expr_runtime, void *expr_schema, 
                         const char *line, int valid_column)
 {
-    if (NULL == expr_runtime || NULL == expr_schema) {
+    if (NULL == expr_runtime || NULL == expr_schema ||
+        NULL == line) {
         return -1;
     }
 
@@ -604,7 +602,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
         item = maat_item_new(item_id, expr_item->group_id, u_para);
         HASH_ADD_INT(expr_rt->item_hash, item_id, item);
 
-        expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger);
+        expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger);
         expr_item_free(expr_item);
         if (NULL == expr_rule) {
             log_error(expr_rt->logger, MODULE_EXPR, 
@@ -615,8 +613,8 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
     }
 
     char *key = (char *)&item_id;
-    ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, 
+    ret = expr_runtime_update_row(expr_rt, key, sizeof(int), item_id, 
-                                  is_valid, expr_rt->logger);
+                                  expr_rule, is_valid, expr_rt->logger);
     if (ret < 0) {
         if (expr_rule != NULL) {
             expr_rule_free(expr_rule);
@@ -653,8 +651,9 @@ int expr_runtime_commit(void *expr_runtime)
 
     for (size_t i = 0; i < rule_cnt; i++) {
         rules[i] = *(and_expr_t *)ex_data_array[i];
+        printf("rule_id:%d\n", rules[i].expr_id);
     }
-
+    printf("\n\n");
     struct adapter_hs *new_adapter_hs = NULL;
     struct adapter_hs *old_adapter_hs = NULL;
 
@@ -689,14 +688,14 @@ int expr_runtime_commit(void *expr_runtime)
 
 int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, 
                              const char *data, size_t data_len,
-                            int *group_ids, size_t group_ids_size, 
+                             int *group_ids, size_t group_ids_size, 
-                            int vtable_id, struct maat_state *state)
+                             int vtable_id, struct maat_state *state)
 {
     size_t n_hit_item = 0;
-    struct hs_scan_results hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
+    struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
     
     int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, 
-                              hit_item_ids, MAX_SCANNER_HIT_ITEM_NUM, 
+                              hit_results, MAX_SCANNER_HIT_ITEM_NUM, 
                               &n_hit_item);
     if (ret < 0) {
         return -1;
@@ -713,9 +712,20 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id,
         n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
     }
     
+    int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
+    struct maat_item_inner *item = NULL;
+    int real_hit_item_cnt = 0;
+    int district_id = state->district_id;
+    for (size_t i = 0; i < n_hit_item; i++) {
+        item = (struct maat_item_inner *)(hit_results[i].user_tag);
+        if (item->district_id == district_id || district_id == DISTRICT_ANY) {
+            hit_item_ids[real_hit_item_cnt++] = hit_results[i].item_id;
+        }
+    }
+
     size_t group_hit_cnt = 0;
     ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, 
-                                    n_hit_item, group_ids, group_ids_size, 
+                                    real_hit_item_cnt, group_ids, group_ids_size, 
                                     &group_hit_cnt, state);
     if (ret < 0) {
         return -1;
@@ -735,14 +745,47 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
 }
 
 int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, 
-                             size_t data_len, int result[], size_t *n_result)
+                             size_t data_len, int *group_ids, size_t group_ids_size, 
+                             int vtable_id, struct maat_state *state)
 {
     if (NULL == expr_rt) {
         return -1;
     }
 
-    return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, 
+    size_t n_hit_item = 0;
-                                  result, n_result);
+    struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
+    int ret = adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, 
+                                     hit_results, MAX_SCANNER_HIT_ITEM_NUM, 
+                                     &n_hit_item);
+    if (ret < 0) {
+        return -1;
+    }
+
+    if (0 == n_hit_item) {
+        return 0;
+    }
+
+    if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
+        log_info(expr_rt->logger, MODULE_EXPR, 
+                 "hit expr item count:%d exceed maxium:%d", 
+                 n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
+        n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
+    }
+    
+    int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
+    for (size_t i = 0; i < n_hit_item; i++) {
+        hit_item_ids[i] = hit_results[i].item_id;
+    }
+
+    size_t group_hit_cnt = 0;
+    ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, 
+                                    n_hit_item, group_ids, group_ids_size, 
+                                    &group_hit_cnt, state);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return group_hit_cnt;
 }
 
 void expr_runtime_stream_close(struct expr_runtime *expr_rt)

src/maat_flag.cpp

@@ -8,64 +8,449 @@
 ***********************************************************************************************
 */
 
+#include <assert.h>
+
 #include "maat_flag.h"
+#include "flag_matcher.h"
+#include "maat_utils.h"
+#include "maat_rule.h"
+#include "log/log.h"
+#include "uthash/uthash.h"
+#include "rcu_hash.h"
+#include "maat_table.h"
+#include "alignment.h"
+#include "maat_compile.h"
+#include "maat_garbage_collection.h"
+
+#define MODULE_FLAG	 module_name_str("maat.flag")
 
 struct flag_schema {
+    int item_id_column;
+    int group_id_column;
+    int flag_column;
+    int flag_mask_column;
+    int table_id;
+    struct table_manager *ref_tbl_mgr;
+};
 
+struct flag_item {
+    int item_id;
+    int group_id;
+    uint64_t flag;
+    uint64_t flag_mask;
 };
 
 struct flag_runtime {
+    struct flag_matcher *matcher;
+    struct rcu_hash_table *htable;
 
+    uint32_t rule_num;
+    uint32_t updating_rule_num;
+    struct maat_item *item_hash;
+    void (*item_user_data_free)(void *);
+
+    struct maat_garbage_bin *ref_garbage_bin;
+    struct log_handle *logger;
+
+    long long *scan_cnt;
+    long long *hit_cnt;
 };
 
 void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
                       const char *table_name, struct log_handle *logger)
 {
+    int read_cnt = 0;
+    struct flag_schema *schema = ALLOC(struct flag_schema, 1);
 
+    cJSON *custom_item = NULL;
+    cJSON *item = cJSON_GetObjectItem(json, "table_id");
+    if (item != NULL && item->type == cJSON_Number) {
+        schema->table_id = item->valueint;
+        read_cnt++;
+    }
+
+    item = cJSON_GetObjectItem(json, "custom");
+    if (item == NULL || item->type != cJSON_Object) {
+        log_error(logger, MODULE_FLAG, 
+                  "table %s has no custom column", table_name);
+        goto error;
+    }
+
+    custom_item = cJSON_GetObjectItem(item, "item_id");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->item_id_column = custom_item->valueint;
+        read_cnt++;
+    }
+
+    custom_item = cJSON_GetObjectItem(item, "group_id");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->group_id_column = custom_item->valueint;
+        read_cnt++;
+    }
+
+    custom_item = cJSON_GetObjectItem(item, "flag");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->flag_column = custom_item->valueint;
+        read_cnt++;
+    }
+
+    custom_item = cJSON_GetObjectItem(item, "flag_mask");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->flag_mask_column = custom_item->valueint;
+        read_cnt++;
+    }
+
+    schema->ref_tbl_mgr = tbl_mgr;
+
+    if (read_cnt < 5) {
+        goto error;
+    }
+
+    return schema;
+error:
+    FREE(schema);
+    return NULL;
 }
 
 void flag_schema_free(void *flag_schema)
 {
+    FREE(flag_schema);
+}
 
+void flag_ex_data_free(void *user_ctx, void *data)
+{
+    struct flag_item *flag_item = (struct flag_item *)data;
+    FREE(flag_item);
 }
 
 void *flag_runtime_new(void *flag_schema, int max_thread_num, 
                        struct maat_garbage_bin *garbage_bin,
                        struct log_handle *logger)
 {
+    if (NULL == flag_schema) {
+        return NULL;
+    }
 
+    struct flag_runtime *flag_rt = ALLOC(struct flag_runtime, 1);
+
+    flag_rt->htable = rcu_hash_new(flag_ex_data_free);
+    flag_rt->item_user_data_free = maat_item_inner_free;
+    flag_rt->ref_garbage_bin = garbage_bin;
+    flag_rt->logger = logger;
+
+    flag_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
+    flag_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
+
+    return flag_rt;
 }
 
 void flag_runtime_free(void *flag_runtime)
 {
+    if (NULL == flag_runtime) {
+        return;
+    }
 
+    struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
+    if (flag_rt->htable != NULL) {
+        rcu_hash_free(flag_rt->htable);
+        flag_rt->htable = NULL;
+    }
+
+    struct maat_item *item = NULL, *tmp = NULL;
+    HASH_ITER(hh, flag_rt->item_hash, item, tmp) {
+        HASH_DELETE(hh, flag_rt->item_hash, item);
+        maat_item_free(item, flag_rt->item_user_data_free);
+    }
+
+    if (flag_rt->hit_cnt != NULL) {
+        alignment_int64_array_free(flag_rt->hit_cnt);
+        flag_rt->hit_cnt = NULL;
+    }
+
+    if (flag_rt->scan_cnt != NULL) {
+        alignment_int64_array_free(flag_rt->scan_cnt);
+        flag_rt->scan_cnt = NULL;
+    }
+
+    FREE(flag_rt);
+}
+
+int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len,
+                            int item_id, struct flag_rule *rule, int is_valid)
+{
+    void *data = NULL;
+
+    if (0 == is_valid) {
+        //delete
+        data = rcu_hash_find(flag_rt->htable, key, key_len);
+        if (NULL == data) {
+            log_error(flag_rt->logger, MODULE_FLAG, 
+                      "the key of flag rule not exist, can't be deleted, item_id:%d",
+                      item_id);
+            return -1;
+        }
+        rcu_hash_del(flag_rt->htable, key, key_len);
+    } else {
+        //add
+        data = rcu_hash_find(flag_rt->htable, key, key_len);
+        if (data != NULL) {
+            log_error(flag_rt->logger, MODULE_FLAG,
+                      "the key of flag rule already exist, can't be added, item_id:%d", 
+                      item_id);
+            return -1;
+        }
+        rcu_hash_add(flag_rt->htable, key, key_len, (void *)rule);
+    }
+    
+    return 0;
+}
+
+struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schema,
+                                struct log_handle *logger)
+{
+    size_t column_offset = 0;
+    size_t column_len = 0;
+    struct flag_item *flag_item = ALLOC(struct flag_item, 1);
+
+    int ret = get_column_pos(line, flag_schema->item_id_column, &column_offset, &column_len);
+    if (ret < 0) {
+        log_error(logger, MODULE_FLAG, 
+                  "flag table(table_id:%d) line:%s has no item_id",
+                  flag_schema->table_id, line);
+        goto error;
+    }
+    flag_item->item_id = atoi(line + column_offset);
+
+    ret = get_column_pos(line, flag_schema->group_id_column, &column_offset, &column_len);
+    if (ret < 0) {
+        log_error(logger, MODULE_FLAG, 
+                  "flag table(table_id:%d) line:%s has no group_id",
+                  flag_schema->table_id, line);
+        goto error;
+    }
+    flag_item->group_id = atoi(line + column_offset);
+
+    ret = get_column_pos(line, flag_schema->flag_column, &column_offset, &column_len);
+    if (ret < 0) {
+        log_error(logger, MODULE_FLAG, 
+                  "flag table(table_id:%d) line:%s has no flag",
+                  flag_schema->table_id, line);
+        goto error;
+    }
+
+    flag_item->flag = strtoull(line + column_offset, NULL, 0);
+
+     ret = get_column_pos(line, flag_schema->flag_mask_column, &column_offset, &column_len);
+    if (ret < 0) {
+        log_error(logger, MODULE_FLAG, 
+                  "flag table(table_id:%d) line:%s has no flag_mask",
+                  flag_schema->table_id, line);
+        goto error;
+    }
+    flag_item->flag_mask = strtoull(line + column_offset, NULL, 0);
+
+    return flag_item;
+error:
+    FREE(flag_item);
+    return NULL;
+}
+
+void flag_item_free(struct flag_item *item)
+{
+    FREE(item);
+}
+
+struct flag_rule *flag_item_to_flag_rule(struct flag_item *item, void *user_data)
+{
+    struct flag_rule *rule = ALLOC(struct flag_rule, 1);
+
+    rule->rule_id = item->item_id;
+    rule->flag = item->flag;
+    rule->mask = item->flag_mask;
+    rule->user_tag = user_data;
+
+    return rule;
+}
+
+void flag_rule_free(struct flag_rule *rule)
+{
+    FREE(rule);
 }
 
 int flag_runtime_update(void *flag_runtime, void *flag_schema, 
                         const char *line, int valid_column)
 {
+    if (NULL == flag_runtime || NULL == flag_schema ||
+        NULL == line) {
+        return -1;
+    }
 
+    struct maat_item_inner *u_para = NULL;
+    struct maat_item *item = NULL;
+    struct flag_rule *flag_rule = NULL;
+    struct flag_schema *schema = (struct flag_schema *)flag_schema;
+    struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
+
+    int item_id = get_column_value(line, schema->item_id_column);
+    int is_valid = get_column_value(line, valid_column);
+    if (is_valid < 0) {
+        return -1;
+    } else if (0 == is_valid) {
+        //delete
+        HASH_FIND_INT(flag_rt->item_hash, &item_id, item);
+        if (NULL == item) {
+            return -1;
+        }
+
+        u_para = (struct maat_item_inner *)item->user_data;
+        item->user_data = NULL;
+        
+	    if (NULL == u_para) {
+		    return -1;
+	    }
+
+        HASH_DELETE(hh, flag_rt->item_hash, item);
+        maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free);
+    } else {
+        //add
+        HASH_FIND_INT(flag_rt->item_hash, &item_id, item);
+        if (item) {
+            log_error(flag_rt->logger, MODULE_FLAG, 
+                      "flag runtime add item %d to item_hash failed, already exist", 
+                      item_id);
+            return -1;
+        }
+
+        struct flag_item *flag_item = flag_item_new(line, schema, flag_rt->logger);
+        if (NULL == flag_item) {
+            return -1;
+        }
+
+        u_para = maat_item_inner_new(flag_item->group_id, item_id, 0);
+        item = maat_item_new(item_id, flag_item->group_id, u_para);
+        HASH_ADD_INT(flag_rt->item_hash, item_id, item);
+
+        flag_rule = flag_item_to_flag_rule(flag_item, u_para);
+        flag_item_free(flag_item);
+        if (NULL == flag_rule) {
+            log_error(flag_rt->logger, MODULE_FLAG, 
+                      "transform flag table(table_id:%d) item to flag_rule failed, item_id:%d",
+                      schema->table_id, item_id);
+            return -1;
+        }
+    }
+
+    char *key = (char *)&item_id;
+    int ret = flag_runtime_update_row(flag_rt, key, sizeof(int), item_id, flag_rule, is_valid);
+    if (ret < 0) {
+        if (flag_rule != NULL) {
+            flag_rule_free(flag_rule);
+            flag_rule = NULL;
+        }
+        return -1;
+    } else {
+        if (0 == is_valid) {
+            flag_rt->rule_num--;
+        } else {
+            flag_rt->rule_num++;
+        }
+    }
+
+    return 0;
 }
 
 int flag_runtime_commit(void *flag_runtime)
 {
+    if (NULL == flag_runtime) {
+        return -1;
+    }
 
+    int ret = 0;
+    struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
+    void **ex_data_array = NULL;
+
+    size_t rule_cnt = rcu_hash_list_updating_data(flag_rt->htable, &ex_data_array);
+    if (0 == rule_cnt) {
+        FREE(ex_data_array);
+        return 0;
+    }
+
+    struct flag_rule *rules = ALLOC(struct flag_rule, rule_cnt);
+
+    for (size_t i = 0; i < rule_cnt; i++) {
+        rules[i] = *(struct flag_rule *)ex_data_array[i];
+    }
+
+    struct flag_matcher *new_flag_matcher = NULL;
+    struct flag_matcher *old_flag_matcher = NULL;
+
+    log_info(flag_rt->logger, MODULE_FLAG,
+             "committing %zu flag rules for rebuilding flag_matcher engine", 
+             rule_cnt);
+    new_flag_matcher = flag_matcher_new(rules, rule_cnt);
+    if (NULL == new_flag_matcher) {
+        log_error(flag_rt->logger, MODULE_FLAG,
+                  "rebuild flag_matcher engine failed when update %zu flag rules", 
+                  rule_cnt);
+        ret = -1;
+    }
+
+    old_flag_matcher = flag_rt->matcher;
+    flag_rt->matcher = new_flag_matcher;
+
+    maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher, 
+                         (void (*)(void*))flag_matcher_free);
+    rcu_hash_commit(flag_rt->htable);
+    rule_cnt = rcu_hash_updating_count(flag_rt->htable);
+    assert(rule_cnt == 0);
+    
+    FREE(rules);
+    FREE(ex_data_array);
+
+    return ret;
 }
 
 int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, 
-                             const char *data, size_t data_len, 
+                           uint64_t flag, int *group_ids, size_t group_ids_size, 
-                             int *group_ids, size_t group_ids_size, 
+                           int vtable_id, struct maat_state *state)
-                             int vtable_id, struct maat_state *state)
 {
+    struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
     
+    int n_hit_item = flag_matcher_match(flag_rt->matcher, flag, 
+                                        hit_results, MAX_SCANNER_HIT_ITEM_NUM);
+    if (n_hit_item <= 0) {
+        return n_hit_item;
+    }
+
+    if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
+        log_info(flag_rt->logger, MODULE_FLAG, 
+                 "hit flag item count:%d exceed maxium:%d", 
+                 n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
+        n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
+    }
+    
+    int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
+    for (int i = 0; i < n_hit_item; i++) {
+        hit_item_ids[i] = hit_results[i].rule_id;
+    }
+
+    size_t group_hit_cnt = 0;
+    int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids, 
+                                        n_hit_item, group_ids, group_ids_size, 
+                                        &group_hit_cnt, state);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return group_hit_cnt;
 }
 
 void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id)
 {
-
+    alignment_int64_array_add(flag_rt->hit_cnt, thread_id, 1);
 }
 
 long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread)
 {
-
+    return alignment_int64_array_sum(flag_rt->hit_cnt, n_thread);
 }

src/maat_interval.cpp

@@ -0,0 +1,486 @@
+/*
+**********************************************************************************************
+*	File: maat_interval.cpp
+*   Description: 
+*	Authors: Liu WenTan <liuwentan@geedgenetworks.com>
+*	Date:    2022-10-31
+*   Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
+***********************************************************************************************
+*/
+
+#include <assert.h>
+
+#include "maat_utils.h"
+#include "log/log.h"
+#include "maat_interval.h"
+#include "maat_rule.h"
+#include "rcu_hash.h"
+#include "alignment.h"
+#include "uthash/uthash.h"
+#include "maat_garbage_collection.h"
+#include "maat_compile.h"
+#include "interval_matcher.h"
+
+#define MODULE_INTERVAL	 module_name_str("maat.interval")
+
+struct interval_schema {
+    int item_id_column;
+    int group_id_column;
+    int district_column;
+    int low_bound_column;
+    int up_bound_column;
+    int table_id;
+    struct table_manager *ref_tbl_mgr;
+};
+
+struct interval_item {
+    int item_id;
+    int group_id;
+    int district_id;
+    int low_bound;
+    int up_bound;
+};
+
+struct interval_runtime {
+    struct interval_matcher *matcher;
+    struct rcu_hash_table *htable;
+
+    uint32_t rule_num;
+    uint32_t updating_rule_num;
+    struct maat_item *item_hash;
+    void (*item_user_data_free)(void *);
+
+    struct maat_garbage_bin *ref_garbage_bin;
+    struct log_handle *logger;
+
+    long long *scan_cnt;
+    long long *hit_cnt;
+};
+
+void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
+                          const char *table_name, struct log_handle *logger)
+{
+    int read_cnt = 0;
+    struct interval_schema *schema = ALLOC(struct interval_schema, 1);
+
+    cJSON *custom_item = NULL;
+    cJSON *item = cJSON_GetObjectItem(json, "table_id");
+    if (item != NULL && item->type == cJSON_Number) {
+        schema->table_id = item->valueint;
+        read_cnt++;
+    }
+
+    item = cJSON_GetObjectItem(json, "custom");
+    if (item == NULL || item->type != cJSON_Object) {
+        log_error(logger, MODULE_INTERVAL, 
+                  "table %s has no custom column", table_name);
+        goto error;
+    }
+
+    custom_item = cJSON_GetObjectItem(item, "item_id");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->item_id_column = custom_item->valueint;
+        read_cnt++;
+    }
+
+    custom_item = cJSON_GetObjectItem(item, "group_id");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->group_id_column = custom_item->valueint;
+        read_cnt++;
+    }
+
+    custom_item = cJSON_GetObjectItem(item, "district");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->district_column = custom_item->valueint;
+        read_cnt++;
+    }
+
+    custom_item = cJSON_GetObjectItem(item, "low_bound");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->low_bound_column = custom_item->valueint;
+        read_cnt++;
+    }
+
+    custom_item = cJSON_GetObjectItem(item, "up_bound");
+    if (custom_item != NULL && custom_item->type == cJSON_Number) {
+        schema->up_bound_column = custom_item->valueint;
+        read_cnt++;
+    }
+
+    schema->ref_tbl_mgr = tbl_mgr;
+
+    if (read_cnt < 5) {
+        goto error;
+    }
+
+    return schema;
+error:
+    FREE(schema);
+    return NULL;
+}
+
+void interval_schema_free(void *interval_schema)
+{
+    FREE(interval_schema);
+}
+
+void interval_ex_data_free(void *user_ctx, void *data)
+{
+    struct interval_item *item = (struct interval_item *)data;
+    FREE(item);
+}
+
+void *interval_runtime_new(void *interval_schema, int max_thread_num, 
+                           struct maat_garbage_bin *garbage_bin,
+                           struct log_handle *logger)
+{
+    if (NULL == interval_schema) {
+        return NULL;
+    }
+
+    struct interval_runtime *interval_rt = ALLOC(struct interval_runtime, 1);
+
+    interval_rt->htable = rcu_hash_new(interval_ex_data_free);
+    interval_rt->item_user_data_free = maat_item_inner_free;
+    interval_rt->ref_garbage_bin = garbage_bin;
+    interval_rt->logger = logger;
+
+    interval_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num);
+    interval_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num);
+
+    return interval_rt;
+}
+
+void interval_runtime_free(void *interval_runtime)
+{
+    if (NULL == interval_runtime) {
+        return;
+    }
+
+    struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
+    if (interval_rt->htable != NULL) {
+        rcu_hash_free(interval_rt->htable);
+        interval_rt->htable = NULL;
+    }
+
+    struct maat_item *item = NULL, *tmp = NULL;
+    HASH_ITER(hh, interval_rt->item_hash, item, tmp) {
+        HASH_DELETE(hh, interval_rt->item_hash, item);
+        maat_item_free(item, interval_rt->item_user_data_free);
+    }
+
+    if (interval_rt->hit_cnt != NULL) {
+        alignment_int64_array_free(interval_rt->hit_cnt);
+        interval_rt->hit_cnt = NULL;
+    }
+
+    if (interval_rt->scan_cnt != NULL) {
+        alignment_int64_array_free(interval_rt->scan_cnt);
+        interval_rt->scan_cnt = NULL;
+    }
+
+    FREE(interval_rt);
+}
+
+struct interval_item *interval_item_new(const char *line, struct interval_schema *schema,
+                                        struct log_handle *logger)
+{
+    size_t column_offset = 0;
+    size_t column_len = 0;
+    enum table_type table_type = TABLE_TYPE_INVALID;
+    struct interval_item *item = ALLOC(struct interval_item, 1);
+
+    int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
+    if (ret < 0) {
+        log_error(logger, MODULE_INTERVAL, 
+                  "interval table(table_id:%d) line:%s has no item_id",
+                  schema->table_id, line);
+        goto error;
+    }
+    item->item_id = atoi(line + column_offset);
+
+    ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len);
+    if (ret < 0) {
+        log_error(logger, MODULE_INTERVAL, 
+                  "interval table(table_id:%d) line:%s has no group_id",
+                  schema->table_id, line);
+        goto error;
+    }
+    item->group_id = atoi(line + column_offset);
+
+    table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id);
+    if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
+        ret = get_column_pos(line, schema->district_column, &column_offset, &column_len);
+        if (ret < 0) {
+            goto error;
+        }
+
+        if (column_len >= MAX_DISTRICT_STR) {
+            log_error(logger, MODULE_INTERVAL, 
+                      "interval table(table_id:%d) line:%s district length too long",
+                      schema->table_id, line);
+            goto error;
+        }
+
+        char district[MAX_DISTRICT_STR] = {0};
+        memcpy(district, (line + column_offset), column_len);
+        assert(strlen(district) > 0);
+        str_unescape(district);
+        item->district_id = table_manager_get_district_id(schema->ref_tbl_mgr, district);
+    }
+
+    ret = get_column_pos(line, schema->low_bound_column, &column_offset, &column_len);
+    if (ret < 0) {
+        log_error(logger, MODULE_INTERVAL, 
+                  "interval table(table_id:%d) line:%s has no low_bound",
+                  schema->table_id, line);
+        goto error;
+    }
+    item->low_bound = atoi(line + column_offset);
+
+    ret = get_column_pos(line, schema->up_bound_column, &column_offset, &column_len);
+    if (ret < 0) {
+        log_error(logger, MODULE_INTERVAL, 
+                  "interval table(table_id:%d) line:%s has no up_bound",
+                  schema->table_id, line);
+        goto error;
+    }
+    item->up_bound = atoi(line + column_offset);
+
+    return item;
+error:
+    FREE(item);
+    return NULL;
+}
+
+void interval_item_free(void *interval_item)
+{
+    FREE(interval_item);
+}
+
+struct interval_rule *interval_item_to_interval_rule(struct interval_item *item, void *user_data)
+{
+    struct interval_rule *rule = ALLOC(struct interval_rule, 1);
+
+    rule->start = item->low_bound;
+    rule->end = item->up_bound;
+    rule->result.rule_id = item->item_id;
+    rule->result.user_tag = user_data;
+
+    return rule;
+}
+
+void interval_rule_free(struct interval_rule *rule)
+{
+    FREE(rule);
+}
+
+int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, size_t key_len,
+                                int item_id, struct interval_rule *rule, int is_valid)
+{
+    void *data = NULL;
+
+    if (0 == is_valid) {
+        //delete
+        data = rcu_hash_find(interval_rt->htable, key, key_len);
+        if (NULL == data) {
+            log_error(interval_rt->logger, MODULE_INTERVAL, 
+                      "the key of interval rule not exist, can't be deleted, item_id:%d",
+                      item_id);
+            return -1;
+        }
+        rcu_hash_del(interval_rt->htable, key, key_len);
+    } else {
+        //add
+        data = rcu_hash_find(interval_rt->htable, key, key_len);
+        if (data != NULL) {
+            log_error(interval_rt->logger, MODULE_INTERVAL,
+                      "the key of interval rule already exist, can't be added, item_id:%d", 
+                      item_id);
+            return -1;
+        }
+        rcu_hash_add(interval_rt->htable, key, key_len, (void *)rule);
+    }
+    
+    return 0;
+}
+
+int interval_runtime_update(void *interval_runtime, void *interval_schema, 
+                            const char *line, int valid_column)
+{
+    if (NULL == interval_runtime || NULL == interval_schema ||
+        NULL == line) {
+        return -1;
+    }
+
+    struct maat_item_inner *u_para = NULL;
+    struct maat_item *item = NULL;
+    struct interval_rule *interval_rule = NULL;
+    struct interval_schema *schema = (struct interval_schema *)interval_schema;
+    struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
+
+    int item_id = get_column_value(line, schema->item_id_column);
+    int is_valid = get_column_value(line, valid_column);
+    if (is_valid < 0) {
+        return -1;
+    } else if (0 == is_valid) {
+        //delete
+        HASH_FIND_INT(interval_rt->item_hash, &item_id, item);
+        if (NULL == item) {
+            return -1;
+        }
+
+        u_para = (struct maat_item_inner *)item->user_data;
+        item->user_data = NULL;
+        
+	    if (NULL == u_para) {
+		    return -1;
+	    }
+
+        HASH_DELETE(hh, interval_rt->item_hash, item);
+        maat_garbage_bagging(interval_rt->ref_garbage_bin, u_para, maat_item_inner_free);
+    } else {
+        //add
+        HASH_FIND_INT(interval_rt->item_hash, &item_id, item);
+        if (item) {
+            log_error(interval_rt->logger, MODULE_INTERVAL, 
+                      "interval runtime add item %d to item_hash failed, already exist", 
+                      item_id);
+            return -1;
+        }
+
+        struct interval_item *interval_item = interval_item_new(line, schema, interval_rt->logger);
+        if (NULL == interval_item) {
+            return -1;
+        }
+
+        u_para = maat_item_inner_new(interval_item->group_id, item_id, 0);
+        item = maat_item_new(item_id, interval_item->group_id, u_para);
+        HASH_ADD_INT(interval_rt->item_hash, item_id, item);
+
+        interval_rule = interval_item_to_interval_rule(interval_item, u_para);
+        interval_item_free(interval_item);
+        if (NULL == interval_rule) {
+            log_error(interval_rt->logger, MODULE_INTERVAL, 
+                      "transform interval table(table_id:%d) item to interval_rule failed, item_id:%d",
+                      schema->table_id, item_id);
+            return -1;
+        }
+    }
+
+    char *key = (char *)&item_id;
+    int ret = interval_runtime_update_row(interval_rt, key, sizeof(int), item_id, interval_rule, is_valid);
+    if (ret < 0) {
+        if (interval_rule != NULL) {
+            interval_rule_free(interval_rule);
+            interval_rule = NULL;
+        }
+        return -1;
+    } else {
+        if (0 == is_valid) {
+            interval_rt->rule_num--;
+        } else {
+            interval_rt->rule_num++;
+        }
+    }
+
+    return 0;
+}
+
+int interval_runtime_commit(void *interval_runtime)
+{
+    if (NULL == interval_runtime) {
+        return -1;
+    }
+
+    int ret = 0;
+    struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
+    void **ex_data_array = NULL;
+
+    size_t rule_cnt = rcu_hash_list_updating_data(interval_rt->htable, &ex_data_array);
+    if (0 == rule_cnt) {
+        FREE(ex_data_array);
+        return 0;
+    }
+
+    struct interval_rule *rules = ALLOC(struct interval_rule, rule_cnt);
+
+    for (size_t i = 0; i < rule_cnt; i++) {
+        rules[i] = *(struct interval_rule *)ex_data_array[i];
+    }
+
+    struct interval_matcher *new_interval_matcher = NULL;
+    struct interval_matcher *old_interval_matcher = NULL;
+
+    log_info(interval_rt->logger, MODULE_INTERVAL,
+             "committing %zu interval rules for rebuilding interval_matcher engine", 
+             rule_cnt);
+
+    new_interval_matcher = interval_matcher_new(rules, rule_cnt);
+    if (NULL == new_interval_matcher) {
+        log_error(interval_rt->logger, MODULE_INTERVAL,
+                  "rebuild interval_matcher engine failed when update %zu interval rules", 
+                  rule_cnt);
+        ret = -1;
+    }
+
+    old_interval_matcher = interval_rt->matcher;
+    interval_rt->matcher = new_interval_matcher;
+
+    maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher, 
+                         (void (*)(void*))interval_matcher_free);
+    rcu_hash_commit(interval_rt->htable);
+    rule_cnt = rcu_hash_updating_count(interval_rt->htable);
+    assert(rule_cnt == 0);
+    
+    FREE(rules);
+    FREE(ex_data_array);
+
+    return ret;
+}
+
+int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id, 
+                                 uint64_t integer, 
+                                 int *group_ids, size_t group_ids_size, 
+                                 int vtable_id, struct maat_state *state)
+{
+    struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
+    
+    int n_hit_item = interval_matcher_match(interval_rt->matcher, integer, 
+                                            hit_results, MAX_SCANNER_HIT_ITEM_NUM);
+    if (n_hit_item <= 0) {
+        return n_hit_item;
+    }
+
+    if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) {
+        log_info(interval_rt->logger, MODULE_INTERVAL, 
+                 "hit interval item count:%d exceed maxium:%d", 
+                 n_hit_item, MAX_SCANNER_HIT_ITEM_NUM);
+        n_hit_item = MAX_SCANNER_HIT_ITEM_NUM;
+    }
+    
+    int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1};
+    for (int i = 0; i < n_hit_item; i++) {
+        hit_item_ids[i] = hit_results[i].rule_id;
+    }
+
+    size_t group_hit_cnt = 0;
+    int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids, 
+                                        n_hit_item, group_ids, group_ids_size, 
+                                        &group_hit_cnt, state);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return group_hit_cnt;
+}
+
+void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id)
+{
+    alignment_int64_array_add(interval_rt->hit_cnt, thread_id, 1);
+}
+
+long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread)
+{
+    return alignment_int64_array_sum(interval_rt->hit_cnt, n_thread);
+}

src/maat_intval.cpp

@@ -1,72 +0,0 @@
-/*
-**********************************************************************************************
-*	File: maat_intval.cpp
-*   Description: 
-*	Authors: Liu WenTan <liuwentan@geedgenetworks.com>
-*	Date:    2022-10-31
-*   Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
-***********************************************************************************************
-*/
-
-#include "maat_intval.h"
-
-struct intval_schema {
-
-};
-
-struct intval_runtime {
-
-};
-
-void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
-                      const char *table_name, struct log_handle *logger)
-{
-
-}
-
-void _schema_free(void *intval_schema)
-{
-
-}
-
-/* intval runtime API */
-void *intval_runtime_new(void *intval_schema, int max_thread_num, 
-                       struct maat_garbage_bin *garbage_bin,
-                       struct log_handle *logger)
-{
-
-}
-
-void intval_runtime_free(void *intval_runtime)
-{
-
-}
-
-int intval_runtime_update(void *intval_runtime, void *intval_schema, 
-                        const char *line, int valid_column)
-{
-
-}
-
-int intval_runtime_commit(void *intval_runtime)
-{
-
-}
-
-int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id, 
-                             const char *data, size_t data_len, 
-                             int *group_ids, size_t group_ids_size, 
-                             int vtable_id, struct maat_state *state)
-{
-
-}
-
-void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id)
-{
-
-}
-
-long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread)
-{
-    
-}

src/maat_plugin.cpp

@@ -369,7 +369,8 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *line,
 int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, 
                           const char *line, int valid_column)
 {
-    if (NULL == plugin_runtime || NULL == plugin_schema) {
+    if (NULL == plugin_runtime || NULL == plugin_schema ||
+        NULL == line) {
         return -1;
     }
 

src/maat_rule.cpp

@@ -350,7 +350,7 @@ uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt)
 void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr)
 {
     size_t table_cnt = table_manager_table_count(tbl_mgr);
-    enum table_type table_type = TABLE_TYPE_MAX;
+    enum table_type table_type = TABLE_TYPE_INVALID;
 
     for (size_t i = 0; i < table_cnt; i++) {
         table_type = table_manager_get_table_type(tbl_mgr, i);

src/maat_table.cpp

@@ -21,6 +21,7 @@
 #include "maat_ip.h"
 #include "maat_compile.h"
 #include "maat_group.h"
+#include "maat_flag.h"
 #include "maat_plugin.h"
 #include "maat_ip_plugin.h"
 #include "maat_virtual.h"
@@ -75,14 +76,17 @@ struct table_operations {
 };
 
 struct table_operations table_ops[TABLE_TYPE_MAX] = {
+    // {
+    //     .type = TABLE_TYPE_FLAG,
+    //     .new_schema = flag_schema_new,
+    //     .free_schema = flag_schema_free,
+    //     .new_runtime = flag_runtime_new,
+    //     .free_runtime = flag_runtime_free,
+    //     .update_runtime = flag_runtime_update,
+    //     .commit_runtime = flag_runtime_commit
+    // },
     {
-        .type = TABLE_TYPE_FLAG,
+        .type = TABLE_TYPE_FLAG
-        .new_schema = flag_schema_new,
-        .free_schema = flag_schema_free,
-        .new_runtime = flag_runtime_new,
-        .free_runtime = flag_runtime_free,
-        .update_runtime = flag_runtime_update,
-        .commit_runtime = flag_runtime_commit
     },
     {
         .type = TABLE_TYPE_EXPR,
@@ -130,22 +134,10 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
         .commit_runtime = NULL
     },
     {
-        .type = TABLE_TYPE_DIGEST,
+        .type = TABLE_TYPE_DIGEST
-        .new_schema = NULL,
-        .free_schema = NULL,
-        .new_runtime = NULL,
-        .free_runtime = NULL,
-        .update_runtime = NULL,
-        .commit_runtime = NULL
     },
     {
-        .type = TABLE_TYPE_SIMILARITY,
+        .type = TABLE_TYPE_SIMILARITY
-        .new_schema = NULL,
-        .free_schema = NULL,
-        .new_runtime = NULL,
-        .free_runtime = NULL,
-        .update_runtime = NULL,
-        .commit_runtime = NULL
     },
     {   
         .type = TABLE_TYPE_CONJUNCTION,
@@ -262,9 +254,13 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map)
     maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP);
     maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR);
     maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS);
+    maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL);
+    maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS);
     maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS);
     maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
     maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);
+    maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN);
+    maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
     maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL);
 }
 
@@ -511,7 +507,7 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n
     assert(tbl_mgr->n_table != 0);
 
     size_t i = 0;
-    enum table_type table_type = TABLE_TYPE_MAX;
+    enum table_type table_type = TABLE_TYPE_INVALID;
 
     for (i = 0; i < MAX_TABLE_NUM; i++) {
         void *schema = table_manager_get_schema(tbl_mgr, i);
@@ -629,11 +625,11 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name)
 enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id)
 {
     if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) {
-        return TABLE_TYPE_MAX;
+        return TABLE_TYPE_INVALID;
     }
 
     if (NULL == tbl_mgr->tbl[table_id]) {
-        return TABLE_TYPE_MAX;
+        return TABLE_TYPE_INVALID;
     }
 
     return tbl_mgr->tbl[table_id]->table_type;
@@ -750,7 +746,7 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id,
     }
 
     enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
-    if (table_type == TABLE_TYPE_MAX) {
+    if (table_type == TABLE_TYPE_INVALID) {
         return -1;
     }
 
@@ -773,7 +769,7 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id,
 void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
 {
     enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
-    if (table_type == TABLE_TYPE_MAX) {
+    if (table_type == TABLE_TYPE_INVALID) {
         return;
     }
 
@@ -782,6 +778,7 @@ void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id)
         return;
     }
 
+    printf("table_id:%d\n", table_id);
     if ( table_ops[table_type].commit_runtime != NULL) {
         table_ops[table_type].commit_runtime(runtime);
     }

src/maat_virtual.cpp

@@ -8,11 +8,15 @@
 ***********************************************************************************************
 */
 
+#include <assert.h>
+
 #include "cJSON/cJSON.h"
 #include "maat_kv.h"
 #include "maat_utils.h"
 #include "log/log.h"
 #include "maat_virtual.h"
+#include "maat_rule.h"
+#include "maat_table.h"
 
 #define MODULE_VIRTUAL	 module_name_str("maat.virtual")
 
@@ -25,8 +29,6 @@ struct virtual_schema {
 void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
                          const char *table_name, struct log_handle *logger)
 {
-    //size_t read_cnt = 0;
-
     cJSON *item = cJSON_GetObjectItem(json, "physical_table");
     if (NULL == item || item->type != cJSON_Array) {
         log_error(logger, MODULE_VIRTUAL, 
@@ -35,37 +37,43 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
     }
 
     struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
-
     vt_schema->tbl_mgr = tbl_mgr;
-    return vt_schema;
+   
-#if 0
-    struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
     int cnt = cJSON_GetArraySize(item);
     for (int i = 0; i < cnt; i++) {
         cJSON *tmp_item = cJSON_GetArrayItem(item, i);
         if (tmp_item != NULL && tmp_item->type == cJSON_String) {
-            int table_id = -1;
+            int table_id = table_manager_get_table_id(tbl_mgr, tmp_item->valuestring);
             /* physical table should already exist */
-            int ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &table_id);
+            if (table_id < 0) {
-			if (ret < 0) {
+                log_error(logger, MODULE_VIRTUAL, "table:%s is not registered",
-				return -1;		
+                            tmp_item->valuestring);
-			}
+                FREE(vt_schema);
+                return NULL;
+            }
 
-            enum scan_type table_scan_type = table_schema_get_scan_type(table_array[table_id]);
+            enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
-            vt_schema->physical_table_id[table_scan_type]= table_id;
+            assert(table_type != TABLE_TYPE_INVALID);
+            enum scan_type scan_type = maat_table_get_scan_type(table_type);
+            assert(scan_type != SCAN_TYPE_INVALID);
+            vt_schema->physical_table_id[scan_type]= table_id;
         }
     }
 
-    if (read_cnt < 3) {
-        FREE(vt_schema);
-        return NULL;
-    }
-
     return vt_schema;
-#endif
 }
 
 void virtual_schema_free(void *virtual_schema)
 {
     FREE(virtual_schema);
 }
+
+int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type)
+{
+    if (NULL == virtual_schema) {
+        return -1;
+    }
+
+    struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
+    return schema->physical_table_id[type];
+}

test/maat_framework_gtest.cpp

@@ -113,7 +113,7 @@ TEST_F(MaatIPScan, hit_ip_and_port) {
 
     state = NULL;
     ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
-    EXPECT_EQ(ret, MAAT_OK);
+    EXPECT_EQ(ret, MAAT_SCAN_OK);
     EXPECT_EQ(n_hit_result, 0);
 
     maat_state_free(&state);
@@ -148,7 +148,7 @@ TEST_F(MaatStringScan, hit_ip_and_port_range) {
     n_hit_result = 0;
     state = NULL;
     ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
-    EXPECT_EQ(ret, MAAT_OK);
+    EXPECT_EQ(ret, MAAT_SCAN_OK);
     EXPECT_EQ(n_hit_result, 0);
 }
 
@@ -183,7 +183,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) {
     EXPECT_EQ(ret, 1);
     state = NULL;
     ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
-    EXPECT_EQ(ret, MAAT_OK);
+    EXPECT_EQ(ret, MAAT_SCAN_OK);
     EXPECT_EQ(n_hit_result, 0);
     maat_state_free(&state);
 }
@@ -209,7 +209,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) {
     EXPECT_EQ(ret, 1);
     state = NULL;
     ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
-    EXPECT_EQ(ret, MAAT_OK);
+    EXPECT_EQ(ret, MAAT_SCAN_OK);
     EXPECT_EQ(n_hit_result, 0);
     maat_state_free(&state);
 }
@@ -233,7 +233,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) {
 
     state = NULL;
     ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
-    EXPECT_EQ(ret, MAAT_OK);
+    EXPECT_EQ(ret, MAAT_SCAN_OK);
     EXPECT_EQ(n_hit_result, 0);
     maat_state_free(&state);
 }
@@ -256,7 +256,7 @@ TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) {
 
     state = NULL;
     ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state);
-    EXPECT_EQ(ret, MAAT_OK);
+    EXPECT_EQ(ret, MAAT_SCAN_OK);
     EXPECT_EQ(n_hit_result, 0);
     maat_state_free(&state);
 }
@@ -269,7 +269,7 @@ TEST(maat_scan_string, dynamic_config) {
     size_t n_hit_result = 0;
     struct maat_state *state = NULL;
     int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state);
-    EXPECT_EQ(ret, MAAT_OK);
+    EXPECT_EQ(ret, MAAT_SCAN_OK);
     EXPECT_EQ(n_hit_result, 0);
     maat_state_free(&state);
 
@@ -303,7 +303,7 @@ TEST(maat_scan_ip, dynamic_config) {
     size_t n_hit_result = 0;
     struct maat_state *state = NULL;
     ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state);
-    EXPECT_EQ(ret, MAAT_OK);
+    EXPECT_EQ(ret, MAAT_SCAN_OK);
     EXPECT_EQ(n_hit_result, 0);
     maat_state_free(&state);
 

test/maat_input_mode_gtest.cpp

@@ -10,6 +10,7 @@
 const char *table_info_path = "./table_info.conf";
 const char *json_filename = "maat_json.json";
 
+#if 0
 TEST(json_mode, maat_scan_string) {
     char json_iris_path[128] = {0};
     snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
@@ -26,16 +27,16 @@ TEST(json_mode, maat_scan_string) {
     struct maat *maat_instance = maat_new(opts, table_info_path);
     EXPECT_NE(maat_instance, nullptr);
 
-    int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL");
+    int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE");
    
-    char scan_data[128] = "hello";
+    char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
     int results[5] = {0};
     size_t n_hit_result = 0;
     struct maat_state *state = NULL;
     int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
     EXPECT_EQ(ret, MAAT_SCAN_HIT);
     EXPECT_EQ(n_hit_result, 1);
-    EXPECT_EQ(results[0], 191);
+    EXPECT_EQ(results[0], 132);
 
     maat_options_free(opts);
     maat_state_free(&state);
@@ -69,22 +70,22 @@ TEST(iris_mode, maat_scan_string) {
     struct maat *maat_instance = maat_new(opts, table_info_path);
     EXPECT_NE(maat_instance, nullptr);
 
-    int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL");
+    int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE");
    
-    char scan_data[128] = "hello";
+    char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
     int results[5] = {0};
     size_t n_hit_result = 0;
     struct maat_state *state = NULL;
     int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
     EXPECT_EQ(ret, MAAT_SCAN_HIT);
     EXPECT_EQ(n_hit_result, 1);
-    EXPECT_EQ(results[0], 191);
+    EXPECT_EQ(results[0], 132);
 
     maat_options_free(opts);
     maat_state_free(&state);
     maat_free(maat_instance);
 }
-
+#endif
 int count_line_num_cb(const char *table_name, const char *line, void *u_para)
 {
 	(*((unsigned int *)u_para))++;
@@ -131,6 +132,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
 	return 0;
 }
 
+
+#if 1
 TEST(redis_mode, maat_scan_string) {
     char json_iris_path[128] = {0};
     char redis_ip[64] = "127.0.0.1";
@@ -189,23 +192,23 @@ TEST(redis_mode, maat_scan_string) {
     maat_options_set_logger(opts, logger);
 
     struct maat *maat_instance = maat_new(opts, table_info_path);
-    int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL");
+    int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE");
    
-    char scan_data[128] = "hello";
+    char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
     int results[5] = {0};
     size_t n_hit_result = 0;
     struct maat_state *state = NULL;
     int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state);
     EXPECT_EQ(ret, MAAT_SCAN_HIT);
     EXPECT_EQ(n_hit_result, 1);
-    EXPECT_EQ(results[0], 191);
+    EXPECT_EQ(results[0], 132);
 
     maat_options_free(opts);
     maat_state_free(&state);
     log_handle_destroy(maat_instance->logger);
     maat_free(maat_instance);
 }
-
+#endif
 int main(int argc, char ** argv)
 {
 	int ret=0;

test/table_info.conf

@@ -20,7 +20,7 @@
         "table_id":1,
         "table_name":"GROUP2COMPILE",
         "table_type":"group2compile",
-        "associated_compile_table_id":1,
+        "associated_compile_table_id":0,
         "valid_column":3,
         "custom": {
             "group_id":1,
@@ -113,6 +113,7 @@
         "table_type":"expr_plus",
         "valid_column":8,
         "custom": {
+            "scan_mode":"block",
             "item_id":1,
             "group_id":2,
             "district":3,
@@ -210,7 +211,6 @@
             "expr_type":5,
             "match_method":6,
             "is_hexbin":7
-        
         }
     },
     {
@@ -377,12 +377,6 @@
             "fqdn":3
         }
     },
-    {
-        "table_id":37,
-        "table_name":"VIRTUAL_SSL_SNI",
-        "table_type":"virtual",
-        "physical_table": ["KEYWORDS_TABLE", "INTERGER_PLUS"]
-    },
     {
         "table_id":38,
         "table_name":"APP_ID",

vendor/CMakeLists.txt

@@ -100,3 +100,29 @@ add_library(igraph-static STATIC IMPORTED GLOBAL)
 add_dependencies(igraph-static igraph)
 set_property(TARGET igraph-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libigraph.a)
 set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)
+
+#flag-matcher
+ExternalProject_Add(flagmatcher PREFIX flagmatcher
+        URL ${CMAKE_CURRENT_SOURCE_DIR}/flag-matcher.tar.gz
+		CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC")
+
+ExternalProject_Get_Property(flagmatcher INSTALL_DIR)
+file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
+
+add_library(flagmatcher-static STATIC IMPORTED GLOBAL)
+add_dependencies(flagmatcher-static flagmatcher)
+set_property(TARGET flagmatcher-static  PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/libflag_matcher.a)
+set_property(TARGET flagmatcher-static  PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)
+
+#interval-matcher
+ExternalProject_Add(intervalmatcher PREFIX intervalmatcher
+        URL ${CMAKE_CURRENT_SOURCE_DIR}/interval-matcher.tar.gz
+		CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC")
+
+ExternalProject_Get_Property(intervalmatcher INSTALL_DIR)
+file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
+
+add_library(intervalmatcher-static STATIC IMPORTED GLOBAL)
+add_dependencies(intervalmatcher-static flagmatcher)
+set_property(TARGET intervalmatcher-static  PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/libinterval_matcher.a)
+set_property(TARGET intervalmatcher-static  PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)