add json/redis rule parser

2022-12-03 22:23:41 +08:00
parent 84a271144b
commit ea4c1ba4c3
32 changed files with 6734 additions and 177 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,4 +14,5 @@ enable_testing()
 add_subdirectory(vendor)
 add_subdirectory(src)
 add_subdirectory(test)
+add_subdirectory(tools)
 add_subdirectory(scanner)
--- a/include/maat/maat.h
+++ b/include/maat/maat.h
@@ -66,6 +66,10 @@ int maat_options_set_instance_name(struct maat_options *opts, const char *instan
 int maat_options_set_deferred_load_on(struct maat_options *opts);
 int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_dir);
 int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir);
+int maat_options_set_json_file(struct maat_options *opts, const char *json_filename);
+int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip);
+int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port);
+int maat_options_set_redis_db_index(struct maat_options *opts, int db_index);

 /* maat_instance API */
 struct maat *maat_new(struct maat_options *opts, const char *table_info_path);
--- a/include/utils.h
+++ b/include/utils.h
@@ -20,6 +20,15 @@ extern "C"

 #define ALLOC(type, number) ((type *)calloc(sizeof(type), number))

+#define FREE(ptr)  \
+    {                   \
+        if (ptr)        \
+        {               \
+            free(ptr);  \
+            ptr = NULL; \
+        }               \
+    }
+
 #ifdef __cpluscplus
 }
 #endif
--- a/scanner/IPMatcher.h
+++ b/scanner/IPMatcher.h
@@ -1,117 +0,0 @@
-/*
- *
- * Copyright (c) 2020
- * String Algorithms Research Group
- * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS)
- * National Engineering Laboratory for Information Security Technologies (NELIST)
- * All rights reserved
- *
- * Written by: LU YUHAI    (luyuhai@iie.ac.cn)
- * Last modification: 2020-04-20
- *
- * This code is the exclusive and proprietary property of IIE-CAS and NELIST. 
- * Usage for direct or indirect commercial advantage is not allowed without 
- * written permission from the authors.
- *
- */
-
-#ifndef H_IP_MATCHER_H
-#define H_IP_MATCHER_H
-#include <stddef.h>
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-
-	 enum IP_TYPE
-	{
-		IPv4,
-		IPv6
-	};
-
-	/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĵ<EFBFBD><C4B5><EFBFBD>IPv4<76><34><EFBFBD><EFBFBD> */
-	struct ipv4_range
-	{
-		unsigned int	start_ip;     /* IP<49><50>Χ<EFBFBD>½<EFBFBD> */
-		unsigned int	end_ip;     /* IP<49><50>Χ<EFBFBD>Ͻ<EFBFBD> */
-	};
-
-	/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĵ<EFBFBD><C4B5><EFBFBD>IPv6<76><36><EFBFBD><EFBFBD> */
-	struct ipv6_range
-	{
-		unsigned int	start_ip[4];  /* IP<49><50>Χ<EFBFBD>½<EFBFBD> */
-		unsigned int	end_ip[4];  /* IP<49><50>Χ<EFBFBD>Ͻ<EFBFBD> */
-	};
-
-	/* ͨ<>õ<EFBFBD>ip<69><70><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
-	struct ip_rule
-	{
-		enum IP_TYPE type;           /* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͣ<EFBFBD>ipv4<76><34>ipv6 */
-		unsigned int rule_id;		 /* <20><><EFBFBD><EFBFBD>ID */
-		void* user_tag;              /* <20>û<EFBFBD><C3BB>Զ<EFBFBD><D4B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD><DDA3><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1>ƥ<EFBFBD><C6A5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
-		union								
-		{
-			struct ipv4_range ipv4_rule;  /*<2A><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĵ<EFBFBD><C4B5><EFBFBD>IPv4<76><34><EFBFBD><EFBFBD>*/
-			struct ipv6_range ipv6_rule;  /*<2A><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ĵ<EFBFBD><C4B5><EFBFBD>IPv6<76><36><EFBFBD><EFBFBD>*/
-		};
-	};
-
-	/* ͨ<>õĴ<C3B5>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
-	struct ip_data
-	{
-		enum IP_TYPE type;			/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͣ<EFBFBD>ipv4<76><34>ipv6 */
-		union                       /* <20><><EFBFBD><EFBFBD>rule_type<70><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݸ<EFBFBD><DDB8><EFBFBD><EFBFBD><EFBFBD>ipv4<76><34><EFBFBD><EFBFBD>ipv6 */
-		{
-			unsigned int ipv4;		/* ipv4<76><34><EFBFBD><EFBFBD>*/
-			unsigned int ipv6[4];	/* ipv6<76><36><EFBFBD><EFBFBD>*/
-		};
-	};
-
-
-		/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʽ<EFBFBD><CABD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
-	struct scan_result
-	{
-		unsigned int  rule_id;	/* <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ID */		
-		void *        tag;		/* <20>û<EFBFBD><C3BB>Զ<EFBFBD><D4B6><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ݣ<EFBFBD><DDA3><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1>ƥ<EFBFBD><C6A5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> */
-	};
-
-
-	struct ip_matcher;
-
-	/* 
-	<09><><EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĺ<EFBFBD><C4B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD> 
-	<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
-		rules[in]<5D><>һ<EFBFBD><D2BB>ip<69><70><EFBFBD><EFBFBD>
-		rule_num[in]<5D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĺ<EFBFBD><C4B9><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
-		mem_use[out]<5D><><EFBFBD>ڴ<EFBFBD><DAB4><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
-	<09><><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5>
-		ipɨ<70><C9A8><EFBFBD><EFBFBD>,<2C><><EFBFBD>ؿ<EFBFBD>ָ<EFBFBD><D6B8><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɨ<EFBFBD><C9A8><EFBFBD><EFBFBD>ʧ<EFBFBD><CAA7>
-	*/
-	struct ip_matcher* ip_matcher_new(struct ip_rule * rules, size_t rule_num, size_t * mem_use);
-
-	/*
-	<09><><EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD><EFBFBD><EFBFBD>ipɨ<70><C9A8><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ip<69><70><EFBFBD>ݽ<EFBFBD><DDBD><EFBFBD>ɨ<EFBFBD><C9A8>
-	<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
-		matcher[in]<5D><>ipɨ<70><C9A8><EFBFBD><EFBFBD>
-		data[in]<5D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĵ<EFBFBD>ɨ<EFBFBD><C9A8>ip<69><70><EFBFBD><EFBFBD>
-		result[in]<5D><><EFBFBD><EFBFBD><EFBFBD>ؽ<EFBFBD><D8BD><EFBFBD><EFBFBD>洢<EFBFBD><E6B4A2><EFBFBD><EFBFBD>
-		size[in]<5D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ĵ<EFBFBD>С
-	<09><><EFBFBD><EFBFBD>ֵ<EFBFBD><D6B5>
-		<09><><EFBFBD>н<EFBFBD><D0BD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><=size<7A><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ֵΪ-1<><31>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
-		
-	*/
-	int ip_matcher_match(struct ip_matcher* matcher, struct ip_data * data, struct scan_result* result, size_t size);
-
-	/*
-	<09><><EFBFBD>ܣ<EFBFBD><DCA3><EFBFBD><EFBFBD><EFBFBD>һ<EFBFBD><D2BB>ipɨ<70><C9A8><EFBFBD><EFBFBD>
-	<09><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
-		matcher[in]<5D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ٵ<EFBFBD>ipɨ<70><C9A8><EFBFBD><EFBFBD>ָ<EFBFBD><D6B8>
-	*/
-	void ip_matcher_free(struct ip_matcher* matcher);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* !defined(H_IP_MATCHER_H) */
--- a/scanner/adapter_hs.cpp
+++ b/scanner/adapter_hs.cpp
@@ -184,7 +184,7 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt

 struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array)
 {
-    if ((scan_mode != SCAN_MODE_BLOCK && scan_mode != SCAN_MODE_STREAM) || 
+    if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) || 
         0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) {
        fprintf(stderr, "%s input parameters illegal!", __func__);
        return NULL;
@@ -407,11 +407,12 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d
    utarray_new(pattern_id_set, &ut_pattern_id_icd);
 	utarray_reserve(pattern_id_set, hs_instance->n_patterns);

+    int err_count = 0;
    if (hs_rt->literal_db != NULL) {
        err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set);
        if (err != HS_SUCCESS) {
            //log_error()
-            return -1;
+            err_count++;
        }
    }
    
@@ -419,10 +420,14 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d
        err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set);
        if (err != HS_SUCCESS) {
            //log_error()
-            return -1;
+            err_count++;
        }
    }
    
+    if (2 == err_count) {
+        return -1;
+    }
+
    size_t pattern_set_size = utarray_len(pattern_id_set);
    unsigned long long items[pattern_set_size];
    memset(items, 0, sizeof(unsigned long long) * pattern_set_size);
--- a/scanner/adapter_hs.h
+++ b/scanner/adapter_hs.h
@@ -24,10 +24,10 @@ extern "C"
 struct adapter_hs;

 /* scan mode */
-enum scan_mode {
-    SCAN_MODE_BLOCK = 1,
-    SCAN_MODE_STREAM,
-    SCAN_MODE_MAX
+enum hs_scan_mode {
+    HS_SCAN_MODE_BLOCK = 1,
+    HS_SCAN_MODE_STREAM,
+    HS_SCAN_MODE_MAX
 };

 /* pattern type: PATTERN_TYPE_STR(pure literal string) or PATTERN_TYPE_REG(regex expression) */
--- a/scanner/adapter_hs_gtest.cpp
+++ b/scanner/adapter_hs_gtest.cpp
@@ -66,11 +66,11 @@ TEST(block_mode_initialize, invalid_input_parameter)
    EXPECT_EQ(hs_instance, nullptr);

    /* case2: invalid expr parameter */
-    hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, nullptr, 1);
+    hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 1);
    EXPECT_EQ(hs_instance, nullptr);

    /* case3: invalid expr num */
-    hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, exprs, 0);
+    hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, exprs, 0);
    EXPECT_EQ(hs_instance, nullptr);
 }

@@ -79,7 +79,7 @@ TEST(block_mode_scan, invalid_input_parameter)
    and_expr_t expr_array[64];
    size_t n_expr_array = 0;

-    struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, nullptr, 0);
+    struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 0);
    EXPECT_EQ(hs_instance, nullptr);

    hs_instance = adapter_hs_initialize(0, 1, expr_array, n_expr_array);
@@ -88,7 +88,7 @@ TEST(block_mode_scan, invalid_input_parameter)
    n_expr_array = 1;
    expr_array[0].expr_id = 101;
    expr_array[0].n_patterns = 10;
-    hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
+    hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
    EXPECT_EQ(hs_instance, nullptr);

    memset(expr_array, 0, sizeof(expr_array));
@@ -96,7 +96,7 @@ TEST(block_mode_scan, invalid_input_parameter)
    expr_array[0].expr_id = 101;
    expr_array[0].n_patterns = 1;
    expr_array[0].patterns[0].type = 0;
-    hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
+    hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
    EXPECT_EQ(hs_instance, nullptr);
 }

@@ -109,7 +109,7 @@ TEST(block_mode_scan, hit_one_expr)
    EXPECT_EQ(ret, 0);
    EXPECT_EQ(n_expr_array, 6);

-    struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
+    struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
    EXPECT_NE(hs_instance, nullptr);
    expr_array_free(expr_array, n_expr_array);

@@ -149,7 +149,7 @@ TEST(block_mode_scan, hit_two_expr)
    EXPECT_EQ(ret, 0);
    EXPECT_EQ(n_expr_array, 6);

-    struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
+    struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
    EXPECT_NE(hs_instance, nullptr);
    expr_array_free(expr_array, n_expr_array);

@@ -184,7 +184,7 @@ TEST(block_mode_scan, hit_three_expr)
    EXPECT_EQ(ret, 0);
    EXPECT_EQ(n_expr_array, 6);

-    struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
+    struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
    EXPECT_NE(hs_instance, nullptr);
    expr_array_free(expr_array, n_expr_array);

@@ -221,7 +221,7 @@ TEST(block_mode_scan, hit_four_expr)
    EXPECT_EQ(ret, 0);
    EXPECT_EQ(n_expr_array, 6);

-    struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
+    struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
    EXPECT_NE(hs_instance, nullptr);
    expr_array_free(expr_array, n_expr_array);

@@ -260,7 +260,7 @@ TEST(block_mode_scan, hit_five_expr)
    EXPECT_EQ(ret, 0);
    EXPECT_EQ(n_expr_array, 6);

-    struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
+    struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array);
    EXPECT_NE(hs_instance, nullptr);
    expr_array_free(expr_array, n_expr_array);

--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -8,8 +8,9 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
 message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")

 add_definitions(-fPIC)
-set(MAAT_SRC maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp 
-    maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp)
+set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp 
+    maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp
+    maat_command.cpp maat_redis_monitor.cpp)

 set(LIB_SOURCE_FILES
 	${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c)
@@ -25,7 +26,7 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX)
 set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe)
 set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)

-target_link_libraries(maat_frame_static adapter-static pthread)
+target_link_libraries(maat_frame_static adapter-static hiredis-static pthread crypto z)

 # Shared Library Output
 #add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES})
--- a/src/inc_internal/json2iris.h
+++ b/src/inc_internal/json2iris.h
@@ -0,0 +1,29 @@
+/*
+**********************************************************************************************
+*	File: json2iris.h
+*   Description: rule for transform json2iris
+*	Authors: Liu WenTan <liuwentan@geedgenetworks.com>
+*	Date:    2022-10-31
+*   Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
+***********************************************************************************************
+*/
+
+#ifndef _JSON2IRIS_H_
+#define _JSON2IRIS_H_
+
+#ifdef __cpluscplus
+extern "C"
+{
+#endif
+
+#include "hiredis/hiredis.h"
+
+int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, 
+              const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, 
+              char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo);
+
+#ifdef __cpluscplus
+}
+#endif
+
+#endif
--- a/src/inc_internal/maat_command.h
+++ b/src/inc_internal/maat_command.h
@@ -0,0 +1,39 @@
+/*
+**********************************************************************************************
+*	File: maat_command.h
+*   Description: 
+*	Authors: Liu WenTan <liuwentan@geedgenetworks.com>
+*	Date:    2022-10-31
+*   Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
+***********************************************************************************************
+*/
+
+#ifndef _MAAT_COMMAND_H_
+#define _MAAT_COMMAND_H_
+
+#ifdef __cpluscplus
+extern "C"
+{
+#endif
+
+enum maat_operation {
+	MAAT_OP_DEL = 0,
+	MAAT_OP_ADD,
+	MAAT_OP_RENEW_TIMEOUT	//Rule expire time is changed to now+cmd->expire_after
+};
+
+struct maat_cmd_line
+{
+	const char *table_name;
+	const char *table_line;
+	int rule_id;		// for MAAT_OP_DEL, only rule_id and table_name are necessary.
+	int expire_after;	//expired after $timeout$ seconds, set to 0 for never timeout.
+};
+
+int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule);
+
+#ifdef __cpluscplus
+}
+#endif
+
+#endif
--- a/src/inc_internal/maat_common.h
+++ b/src/inc_internal/maat_common.h
@@ -18,6 +18,8 @@ extern "C"

 #include <stddef.h>

+#include "maat_rule.h"
+
 struct maat_options {
    size_t nr_worker_threads;
    int rule_effect_interval_ms;
@@ -25,8 +27,11 @@ struct maat_options {
    int gc_timeout_ms;
    int deferred_load_on;
    enum data_source input_mode;
-    char iris_full_dir[NAME_MAX];
-    char iris_inc_dir[NAME_MAX];
+    union {
+        struct source_iris_ctx iris_ctx;
+        struct source_json_ctx json_ctx;
+        struct source_redis_ctx redis_ctx;
+    };
 };

 #ifdef __cpluscplus
--- a/src/inc_internal/maat_config_monitor.h
+++ b/src/inc_internal/maat_config_monitor.h
@@ -28,6 +28,8 @@ void config_monitor_traverse(long long version, const char *idx_dir,
 							void (*finish_fn)(void *),
 							void *u_param);

+int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz);
+
 #ifdef __cpluscplus
 }
 #endif
--- a/src/inc_internal/maat_redis_monitor.h
+++ b/src/inc_internal/maat_redis_monitor.h
@@ -0,0 +1,33 @@
+/*
+**********************************************************************************************
+*	File: maat_redis_monitor.h
+*   Description: maat redis monitor api
+*	Authors: Liu WenTan <liuwentan@geedgenetworks.com>
+*	Date:    2022-11-29
+*   Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
+***********************************************************************************************
+*/
+
+#ifndef _MAAT_REDIS_MONITOR_H_
+#define _MAAT_REDIS_MONITOR_H_
+
+#ifdef __cpluscplus
+extern "C"
+{
+#endif
+
+#include "maat_rule.h"
+
+#include <stdint.h>
+
+void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx,
+							void (*start_fn)(long long, int, void *),
+							int (*update_fn)(const char *, const char *, void *),
+							void (*finish_fn)(void *),
+							void *u_param);
+
+#ifdef __cpluscplus
+}
+#endif
+
+#endif
--- a/src/inc_internal/maat_rule.h
+++ b/src/inc_internal/maat_rule.h
@@ -21,6 +21,13 @@ extern "C"
 #include <limits.h>
 #include <sys/time.h>
 #include <pthread.h>
+#include <sys/queue.h>
+#include <openssl/md5.h>
+
+#include "hiredis/hiredis.h"
+#include "uthash/uthash.h"
+#include "maat_table_schema.h"
+#include "maat_command.h"

 struct maat_runtime {
    /* maat_runtime can be created and destroy dynamic, so need version info */
@@ -37,7 +44,9 @@ struct maat_runtime {

 enum data_source {
    DATA_SOURCE_NONE = 0,
-    DATA_SOURCE_IRIS_FILE
+    DATA_SOURCE_REDIS,
+    DATA_SOURCE_IRIS_FILE,
+    DATA_SOURCE_JSON_FILE
 };

 struct source_iris_ctx {
@@ -45,6 +54,52 @@ struct source_iris_ctx {
    char full_dir[NAME_MAX];
 };

+struct source_json_ctx
+{
+	char json_file[NAME_MAX];
+	char iris_file[NAME_MAX];
+	char effective_json_md5[MD5_DIGEST_LENGTH*2+1];
+	struct timespec last_md5_time;
+};
+
+struct source_redis_ctx
+{
+	redisContext *read_ctx;
+	redisContext *write_ctx;
+	char redis_ip[64];
+	uint16_t redis_port;
+	int redis_db;
+	time_t last_reconnect_time;
+};
+
+struct foreign_key {
+	int column;
+	char *key;
+    size_t key_len;
+	char *filename;
+};
+
+//rm= Redis Maat
+struct serial_rule {
+	enum maat_operation op;//0: delete, 1: add.
+	unsigned long rule_id;
+	int label_id;
+	long long timeout;			// absolute unix time.
+	char table_name[NAME_MAX];
+	char *table_line;
+	int n_foreign;
+	struct foreign_key *f_keys;
+	TAILQ_ENTRY(serial_rule) entries;
+    UT_hash_handle hh;
+};
+
+#define	POSSIBLE_REDIS_REPLY_SIZE	2
+struct expected_reply {
+	int s_rule_seq;
+	int possible_reply_num;
+	redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE];
+};
+
 struct maat {
    char instance_name[NAME_MAX];

@@ -56,6 +111,8 @@ struct maat {
    enum data_source input_mode;
    union {
        struct source_iris_ctx iris_ctx;
+        struct source_json_ctx json_ctx;
+        struct source_redis_ctx mr_ctx;
    };
    
    int deferred_load;
@@ -72,7 +129,24 @@ struct maat {
    int rule_update_checking_interval_ms;
    int gc_timeout_ms; //garbage collection timeout_ms;

+    int cumulative_update_off; //Default: cumulative update on
+
    struct maat_garbage_bin *garbage_bin;
+
+    char compile_tn[NAME_MAX];
+    char group_tn[NAME_MAX];
+	char group2compile_tn[NAME_MAX];
+	char group2group_tn[NAME_MAX];
+
+    char decrypt_key[NAME_MAX];
+	char decrypt_algo[NAME_MAX];
+    int maat_json_is_gzipped;
+
+    long long load_specific_version;  //Default: Load the Latest. Only valid in redis mode, and maybe failed for too old
+    char foreign_cont_dir[NAME_MAX];
+
+    /* statistics */
+    long long line_cmd_acc_num;
 };

 void maat_start_cb(long long new_version, int update_type, void *u_para);
@@ -85,6 +159,40 @@ void *rule_monitor_loop(void *arg);

 void maat_read_full_config(struct maat *maat_instance);

+/* maat command API for internal */
+redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db);
+
+redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...);
+
+int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply);
+
+long long maat_cmd_redis_server_time_s(redisContext *c);
+
+long long maat_cmd_read_redis_integer(const redisReply *reply);
+
+int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq);
+
+const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len);
+
+int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time);
+
+void maat_cmd_empty_serial_rule(struct serial_rule *s_rule);
+
+int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, 
+                             long long *new_version, struct table_schema_manager* table_schema_mgr, 
+                             struct serial_rule **list, int *update_type, int cumulative_off);
+
+int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process);
+
+int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num, const char* dir);
+
+void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn);
+
+void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule);
+
+void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id, 
+                              const char *table_name, const char *line, long long timeout);
+
 #ifdef __cpluscplus
 }
 #endif
--- a/src/inc_internal/maat_table_schema.h
+++ b/src/inc_internal/maat_table_schema.h
@@ -25,13 +25,28 @@ extern "C"
 #define MAX_DISTRICT_STR 128
 #define MAX_IP_STR       128
 #define MAX_KEYWORDS_STR 1024
+#define	MAX_FOREIGN_CLMN_NUM 8

 enum table_type {
 	TABLE_TYPE_EXPR = 0,
    TABLE_TYPE_EXPR_PLUS,
    TABLE_TYPE_IP,
+    TABLE_TYPE_IP_PLUS,
+    TABLE_TYPE_INTERVAL,
+    TABLE_TYPE_INTERVAL_PLUS,
+    TABLE_TYPE_DIGEST,
+    TABLE_TYPE_SIMILARITY,
    TABLE_TYPE_PLUGIN,
    TABLE_TYPE_IP_PLUGIN,
+    TABLE_TYPE_FQDN_PLUGIN,
+    TABLE_TYPE_BOOL_PLUGIN,
+    //above are physical table
+    TABLE_TYPE_VIRTUAL,
+    TABLE_TYPE_COMPOSITION,
+    TABLE_TYPE_COMPILE,
+    TABLE_TYPE_GROUP,
+    TABLE_TYPE_GROUP2GROUP,
+	TABLE_TYPE_GROUP2COMPILE,
 	TABLE_TYPE_MAX
 };

@@ -42,8 +57,21 @@ enum expr_type {
    EXPR_TYPE_MAX
 };

+enum scan_type {
+    SCAN_TYPE_INVALID = -1,
+	SCAN_TYPE_NONE = 0,
+	SCAN_TYPE_PLUGIN,
+	SCAN_TYPE_IP_PLUGIN,
+	SCAN_TYPE_FQDN_PLUGIN,
+	SCAN_TYPE_BOOL_PLUGIN,
+	SCAN_TYPE_IP,
+	SCAN_TYPE_INTERVAL,
+	SCAN_TYPE_STRING,
+	SCAN_TYPE_MAX
+};
+
 enum match_method {
-    MATCH_METHOD_SUB=0,
+    MATCH_METHOD_SUB = 0,
 	MATCH_METHOD_RIGHT,
 	MATCH_METHOD_LEFT,
 	MATCH_METHOD_COMPLETE,
@@ -60,10 +88,6 @@ struct expr_item {
    int is_hexbin;
    int is_case_sensitive;
    int is_valid;
-   
-     //rule_tag; 只存在schema里
-	//int have_exdata;
-    //struct ex_data *ex_data; //hash表
 };

 struct plugin_item {
@@ -129,14 +153,21 @@ void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* tabl
 /* table schema generic API */
 struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id);

+struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr, 
+                                                   int table_id, enum scan_type type, int *virtual_table_id);
+
 enum table_type table_schema_get_table_type(struct table_schema *table_schema);

 int table_schema_get_table_id(struct table_schema *table_schema);

+enum scan_type table_schema_get_scan_type(struct table_schema *table_schema);
+
 struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema);

+int table_schema_get_valid_flag_column(struct table_schema *table_schema);
+
 /* expr table schema API */
-enum scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);
+enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema);

 /* plugin table schema API */
 int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
@@ -165,6 +196,8 @@ size_t plugin_table_schema_callback_count(struct table_schema *table_schema);

 void plugin_table_schema_all_cb_update(struct table_schema *table_schema, const char *row);

+int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns);
+
 #ifdef __cpluscplus
 }
 #endif
--- a/src/inc_internal/maat_utils.h
+++ b/src/inc_internal/maat_utils.h
@@ -30,6 +30,8 @@ extern "C"
 #define MIN(a, b)  	(((a) < (b)) ? (a) : (b))
 #endif

+#define UNUSED	__attribute__((unused))
+
 char *maat_strdup(const char *s);

 int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len);
@@ -42,6 +44,19 @@ char *str_unescape_and(char *s);

 char *str_unescape(char *s);

+char *md5_file(const char *filename, char *md5string);
+
+int decrypt_open(const char* file_name, const char* key, const char* algorithm, 
+                 unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz);
+
+int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz, 
+                 const char *key, const char *algorithm, int do_encrypt, char *err_str, size_t err_str_sz);
+
+int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, 
+                    unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz);
+
+size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_t n_src);
+
 /* system cmd wrapper */
 int system_cmd_mkdir(const char* path);

--- a/src/json2iris.cpp
+++ b/src/json2iris.cpp
--- a/src/maat_api.cpp
+++ b/src/maat_api.cpp
@@ -12,13 +12,16 @@
 #include <string.h>

 #include "utils.h"
+#include "json2iris.h"
 #include "maat/maat.h"
 #include "maat_rule.h"
 #include "maat_common.h"
 #include "maat_kv.h"
+#include "maat_command.h"
 #include "maat_table_schema.h"
 #include "maat_table_runtime.h"
 #include "maat_config_monitor.h"
+#include "maat_redis_monitor.h"

 struct maat_options* maat_options_new(void)
 {
@@ -65,6 +68,7 @@ int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms)
 int maat_options_set_deferred_load_on(struct maat_options *opts)
 {
    opts->deferred_load_on = 1;
+
    return 0;
 }

@@ -74,8 +78,9 @@ int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_d
        return -1;
    }

-    memcpy(opts->iris_full_dir, full_dir, strlen(full_dir));
+    memcpy(opts->iris_ctx.full_dir, full_dir, strlen(full_dir));
    opts->input_mode = DATA_SOURCE_IRIS_FILE;
+
    return 0;
 }

@@ -85,26 +90,91 @@ int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir
        return -1;
    }

-    memcpy(opts->iris_inc_dir, inc_dir, strlen(inc_dir));
+    memcpy(opts->iris_ctx.inc_dir, inc_dir, strlen(inc_dir));
    opts->input_mode = DATA_SOURCE_IRIS_FILE;
+
+    return 0;
+}
+
+int maat_options_set_json_file(struct maat_options *opts, const char *json_filename)
+{
+    strncpy(opts->json_ctx.json_file, json_filename, sizeof(opts->json_ctx.json_file));
+    opts->input_mode = DATA_SOURCE_JSON_FILE;
+
+    return 0;
+}
+
+int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip)
+{
+    memcpy(opts->redis_ctx.redis_ip, redis_ip, strlen(redis_ip));
+    opts->input_mode = DATA_SOURCE_REDIS;
+
+    return 0;
+}
+
+int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port)
+{
+    opts->redis_ctx.redis_port = redis_port;
+
+    return 0;
+}
+
+int maat_options_set_redis_db_index(struct maat_options *opts, int db_index)
+{
+    opts->redis_ctx.redis_db = db_index;
+
    return 0;
 }

 void maat_read_full_config(struct maat *maat_instance)
 {
+    int ret = -1;
+    char err_str[NAME_MAX] = {0};
+    struct source_redis_ctx *mr_ctx = NULL;
+
    switch (maat_instance->input_mode) {
+        case DATA_SOURCE_REDIS:
+            mr_ctx = &(maat_instance->mr_ctx);
+            fprintf(stdout, "Maat initiate from Redis %s:%hu db%d.", 
+                            mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
+            mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
+            if (mr_ctx->read_ctx != NULL) {
+                redis_monitor_traverse(maat_instance->maat_version, mr_ctx,
+									   maat_start_cb, maat_update_cb, maat_finish_cb,
+									   maat_instance);
+            }
+
+            if (NULL == maat_instance->creating_maat_rt) {
+                fprintf(stderr, "At initiation: NO effective rule in %s.",
+                                maat_instance->iris_ctx.full_dir);
+            }
+            break;
        case DATA_SOURCE_IRIS_FILE:
 			config_monitor_traverse(maat_instance->maat_version,
                                    maat_instance->iris_ctx.full_dir,
-                                    maat_start_cb,
-                                    maat_update_cb,
-                                    maat_finish_cb,
+                                    maat_start_cb, maat_update_cb, maat_finish_cb,
                                    maat_instance);
            if (NULL == maat_instance->creating_maat_rt) {
                fprintf(stderr, "At initiation: NO effective rule in %s.",
                                maat_instance->iris_ctx.full_dir);
            }
            break;
+        case DATA_SOURCE_JSON_FILE:
+            ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str));
+            if (ret < 0) {
+                fprintf(stderr, "Maat re-initiate with JSON file %s failed: %s", maat_instance->json_ctx.json_file, err_str);
+                return;
+            }
+            
+            config_monitor_traverse(maat_instance->maat_version,
+                                    maat_instance->json_ctx.iris_file,
+                                    maat_start_cb, maat_update_cb, maat_finish_cb,
+                                    maat_instance);
+            if (NULL == maat_instance->creating_maat_rt) {
+                fprintf(stderr, "At initiation: NO effective rule in %s.",
+                                maat_instance->json_ctx.iris_file);
+            }
+            break;
        default:
            break;
    }
@@ -134,9 +204,17 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path)

    maat_instance->input_mode = opts->input_mode;
    switch (maat_instance->input_mode) {
+        case DATA_SOURCE_REDIS:
+            memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip, strlen(opts->redis_ctx.redis_ip));
+            maat_instance->mr_ctx.redis_port = opts->redis_ctx.redis_port;
+            maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db;
+            break;
        case DATA_SOURCE_IRIS_FILE:
-            memcpy(maat_instance->iris_ctx.full_dir, opts->iris_full_dir, strlen(opts->iris_full_dir));
-            memcpy(maat_instance->iris_ctx.inc_dir, opts->iris_inc_dir, strlen(opts->iris_inc_dir));
+            memcpy(maat_instance->iris_ctx.full_dir, opts->iris_ctx.full_dir, strlen(opts->iris_ctx.full_dir));
+            memcpy(maat_instance->iris_ctx.inc_dir, opts->iris_ctx.inc_dir, strlen(opts->iris_ctx.inc_dir));
+            break;
+        case DATA_SOURCE_JSON_FILE:
+            memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file));
            break;
        default:
            fprintf(stderr, "data source unsupported:%d\n", maat_instance->input_mode);
--- a/src/maat_command.cpp
+++ b/src/maat_command.cpp
@@ -0,0 +1,348 @@
+/*
+**********************************************************************************************
+*	File: maat_command.cpp
+*   Description:
+*	Authors: Liu WenTan <liuwentan@geedgenetworks.com>
+*	Date:    2022-10-31
+*   Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
+***********************************************************************************************
+*/
+
+#include <stdlib.h>
+#include <assert.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "utils.h"
+#include "maat_utils.h"
+#include "maat_command.h"
+#include "maat_rule.h"
+#include "hiredis/hiredis.h"
+#include "maat_config_monitor.h"
+#include "maat_table_schema.h"
+
+extern const char *foreign_source_prefix;
+extern const char *mr_key_prefix;
+
+extern const char *mr_expire_lock;
+extern const long mr_expire_lock_time;
+
+extern const char *mr_status_sset;
+extern const char *mr_version_sset;
+extern const char *mr_label_sset;
+
+redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...)
+{
+	va_list ap;
+	void *reply = NULL;
+	int ret = REDIS_ERR;
+    int retry = 0;
+
+	while (reply == NULL && retry < 2 && ret != REDIS_OK) {
+		va_start(ap,format);
+		reply = redisvCommand(c,format,ap);
+		va_end(ap);
+		if (NULL == reply) {
+			ret = redisReconnect(c);
+			retry++;
+		}
+	}
+
+	return (redisReply *)reply;
+}
+
+redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db)
+{
+	struct timeval connect_timeout;
+	connect_timeout.tv_sec = 0;
+	connect_timeout.tv_usec = 100 * 1000; // 100 ms
+
+	redisContext *c = redisConnectWithTimeout(redis_ip, redis_port, connect_timeout);
+	if (NULL == c || c->err) {
+		fprintf(stderr, "Unable to connect redis server %s:%d db%d, error: %s",
+						redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr);
+		
+		if (c != NULL) {
+            redisFree(c);
+        }
+		return NULL;
+	}
+
+	redisEnableKeepAlive(c);
+	redisReply *reply = maat_cmd_wrap_redis_command(c, "select %d", redis_db);
+	freeReplyObject(reply);
+	reply = NULL;
+
+	return c;
+}
+
+struct s_rule_array
+{
+	int cnt;
+	int size;
+	struct serial_rule *array;
+};
+
+void save_serial_rule(void *data, void *user)
+{
+	struct s_rule_array *array = (struct s_rule_array *)user;
+	int i = array->cnt;
+	memcpy(&(array->array[i]), data, sizeof(struct serial_rule));
+	array->array[i].op = MAAT_OP_ADD;
+}
+
+void maat_cmd_empty_serial_rule(struct serial_rule *s_rule)
+{
+	if (s_rule->table_line != NULL) {
+		FREE(s_rule->table_line);
+	}
+
+	if (s_rule->n_foreign > 0) {
+		for (int i = 0; i < s_rule->n_foreign; i++) {
+			FREE(s_rule->f_keys[i].filename);
+			FREE(s_rule->f_keys[i].key);
+		}
+
+		FREE(s_rule->f_keys);
+	}
+
+	memset(s_rule, 0, sizeof(struct serial_rule));
+}
+
+int connect_redis_for_write(struct source_redis_ctx *mr_ctx)
+{
+	assert(mr_ctx->write_ctx == NULL);
+	mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db);
+	if (NULL == mr_ctx->write_ctx) {
+		return -1;
+	} else {
+		return 0;
+	}
+}
+
+redisContext *get_redis_ctx_for_write(struct maat *maat_instance)
+{
+	if (NULL == maat_instance->mr_ctx.write_ctx) {
+		int ret = connect_redis_for_write(&(maat_instance->mr_ctx));
+		if(ret!=0)
+		{
+			return NULL;
+		}
+	}
+	return maat_instance->mr_ctx.write_ctx;
+}
+
+void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id, 
+                              const char *table_name, const char *line, long long timeout)
+{
+	memset(rule, 0, sizeof(struct serial_rule));
+    rule->op = op;
+	rule->rule_id = rule_id;
+	rule->timeout = timeout;
+	assert(strlen(table_name) < sizeof(rule->table_name));
+	strncpy(rule->table_name, table_name, sizeof(rule->table_name));
+	if (line != NULL) {
+		rule->table_line = maat_strdup(line);
+	}
+}
+
+int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq)
+{
+	int column_seq = 0;
+
+ 	switch (table_type) {
+		case TABLE_TYPE_EXPR:
+			column_seq = 7;
+			break;
+		case TABLE_TYPE_IP:
+			column_seq = 14;
+			break;
+		case TABLE_TYPE_IP_PLUS:
+			column_seq = 18;
+			break;
+		case TABLE_TYPE_COMPILE:
+			column_seq = 8;
+			break;
+		case TABLE_TYPE_PLUGIN:
+		case TABLE_TYPE_IP_PLUGIN:
+		case TABLE_TYPE_FQDN_PLUGIN:
+		case TABLE_TYPE_BOOL_PLUGIN:
+			if (valid_column_seq < 0) {
+				return -1;
+			}
+
+			column_seq = valid_column_seq;
+			break;
+		case TABLE_TYPE_INTERVAL:
+			column_seq = 5;
+			break;
+		case TABLE_TYPE_INTERVAL_PLUS:
+			column_seq = 6;
+			break;
+		case TABLE_TYPE_DIGEST:
+			column_seq = 6;
+			break;
+		case TABLE_TYPE_SIMILARITY:
+			column_seq = 5;
+			break;
+		case TABLE_TYPE_EXPR_PLUS:
+			column_seq = 8;
+			break;
+		case TABLE_TYPE_GROUP2COMPILE:
+		case TABLE_TYPE_GROUP2GROUP:
+			column_seq = 3;
+			break;
+		default:
+			assert(0);
+	}
+	
+    size_t offset = 0;
+    size_t len = 0;
+	int ret = get_column_pos(line, column_seq, &offset, &len);
+    // 0 is also a valid value for some non-MAAT producer.
+	if (ret < 0 || offset >= strlen(line) || (line[offset] != '1' && line[offset] != '0')) {
+		return -1;
+	}
+
+	return offset;
+}
+
+long long maat_cmd_redis_server_time_s(redisContext *c)
+{
+	long long server_time = 0;
+
+	redisReply *data_reply = maat_cmd_wrap_redis_command(c, "TIME"); 
+	if (data_reply->type == REDIS_REPLY_ARRAY) {
+		server_time = atoll(data_reply->element[0]->str);
+		freeReplyObject(data_reply);
+		data_reply = NULL;
+	}
+
+	return server_time;
+}
+
+const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len)
+{
+    size_t i = 0;
+	int j = 0;
+	size_t start=0, end=0;
+	size_t line_len = strlen(line);
+
+	for (i = 0; i < line_len; i++) {
+		if (line[i] != ' ' && line[i] != '\t') {
+			continue;
+		}
+
+		j++;
+		if (j == Nth - 1) {
+			start = i + 1;
+		}
+
+		if(j == Nth) {
+			end = i;
+			break;
+		}
+	}
+
+	if (start == end) {
+		return NULL;
+	}
+
+	if (end == 0) {
+		end = i;
+	}
+
+	*column_len = end - start;
+
+	return line + start;
+}
+
+long long maat_cmd_read_redis_integer(const redisReply *reply)
+{
+	switch (reply->type) {
+		case REDIS_REPLY_INTEGER:
+			return reply->integer;
+			break;
+		case REDIS_REPLY_ARRAY:
+			assert(reply->element[0]->type == REDIS_REPLY_INTEGER);
+			return reply->element[0]->integer;
+			break;
+		case REDIS_REPLY_STRING:
+			return atoll(reply->str);
+			break;
+		default:
+			return -1;
+			break;
+	}	
+	return 0;
+}
+
+int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply)
+{
+	return redisGetReply(c, (void **)reply);
+}
+
+int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule)
+{
+    int i = 0;
+	int ret = 0;
+	long long absolute_expire_time = 0;
+
+	redisContext *write_ctx = get_redis_ctx_for_write(maat_instance);
+	if (NULL == write_ctx) {
+		return -1;
+	}
+
+	long long server_time = maat_cmd_redis_server_time_s(write_ctx);
+	if(!server_time) {
+		return -1;
+	}
+
+	struct serial_rule *s_rule = ALLOC(struct serial_rule, 1);
+
+    int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name);
+    if (table_id < 0) {
+        fprintf(stderr, "Command set line id %d failed: unknown table %s.", line_rule->rule_id, line_rule->table_name);
+        FREE(s_rule);
+        return -1;
+    }
+
+    struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
+    if (!table_schema) {
+        FREE(s_rule);
+        return -1;
+    }
+
+    int valid_flag_column = table_schema_get_valid_flag_column(table_schema);
+    if (valid_flag_column < 0) {
+        fprintf(stderr, "Command set line id %d failed: table %s is not a plugin or ip_plugin table.",
+                        line_rule->rule_id, line_rule->table_name);
+        FREE(s_rule);
+        return -1;
+    }
+
+    enum table_type table_type = table_schema_get_table_type(table_schema);
+    int is_valid = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_flag_column);
+
+    if (line_rule->expire_after > 0) {
+        absolute_expire_time = server_time + line_rule->expire_after;
+    }
+
+    maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name,
+                             line_rule->table_line, absolute_expire_time);
+
+    int success_cnt = maat_cmd_exec_serial_rule(write_ctx, s_rule, 1, server_time);
+	if (success_cnt != 1) {
+		ret = -1;
+		goto error_out;
+	}
+
+	ret = success_cnt;
+	maat_instance->line_cmd_acc_num += success_cnt;
+
+error_out:
+	maat_cmd_empty_serial_rule(s_rule);
+	FREE(s_rule);
+
+	return ret;
+}
--- a/src/maat_config_monitor.cpp
+++ b/src/maat_config_monitor.cpp
@@ -14,10 +14,13 @@
 #include <limits.h>
 #include <sys/stat.h>
 #include <dirent.h>
+#include <unistd.h>

-#include "maat_config_monitor.h"
-#include "maat_utils.h"
 #include "utils.h"
+#include "maat_utils.h"
+#include "maat_rule.h"
+#include "json2iris.h"
+#include "maat_config_monitor.h"

 #define CM_MAX_TABLE_NUM 256
 #define	MAX_CONFIG_LINE	(1024 * 16)
@@ -346,3 +349,80 @@ void config_monitor_traverse(long long current_version, const char *idx_dir,
 	
 	free(idx_path_array);
 }
+
+int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz)
+{
+	int ret = 0;
+	struct stat fstat_buf;
+	unsigned char *json_buff = NULL; 
+    unsigned char *decrypted_buff = NULL; 
+    unsigned char *uncompressed_buff = NULL;
+	size_t json_buff_sz = 0;
+    size_t decrypted_buff_sz = 0; 
+    size_t uncompressed_buff_sz = 0;
+
+	fprintf(stdout, "Maat initial with JSON file %s, formating..", json_filename);
+
+	if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) {
+		ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo, 
+                            (unsigned char **)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz);
+		if (ret < 0) {
+			fprintf(stderr, "Decrypt Maat JSON file %s failed.", json_filename);
+			return -1;
+		}
+
+		json_buff=decrypted_buff;
+		json_buff_sz=decrypted_buff_sz;
+	}
+    
+	if (maat_instance->maat_json_is_gzipped) {
+		ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz);
+		free(json_buff);
+		if (ret < 0) {
+			fprintf(stderr, "Uncompress Maat JSON file %s failed.", json_filename);
+			return -1;
+		}
+
+		json_buff = uncompressed_buff;
+		json_buff_sz = uncompressed_buff_sz;
+	}
+
+    //decryption failed or no decryption
+	if (NULL == json_buff) {	
+		ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz);
+		if (ret < 0) {
+			fprintf(stderr, "Read Maat JSON file %s failed.", json_filename);
+			return -1;
+		}
+	}
+
+	ret = json2iris((const char*)json_buff, json_filename,
+			maat_instance->compile_tn, maat_instance->group2compile_tn, maat_instance->group2group_tn,
+			NULL,
+			maat_instance->json_ctx.iris_file,
+			sizeof(maat_instance->json_ctx.iris_file),
+			strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL,
+			strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL);
+
+	free(json_buff);
+	json_buff = NULL;
+	if (ret < 0) {
+		return -1;
+	}
+
+	if (!maat_instance->is_running) {
+		strncpy(maat_instance->json_ctx.json_file, json_filename, sizeof(maat_instance->json_ctx.json_file));
+	}
+	
+	ret=stat(json_filename, &fstat_buf);
+	maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim;
+	
+	md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5);
+	fprintf(stdout, "JSON file %s md5: %s, generate index file %s OK.",
+					maat_instance->json_ctx.json_file,
+					maat_instance->json_ctx.effective_json_md5,
+					maat_instance->json_ctx.iris_file);
+	maat_instance->input_mode = DATA_SOURCE_JSON_FILE;
+
+	return 0;	
+}
--- a/src/maat_redis_monitor.cpp
+++ b/src/maat_redis_monitor.cpp
--- a/src/maat_rule.cpp
+++ b/src/maat_rule.cpp
@@ -14,12 +14,16 @@
 #include <limits.h>
 #include <string.h>
 #include <sys/prctl.h>
+#include <sys/stat.h>
 #include <assert.h>

+#include "utils.h"
+#include "json2iris.h"
+#include "maat_utils.h"
 #include "maat_rule.h"
 #include "maat_config_monitor.h"
-#include "utils.h"
-#include "maat_utils.h"
+#include "maat_redis_monitor.h"
+
 #include "maat_table_runtime.h"
 #include "maat_table_schema.h"

@@ -100,7 +104,16 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param)
 	struct maat *maat_instance =(struct maat *)u_param; 
 	struct maat_runtime* maat_rt = NULL;
 	int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name);
+    if (table_id < 0) {
+        fprintf(stderr, "update warning, unknown table name %s\n", table_name);
+        return -1;
+    }
+
 	struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id);
+    if (NULL == table_schema) {
+        fprintf(stderr, "update warning, table name %s doesn't have table schema\n", table_name);
+        return -1;
+    }

 	if (maat_instance->creating_maat_rt != NULL) {
 		maat_rt = maat_instance->creating_maat_rt;
@@ -177,10 +190,21 @@ void *rule_monitor_loop(void *arg)
    }
    pthread_mutex_unlock(&(maat_instance->background_update_mutex));

+    char md5_tmp[MD5_DIGEST_LENGTH * 2 + 1] = {0};
+    char err_str[NAME_MAX] = {0};
+    struct stat attrib;
    while (maat_instance->is_running) {
        usleep(maat_instance->rule_update_checking_interval_ms * 1000);
 		if( 0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
 			switch (maat_instance->input_mode) {
+                case DATA_SOURCE_REDIS:
+                    redis_monitor_traverse(maat_instance->maat_version,
+                                            &(maat_instance->mr_ctx),
+                                            maat_start_cb,
+                                            maat_update_cb,
+                                            maat_finish_cb,
+                                            maat_instance);
+                    break;
 				case DATA_SOURCE_IRIS_FILE:
 					config_monitor_traverse(maat_instance->maat_version, 
 											maat_instance->iris_ctx.inc_dir,
@@ -189,6 +213,29 @@ void *rule_monitor_loop(void *arg)
 											maat_finish_cb,
 											maat_instance);
 					break;
+                case DATA_SOURCE_JSON_FILE:
+                    memset(md5_tmp, 0, sizeof(md5_tmp));
+				    stat(maat_instance->json_ctx.json_file, &attrib);
+					if (memcmp(&attrib.st_ctim, &(maat_instance->json_ctx.last_md5_time), sizeof(attrib.st_ctim))) {
+						maat_instance->json_ctx.last_md5_time = attrib.st_ctim;
+						md5_file(maat_instance->json_ctx.json_file, md5_tmp);
+						if (0 != strcmp(md5_tmp, maat_instance->json_ctx.effective_json_md5)) {
+							ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str));
+							if (ret < 0) {
+								fprintf(stdout, "Maat re-initiate with JSON file %s (md5=%s)failed: %s",
+												maat_instance->json_ctx.json_file, md5_tmp, err_str);
+							} else {
+								config_monitor_traverse(0, maat_instance->json_ctx.iris_file,
+														maat_start_cb,
+														maat_update_cb,
+														maat_finish_cb,
+														maat_instance);
+								fprintf(stdout, "Maat re-initiate with JSON file %s success, md5: %s",
+												maat_instance->json_ctx.json_file, md5_tmp);
+							}
+						}
+					}
+                    break;
 				default:
 					break;
 			}
@@ -231,7 +278,20 @@ void *rule_monitor_loop(void *arg)
    maat_runtime_destroy(maat_instance->maat_rt);
 	maat_garbage_bin_free(maat_instance->garbage_bin);
 	table_schema_manager_destroy(maat_instance->table_schema_mgr);
-    free(maat_instance);
+
+    if (maat_instance->input_mode == DATA_SOURCE_REDIS) {
+        if (maat_instance->mr_ctx.read_ctx != NULL) {
+            redisFree(maat_instance->mr_ctx.read_ctx);
+            maat_instance->mr_ctx.read_ctx = NULL;
+        }
+
+        if (maat_instance->mr_ctx.write_ctx != NULL) {
+            redisFree(maat_instance->mr_ctx.write_ctx);
+            maat_instance->mr_ctx.write_ctx = NULL;
+        }
+    }
+
+    FREE(maat_instance);

    return NULL;
 }
--- a/src/maat_table_runtime.cpp
+++ b/src/maat_table_runtime.cpp
@@ -36,7 +36,7 @@ struct plugin_user_ctx {
 };

 struct expr_runtime {
-    enum scan_mode scan_mode;
+    enum hs_scan_mode scan_mode;
    struct adapter_hs *hs;
    struct adapter_hs_stream *hs_stream;
    struct rcu_hash_table *htable;
--- a/src/maat_table_schema.cpp
+++ b/src/maat_table_schema.cpp
@@ -33,7 +33,7 @@ struct expr_table_schema {
    int match_method_column;
    int is_hexbin_column;
 	int is_valid_column;  /* valid means add, invalid means delete */
-    enum scan_mode scan_mode; /* adapter_hs scan mode */
+    enum hs_scan_mode scan_mode; /* adapter_hs scan mode */
 };

 #define	MAX_PLUGIN_PER_TABLE 32
@@ -60,6 +60,10 @@ struct ip_plugin_table_schema {
 	struct ex_data_schema ex_schema;
 };

+struct virtual_table_schema {
+	int physical_table_id[SCAN_TYPE_MAX];
+};
+
 struct table_schema {
    int table_id;
    char table_name[NAME_MAX];
@@ -68,6 +72,7 @@ struct table_schema {
        struct expr_table_schema expr;
        struct plugin_table_schema plugin;
        struct ip_plugin_table_schema ip_plugin;
+        struct virtual_table_schema virtual_table;
    };
 };

@@ -388,6 +393,10 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
    unsigned char *json_buff = NULL;
    size_t json_buff_sz = 0;

+    if (NULL == table_info_path) {
+        return NULL;
+    }
+
    int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz);
    if (ret < 0) {
        fprintf(stderr, "Maat read table info %s error.\n", table_info_path);
@@ -417,8 +426,8 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_
    maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN);
    maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN);

-    maat_kv_register(reserved_word_map, "block", SCAN_MODE_BLOCK);
-    maat_kv_register(reserved_word_map, "stream", SCAN_MODE_STREAM);
+    maat_kv_register(reserved_word_map, "block", HS_SCAN_MODE_BLOCK);
+    maat_kv_register(reserved_word_map, "stream", HS_SCAN_MODE_STREAM);

    struct table_schema_manager *table_schema_mgr = ALLOC(struct table_schema_manager, 1);
    struct table_schema **pptable = table_schema_mgr->schema_table;
@@ -471,6 +480,10 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_

 void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr)
 {
+    if (NULL == table_schema_mgr) {
+        return;
+    }
+
 	for (size_t i = 0; i < MAX_TABLE_NUM; i++) {
 		if (NULL == table_schema_mgr->schema_table[i]) {
 			continue;
@@ -480,11 +493,15 @@ void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr)
 	}

 	maat_kv_store_free(table_schema_mgr->tablename2id_map);
-	free(table_schema_mgr);
+	FREE(table_schema_mgr);
 }

 int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name)
 {
+    if (NULL == table_schema_mgr || NULL == table_name) {
+        return -1;
+    }
+
    int table_id = -1;

 	int ret = maat_kv_read(table_schema_mgr->tablename2id_map, table_name, &table_id);
@@ -497,6 +514,10 @@ int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_

 enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int id)
 {
+    if (NULL == table_schema_mgr) {
+        return TABLE_TYPE_MAX;
+    }
+
    if (table_schema_mgr->schema_table[id] == NULL) {
        return TABLE_TYPE_MAX;
    }
@@ -511,6 +532,10 @@ size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_m

 void table_schema_manager_all_plugin_cb_start(struct table_schema_manager *table_schema_mgr, int update_type)
 {
+    if (NULL == table_schema_mgr) {
+        return;
+    }
+
    struct table_schema *ptable = NULL;
    struct plugin_table_schema *plugin_schema = NULL;

@@ -536,6 +561,10 @@ void table_schema_manager_all_plugin_cb_start(struct table_schema_manager *table

 void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* table_schema_mgr)
 {
+    if (NULL == table_schema_mgr) {
+        return;
+    }
+
    struct table_schema *ptable = NULL;
    struct plugin_table_schema *plugin_schema = NULL;

@@ -606,19 +635,149 @@ enum match_method int_to_match_method_type(int match_method_type)

 struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id)
 {
+    if ((NULL == table_schema_mgr) || (table_id < 0)) {
+        return NULL;
+    }
+
    return table_schema_mgr->schema_table[table_id];
 }

+struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr, 
+                                                   int table_id, enum scan_type scan_type, int *virtual_table_id)
+{
+    enum scan_type table_scan_type;
+	struct table_schema **pptable = table_schema_mgr->schema_table;
+	size_t n_table = MAX_TABLE_NUM;
+
+	if ((unsigned int)table_id > n_table) {
+		return NULL;
+	}
+
+	if (NULL == pptable[table_id]) {
+		return NULL;
+	}
+
+	struct table_schema *ptable = pptable[table_id];
+	if (NULL == ptable) {
+		return NULL;
+	}
+
+    struct table_schema *p_physical_table = NULL;
+	if (ptable->table_type == TABLE_TYPE_VIRTUAL) {
+		p_physical_table = pptable[ptable->virtual_table.physical_table_id[scan_type]];
+		*virtual_table_id = table_id;
+	} else {
+		p_physical_table = ptable;
+		if(virtual_table_id) {
+            *virtual_table_id = 0;
+        }
+	}
+
+	table_scan_type = table_schema_get_scan_type(p_physical_table);
+	if (table_scan_type != scan_type) {
+			return NULL;	
+	}
+
+	return p_physical_table;
+}
+
 enum table_type table_schema_get_table_type(struct table_schema *table_schema)
 {
+    if (NULL == table_schema) {
+        return TABLE_TYPE_MAX;
+    }   
+
    return table_schema->table_type;
 }

 int table_schema_get_table_id(struct table_schema *table_schema)
 {
+    if (NULL == table_schema) {
+        return -1;
+    }
+
    return table_schema->table_id;
 }

+enum scan_type table_schema_get_scan_type(struct table_schema *table_schema)
+{
+    enum scan_type ret = SCAN_TYPE_INVALID;
+
+    if (NULL == table_schema) {
+        return ret;
+    }
+
+	switch (table_schema->table_type) {
+		case TABLE_TYPE_EXPR:
+		case TABLE_TYPE_EXPR_PLUS:
+		case TABLE_TYPE_SIMILARITY:
+		case TABLE_TYPE_DIGEST:
+			ret = SCAN_TYPE_STRING;
+			break;
+		case TABLE_TYPE_INTERVAL:
+		case TABLE_TYPE_INTERVAL_PLUS:
+			ret = SCAN_TYPE_INTERVAL;
+			break;
+		case TABLE_TYPE_IP:
+		case TABLE_TYPE_IP_PLUS:
+		case TABLE_TYPE_COMPOSITION:
+			ret = SCAN_TYPE_IP;
+			break;
+		case TABLE_TYPE_PLUGIN:
+			ret = SCAN_TYPE_PLUGIN;
+			break;
+		case TABLE_TYPE_IP_PLUGIN:
+			ret = SCAN_TYPE_IP;
+			break;
+		case TABLE_TYPE_FQDN_PLUGIN:
+			ret = SCAN_TYPE_FQDN_PLUGIN;
+			break;
+		case TABLE_TYPE_BOOL_PLUGIN:
+			ret = SCAN_TYPE_BOOL_PLUGIN;
+			break;
+		case TABLE_TYPE_COMPILE:			
+			ret = SCAN_TYPE_NONE;
+			break;
+		default:
+			break;
+	}
+
+	return ret;
+}
+
+int table_schema_get_valid_flag_column(struct table_schema *table_schema)
+{
+	int valid_flag_column = -1;
+
+    if (NULL == table_schema) {
+        return valid_flag_column;
+    }
+
+	switch (table_schema->table_type) {
+        case TABLE_TYPE_EXPR:
+            valid_flag_column = table_schema->expr.is_valid_column;
+            break;
+		case TABLE_TYPE_PLUGIN:
+			valid_flag_column = table_schema->plugin.is_valid_column;
+			break;
+		case TABLE_TYPE_IP_PLUGIN:
+			valid_flag_column = table_schema->ip_plugin.is_valid_column;
+			break;
+            /*
+		case TABLE_TYPE_FQDN_PLUGIN:
+			valid_flag_column = table_schema->fqdn_plugin.valid_flag_column;
+			break;
+		case TABLE_TYPE_BOOL_PLUGIN:
+			valid_flag_column = table_schema->bool_plugin.valid_flag_column;
+			break;*/
+		default:
+			valid_flag_column = -1;
+			break;
+	}
+
+	return valid_flag_column;
+}
+
 int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, struct expr_item *expr_item)
 {
    size_t column_offset = 0;
@@ -778,6 +937,10 @@ int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schem
 struct table_item *
 table_schema_line_to_item(const char *line, struct table_schema *table_schema)
 {
+    if (NULL == line || NULL == table_schema) {
+        return NULL;
+    }
+
    int ret = -1;
    struct table_item *table_item = ALLOC(struct table_item, 1);

@@ -812,10 +975,14 @@ error:
    return NULL;
 }

-enum scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema)
+enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema)
 {
+    if (NULL == table_schema) {
+        return HS_SCAN_MODE_MAX;
+    }
+
    if (table_schema->table_type != TABLE_TYPE_EXPR) {
-        return SCAN_MODE_MAX;
+        return HS_SCAN_MODE_MAX;
    }

    return table_schema->expr.scan_mode;
@@ -827,7 +994,7 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,
 									        maat_plugin_ex_dup_func_t *dup_func,
 									        long argl, void *argp)
 {
-    if (NULL == new_func || NULL == free_func || NULL == dup_func) {
+    if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) {
        assert(0);
        fprintf(stderr, "%s failed: invalid parameter", __FUNCTION__);
        return -1;
@@ -857,6 +1024,10 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema,

 struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schema *table_schema)
 {
+    if (NULL == table_schema) {
+        return NULL;
+    }
+
    struct ex_data_schema *ex_schema = NULL;

    switch (table_schema->table_type) {
@@ -875,6 +1046,10 @@ struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schem

 int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema)
 {
+    if (NULL == table_schema) {
+        return -1;
+    }
+
    struct ex_data_schema *ex_schema = NULL;

    switch (table_schema->table_type) {
@@ -897,6 +1072,10 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m
 							         maat_finish_callback_t *finish,
 							         void *u_para)
 {
+    if ((NULL == table_schema_mgr) || (table_id < 0)) {
+        return -1;
+    }
+
 	struct table_schema *ptable = table_schema_get(table_schema_mgr, table_id);
    if (NULL == ptable) {
        fprintf(stderr, "table_id:%d unregistered, can't add callback func", table_id);
@@ -945,6 +1124,10 @@ size_t plugin_table_schema_callback_count(struct table_schema *table_schema)

 void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const char *row)
 {
+    if (NULL == table_schema || NULL == row) {
+        return;
+    }
+
    struct plugin_table_schema *plugin_schema = NULL;

    switch (table_schema->table_type) {
@@ -964,3 +1147,21 @@ void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const
            break;
    }
 }
+
+int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns)
+{
+    if (NULL == table_schema) {
+        return -1;
+    }
+
+    if (table_schema->table_type != TABLE_TYPE_PLUGIN) {
+        return 0;
+    }
+
+    int n_foreign = table_schema->plugin.n_foreign;
+    for (int i = 0; i < n_foreign; i++) {
+        foreign_columns[i] = table_schema->plugin.foreign_columns[i];
+    }
+
+    return n_foreign;
+}
--- a/src/maat_utils.cpp
+++ b/src/maat_utils.cpp
@@ -11,6 +11,9 @@
 #include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <zlib.h>
+#include <openssl/md5.h>
+#include <openssl/evp.h>

 #include "maat_utils.h"

@@ -19,6 +22,7 @@ char *maat_strdup(const char *s)
 	if (NULL == s) {
 		return NULL;
 	}
+
    char *d = (char *)malloc(strlen(s) + 1);
 	memcpy(d, s, strlen(s) + 1);

@@ -27,10 +31,14 @@ char *maat_strdup(const char *s)

 int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len)
 {
-	const char *seps=" \t";
-	char *saveptr=NULL, *subtoken=NULL, *str=NULL;
+	int i = 0;
+    int ret = -1;
+    char *str = NULL;
+	char *saveptr = NULL;
+    char *subtoken = NULL;
+    const char *seps = " \t";
 	char *dup_line = maat_strdup(line);
-	int i = 0, ret = -1;
+	
 	for (str = dup_line; ; str = NULL) {
 		subtoken = strtok_r(str, seps, &saveptr);
 		if (subtoken == NULL)
@@ -53,7 +61,7 @@ int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *o
 	int ret = 0;
 	FILE *fp = NULL;		
 	struct stat fstat_buf;
-	size_t read_size=0;
+	size_t read_size = 0;
 	
 	ret = stat(file_name, &fstat_buf);
 	if (ret != 0) {
@@ -187,3 +195,180 @@ int system_cmd_mkdir(const char *path)
 	snprintf(cmd, sizeof(cmd), "mkdir -p %s", path);
 	return system(cmd);
 }
+
+char *md5_file(const char *filename, char *md5string)
+{
+	unsigned char md5[MD5_DIGEST_LENGTH] = {0};
+	struct stat file_info;	
+	stat(filename, &file_info);
+	size_t file_size = file_info.st_size;
+
+	FILE *fp = fopen(filename,"r");
+	if (NULL == fp) {
+		return NULL;
+	}
+
+	char *file_buff = (char *)malloc(file_size);
+	fread(file_buff, 1, file_size, fp);
+	fclose(fp);
+	
+	MD5((const unsigned char *)(file_buff), (unsigned long)(file_size), md5);
+	for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
+		sprintf(&md5string[i*2], "%02x", (unsigned int)md5[i]);
+	}
+
+	free(file_buff);
+	return md5string;
+}
+
+int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz, 
+                 const char *key, const char *algorithm, int do_encrypt, 
+                 char *err_str, size_t err_str_sz)
+{
+	OpenSSL_add_all_algorithms();
+	const EVP_CIPHER *cipher = EVP_get_cipherbyname(algorithm);
+	if (NULL == cipher) {
+		snprintf(err_str, err_str_sz, "Cipher %s is not supported.", algorithm);
+		return 0;
+	}
+
+	const EVP_MD *dgst = EVP_get_digestbyname("md5");
+	if (NULL == dgst) {
+		snprintf(err_str, err_str_sz, "Get MD5 object failed.");
+		return 0;
+	}
+
+    const unsigned char *salt = NULL;
+    unsigned char cipher_key[EVP_MAX_KEY_LENGTH];
+	unsigned char cipher_iv[EVP_MAX_IV_LENGTH];
+
+	memset(cipher_key,0,sizeof(cipher_key));
+	memset(cipher_iv,0,sizeof(cipher_iv));
+
+	int ret = EVP_BytesToKey(cipher, dgst, salt, (unsigned char *)key, 
+                             strlen((const char *)key), 1, cipher_key, cipher_iv);
+	if(0 == ret) {
+		snprintf(err_str, err_str_sz, "Key and IV generatioin failed.");
+		return 0;
+	}
+
+    /* Don't set key or IV right away; we want to check lengths */
+	EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
+	EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, do_encrypt);
+	OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16 == 0);
+	OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16);
+	 
+	/* Now we can set key and IV */
+	//It should be set to 1 for encryption, 0 for decryption and -1 to leave the value unchanged (the actual value of 'enc' being supplied in a previous call).
+	EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, -1);
+    int out_blk_len = 0;
+    int out_buff_offset = 0;
+	int out_buff_len = inlen + EVP_CIPHER_block_size(cipher) - 1;
+	*pp_out = (unsigned char *)malloc(out_buff_len * sizeof(unsigned char));
+	if (!EVP_CipherUpdate(ctx, *pp_out + out_buff_offset, &out_blk_len, inbuf, inlen)) {
+		snprintf(err_str, err_str_sz, "EVP_CipherUpdate failed.");
+		EVP_CIPHER_CTX_free(ctx);
+		goto error_out;
+	}
+
+	out_buff_offset += out_blk_len;
+	if (!EVP_CipherFinal_ex(ctx, *pp_out+out_buff_offset, &out_blk_len)) {
+		snprintf(err_str, err_str_sz, "EVP_CipherFinal_ex failed. Maybe password is wrong?");
+		EVP_CIPHER_CTX_free(ctx);
+		goto error_out;
+	}
+
+	out_buff_offset += out_blk_len;
+	EVP_CIPHER_CTX_free(ctx);
+	*out_sz = out_buff_offset;
+	return 0;	
+	
+error_out:	
+	free(*pp_out);
+	*pp_out = NULL;
+	return -1;
+}
+
+int decrypt_open(const char* file_name, const char* key, const char* algorithm, 
+                 unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz)
+{
+	size_t file_sz = 0;
+	unsigned char *file_buff = NULL;
+	int ret = load_file_to_memory(file_name, &file_buff, &file_sz);
+	if (ret < 0) {
+		return -1;
+	}
+
+	ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz);
+	free(file_buff);
+	file_buff = NULL;
+
+	return ret;
+}
+
+int gzip_uncompress_one_try(const unsigned char *in_compressed_data, size_t in_compressed_sz, 
+                            unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz)
+{
+	z_stream strm;
+	strm.zalloc = NULL;
+	strm.zfree = NULL;
+	strm.opaque = NULL;
+
+	strm.avail_in = in_compressed_sz;
+	strm.avail_out = *out_uncompressed_sz;
+	strm.next_in = (Bytef *) in_compressed_data;
+	strm.next_out = *out_uncompressed_data;
+
+	int ret = -1;
+	ret = inflateInit2(&strm, MAX_WBITS+16);
+	if (ret == Z_OK) {
+		ret = inflate(&strm, Z_FINISH);
+		if (ret == Z_STREAM_END) {
+			*out_uncompressed_sz = strm.total_out;
+			ret = inflateEnd(&strm);
+			return ret;
+		}
+	}
+	inflateEnd(&strm);
+	return ret;
+}
+
+int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, 
+                    unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz)
+{
+	int z_result;
+	int ret = -1;
+	size_t buffer_sz = in_compressed_sz * 2;	
+	*out_uncompressed_data = (unsigned char *)malloc(buffer_sz);
+
+	do {
+		*out_uncompressed_sz=buffer_sz;
+		z_result = gzip_uncompress_one_try(in_compressed_data, in_compressed_sz,
+			                                out_uncompressed_data, out_uncompressed_sz);
+		switch (z_result) {
+			case Z_OK:
+				ret = 0;
+				break;			
+			case Z_BUF_ERROR:
+				buffer_sz *= 2;
+				*out_uncompressed_data = (unsigned char *)realloc(*out_uncompressed_data, buffer_sz);
+				break;
+			default:
+				ret = -1;
+				break;
+		}
+	} while (z_result == Z_BUF_ERROR);
+
+	return ret;
+}
+
+size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_t n_src)
+{
+	if (*n_dest < offset + n_src) {
+		*n_dest = (offset + n_src) * 2;
+		*dest = realloc(*dest, sizeof(char) * (*n_dest));
+	}
+	memcpy((char *) * dest + offset, src, n_src);
+    
+	return n_src;
+}
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -14,3 +14,4 @@ target_link_libraries(maat_framework_gtest maat_frame_static gtest_static)
 file(COPY rule DESTINATION ./)
 file(COPY table_info.conf DESTINATION ./)
 file(COPY and_expr.conf DESTINATION ./)
+file(COPY maat_json.json DESTINATION ./)
--- a/test/maat_framework_gtest.cpp
+++ b/test/maat_framework_gtest.cpp
@@ -5,11 +5,14 @@
 #include "maat_utils.h"
 #include "maat_table_schema.h"
 #include "maat_table_runtime.h"
+#include "maat_command.h"

 struct maat *g_maat_instance = NULL;
 const char *table_info_path = "./table_info.conf";
 const char *rule_full_path = "./rule/full/index";
 const char *rule_inc_path = "./rule/inc/index";
+const char *json_path="./maat_json.json";
+const char *iris_file = "./HTTP_URL.000001";

 TEST(maat_scan_string, hit_one_expr) {
    struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
@@ -31,20 +34,22 @@ TEST(maat_scan_string, hit_one_expr) {
    EXPECT_EQ(n_result_array, 1);
    EXPECT_EQ(result_array[0], 101);

+/*
    memset(result_array, 0, sizeof(result_array));
    char data3[64] = "maat";
    ret = maat_scan_string(g_maat_instance, table_id, 0, data3, strlen(data3), result_array, &n_result_array, NULL);
    EXPECT_EQ(ret, 0);
    EXPECT_EQ(n_result_array, 1);
-    EXPECT_EQ(result_array[0], 102);
+    EXPECT_EQ(result_array[0], 102); */

+/*
    memset(result_array, 0, sizeof(result_array));
    char data4[64] = "world";
    ret = maat_scan_string(g_maat_instance, table_id, 0, data4, strlen(data4), result_array, &n_result_array, NULL);
    EXPECT_EQ(n_result_array, 1);
-    EXPECT_EQ(result_array[0], 103);
+    EXPECT_EQ(result_array[0], 103); */
 }
-
+#if 0
 TEST(maat_scan_string, hit_two_expr) {
    struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
    int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
@@ -159,14 +164,82 @@ TEST(maat_scan_string, config_dynamic_update) {
    EXPECT_EQ(result_array[4], 101);
 }

+
+TEST(maat_scan_string, hit_one_expr) {
+    struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
+    int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
+   
+    char data[128] = "i.ytimg.com";
+    int result_array[5] = {0};
+    size_t n_result_array = 0;
+    int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
+    EXPECT_EQ(ret, 0);
+    EXPECT_EQ(n_result_array, 1);
+    EXPECT_EQ(result_array[0], 30);
+}
+
+TEST(maat_scan_string, hit_two_expr) {
+    struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
+    int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
+   
+    char data[128] = "should hit aaa bbb";
+    int result_array[5] = {0};
+    size_t n_result_array = 0;
+    int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
+    EXPECT_EQ(ret, 0);
+    EXPECT_EQ(n_result_array, 2);
+    EXPECT_EQ(result_array[0], 28);
+    EXPECT_EQ(result_array[1], 27);
+}
+
+TEST(maat_scan_string, hit_three_expr) {
+    struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr;
+    int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL");
+   
+    char data[128] = "should hit aaa bbb C#中国";
+    int result_array[5] = {0};
+    size_t n_result_array = 0;
+    int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL);
+    EXPECT_EQ(ret, 0);
+    EXPECT_EQ(n_result_array, 3);
+    EXPECT_EQ(result_array[0], 28);
+    EXPECT_EQ(result_array[1], 27);
+    EXPECT_EQ(result_array[2], 18);
+}
+#endif
+TEST(maat_redis, write) {
+    char line[1024] = {0};
+    char table_name[64] = "HTTP_URL";
+    struct maat_cmd_line rule_line;
+    rule_line.table_name = (char *)malloc(sizeof(char) * 64);
+
+    memcpy((void *)rule_line.table_name, table_name, strlen(table_name));
+    FILE *fp = fopen(iris_file, "r");
+    EXPECT_NE(fp, nullptr);
+
+    int rule_num = 0;
+    fscanf(fp, "%d\n", &rule_num);
+
+    while (NULL != fgets(line, sizeof(line), fp)) {
+        rule_line.table_line = (char *)malloc(sizeof(line));
+        memcpy((void *)rule_line.table_line, line, strlen(line));
+        maat_cmd_set_line(g_maat_instance, &rule_line);
+        break;
+    }
+
+}
+
 int main(int argc, char ** argv)
 {
 	int ret=0;
 	::testing::InitGoogleTest(&argc, argv);

    struct maat_options *opts = maat_options_new();
-    maat_options_set_iris_full_dir(opts, rule_full_path);
-    maat_options_set_iris_inc_dir(opts, rule_inc_path);
+    //maat_options_set_iris_full_dir(opts, rule_full_path);
+    //maat_options_set_iris_inc_dir(opts, rule_inc_path);
+    maat_options_set_redis_ip(opts, "127.0.0.1");
+    maat_options_set_redis_port(opts, 6379);
+    //maat_options_set_json_file(opts, json_path);

    g_maat_instance = maat_new(opts, table_info_path);
    EXPECT_NE(g_maat_instance, nullptr);
--- a/test/maat_json.json
+++ b/test/maat_json.json
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -0,0 +1,8 @@
+include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal)
+include_directories(${PROJECT_SOURCE_DIR}/deps)
+include_directories(${PROJECT_SOURCE_DIR}/scanner)
+
+add_executable(maat_redis_tool maat_redis_tool.cpp)
+target_link_libraries(maat_redis_tool maat_frame_static)
+
+install(TARGETS maat_redis_tool DESTINATION /usr/local/bin/ COMPONENT TOOLS)
--- a/tools/maat_redis_tool.cpp
+++ b/tools/maat_redis_tool.cpp
@@ -0,0 +1,394 @@
+#include <unistd.h>
+#include <stdio.h>
+#include <errno.h>
+#include <assert.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "utils.h"
+#include "maat_rule.h"
+#include "maat_utils.h"
+#include "maat_command.h"
+#include "cJSON/cJSON.h"
+#include "maat_config_monitor.h"
+#include "json2iris.h"
+#include "hiredis/hiredis.h"
+
+const char *redis_dump_dir = "./redis_dump";
+const char *default_table_info = "./table_info.conf";
+
+void maat_tool_print_usage(void)
+{
+	printf("maat_redis_tool manipulate rules from redis.\n");
+	printf("Usage:\n");
+	printf("\t-h [host], redis IP, 127.0.0.1 as default.\n");
+	printf("\t-p [port], redis port, 6379 as default.\n");
+	printf("\t-n [db], redis db, 0 as default.\n");
+	printf("\t-d [dir], dump rules from redis to [dir], %s as default.\n",redis_dump_dir);
+	printf("\t-v [version], dump specific [version] from redis, dump latest version as default.\n");
+	printf("\t-j [payload.json], add or delete rules as maat json. Must have field compile_table field, and plugin table's valid flag must be in the last column.\n");
+	printf("\t-t [timeout], timeout config after t seconds, default is 0 which means never timeout.\n");	
+	printf("example: ./maat_redis_tool -h 127.0.0.1 -p 6379 -d %s\n",redis_dump_dir);
+	printf("	./maat_redis_tool -h 127.0.0.1 -p 6379 -j payload.json -t 300\n");
+}
+
+static int compare_serial_rule(const void *a, const void *b)
+{
+	struct serial_rule *ra=(struct serial_rule *)a;
+	struct serial_rule *rb=(struct serial_rule *)b;
+
+	int ret = strcmp(ra->table_name, rb->table_name);
+	if (0 == ret) {
+		ret = ra->rule_id - rb->rule_id;
+	}
+
+	return ret;
+}
+
+int set_file_rulenum(const char *path, int rule_num)
+{
+	FILE* fp=NULL;
+	if (0 == rule_num) {
+		fp = fopen(path, "w");
+	} else {
+		fp = fopen(path, "r+");
+	}
+
+	if (NULL == fp) {
+		fprintf(stderr, "fopen %s failed %s at set rule num.", path, strerror(errno));
+		return -1;
+	}
+
+	fprintf(fp, "%010d\n", rule_num);
+	fclose(fp);
+
+	return 0;
+}
+
+void read_rule_from_redis(redisContext *c, long long desire_version, const char* output_path)
+{
+	struct serial_rule *rule_list;
+	int line_count=0;
+	int i=0,ret=0;
+	int update_type = CM_UPDATE_TYPE_INC;
+	long long version=0;
+	const char* cur_table=NULL;
+    char foreign_files_dir[256] = {0};
+	char table_path[256],index_path[256];
+	FILE *table_fp=NULL, *index_fp=NULL;
+
+	int rule_num = maat_cmd_get_rm_key_list(c, 0, desire_version, &version, NULL, &rule_list, &update_type, 0);
+	if (0 == rule_num) {
+		if (desire_version != 0) {
+			printf("Read desired version %lld failed.\n", desire_version);
+		} else {
+			printf("No Effective Rules.\n");
+		}
+		return;
+	}
+
+	if (rule_num < 0) {
+		printf("Read Redis Error.\n");
+		return;
+	}
+
+	assert(update_type == CM_UPDATE_TYPE_FULL);
+	printf("MAAT Version: %lld, key number: %d\n", version, rule_num);
+	if (0 == rule_num) {
+		goto clean_up;
+	}
+
+	printf("Reading value: \n");
+	ret = maat_cmd_get_redis_value(c, rule_list, rule_num, 1);
+	if (ret < 0) {
+		goto clean_up;
+	}
+
+	printf("Sorting.\n");
+	qsort(rule_list, rule_num, sizeof(struct serial_rule), compare_serial_rule);
+	if ((access(output_path, F_OK)) <0) {	
+        if ((mkdir(output_path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) < 0) {
+            printf("mkdir %s error\n", output_path);
+        }
+	}
+
+	snprintf(foreign_files_dir, sizeof(foreign_files_dir), "%s/foreign_files/",output_path);
+
+	if ((access(foreign_files_dir, F_OK)) <0) {	
+        if((mkdir(foreign_files_dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) < 0) {
+            printf("mkdir %s error\n", foreign_files_dir);
+        }
+	}
+
+	ret = maat_cmd_get_foreign_keys_by_prefix(c, rule_list, rule_num, foreign_files_dir);
+	if (ret > 0) {
+		printf("%d lines has foreign content.\n", ret);
+		maat_cmd_get_foreign_conts(c, rule_list, rule_num, 1);
+	}
+
+	snprintf(index_path,sizeof(index_path),"%s/full_config_index.%020lld",output_path,version);
+	index_fp = fopen(index_path, "w");
+	if (NULL == index_fp) {
+		printf("Open %s failed.\n",index_path);
+		goto clean_up;
+	}
+
+	for (i = 0; i < rule_num; i++) {
+		if (rule_list[i].n_foreign > 0) {
+			maat_cmd_rewrite_table_line_with_foreign(rule_list+i);
+		}
+
+		if (NULL == cur_table || 0 != strcmp(cur_table,rule_list[i].table_name)) {
+			if (table_fp != NULL) {
+				fprintf(index_fp, "%s\t%d\t%s\n", cur_table, line_count, table_path);
+				fclose(table_fp);
+				table_fp = NULL;
+				set_file_rulenum(table_path, line_count);
+				line_count = 0;
+				printf("Written table %s\n",table_path);
+			}
+
+			snprintf(table_path, sizeof(table_path), "%s/%s.%020lld", output_path, 
+                                                     rule_list[i].table_name, version);
+			set_file_rulenum(table_path, 0);
+
+			table_fp = fopen(table_path, "a");
+			if (NULL == table_fp) {
+				printf("Open %s failed.\n", table_path);
+				goto clean_up;
+			}
+			
+			cur_table = rule_list[i].table_name;
+		}
+		fprintf(table_fp, "%s\tkey=%ld\n", rule_list[i].table_line, rule_list[i].rule_id);
+		line_count++;
+	}
+
+	fclose(table_fp);
+	table_fp = NULL;
+
+	fprintf(index_fp, "%s\t%d\t%s\n", cur_table, line_count, table_path);
+	set_file_rulenum(table_path, line_count);
+	printf("Written table %s\n", table_path);
+	printf("Written complete: %s\n", index_path);
+
+clean_up:
+	for (i = 0; i < rule_num; i++) {
+		maat_cmd_empty_serial_rule(rule_list+i);
+	}
+
+	free(rule_list);
+	rule_list = NULL;
+
+	if (c != NULL) {
+		redisFree(c);
+	}
+
+	if (index_fp != NULL) {
+		fclose(index_fp);
+	}
+
+	if (table_fp != NULL) {
+		fclose(table_fp);
+	}
+}
+
+int count_line_num(const char *table_name,const char *line, void *u_para)
+{
+	(*((unsigned int *)u_para))++;
+	return 0;
+}
+
+int line_idx = 0;
+long long absolute_expire_time=0;
+int make_serial_rule(const char *table_name, const char *line,void *u_para)
+{
+	struct serial_rule *s_rule=(struct serial_rule *)u_para;
+	int rule_id=0,is_valid=0;
+	char *buff = ALLOC(char, strlen(line) + 1);
+    
+    memcpy(buff, line, strlen(line) + 1);
+
+	while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') {
+		buff[strlen(buff) - 1] = '\0';
+	}
+
+    int j = 0;
+    char *str1 = NULL;
+    char *token = NULL;
+    char *saveptr1 = NULL;
+    char *last1 =NULL;
+	for (j = 0,str1 = buff; ; j++, str1 = NULL) {
+		 token = strtok_r(str1, "\t ", &saveptr1);
+		 if (token == NULL)
+			 break;
+		 if(j==0)
+		 {
+			sscanf(token,"%d",&rule_id);
+		 }
+		 last1=token;
+ 	 }
+	sscanf(last1,"%d",&is_valid);
+	
+	memcpy(buff,line, strlen(line)+1);
+	while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t')
+	{
+		buff[strlen(buff)-1]='\0';
+	}
+	//printf("Writing table %s, rule_id=%d, timeout=%lld, is_valid=%d\n", table_name, rule_id,absolute_expire_time,is_valid);
+	maat_cmd_set_serial_rule(s_rule + line_idx, (enum maat_operation)is_valid, rule_id, table_name, buff, absolute_expire_time);
+	line_idx++;
+	free(str1);
+
+	return 0;
+}
+
+#define	WORK_MODE_DUMP	0
+#define WORK_MODE_JSON	1
+int main(int argc, char * argv[])
+{
+	int oc=0,ret=0;
+	int model=0;
+	char redis_ip[64];
+	int redis_port=6379;
+	int redis_db=0;
+	strncpy(redis_ip, "127.0.0.1", sizeof(redis_ip));
+	char dump_dir[128], json_file[128], tmp_iris_path[128];
+	strncpy(dump_dir,redis_dump_dir,sizeof(dump_dir));
+	redisContext * ctx=NULL;
+	unsigned int total_line_cnt=0, success_cnt=0, i=0;
+	int timeout=0;
+	FILE* json_fp=NULL;
+	cJSON *json=NULL, *tmp_obj=NULL;
+	struct stat fstat_buf;
+	unsigned long json_file_size=0,read_size=0;
+	long long desired_version=0;
+	char* json_buff=NULL;
+	size_t json_buff_sz=0;
+    char compile_table_name[128] = {0};
+
+	while ((oc = getopt(argc,argv,"h:p:n:d:v:f:j:t:")) != -1) {
+		switch (oc) {
+			case 'h':
+				strncpy(redis_ip, optarg, sizeof(redis_ip));
+				break;
+			case 'p':
+				sscanf(optarg, "%d", &redis_port);
+				break;
+			case 'n':
+				sscanf(optarg, "%d", &redis_db);
+				break;
+			case 'd':
+				model = WORK_MODE_DUMP;
+				strncpy(dump_dir, optarg, sizeof(dump_dir));
+				if (dump_dir[strlen(dump_dir)-1] == '/') {
+					dump_dir[strlen(dump_dir)-1] = '\0';
+				}
+				break;
+			case 'v':
+				sscanf(optarg, "%lld", &desired_version);
+				break;
+			case 'j':
+				strncpy(json_file, optarg, sizeof(json_file));
+				model = WORK_MODE_JSON;
+				ret = stat(json_file, &fstat_buf);
+				if (ret != 0) {
+					printf("fstat file %s error.\n", json_file);
+					return -1;
+				}
+
+				json_fp = fopen(json_file, "r");
+				if (NULL == json_fp) {
+					printf("fopen file %s error %s.\n", json_file, strerror(errno));
+					return -1;
+				}
+
+				json_file_size = fstat_buf.st_size;
+				json_buff = ALLOC(char, json_file_size);
+				read_size = fread(json_buff, 1, json_file_size, json_fp);
+				if (read_size != json_file_size) {
+					printf("fread file %s error.\n", json_file);
+					return -1;
+				}
+
+				json = cJSON_Parse(json_buff);
+				if (!json) {
+					printf("%s format is error before: %-200.200s\n", json_file, cJSON_GetErrorPtr());
+					return -1;
+				}
+
+				tmp_obj=cJSON_GetObjectItem(json, "compile_table");
+				if (NULL == tmp_obj) {
+					printf("No \"compile_table\" in %s.\n",json_file);
+					return -1;
+				}
+				strncpy(compile_table_name, tmp_obj->valuestring, sizeof(compile_table_name));
+				free(json_buff);
+				cJSON_Delete(json);
+				fclose(json_fp);
+				break;
+			case 't':
+				sscanf(optarg,"%d",&timeout);
+				break;
+			case '?':
+			default:
+				maat_tool_print_usage();
+				return 0;
+				break;
+		}
+	}
+
+	ctx = maat_cmd_connect_redis(redis_ip, redis_port, redis_db);
+	if (NULL == ctx) {
+		return -1;
+	}
+
+	if (model == WORK_MODE_DUMP) {
+		printf("Reading key list from %s:%d db%d.\n", redis_ip, redis_port, redis_db);
+		read_rule_from_redis(ctx, desired_version, dump_dir);
+	} else if(model == WORK_MODE_JSON) {
+		ret = load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz);
+		if (ret < 0) {
+			printf("open %s failed.\n", json_file);
+		}
+
+		ret = json2iris(json_buff, json_file, NULL, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL);
+		if (ret < 0) {
+			printf("Invalid json format.\n");
+		}
+
+		total_line_cnt = 0;
+		config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num, NULL, &total_line_cnt);
+		printf("Serialize %s to %d lines, write temp file to %s .\n",json_file, total_line_cnt, tmp_iris_path);
+
+		struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
+		long long server_time = maat_cmd_redis_server_time_s(ctx);
+		if (!server_time) {
+			printf("Get Redis Time failed.\n");
+		}
+
+		if (timeout > 0) {
+			absolute_expire_time = server_time + timeout;
+		}
+
+		config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule);
+		printf("Timeout = %lld\n", absolute_expire_time);
+
+		ret = 0;
+		do {
+			success_cnt = maat_cmd_exec_serial_rule(ctx, s_rule, total_line_cnt, server_time);
+		} while(success_cnt < 0);
+
+		if (success_cnt != total_line_cnt) {
+			printf("Only Add %d of %d, rule id maybe conflicts.\n", success_cnt, total_line_cnt);
+		}
+
+		for (i = 0; i < total_line_cnt; i++) {
+			maat_cmd_empty_serial_rule(s_rule+i);
+		}
+		free(s_rule);	
+		redisFree(ctx);
+	}
+
+	return 0;
+}
--- a/vendor/CMakeLists.txt
+++ b/vendor/CMakeLists.txt
@@ -70,3 +70,16 @@ add_library(ipmatcher-static STATIC IMPORTED GLOBAL)
 add_dependencies(ipmatcher-static ipmatcher)
 set_property(TARGET ipmatcher-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/ipmatcher.a)
 set_property(TARGET ipmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include)
+
+# hiredis-1.1.0
+ExternalProject_Add(hiredis PREFIX hiredis
+        URL ${CMAKE_CURRENT_SOURCE_DIR}/hiredis-1.1.0.tar.gz
+		CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC")
+
+ExternalProject_Get_Property(hiredis INSTALL_DIR)
+file(MAKE_DIRECTORY ${VENDOR_BUILD}/include)
+
+add_library(hiredis-static STATIC IMPORTED GLOBAL)
+add_dependencies(hiredis-static hiredis)
+set_property(TARGET hiredis-static  PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a)
+set_property(TARGET hiredis-static  PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include)
--- a/vendor/hiredis-1.1.0.tar.gz
+++ b/vendor/hiredis-1.1.0.tar.gz