add basic code without test case, just compile success

include/maat.h

@@ -77,6 +77,11 @@ struct ip_addr {
     };
 };
 
+enum maat_operation {
+	MAAT_OP_DEL = 0,
+	MAAT_OP_ADD
+};
+
 enum log_level { 
     LOG_LEVEL_TRACE, 
     LOG_LEVEL_DEBUG, 
@@ -89,7 +94,7 @@ enum log_level {
 /* update_type: MAAT_UPDATE_TYPE_FULL or MAAT_UPDATE_TYPE_INC */
 typedef void maat_start_callback_t(int update_type, void *u_param);
 
-typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para);
+typedef void maat_update_callback_t(int table_id, const char *table_line, enum maat_operation op, void *u_para);
 
 typedef void maat_finish_callback_t(void *u_para);
 
@@ -135,9 +140,6 @@ int maat_options_set_logger(struct maat_options *opts, const char *log_path,
 
 int maat_options_set_log_file_max_size(struct maat_options *opts,  size_t max_size_mb);//default is 500MB, 0 for unlimited
 
-int maat_options_set_iris(struct maat_options *opts, const char *full_directory, 
-                          const char *increment_directory);
-
 int maat_options_set_json_file(struct maat_options *opts, const char *json_filename);
 
 /** 
@@ -186,7 +188,7 @@ int maat_helper_verify_regex_expression(const char *expression);
 
 /* maat table API */
 int maat_get_table_id(struct maat *instance, const char *table_name);
-int maat_get_attribute_id(struct maat *instance, const char *attribute_name);//TODO: new API
+int maat_get_attribute_id(struct maat *instance, const char *attribute_name);
 
 const char *maat_get_table_schema_tag(struct maat *instance, int table_id);//TODO: delete
 
@@ -256,11 +258,11 @@ struct maat_state;
  *         MAAT_SCAN_HALF_HIT
  *         MAAT_SCAN_HIT       
 */
-int maat_scan_flag(struct maat *instance, int table_id, long long flag,
+int maat_scan_flag(struct maat *instance, int attribute_id, long long flag,
                    long long *results, size_t n_result, size_t *n_hit_result,
                    struct maat_state *state);
 
-int maat_scan_integer(struct maat *instance, int table_id, long long integer,
+int maat_scan_integer(struct maat *instance, int attribute_id, long long integer,
                       long long *results, size_t n_result, size_t *n_hit_result,
                       struct maat_state *state);
 
@@ -268,37 +270,37 @@ int maat_scan_integer(struct maat *instance, int table_id, long long integer,
  * @param ip_addr: ipv4 address in network order
  * @param port:    port in host order. If the port is not specified, use -1. Note that 0 is a valid port.
 */
-int maat_scan_ipv4_port(struct maat *instance, int table_id, uint32_t ip_addr, int port,
+int maat_scan_ipv4_port(struct maat *instance, int attribute_id, uint32_t ip_addr, int port,
                    long long *results, size_t n_result, size_t *n_hit_result,
                    struct maat_state *state);
 
-int maat_scan_ipv6_port(struct maat *instance, int table_id, uint8_t *ip_addr, int port,
+int maat_scan_ipv6_port(struct maat *instance, int attribute_id, uint8_t *ip_addr, int port,
                    long long *results, size_t n_result, size_t *n_hit_result,
                    struct maat_state *state);
-int maat_scan_ipv4(struct maat *instance, int table_id, uint32_t ip_addr,
+int maat_scan_ipv4(struct maat *instance, int attribute_id, uint32_t ip_addr,
                    long long *results, size_t n_result, size_t *n_hit_result,
                    struct maat_state *state);
 
-int maat_scan_ipv6(struct maat *instance, int table_id, uint8_t *ip_addr,
+int maat_scan_ipv6(struct maat *instance, int attribute_id, uint8_t *ip_addr,
                    long long *results, size_t n_result, size_t *n_hit_result,
                    struct maat_state *state);
 
-int maat_scan_string(struct maat *instance, int table_id,
+int maat_scan_string(struct maat *instance, int attribute_id,
                      const char *data, size_t data_len,
                      long long *results, size_t n_result,
                      size_t *n_hit_result, struct maat_state *state);
 
-int maat_scan_object(struct maat *instance, int table_id,
+int maat_scan_object(struct maat *instance, int attribute_id,
                     struct maat_hit_object *objects, size_t n_object,
                     long long *results, size_t n_result,
                     size_t *n_hit_result, struct maat_state *state);
 
-int maat_scan_not_logic(struct maat *instance, int table_id,
+int maat_scan_not_logic(struct maat *instance, int attribute_id,
                         long long *results, size_t n_result,
                         size_t *n_hit_result, struct maat_state *state);
 
 struct maat_stream;
-struct maat_stream *maat_stream_new(struct maat *instance, int table_id,
+struct maat_stream *maat_stream_new(struct maat *instance, int attribute_id,
                                     struct maat_state *state);
 
 int maat_stream_scan(struct maat_stream *stream, const char *data, int data_len, 
@@ -314,7 +316,7 @@ void maat_state_reset(struct maat_state *state);
 
 void maat_state_free(struct maat_state *state);
 
-int maat_state_set_scan_district(struct maat_state *state, int table_id, 
+int maat_state_set_scan_district(struct maat_state *state, int attribute_id, 
                                  const char *district, size_t district_len);
 
 int maat_state_set_scan_rule_table(struct maat_state *state, int rule_table_id);

include/maat_command.h

@@ -20,12 +20,6 @@ extern "C"
 
 #include "maat.h"
 
-enum maat_operation {
-	MAAT_OP_DEL = 0,
-	MAAT_OP_ADD,
-	MAAT_OP_RENEW_TIMEOUT	//Rule expire time is changed to now+cmd->expire_after
-};
-
 struct maat_cmd_line {
 	const char *table_name;
 	const char *table_line;
@@ -40,7 +34,7 @@ struct maat_cmd_line {
  *      success: number of successfully updated rule.
  *      failed: -1
 */
-int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule);
+int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule, enum maat_operation op);
 
 int maat_cmd_set_file(struct maat *maat_instance, const char *key, const char *value, 
                       size_t size, enum maat_operation op);

src/CMakeLists.txt

@@ -11,7 +11,7 @@ LINK_DIRECTORIES(/opt/MESA/lib /usr/lib64)
 
 add_definitions(-D_GNU_SOURCE) 
 add_definitions(-fPIC)
-set(MAAT_SRC alignment.c json2iris.c maat_api.c rcu_hash.c maat_garbage_collection.c maat_config_monitor.c 
+set(MAAT_SRC alignment.c maat_api.c rcu_hash.c maat_garbage_collection.c maat_config_monitor.c 
     maat_core.c maat_kv.c maat_ex_data.c maat_utils.c maat_command.c maat_redis_monitor.c maat_table.c 
     maat_rule.c maat_object.c maat_ip.c maat_flag.c maat_interval.c maat_expr.c maat_plugin.c 
     maat_ip_plugin.c maat_ipport_plugin.c maat_bool_plugin.c maat_fqdn_plugin.c maat_attribute.c maat_stat.c)

src/inc_internal/json2iris.h

@@ -1,31 +0,0 @@
-/*
-**********************************************************************************************
-*	File: json2iris.h
-*   Description: rule for transform json2iris
-*	Authors: Zheng chao <zhengchao@geedgenetworks.com>
-*	Date:    2022-10-31
-*   Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
-***********************************************************************************************
-*/
-
-#ifndef _JSON2IRIS_H_
-#define _JSON2IRIS_H_
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-
-#include "hiredis/hiredis.h"
-#include "log/log.h"
-
-int json2iris(const char *json_buff, const char *json_filename,
-              redisContext *redis_write_ctx, char *iris_dir_buf, 
-              int buf_len, char *encrypt_key, char *encrypt_algo,
-              struct log_handle *logger);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/inc_internal/maat_attribute.h

@@ -54,7 +54,6 @@ void attribute_runtime_hit_item_num_add(struct attribute_runtime *virt_rt,
 
 long long attribute_runtime_hit_item_num(void *attribute_runtime);
 
-int attribute_get_physical_table_id(struct table_manager *tbl_mgr, int attribute_id);
 
 #ifdef __cplusplus
 }

src/inc_internal/maat_bool_plugin.h

@@ -43,7 +43,7 @@ void *bool_plugin_runtime_new(void *bool_plugin_schema, size_t max_thread_num,
 void bool_plugin_runtime_free(void *bool_plugin_runtime);
 
 int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_schema, 
-                                const char *table_name, const char *line, int valid_column);
+                                const char *table_name, const char *line, enum maat_operation op);
 
 int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name,
                                long long maat_rt_version);

src/inc_internal/maat_config_monitor.h

@@ -18,15 +18,18 @@ extern "C"
 
 #include <stdint.h>
 #include "log/log.h"
+#include "maat.h"
+#include "maat_utils.h"
 
-void config_monitor_traverse(long long version, const char *idx_dir,
+void config_monitor_traverse(long long version, const cJSON *json_root,
 							void (*start_fn)(long long, int, void *),
-							int (*update_fn)(const char *, const char *, void *),
+							int (*update_fn)(const char *, const char *, void *, enum maat_operation),
 							void (*finish_fn)(void *), void *u_param,
 							const char *dec_key, struct log_handle *logger);
 
-int load_maat_json_file(struct maat *maat_instance, const char *json_filename, 
-                        char *err_str, size_t err_str_sz);
+int load_maat_json_rule_file(struct maat *maat_inst, const char *json_filename, 
+                        cJSON *json_root, char *err_str, size_t err_str_sz);
+void convert_maat_json_rule(cJSON *json_root, unsigned char *json_buff);
 
 #ifdef __cplusplus
 }

src/inc_internal/maat_core.h

@@ -36,6 +36,7 @@ extern "C"
 #include "hiredis/hiredis.h"
 
 #define MAX_TABLE_NUM	1024
+#define MAX_ATTRIBUTE_NUM	1024
 
 #define	DISTRICT_ANY	   -1
 #define	DISTRICT_UNKNOWN   -2
@@ -81,7 +82,6 @@ struct rule_tag {
 enum data_source {
     DATA_SOURCE_NONE = 0,
     DATA_SOURCE_REDIS,
-    DATA_SOURCE_IRIS_FILE,
     DATA_SOURCE_JSON_FILE
 };
 
@@ -132,7 +132,6 @@ struct maat_options {
     enum maat_expr_engine expr_engine;
     enum data_source input_mode;
     union {
-        struct source_iris_ctx iris_ctx;
         struct source_json_ctx json_ctx;
         struct source_redis_ctx redis_ctx;
     };

src/inc_internal/maat_ex_data.h

@@ -40,6 +40,11 @@ struct ex_container_schema {
     void (*custom_data_free)(void *);
 };
 
+struct ex_data_row {
+    char *row;
+    enum maat_operation op;
+};
+
 struct ex_data_runtime;
 
 /* ex_data_runtime API */
@@ -51,9 +56,9 @@ void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt);
 void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt);
 
 /* ex_data_runtime cache row API */
-void ex_data_runtime_cache_row_put(struct ex_data_runtime *ex_data_rt, const char *row);
+void ex_data_runtime_cache_row_put(struct ex_data_runtime *ex_data_rt, const char *row, enum maat_operation op);
 
-const char *ex_data_runtime_cached_row_get(struct ex_data_runtime *ex_data_rt, size_t index);
+const struct ex_data_row *ex_data_runtime_cached_row_get(struct ex_data_runtime *ex_data_rt, size_t index);
 
 size_t ex_data_runtime_cached_row_count(struct ex_data_runtime *ex_data_rt);
 

src/inc_internal/maat_expr.h

@@ -33,7 +33,7 @@ void *expr_runtime_new(void *expr_schema, size_t max_thread_num,
 void expr_runtime_free(void *expr_runtime);
 
 int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *table_name, 
-                        const char *line, int valid_column);
+                        const char *line, enum maat_operation op);
 
 int expr_runtime_commit(void *expr_runtime, const char *table_name, long long maat_rt_version);
 

src/inc_internal/maat_flag.h

@@ -35,7 +35,7 @@ void *flag_runtime_new(void *flag_schema, size_t max_thread_num,
 void flag_runtime_free(void *flag_runtime);
 
 int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *table_name, 
-                        const char *line, int valid_column);
+                        const char *line, enum maat_operation op);
 
 int flag_runtime_commit(void *flag_runtime, const char *table_name, long long maat_rt_version);
 

src/inc_internal/maat_fqdn_plugin.h

@@ -45,7 +45,7 @@ void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, size_t max_thread_num,
 void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime);
 
 int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime, void *fqdn_plugin_schema, 
-                                const char *table_name, const char *line, int valid_column);
+                                const char *table_name, const char *line, enum maat_operation op);
 
 int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name,
                                long long maat_rt_version);

src/inc_internal/maat_interval.h

@@ -35,7 +35,7 @@ void interval_runtime_free(void *interval_runtime);
 
 int interval_runtime_update(void *interval_runtime, void *interval_schema,
                             const char *table_name, const char *line,
-                            int valid_column);
+                            enum maat_operation op);
 
 int interval_runtime_commit(void *interval_runtime, const char *table_name,
                             long long maat_rt_version);

src/inc_internal/maat_ip.h

@@ -32,7 +32,7 @@ void *ip_runtime_new(void *ip_schema, size_t max_thread_num,
 void ip_runtime_free(void *ip_runtime);
 
 int ip_runtime_update(void *ip_runtime, void *ip_schema, const char *table_name, 
-                      const char *line, int valid_column);
+                      const char *line, enum maat_operation op);
 
 int ip_runtime_commit(void *ip_runtime, const char *table_name, long long maat_rt_version);
 

src/inc_internal/maat_ip_plugin.h

@@ -44,7 +44,7 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, size_t max_thread_num,
 void ip_plugin_runtime_free(void *ip_plugin_runtime);
 
 int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, 
-                             const char *table_name, const char *line, int valid_column);
+                             const char *table_name, const char *line, enum maat_operation op);
 
 int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name, long long maat_rt_version);
 

src/inc_internal/maat_ipport_plugin.h

@@ -45,7 +45,7 @@ void *ipport_plugin_runtime_new(void *ipport_plugin_schema, size_t max_thread_nu
 void ipport_plugin_runtime_free(void *ipport_plugin_runtime);
 
 int ipport_plugin_runtime_update(void *ipport_plugin_runtime, void *ipport_plugin_schema, 
-                                 const char *table_name, const char *line, int valid_column);
+                                 const char *table_name, const char *line, enum maat_operation op);
 
 int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_name,
                                  long long maat_rt_version);

src/inc_internal/maat_object.h

@@ -37,7 +37,7 @@ void *object2object_runtime_new(void *g2g_schema, size_t max_thread_num,
 void object2object_runtime_free(void *g2g_runtime);
 
 int object2object_runtime_update(void *g2g_runtime, void *g2g_schema, const char *table_name, 
-                               const char *line, int valid_column);
+                               const char *line, enum maat_operation op);
 
 int object2object_runtime_commit(void *g2g_runtime, const char *table_name, long long maat_rt_version);
 

src/inc_internal/maat_plugin.h

@@ -21,6 +21,7 @@ extern "C"
 #include "maat_ex_data.h"
 
 #define	MAX_FOREIGN_CLMN_NUM 8
+#define MAX_FOREIGN_NAME_LEN 31
 
 /* plugin schema API */
 void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
@@ -38,7 +39,7 @@ int plugin_table_add_callback(void *plugin_schema, int table_id,
 void plugin_table_all_callback_start(struct plugin_schema *plugin_schema, int update_type);
 void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema);
 
-int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns);
+int plugin_table_get_foreign_names(struct plugin_schema *plugin_schema, char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1]);
 
 /* plugin table ex data API */
 int plugin_table_set_ex_container_schema(void *plugin_schema, int table_id,
@@ -56,7 +57,7 @@ void *plugin_runtime_new(void *plugin_schema, size_t max_thread_num,
 void plugin_runtime_free(void *plugin_runtime);
 
 int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *table_name, 
-                          const char *line, int valid_column);
+                          const char *line, enum maat_operation op);
 
 int plugin_runtime_commit(void *plugin_runtime, const char *table_name, long long maat_rt_version);
 
@@ -68,7 +69,7 @@ struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime);
 
 size_t plugin_runtime_cached_row_count(void *plugin_runtime);
 
-const char *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index);
+const struct ex_data_row *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index);
 
 void *plugin_runtime_get_ex_data(void *plugin_runtime, void *plugin_schema,
                                  const char *key, size_t key_len);

src/inc_internal/maat_redis_monitor.h

@@ -26,7 +26,6 @@ extern "C"
 #include "uthash/uthash.h"
 
 struct foreign_key {
-	int column;
 	char *key;
     size_t key_len;
 	char *filename;
@@ -88,7 +87,7 @@ int maat_get_rm_key_list(redisContext *c, long long instance_version,
 
 void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
 							void (*start_fn)(long long, int, void *),
-							int (*update_fn)(const char *, const char *, void *),
+							int (*update_fn)(const char *, const char *, void *, enum maat_operation),
 							void (*finish_fn)(void *), void *u_param);
 
 #ifdef __cplusplus

src/inc_internal/maat_rule.h

@@ -32,11 +32,6 @@ void *rule_schema_new(cJSON *json, struct table_manager *tbl_mgr,
                          const char *table_name, struct log_handle *logger);
 void rule_schema_free(void *rule_schema);
 
-void *object2rule_schema_new(cJSON *json, struct table_manager *tbl_mgr,
-                               const char *table_name, struct log_handle *logger);
-void object2rule_schema_free(void *g2c_schema);
-int object2rule_associated_rule_table_id(void *g2c_schema);
-
 /* rule runtime API */
 void *rule_runtime_new(void *rule_schema, size_t max_thread_num,
                           struct maat_garbage_bin *garbage_bin, 
@@ -47,7 +42,7 @@ void rule_runtime_init(void *rule_runtime, struct maat_runtime *maat_rt);
 
 int rule_runtime_update(void *rule_runtime, void *rule_schema,
                            const char *table_name, const char *line,
-                           int valid_column);
+                           enum maat_operation op);
 
 int rule_runtime_commit(void *rule_runtime, const char *table_name,
                            long long maat_rt_version);
@@ -64,25 +59,6 @@ size_t rule_runtime_get_hit_paths(struct rule_runtime *rule_rt, int thread_id,
                                   	 struct maat_hit_path *hit_path_array, 
                                      size_t array_size, size_t n_hit_path);
 
-/* object2rule runtime API */
-void *object2rule_runtime_new(void *g2c_schema, size_t max_thread_num,
-                                struct maat_garbage_bin *garbage_bin, 
-                                struct log_handle *logger);
-
-void object2rule_runtime_init(void *g2c_runtime, void *rule_runtime);
-
-void object2rule_runtime_free(void *g2c_runtime);
-
-int object2rule_runtime_update(void *g2c_runtime, void *g2c_schema,
-                                 const char *table_name, const char *line,
-                                 int valid_column);
-
-long long object2rule_runtime_not_condition_count(void *g2c_runtime);
-
-long long object2rule_runtime_rule_count(void *g2c_runtime);
-
-long long object2rule_runtime_update_err_count(void *g2c_runtime);
-
 /* maat rule state API */
 struct rule_compile_state;
 struct rule_compile_state *rule_compile_state_new(void);

src/inc_internal/maat_table.h

@@ -38,10 +38,8 @@ enum table_type {
     TABLE_TYPE_FQDN_PLUGIN,
     TABLE_TYPE_BOOL_PLUGIN,
     //above are physical table
-    TABLE_TYPE_ATTRIBUTE,
     TABLE_TYPE_RULE,
     TABLE_TYPE_OBJECT2OBJECT,
-	TABLE_TYPE_OBJECT2RULE,
     TABLE_TYPE_MAX
 };
 
@@ -61,6 +59,8 @@ size_t table_manager_table_size(struct table_manager *tbl_mgr);
 size_t table_manager_table_num(struct table_manager *tbl_mgr);
 
 int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_name);
+int table_manager_get_attribute_id(struct table_manager *tbl_mgr, const char *attribute_name);
+int table_manager_attribute_get_table_id(struct table_manager *tbl_mgr, int attr_id);
 /**
  * @brief get table_name's all conjunction parents' table_id
  * 
@@ -79,6 +79,7 @@ int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const
 
 const char *table_manager_get_table_name(struct table_manager *tbl_mgr,
                                          int table_id);
+const char *table_manager_get_attribute_name(struct table_manager *tbl_mgr, int attr_id);
 
 const char *table_manager_get_table_schema_tag(struct table_manager *tbl_mgr, int table_id);
 
@@ -87,8 +88,6 @@ enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int
 int table_manager_get_default_rule_table_id(struct table_manager *tbl_mgr);
 int table_manager_get_object2object_table_id(struct table_manager *tbl_mgr);
 
-int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id);
-
 enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr);
 
 size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr);
@@ -99,7 +98,8 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id);
 void *table_manager_get_updating_runtime(struct table_manager *tbl_mgr, int table_id);
 
 int table_manager_update_runtime(struct table_manager *tbl_mgr, const char *table_name, 
-                                 int table_id, const char *line, int update_type);
+                                 int table_id, const char *line, enum maat_operation op,
+                                  int update_type);
 
 void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id, 
                                   int update_type, long long maat_rt_version);

src/inc_internal/maat_utils.h

@@ -21,6 +21,7 @@ extern "C"
 #include <arpa/inet.h>
 
 #include "uthash/utarray.h"
+#include "cJSON/cJSON.h"
 
 #define	TRUE	1
 #define	FALSE	0

src/maat_attribute.c

@@ -246,18 +246,3 @@ long long attribute_runtime_hit_item_num(void *attribute_runtime)
     
     return sum;
 }
-
-int attribute_get_physical_table_id(struct table_manager *tbl_mgr, int table_id)
-{
-    enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
-
-    if (table_type != TABLE_TYPE_ATTRIBUTE) {
-        return -1;
-    }
-
-    // find physical table id
-    struct attribute_schema *attribute_schema = table_manager_get_schema(tbl_mgr, table_id);
-    assert(attribute_schema != NULL);
-
-    return attribute_schema->physical_table_id;
-}

src/maat_bool_plugin.c

@@ -20,10 +20,8 @@
 #define MODULE_BOOL_PLUGIN	 module_name_str("maat.bool_plugin")
 
 struct bool_plugin_schema {
-    int item_id_column;
-    int bool_expr_column;
-    int rule_tag_column;
     int gc_timeout_s;
+    char key_name[MAX_NAME_STR_LEN];
     struct ex_container_schema container_schema;
     int table_id;
     struct table_manager *ref_tbl_mgr;
@@ -68,32 +66,16 @@ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         goto error;
     }
 
-    custom_item = cJSON_GetObjectItem(item, "item_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->item_id_column = custom_item->valueint;
+    custom_item = cJSON_GetObjectItem(item, "key_name");
+    if (custom_item != NULL && custom_item->type == cJSON_String) {
+        strncpy(schema->key_name, custom_item->valuestring, sizeof(schema->key_name) - 1);
     } else {
         log_fatal(logger, MODULE_BOOL_PLUGIN, 
-                  "[%s:%d] bool_plugin table:<%s> schema has no item_id column", 
+                  "[%s:%d] bool_plugin table:<%s> schema has no key_name column", 
                   __FUNCTION__, __LINE__, table_name);
         goto error;
     }
 
-    custom_item = cJSON_GetObjectItem(item, "bool_expr");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->bool_expr_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_BOOL_PLUGIN, 
-                  "[%s:%d] bool_plugin table:<%s> schema has no bool_expr column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    // rule_tag is optional
-    custom_item = cJSON_GetObjectItem(item, "rule_tag");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->rule_tag_column = custom_item->valueint;
-    }
-
     //gc_timeout_s is optional
     custom_item = cJSON_GetObjectItem(item, "gc_timeout_s");
     if (custom_item != NULL && custom_item->type == cJSON_Number) {
@@ -240,12 +222,12 @@ static int
 bool_plugin_runtime_update_row(struct bool_plugin_runtime *bool_plugin_rt,
                                const char *table_name, const char *row, 
                                const char *key, size_t key_len, 
-                               struct bool_expr *expr, int is_valid)
+                               struct bool_expr *expr, enum maat_operation op)
 {
     int ret = -1;
     struct ex_data_runtime *ex_data_rt = bool_plugin_rt->ex_data_rt;
 
-    if (0 == is_valid) {
+    if (MAAT_OP_DEL == op) {
         // delete
         ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len);
         if (ret < 0) {
@@ -271,44 +253,37 @@ bool_plugin_runtime_update_row(struct bool_plugin_runtime *bool_plugin_rt,
 
 static int
 bool_plugin_accept_tag_match(struct bool_plugin_schema *schema,
-                             const char *table_name, const char *line,
+                             const char *table_name, const cJSON *json,
                              struct log_handle *logger)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
     size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr);
+    cJSON *tmp_obj = NULL;
 
-    if (schema->rule_tag_column > 0 && n_tag > 0) {
-        int ret = get_column_pos(line, schema->rule_tag_column, 
-                                 &column_offset, &column_len);
-        if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "tag");
+    if (tmp_obj && n_tag > 0) {
+        if (tmp_obj->type != cJSON_String) {
             log_fatal(logger, MODULE_BOOL_PLUGIN,
-                      "[%s:%d] bool_plugin table:<%s> has no rule_tag"
+                      "[%s:%d] bool_plugin table:<%s> has invalid tag format"
                       " in line:%s", __FUNCTION__, __LINE__, table_name,
-                      line);
+                      cJSON_Print(json));
             return TAG_MATCH_ERR;
         }
 
-        if (column_len > 2) {
-            char *tag_str = ALLOC(char, column_len + 1);
-            memcpy(tag_str, (line + column_offset), column_len);
-            ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str);
-            FREE(tag_str);
-            if (TAG_MATCH_ERR == ret) {
-                log_fatal(logger, MODULE_BOOL_PLUGIN,
-                          "[%s:%d] bool_plugin table:<%s> has invalid "
-                          "tag format in line:%s", __FUNCTION__, __LINE__,
-                          table_name, line);
-                return TAG_MATCH_ERR;
-            }
+        int ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tmp_obj->valuestring);
+        if (TAG_MATCH_ERR == ret) {
+            log_fatal(logger, MODULE_BOOL_PLUGIN,
+                      "[%s:%d] bool_plugin table:<%s> has invalid tag"
+                      " in line:%s", __FUNCTION__, __LINE__, table_name,
+                      cJSON_Print(json));
+            return TAG_MATCH_ERR;
+        }
 
-            if (TAG_MATCH_UNMATCHED == ret) {
-                log_fatal(logger, MODULE_BOOL_PLUGIN,
-                          "[%s:%d] bool_plugin table:<%s> has unmatched tag"
-                          " in line:%s", __FUNCTION__, __LINE__, table_name,
-                          line);
-                return TAG_MATCH_UNMATCHED;
-            }
+        if (TAG_MATCH_UNMATCHED == ret) {
+            log_fatal(logger, MODULE_BOOL_PLUGIN,
+                      "[%s:%d] bool_plugin table:<%s> has unmatched tag"
+                      " in line:%s", __FUNCTION__, __LINE__, table_name,
+                      cJSON_Print(json));
+            return TAG_MATCH_UNMATCHED;
         }
     }
 
@@ -317,44 +292,41 @@ bool_plugin_accept_tag_match(struct bool_plugin_schema *schema,
 
 static struct bool_expr *
 bool_plugin_expr_new(struct bool_plugin_schema *schema, const char *table_name,
-                     const char *line, struct log_handle *logger)
+                     const cJSON *json, struct log_handle *logger)
 {
-    int ret = bool_plugin_accept_tag_match(schema, table_name, line, logger);
+    int ret = bool_plugin_accept_tag_match(schema, table_name, json, logger);
     if (ret == TAG_MATCH_UNMATCHED) {
         return NULL;
     }
 
-    size_t column_offset = 0;
-    size_t column_len = 0;
     size_t n_item = 0;
     char expr_buffer[BUFSIZ + 1] = {0};
     unsigned long long items[MAX_ITEMS_PER_BOOL_EXPR] = {0};
     char *token = NULL, *sub_token = NULL, *saveptr;
     struct bool_expr *bool_expr = ALLOC(struct bool_expr, 1);
+    cJSON *tmp_obj = NULL;
 
-    ret = get_column_pos(line, schema->item_id_column,
-                         &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, schema->key_name);
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_BOOL_PLUGIN, 
                   "[%s:%d] bool_plugin table:<%s> has "
-                  "no item_id in line:%s", __FUNCTION__,
-                  __LINE__, table_name, line);
+                  "no key_name %s or invalid format in line:%s", __FUNCTION__,
+                  __LINE__, table_name, schema->key_name, cJSON_Print(json));
         goto error;
     }
-    bool_expr->expr_id = atoll(line + column_offset);
+    bool_expr->expr_id = atoll(tmp_obj->valuestring);
 
-    ret = get_column_pos(line, schema->bool_expr_column,
-                         &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "bool_expr");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_BOOL_PLUGIN, 
                   "[%s:%d] bool_plugin table:<%s> has "
-                  "no bool_expr in line:%s", __FUNCTION__,
-                  __LINE__, table_name, line);
+                  "no bool_expr or invalid format in line:%s", __FUNCTION__,
+                  __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
 
     memset(expr_buffer, 0, sizeof(expr_buffer));
-    memcpy(expr_buffer, line + column_offset, column_len);
+    memcpy(expr_buffer, tmp_obj->valuestring, strlen(tmp_obj->valuestring));
     for (token = expr_buffer; ; token = NULL) {
         sub_token = strtok_r(token, "&", &saveptr);
         if (NULL == sub_token) {
@@ -367,7 +339,7 @@ bool_plugin_expr_new(struct bool_plugin_schema *schema, const char *table_name,
             log_fatal(logger, MODULE_BOOL_PLUGIN, 
                       "[%s:%d] bool_plugin table:<%s> has "
                       "invalid format of bool_expr in line:%s",
-                      __FUNCTION__, __LINE__, table_name, line);
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             goto error;
         }
     }
@@ -394,7 +366,7 @@ void bool_plugin_expr_free(void *bool_expr)
 int bool_plugin_runtime_update(void *bool_plugin_runtime,
                                void *bool_plugin_schema, 
                                const char *table_name,
-                               const char *line, int valid_column)
+                               const char *line, enum maat_operation op)
 {
     if (NULL == bool_plugin_runtime || NULL == bool_plugin_schema || 
         NULL == line) {
@@ -408,64 +380,72 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime,
     struct bool_plugin_runtime *bool_plugin_rt =
         (struct bool_plugin_runtime *)bool_plugin_runtime;
 
-    size_t item_id_offset = 0, item_id_len = 0;
+    int ret = 0;
+    cJSON *tmp_obj = NULL;
+    cJSON *json = cJSON_Parse(line);
 
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
+    if (NULL == json) {
         log_fatal(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
-                  "[%s:%d] bool_plugin table:<%s> has no "
-                  "is_valid(column seq:%d) in table_line:%s",
-                  __FUNCTION__, __LINE__, table_name,
-                  valid_column, line);
+                  "[%s:%d] bool_plugin table:<%s> has invalid json format"
+                  " in table_line:%s", __FUNCTION__, __LINE__, table_name, line);
         bool_plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
-    int ret = get_column_pos(line, schema->item_id_column,
-                             &item_id_offset, &item_id_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, schema->key_name);
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
-                  "[%s:%d] bool_plugin table:<%s> has no item_id(column seq:%d)"
+                  "[%s:%d] bool_plugin table:<%s> has no key_name(field:%s) or invalid format"
                   " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->item_id_column, line);
+                  schema->key_name, line);
         bool_plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
     if (1 == schema->container_schema.set_flag) {
-        if (1 == is_valid) {
+        if (MAAT_OP_ADD == op) {
             // add
-            bool_expr = bool_plugin_expr_new(schema, table_name, line,
+            bool_expr = bool_plugin_expr_new(schema, table_name, json,
                                              bool_plugin_rt->logger);
             if (NULL == bool_expr) {
                 bool_plugin_rt->update_err_cnt++;
-                return -1;
+                goto ERROR;
             }
         }
 
-        const char *key = line + item_id_offset;
-        size_t key_len = item_id_len;
+        const char *key = tmp_obj->valuestring;
+        size_t key_len = strlen(key);
         ret = bool_plugin_runtime_update_row(bool_plugin_rt, table_name, line,
-                                             key, key_len, bool_expr, is_valid);
+                                             key, key_len, bool_expr, op);
         if (ret < 0) {
             log_fatal(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
                       "[%s:%d]bool_plugin table:<%s> update one line failed, "
                       "line:%s", __FUNCTION__, __LINE__, table_name, line);
             bool_plugin_rt->update_err_cnt++;
-            return -1;
+            goto ERROR;
         }
 
         log_debug(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
                   "bool_plugin table:<%s> update one line, key:%s, key_len:%zu,"
-                  " is_valid:%d", table_name, key, key_len, is_valid);
+                  " maat_op:%d", table_name, key, key_len, op);
     } else {
         //ex_schema not set
-        ex_data_runtime_cache_row_put(bool_plugin_rt->ex_data_rt, line);
+        ex_data_runtime_cache_row_put(bool_plugin_rt->ex_data_rt, line, op);
         bool_plugin_rt->rule_num = 
             ex_data_runtime_cached_row_count(bool_plugin_rt->ex_data_rt);
     }
 
     return 0;
+
+ERROR:
+    if (json != NULL) {
+        cJSON_Delete(json);
+    }
+    if (bool_expr != NULL) {
+        bool_plugin_expr_free(bool_expr);
+    }
+
+    return -1;
 }
 
 void garbage_bool_matcher_free(void *matcher, void *arg)

src/maat_command.c

@@ -265,7 +265,8 @@ int maat_wrap_redis_get_reply(redisContext *c, redisReply **reply)
 }
 
 int maat_cmd_set_line(struct maat *maat_inst,
-					  const struct maat_cmd_line *line_rule)
+					  const struct maat_cmd_line *line_rule,
+					  enum maat_operation op)
 {
 	int ret = 0;
 	long long absolute_expire_time = 0;
@@ -292,36 +293,12 @@ int maat_cmd_set_line(struct maat *maat_inst,
 		FREE(s_rule);
 		return -1;
 	}
-
-	int valid_column = table_manager_get_valid_column(maat_inst->tbl_mgr,
-													  table_id);
-	if (valid_column < 0) {
-		log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND,
-				  "[%s:%d] Command set line id %lld failed: "
-				  "table %s is not a plugin or ip_plugin table",
-				  __FUNCTION__, __LINE__, line_rule->rule_id,
-				  line_rule->table_name);
-		FREE(s_rule);
-		return -1;
-	}
-
-	int valid_offset = maat_get_valid_flag_offset(line_rule->table_line,
-												  valid_column);
-	if (valid_offset < 0) {
-		log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND,
-				  "[%s:%d] Command set line id %lld failed: "
-				  "table %s valid_offset error", __FUNCTION__, __LINE__,
-				  line_rule->rule_id, line_rule->table_name);
-		FREE(s_rule);
-		return -1;
-	}
-
-	int is_valid = atoi(line_rule->table_line + valid_offset);
+	
 	if (line_rule->expire_after > 0) {
 		absolute_expire_time = server_time + line_rule->expire_after;
 	}
 
-    maat_set_serial_rule(s_rule, (enum maat_operation)is_valid,
+    maat_set_serial_rule(s_rule, op,
 						 line_rule->rule_id, line_rule->table_name,
 						 line_rule->table_line, absolute_expire_time);
 

src/maat_config_monitor.c

@@ -16,9 +16,7 @@
 #include <dirent.h>
 #include <unistd.h>
 
-#include "maat_utils.h"
 #include "maat_core.h"
-#include "json2iris.h"
 #include "maat_config_monitor.h"
 
 #define MODULE_CONFIG_MONITOR	 module_name_str("maat.config_monitor")
@@ -32,162 +30,6 @@ struct cm_table_info_t {
 	char encrypt_algo[NAME_MAX];
 };
 
-static int cm_read_cfg_index_file(const char *path, struct cm_table_info_t *idx,
-								  int size, struct log_handle *logger)
-{
-	int ret = 0;
-	int i = 0;
-	char line[MAX_CONFIG_LINE];
-	struct stat file_info;
-
-	FILE* fp = fopen(path,"r");
-	while (!feof(fp)) {
-		memset(line, 0, sizeof(line));
-		fgets(line, sizeof(line), fp);
-		ret = sscanf(line, "%s\t%d\t%s\t%s", idx[i].table_name,
-					 &(idx[i].cfg_num), idx[i].cfg_path, idx[i].encrypt_algo);
-
-		//jump over empty line
-		if (!(ret == 3 || ret == 4) || idx[i].cfg_num == 0) {
-			continue;
-		}
-
-		ret = stat(idx[i].cfg_path, &file_info);
-		if (ret != 0) {
-			log_fatal(logger, MODULE_CONFIG_MONITOR, "%s of %s not exist",
-					  idx[i].cfg_path, path);
-			fclose(fp);
-			return -1;
-		}
-
-		i++;
-		if (i == size) {
-			log_fatal(logger, MODULE_CONFIG_MONITOR, "Too much lines in %s", path);
-			break;
-		}
-	}
-	fclose(fp);
-
-	return i;
-}
-
-static const char *path2filename(const char *path)
-{
-	int i = 0;
-	for (i = strlen(path);i > 0;i--) {
-		if (path[i]=='/') {
-			break;
-		}
-	}
-	return path + i + 1;
-}
-
-static char *read_nxt_line_from_buff(const char *buff, size_t buff_size, 
-                              		 size_t *offset, char *line, int line_size)
-{
-	int this_offset=0;
-	const char* p;
-
-	//search for CRLF, aka CR '\r'(old Mac), LF '\n'(UNIX) or CRLF"\r\n" (Windows)
-	p = (const char *)memchr(buff + *offset, '\n', buff_size - *offset);
-	if (p == NULL) { // NOT "\n" or "\r\n"
-		p = (const char *)memchr(buff + *offset, '\r', buff_size - *offset);
-	}
-
-	if (p != NULL) { //point to next character
-		p++;
-	} else { //Treat rest buff has no CRLF as a line.
-		p = buff + buff_size;
-	}
-
-	this_offset = p - (buff + *offset);
-	memcpy(line, buff + *offset, MIN(this_offset, line_size - 1));
-
-	*offset += this_offset;
-	line[MIN(this_offset, line_size - 1)] = '\0';
-
-	return line;
-}
-
-static int cm_read_table_file(struct cm_table_info_t *index,
-							  int (*update_fn)(const char *, const char *, void *),
-							  void *u_param, const char *dec_key,
-							  struct log_handle *logger)
-{
-	int ret = 0;
-    size_t file_sz = 0;
-	char *file_buff = NULL;
-    char error_string[NAME_MAX];
-	
-    if (strlen(index->encrypt_algo) > 0) {
-        //JSON file has been encrypted
-        if (NULL == dec_key || 0 == strlen(dec_key)) {
-            log_fatal(logger, MODULE_CONFIG_MONITOR, 
-                      "update error, no key to decrypt %s.",
-					  index->cfg_path);
-            return -1;
-        }
-
-        ret = decrypt_open(index->cfg_path, dec_key, index->encrypt_algo,
-                          (unsigned char**)&file_buff, &file_sz, error_string,
-                          sizeof(error_string));
-        if (ret < 0) {
-            log_fatal(logger, MODULE_CONFIG_MONITOR,
-                      "update error, decrypt %s failed: %s",
-					  index->cfg_path, error_string);
-            return -1;
-        }
-    } else {
-        // not encrypted
-        ret = load_file_to_memory(index->cfg_path, (unsigned char**)&file_buff, &file_sz);
-	    if (ret < 0) {
-		    log_fatal(logger, MODULE_CONFIG_MONITOR, "[%s:%d] open %s failed.", 
-                      __FUNCTION__, __LINE__, index->cfg_path);
-		    return -1;
-	    }
-    }
-
-    size_t file_offset = 0;
-    char line[MAX_CONFIG_LINE] = {0};
-	read_nxt_line_from_buff(file_buff, file_sz, &file_offset, line, sizeof(line));
-    int cfg_num = 0;
-	sscanf(line, "%d\n", &cfg_num);
-
-	if(cfg_num != index->cfg_num) {
-        FREE(file_buff);
-        log_fatal(logger, MODULE_CONFIG_MONITOR,
-				  "[%s:%d] file %s config num not matched",
-                  __FUNCTION__, __LINE__, index->cfg_path);
-		return -1;
-	}
-
-	for (int i = 0; i < cfg_num; i++) {
-		line[sizeof(line) - 1] = '\0';
-
-		char *ret_str = read_nxt_line_from_buff(file_buff, file_sz, &file_offset,
-												line, sizeof(line));
-		if (ret_str == NULL) {
-			log_fatal(logger, MODULE_CONFIG_MONITOR,
-					  "[%s:%d] file %s line_num %d less than claimed %d",
-                       __FUNCTION__, __LINE__, index->cfg_path, i, cfg_num);
-			break;
-		}
-
-		if(line[sizeof(line) - 1] != '\0') {
-			log_fatal(logger, MODULE_CONFIG_MONITOR,
-					  "[%s:%d] line size more than %u at of file %s:%d",
-                       __FUNCTION__, __LINE__, sizeof(line), index->cfg_path, i);
-			continue;
-		}
-
-		update_fn(index->table_name, line, u_param);
-	}
-
-	FREE(file_buff);
-	
-	return 0;
-}
-
 //replacement of glibc scandir, to adapt dictator malloc wrap
 #define	ENLARGE_STEP	1024
 int my_scandir(const char *dir, struct dirent ***namelist,
@@ -243,176 +85,140 @@ int my_scandir(const char *dir, struct dirent ***namelist,
 	return num; 
 }
 
-static int filter_fn(const struct dirent *ent)
+static void config_load_json_content(const cJSON *json_root, const char *table_name, const char *key, void *u_param,
+									int (*update_fn)(const char *, const char *, void *, enum maat_operation))
 {
-    return (strncmp(ent->d_name,"full_config_index",strlen("full_config_index")) == 0 ||
-   	        strncmp(ent->d_name,"inc_config_index",strlen("inc_config_index")) == 0);
+	cJSON *tmp_item = NULL;
+	cJSON *array_item = NULL;
+	int i;
+
+	tmp_item = cJSON_GetObjectItem(json_root, table_name);
+	array_item = cJSON_GetObjectItem(json_root, key);
+	if (array_item != NULL) {
+		for (i = 0; i < cJSON_GetArraySize(tmp_item); i++) {
+			cJSON *rule = cJSON_GetArrayItem(tmp_item, i);
+			if (rule == NULL) {
+				continue;
+			}
+
+			char *rule_str = cJSON_PrintUnformatted(rule);
+			if (rule_str == NULL) {
+				continue;
+			}
+
+			update_fn(tmp_item->valuestring, rule_str, u_param, MAAT_OP_ADD);
+			FREE(rule_str);
+		}
+	}
 }
 
-static int get_new_idx_path(long long current_version, const char *file_dir, 
-                     	    char ***idx_path, size_t *idx_num,
-							struct log_handle *logger)
-{
-	struct dirent **namelist = NULL;
-	int update_type = MAAT_UPDATE_TYPE_INVALID;
-	
-	int n = my_scandir(file_dir, &namelist, filter_fn,
-					   (int (*)(const void*, const void*))alphasort);
-	if (n < 0) {
-		log_fatal(logger, MODULE_CONFIG_MONITOR,
-				  "[%s:%d] scan dir error", __FUNCTION__, __LINE__);
-		return update_type;
-	}
-
-	int i = 0;
-    int full_file_idx = 0;
-    int *inc_file_idx = ALLOC(int, n);
-	int inc_idx_num = 0;
-    long long latest_inc_version = 0;
-	long long latest_full_version = 0;
-
-	for (i = 0; i < n; i++) {
-		long long config_seq = 0;
-        char update_str[32] = {0};
-        
-		if ((strcmp(namelist[i]->d_name, ".") == 0) ||
-			(strcmp(namelist[i]->d_name, "..") == 0)) {
-			continue;
-		}
-
-		if (strlen(namelist[i]->d_name) > 42) {
-			log_fatal(logger, MODULE_CONFIG_MONITOR, 
-                      "[%s:%d] config file %s filename too long, should like"
-					  " full_config_index.000000000001",
-					  __FUNCTION__, __LINE__, namelist[i]->d_name);
-			continue;
-		}
-
-		int ret = sscanf(namelist[i]->d_name,"%[a-zA-Z]_config_index.%lld",
-						 update_str, &config_seq);
-		if (ret != 2) {
-			log_fatal(logger, MODULE_CONFIG_MONITOR, 
-                      "[%s:%d] config file %s filename error, should like"
-					  " full_config_index.000000000001",
-                      __FUNCTION__, __LINE__, namelist[i]->d_name);
-			continue;
-		}
-
-		if (strncasecmp(update_str, "full", strlen(update_str)) == 0) {
-			if (config_seq > latest_full_version) {
-				latest_full_version = config_seq;
-				full_file_idx = i;
-			}
-		} else if(strncasecmp(update_str,"inc",strlen(update_str))==0) {
-			if (config_seq > current_version) {
-				inc_file_idx[inc_idx_num] = i;
-				inc_idx_num++;
-				if (config_seq > latest_inc_version) {
-					latest_inc_version = config_seq;
-				}
-			}
-		} else {
-			log_fatal(logger, MODULE_CONFIG_MONITOR, 
-                      "[%s:%d] config file %s, not full or inc config", 
-                      __FUNCTION__, __LINE__, namelist[i]->d_name);
-		}
-	}
-
-    size_t path_len = 0;
-	//full update
-	if (latest_full_version > current_version) {
-        *idx_path = (char **)malloc(sizeof(char **));
-		path_len = strlen(file_dir) + strlen(namelist[full_file_idx]->d_name) + 1 + 1;
-		(*idx_path)[0] = (char *)malloc(path_len);
-		snprintf((*idx_path)[0], path_len, "%s/%s", file_dir,
-				 namelist[full_file_idx]->d_name);
-		*idx_num = 1;
-		update_type = MAAT_UPDATE_TYPE_FULL;
-	} else if (latest_inc_version > current_version) {
-		/* Inc update, it's possible that do inc after Full update, 
-		   but we'll process it at next loop.
-		*/
-		*idx_path = (char **)malloc(sizeof(char **) * inc_idx_num);
-		for (i = 0; i < inc_idx_num; i++) {
-			path_len = strlen(file_dir) + strlen(namelist[inc_file_idx[i]]->d_name) + 1 + 1;
-			(*idx_path)[i] = (char *)malloc(path_len);
-			snprintf((*idx_path)[i], path_len, "%s/%s", file_dir,
-					 namelist[inc_file_idx[i]]->d_name);
-		}
-		*idx_num = inc_idx_num;
-		update_type = MAAT_UPDATE_TYPE_INC;
-	} else {
-		update_type = MAAT_UPDATE_TYPE_INVALID;
-	}
-
-	FREE(inc_file_idx);
-	for (i = 0; i < n; i++) {
-		FREE(namelist[i]);
-	}
-	FREE(namelist);
-
-	return update_type;
-}
-
-void config_monitor_traverse(long long current_version, const char *idx_dir,
+void config_monitor_traverse(long long current_version, const cJSON *json_root,
 							 void (*start_fn)(long long, int, void *),
-							 int (*update_fn)(const char *, const char *, void *),
+							 int (*update_fn)(const char *, const char *, void *, enum maat_operation),
 							 void (*finish_fn)(void *), void *u_param,
 							 const char *dec_key, struct log_handle *logger)
 {
 	size_t i = 0;
 	long long new_version = 0;
-	char **idx_path_array = NULL;
-	size_t idx_path_num = 0;
-	struct cm_table_info_t table_array[MAX_TABLE_NUM];
+	int update_type = MAAT_UPDATE_TYPE_FULL;
 
-	memset(table_array, 0, sizeof(table_array));
-
-	int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array,
-									   &idx_path_num, logger);
-	if (update_type != MAAT_UPDATE_TYPE_INVALID) {
-		for (i = 0; i < idx_path_num; i++) {
-            log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]);
-			int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array,
-												   MAX_TABLE_NUM, logger);
-			if (table_num < 0) {
-				log_fatal(logger, MODULE_CONFIG_MONITOR, 
-                          "[%s:%d] load %s failed, abandon update", 
-                          __FUNCTION__, __LINE__, idx_path_array[i]);
-				break;
-			}
-
-			char str_not_care[256] = {0};
-			const char *table_filename = path2filename(idx_path_array[i]);
-			sscanf(table_filename, "%[a-zA-Z]_config_index.%lld",
-				   str_not_care, &new_version);
-
-			if (start_fn != NULL) {
-				start_fn(new_version, update_type, u_param);
-			}
-
-			for (int j = 0; j < table_num; j++) {
-				cm_read_table_file(table_array + j, update_fn, u_param, dec_key, logger);
-			}
-
-			if (finish_fn != NULL) {
-				finish_fn(u_param);
-			}
-		}
-	}
-
-	for (i = 0; i < idx_path_num; i++) {
-		FREE(idx_path_array[i]);
+	if (start_fn != NULL) {
+		start_fn(new_version, update_type, u_param);
 	}
 	
-	FREE(idx_path_array);
+	cJSON *tmp_obj = NULL;
+
+	tmp_obj = cJSON_GetObjectItem(json_root, "objects");
+	if (tmp_obj != NULL) {
+		for (i = 0; i < cJSON_GetArraySize(tmp_obj); i++) {
+			cJSON *object = cJSON_GetArrayItem(tmp_obj, i);
+			if (object == NULL) {
+				continue;
+			}
+
+			cJSON *regions = cJSON_GetObjectItem(object, "regions");
+			if (regions == NULL) {
+				continue;
+			}
+
+			cJSON *table_name = cJSON_GetObjectItem(object, "table_name");
+			if (table_name == NULL) {
+				continue;
+			}
+
+			cJSON *table_content = cJSON_GetObjectItem(object, "table_content");
+			if (table_content == NULL) {
+				continue;
+			}
+
+			char *table_content_str = cJSON_PrintUnformatted(table_content);
+
+			update_fn(table_name->valuestring, table_content_str, u_param, MAAT_OP_ADD);
+			FREE(table_content_str);
+		}
+	}
+	
+	config_load_json_content(json_root, "object2object_table", "object_groups", u_param, update_fn);
+	config_load_json_content(json_root, "rule_table", "rules", u_param, update_fn);
+
+	if (finish_fn != NULL) {
+		finish_fn(u_param);
+	}
 }
 
-int load_maat_json_file(struct maat *maat_inst, const char *json_filename, 
-                        char *err_str, size_t err_str_sz)
+void convert_maat_json_rule(cJSON *json_root, unsigned char *json_buff)
+{
+	json_root = cJSON_Parse((const char *)json_buff);
+	cJSON *top_objects = cJSON_GetObjectItem(json_root, "objects");
+	//cJSON *object_groups = cJSON_GetObjectItem(json_root, "object_groups");//TODO: object include object in rules
+	cJSON *rules = cJSON_GetObjectItem(json_root, "rules");
+	cJSON *tmp_rule = NULL;
+	long long item_id = 1;
+	long long object_id = 1;
+	char str[10];
+
+	cJSON_ArrayForEach(tmp_rule, rules) {
+		cJSON *tmp_condition = NULL;
+		cJSON *condition_array = cJSON_GetObjectItem(tmp_rule, "conditions");
+		cJSON_ArrayForEach(tmp_condition, condition_array) {
+			cJSON *tmp_object = NULL;
+			cJSON *object_id_array = cJSON_CreateArray();
+			cJSON *object_array = cJSON_GetObjectItem(tmp_condition, "objects");
+			cJSON_ArrayForEach(tmp_object, object_array) {
+				//find items, generate item_id and object_id
+				cJSON *table_name = cJSON_GetObjectItem(tmp_object, "table_name");
+				cJSON *tmp_item = cJSON_CreateObject();
+				cJSON_AddItemToObject(tmp_item, "table_name", cJSON_CreateString(table_name->valuestring));
+				cJSON *dup = cJSON_Duplicate(tmp_object, 0);
+				if (cJSON_GetObjectItem(dup, "item_id") == NULL) {
+					memset(str, 0, sizeof(str));
+					snprintf(str, sizeof(str), "%lld", item_id);
+					cJSON_AddItemToObject(dup, "item_id", cJSON_CreateString(str));
+					item_id++;
+				}
+				if (cJSON_GetObjectItem(dup, "object_id") == NULL) {
+					memset(str, 0, sizeof(str));
+					snprintf(str, sizeof(str), "%lld", object_id);
+					cJSON_AddItemToObject(dup, "object_id", cJSON_CreateString(str));
+					object_id++;
+				}
+				cJSON_AddItemToObject(tmp_item, "table_content", dup);
+				cJSON_AddItemToArray(top_objects, tmp_item);
+
+				cJSON_AddItemToArray(object_id_array, cJSON_CreateString(str));
+			}
+			//replace object content with object_id
+			cJSON_DeleteItemFromObject(tmp_condition, "objects");
+			cJSON_AddItemToObject(tmp_condition, "object_ids", object_id_array);
+		}
+			
+	}
+}
+
+int load_maat_json_rule_file(struct maat *maat_inst, const char *json_filename, 
+                        cJSON *json_root, char *err_str, size_t err_str_sz)
 {
 	int ret = 0;
-	struct stat fstat_buf;
 	unsigned char *json_buff = NULL; 
     unsigned char *decrypted_buff = NULL; 
     unsigned char *uncompressed_buff = NULL;
@@ -466,29 +272,9 @@ int load_maat_json_file(struct maat *maat_inst, const char *json_filename,
 			return -1;
 		}
 	}
-
-	ret = json2iris((const char*)json_buff, json_filename, NULL,
-			        maat_inst->opts.json_ctx.iris_file, 
-                    sizeof(maat_inst->opts.json_ctx.iris_file),
-			        strlen(maat_inst->opts.decrypt_key) ? maat_inst->opts.decrypt_key : NULL,
-			        strlen(maat_inst->opts.decrypt_algo) ? maat_inst->opts.decrypt_algo : NULL,
-                    maat_inst->logger);
-	FREE(json_buff);
-	if (ret < 0) {
-		return -1;
-	}
-	
-	ret = stat(json_filename, &fstat_buf);
-	maat_inst->opts.json_ctx.last_md5_time = fstat_buf.st_ctim;
-	
-	md5_file(maat_inst->opts.json_ctx.json_file, maat_inst->opts.json_ctx.effective_json_md5);
-	log_info(maat_inst->logger, MODULE_CONFIG_MONITOR, 
-            "JSON file %s md5: %s, generate index file %s OK",
-			maat_inst->opts.json_ctx.json_file,
-			maat_inst->opts.json_ctx.effective_json_md5,
-		    maat_inst->opts.json_ctx.iris_file);
-
 	maat_inst->opts.input_mode = DATA_SOURCE_JSON_FILE;
+	
+	convert_maat_json_rule(json_root, json_buff);
 
 	return 0;	
 }

src/maat_core.c

@@ -16,7 +16,6 @@
 #include <sys/stat.h>
 #include <assert.h>
 
-#include "json2iris.h"
 #include "log/log.h"
 #include "maat_utils.h"
 #include "maat_core.h"
@@ -131,7 +130,7 @@ maat_start_cb(long long new_version, int update_type, void *u_param)
 }
 
 static int
-maat_update_cb(const char *table_name, const char *line, void *u_param)
+maat_update_cb(const char *table_name, const char *line, void *u_param, enum maat_operation op)
 {
 	if (NULL == table_name || NULL == line || NULL == u_param) {
 		return 0;
@@ -161,7 +160,7 @@ maat_update_cb(const char *table_name, const char *line, void *u_param)
         for (int i = 0; i < conj_parent_table_cnt; i++) {
             int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name,
                                                    (int)conj_parent_table_ids[i],
-                                                   line, update_type);
+                                                   line, op, update_type);
             if (ret < 0) {
                 log_fatal(maat_inst->logger, MODULE_MAAT_RULE,
                           "[%s:%d] table<%s> update runtime error for rule:%s",
@@ -171,7 +170,7 @@ maat_update_cb(const char *table_name, const char *line, void *u_param)
         }
     } else {
         int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name,
-                                               table_id, line, update_type);
+                                               table_id, line, op, update_type);
             if (ret < 0) {
                 log_fatal(maat_inst->logger, MODULE_MAAT_RULE,
                           "[%s:%d] table<%s> update runtime error for rules:%s",
@@ -307,6 +306,7 @@ void maat_read_full_config(struct maat *maat_inst)
 {
     int ret = -1;
     char err_str[NAME_MAX] = {0};
+    cJSON *json_root = NULL;
     struct source_redis_ctx *redis_ctx = NULL;
 
     switch (maat_inst->opts.input_mode) {
@@ -334,21 +334,9 @@ void maat_read_full_config(struct maat *maat_inst)
                           redis_ctx->redis_db);
             }
             break;
-        case DATA_SOURCE_IRIS_FILE:
-			config_monitor_traverse(maat_inst->maat_version,
-                                    maat_inst->opts.iris_ctx.full_idx_dir,
-                                    maat_start_cb, maat_update_cb, maat_finish_cb,
-                                    maat_inst, maat_inst->opts.decrypt_key,
-                                    maat_inst->logger);
-            if (NULL == maat_inst->creating_maat_rt) {
-                log_fatal(maat_inst->logger, MODULE_MAAT_RULE, 
-                          "[%s:%d] At initiation: NO effective rule in %s",
-                          __FUNCTION__, __LINE__, maat_inst->opts.iris_ctx.full_idx_dir);
-            }
-            break;
         case DATA_SOURCE_JSON_FILE:
-            ret = load_maat_json_file(maat_inst, maat_inst->opts.json_ctx.json_file,
-                                      err_str, sizeof(err_str));
+            ret = load_maat_json_rule_file(maat_inst, maat_inst->opts.json_ctx.json_file,
+                                      json_root, err_str, sizeof(err_str));
             if (ret < 0) {
                 log_fatal(maat_inst->logger, MODULE_MAAT_RULE, 
                           "[%s:%d] Maat re-initiate with JSON file %s failed: %s", 
@@ -357,7 +345,7 @@ void maat_read_full_config(struct maat *maat_inst)
             }
             
             config_monitor_traverse(maat_inst->maat_version,
-                                    maat_inst->opts.json_ctx.iris_file,
+                                    json_root,
                                     maat_start_cb, maat_update_cb, maat_finish_cb,
                                     maat_inst, maat_inst->opts.decrypt_key,
                                     maat_inst->logger);
@@ -366,6 +354,7 @@ void maat_read_full_config(struct maat *maat_inst)
                           "[%s:%d] At initiation: NO effective rule in %s",
                           __FUNCTION__, __LINE__, maat_inst->opts.json_ctx.iris_file);
             }
+            cJSON_Delete(json_root);
             break;
         default:
             break;
@@ -444,6 +433,7 @@ void *rule_monitor_loop(void *arg)
     char md5_tmp[MD5_DIGEST_LENGTH * 2 + 1] = {0};
     char err_str[NAME_MAX] = {0};
     struct stat attrib;
+    cJSON *json_root = NULL;
 
     while (maat_inst->is_running) {
         if (time(NULL) % 10 == 0) {
@@ -461,13 +451,6 @@ void *rule_monitor_loop(void *arg)
                                         maat_start_cb, maat_update_cb, maat_finish_cb,
                                         maat_inst);
                 break;
-            case DATA_SOURCE_IRIS_FILE:
-                config_monitor_traverse(maat_inst->maat_version, 
-                                        maat_inst->opts.iris_ctx.inc_idx_dir,
-                                        maat_start_cb, maat_update_cb, maat_finish_cb,
-                                        maat_inst, maat_inst->opts.decrypt_key,
-                                        maat_inst->logger);
-                break;
             case DATA_SOURCE_JSON_FILE:
                 memset(md5_tmp, 0, sizeof(md5_tmp));
                 stat(maat_inst->opts.json_ctx.json_file, &attrib);
@@ -476,15 +459,15 @@ void *rule_monitor_loop(void *arg)
                     maat_inst->opts.json_ctx.last_md5_time = attrib.st_ctim;
                     md5_file(maat_inst->opts.json_ctx.json_file, md5_tmp);
                     if (0 != strcmp(md5_tmp, maat_inst->opts.json_ctx.effective_json_md5)) {
-                        ret = load_maat_json_file(maat_inst, maat_inst->opts.json_ctx.json_file, 
-                                                    err_str, sizeof(err_str));
+                        ret = load_maat_json_rule_file(maat_inst, maat_inst->opts.json_ctx.json_file, 
+                                                    json_root, err_str, sizeof(err_str));
                         if (ret < 0) {
                             log_fatal(maat_inst->logger, MODULE_MAAT_RULE, 
                                         "[%s:%d] Maat re-initiate with JSON file %s (md5=%s)failed: %s\n",
                                         __FUNCTION__, __LINE__, maat_inst->opts.json_ctx.json_file, 
                                         md5_tmp, err_str);
                         } else {
-                            config_monitor_traverse(0, maat_inst->opts.json_ctx.iris_file,
+                            config_monitor_traverse(0, json_root,
                                                     maat_start_cb, maat_update_cb, maat_finish_cb,
                                                     maat_inst, maat_inst->opts.decrypt_key,
                                                     maat_inst->logger);
@@ -492,6 +475,7 @@ void *rule_monitor_loop(void *arg)
                                         "Maat re-initiate with JSON file %s success, md5: %s\n",
                                         maat_inst->opts.json_ctx.json_file, md5_tmp);
                         }
+                        cJSON_Delete(json_root);
                     }
                 }
                 break;

src/maat_ex_data.c

@@ -35,6 +35,12 @@ struct ex_data_runtime {
 
 void cache_row_free(void *p)
 {
+    struct ex_data_row *ex_data_row = (struct ex_data_row *)(char*)p;
+
+    if (ex_data_row->row != NULL) {
+        FREE(ex_data_row->row);
+        ex_data_row->row = NULL;
+    }
 	free(*(char **)p);
 }
 
@@ -87,22 +93,24 @@ void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt)
     rcu_hash_commit(ex_data_rt->htable);
 }
 
-void ex_data_runtime_cache_row_put(struct ex_data_runtime *ex_data_rt, const char *row)
+void ex_data_runtime_cache_row_put(struct ex_data_runtime *ex_data_rt, const char *row, enum maat_operation op)
 {
     if (NULL == ex_data_rt || NULL == row) {
         return;
     }
 
 	size_t row_len = strlen(row);
-	char *row_copy = ALLOC(char, row_len + 1);
+    struct ex_data_row *ex_data_row = ALLOC(struct ex_data_row, 1);
+    ex_data_row->row = ALLOC(char, row_len + 1);
 
-	memcpy(row_copy, row, row_len);
+    ex_data_row->op = op;
+	memcpy(ex_data_row->row, row, row_len);
 	ex_data_rt->cache_size += row_len;
-	utarray_push_back(ex_data_rt->cache_rows, &row_copy);
+	utarray_push_back(ex_data_rt->cache_rows, &ex_data_row);
 	ex_data_rt->cache_row_num++;	
 }
 
-const char *ex_data_runtime_cached_row_get(struct ex_data_runtime *ex_data_rt, size_t index)
+const struct ex_data_row *ex_data_runtime_cached_row_get(struct ex_data_runtime *ex_data_rt, size_t index)
 {
     if (NULL == ex_data_rt) {
         return NULL;
@@ -110,7 +118,7 @@ const char *ex_data_runtime_cached_row_get(struct ex_data_runtime *ex_data_rt, s
 
     const char **row = NULL;
 	row = (const char **)utarray_eltptr(ex_data_rt->cache_rows, index);
-	return *row;
+	return (struct ex_data_row *)*row;
 }
 
 size_t ex_data_runtime_cached_row_count(struct ex_data_runtime *ex_data_rt)

src/maat_expr.c

@@ -34,11 +34,6 @@
 #define ENGINE_TYPE_SWITCH_THRESHOLD 50000
 
 struct expr_schema {
-	int item_id_column;
-    int object_id_column;
-	int district_column;
-	int keywords_column;
-    int expr_type_column;
     int table_id;
     enum maat_expr_engine engine_type;
     struct table_manager *ref_tbl_mgr;
@@ -157,64 +152,59 @@ int expr_runtime_set_scan_district(struct expr_runtime *expr_rt, const char *dis
 
 static struct expr_item *
 expr_item_new(struct expr_schema *expr_schema, const char *table_name,
-              const char *line, struct expr_runtime *expr_rt)
+              const cJSON *json, struct expr_runtime *expr_rt, long long item_id)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
     int expr_type = -1;
     enum table_type table_type = TABLE_TYPE_INVALID;
     struct expr_item *expr_item = ALLOC(struct expr_item, 1);
+    cJSON *tmp_obj = NULL;
+    size_t len = 0;
+    int ret;
 
-    int ret = get_column_pos(line, expr_schema->item_id_column, &column_offset,
-                             &column_len);
-    if (ret < 0) {
-        log_fatal(expr_rt->logger, MODULE_EXPR, 
-                  "[%s:%d] expr table:<%s> has no item_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
-        goto error;
-    }
-    expr_item->item_id = atoll(line + column_offset);
+    expr_item->item_id = item_id;
 
-    ret = get_column_pos(line, expr_schema->object_id_column, &column_offset,
-                         &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "object_id");
+    if (tmp_obj != NULL && tmp_obj->type == cJSON_String) {
+        expr_item->object_id = tmp_obj->valueint;
+    } else {
         log_fatal(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table:<%s> has no object_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    expr_item->object_id = atoll(line + column_offset);
+    expr_item->object_id = atoll(tmp_obj->valuestring);
 
-    ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "keywords");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
         log_fatal(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table:<%s> has no keywords in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
+    len = strlen(tmp_obj->valuestring);
 
-    if (column_len > MAX_KEYWORDS_STR_LEN) {
+    if (len > MAX_KEYWORDS_STR_LEN) {
         log_fatal(expr_rt->logger, MODULE_EXPR, 
                   "[%s:%d] expr table:<%s> keywords length too long in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    memcpy(expr_item->keywords, (line + column_offset), column_len);
+    memcpy(expr_item->keywords, tmp_obj->valuestring, len);
 
-    ret = get_column_pos(line, expr_schema->expr_type_column, &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "expr_type");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
         log_fatal(expr_rt->logger, MODULE_EXPR,
                   "[%s:%d] expr table:<%s> has no expr_type in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
 
-    expr_type = atoi(line + column_offset);
+    expr_type = atoi(tmp_obj->valuestring);
     expr_item->expr_type = int_to_expr_type(expr_type);
     if (expr_item->expr_type == EXPR_TYPE_INVALID) {
         log_fatal(expr_rt->logger, MODULE_EXPR,
                   "[%s:%d] expr table:<%s> has invalid expr_type in line:%s",
-                 __FUNCTION__, __LINE__, table_name, line);
+                 __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     } else if (expr_item->expr_type == EXPR_TYPE_REGEX) {
         ret = expr_matcher_verify_regex_expression(expr_item->keywords, expr_rt->logger);
@@ -229,21 +219,25 @@ expr_item_new(struct expr_schema *expr_schema, const char *table_name,
    
     table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id);
     if (table_type == TABLE_TYPE_EXPR_PLUS) {
-        ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len);
-        if (ret < 0) {
+        tmp_obj = cJSON_GetObjectItem(json, "district");
+        if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
+            log_fatal(expr_rt->logger, MODULE_EXPR, 
+                      "[%s:%d] expr table:<%s> has no district in line:%s",
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             goto error;
         }
 
-        if (column_len > MAX_DISTRICT_STR_LEN) {
+        len = strlen(tmp_obj->valuestring);
+        if (len > MAX_DISTRICT_STR_LEN) {
             log_fatal(expr_rt->logger, MODULE_EXPR, 
                       "[%s:%d] expr table:<%s> district length exceed maximum:%d"
                       " in line:%s", __FUNCTION__, __LINE__, table_name,
-                      MAX_DISTRICT_STR_LEN, line);
+                      MAX_DISTRICT_STR_LEN, cJSON_Print(json));
             goto error;
         }
 
         char district[MAX_DISTRICT_STR_LEN + 1] = {0};
-        memcpy(district, (line + column_offset), column_len);
+        memcpy(district, tmp_obj->valuestring, len);
         assert(strlen(district) > 0);
         str_unescape(district);
         expr_item->district_id = expr_runtime_get_district_id(expr_rt, district);
@@ -264,7 +258,6 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 	struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1);
     expr_schema->engine_type = MAAT_EXPR_ENGINE_AUTO;
 
-    cJSON *custom_item = NULL;
     cJSON *item = cJSON_GetObjectItem(json, "table_id");
     if (item != NULL && item->type == cJSON_Number) {
         expr_schema->table_id = item->valueint;
@@ -293,67 +286,6 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         }
     }
 
-    item = cJSON_GetObjectItem(json, "custom");
-    if (item == NULL || item->type != cJSON_Object) {
-        log_fatal(logger, MODULE_EXPR, 
-                  "[%s:%d] expr table:<%s> schema has no custom column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-	custom_item = cJSON_GetObjectItem(item, "item_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        expr_schema->item_id_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_EXPR, 
-                  "[%s:%d] expr table:<%s> schema has no item_id column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-	custom_item = cJSON_GetObjectItem(item, "object_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        expr_schema->object_id_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_EXPR, 
-                  "[%s:%d] expr table:<%s> schema has no object_id column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "keywords");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        expr_schema->keywords_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_EXPR, 
-                  "[%s:%d] expr table:<%s> schema has no keywords column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    /* expr_plus has district */
-    if (strcmp(table_type, "expr_plus") == 0) {
-        custom_item = cJSON_GetObjectItem(item, "district");
-        if (custom_item != NULL && custom_item->type == cJSON_Number) {
-            expr_schema->district_column = custom_item->valueint;
-        } else {
-            log_fatal(logger, MODULE_EXPR, 
-                  "[%s:%d] expr_plus table:<%s> schema has no district column", 
-                  __FUNCTION__, __LINE__, table_name);
-            goto error;
-        }
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "expr_type");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        expr_schema->expr_type_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_EXPR,
-                  "[%s:%d] expr table:<%s> schema has no expr_type column",
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
     expr_schema->ref_tbl_mgr = tbl_mgr;
 	
     return expr_schema;
@@ -489,11 +421,11 @@ void expr_runtime_free(void *expr_runtime)
 
 static int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key,
                                    size_t key_len, struct expr_item *item,
-                                   int is_valid)
+                                   enum maat_operation op)
 {
     int ret = -1;
 
-    if (0 == is_valid) {
+    if (MAAT_OP_DEL == op) {
         //delete
         rcu_hash_del(expr_rt->item_hash, key, key_len);
     } else {
@@ -729,7 +661,7 @@ static int expr_item_to_expr_rule(struct expr_item *expr_item,
 
 int expr_runtime_update(void *expr_runtime, void *expr_schema, 
                         const char *table_name, const char *line, 
-                        int valid_column)
+                        enum maat_operation op)
 {
     if (NULL == expr_runtime || NULL == expr_schema ||
         NULL == line) {
@@ -738,34 +670,43 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
 
     struct expr_schema *schema = (struct expr_schema *)expr_schema;
     struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime;
-
-    long long item_id = get_column_value(line, schema->item_id_column);
-    if (item_id < 0) {
+    cJSON *tmp_obj = NULL;
+    cJSON *json = cJSON_Parse(line);
+    
+    if (NULL == json) {
         log_fatal(expr_rt->logger, MODULE_EXPR,
-                  "[%s:%d] expr table:<%s> has no item_id(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->item_id_column, line);
+                  "[%s:%d] expr table:<%s> line:%s is not a valid json",
+                  __FUNCTION__, __LINE__, table_name, line);
         expr_rt->update_err_cnt++;
         return -1;
     }
 
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "item_id");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
         log_fatal(expr_rt->logger, MODULE_EXPR,
-                  "[%s:%d] expr table:<%s> has no is_valid(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  valid_column, line);
+                  "[%s:%d] expr table:<%s> has no item_id in line:%s",
+                  __FUNCTION__, __LINE__, table_name, line);
         expr_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
+    }
+
+    long long item_id = atoll(tmp_obj->valuestring);
+    if (item_id < 0) {
+        log_fatal(expr_rt->logger, MODULE_EXPR,
+                  "[%s:%d] expr table:<%s> item_id wrong"
+                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
+                  cJSON_Print(json));
+        expr_rt->update_err_cnt++;
+        goto ERROR;
     }
     
     struct expr_item *expr_item = NULL;
-    if (1 == is_valid) {
+    if (MAAT_OP_ADD == op) {
         //add
-        expr_item = expr_item_new(schema, table_name, line, expr_rt);
+        expr_item = expr_item_new(schema, table_name, json, expr_rt, item_id);
         if (NULL == expr_item) {
             expr_rt->update_err_cnt++;
-            return -1;
+            goto ERROR;
         }
 
         int *item_district_id = ALLOC(int, 1);
@@ -775,7 +716,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
     }
 
     int ret = expr_runtime_update_row(expr_rt, (char *)&item_id, sizeof(long long),
-                                      expr_item, is_valid);
+                                      expr_item, op);
     if (ret < 0) {
         if (expr_item != NULL) {
             expr_item_free(expr_item);
@@ -783,7 +724,13 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema,
         //don't return failed, ignore the case of adding duplicate keys
     }
 
+    cJSON_Delete(json);
     return 0;
+ERROR:
+    if (json != NULL) {
+        cJSON_Delete(json);
+    }
+    return -1;
 }
 
 static void garbage_expr_matcher_free(void *expr_matcher, void *arg)

src/maat_flag.c

@@ -25,11 +25,6 @@
 #define MODULE_FLAG	 module_name_str("maat.flag")
 
 struct flag_schema {
-    int item_id_column;
-    int object_id_column;
-    int district_column;
-    int flag_column;
-    int flag_mask_column;
     int table_id;
     struct table_manager *ref_tbl_mgr;
 };
@@ -71,7 +66,6 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
     struct flag_schema *schema = ALLOC(struct flag_schema, 1);
 
     char table_type[NAME_MAX] = {0};
-    cJSON *custom_item = NULL;
     cJSON *item = cJSON_GetObjectItem(json, "table_id");
     if (item != NULL && item->type == cJSON_Number) {
         schema->table_id = item->valueint;
@@ -86,67 +80,6 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr,
     item = cJSON_GetObjectItem(json, "table_type");
     memcpy(table_type, item->valuestring, strlen(item->valuestring));
 
-    item = cJSON_GetObjectItem(json, "custom");
-    if (item == NULL || item->type != cJSON_Object) {
-        log_fatal(logger, MODULE_FLAG, 
-                  "[%s:%d] flag table:<%s> schema has no custom column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "item_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->item_id_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_FLAG, 
-                  "[%s:%d] flag table:<%s> schema has no item_id column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "object_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->object_id_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_FLAG, 
-                  "[%s:%d] flag table:<%s> schema has no object_id column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    /* flag_plus has district */
-    if (strcmp(table_type, "flag_plus") == 0) {
-        custom_item = cJSON_GetObjectItem(item, "district");
-        if (custom_item != NULL && custom_item->type == cJSON_Number) {
-            schema->district_column = custom_item->valueint;
-        } else {
-            log_fatal(logger, MODULE_FLAG, 
-                  "[%s:%d] flag_plus table:<%s> schema has no district column", 
-                  __FUNCTION__, __LINE__, table_name);
-            goto error;
-        }
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "flag");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->flag_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_FLAG, 
-                  "[%s:%d] flag table:<%s> schema has no flag column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "flag_mask");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->flag_mask_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_FLAG, 
-                  "[%s:%d] flag table:<%s> schema has no flag_mask column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
     schema->ref_tbl_mgr = tbl_mgr;
     return schema;
 error:
@@ -253,11 +186,11 @@ void flag_runtime_free(void *flag_runtime)
 
 static int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key,
                                    size_t key_len, struct flag_item *item,
-                                   int is_valid)
+                                   enum maat_operation op)
 {
     int ret = -1;
 
-    if (0 == is_valid) {
+    if (MAAT_OP_DEL == op) {
         //delete
         rcu_hash_del(flag_rt->item_hash, key, key_len);
     } else {
@@ -309,75 +242,69 @@ int flag_runtime_set_scan_district(struct flag_runtime *flag_rt, const char *dis
 
 static struct flag_item *
 flag_item_new(struct flag_schema *schema, const char *table_name,
-              const char *line, struct flag_runtime *flag_rt)
+              const cJSON *json, struct flag_runtime *flag_rt, long long item_id)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
+    cJSON *tmp_obj = NULL;
     enum table_type table_type = TABLE_TYPE_INVALID;
     struct flag_item *item = ALLOC(struct flag_item, 1);
 
-    int ret = get_column_pos(line, schema->item_id_column, &column_offset,
-                             &column_len);
-    if (ret < 0) {
-        log_fatal(flag_rt->logger, MODULE_FLAG, 
-                  "[%s:%d] flag table:<%s> has no item_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
-        goto error;
-    }
-    item->item_id = atoll(line + column_offset);
+    item->item_id = item_id;
 
-    ret = get_column_pos(line, schema->object_id_column, &column_offset,
-                         &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "object_id");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
         log_fatal(flag_rt->logger, MODULE_FLAG, 
-                  "[%s:%d] flag table:<%s> has no object_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] flag table:<%s> has no object_id in json:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    item->object_id = atoll(line + column_offset);
+    item->object_id = atoll(tmp_obj->valuestring);
 
     table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id);
     if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
-        ret = get_column_pos(line, schema->district_column, &column_offset,
-                             &column_len);
-        if (ret < 0) {
+        tmp_obj = cJSON_GetObjectItem(json, "district");
+        if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
+            log_fatal(flag_rt->logger, MODULE_FLAG, 
+                      "[%s:%d] flag_plus table:<%s> has no district in json:%s",
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             goto error;
         }
 
-        if (column_len > MAX_DISTRICT_STR_LEN) {
+        size_t len = strlen(tmp_obj->valuestring);
+        if (len > MAX_DISTRICT_STR_LEN) {
             log_fatal(flag_rt->logger, MODULE_FLAG, 
                       "[%s:%d] flag_plus table:<%s> district length exceed "
-                      "maximum:%d in line:%s", __FUNCTION__, __LINE__,
-                      table_name, MAX_DISTRICT_STR_LEN, line);
+                      "maximum:%d in json:%s", __FUNCTION__, __LINE__,
+                      table_name, MAX_DISTRICT_STR_LEN, cJSON_Print(json));
             goto error;
         }
 
         char district[MAX_DISTRICT_STR_LEN + 1] = {0};
-        memcpy(district, (line + column_offset), column_len);
+        memcpy(district, tmp_obj->valuestring, len);
         assert(strlen(district) > 0);
         item->district_id = flag_runtime_get_district_id(flag_rt, district);
     } else {
         item->district_id = DISTRICT_ANY;
     }
 
-    ret = get_column_pos(line, schema->flag_column, &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "flag");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
         log_fatal(flag_rt->logger, MODULE_FLAG, 
-                  "[%s:%d] flag table:<%s> has no flag in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] flag table:<%s> has no flag in json:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
 
-    item->flag = strtoull(line + column_offset, NULL, 0);
+    item->flag = strtoull(tmp_obj->valuestring, NULL, 0);
 
-    ret = get_column_pos(line, schema->flag_mask_column, &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "flag_mask");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
         log_fatal(flag_rt->logger, MODULE_FLAG, 
-                  "[%s:%d] flag table:<%s> has no flag_mask in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] flag table:<%s> has no flag_mask in json:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    item->flag_mask = strtoull(line + column_offset, NULL, 0);
+    
+    item->flag_mask = strtoull(tmp_obj->valuestring, NULL, 0);
 
     return item;
 error:
@@ -399,7 +326,7 @@ static struct flag_rule flag_item_to_flag_rule(struct flag_item *item)
 
 int flag_runtime_update(void *flag_runtime, void *flag_schema,
                         const char *table_name, const char *line,
-                        int valid_column)
+                        enum maat_operation op)
 {
     if (NULL == flag_runtime || NULL == flag_schema ||
         NULL == line) {
@@ -409,31 +336,30 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
     struct flag_schema *schema = (struct flag_schema *)flag_schema;
     struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime;
 
-    long long item_id = get_column_value(line, schema->item_id_column);
-    if (item_id < 0) {
+    cJSON *json = cJSON_Parse(line);
+    if (NULL == json) {
         log_fatal(flag_rt->logger, MODULE_FLAG,
-                  "[%s:%d] flag table:<%s> has no item_id(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->item_id_column, line);
+                  "[%s:%d] flag table:<%s> line:%s is not a valid json",
+                  __FUNCTION__, __LINE__, table_name, line);
         return -1;
     }
 
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
+    cJSON *item = cJSON_GetObjectItem(json, "item_id");
+    if (NULL == item || item->type != cJSON_String) {
         log_fatal(flag_rt->logger, MODULE_FLAG,
-                  "[%s:%d] flag table:<%s> has no is_valid(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  valid_column, line);
-        flag_rt->update_err_cnt++;
-        return -1;
+                  "[%s:%d] flag table:<%s> line:%s has no item_id",
+                  __FUNCTION__, __LINE__, table_name, line);
+        goto ERROR;
     }
 
+    long long item_id = atoll(item->valuestring);
+
     struct flag_item *flag_item = NULL;
-    if (1 == is_valid) {
-        flag_item = flag_item_new(schema, table_name, line, flag_rt);
+    if (MAAT_OP_ADD == op) {
+        flag_item = flag_item_new(schema, table_name, json, flag_rt, item_id);
         if (NULL == flag_item) {
             flag_rt->update_err_cnt++;
-            return -1;
+            goto ERROR;
         }
 
         int *item_district_id = ALLOC(int, 1);
@@ -443,7 +369,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
     }
 
     int ret = flag_runtime_update_row(flag_rt, (char *)&item_id, sizeof(long long),
-                                      flag_item, is_valid);
+                                      flag_item, op);
     if (ret < 0) {
         if (flag_item != NULL) {
             flag_item_free(flag_item);
@@ -451,7 +377,14 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema,
         //don't return failed, ignore the case of adding duplicate keys
     }
 
+    cJSON_Delete(json);
+
     return 0;
+ERROR:
+    if (json != NULL) {
+        cJSON_Delete(json);
+    }
+    return -1;
 }
 
 static void garbage_flag_matcher_free(void *flag_matcher, void *arg)

src/maat_fqdn_plugin.c

@@ -20,11 +20,9 @@
 #define MODULE_FQDN_PLUGIN	 module_name_str("maat.bool_plugin")
 
 struct fqdn_plugin_schema {
-    int item_id_column;
-    int fqdn_column;
-    int rule_tag_column;
     int gc_timeout_s;
     int table_id;
+    char key_name[MAX_NAME_STR_LEN];
     struct ex_container_schema container_schema;
     struct table_manager *ref_tbl_mgr;
     struct log_handle *logger;
@@ -67,9 +65,9 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         goto error;
     }
 
-    custom_item = cJSON_GetObjectItem(item, "item_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->item_id_column = custom_item->valueint;
+    custom_item = cJSON_GetObjectItem(item, "key_name");
+    if (custom_item != NULL && custom_item->type == cJSON_String) {
+        strncpy(schema->key_name, custom_item->valuestring, sizeof(schema->key_name) - 1);
     } else {
         log_fatal(logger, MODULE_FQDN_PLUGIN, 
                   "[%s:%d] fqdn_plugin table:<%s> schema has no item_id column", 
@@ -77,23 +75,6 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         goto error;
     }
 
-    // rule_tag is optional
-    custom_item = cJSON_GetObjectItem(item, "fqdn");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->fqdn_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_FQDN_PLUGIN, 
-                  "[%s:%d] fqdn_plugin table:<%s> schema has no fqdn column",
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    // rule_tag is optional
-    custom_item = cJSON_GetObjectItem(item, "rule_tag");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->rule_tag_column = custom_item->valueint;
-    }
-
     //gc_timeout_s is optional
     custom_item = cJSON_GetObjectItem(item, "gc_timeout_s");
     if (custom_item != NULL && custom_item->type == cJSON_Number) {
@@ -223,34 +204,30 @@ void fqdn_plugin_runtime_free(void *fqdn_plugin_runtime)
 
 static int
 fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema,
-                             const char *table_name, const char *line,
+                             const char *table_name, const cJSON *json,
                              struct log_handle *logger)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
     size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr);
+    cJSON *tmp_obj = NULL;
 
-    if (schema->rule_tag_column > 0 && n_tag > 0) {
-        int ret = get_column_pos(line, schema->rule_tag_column, 
-                                 &column_offset, &column_len);
-        if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "tag");
+    if (tmp_obj != NULL && n_tag > 0) {
+        if (tmp_obj->type != cJSON_String) {
             log_fatal(logger, MODULE_FQDN_PLUGIN,
-                      "[%s:%d] fqdn_plugin table:<%s> has no rule_tag"
+                      "[%s:%d] fqdn_plugin table:<%s> has invalid tag format"
                       " in line:%s", __FUNCTION__, __LINE__, table_name,
-                      line);
+                      cJSON_Print(json));
             return TAG_MATCH_ERR;
         }
 
-        if (column_len > 2) {
-            char *tag_str = ALLOC(char, column_len + 1);
-            memcpy(tag_str, (line + column_offset), column_len);
-            ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str);
-            FREE(tag_str);
+        const char *tag = tmp_obj->valuestring;
+        if (strlen(tag) > 2) {
+            int ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag);
             if (TAG_MATCH_ERR == ret) {
                 log_fatal(logger, MODULE_FQDN_PLUGIN,
-                          "[%s:%d] fqdn_plugin table:<%s> has invalid "
-                          "tag format in line:%s", __FUNCTION__, __LINE__,
-                          table_name, line);
+                          "[%s:%d] fqdn_plugin table:<%s> has invalid tag"
+                          " format in line:%s", __FUNCTION__, __LINE__,
+                          table_name, cJSON_Print(json));
                 return TAG_MATCH_ERR;
             }
 
@@ -258,7 +235,7 @@ fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema,
                 log_fatal(logger, MODULE_FQDN_PLUGIN,
                           "[%s:%d] fqdn_plugin table:<%s> has unmatched tag"
                           " in line:%s", __FUNCTION__, __LINE__, table_name,
-                          line);
+                          cJSON_Print(json));
                 return TAG_MATCH_UNMATCHED;
             }
         }
@@ -268,41 +245,38 @@ fqdn_plugin_accept_tag_match(struct fqdn_plugin_schema *schema,
 }
 
 static struct FQDN_rule *
-fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
+fqdn_plugin_rule_new(const cJSON *json, struct fqdn_plugin_schema *schema,
                      const char *table_name, struct log_handle *logger)
 {
-     int ret = fqdn_plugin_accept_tag_match(schema, table_name, line, logger);
+     int ret = fqdn_plugin_accept_tag_match(schema, table_name, json, logger);
     if (ret == TAG_MATCH_UNMATCHED) {
         return NULL;
     }
 
-    size_t column_offset = 0;
-    size_t column_len = 0;
     const char *fqdn = NULL;
     size_t fqdn_len = 0;
     struct FQDN_rule *fqdn_plugin_rule = ALLOC(struct FQDN_rule, 1);
+    cJSON *tmp_obj = NULL;
 
-    ret = get_column_pos(line, schema->item_id_column,
-                         &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, schema->key_name);
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_FQDN_PLUGIN, 
-                  "[%s:%d] fqdn_plugin table:<%s> has no item_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] fqdn_plugin table:<%s> has no key_name or invalid format in line:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    fqdn_plugin_rule->id = atoi(line + column_offset);
+    fqdn_plugin_rule->id = atoi(tmp_obj->valuestring);
 
-    ret = get_column_pos(line, schema->fqdn_column,
-                         &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "fqdn");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_FQDN_PLUGIN, 
                   "[%s:%d] fqdn_plugin table:<%s> has no fqdn in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
 
-    fqdn = line + column_offset;
-    fqdn_len = column_len;
+    fqdn = tmp_obj->valuestring;
+    fqdn_len = strlen(fqdn);
 
     if (fqdn[0] == '*') {
         fqdn_plugin_rule->is_suffix_match = 1;
@@ -334,12 +308,12 @@ fqdn_plugin_runtime_update_row(struct fqdn_plugin_runtime *fqdn_plugin_rt,
                                const char *table_name, const char *row, 
                                const char *key, size_t key_len, 
                                struct FQDN_rule *fqdn_plugin_rule,
-                               int is_valid)
+                               enum maat_operation op)
 {
     int ret = -1;
     struct ex_data_runtime *ex_data_rt = fqdn_plugin_rt->ex_data_rt;
 
-    if (0 == is_valid) {
+    if (MAAT_OP_DEL == op) {
         // delete
         ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len);
         if (ret < 0) {
@@ -370,7 +344,7 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime,
                                void *fqdn_plugin_schema,
                                const char *table_name,
                                const char *line,
-                               int valid_column)
+                               enum maat_operation op)
 {
     if (NULL == fqdn_plugin_runtime || NULL == fqdn_plugin_schema || 
         NULL == line) {
@@ -384,64 +358,72 @@ int fqdn_plugin_runtime_update(void *fqdn_plugin_runtime,
     struct fqdn_plugin_runtime *fqdn_plugin_rt =
         (struct fqdn_plugin_runtime *)fqdn_plugin_runtime;
 
-    size_t item_id_offset = 0, item_id_len = 0;
+    int ret = 0;
+    cJSON *tmp_obj = NULL;
+    cJSON *json = cJSON_Parse(line);
 
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
+    if (NULL == json) {
         log_fatal(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
-                  "[%s:%d] fqdn_plugin table:<%s> has no is_valid(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  valid_column, line);
+                  "[%s:%d] fqdn_plugin table:<%s> line is not a json format, line: %s",
+                  __FUNCTION__, __LINE__, table_name, line);
         fqdn_plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
-    int ret = get_column_pos(line, schema->item_id_column,
-                             &item_id_offset, &item_id_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, schema->key_name);
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
-                  "[%s:%d] fqdn_plugin table:<%s> has no item_id(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->item_id_column, line);
+                  "[%s:%d] fqdn_plugin table:<%s> has no key_name or invalid format in line:%s",
+                  __FUNCTION__, __LINE__, table_name, line);
         fqdn_plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
     if (1 == schema->container_schema.set_flag) {
-        if (1 == is_valid) {
+        if (MAAT_OP_ADD == op) {
             // add
-            fqdn_plugin_rule = fqdn_plugin_rule_new(line, schema, table_name,
+            fqdn_plugin_rule = fqdn_plugin_rule_new(json, schema, table_name,
                                                     fqdn_plugin_rt->logger);
             if (NULL == fqdn_plugin_rule) {
                 fqdn_plugin_rt->update_err_cnt++;
-                return -1;
+                goto ERROR;
             }
         }
 
-        const char *key = line + item_id_offset;
-        size_t key_len = item_id_len;
+        const char *key = tmp_obj->valuestring;
+        size_t key_len = strlen(key);
         ret = fqdn_plugin_runtime_update_row(fqdn_plugin_rt, table_name, line,
                                              key, key_len, fqdn_plugin_rule,
-                                             is_valid);
+                                             op);
         if (ret < 0) {
             log_fatal(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
                       "[%s:%d]fqdn_plugin table:<%s> update one line failed, "
                       "line:%s", __FUNCTION__, __LINE__, table_name, line);
             fqdn_plugin_rt->update_err_cnt++;
-            return -1;
+            goto ERROR;
         }
 
         log_debug(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
                   "fqdn_plugin table:<%s> update one line, key:%s, key_len:%zu,"
-                  "is_valid:%d", table_name, key, key_len, is_valid);
+                  "maat_op:%d", table_name, key, key_len, op);
     } else {
         //ex_schema not set
-        ex_data_runtime_cache_row_put(fqdn_plugin_rt->ex_data_rt, line);
+        ex_data_runtime_cache_row_put(fqdn_plugin_rt->ex_data_rt, line, op);
         fqdn_plugin_rt->rule_num =
             ex_data_runtime_cached_row_count(fqdn_plugin_rt->ex_data_rt);
     }
 
     return 0;
+
+ERROR:
+    if (fqdn_plugin_rule != NULL) {
+        fqdn_rule_free(fqdn_plugin_rule);
+    }
+    if (json != NULL) {
+        cJSON_Delete(json);
+    }
+
+    return -1;
 }
 
 static void garbage_fqdn_engine_free(void *fqdn_engine, void *arg)

src/maat_interval.c

@@ -22,10 +22,6 @@
 #define MODULE_INTERVAL	 module_name_str("maat.interval")
 
 struct interval_schema {
-    int item_id_column;
-    int object_id_column;
-    int district_column;
-    int interval_column;
     int table_id;
     struct table_manager *ref_tbl_mgr;
 };
@@ -67,7 +63,6 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
     struct interval_schema *schema = ALLOC(struct interval_schema, 1);
 
     char table_type[NAME_MAX] = {0};
-    cJSON *custom_item = NULL;
     cJSON *item = cJSON_GetObjectItem(json, "table_id");
     if (item != NULL && item->type == cJSON_Number) {
         schema->table_id = item->valueint;
@@ -82,57 +77,6 @@ void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr,
     item = cJSON_GetObjectItem(json, "table_type");
     memcpy(table_type, item->valuestring, strlen(item->valuestring));
 
-    item = cJSON_GetObjectItem(json, "custom");
-    if (item == NULL || item->type != cJSON_Object) {
-        log_fatal(logger, MODULE_INTERVAL, 
-                  "[%s:%d] interval table:<%s> schema has no custom column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "item_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->item_id_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_INTERVAL, 
-                  "[%s:%d] interval table:<%s> schema has no item_id column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "object_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->object_id_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_INTERVAL, 
-                  "[%s:%d] interval table:<%s> schema has no object_id column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    /* interval_plus has district */
-    if (strcmp(table_type, "interval_plus") == 0) {
-        custom_item = cJSON_GetObjectItem(item, "district");
-        if (custom_item != NULL && custom_item->type == cJSON_Number) {
-            schema->district_column = custom_item->valueint;
-        } else {
-            log_fatal(logger, MODULE_INTERVAL, 
-                  "[%s:%d] interval_plus table:<%s> schema has no district column", 
-                  __FUNCTION__, __LINE__, table_name);
-            goto error;
-        }
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "interval");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->interval_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_INTERVAL, 
-                  "[%s:%d] interval table:<%s> schema has no interval column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
     schema->ref_tbl_mgr = tbl_mgr;
     return schema;
 error:
@@ -271,66 +215,60 @@ int interval_runtime_set_scan_district(struct interval_runtime *interval_rt,
 
 static struct interval_item *
 interval_item_new(struct interval_schema *schema, const char *table_name,
-                  const char *line, struct interval_runtime *interval_rt)
+                  const cJSON *json, struct interval_runtime *interval_rt, long long item_id)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
     enum table_type table_type = TABLE_TYPE_INVALID;
     char port_str[16] = {0};
     struct interval_item *item = ALLOC(struct interval_item, 1);
+    cJSON *tmp_obj = NULL;
 
-    int ret = get_column_pos(line, schema->item_id_column, &column_offset,
-                             &column_len);
-    if (ret < 0) {
-        log_fatal(interval_rt->logger, MODULE_INTERVAL, 
-                  "[%s:%d] interval table:<%s> has no item_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
-        goto error;
-    }
-    item->item_id = atoll(line + column_offset);
+    item->item_id = item_id;
 
-    ret = get_column_pos(line, schema->object_id_column, &column_offset,
-                         &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "object_id");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(interval_rt->logger, MODULE_INTERVAL, 
                   "[%s:%d] interval table:<%s> has no object_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    item->object_id = atoll(line + column_offset);
+    item->object_id = atoll(tmp_obj->valuestring);
 
     table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id);
     if (table_type == TABLE_TYPE_INTERVAL_PLUS) {
-        ret = get_column_pos(line, schema->district_column, &column_offset,
-                             &column_len);
-        if (ret < 0) {
+        tmp_obj = cJSON_GetObjectItem(json, "district");
+        if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
+            log_fatal(interval_rt->logger, MODULE_INTERVAL, 
+                      "[%s:%d] interval_plus table:<%s> has no district in line:%s",
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             goto error;
         }
 
-        if (column_len > MAX_DISTRICT_STR_LEN) {
+        size_t len = strlen(tmp_obj->valuestring);
+
+        if (len > MAX_DISTRICT_STR_LEN) {
             log_fatal(interval_rt->logger, MODULE_INTERVAL, 
                       "[%s:%d] interval_plus table:<%s> district length exceed "
                       "maximum:%d in line:%s", __FUNCTION__, __LINE__, table_name,
-                      MAX_DISTRICT_STR_LEN, line);
+                      MAX_DISTRICT_STR_LEN, cJSON_Print(json));
             goto error;
         }
 
         char district[MAX_DISTRICT_STR_LEN + 1] = {0};
-        memcpy(district, (line + column_offset), column_len);
+        memcpy(district, tmp_obj->valuestring, len);
         assert(strlen(district) > 0);
         item->district_id = interval_runtime_get_district_id(interval_rt, district);
     } else {
         item->district_id = DISTRICT_ANY;
     }
 
-    ret = get_column_pos(line, schema->interval_column, &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "interval");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(interval_rt->logger, MODULE_INTERVAL, 
                   "[%s:%d] interval table:<%s> has no interval in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    memcpy(port_str, line + column_offset, column_len);
+    memcpy(port_str, tmp_obj->valuestring, strlen(tmp_obj->valuestring));
     if (strchr(port_str, '-') != NULL) {
         sscanf(port_str, "%d-%d", &item->low_boundary, &item->up_boundary);
     } else {
@@ -359,11 +297,11 @@ interval_item_to_interval_rule(struct interval_item *item)
 
 static int interval_runtime_update_row(struct interval_runtime *interval_rt,
                                        char *key, size_t key_len,
-                                       struct interval_item *item, int is_valid)
+                                       struct interval_item *item, enum maat_operation op)
 {
     int ret = -1;
 
-    if (0 == is_valid) {
+    if (MAAT_OP_DEL == op) {
         //delete
         rcu_hash_del(interval_rt->item_hash, key, key_len);
     } else {
@@ -383,7 +321,7 @@ static int interval_runtime_update_row(struct interval_runtime *interval_rt,
 
 int interval_runtime_update(void *interval_runtime, void *interval_schema, 
                             const char *table_name, const char *line, 
-                            int valid_column)
+                            enum maat_operation op)
 {
     if (NULL == interval_runtime || NULL == interval_schema ||
         NULL == line) {
@@ -392,34 +330,44 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
 
     struct interval_schema *schema = (struct interval_schema *)interval_schema;
     struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime;
-
-    long long item_id = get_column_value(line, schema->item_id_column);
-    if (item_id < 0) {
+    cJSON *tmp_obj = NULL;
+    cJSON *json = NULL;
+    
+    json = cJSON_Parse(line);
+    if (NULL == json) {
         log_fatal(interval_rt->logger, MODULE_INTERVAL,
-                  "[%s:%d] interval table:<%s> has no item_id(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->item_id_column, line);
+                  "[%s:%d] interval table:<%s> line:%s is not a valid json",
+                  __FUNCTION__, __LINE__, table_name, line);
         interval_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "item_id");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(interval_rt->logger, MODULE_INTERVAL,
-                  "[%s:%d] interval table:<%s> has no is_valid(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  valid_column, line);
+                  "[%s:%d] interval table:<%s> has no item_id in line:%s",
+                  __FUNCTION__, __LINE__, table_name, line);
         interval_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
+    }
+
+    long long item_id = atoll(tmp_obj->valuestring);
+    if (item_id < 0) {
+        log_fatal(interval_rt->logger, MODULE_INTERVAL,
+                  "[%s:%d] interval table:<%s> item_id wrong"
+                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
+                  line);
+        interval_rt->update_err_cnt++;
+        goto ERROR;
     }
     
     struct interval_item *interval_item = NULL;
-    if (1 == is_valid) {
+    if (MAAT_OP_ADD == op) {
         //add
-        interval_item = interval_item_new(schema, table_name, line, interval_rt);
+        interval_item = interval_item_new(schema, table_name, json, interval_rt, item_id);
         if (NULL == interval_item) {
             interval_rt->update_err_cnt++;
-            return -1;
+            goto ERROR;
         }
 
         int *item_district_id = ALLOC(int, 1);
@@ -429,7 +377,7 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
     }
 
     int ret = interval_runtime_update_row(interval_rt, (char *)&item_id, sizeof(long long),
-                                          interval_item, is_valid);
+                                          interval_item, op);
     if (ret < 0) {
         if (interval_item != NULL) {
             interval_item_free(interval_item);
@@ -437,7 +385,14 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema,
         //don't return failed, ignore the case of adding duplicate keys
     }
 
+    cJSON_Delete(json);
     return 0;
+    
+ERROR:
+    if (json != NULL) {
+        cJSON_Delete(json);
+    }
+    return -1;
 }
 
 void garbage_interval_matcher_free(void *interval_matcher, void *arg)

src/maat_ip.c

@@ -24,11 +24,7 @@
 #define MODULE_IP	 module_name_str("maat.ip")
 
 struct ip_schema {
-    int item_id_column;
-    int object_id_column;
-    int ip_column;
     int table_id;
-    int port_column;
     struct table_manager *ref_tbl_mgr;
 };
 
@@ -78,7 +74,6 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 {
     struct ip_schema *ip_schema = ALLOC(struct ip_schema, 1);
 
-    cJSON *custom_item = NULL;
     cJSON *item = cJSON_GetObjectItem(json, "table_id");
     if (item != NULL && item->type == cJSON_Number) {
         ip_schema->table_id = item->valueint;
@@ -89,50 +84,6 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         goto error;
     }
 
-    item = cJSON_GetObjectItem(json, "custom");
-    if (NULL == item || item->type != cJSON_Object) {
-        log_fatal(logger, MODULE_IP, 
-                  "[%s:%d] ip table:<%s> schema has no custom column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "item_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        ip_schema->item_id_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_IP, 
-                  "[%s:%d] ip table:<%s> schema has no item_id column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "object_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        ip_schema->object_id_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_IP, 
-                  "[%s:%d] ip table:<%s> schema has no object_id column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "ip");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        ip_schema->ip_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_IP, 
-                  "[%s:%d] ip table:<%s> schema has no ip column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "port");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        ip_schema->port_column = custom_item->valueint;
-    } else {
-        ip_schema->port_column = 0;
-    }
     ip_schema->ref_tbl_mgr = tbl_mgr;
     return ip_schema;
 error:
@@ -147,42 +98,32 @@ void ip_schema_free(void *ip_schema)
 
 static struct ip_item *
 ip_item_new(struct ip_schema *ip_schema, const char *table_name,
-            const char *line, struct log_handle *logger)
+            const cJSON *json, struct log_handle *logger, long long item_id)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
     char ip_str[128] = {0};
     struct ip_item *ip_item = ALLOC(struct ip_item, 1);
+    cJSON *tmp_obj = NULL;
+    int ret = 0;
     
-    int ret = get_column_pos(line, ip_schema->item_id_column, &column_offset,
-                             &column_len);
-    if (ret < 0) {
-        log_fatal(logger, MODULE_IP, 
-                  "[%s:%d] ip table:<%s> has no item_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
-        goto error;
-    }
-    ip_item->item_id = atoll(line + column_offset);
+    ip_item->item_id = item_id;
 
-    ret = get_column_pos(line, ip_schema->object_id_column, &column_offset,
-                         &column_len);
-    if (ret < 0) {
-        log_fatal(logger, MODULE_IP,
+    tmp_obj = cJSON_GetObjectItem(json, "object_id");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
+        log_fatal(logger, MODULE_IP, 
                   "[%s:%d] ip table:<%s> has no object_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    ip_item->object_id = atoll(line + column_offset);
+    ip_item->object_id = atoll(tmp_obj->valuestring);
 
-    ret = get_column_pos(line, ip_schema->ip_column, &column_offset,
-                         &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "ip");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_IP, 
-                  "[%s:%d] ip table:<%s> has no ip1 in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] ip table:<%s> has no ip in line:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    memcpy(ip_str, (line + column_offset), column_len);
+    memcpy(ip_str, tmp_obj->valuestring, strlen(tmp_obj->valuestring));
 
     if (strchr(ip_str, ':') != NULL) {
 		ip_item->addr_type = IPV6;
@@ -195,7 +136,7 @@ ip_item_new(struct ip_schema *ip_schema, const char *table_name,
         if (ret < 0) {
             log_fatal(logger, MODULE_IP, 
                       "[%s:%d] ip table:<%s> ip_format2range(ip4) failed in line:%s",
-                      __FUNCTION__, __LINE__, table_name, line);
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             goto error;
         }
     } else {
@@ -204,23 +145,16 @@ ip_item_new(struct ip_schema *ip_schema, const char *table_name,
         if (ret < 0) {
             log_fatal(logger, MODULE_IP, 
                       "[%s:%d] ip table:<%s> ip_format2range(ip6) failed in line:%s",
-                      __FUNCTION__, __LINE__, table_name, line);
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             goto error;
         }
     }
 
-    if(ip_schema->port_column>0)
+    tmp_obj = cJSON_GetObjectItem(json, "port");
+    if(tmp_obj && tmp_obj->type == cJSON_String)
     {
-        ret = get_column_pos(line, ip_schema->port_column, &column_offset,
-                         &column_len);
-        if (ret < 0) {
-            log_fatal(logger, MODULE_IP, 
-                      "[%s:%d] ip table:<%s> has no port in line:%s",
-                      __FUNCTION__, __LINE__, table_name, line);
-            goto error;
-        }
         char port_range[20] = {0};
-        memcpy(port_range, (line + column_offset), column_len);
+        memcpy(port_range, tmp_obj->valuestring, strlen(tmp_obj->valuestring));
 
         //port range is port or port_start-port_end
         if(strchr(port_range,'-')!=NULL){
@@ -338,11 +272,11 @@ static void ip_item_to_ip_rule(struct ip_item *item, struct ip_rule *rule)
 }
 
 static int ip_runtime_update_row(struct ip_runtime *ip_rt, char *key, size_t key_len, 
-                                 struct ip_item *item, int is_valid)
+                                 struct ip_item *item, enum maat_operation op)
 {
     int ret = -1;
     
-    if (0 == is_valid) {
+    if (MAAT_OP_DEL == op) {
         // delete
         rcu_hash_del(ip_rt->item_hash, key, key_len);
     } else {
@@ -361,7 +295,7 @@ static int ip_runtime_update_row(struct ip_runtime *ip_rt, char *key, size_t key
 
 int ip_runtime_update(void *ip_runtime, void *ip_schema, 
                       const char *table_name, const char *line, 
-                      int valid_column)
+                      enum maat_operation op)
 {
     if (NULL == ip_runtime || NULL == ip_schema || NULL == line) {
         return -1;
@@ -369,39 +303,40 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
 
     struct ip_schema *schema = (struct ip_schema *)ip_schema;
     struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime;
-
-    long long item_id = get_column_value(line, schema->item_id_column);
-    if (item_id < 0) {
+    cJSON *json = NULL;
+    cJSON *tmp_obj = NULL;
+    
+    json = cJSON_Parse(line);
+    if (NULL == json) {
         log_fatal(ip_rt->logger, MODULE_IP,
-                  "[%s:%d] ip table:<%s> has no item_id(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->item_id_column, line);
+                  "[%s:%d] ip table:<%s> line is not a valid json string",
+                  __FUNCTION__, __LINE__, table_name);
         ip_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "item_id");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(ip_rt->logger, MODULE_IP,
-                  "[%s:%d] ip table:<%s> has no is_valid(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  valid_column, line);
+                  "[%s:%d] ip table:<%s> has no item_id in line:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         ip_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
+    long long item_id = atoll(tmp_obj->valuestring);
     
     struct ip_item *ip_item = NULL;
-    if (1 == is_valid) {
+    if (MAAT_OP_ADD == op) {
         //add
-        ip_item = ip_item_new(schema, table_name, line, ip_rt->logger);
+        ip_item = ip_item_new(schema, table_name, json, ip_rt->logger, item_id);
         if (NULL == ip_item) {
             ip_rt->update_err_cnt++;
-            return -1;
+            goto ERROR;
         }
     }
     
     int ret = ip_runtime_update_row(ip_rt, (char *)&item_id, sizeof(long long),
-                                    ip_item, is_valid);
+                                    ip_item, op);
     if (ret < 0) {
         if (ip_item != NULL) {
             ip_item_free(ip_item);
@@ -409,7 +344,13 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema,
         //don't return failed, ignore the case of adding duplicate keys
     }
 
+    cJSON_Delete(json);
     return 0;
+ERROR:
+    if (json != NULL) {
+        cJSON_Delete(json);
+    }
+    return -1;
 }
 
 void garbage_ip_matcher_free(void *ip_matcher, void *arg)

src/maat_ip_plugin.c

@@ -22,11 +22,9 @@
 #define MAX_IP_STR    128
 
 struct ip_plugin_schema {
-    int item_id_column;
-	int ip_column;
-	int rule_tag_column;
     int gc_timeout_s;
     int table_id;
+    char key_name[MAX_NAME_STR_LEN];
 	struct ex_container_schema container_schema;
     struct table_manager *ref_tbl_mgr;
     struct log_handle *logger;
@@ -71,33 +69,15 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         goto error;
     }
 
-    custom_item = cJSON_GetObjectItem(item, "item_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->item_id_column = custom_item->valueint;
-    } else {
+    custom_item = cJSON_GetObjectItem(item, "key_name");
+    if (custom_item == NULL || custom_item->type != cJSON_String) {
         log_fatal(logger, MODULE_IP_PLUGIN, 
                   "[%s:%d] ip_plugin table:<%s> schema has no"
-                  " item_id column", __FUNCTION__, __LINE__,
+                  " key_name column", __FUNCTION__, __LINE__,
                   table_name);
         goto error;
     }
-
-    custom_item = cJSON_GetObjectItem(item, "ip");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->ip_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_IP_PLUGIN, 
-                  "[%s:%d] ip_plugin table:<%s> schema has no"
-                  " end_ip column", __FUNCTION__, __LINE__,
-                  table_name);
-        goto error;
-    }
-
-    // rule_tag is optional
-    custom_item = cJSON_GetObjectItem(item, "rule_tag");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->rule_tag_column = custom_item->valueint;
-    }
+    strncpy(schema->key_name, custom_item->valuestring, sizeof(schema->key_name) - 1);
 
     //gc_timeout_s is optional
     custom_item = cJSON_GetObjectItem(item, "gc_timeout_s");
@@ -124,80 +104,71 @@ void ip_plugin_schema_free(void *ip_plugin_schema)
 
 static int
 ip_plugin_accept_tag_match(struct ip_plugin_schema *schema,
-                           const char *table_name, const char *line,
+                           const char *table_name, const cJSON *json,
                            struct log_handle *logger)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
+    cJSON *tmp_obj = NULL;
     size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr);
 
-    if (schema->rule_tag_column > 0 && n_tag > 0) {
-        int ret = get_column_pos(line, schema->rule_tag_column, 
-                                 &column_offset, &column_len);
-        if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "tag");
+    if (tmp_obj && n_tag > 0) {
+        if (tmp_obj->type != cJSON_String) {
             log_fatal(logger, MODULE_IP_PLUGIN,
-                      "[%s:%d] ip_plugin table:<%s> has no rule_tag in line:%s",
-                      __FUNCTION__, __LINE__, table_name, line);
+                      "[%s:%d] ip_plugin table:<%s> has invalid tag format"
+                      " in line:%s", __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             return TAG_MATCH_ERR;
         }
 
-        if (column_len > 2) {
-            char *tag_str = ALLOC(char, column_len + 1);
-            memcpy(tag_str, (line + column_offset), column_len);
-            ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str);
-            FREE(tag_str);
-            if (TAG_MATCH_ERR == ret) {
-                log_fatal(logger, MODULE_IP_PLUGIN,
-                          "[%s:%d] ip_plugin table:<%s> has invalid tag format"
-                          " in line:%s", __FUNCTION__, __LINE__, table_name, line);
-                return TAG_MATCH_ERR;
-            }
+        int ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tmp_obj->valuestring);
+        if (TAG_MATCH_ERR == ret) {
+            log_fatal(logger, MODULE_IP_PLUGIN,
+                      "[%s:%d] ip_plugin table:<%s> has invalid tag format"
+                      " in line:%s", __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
+            return TAG_MATCH_ERR;
+        }
 
-            if (TAG_MATCH_UNMATCHED == ret) {
-                log_fatal(logger, MODULE_IP_PLUGIN,
-                          "[%s:%d] ip_plugin table:<%s> has unmatched tag in line:%s",
-                          __FUNCTION__, __LINE__, table_name, line);
-                return TAG_MATCH_UNMATCHED;
-            }
+        if (TAG_MATCH_UNMATCHED == ret) {
+            log_fatal(logger, MODULE_IP_PLUGIN,
+                      "[%s:%d] ip_plugin table:<%s> has unmatched tag in line:%s",
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
+            return TAG_MATCH_UNMATCHED;
         }
     }
 
+
     return TAG_MATCH_MATCHED;
 }
 
 static struct ip_rule *
 ip_plugin_rule_new(struct ip_plugin_schema *schema, const char *table_name,
-                   const char *line, struct log_handle *logger)
+                   const cJSON *json, struct log_handle *logger)
 {
-    int ret = ip_plugin_accept_tag_match(schema, table_name, line, logger);
+    int ret = ip_plugin_accept_tag_match(schema, table_name, json, logger);
     if (ret == TAG_MATCH_UNMATCHED) {
         return NULL;
     }
 
-    size_t column_offset = 0;
-    size_t column_len = 0;
     char ip_str[128] = {0};
+    cJSON *tmp_obj = NULL;
     struct ip_rule *ip_plugin_rule = ALLOC(struct ip_rule, 1);
 
-    ret = get_column_pos(line, schema->item_id_column,
-                         &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, schema->key_name);
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_IP_PLUGIN, 
-                  "[%s:%d] ip_plugin table:<%s> has no item_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] ip_plugin table:<%s> has no key %s or invalid format in line:%s",
+                  __FUNCTION__, __LINE__, table_name, schema->key_name, cJSON_Print(json));
         goto error;
     }
-    ip_plugin_rule->rule_id = atoll(line + column_offset);
+    ip_plugin_rule->rule_id = atoll(tmp_obj->valuestring);
     
-    ret = get_column_pos(line, schema->ip_column,
-                         &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "ip");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_IP_PLUGIN, 
-                  "[%s:%d] ip_plugin table:<%s> has no end_ip in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] ip_plugin table:<%s> has no ip field or invalid format in line:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    strncpy(ip_str, line + column_offset, column_len);
+    strncpy(ip_str, tmp_obj->valuestring, strlen(tmp_obj->valuestring));
 
     if (strchr(ip_str, ':') != NULL) {
 		ip_plugin_rule->type = IPV6;
@@ -211,7 +182,7 @@ ip_plugin_rule_new(struct ip_plugin_schema *schema, const char *table_name,
             log_fatal(logger, MODULE_IP_PLUGIN, 
                       "[%s:%d] ip_plugin table:<%s>> ip_format2range(ip4)"
                       " failed in line:%s", __FUNCTION__, __LINE__,
-                      table_name, line);
+                      table_name, cJSON_Print(json));
             goto error;
         }
     } else {
@@ -221,7 +192,7 @@ ip_plugin_rule_new(struct ip_plugin_schema *schema, const char *table_name,
             log_fatal(logger, MODULE_IP_PLUGIN, 
                       "[%s:%d] ip_plugin table:<%s> ip_format2range(ip6)"
                       " failed in line:%s", __FUNCTION__, __LINE__,
-                      table_name, line);
+                      table_name, cJSON_Print(json));
             goto error;
         }
     }
@@ -279,7 +250,7 @@ static int
 ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt,
                              const char *table_name, const char *row, 
                              const char *key, size_t key_len, 
-                             struct ip_rule *ip_plugin_rule, int is_valid)
+                             struct ip_rule *ip_plugin_rule, enum maat_operation op)
 {
     int ret = -1;
     struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt;
@@ -287,7 +258,7 @@ ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt,
         return -1;
     }
 
-    if (0 == is_valid) {
+    if (MAAT_OP_DEL == op) {
         // delete
         ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len);
         if (ret < 0) {
@@ -372,7 +343,7 @@ void ip_plugin_runtime_free(void *ip_plugin_runtime)
 
 int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema, 
                              const char *table_name, const char *line, 
-                             int valid_column)
+                             enum maat_operation op)
 {
     if (NULL == ip_plugin_runtime || NULL == ip_plugin_schema || 
         NULL == line) {
@@ -386,55 +357,71 @@ int ip_plugin_runtime_update(void *ip_plugin_runtime, void *ip_plugin_schema,
     struct ip_plugin_runtime *ip_plugin_rt =
         (struct ip_plugin_runtime *)ip_plugin_runtime;
 
-    size_t item_id_offset = 0, item_id_len = 0;
+    int ret = 0;
+    cJSON *tmp_obj = NULL;
+    cJSON *json = cJSON_Parse(line);
 
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
+    if (NULL == json) {
+        log_fatal(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
+                  "[%s:%d]ip_plugin table:<%s> parse json failed, line:%s",
+                  __FUNCTION__, __LINE__, table_name, line);
         ip_plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
-    int ret = get_column_pos(line, schema->item_id_column,
-                             &item_id_offset, &item_id_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, schema->key_name);
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
+        log_fatal(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
+                  "[%s:%d]ip_plugin table:<%s> has no entry_id in line:%s",
+                  __FUNCTION__, __LINE__, table_name, line);
         ip_plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
     if (1 == schema->container_schema.set_flag) {
-        if (1 == is_valid) {
+        if (MAAT_OP_ADD == op) {
             // add
-            ip_plugin_rule = ip_plugin_rule_new(schema, table_name, line,
+            ip_plugin_rule = ip_plugin_rule_new(schema, table_name, json,
                                                 ip_plugin_rt->logger);
             if (NULL == ip_plugin_rule) {
                 ip_plugin_rt->update_err_cnt++;
-                return -1;
+                goto ERROR;
             }
         }
 
-        const char *key = line + item_id_offset;
-        size_t key_len = item_id_len;
+        const char *key = tmp_obj->valuestring;
+        size_t key_len = strlen(key);
         ret = ip_plugin_runtime_update_row(ip_plugin_rt, table_name, line, key,
-                                           key_len, ip_plugin_rule, is_valid);
+                                           key_len, ip_plugin_rule, op);
         if (ret < 0) {
             log_fatal(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
                       "[%s:%d]ip_plugin table:<%s> update one line failed, "
                       "line:%s", __FUNCTION__, __LINE__, table_name, line);
             ip_plugin_rt->update_err_cnt++;
-            return -1;
+            goto ERROR;
         }
 
         log_debug(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
                  "ip_plugin table:<%s> update one line, key:%s, key_len:%zu,"
-                 " is_valid:%d", table_name, key, key_len, is_valid);
+                 " maat_op:%d", table_name, key, key_len, op);
     } else {
         //ex_schema not set
-        ex_data_runtime_cache_row_put(ip_plugin_rt->ex_data_rt, line);
+        ex_data_runtime_cache_row_put(ip_plugin_rt->ex_data_rt, line, op);
         ip_plugin_rt->rule_num =
             ex_data_runtime_cached_row_count(ip_plugin_rt->ex_data_rt);
     }
 
     return 0;
+
+ERROR:
+    if (ip_plugin_rule != NULL) {
+        ip_plugin_rule_free(ip_plugin_rule);
+    }
+    if (json != NULL) {
+        cJSON_Delete(json);
+    }
+
+    return -1;
 }
 
 int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,

src/maat_ipport_plugin.c

@@ -23,13 +23,9 @@
 #define MODULE_IPPORT_PLUGIN	 module_name_str("maat.ipport_plugin")
 
 struct ipport_plugin_schema {
-    int item_id_column;
-    int ip_type_column;
-    int ip_addr_column;
-    int port1_column;
-    int port2_column;
     int gc_timeout_s;
     int table_id;
+    char key_name[MAX_NAME_STR_LEN];
     struct ex_container_schema container_schema;
     struct table_manager *ref_tbl_mgr;
     struct log_handle *logger;
@@ -95,9 +91,9 @@ void *ipport_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         goto error;
     }
 
-    custom_item = cJSON_GetObjectItem(item, "item_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->item_id_column = custom_item->valueint;
+    custom_item = cJSON_GetObjectItem(item, "key_name");
+    if (custom_item != NULL && custom_item->type == cJSON_String) {
+        strncpy(schema->key_name, custom_item->valuestring, sizeof(schema->key_name) - 1);
     } else {
         log_fatal(logger, MODULE_IPPORT_PLUGIN, 
                   "[%s:%d] ipport_plugin table:<%s> schema has no"
@@ -106,48 +102,6 @@ void *ipport_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         goto error;
     }
 
-    custom_item = cJSON_GetObjectItem(item, "ip_type");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->ip_type_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport_plugin table:<%s> schema has no"
-                  " ip_type column", __FUNCTION__, __LINE__,
-                  table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "ip_addr");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->ip_addr_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport_plugin table:<%s> schema has no"
-                  " ip_addr column", __FUNCTION__, __LINE__,
-                  table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "port1");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->port1_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport_plugin table:<%s> schema has no"
-                  " port1 column", __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "port2");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        schema->port2_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport_plugin table:<%s> schema has no"
-                  " port2 column", __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
     //gc_timeout_s is optional
     custom_item = cJSON_GetObjectItem(item, "gc_timeout_s");
     if (custom_item != NULL && custom_item->type == cJSON_Number) {
@@ -269,93 +223,78 @@ void ipport_plugin_runtime_free(void *ipport_plugin_runtime)
 
 static struct ipport_item *
 ipport_item_new(struct ipport_plugin_schema *schema, const char *table_name,
-                const char *line, struct log_handle *logger)
+                const cJSON *json, struct log_handle *logger)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
-    char ip_str[40] = {0};
+    char ip_str[128] = {0};
     struct ipport_item *ipport_item = ALLOC(struct ipport_item, 1);
+    cJSON *tmp_obj = NULL;
+    int ret = 0;
 
-    int ret = get_column_pos(line, schema->item_id_column, &column_offset,
-                             &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, schema->key_name);
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_Number) {
         log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport table:<%s> has no item_id in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] ipport table:<%s> has no key or invalid format, line:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    ipport_item->item_id = atoll(line + column_offset);
+    ipport_item->item_id = atoll(tmp_obj->valuestring);
 
-    ret = get_column_pos(line, schema->ip_type_column, &column_offset,
-                         &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "ip");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport table:<%s> has no ip_type in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] ipport table:<%s> has no ip or invalid format in line:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    ipport_item->ip_type = atoi(line + column_offset);
+    strncpy(ip_str, tmp_obj->valuestring, strlen(tmp_obj->valuestring));
 
-    if (ipport_item->ip_type != IPV4 && ipport_item->ip_type != IPV6) {
-        log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport table:<%s> has invalid ip type:%d in line:%s",
-                  __FUNCTION__, __LINE__, table_name, ipport_item->ip_type, line);
-        goto error;
+    if (strchr(ip_str, ':') != NULL) {
+		ipport_item->ip_type = IPV6;
+	} else {
+		ipport_item->ip_type = IPV4;
     }
 
-    ret = get_column_pos(line, schema->ip_addr_column, &column_offset,
-                         &column_len);
-    if (ret < 0) {
-        log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport table:<%s> has no ip_addr in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
-        goto error;
-    }
-    memcpy(ip_str, (line + column_offset), column_len);
-
     if (IPV4 == ipport_item->ip_type) {
-        uint32_t ipv4_addr = 0;
-		ret = inet_pton(AF_INET, ip_str, &ipv4_addr);
+        ret = ip_format2range(ip_str, ipport_item->ip_type, &ipport_item->ipv4.min_ip, &ipport_item->ipv4.max_ip);
         if (ret < 0) {
             log_fatal(logger, MODULE_IPPORT_PLUGIN, 
                       "[%s:%d] ipport table:<%s> ip_format2range(ip4) failed in line:%s",
-                      __FUNCTION__, __LINE__, table_name, line);
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             goto error;
         }
-        ipport_item->ipv4.min_ip = ipv4_addr;
-        ipport_item->ipv4.max_ip = ipv4_addr;
     } else {
         //ipv6
-        uint32_t ipv6_addr[4] = {0};
-		ret = inet_pton(AF_INET6, ip_str, ipv6_addr);
+        ret = ip_format2range(ip_str, ipport_item->ip_type, ipport_item->ipv6.min_ip, ipport_item->ipv6.max_ip);
         if (ret < 0) {
             log_fatal(logger, MODULE_IPPORT_PLUGIN, 
                       "[%s:%d] ipport table:<%s> ip_format2range(ip6) failed in line:%s",
-                      __FUNCTION__, __LINE__, table_name, line);
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             goto error;
         }
-        memcpy(ipport_item->ipv6.min_ip, ipv6_addr, sizeof(ipv6_addr));
     }
 
-    ret = get_column_pos(line, schema->port1_column, &column_offset,
-                         &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "port");
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport table:<%s>) has no port1 in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
+                  "[%s:%d] ipport table:<%s> has no port or invalid format in line:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         goto error;
     }
-    ipport_item->min_port = atoi(line + column_offset);
 
-    ret = get_column_pos(line, schema->port2_column, &column_offset,
-                         &column_len);
-    if (ret < 0) {
-        log_fatal(logger, MODULE_IPPORT_PLUGIN, 
-                  "[%s:%d] ipport table:<%s> has no port2 in line:%s",
-                  __FUNCTION__, __LINE__, table_name, line);
-        goto error;
+    char port_range[20] = {0};
+    memcpy(port_range, tmp_obj->valuestring, strlen(tmp_obj->valuestring));
+
+    //port range is port or port_start-port_end
+    if(strchr(port_range,'-')!=NULL){
+        char *saveptr = NULL;
+        char *port_start = strtok_r(port_range,"-", &saveptr);
+        char *port_end = strtok_r(NULL,"-", &saveptr);
+        ipport_item->min_port = atoi(port_start);
+        ipport_item->max_port = atoi(port_end);
+    } else {
+        ipport_item->min_port = atoi(port_range);
+        ipport_item->max_port = atoi(port_range);
     }
-    ipport_item->max_port = atoi(line + column_offset);
 
     return ipport_item;
 error:
@@ -376,7 +315,7 @@ static int
 ipport_plugin_runtime_update_row(struct ipport_plugin_runtime *ipport_plugin_rt,
                                  const char *table_name, const char *row, 
                                  const char *key, size_t key_len, 
-                                 struct ipport_item *ipport_item, int is_valid)
+                                 struct ipport_item *ipport_item, enum maat_operation op)
 {
     int ret = -1;
     struct ex_data_runtime *ex_data_rt = ipport_plugin_rt->ex_data_rt;
@@ -384,7 +323,7 @@ ipport_plugin_runtime_update_row(struct ipport_plugin_runtime *ipport_plugin_rt,
         return -1;
     }
 
-    if (0 == is_valid) {
+    if (MAAT_OP_DEL == op) {
         // delete
         ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len);
         if (ret < 0) {
@@ -415,7 +354,7 @@ ipport_plugin_runtime_update_row(struct ipport_plugin_runtime *ipport_plugin_rt,
 int ipport_plugin_runtime_update(void *ipport_plugin_runtime,
                                  void *ipport_plugin_schema, 
                                  const char *table_name,
-                                 const char *line, int valid_column)
+                                 const char *line, enum maat_operation op)
 {
     if (NULL == ipport_plugin_runtime || NULL == ipport_plugin_schema || 
         NULL == line) {
@@ -429,56 +368,64 @@ int ipport_plugin_runtime_update(void *ipport_plugin_runtime,
     struct ipport_plugin_runtime *ipport_plugin_rt =
         (struct ipport_plugin_runtime *)ipport_plugin_runtime;
 
-    size_t item_id_offset = 0, item_id_len = 0;
+    int ret = 0;
+    cJSON *tmp_obj = NULL;
+    cJSON *json = cJSON_Parse(line);
 
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, schema->key_name);
+    if (NULL == tmp_obj || tmp_obj->type != cJSON_String) {
+        log_fatal(ipport_plugin_rt->logger, MODULE_IPPORT_PLUGIN,
+                  "[%s:%d]ipport_plugin table:<%s> has no key or invalid format, line:%s",
+                  __FUNCTION__, __LINE__, table_name, line);
         ipport_plugin_rt->update_err_cnt++;
-        return -1;
-    }
-
-    int ret = get_column_pos(line, schema->item_id_column,
-                             &item_id_offset, &item_id_len);
-    if (ret < 0) {
-        ipport_plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
     if (1 == schema->container_schema.set_flag) {
-        if (1 == is_valid) {
+        if (MAAT_OP_ADD == op) {
             // add
-            ipport_item = ipport_item_new(schema, table_name, line,
+            ipport_item = ipport_item_new(schema, table_name, json,
                                           ipport_plugin_rt->logger);
             if (NULL == ipport_item) {
                 ipport_plugin_rt->update_err_cnt++;
-                return -1;
+                goto ERROR;
             }
         }
 
-        const char *key = line + item_id_offset;
-        size_t key_len = item_id_len;
+        const char *key = tmp_obj->valuestring;
+        size_t key_len = strlen(key);
         ret = ipport_plugin_runtime_update_row(ipport_plugin_rt, table_name,
                                                line, key, key_len, ipport_item,
-                                               is_valid);
+                                               op);
         if (ret < 0) {
             log_fatal(ipport_plugin_rt->logger, MODULE_IPPORT_PLUGIN,
                       "[%s:%d]ipport_plugin table:<%s> update one line failed, "
                       "line:%s", __FUNCTION__, __LINE__, table_name, line);
             ipport_plugin_rt->update_err_cnt++;
-            return -1;
+            goto ERROR;
         }
 
         log_debug(ipport_plugin_rt->logger, MODULE_IPPORT_PLUGIN,
                  "ipport_plugin table:<%s> update one line, key:%s, key_len:%zu,"
-                 " is_valid:%d", table_name, key, key_len, is_valid);
+                 " maat_op:%d", table_name, key, key_len, op);
     } else {
         //ex_schema not set
-        ex_data_runtime_cache_row_put(ipport_plugin_rt->ex_data_rt, line);
+        ex_data_runtime_cache_row_put(ipport_plugin_rt->ex_data_rt, line, op);
         ipport_plugin_rt->rule_num =
             ex_data_runtime_cached_row_count(ipport_plugin_rt->ex_data_rt);
     }
 
     return 0;
+
+ERROR:
+    if (NULL != ipport_item) {
+        ipport_item_free(ipport_item);
+    }
+    if (NULL != json) {
+        cJSON_Delete(json);
+    }
+
+    return -1;
 }
 
 static void 

src/maat_object.c

@@ -29,9 +29,6 @@ struct object2object_item {
 };
 
 struct object2object_schema {
-    int object_id_column;
-    int incl_sub_object_ids_column;
-    int excl_sub_object_ids_column;
     int table_id;
     struct table_manager *ref_tbl_mgr;
 };
@@ -92,7 +89,6 @@ void *object2object_schema_new(cJSON *json, struct table_manager *tbl_mgr,
 {
 	struct object2object_schema *g2g_schema = ALLOC(struct object2object_schema, 1);
 	
-	cJSON *custom_item = NULL;
     cJSON *item = cJSON_GetObjectItem(json, "table_id");
     if (item != NULL && item->type == cJSON_Number) {
         g2g_schema->table_id = item->valueint;
@@ -103,44 +99,6 @@ void *object2object_schema_new(cJSON *json, struct table_manager *tbl_mgr,
         goto error;
     }
 
-    item = cJSON_GetObjectItem(json, "custom");
-    if (item == NULL || item->type != cJSON_Object) {
-        log_fatal(logger, MODULE_OBJECT, 
-                  "[%s:%d] g2g table:<%s> schema has no custom column", 
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "object_id");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        g2g_schema->object_id_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_OBJECT, 
-                  "[%s:%d] g2g table:<%s> schema has no object_id column",
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "included_sub_object_ids");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        g2g_schema->incl_sub_object_ids_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_OBJECT, 
-                  "[%s:%d] g2g table:<%s> schema has no included_sub_object_ids column",
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
-    custom_item = cJSON_GetObjectItem(item, "excluded_sub_object_ids");
-    if (custom_item != NULL && custom_item->type == cJSON_Number) {
-        g2g_schema->excl_sub_object_ids_column = custom_item->valueint;
-    } else {
-        log_fatal(logger, MODULE_OBJECT, 
-                  "[%s:%d] g2g table:<%s> schema has no excluded_sub_object_ids column",
-                  __FUNCTION__, __LINE__, table_name);
-        goto error;
-    }
-
     g2g_schema->ref_tbl_mgr = tbl_mgr;
 	return g2g_schema;
 error:
@@ -323,40 +281,47 @@ static struct object2object_item *
 object2object_item_new(const char *line, struct object2object_schema *g2g_schema,
                      const char *table_name, struct log_handle *logger)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
+    cJSON *tmp_obj = NULL;
+    cJSON *json = cJSON_Parse(line);
+
+    if (json == NULL) {
+        log_fatal(logger, MODULE_OBJECT,
+                  "[%s:%d] g2g table:<%s> line:<%s> parse json failed",
+                  __FUNCTION__, __LINE__, table_name, line);
+        return NULL;
+    }
+
     struct object2object_item *g2g_item = ALLOC(struct object2object_item, 1);
     utarray_new(g2g_item->incl_sub_object_ids, &ut_object_id_icd);
     utarray_new(g2g_item->excl_sub_object_ids, &ut_object_id_icd);
 
-    int ret = get_column_pos(line, g2g_schema->object_id_column, 
-                             &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "object_id");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_OBJECT,
-                  "[%s:%d] g2 table:<%s> has no object_id in line:%s",
+                  "[%s:%d] g2g table:<%s> has no object_id or format is not string in line:%s",
                   __FUNCTION__, __LINE__, table_name, line);
         goto error;
     }
-    g2g_item->object_id = atoll(line + column_offset);
+    g2g_item->object_id = atoll(tmp_obj->valuestring);
 
-    ret = get_column_pos(line, g2g_schema->incl_sub_object_ids_column,
-                         &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "include_object_ids");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_Array) {
         log_fatal(logger, MODULE_OBJECT,
-                  "[%s:%d] g2g table:<%s> has no included_sub_object_ids in line:%s",
+                  "[%s:%d] g2g table:<%s> has no included_sub_object_ids or format is not array in line:%s",
                   __FUNCTION__, __LINE__, table_name, line);
         goto error;
     }
 
-    char object_ids_str[MAX_OBJECT_IDS_STR_LEN] = {0};
-    memcpy(object_ids_str, line + column_offset, MIN(MAX_OBJECT_IDS_STR_LEN, column_len));
-
-    ret = ids_str2longlong_array(object_ids_str, g2g_item->incl_sub_object_ids);
-    if (ret < 0) {
-        log_fatal(logger, MODULE_OBJECT,
-				  "[%s:%d] g2c table:<%s> included_sub_object_ids str2longlong failed in line:%s",
-				  __FUNCTION__, __LINE__, table_name, line);
-		goto error; 
+    for (int i = 0; i < cJSON_GetArraySize(tmp_obj); i++) {
+        cJSON *item = cJSON_GetArrayItem(tmp_obj, i);
+        if (item == NULL || item->type != cJSON_String) {
+            log_fatal(logger, MODULE_OBJECT,
+                      "[%s:%d] g2g table:<%s> included_sub_object_ids format error in line:%s",
+                      __FUNCTION__, __LINE__, table_name, line);
+            goto error;
+        }
+        long long object_id = atoll(item->valuestring);
+        utarray_push_back(g2g_item->incl_sub_object_ids, &object_id);
     }
 
     if (utarray_len(g2g_item->incl_sub_object_ids) > MAX_OBJECT_CNT) {
@@ -366,24 +331,24 @@ object2object_item_new(const char *line, struct object2object_schema *g2g_schema
 		goto error;
     }
 
-    ret = get_column_pos(line, g2g_schema->excl_sub_object_ids_column, 
-                         &column_offset, &column_len);
-    if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "exclude_object_ids");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_Array) {
         log_fatal(logger, MODULE_OBJECT,
-                  "[%s:%d] g2g table:<%s> has no excluded_sub_object_ids in line:%s",
+                  "[%s:%d] g2g table:<%s> has no excluded_sub_object_ids or format is not array in line:%s",
                   __FUNCTION__, __LINE__, table_name, line);
         goto error;
     }
 
-    memset(object_ids_str, 0, sizeof(object_ids_str));
-    memcpy(object_ids_str, line + column_offset, MIN(MAX_OBJECT_IDS_STR_LEN, column_len));
-
-    ret = ids_str2longlong_array(object_ids_str, g2g_item->excl_sub_object_ids);
-    if (ret < 0) {
-        log_fatal(logger, MODULE_OBJECT,
-				  "[%s:%d] g2c table:<%s> excluded_sub_object_ids str2longlong failed in line:%s",
-				  __FUNCTION__, __LINE__, table_name, line);
-		goto error;
+    for (int i = 0; i < cJSON_GetArraySize(tmp_obj); i++) {
+        cJSON *item = cJSON_GetArrayItem(tmp_obj, i);
+        if (item == NULL || item->type != cJSON_String) {
+            log_fatal(logger, MODULE_OBJECT,
+                      "[%s:%d] g2g table:<%s> excluded_sub_object_ids format error in line:%s",
+                      __FUNCTION__, __LINE__, table_name, line);
+            goto error;
+        }
+        long long object_id = atoll(item->valuestring);
+        utarray_push_back(g2g_item->excl_sub_object_ids, &object_id);
     }
 
     if (utarray_len(g2g_item->excl_sub_object_ids) > MAX_OBJECT_CNT) {
@@ -743,7 +708,7 @@ static int object_topology_build_super_objects(struct maat_object_topology *obje
 
 int object2object_runtime_update(void *g2g_runtime, void *g2g_schema, 
                                const char *table_name, const char *line, 
-                               int valid_column)
+                               enum maat_operation op)
 {
     if (NULL == g2g_runtime || NULL == g2g_schema || 
         NULL == line) {
@@ -752,15 +717,6 @@ int object2object_runtime_update(void *g2g_runtime, void *g2g_schema,
 
     struct object2object_schema *schema = (struct object2object_schema *)g2g_schema;
 	struct object2object_runtime *g2g_rt = (struct object2object_runtime *)g2g_runtime;
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
-        log_fatal(g2g_rt->logger, MODULE_OBJECT,
-                  "[%s:%d] g2g table:<%s> has no is_valid(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  valid_column, line);
-        g2g_rt->update_err_cnt++;
-        return -1;
-    }
 
     struct object2object_item *g2g_item = object2object_item_new(line, schema, table_name,
                                                              g2g_rt->logger);
@@ -779,7 +735,7 @@ int object2object_runtime_update(void *g2g_runtime, void *g2g_schema,
     size_t i = 0;
     int err_flag = 0;
     long long *sub_object_id = NULL;
-    if (0 == is_valid) {
+    if (MAAT_OP_DEL == op) {
         //delete
         for (i = 0; i < utarray_len(g2g_item->incl_sub_object_ids); i++) {
             sub_object_id = (long long *)utarray_eltptr(g2g_item->incl_sub_object_ids, i);

src/maat_plugin.c

@@ -54,12 +54,10 @@ enum plugin_key_type {
 struct plugin_schema {
     enum plugin_key_type key_type;
     int key_len;
-    int key_column;
-    int addr_type_column;
-    int rule_tag_column;
+    char key_name[MAX_NAME_STR_LEN];
     int gc_timeout_s;
     int n_foreign;
-    int foreign_columns[MAX_FOREIGN_CLMN_NUM];
+    char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1];
     size_t cb_cnt;
     struct plugin_callback_schema cb[MAX_PLUGIN_PER_TABLE];
     struct ex_container_schema container_schema;
@@ -68,23 +66,6 @@ struct plugin_schema {
     struct log_handle *logger;
 };
 
-static int read_integer_array(char *string, int *array, int size)
-{
-    int i = 0;
-	char *token = NULL, *sub_token = NULL, *saveptr;
-	
-	for (i = 0, token = string; i < size; token= NULL, i++) {
-		sub_token = strtok_r(token, ",", &saveptr);
-		if (sub_token == NULL) {
-            break;
-        }
-		
-		sscanf(sub_token, "%d", array + i);
-	}
-
-	return i;
-}
-
 void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, 
                         const char *table_name, struct log_handle *logger)
 {
@@ -105,14 +86,14 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
     /* custom is optional */
     item = cJSON_GetObjectItem(json, "custom");
     if (item != NULL && item->type == cJSON_Object) {
-        custom_item = cJSON_GetObjectItem(item, "key");
-        if (NULL == custom_item || custom_item->type != cJSON_Number) {
+        custom_item = cJSON_GetObjectItem(item, "key_name");
+        if (NULL == custom_item || custom_item->type != cJSON_String) {
             log_fatal(logger, MODULE_PLUGIN,
-                      "[%s:%d]plugin table:<%s> schema has no key column",
+                      "[%s:%d]plugin table:<%s> schema has no key_name column",
                       __FUNCTION__, __LINE__, table_name);
             goto error;
         }
-        schema->key_column = custom_item->valueint;
+        strncpy(schema->key_name, custom_item->valuestring, sizeof(schema->key_name) - 1);
 
         custom_item = cJSON_GetObjectItem(item, "key_type");
         if (NULL == custom_item || custom_item->type != cJSON_String) {
@@ -137,15 +118,6 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
             schema->key_len = custom_item->valueint;
         } else if (strcmp(custom_item->valuestring, "ip_addr") == 0) {
             schema->key_type = PLUGIN_KEY_TYPE_IP_ADDR;
-            custom_item = cJSON_GetObjectItem(item, "addr_type");
-            if (NULL == custom_item || custom_item->type != cJSON_Number) {
-                log_fatal(logger, MODULE_PLUGIN,
-                          "[%s:%d]plugin table:<%s> schema ip_addr key must"
-                          " have addr_type column", __FUNCTION__, __LINE__,
-                          table_name);
-                goto error;
-            }
-            schema->addr_type_column = custom_item->valueint;
         } else {
             log_fatal(logger, MODULE_PLUGIN,
                       "[%s:%d]plugin table:<%s> schema key_type:%s is illegal, "
@@ -154,31 +126,33 @@ void *plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
             goto error;
         }
 
-        custom_item = cJSON_GetObjectItem(item, "tag");
-        if (custom_item != NULL && custom_item->type == cJSON_Number) {
-            schema->rule_tag_column = custom_item->valueint;
-        }
-
-        custom_item = cJSON_GetObjectItem(item, "foreign");
-        if (custom_item != NULL) {
-            if (custom_item->type == cJSON_String) {
-                schema->n_foreign = read_integer_array(custom_item->valuestring,
-                                                       schema->foreign_columns,
-                                                       MAX_FOREIGN_CLMN_NUM);
-            } else if (custom_item->type == cJSON_Array) {
-                schema->n_foreign = cJSON_GetArraySize(custom_item);
-                for (int i = 0; i < schema->n_foreign; i++) {
-                    cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i);
-                    assert(foreign_item->type == cJSON_Number);
-                    schema->foreign_columns[i] = foreign_item->valueint;
-                }
-            }
-        }
-
         custom_item = cJSON_GetObjectItem(item, "gc_timeout_s");
         if (custom_item != NULL && custom_item->type == cJSON_Number) {
             schema->gc_timeout_s = custom_item->valueint;
         }
+
+        custom_item = cJSON_GetObjectItem(item, "foreign_names");
+        if (custom_item != NULL && custom_item->type == cJSON_Array) {
+            schema->n_foreign = cJSON_GetArraySize(custom_item);
+            if (schema->n_foreign > MAX_FOREIGN_CLMN_NUM) {
+                log_fatal(logger, MODULE_PLUGIN,
+                          "[%s:%d]plugin table:<%s> schema foreign_names number"
+                          " exceed maxium:%d", __FUNCTION__, __LINE__, table_name,
+                          MAX_FOREIGN_CLMN_NUM);
+                goto error;
+            }
+
+            for (int i = 0; i < schema->n_foreign; i++) {
+                cJSON *foreign_item = cJSON_GetArrayItem(custom_item, i);
+                if (foreign_item == NULL || foreign_item->type != cJSON_String) {
+                    log_fatal(logger, MODULE_PLUGIN,
+                              "[%s:%d]plugin table:<%s> schema foreign_names"
+                              " has invalid format", __FUNCTION__, __LINE__, table_name);
+                    goto error;
+                }
+                strncpy(schema->foreign_names[i], foreign_item->valuestring, sizeof(schema->foreign_names[i]) - 1);
+            }
+        }
     }
 
     schema->ref_tbl_mgr = tbl_mgr;
@@ -245,19 +219,18 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema)
     }
 }
 
-int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, 
-                                    int *foreign_columns)
+int plugin_table_get_foreign_names(struct plugin_schema *plugin_schema, 
+                                    char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1])
 {
     if (NULL == plugin_schema) {
         return -1;
     }
 
-    int n_foreign = plugin_schema->n_foreign;
-    for (int i = 0; i < n_foreign; i++) {
-        foreign_columns[i] = plugin_schema->foreign_columns[i];
+    for (int i = 0; i < plugin_schema->n_foreign; i++) {
+        strncpy(foreign_names[i], plugin_schema->foreign_names[i], MAX_FOREIGN_NAME_LEN);
     }
 
-    return n_foreign;
+    return plugin_schema->n_foreign;
 }
 
 int plugin_table_set_ex_container_schema(void *plugin_schema, int table_id,
@@ -347,14 +320,14 @@ void plugin_runtime_free(void *plugin_runtime)
 static int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, 
                                      struct plugin_schema *plugin_schema, 
                                      const char *table_name, const char *row, 
-                                     const char *key, size_t key_len, int is_valid)
+                                     const char *key, size_t key_len, enum maat_operation op)
 {
     int ret = -1;
     struct ex_container_schema *container_schema = &(plugin_schema->container_schema);
 
     /* already set plugin_table_schema's ex_data_schema */
     if (1 == container_schema->set_flag) {
-        if (is_valid == 0) {
+        if (MAAT_OP_DEL == op) {
             // delete
             ret = ex_data_runtime_del_ex_container(plugin_rt->ex_data_rt, key, key_len);
             if (ret < 0) {
@@ -382,13 +355,13 @@ static int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
     size_t cb_count = plugin_schema->cb_cnt;
     if (cb_count > 0) {
         for (size_t i = 0; i < cb_count; i++) {
-            plugin_schema->cb[i].update(plugin_schema->table_id, row,
+            plugin_schema->cb[i].update(plugin_schema->table_id, row, op,
                                         plugin_schema->cb[i].u_para);
         }
     }
     
     if (0 == container_schema->set_flag) {
-        ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row);
+        ex_data_runtime_cache_row_put(plugin_rt->ex_data_rt, row, op);
         plugin_rt->rule_num = ex_data_runtime_cached_row_count(plugin_rt->ex_data_rt);
     }
 
@@ -396,40 +369,42 @@ static int plugin_runtime_update_row(struct plugin_runtime *plugin_rt,
 }
 
 static int plugin_accept_tag_match(struct plugin_schema *schema,
-                                   const char *table_name, const char *line,
+                                   const char *table_name, const cJSON *json,
                                    struct log_handle *logger)
 {
-    size_t column_offset = 0;
-    size_t column_len = 0;
+    size_t tag_len = 0;
     size_t n_tag = table_manager_accept_tags_count(schema->ref_tbl_mgr);
+    cJSON *tmp_obj = NULL;
+    int ret = 0;
 
-    if (schema->rule_tag_column > 0 && n_tag > 0) {
-        int ret = get_column_pos(line, schema->rule_tag_column, &column_offset,
-                                 &column_len);
-        if (ret < 0) {
+    tmp_obj = cJSON_GetObjectItem(json, "tag");
+
+    if (tmp_obj != NULL && n_tag > 0) {
+        if (tmp_obj->type != cJSON_String) {
             log_fatal(logger, MODULE_PLUGIN,
-                      "[%s:%d] table: <%s> has no rule_tag(column_seq:%d) "
-                      "in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                      schema->rule_tag_column, line);
+                      "[%s:%d] table: <%s> has invalid tag format in json, line %s",
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             return TAG_MATCH_ERR;
         }
 
-        if (column_len > 2) {
-            char *tag_str = ALLOC(char, column_len + 1);
-            memcpy(tag_str, (line + column_offset), column_len);
+        tag_len = strlen(tmp_obj->valuestring);
+
+        if (tag_len > 2) {
+            char *tag_str = ALLOC(char, tag_len + 1);
+            memcpy(tag_str, tmp_obj->valuestring, tag_len);
             ret = table_manager_accept_tags_match(schema->ref_tbl_mgr, tag_str);
             FREE(tag_str);
             if (TAG_MATCH_ERR == ret) {
                 log_fatal(logger, MODULE_PLUGIN,
                           "[%s:%d] table: <%s> has invalid tag format in table_line:%s",
-                          __FUNCTION__, __LINE__, table_name, line);
+                          __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
                 return TAG_MATCH_ERR;
             }
 
             if (TAG_MATCH_UNMATCHED == ret) {
                 log_fatal(logger, MODULE_PLUGIN,
                           "[%s:%d] table: <%s> has unmatched tag in table_line:%s",
-                          __FUNCTION__, __LINE__, table_name, line);
+                          __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
                 return TAG_MATCH_UNMATCHED;
             }
         }
@@ -439,7 +414,7 @@ static int plugin_accept_tag_match(struct plugin_schema *schema,
 }
 
 static int plugin_table_line_get_ip_key(struct plugin_schema *schema,
-                                        const char *table_name, const char *line,
+                                        const char *table_name, const cJSON *json,
                                         const char *src_key, size_t src_key_len,
                                         char *dst_key, size_t *dst_key_len,
                                         struct log_handle *logger)
@@ -448,18 +423,18 @@ static int plugin_table_line_get_ip_key(struct plugin_schema *schema,
         log_fatal(logger, MODULE_PLUGIN,
                   "[%s:%d] plugin table:<%s> ip_key too long exceed maximum:%d in "
                   "table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  INET6_ADDRSTRLEN, line);
+                  INET6_ADDRSTRLEN, cJSON_Print(json));
         return -1;
     }
 
-    size_t addr_type_offset = 0, addr_type_len = 0;
-    int ret = get_column_pos(line, schema->addr_type_column,
-                             &addr_type_offset, &addr_type_len);
-    if (ret < 0) {
+    cJSON *tmp_obj = NULL;
+    int ret = 0;
+
+    tmp_obj = cJSON_GetObjectItem(json, "addr_type");
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
         log_fatal(logger, MODULE_PLUGIN,
-                  "[%s:%d] plugin table:<%s> has no addr_type(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->addr_type_column, line);
+                  "[%s:%d] plugin table:<%s> has no addr_type or not string format in table_line:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
         return -1;
     }
 
@@ -467,15 +442,15 @@ static int plugin_table_line_get_ip_key(struct plugin_schema *schema,
     // snprintf() write at most (key_len+1) bytes (including the terminating null{'\0}) to ip_key.
     snprintf(ip_key, src_key_len + 1, "%s", src_key);
 
-    int addr_type = atoi(line + addr_type_offset);
+    int addr_type = atoi(tmp_obj->valuestring);
     if (IPV4 == addr_type) {
         uint32_t ipv4_addr;
         ret = inet_pton(AF_INET, ip_key, &ipv4_addr);
         if (ret <= 0) {
             log_fatal(logger, MODULE_PLUGIN,
-                      "[%s:%d] plugin table:<%s> ipv4 key(column seq:%d)"
+                      "[%s:%d] plugin table:<%s> ipv4 key"
                       " illegal in table_line:%s",
-                      __FUNCTION__, __LINE__, table_name, schema->key_column, line);
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             return -1;
         }
 
@@ -486,9 +461,9 @@ static int plugin_table_line_get_ip_key(struct plugin_schema *schema,
         ret = inet_pton(AF_INET6, ip_key, ipv6_addr);
         if (ret <= 0) {
             log_fatal(logger, MODULE_PLUGIN,
-                      "[%s:%d] plugin table:<%s> ipv6 key(column seq:%d)"
+                      "[%s:%d] plugin table:<%s> ipv6 key"
                       " illegal in table_line:%s",
-                      __FUNCTION__, __LINE__, table_name, schema->key_column, line);
+                      __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
             return -1;
         }
 
@@ -498,7 +473,7 @@ static int plugin_table_line_get_ip_key(struct plugin_schema *schema,
         log_fatal(logger, MODULE_PLUGIN,
                   "[%s:%d] plugin table:<%s> addr_type:%d illegal, just"
                   " allow{4, 6}, table_line:%s",
-                  __FUNCTION__, __LINE__, table_name, addr_type, line);
+                  __FUNCTION__, __LINE__, table_name, addr_type, cJSON_Print(json));
         return -1;
     }
 
@@ -506,24 +481,11 @@ static int plugin_table_line_get_ip_key(struct plugin_schema *schema,
 }
 
 static int plugin_table_line_get_key(struct plugin_schema *schema,
-                                     const char *table_name, const char *line,
+                                     const char *table_name, const cJSON *json,
                                      const char *src_key, size_t src_key_len,
                                      char *dst_key, size_t *dst_key_len,
                                      struct log_handle *logger)
 {
-    size_t plugin_key_offset = 0, plugin_key_len = 0;
-    
-    int ret = get_column_pos(line, schema->key_column, &plugin_key_offset, &plugin_key_len);
-    if (ret < 0) {
-        log_fatal(logger, MODULE_PLUGIN,
-                  "[%s:%d] plugin table:<%s> has no key(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->key_column, line);
-        return -1;
-    }
-
-    
-
     if (schema->key_type == PLUGIN_KEY_TYPE_POINTER) {
         memcpy(dst_key, src_key, src_key_len);
         *dst_key_len = src_key_len;
@@ -538,7 +500,7 @@ static int plugin_table_line_get_key(struct plugin_schema *schema,
         *dst_key_len = schema->key_len;
     } else {
         //PLUGIN_KEY_TYPE_IP_ADDR
-        return plugin_table_line_get_ip_key(schema, table_name, line, src_key,
+        return plugin_table_line_get_ip_key(schema, table_name, json, src_key,
                                             src_key_len, dst_key, dst_key_len,
                                             logger);
     }
@@ -548,7 +510,7 @@ static int plugin_table_line_get_key(struct plugin_schema *schema,
 
 int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, 
                           const char *table_name, const char *line, 
-                          int valid_column)
+                          enum maat_operation op)
 {
     if (NULL == plugin_runtime || NULL == plugin_schema ||
         NULL == line) {
@@ -557,65 +519,71 @@ int plugin_runtime_update(void *plugin_runtime, void *plugin_schema,
 
     struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
     struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime;
-    int is_valid = get_column_value(line, valid_column);
-    if (is_valid < 0) {
+    cJSON *tmp_obj = NULL;
+    cJSON *json = cJSON_Parse(line);
+
+    if (NULL == json) {
         log_fatal(plugin_rt->logger, MODULE_PLUGIN,
-                  "[%s:%d] plugin table:<%s> has no is_valid(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  valid_column, line);
-        plugin_rt->update_err_cnt++;
+                  "[%s:%d] plugin table:<%s> line is not json format, line:%s",
+                  __FUNCTION__, __LINE__, table_name, line);
         return -1;
     }
 
-    int ret = plugin_accept_tag_match(schema, table_name, line, plugin_rt->logger);
+    int ret = plugin_accept_tag_match(schema, table_name, json, plugin_rt->logger);
     if (ret == TAG_MATCH_UNMATCHED) {
         plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
-    size_t raw_key_offset = 0, raw_key_len = 0;
-    ret = get_column_pos(line, schema->key_column, &raw_key_offset, &raw_key_len);
-    if (ret < 0) {
+    size_t raw_key_len = 0;
+    tmp_obj = cJSON_GetObjectItem(json, schema->key_name);
+    if (tmp_obj == NULL || tmp_obj->type != cJSON_String) {
         log_fatal(plugin_rt->logger, MODULE_PLUGIN,
-                  "[%s:%d] plugin table:<%s> has no key(column seq:%d)"
-                  " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->key_column, line);
-        return -1;
+                  "[%s:%d] plugin table:<%s> has no key_name or invalid format in table_line:%s",
+                  __FUNCTION__, __LINE__, table_name, cJSON_Print(json));
+        goto ERROR;
     }
+    raw_key_len = strlen(tmp_obj->valuestring);
 
     if (raw_key_len > MAX_KEYWORDS_STR_LEN) {
         log_fatal(plugin_rt->logger, MODULE_PLUGIN,
-                  "[%s:%d] plugin table:<%s> key(column seq:%d) length exceed maxium:%d"
+                  "[%s:%d] plugin table:<%s> key length exceed maxium:%d"
                   " in table_line:%s", __FUNCTION__, __LINE__, table_name,
-                  schema->key_column, MAX_KEYWORDS_STR_LEN, line); 
-        return -1;
+                  MAX_KEYWORDS_STR_LEN, line); 
+        goto ERROR;
     }
 
-    const char *raw_key = line + raw_key_offset;
+    const char *raw_key = tmp_obj->valuestring;
     char hash_key[MAX_KEYWORDS_STR_LEN + 1] = {0};
     size_t hash_key_len = 0;
 
-    ret = plugin_table_line_get_key(schema, table_name, line, raw_key, raw_key_len,
+    ret = plugin_table_line_get_key(schema, table_name, json, raw_key, raw_key_len,
                                     hash_key, &hash_key_len, plugin_rt->logger);
     if (ret < 0) {
         plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
 
     ret = plugin_runtime_update_row(plugin_rt, schema, table_name, line, 
-                                    hash_key, hash_key_len, is_valid);
+                                    hash_key, hash_key_len, op);
     if (ret < 0) {
         plugin_rt->update_err_cnt++;
-        return -1;
+        goto ERROR;
     }
     
     char print_key[MAX_KEYWORDS_STR_LEN + 1] = {0};
     memcpy(print_key, raw_key, raw_key_len);
     log_debug(plugin_rt->logger, MODULE_PLUGIN,
-              "plugin table:<%s> update one line, key:%s, key_len:%zu, is_valid:%d",
-              table_name, print_key, raw_key_len, is_valid);
+              "plugin table:<%s> update one line, key:%s, key_len:%zu, maat_operation:%d",
+              table_name, print_key, raw_key_len, op);
     
     return 0;
+
+ERROR:
+    if (json) {
+        cJSON_Delete(json);
+    }
+    return -1;
 }
 
 int plugin_runtime_commit(void *plugin_runtime, const char *table_name,
@@ -691,7 +659,7 @@ size_t plugin_runtime_cached_row_count(void *plugin_runtime)
     return ex_data_runtime_cached_row_count(plugin_rt->ex_data_rt);
 }
 
-const char *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index)
+const struct ex_data_row *plugin_runtime_cached_row_get(void *plugin_runtime, size_t index)
 {
     if (NULL == plugin_runtime) {
         return NULL;

src/maat_redis_monitor.c

@@ -64,62 +64,17 @@ get_foreign_cont_filename(const char *table_name, long long rule_id,
 	return filename;
 }
 
-static const char *
-maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len)
-{
-    size_t i = 0;
-	int j = 0;
-	size_t start=0, end=0;
-	size_t line_len = strlen(line);
-
-	for (i = 0; i < line_len; i++) {
-		if (line[i] != ' ' && line[i] != '\t') {
-			continue;
-		}
-
-		j++;
-		if (j == Nth - 1) {
-			start = i + 1;
-		}
-
-		if(j == Nth) {
-			end = i;
-			break;
-		}
-	}
-
-	if (start == end) {
-		return NULL;
-	}
-
-	if (end == 0) {
-		end = i;
-	}
-
-	*column_len = end - start;
-
-	return line + start;
-}
-
 static void
-get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, 
+get_foregin_keys(struct serial_rule *p_rule, const char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1], 
                  int n_foreign, const char *dir, struct log_handle *logger)
 {
 	int foreign_key_size = 0;
 	p_rule->f_keys = ALLOC(struct foreign_key, n_foreign);
 
 	for (int i = 0; i < n_foreign; i++) {
-		const char *p_foreign =
-			maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], 
-                                     &foreign_key_size);
-		if (NULL == p_foreign) {
-			log_fatal(logger, MODULE_REDIS_MONITOR, 
-                      "[%s:%d] Get %s,%lld foreign keys failed: No %dth column",
-					  __FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_id, 
-                      foreign_columns[i]);
-			continue;
-		}
+		const char *p_foreign = foreign_names[i];
 
+		foreign_key_size = strlen(p_foreign);
         //emtpy file
 		if (0 == strncasecmp(p_foreign, "null", strlen("null"))) {
 			continue;
@@ -134,7 +89,6 @@ get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns,
 			continue;
 		}
 
-		p_rule->f_keys[p_rule->n_foreign].column = foreign_columns[i];
 		foreign_key_size = foreign_key_size - strlen(foreign_source_prefix);
 		p_foreign += strlen(foreign_source_prefix);
 
@@ -172,7 +126,7 @@ get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list,
 
         int table_id = table_manager_get_table_id(maat_inst->tbl_mgr,
 												  rule_list[i].table_name);
-		void *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id);
+		struct plugin_schema *schema = table_manager_get_schema(maat_inst->tbl_mgr, table_id);
 
         enum table_type table_type =
 			table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
@@ -181,15 +135,15 @@ get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list,
 			continue;
 		}
 
-        int foreign_columns[8];
-        int n_foreign_column =
-			plugin_table_get_foreign_column((struct plugin_schema *)schema, 
-                                            foreign_columns);
-        if (0 == n_foreign_column) {
+        char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1];
+        int n_foreign =
+			plugin_table_get_foreign_names(schema, 
+                                            foreign_names);
+        if (0 == n_foreign) {
             continue;
         }
 
-		get_foregin_keys(rule_list+i, foreign_columns, n_foreign_column,
+		get_foregin_keys(rule_list+i, foreign_names, n_foreign,
 						 dir, maat_inst->logger);
 		rule_with_foreign_key++;
 	}
@@ -202,33 +156,33 @@ int maat_get_foreign_keys_by_prefix(redisContext *ctx,
                                     int rule_num, const char* dir,
 									struct log_handle *logger)
 {
-	int j = 0;
-    int foreign_key_size = 0;
 	int rule_with_foreign_key = 0;
-	const char *p_foreign = NULL;
 
 	int n_foreign = 0;
-	int foreign_columns[MAX_FOREIGN_CLMN_NUM];
+	char foreign_names[MAX_FOREIGN_CLMN_NUM][MAX_FOREIGN_NAME_LEN + 1];
 
 	for (int i = 0; i < rule_num; i++) {
-		j = 1;
 		n_foreign = 0;
-		do {
-			p_foreign = maat_cmd_find_Nth_column(rule_list[i].table_line, j,
-												 &foreign_key_size);
+		memset(foreign_names, 0, sizeof(foreign_names));
 
-			if (p_foreign != NULL &&
-				foreign_key_size > (int)strlen(foreign_source_prefix) && 
-                0 == strncmp(p_foreign, foreign_source_prefix,
-							 strlen(foreign_source_prefix))) {
-				foreign_columns[n_foreign] = j;
-				n_foreign++;
+		cJSON *json = cJSON_Parse(rule_list[i].table_line);
+		if (json != NULL) {
+			cJSON *item = NULL;
+			cJSON_ArrayForEach(item, json) {
+				if (item->type == cJSON_String) {
+					if (strlen(item->valuestring) > strlen(foreign_source_prefix) &&
+					0 == strncmp(item->valuestring, foreign_key_prefix, strlen(foreign_key_prefix))) {
+						
+						strncpy(foreign_names[n_foreign], item->valuestring,
+							MAX_FOREIGN_NAME_LEN);
+						n_foreign++;
+					}
+				}
 			}
-			j++;
-		} while (p_foreign != NULL && n_foreign < MAX_FOREIGN_CLMN_NUM);
+		}
 
 		if (n_foreign > 0) {
-			get_foregin_keys(rule_list+i, foreign_columns, n_foreign,
+			get_foregin_keys(rule_list+i, foreign_names, n_foreign,
 							 dir, logger);
 			rule_with_foreign_key++;
 		}
@@ -799,53 +753,31 @@ void maat_get_foreign_conts(redisContext *c,
 	}
 }
 
-static int validate_line(char *line, int column_seq)
-{
-    if (NULL == line || column_seq < 0) {
-        return -1;
-    }
-
-	int offset = maat_get_valid_flag_offset(line, column_seq);
-	if (offset < 0) {
-		return -1;
-	}
-
-	line[offset] = '0';
-
-	return 0;
-}
-
 void maat_rewrite_table_line_with_foreign(struct serial_rule *s_rule)
 {
-    int i = 0;
-	size_t fn_size = 0;
-	
-	for (i = 0; i < s_rule->n_foreign; i++) {
-		fn_size += strlen(s_rule->f_keys[i].filename);
-	}
-	
-    char *rewrite_line = ALLOC(char, strlen(s_rule->table_line) + fn_size + 1);
-    char *pos_rewrite_line = rewrite_line;
-	const char *pos_origin_line = s_rule->table_line;
-	
-	for (i = 0; i < s_rule->n_foreign; i++) {
-        int origin_column_size = 0;
-		const char *origin_column =
-			maat_cmd_find_Nth_column(s_rule->table_line, s_rule->f_keys[i].column, 
-                                     &origin_column_size);
+	int i = 0;
 
-		strncat(pos_rewrite_line, pos_origin_line, origin_column - pos_origin_line);
-		pos_rewrite_line += origin_column - pos_origin_line;
-		pos_origin_line = origin_column+origin_column_size;
-		strncat(pos_rewrite_line, s_rule->f_keys[i].filename,
-				strlen(s_rule->f_keys[i].filename));
-		pos_rewrite_line += strlen(s_rule->f_keys[i].filename);
+	cJSON *json = cJSON_Parse(s_rule->table_line);
+	if (NULL == json) {
+		return;
 	}
 
-	strncat(pos_rewrite_line, pos_origin_line, 
-            strlen(s_rule->table_line) - (pos_origin_line - s_rule->table_line));
+	for (i = 0; i < s_rule->n_foreign; i++) {
+		cJSON *tmp_obj = cJSON_GetObjectItem(json, s_rule->f_keys[i].key);
+		if (NULL == tmp_obj) {
+			continue;
+		}
+
+		cJSON_DeleteItemFromObject(json, s_rule->f_keys[i].key);
+		cJSON_AddStringToObject(json, s_rule->f_keys[i].key, s_rule->f_keys[i].filename);
+	}
+
+	char *rewrite_line_json = cJSON_PrintUnformatted(json);
+	cJSON_Delete(json);
 	FREE(s_rule->table_line);
-	s_rule->table_line = rewrite_line;
+	s_rule->table_line = rewrite_line_json;
+
+	return;
 }
 
 static void
@@ -966,7 +898,7 @@ static void
 exec_serial_rule(redisContext *c, const char *transaction_list, 
                  struct serial_rule *s_rule, size_t rule_num,  
                  struct expected_reply *expect_reply, size_t *cnt, 
-                 size_t offset, int renew_allowed)
+                 size_t offset)
 {
     size_t i = 0;
     size_t append_cmd_cnt = 0;
@@ -1055,21 +987,6 @@ exec_serial_rule(redisContext *c, const char *transaction_list,
 				expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
 				(*cnt)++;
 				append_cmd_cnt++;
-				break;
-			case	MAAT_OP_RENEW_TIMEOUT:
-				if (renew_allowed != 1) {
-					continue;
-				}
-				//s_rule[i].timeout>0 was checked by caller.
-				redisAppendCommand(c, "ZADD %s %lld %s,%lld",
-						              mr_expire_sset,
-						              s_rule[i].timeout,
-						              s_rule[i].table_name,
-						              s_rule[i].rule_id);
-				expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0);
-				(*cnt)++;
-				append_cmd_cnt++;
-								
 				break;
 			default:
 				assert(0);
@@ -1129,12 +1046,6 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
 	size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2; // 2 for operation in exec_serial_rule_end()
 	struct expected_reply *expected_reply =
 		ALLOC(struct expected_reply, max_multi_cmd_num);
-	
-	for (i = 0; i < serial_rule_num; i++) {
-		if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) {
-			renew_num++;
-		}
-	}
 
 	int ret = exec_serial_rule_begin(c, serial_rule_num, renew_num,
 									 &renew_allowed, &transaction_version);
@@ -1152,7 +1063,7 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
 	while (success_cnt < serial_rule_num) {
 		size_t batch_cnt = MIN(serial_rule_num - success_cnt, max_redis_batch);
 		exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt, 
-                         expected_reply, &multi_cmd_cnt, success_cnt, renew_allowed);
+                         expected_reply, &multi_cmd_cnt, success_cnt);
 		assert(multi_cmd_cnt<max_multi_cmd_num);
 		success_cnt+=batch_cnt;
 	}
@@ -1199,21 +1110,6 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
 	transaction_reply = NULL;
 	
 error_out:
-	if (renew_num > 0 && renew_allowed != 1) {
-		for (i = 0; i < (unsigned int)serial_rule_num; i++) {
-			if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) {
-				log_fatal(logger, MODULE_REDIS_MONITOR, 
-                          "[%s:%d] %s %s %lld is not allowed due to lock contention",
-                          __FUNCTION__, __LINE__, mr_op_str[MAAT_OP_RENEW_TIMEOUT],
-                          s_rule[i].table_name, s_rule[i].rule_id);
-			}
-		}
-
-		if (success_cnt > 0) {
-			success_cnt -= renew_num;
-		}
-	}
-
 	FREE(expected_reply);
 
 	return success_cnt;
@@ -1346,7 +1242,7 @@ check_maat_expiration(redisContext *c, struct log_handle *logger)
 
 void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
 							void (*start_fn)(long long, int, void *),
-							int (*update_fn)(const char *, const char *, void *),
+							int (*update_fn)(const char *, const char *, void *, enum maat_operation),
 							void (*finish_fn)(void *), void *u_param)
 {
     int i = 0;
@@ -1355,7 +1251,6 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
     int empty_value_num = 0;
     int no_table_num = 0;
     int call_update_num = 0;
-    int valid_column = -1;
     struct maat *maat_inst = (struct maat *)u_param;
 
     //authorized to write
@@ -1466,23 +1361,11 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
 			continue;
 		}
 
-		if (rule_list[i].op == MAAT_OP_DEL) {
-			valid_column = table_manager_get_valid_column(maat_inst->tbl_mgr,
-														  table_id);
-			ret = validate_line(rule_list[i].table_line, valid_column);
-			if (ret < 0) {
-				log_fatal(maat_inst->logger, MODULE_REDIS_MONITOR, 
-                          "[%s:%d] Validate line failed, invaid format %s", 
-                          __FUNCTION__, __LINE__, rule_list[i].table_line);
-				continue;
-			}
-		}
-
 		if (rule_list[i].n_foreign > 0) {
 			maat_rewrite_table_line_with_foreign(rule_list+i);
 		}
 
-		update_fn(rule_list[i].table_name, rule_list[i].table_line, u_param);
+		update_fn(rule_list[i].table_name, rule_list[i].table_line, u_param, rule_list[i].op);
 		call_update_num++;
 	}
 

src/maat_stat.c

@@ -272,9 +272,6 @@ static void fs_table_row_refresh(struct maat_stat *stat, int perf_on)
                 plugin_cache_num += plugin_runtime_cached_row_count(runtime);
                 plugin_rule_num += plugin_runtime_rule_count(runtime);
                 break;
-            case TABLE_TYPE_OBJECT2RULE:
-                g2c_not_condition_num += object2rule_runtime_not_condition_count(runtime);
-                break;
             case TABLE_TYPE_OBJECT2OBJECT:
                 g2g_excl_rule_num += object2object_runtime_exclude_rule_count(runtime);
                 break;
@@ -320,11 +317,10 @@ static void fs_table_row_refresh(struct maat_stat *stat, int perf_on)
         fieldstat_easy_counter_set(stat->fs_handle, 0,
                                    stat->fs_column_id[COLUMN_HIT_ITEM_NUM],
                                    &cell_tag, 1, hit_item_num); 
-        if (table_type != TABLE_TYPE_ATTRIBUTE) {
-            total_regv6_num += regex_rule_num;
-            total_regv6_num += ipv6_rule_num;
-            total_rule_num += rule_num;
-        }
+        
+        total_regv6_num += regex_rule_num;
+        total_regv6_num += ipv6_rule_num;
+        total_rule_num += rule_num;
         
         total_scan_times += scan_times;
         total_hit_times += hit_times;

src/maat_table.c

@@ -38,15 +38,22 @@ struct maat_table {
     char schema_tag[MAX_TAG_STR_LEN + 1];
     size_t schema_tag_len;
     enum table_type table_type;
-    int valid_column;
     void *schema;
     void *runtime;
     void *updating_runtime;
 };
 
+struct maat_attribute {
+    int table_id;
+    int attr_id;
+    char attr_name[MAX_NAME_STR_LEN + 1];
+};
+
 struct table_manager {
     struct maat_table *tbl[MAX_TABLE_NUM];
     size_t n_table;
+    struct maat_attribute *attr[MAX_ATTRIBUTE_NUM];
+    size_t n_attr;
 
     struct rule_tag *accept_tags;
     size_t n_accept_tag;
@@ -56,6 +63,8 @@ struct table_manager {
     int g2g_table_id;
     struct maat_kv_store *tbl_name2id_map;
     struct maat_kv_store *conj_tbl_name2id_map;
+    struct maat_kv_store *attr_name2id_map;
+    struct maat_kv_store *sequence_map;
 
     struct maat_garbage_bin *ref_garbage_bin;
     struct log_handle *logger;
@@ -75,7 +84,7 @@ struct table_operations {
     void (*free_runtime)(void *runtime);
 
     int (*update_runtime)(void *runtime, void *schema, const char *table_name,
-                          const char *line, int valid_column);
+                          const char *line, enum maat_operation op);
 
     int (*commit_runtime)(void *runtime, const char *table_name, 
                           long long maat_rt_version);
@@ -263,6 +272,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
         .rule_count = bool_plugin_runtime_rule_count,
         .update_err_count = bool_plugin_runtime_update_err_count
     },
+#if 0
     {
         .type = TABLE_TYPE_ATTRIBUTE,
         .new_schema = attribute_schema_new,
@@ -277,6 +287,7 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
         .hit_times = attribute_runtime_hit_times,
         .hit_item_num = attribute_runtime_hit_item_num
     },
+#endif
     {
         .type = TABLE_TYPE_RULE,
         .new_schema = rule_schema_new,
@@ -298,17 +309,6 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = {
         .commit_runtime = object2object_runtime_commit,
         .rule_count = object2object_runtime_rule_count,
         .update_err_count = object2object_runtime_update_err_count
-    },
-    {
-        .type = TABLE_TYPE_OBJECT2RULE,
-        .new_schema = object2rule_schema_new,
-        .free_schema = object2rule_schema_free,
-        .new_runtime = object2rule_runtime_new,
-        .free_runtime = object2rule_runtime_free,
-        .update_runtime = object2rule_runtime_update,
-        .commit_runtime = NULL,
-        .rule_count = object2rule_runtime_rule_count,
-        .update_err_count = object2rule_runtime_update_err_count
     }
 };
 
@@ -500,7 +500,6 @@ static void maat_table_schema_free(void *schema, enum table_type table_type)
 static void register_reserved_word(struct maat_kv_store *reserved_word_map)
 {
     maat_kv_register(reserved_word_map, "rule", TABLE_TYPE_RULE);
-    maat_kv_register(reserved_word_map, "object2rule", TABLE_TYPE_OBJECT2RULE);
     maat_kv_register(reserved_word_map, "object2object", TABLE_TYPE_OBJECT2OBJECT);
     maat_kv_register(reserved_word_map, "flag", TABLE_TYPE_FLAG);
     maat_kv_register(reserved_word_map, "flag_plus", TABLE_TYPE_FLAG_PLUS);
@@ -514,7 +513,6 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map)
     maat_kv_register(reserved_word_map, "ipport_plugin", TABLE_TYPE_IPPORT_PLUGIN);
     maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN);
     maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN);
-    maat_kv_register(reserved_word_map, "attribute", TABLE_TYPE_ATTRIBUTE);
 }
 
 static struct maat_table *
@@ -595,18 +593,6 @@ maat_table_new(cJSON *json, struct maat_kv_store *reserved_word_map,
                   item->valuestring);
         goto error;
     }
-
-    item = cJSON_GetObjectItem(json, "valid_column");
-    if (NULL == item || item->type != cJSON_Number) {
-        if (ptable->table_type != TABLE_TYPE_ATTRIBUTE) {
-            log_fatal(logger, MODULE_TABLE, 
-                      "[%s:%d] table:%s has no valid column", 
-                      __FUNCTION__, __LINE__, ptable->table_name);
-            goto error;
-        }
-    } else {
-        ptable->valid_column = item->valueint;
-    }
     
     return ptable;
 error:
@@ -674,6 +660,31 @@ static int register_single_tbl_name2id(struct maat_kv_store *tbl_name2id_map,
     return 0;
 }
 
+static int register_single_attribute_name2id(struct maat_kv_store *attr_name2id_map,
+                                             const char *attr_name, int attr_id,
+                                             struct log_handle *logger)
+{
+    if (strlen(attr_name) >= NAME_MAX) {
+        log_fatal(logger, MODULE_TABLE,
+                  "[%s:%d] attribute:<%s> name length exceed maxium:%d",
+                  __FUNCTION__, __LINE__, attr_name, NAME_MAX);
+        return -1;
+    }
+
+    long long tmp_attr_id = -1;
+    int ret = maat_kv_read(attr_name2id_map, attr_name, &tmp_attr_id, 1);
+    if (ret > 0 && tmp_attr_id != attr_id) {
+        log_fatal(logger, MODULE_TABLE,
+                  "[%s:%d] attribute:<%s>(attr_id:%lld) has already been registered"
+                  ", can't register again",
+                  __FUNCTION__, __LINE__, attr_name, tmp_attr_id);
+        return -1;
+    }
+
+    maat_kv_register(attr_name2id_map, attr_name, attr_id);
+    return 0;
+}
+
 static int register_conjunction_tbl_name2id(struct maat_kv_store *conj_tbl_name2id_map,
                                              cJSON *root, struct log_handle *logger)
 {
@@ -799,6 +810,71 @@ int maat_default_rule_table_id(cJSON *json, struct log_handle *logger)
     return item->valueint;
 }
 
+static long long maat_table_get_sequence(struct maat_kv_store *sequence_map,
+                                         const char *sequence_name)
+{
+    long long sequence = 1;
+    int map_ret = maat_kv_read(sequence_map, sequence_name, &sequence, 1);
+    if (map_ret < 0) {
+        maat_kv_register(sequence_map, sequence_name, sequence);
+    } else {
+        sequence++;
+        int ret = maat_kv_write(sequence_map, sequence_name, sequence);
+        if (ret < 0) {
+            return -1;
+        }
+    }
+
+    return sequence;
+}
+
+static void maat_table_attribute_update(struct table_manager *tbl_mgr, cJSON *json,
+                                        int table_id, struct log_handle *logger)
+{
+    cJSON *item = cJSON_GetObjectItem(json, "supported_attributes");
+    if (NULL == item) {
+        return;
+    }
+
+    if (item->type != cJSON_Array) {
+        log_fatal(logger, MODULE_TABLE,
+                  "[%s:%d] table_id:%d supported_attributes should be an array",
+                  __FUNCTION__, __LINE__, table_id);
+        return;
+    }
+
+    int n_attr = cJSON_GetArraySize(item);
+    for (int i = 0; i < n_attr; i++) {
+        cJSON *attr = cJSON_GetArrayItem(item, i);
+        if (NULL == attr || attr->type != cJSON_String) {
+            log_fatal(logger, MODULE_TABLE,
+                      "[%s:%d] table_id:%d supported_attributes element should be string",
+                      __FUNCTION__, __LINE__, table_id);
+            return;
+        }
+
+        struct maat_attribute *pattr = ALLOC(struct maat_attribute, 1);
+        pattr->table_id = table_id;
+        pattr->attr_id = maat_table_get_sequence(tbl_mgr->sequence_map, "attribute_id");
+        if (pattr->attr_id < 0) {
+            log_fatal(logger, MODULE_TABLE,
+                      "[%s:%d] table_id:%d attribute %s get id failed",
+                      __FUNCTION__, __LINE__, table_id, attr->valuestring);
+            return;
+        }
+        if (register_single_attribute_name2id(tbl_mgr->attr_name2id_map, attr->valuestring, pattr->attr_id, logger) < 0) {
+            log_fatal(logger, MODULE_TABLE,
+                      "[%s:%d] table_id:%d attribute %s register failed",
+                      __FUNCTION__, __LINE__, table_id, attr->valuestring);
+            return;
+        }
+        strncpy(pattr->attr_name, attr->valuestring, MAX_NAME_STR_LEN);
+
+        tbl_mgr->attr[pattr->attr_id] = pattr;
+        tbl_mgr->n_attr++;
+    }
+}
+
 struct table_manager *
 table_manager_create(const char *table_info_path, const char *accept_tags,
                      enum maat_expr_engine engine_type, struct maat_garbage_bin *garbage_bin,
@@ -844,6 +920,8 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
     tbl_mgr->logger = logger;
     tbl_mgr->tbl_name2id_map = maat_kv_store_new();
     tbl_mgr->conj_tbl_name2id_map = maat_kv_store_new();
+    tbl_mgr->attr_name2id_map = maat_kv_store_new();
+    tbl_mgr->sequence_map = maat_kv_store_new();
     tbl_mgr->engine_type = engine_type;
     tbl_mgr->ref_garbage_bin = garbage_bin;
 
@@ -909,20 +987,16 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
                 g2g_table_id = maat_tbl->table_id;
             }
 
+            if (maat_tbl->table_type >= TABLE_TYPE_FLAG && maat_tbl->table_type <= TABLE_TYPE_INTERVAL_PLUS) {
+                maat_table_attribute_update(tbl_mgr, json, maat_tbl->table_id, logger);
+            }
+
             tbl_mgr->tbl[maat_tbl->table_id] = maat_tbl;
             tbl_mgr->n_table++;
 
-            if (maat_tbl->table_type == TABLE_TYPE_ATTRIBUTE) {
-                int phy_table_id = attribute_get_physical_table_id(tbl_mgr, maat_tbl->table_id);
-                log_info(logger, MODULE_TABLE,
-                         "successfully register attribute[%s]->attribute_id:%d,"
-                         " physical_table_id:%d", maat_tbl->table_name, maat_tbl->table_id,
-                         phy_table_id);
-            } else {
-                log_info(logger, MODULE_TABLE,
-                         "successfully register physical table[%s]->table_id:%d",
-                         maat_tbl->table_name, maat_tbl->table_id);
-            }
+            log_info(logger, MODULE_TABLE,
+                        "successfully register table[%s]->table_id:%d",
+                        maat_tbl->table_name, maat_tbl->table_id);
         }
     }
 
@@ -990,32 +1064,6 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, size_t max_threa
                                                                     garbage_bin, tbl_mgr->logger);
     }
 
-    /* object2rule runtime depends on associated rule runtime,
-       must make sure associated rule runtime already exist */
-    for (i = 0; i < MAX_TABLE_NUM; i++) {
-        table_type = table_manager_get_table_type(tbl_mgr, i);
-        if (table_type != TABLE_TYPE_OBJECT2RULE) {
-            continue;
-        }
-
-        void *schema = table_manager_get_schema(tbl_mgr, i);
-        if (NULL == schema) {
-            log_fatal(tbl_mgr->logger, MODULE_TABLE, 
-                      "[%s:%d] object2rule table(table_id:%d) schema is null",
-                      __FUNCTION__, __LINE__, i);
-            continue;
-        }
-
-        void *g2c_updating_rt = table_manager_get_updating_runtime(tbl_mgr, i);
-        if (NULL == g2c_updating_rt) {
-            continue;
-        }
-
-        int asso_rule_table_id = object2rule_associated_rule_table_id(schema);
-        void *rule_updating_rt = table_manager_get_updating_runtime(tbl_mgr, asso_rule_table_id);
-        object2rule_runtime_init(g2c_updating_rt, rule_updating_rt);
-    }
-
     return 0;
 }
 
@@ -1104,6 +1152,21 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *table_
     return (int)table_id;
 }
 
+int table_manager_get_attribute_id(struct table_manager *tbl_mgr, const char *attr_name)
+{
+    if (NULL == tbl_mgr || NULL == attr_name) {
+        return -1;
+    }
+
+    long long attr_id = -1;
+    int ret = maat_kv_read(tbl_mgr->attr_name2id_map, attr_name, &attr_id, 1);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return (int)attr_id;
+}
+
 int table_manager_get_conj_parent_table_ids(struct table_manager *tbl_mgr, const char *table_name,
                                             long long *table_ids_array, size_t n_table_ids_array)
 {
@@ -1128,6 +1191,32 @@ const char *table_manager_get_table_name(struct table_manager *tbl_mgr, int tabl
     return tbl_mgr->tbl[table_id]->table_name;
 }
 
+const char *table_manager_get_attribute_name(struct table_manager *tbl_mgr, int attr_id)
+{
+    if (NULL == tbl_mgr || attr_id < 0) {
+        return NULL;
+    }
+
+    if (NULL == tbl_mgr->attr[attr_id]) {
+        return NULL;
+    }
+
+    return tbl_mgr->attr[attr_id]->attr_name;
+}
+
+int table_manager_attribute_get_table_id(struct table_manager *tbl_mgr, int attr_id)
+{
+    if (NULL == tbl_mgr || attr_id < 0) {
+        return -1;
+    }
+
+    if (NULL == tbl_mgr->attr[attr_id]) {
+        return -1;
+    }
+
+    return tbl_mgr->attr[attr_id]->table_id;
+}
+
 const char *table_manager_get_table_schema_tag(struct table_manager *tbl_mgr, int table_id)
 {
     if (NULL == tbl_mgr || table_id < 0) {
@@ -1185,19 +1274,6 @@ void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id)
     return tbl_mgr->tbl[table_id]->schema;
 }
 
-int table_manager_get_valid_column(struct table_manager *tbl_mgr, int table_id)
-{
-    if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) {
-        return -1;
-    }
-
-    if (NULL == tbl_mgr->tbl[table_id]) {
-        return -1;
-    }
-
-    return tbl_mgr->tbl[table_id]->valid_column;
-}
-
 enum maat_expr_engine table_manager_get_expr_engine(struct table_manager *tbl_mgr)
 {
     if (NULL == tbl_mgr) {
@@ -1252,7 +1328,8 @@ void *table_manager_get_updating_runtime(struct table_manager *tbl_mgr, int tabl
 }
 
 int table_manager_update_runtime(struct table_manager *tbl_mgr, const char *table_name,
-                                 int table_id, const char *line, int update_type)
+                                 int table_id, const char *line, enum maat_operation op, 
+                                 int update_type)
 {
     void *schema = table_manager_get_schema(tbl_mgr, table_id);
     if (NULL == schema) {
@@ -1270,14 +1347,6 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, const char *tabl
         return -1;
     }
 
-    int valid_column = table_manager_get_valid_column(tbl_mgr, table_id);
-    if (valid_column < 0) {
-        log_fatal(tbl_mgr->logger, MODULE_TABLE, 
-                  "[%s:%d] table:%s has no valid column, can't update runtime",
-                  __FUNCTION__, __LINE__, table_name);
-        return -1;
-    }
-
     enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
     if (table_type == TABLE_TYPE_INVALID) {
         log_fatal(tbl_mgr->logger, MODULE_TABLE, 
@@ -1294,7 +1363,7 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, const char *tabl
     }
     
     return table_ops[table_type].update_runtime(runtime, schema, table_name, 
-                                                line, valid_column);
+                                                line, op);
 }
 
 void table_commit_updating_runtime(struct table_manager *tbl_mgr, int table_id, 

test/CMakeLists.txt

@@ -39,7 +39,7 @@ target_link_libraries(maat_ex_data_gtest maat_frame_static gtest_static)
 
 add_subdirectory(object_nesting)
 add_subdirectory(ipport_plugin)
-add_subdirectory(benchmark)
+#add_subdirectory(benchmark) //TODO: convert iris rule to json rule
 
 configure_file(table_info.json table_info.json COPYONLY)
 configure_file(tsg_table_info.json tsg_table_info.json COPYONLY)

test/benchmark/benchmark_gtest.cpp

@@ -2,7 +2,6 @@
 #include "log/log.h"
 #include "maat_utils.h"
 #include "maat_core.h"
-#include "json2iris.h"
 #include "ip_matcher.h"
 #include "maat_table.h"
 #include "../test_utils.h"

test/ipport_plugin/ipport_plugin_gtest.cpp

@@ -158,7 +158,7 @@ test_add_ipport_plugin_command(struct maat *maat_inst, const char *table_name,
     line_rule.table_name = table_name;
     line_rule.expire_after = 0;
 
-    maat_cmd_set_line(maat_inst, &line_rule);
+    maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
 
     return 0;
 }
@@ -188,7 +188,7 @@ test_del_ipport_plugin_command(struct maat *maat_inst, const char *table_name,
     line_rule.table_name = table_name;
     line_rule.expire_after = 0;
 
-    maat_cmd_set_line(maat_inst, &line_rule);
+    maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_DEL);
 
     return 0;
 }
@@ -235,6 +235,7 @@ void *ipport_plugin_update_thread(void *arg)
     return is_all_hit;
 }
 
+#if 0 //TODO: convert iris rule to json rule
 TEST_F(IPPortPluginTable, WITHOUT_SAME_IP) {
     char redis_ip[32] = "127.0.0.1";
     int redis_port = 6379;
@@ -380,6 +381,7 @@ TEST_F(IPPortPluginTable, WITH_256SAME_IP) {
              "IpportPluginScan with 256 same ip match rate speed"
              " %lld lookups/s/thread", scan_per_second);
 }
+#endif
 
 int main(int argc, char ** argv)
 {

test/maat_ex_data_gtest.cpp

@@ -1,7 +1,6 @@
 #include "maat.h"
 #include "log/log.h"
 #include "maat_utils.h"
-#include "json2iris.h"
 #include "maat_command.h"
 #include "maat_ex_data.h"
 #include "maat_garbage_collection.h"

test/maat_framework_gtest.cpp

@@ -8,7 +8,6 @@
 #include "maat_utils.h"
 #include "maat_command.h"
 #include "ip_matcher.h"
-#include "json2iris.h"
 #include "log/log.h"
 #include "maat_config_monitor.h"
 #include "maat_redis_monitor.h"
@@ -5260,7 +5259,7 @@ void maat_read_entry_start_cb(int update_type, void *u_para)
 {
 }
 
-void maat_read_entry_cb(int table_id, const char *table_line, void *u_para)
+void maat_read_entry_cb(int table_id, const char *table_line, enum maat_operation op, void *u_para)
 {
 	char ip_str[16] = {0};
 	int entry_id = -1, seq = -1;
@@ -6530,7 +6529,7 @@ protected:
 struct maat *Policy::_shared_maat_inst;
 struct log_handle *Policy::logger;
 
-void accept_tags_entry_cb(int table_id, const char *table_line, void *u_para)
+void accept_tags_entry_cb(int table_id, const char *table_line, enum maat_operation op, void *u_para)
 {
 	int* callback_times = (int *)u_para;
 	char status[32] = {0};
@@ -6559,7 +6558,7 @@ TEST_F(Policy, PluginRuleTags1) {
 	EXPECT_EQ(callback_times, 5);
 }
 
-void accept_tags_entry2_cb(int table_id, const char *table_line, void *u_para)
+void accept_tags_entry2_cb(int table_id, const char *table_line, enum maat_operation op, void *u_para)
 {
 	int *callback_times = (int *)u_para;
 	(*callback_times)++;
@@ -6973,6 +6972,7 @@ TEST_F(TableInfo, Conjunction) {
     state = NULL;
 }
 
+#if 0 //TODO: change iris rule to json rule
 class FileTest : public testing::Test
 {
 protected:
@@ -7067,6 +7067,7 @@ TEST_F(FileTest, StreamFiles) {
 
     free(name_list);
 }
+#endif
 
 class ObjectHierarchy : public testing::Test
 {
@@ -8293,7 +8294,7 @@ TEST_F(MaatCmd, SetLines) {
         line_rule.table_line = table_line_add[i];
         line_rule.expire_after = 0;
 
-        ret = maat_cmd_set_line(maat_inst, &line_rule);
+        ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
         EXPECT_GT(ret, 0);
     }
 
@@ -8304,13 +8305,13 @@ TEST_F(MaatCmd, SetLines) {
         line_rule.table_line = table_line_del[i];
         line_rule.expire_after = 0;
 
-        ret = maat_cmd_set_line(maat_inst, &line_rule);
+        ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_DEL);
         EXPECT_GT(ret, 0);
     }
 }
 
 int g_test_update_paused = 0;
-void pause_update_test_entry_cb(int table_id,const char* table_line,void* u_para)
+void pause_update_test_entry_cb(int table_id,const char* table_line, enum maat_operation op, void* u_para)
 {
 	char status[32] = {0};
 	int entry_id = -1, seq = -1;
@@ -8341,7 +8342,7 @@ TEST_F(MaatCmd, PauseUpdate) {
     line_rule.table_line = line;
     line_rule.expire_after = 0;
 
-    ret = maat_cmd_set_line(maat_inst, &line_rule);
+    ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
     EXPECT_EQ(ret, 1);
     free(line);
     
@@ -8394,7 +8395,7 @@ int is_same_file(const char *filename1, const char *filename2)
 int g_test_foregin_read_OK = 0, g_test_foreign_del_OK = 0;
 char file1_to_del[256], file2_to_del[256];
 const char* empty_file_name = "An_empty_file";
-void foreign_key_test_entry_cb(int table_id, const char *table_line, void *u_para)
+void foreign_key_test_entry_cb(int table_id, const char *table_line, enum maat_operation op, void *u_para)
 {
 	int rule_id=-1, not_care=0, tag=0;
 	int is_valid=0;
@@ -8467,7 +8468,7 @@ TEST_F(MaatCmd, SetFile) {
     line_rule.table_line = line;
     line_rule.expire_after = 0;
 
-    ret = maat_cmd_set_line(maat_inst, &line_rule);
+    ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
     EXPECT_EQ(ret, 1);
 	sleep(WAIT_FOR_EFFECTIVE_S);//wait for callback triggered.
 	EXPECT_EQ(g_test_foregin_read_OK, 1);
@@ -8486,7 +8487,7 @@ TEST_F(MaatCmd, SetFile) {
              line_rule.rule_id, tag, file1_name, file1_key, file2_name, file2_key);
     line_rule_del.table_line = line;
     line_rule_del.expire_after = 0;
-    ret = maat_cmd_set_line(maat_inst, &line_rule_del);
+    ret = maat_cmd_set_line(maat_inst, &line_rule_del, MAAT_OP_DEL);
     EXPECT_EQ(ret, 1);
     sleep(WAIT_FOR_EFFECTIVE_S);
 
@@ -8508,7 +8509,7 @@ TEST_F(MaatCmd, SetFile) {
 
     g_test_foregin_read_OK = 0;
 
-    ret = maat_cmd_set_line(maat_inst, &line_rule);
+    ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
     EXPECT_EQ(ret, 1);
 	sleep(WAIT_FOR_EFFECTIVE_S);//wait for callback triggered.
 	EXPECT_EQ(g_test_foregin_read_OK, 1);
@@ -8631,7 +8632,7 @@ TEST_F(MaatCmd, PluginEXData) {
         line_rule.table_line = table_line_add[i];
         line_rule.expire_after = 0;
 
-        ret = maat_cmd_set_line(maat_inst, &line_rule);
+        ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
         EXPECT_GT(ret, 0);
     }
 	
@@ -8662,7 +8663,7 @@ TEST_F(MaatCmd, PluginEXData) {
     line_rule.table_line = table_line_del[1];
     line_rule.expire_after = 0;
 
-	ret = maat_cmd_set_line(maat_inst, &line_rule);
+	ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_DEL);
 	EXPECT_GT(ret, 0);
 
 	sleep(WAIT_FOR_EFFECTIVE_S); //gc_timeout_s == 3 which configured in table_info
@@ -8711,7 +8712,7 @@ TEST_F(MaatCmd, UpdateIPPlugin) {
         line_rule.table_line = table_line_add[i];
         line_rule.expire_after = 0;
 
-        ret = maat_cmd_set_line(maat_inst, &line_rule);
+        ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
         EXPECT_GT(ret, 0);
     }
 
@@ -8756,7 +8757,7 @@ TEST_F(MaatCmd, UpdateIPPlugin) {
         line_rule.table_line = table_line_del[i];
         line_rule.expire_after = 0;
 
-        ret = maat_cmd_set_line(maat_inst, &line_rule);
+        ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_DEL);
         EXPECT_GT(ret, 0);
     }
 
@@ -8807,7 +8808,7 @@ TEST_F(MaatCmd, UpdateFQDNPlugin) {
         line_rule.table_line = table_line_add[i];
         line_rule.expire_after = 0;
 
-        ret = maat_cmd_set_line(maat_inst, &line_rule);
+        ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
         EXPECT_GT(ret, 0);
     }
 
@@ -8839,7 +8840,7 @@ TEST_F(MaatCmd, UpdateFQDNPlugin) {
         line_rule.table_line = table_line_del[i];
         line_rule.expire_after = 0;
 
-        ret = maat_cmd_set_line(maat_inst, &line_rule);
+        ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_DEL);
         EXPECT_GT(ret, 0);
     }
 
@@ -8889,7 +8890,7 @@ TEST_F(MaatCmd, UpdateBoolPlugin) {
         line_rule.table_line = table_line_add[i];
         line_rule.expire_after = 0;
 
-        ret = maat_cmd_set_line(maat_inst, &line_rule);
+        ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
         EXPECT_GT(ret, 0);
     }
 
@@ -8919,7 +8920,7 @@ TEST_F(MaatCmd, UpdateBoolPlugin) {
         line_rule.table_line = table_line_del[i];
         line_rule.expire_after = 0;
 
-        ret = maat_cmd_set_line(maat_inst, &line_rule);
+        ret = maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_DEL);
         EXPECT_GT(ret, 0);
     }
     sleep(WAIT_FOR_EFFECTIVE_S);

test/maat_framework_perf_gtest.cpp

@@ -7,7 +7,6 @@
 #include "maat_utils.h"
 #include "ip_matcher.h"
 #include "maat_command.h"
-#include "json2iris.h"
 #include "maat_config_monitor.h"
 #include "maat_redis_monitor.h"
 
@@ -1208,7 +1207,7 @@ void *perf_fqdn_plugin_update_thread(void *arg)
                  line_rule.rule_id, fqdn_buff);
 		line_rule.table_line = line_buff;
 		line_rule.expire_after = 0;
-		maat_cmd_set_line(maat_inst, &line_rule);
+		maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
 		sleep(WAIT_FOR_EFFECTIVE_S);
 	}
 
@@ -1422,7 +1421,7 @@ void *perf_bool_plugin_update_thread(void *arg)
 		snprintf(line_buff, 1024, "%lld\t1&%d\ttunnel2\t1", line_rule.rule_id, i);
 		line_rule.table_line = line_buff;
 		line_rule.expire_after = 0;
-		maat_cmd_set_line(maat_inst, &line_rule);
+		maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
 		sleep(WAIT_FOR_EFFECTIVE_S);
 	}
 
@@ -1489,43 +1488,6 @@ TEST_F(MaatPerfBoolPluginScan, MultiThread) {
              PERF_THREAD_NUM, scan_per_second);
 }
 
-class MaatPerfFileScan : public testing::Test
-{
-protected:
-    static void SetUpTestCase() {
-        logger = log_handle_create("./maat_framework_perf_gtest.log", 0);        
-        const char *rule_folder = "./tsgrule/index";
-        const char *table_info = "./tsg_table_info.json";
-
-        struct maat_options *opts = maat_options_new();
-        maat_options_set_stat_file(opts, "./stat.log");
-        maat_options_set_perf_on(opts);
-        maat_options_set_iris(opts, rule_folder, rule_folder);
-        maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
-                                LOG_LEVEL_INFO);
-        maat_options_set_caller_thread_number(opts, 5);
-
-        _shared_maat_inst = maat_new(opts, table_info);
-        maat_options_free(opts);
-        if (NULL == _shared_maat_inst) {
-            log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
-                      "[%s:%d] create maat instance in MaatPerfFileScan failed.",
-                      __FUNCTION__, __LINE__);
-        }
-    }
-
-    static void TearDownTestCase() {
-        maat_free(_shared_maat_inst);
-        log_handle_destroy(logger);
-    }
-
-    static struct log_handle *logger;
-    static struct maat *_shared_maat_inst;
-};
-
-struct maat *MaatPerfFileScan::_shared_maat_inst;
-struct log_handle *MaatPerfFileScan::logger;
-
 struct perf_ip_plugin_ud {
 	long long rule_id;
 };
@@ -1565,6 +1527,45 @@ void perf_ip_plugin_EX_dup_cb(int table_id, void **to, void **from,
 	*to = u;
 }
 
+//TODO: convert iris rule to json rule
+#if 0
+class MaatPerfFileScan : public testing::Test
+{
+protected:
+    static void SetUpTestCase() {
+        logger = log_handle_create("./maat_framework_perf_gtest.log", 0);        
+        const char *rule_folder = "./tsgrule/index";
+        const char *table_info = "./tsg_table_info.json";
+
+        struct maat_options *opts = maat_options_new();
+        maat_options_set_stat_file(opts, "./stat.log");
+        maat_options_set_perf_on(opts);
+        maat_options_set_iris(opts, rule_folder, rule_folder);
+        maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
+                                LOG_LEVEL_INFO);
+        maat_options_set_caller_thread_number(opts, 5);
+
+        _shared_maat_inst = maat_new(opts, table_info);
+        maat_options_free(opts);
+        if (NULL == _shared_maat_inst) {
+            log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
+                      "[%s:%d] create maat instance in MaatPerfFileScan failed.",
+                      __FUNCTION__, __LINE__);
+        }
+    }
+
+    static void TearDownTestCase() {
+        maat_free(_shared_maat_inst);
+        log_handle_destroy(logger);
+    }
+
+    static struct log_handle *logger;
+    static struct maat *_shared_maat_inst;
+};
+
+struct maat *MaatPerfFileScan::_shared_maat_inst;
+struct log_handle *MaatPerfFileScan::logger;
+
 static void *ip_plugin_get_thread(void *arg)
 {
 	const char *table_name = "TSG_IP_LOCATION_BUILT_IN";
@@ -1632,6 +1633,7 @@ TEST_F(MaatPerfFileScan, IPPlugin) {
 		free(is_all_hit);
 	}
 }
+#endif
 
 class MaatPerfIPPortPluginScan : public testing::Test
 {
@@ -1730,7 +1732,7 @@ void *perf_ipport_plugin_update_thread(void *arg)
                  line_rule.rule_id, 200+i, 300+i);
 		line_rule.table_line = line_buff;
 		line_rule.expire_after = 0;
-		maat_cmd_set_line(maat_inst, &line_rule);
+		maat_cmd_set_line(maat_inst, &line_rule, MAAT_OP_ADD);
 		sleep(WAIT_FOR_EFFECTIVE_S);
 	}
 

test/maat_input_mode_gtest.cpp

@@ -1,7 +1,6 @@
 #include "maat.h"
 #include "maat_utils.h"
 #include "maat_core.h"
-#include "json2iris.h"
 #include "test_utils.h"
 #include "maat_table.h"
 #include "maat_config_monitor.h"
@@ -16,11 +15,11 @@ const char *g_json_filename = "maat_json.json";
 struct log_handle *g_logger = NULL;
 
 TEST(json_mode, maat_scan_string) {
-    char iris_idx_path[PATH_MAX] = {0};
     char json_iris_path[PATH_MAX] = {0};
     char json_path[PATH_MAX] = {0};
     long long results[5] = {0};
     size_t n_hit_result = 0;
+    int ret = 0;
     int thread_id = 0;
     char scan_data[128] = "string1, string2, string3, string4, "
                           "string5, string6, string7, string8";
@@ -28,13 +27,8 @@ TEST(json_mode, maat_scan_string) {
 
     snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp",
              g_json_filename);
-    snprintf(iris_idx_path, sizeof(iris_idx_path), "%s/index", json_iris_path);
     snprintf(json_path, sizeof(json_path), "./%s", g_json_filename);
 
-    int ret = write_json_to_iris(g_json_filename, iris_idx_path,
-                                 sizeof(iris_idx_path), g_logger);    
-    assert(ret == 0);
-
 	struct maat_options *opts = maat_options_new();
     maat_options_set_json_file(opts, json_path);
     maat_options_set_stat_on(opts);
@@ -58,47 +52,6 @@ TEST(json_mode, maat_scan_string) {
     maat_free(maat_inst);
 }
 
-TEST(iris_mode, maat_scan_string) {
-    char json_iris_path[512] = {0};
-    char iris_idx_path[PATH_MAX] = {0};
-    long long results[5] = {0};
-    size_t n_hit_result = 0;
-    int thread_id = 0;
-    const char *table_name = "KEYWORDS_TABLE";
-    char scan_data[128] = "string1, string2, string3, string4, "
-                          "string5, string6, string7, string8";
-
-    snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp",
-             g_json_filename);
-    snprintf(iris_idx_path, sizeof(iris_idx_path), "%s/index", json_iris_path);
-
-    int ret = write_json_to_iris(g_json_filename, iris_idx_path,
-                                 sizeof(iris_idx_path), g_logger);    
-    assert(ret == 0);
-
-    struct maat_options *opts = maat_options_new();
-    maat_options_set_stat_on(opts);
-    maat_options_set_perf_on(opts);
-    maat_options_set_iris(opts, iris_idx_path, iris_idx_path);
-    maat_options_set_logger(opts, "./maat_input_mode_gtest.log", LOG_LEVEL_INFO);
-     
-    struct maat *maat_inst = maat_new(opts, g_table_info_path);
-    EXPECT_TRUE(maat_inst != NULL);
-
-    int table_id = maat_get_table_id(maat_inst, table_name);
-    struct maat_state *state = maat_state_new(maat_inst, thread_id);
-
-    ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), 
-                           results, sizeof(results), &n_hit_result, state);
-    EXPECT_EQ(ret, MAAT_SCAN_HIT);
-    EXPECT_EQ(n_hit_result, 1);
-    EXPECT_EQ(results[0], 182);
-
-    maat_options_free(opts);
-    maat_state_free(state);
-    maat_free(maat_inst);
-}
-
 TEST(redis_mode, maat_scan_string) {
     char redis_ip[32] = "127.0.0.1";
     int redis_port = 6379;

test/object_nesting/object_nesting_gtest.cpp

@@ -185,7 +185,7 @@ TEST_F(MaatObjectExclude, level_3_function) {
 
         sprintf(table_line, "%lld\t%s\t%s\t%d", rules.add_items[i].object_id,
                 rules.add_items[i].incl_sub_ids_str, rules.add_items[i].excl_sub_ids_str, 1);
-        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_ADD);
     }
 
     object2object_runtime_commit(g2g_runtime, table_name, 1);
@@ -202,7 +202,7 @@ TEST_F(MaatObjectExclude, level_3_function) {
     memset(table_line, 0, sizeof(table_line));
     sprintf(table_line, "%lld\t%s\t%s\t%d", rules.del_items[0].object_id,
             rules.del_items[0].incl_sub_ids_str, rules.del_items[0].excl_sub_ids_str, 0);
-    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_DEL);
     object2object_runtime_commit(g2g_runtime, table_name, 2);
 
     super_object_cnt = object2object_runtime_get_super_objects(g2g_runtime, hit_object_ids, 2,
@@ -216,7 +216,7 @@ TEST_F(MaatObjectExclude, level_3_function) {
     memset(table_line, 0, sizeof(table_line));
     sprintf(table_line, "%lld\t%s\t%s\t%d", rules.del_items[1].object_id,
             rules.del_items[1].incl_sub_ids_str, rules.del_items[1].excl_sub_ids_str, 0);
-    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_DEL);
     object2object_runtime_commit(g2g_runtime, table_name, 3);
 
     super_object_cnt = object2object_runtime_get_super_objects(g2g_runtime, hit_object_ids, 2,
@@ -250,7 +250,7 @@ TEST_F(MaatObjectExclude, level_3_perf) {
 
         sprintf(table_line, "%lld\t%s\t%s\t%d", rules.add_items[i].object_id,
                 rules.add_items[i].incl_sub_ids_str, rules.add_items[i].excl_sub_ids_str, 1);
-        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_ADD);
     }
 
     object2object_runtime_commit(g2g_runtime, table_name, 4);
@@ -274,7 +274,7 @@ TEST_F(MaatObjectExclude, level_3_perf) {
     memset(table_line, 0, sizeof(table_line));
     sprintf(table_line, "%lld\t%s\t%s\t%d", rules.del_items[0].object_id,
             rules.del_items[0].incl_sub_ids_str, rules.del_items[0].excl_sub_ids_str, 0);
-    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_DEL);
     object2object_runtime_commit(g2g_runtime, table_name, 5);
 
     clock_gettime(CLOCK_MONOTONIC, &start);
@@ -291,7 +291,7 @@ TEST_F(MaatObjectExclude, level_3_perf) {
     memset(table_line, 0, sizeof(table_line));
     sprintf(table_line, "%lld\t%s\t%s\t%d", rules.del_items[1].object_id,
             rules.del_items[1].incl_sub_ids_str, rules.del_items[1].excl_sub_ids_str, 0);
-    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_DEL);
     object2object_runtime_commit(g2g_runtime, table_name, 6);
 
     clock_gettime(CLOCK_MONOTONIC, &start);
@@ -327,7 +327,7 @@ TEST_F(MaatObjectExclude, level_4_function) {
 
         sprintf(table_line, "%lld\t%s\t%s\t%d", rules.add_items[i].object_id,
                 rules.add_items[i].incl_sub_ids_str, rules.add_items[i].excl_sub_ids_str, 1);
-        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_ADD);
     }
 
     object2object_runtime_commit(g2g_runtime, table_name, 7);
@@ -347,7 +347,7 @@ TEST_F(MaatObjectExclude, level_4_function) {
     memset(table_line, 0, sizeof(table_line));
     sprintf(table_line, "%lld\t%s\t%s\t%d", rules.del_items[0].object_id,
             rules.del_items[0].incl_sub_ids_str, rules.del_items[0].excl_sub_ids_str, 0);
-    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_DEL);
     object2object_runtime_commit(g2g_runtime, table_name, 8);
 
     super_object_cnt = object2object_runtime_get_super_objects(g2g_runtime, hit_object_ids, 2,
@@ -384,7 +384,7 @@ TEST_F(MaatObjectExclude, level_4_perf) {
 
         sprintf(table_line, "%lld\t%s\t%s\t%d", rules.add_items[i].object_id,
                 rules.add_items[i].incl_sub_ids_str, rules.add_items[i].excl_sub_ids_str, 1);
-        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_ADD);
     }
 
     object2object_runtime_commit(g2g_runtime, table_name, 7);
@@ -408,7 +408,7 @@ TEST_F(MaatObjectExclude, level_4_perf) {
     memset(table_line, 0, sizeof(table_line));
     sprintf(table_line, "%lld\t%s\t%s\t%d", rules.del_items[0].object_id,
             rules.del_items[0].incl_sub_ids_str, rules.del_items[0].excl_sub_ids_str, 0);
-    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_DEL);
     object2object_runtime_commit(g2g_runtime, table_name, 8);
 
     clock_gettime(CLOCK_MONOTONIC, &start);
@@ -444,7 +444,7 @@ TEST_F(MaatObjectExclude, level_exceed_function) {
 
         sprintf(table_line, "%lld\t%s\t%s\t%d", rules.add_items[i].object_id,
                 rules.add_items[i].incl_sub_ids_str, rules.add_items[i].excl_sub_ids_str, 1);
-        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+        object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_ADD);
     }
 
     object2object_runtime_commit(g2g_runtime, table_name, 8);
@@ -465,7 +465,7 @@ TEST_F(MaatObjectExclude, level_exceed_function) {
     memset(table_line, 0, sizeof(table_line));
     sprintf(table_line, "%lld\t%s\t%s\t%d", rules.del_items[0].object_id,
             rules.del_items[0].incl_sub_ids_str, rules.del_items[0].excl_sub_ids_str, 0);
-    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, 4);
+    object2object_runtime_update(g2g_runtime, g2g_schema, table_name, table_line, MAAT_OP_DEL);
     object2object_runtime_commit(g2g_runtime, table_name, 9);
 
     hit_object_ids[0] = 7;

test/test_utils.cpp

@@ -4,7 +4,6 @@
 #include "maat_table.h"
 #include "maat_core.h"
 #include "maat_config_monitor.h"
-#include "json2iris.h"
 
 #include <assert.h>
 #include <unistd.h>
@@ -13,14 +12,14 @@ int line_idx = 0;
 long long absolute_expire_time = 0;
 
 static int
-count_line_num_cb(const char *table_name, const char *line, void *u_para)
+count_line_num_cb(const char *table_name, const char *line, void *u_para, enum maat_operation op)
 {
 	(*((unsigned int *)u_para))++;
 	return 0;
 }
 
 static int
-make_serial_rule(const char *table_name, const char *line, void *u_para)
+make_serial_rule(const char *table_name, const char *line, void *u_para, enum maat_operation op)
 {
 	struct serial_rule *s_rule=(struct serial_rule *)u_para;
     redisContext *ctx = s_rule->ref_ctx;
@@ -56,33 +55,22 @@ make_serial_rule(const char *table_name, const char *line, void *u_para)
 	return 0;
 }
 
-int write_json_to_iris(const char* json_fn, char *iris_path, size_t path_sz,
-                       struct log_handle *logger)
+int write_json_to_redis(const char *json_filename, char *redis_ip, int redis_port,
+                        int redis_db, struct log_handle *logger)
 {
-
     char *json_buff = NULL;
     size_t json_buff_sz = 0;
+    cJSON *json_root = NULL;
     
-    int ret = load_file_to_memory(json_fn, (unsigned char **)&json_buff,
+    int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff,
                                   &json_buff_sz);
     if (ret < 0) {
         return -1;
     }
 
-    ret = json2iris(json_buff, json_fn, NULL, iris_path, path_sz, NULL, NULL, logger);
-    FREE(json_buff);
+    convert_maat_json_rule(json_root, (unsigned char *)json_buff);
 
-    if (ret < 0) {
-        return -1;
-    }
-
-    return 0;
-}
-
-int write_iris_to_redis(const char *iris_path, char *redis_ip, int redis_port,
-                        int redis_db, struct log_handle *logger)
-{
-   redisContext *c = maat_connect_redis(redis_ip, redis_port, redis_db, logger);
+    redisContext *c = maat_connect_redis(redis_ip, redis_port, redis_db, logger);
     if (NULL == c) {
         return -1;
     }
@@ -96,7 +84,7 @@ int write_iris_to_redis(const char *iris_path, char *redis_ip, int redis_port,
     }
 
     size_t total_line_cnt = 0;
-    config_monitor_traverse(0, iris_path, NULL, count_line_num_cb,
+    config_monitor_traverse(0, json_root, NULL, count_line_num_cb,
                             NULL, &total_line_cnt, NULL, logger);
 
     struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
@@ -107,7 +95,7 @@ int write_iris_to_redis(const char *iris_path, char *redis_ip, int redis_port,
     }
 
     absolute_expire_time = server_time + 300;
-    config_monitor_traverse(0, iris_path, NULL, make_serial_rule,
+    config_monitor_traverse(0, json_root, NULL, make_serial_rule,
                             NULL, s_rule, NULL, logger);
     s_rule->ref_ctx = NULL;
     line_idx = 0;
@@ -126,23 +114,7 @@ int write_iris_to_redis(const char *iris_path, char *redis_ip, int redis_port,
     }
     FREE(s_rule);
     redisFree(c);
-    return 0;
-}
-
-int write_json_to_redis(const char *json_filename, char *redis_ip, int redis_port,
-                        int redis_db, struct log_handle *logger)
-{
-    char iris_path[512] = {0};
-
-    int ret = write_json_to_iris(json_filename, iris_path, sizeof(iris_path), logger);
-    if (ret < 0) {
-        return -1;
-    }
-
-    ret = write_iris_to_redis(iris_path, redis_ip, redis_port, redis_db, logger);
-    if (ret < 0) {
-        return -1;
-    }
+    cJSON_Delete(json_root);
 
     return 0;
 }
@@ -162,7 +134,7 @@ int rule_table_set_line(struct maat *maat_inst, const char *table_name,
     line_rule.table_name = table_name;
     line_rule.expire_after = expire_after;
 
-    return maat_cmd_set_line(maat_inst, &line_rule);
+    return maat_cmd_set_line(maat_inst, &line_rule, op);
 }
 
 #define TO_OBJECT2X_KEY(object_id, parent_id, condition_index) \
@@ -184,7 +156,7 @@ int object2rule_table_set_line(struct maat *maat_inst, const char *table_name,
     line_rule.table_name = table_name;
     line_rule.expire_after = expire_after;
 
-    return maat_cmd_set_line(maat_inst, &line_rule);
+    return maat_cmd_set_line(maat_inst, &line_rule, op);
 }
 
 int object2object_table_set_line(struct maat *maat_inst, const char *table_name,
@@ -201,7 +173,7 @@ int object2object_table_set_line(struct maat *maat_inst, const char *table_name,
     line_rule.table_name = table_name;
     line_rule.expire_after = expire_after;
 
-    return maat_cmd_set_line(maat_inst, &line_rule);
+    return maat_cmd_set_line(maat_inst, &line_rule, op);
 }
 
 int expr_table_set_line(struct maat *maat_inst, const char *table_name,
@@ -234,7 +206,7 @@ int expr_table_set_line(struct maat *maat_inst, const char *table_name,
     line_rule.table_name = table_name;
     line_rule.expire_after = expire_after;
 
-    return maat_cmd_set_line(maat_inst, &line_rule);
+    return maat_cmd_set_line(maat_inst, &line_rule, op);
 }
 
 int interval_table_set_line(struct maat *maat_inst, const char *table_name,
@@ -266,7 +238,7 @@ int interval_table_set_line(struct maat *maat_inst, const char *table_name,
     line_rule.table_name = table_name;
     line_rule.expire_after = expire_after;
 
-    return maat_cmd_set_line(maat_inst, &line_rule);
+    return maat_cmd_set_line(maat_inst, &line_rule, op);
 }
 
 int ip_table_set_line(struct maat *maat_inst, const char *table_name,
@@ -288,7 +260,7 @@ int ip_table_set_line(struct maat *maat_inst, const char *table_name,
     line_rule.table_name = table_name;
     line_rule.expire_after = expire_after;
 
-    return maat_cmd_set_line(maat_inst, &line_rule);
+    return maat_cmd_set_line(maat_inst, &line_rule, op);
 }
 int flag_table_set_line(struct maat *maat_inst, const char *table_name,
                     enum maat_operation op, long long item_id,
@@ -310,7 +282,7 @@ int flag_table_set_line(struct maat *maat_inst, const char *table_name,
     line_rule.table_name = table_name;
     line_rule.expire_after = expire_after;
 
-    return maat_cmd_set_line(maat_inst, &line_rule);
+    return maat_cmd_set_line(maat_inst, &line_rule, op);
 }
 
 void random_keyword_generate(char *keyword_buf, size_t sz)

test/test_utils.h

@@ -7,12 +7,6 @@
 int write_json_to_redis(const char *json_filename, char *redis_ip, int redis_port,
                         int redis_db, struct log_handle *logger);
 
-int write_iris_to_redis(const char *iris_path, char *redis_ip, int redis_port,
-                        int redis_db, struct log_handle *logger);
-
-int write_json_to_iris(const char* json_fn, char *iris_path, size_t path_sz,
-                       struct log_handle *logger);
-
 int rule_table_set_line(struct maat *maat_inst, const char *table_name,
                            enum maat_operation op, long long rule_id,
                            const char *user_region, int condition_num,

tools/maat_redis_tool.cpp

@@ -11,7 +11,6 @@
 #include "cJSON/cJSON.h"
 #include "maat_config_monitor.h"
 #include "maat_redis_monitor.h"
-#include "json2iris.h"
 #include "../test/test_utils.h"
 #include "hiredis/hiredis.h"