Merge branch 'feature-encrypt-maat-json' into 'master'

Feature encrypt maat json See merge request MESA_framework/maat!33
2020-01-22 20:55:54 +08:00
parent eb7912a854 0b15441525
commit 02b9914210
29 changed files with 3310 additions and 2024 deletions
--- a/inc/Maat_command.h
+++ b/inc/Maat_command.h
@@ -109,6 +109,7 @@ struct Maat_region_t
 struct Maat_group_t
 {
 	const char* table_name;
+	const char* virtual_table_name;
 	int group_id;	//If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number.
 	int parent_id;
 	int not_flag;
--- a/inc/Maat_rule.h
+++ b/inc/Maat_rule.h
@@ -30,7 +30,9 @@ enum MAAT_CHARSET
 	CHARSET_UNICODE_NCR_DEC,		//SGML Numeric character reference,decimal base, e.g. "&#1575;"
 	CHARSET_UNICODE_NCR_HEX,		//SGML Numeric character reference,hexdecimal base, e.g. "&#x627;"
 	CHARSET_URL_ENCODE_GB2312,		//URL encode with GB2312, e.g. the chinese word "china" was encoded to %D6%D0%B9%FA
-	CHARSET_URL_ENCODE_UTF8		//11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD
+	CHARSET_URL_ENCODE_UTF8,		//11, URL encode with UTF8,e.g. the chinese word "china" was encoded to %E4%B8%AD%E5%9B%BD
+	CHARSET_WINDOWS1251,
+	__CHARSET_MAX
 };
 enum MAAT_ACTION
 {
@@ -158,8 +160,7 @@ enum MAAT_INIT_OPT
 									//This option also disables background update.
 	MAAT_OPT_ENABLE_UPDATE,			//VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled.  DEFAULT: Backgroud update is enabled. Runtime setting is allowed.
 	MAAT_OPT_ACCEPT_TAGS,			//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]}
-	MAAT_OPT_FOREIGN_CONT_DIR,		//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files
- 	MAAT_OPT_FOREIGN_CONT_LINGER	//VALUE is interger *, SIZE=sizeof(int). Greater than 0: delete after VALUE seconds; 0: delete foreign content right after the notification callbacks;  Less than 0: NEVER delete. Default: 0.
+	MAAT_OPT_FOREIGN_CONT_DIR		//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files
 };
 //return -1 if failed, return 0 on success;
 int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT  type,const void* value,int size);
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,14 +1,14 @@
 cmake_minimum_required(VERSION 3.5)

 set(MAAT_FRAME_MAJOR_VERSION 2)
-set(MAAT_FRAME_MINOR_VERSION 7)
+set(MAAT_FRAME_MINOR_VERSION 8)
 set(MAAT_FRAME_PATCH_VERSION 0)
 set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.${MAAT_FRAME_PATCH_VERSION})

 message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")

 add_definitions(-fPIC)
-set(MAAT_SRC entry/cJSON.c  entry/config_monitor.cpp  entry/dynamic_array.cpp  entry/gram_index_engine.c  entry/interval_index.c  entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp  entry/Maat_command.cpp  entry/Maat_rule.cpp  entry/Maat_stat.cpp  entry/map_str2int.cpp  entry/rbtree.c  entry/stream_fuzzy_hash.c  entry/bool_matcher.cpp)
+set(MAAT_SRC entry/cJSON.c  entry/config_monitor.cpp  entry/dynamic_array.cpp  entry/gram_index_engine.c  entry/interval_index.c  entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp  entry/Maat_command.cpp  entry/Maat_rule.cpp entry/Maat_table.cpp entry/Maat_table_runtime.cpp entry/Maat_stat.cpp  entry/map_str2int.cpp  entry/rbtree.c  entry/stream_fuzzy_hash.c  entry/bool_matcher.cpp)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
 include_directories(/opt/MESA/include/MESA/)

--- a/src/entry/Maat_api.cpp
+++ b/src/entry/Maat_api.cpp
--- a/src/entry/Maat_command.cpp
+++ b/src/entry/Maat_command.cpp
@@ -143,9 +143,11 @@ long long redis_server_time(redisContext* ctx)
 	long long server_time=0;
 	redisReply* data_reply=NULL;
 	data_reply=_wrap_redisCommand(ctx,"TIME"); 
-	assert(data_reply->type==REDIS_REPLY_ARRAY);
-	server_time=atoll(data_reply->element[0]->str);
-	freeReplyObject(data_reply);
+	if(data_reply->type==REDIS_REPLY_ARRAY)
+	{
+		server_time=atoll(data_reply->element[0]->str);
+		freeReplyObject(data_reply);
+	}
 	return server_time;
 }
 enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p)
@@ -245,11 +247,17 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
 void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz)
 {
 	if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
-	snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d", p_group->group_id, 
+	const char* vt_name="null";
+	if(p_group->virtual_table_name!=NULL)
+	{
+		vt_name=p_group->virtual_table_name;
+	}
+	snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d\t%s", p_group->group_id, 
 									p_group->parent_id,
 									op,
 									p_group->not_flag,
-									p_group->parent_type);
+									p_group->parent_type,
+									vt_name);
 	return;
 }
 void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz)
@@ -274,65 +282,65 @@ void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, s
 	switch(p->region_type)
 	{
 		case REGION_IP:
-			ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1"
-					,p->region_id
-					,group_id
-					,p->ip_rule.addr_type
-					,p->ip_rule.src_ip
-					,p->ip_rule.mask_src_ip
-					,p->ip_rule.src_port
-					,p->ip_rule.mask_src_port
-					,p->ip_rule.dst_ip
-					,p->ip_rule.mask_dst_ip
-					,p->ip_rule.dst_port
-					,p->ip_rule.mask_dst_port
-					,p->ip_rule.protocol
-					,p->ip_rule.direction);	
+			ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1",
+					p->region_id,
+					group_id,
+					p->ip_rule.addr_type,
+					p->ip_rule.src_ip,
+					p->ip_rule.mask_src_ip,
+					p->ip_rule.src_port,
+					p->ip_rule.mask_src_port,
+					p->ip_rule.dst_ip,
+					p->ip_rule.mask_dst_ip,
+					p->ip_rule.dst_port,
+					p->ip_rule.mask_dst_port,
+					p->ip_rule.protocol,
+					p->ip_rule.direction);	
 			break;
 		case REGION_EXPR:
 			if(p->expr_rule.district==NULL)
 			{
-				ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1"
-						,p->region_id
-						,group_id
-						,p->expr_rule.keywords
-						,p->expr_rule.expr_type
-						,p->expr_rule.match_method
-						,p->expr_rule.hex_bin);	
+				ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1",
+						p->region_id,
+						group_id,
+						p->expr_rule.keywords,
+						p->expr_rule.expr_type,
+						p->expr_rule.match_method,
+						p->expr_rule.hex_bin);	
 			}
 			else	//expr_plus
 			{
-				ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1"
-						,p->region_id
-						,group_id
-						,p->expr_rule.keywords
-						,p->expr_rule.district
-						,p->expr_rule.expr_type
-						,p->expr_rule.match_method
-						,p->expr_rule.hex_bin);	
+				ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1",
+						p->region_id,
+						group_id,
+						p->expr_rule.district,
+						p->expr_rule.keywords,
+						p->expr_rule.expr_type,
+						p->expr_rule.match_method,
+						p->expr_rule.hex_bin);	
 			}
 			break;
 		case REGION_INTERVAL:
-			ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1"
-					,p->region_id
-					,group_id
-					,p->interval_rule.low_boundary
-					,p->interval_rule.up_boundary);
+			ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1",
+					p->region_id,
+					group_id,
+					p->interval_rule.low_boundary,
+					p->interval_rule.up_boundary);
 			break;
 		case REGION_DIGEST:
-			ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1"
-					,p->region_id
-					,group_id
-					,p->digest_rule.orgin_len
-					,p->digest_rule.digest_string
-					,p->digest_rule.confidence_degree);
+			ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1",
+					p->region_id,
+					group_id,
+					p->digest_rule.orgin_len,
+					p->digest_rule.digest_string,
+					p->digest_rule.confidence_degree);
 			break;
 		case REGION_SIMILARITY:
-			ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1"
-					,p->region_id
-					,group_id
-					,p->similarity_rule.target
-					,p->similarity_rule.threshold);
+			ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1",
+					p->region_id,
+					group_id,
+					p->similarity_rule.target,
+					p->similarity_rule.threshold);
 			break;
 		default:
 			assert(0);
@@ -915,7 +923,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
 				continue;
 			}
 			region_cmd=&(group_cmd->regions[j]);
-			region_cmd->table_name=_maat_strdup(feather->p_table_info[region_inner->table_id]->table_name[0]);
+			region_cmd->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, region_inner->table_id));
 			region_cmd->region_id=region_inner->region_id;
 			//NOTICE: region_type only avilable when OP_ADD, 
 			region_cmd->region_type=REGION_EXPR;
@@ -1374,7 +1382,7 @@ error_out:

 int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
 {
-	int i=0,j=0,ret=0;
+	int i=0, j=0;
 	const char *table_name=NULL;
 	int table_id=0;
 	struct Maat_group_t* p_group=NULL;
@@ -1399,8 +1407,8 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
 		{
 			p_region=&(p_group->regions[j]);
 			table_name=p_region->table_name;
-			ret=map_str2int(feather->map_tablename2id, table_name, &table_id);
-			if(ret<0)
+			table_id=Maat_table_get_id_by_name(feather->table_mgr, table_name);
+			if(table_id<0)
 			{
 				MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module 
 						,"Unknown table %s of Maat_cmd_t[%d]->group[%d]->region[%d]."
@@ -1409,7 +1417,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
 				return -1;
 			}
 			table_type=type_region2table(p_region);
-			if(table_type!=feather->p_table_info[table_id]->table_type)
+			if(table_type!=Maat_table_get_type_by_id(feather->table_mgr, table_id))
 			{
 				MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module 
 						,"Table %s not support region type %d of Maat_cmd_t[%d]->group[%d]->region[%d]."
@@ -1419,7 +1427,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
 				return -1;
 			}
 			free((char*)p_region->table_name);
-			p_region->table_name=_maat_strdup(feather->p_table_info[table_id]->table_name[0]);
+			p_region->table_name=_maat_strdup(Maat_table_get_name_by_id(feather->table_mgr, table_id));
 		}
 	}
 	return 0;
@@ -1434,7 +1442,10 @@ void check_maat_expiration(redisContext *ctx, void *logger)
 	long long server_time=0;

 	server_time=redis_server_time(ctx);
-
+	if(!server_time)
+	{
+		return;
+	}
 	data_reply=_wrap_redisCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_expire_sset,server_time);
 	if(data_reply->type!=REDIS_REPLY_ARRAY||data_reply->elements==0)
 	{
@@ -1473,7 +1484,10 @@ void cleanup_update_status(redisContext *ctx, void *logger)
 	long long server_time=0, version_upper_bound=0,version_lower_bound=0,version_num=0,entry_num=0;

 	server_time=redis_server_time(ctx);
-
+	if(!server_time)
+	{
+		return;
+	}
 	reply=_wrap_redisCommand(ctx,"MULTI");
 	freeReplyObject(reply);
 	redisAppendCommand(ctx, "ZRANGEBYSCORE %s -inf %lld",mr_version_sset,server_time-MAAT_REDIS_SYNC_TIME);
@@ -1599,10 +1613,9 @@ void rewrite_table_line_with_foreign(struct serial_rule_t*p)
 }
 void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n_foreign, const char* dir, void* logger)
 {
-	int ret=0, i=0;
+	int i=0;
 	const char* p_foreign=NULL;
 	int foreign_key_size=0;
-	struct stat file_info;
 	p_rule->n_foreign=n_foreign;
 	p_rule->f_keys=ALLOC(struct foreign_key, n_foreign);
 	for(i=0; i<n_foreign; i++)
@@ -1633,13 +1646,7 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n
 		}
 		p_rule->f_keys[i].key=(char*)calloc(sizeof(char),foreign_key_size+1);
 		memcpy(p_rule->f_keys[i].key, p_foreign, foreign_key_size);
-	
 		p_rule->f_keys[i].filename=get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[i].key, dir);
-		ret=stat(p_rule->f_keys[i].filename, &file_info);
-		if(ret==0)
-		{
-			p_rule->f_keys[i].is_existed=1;
-		}
 	}
 	if(i!=n_foreign)
 	{
@@ -1651,7 +1658,7 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n
 }
 int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger)
 {
-	int ret=0, table_id=0, i=0;
+	int i=0;
 	int rule_with_foreign_key=0;
 	struct Maat_table_desc* p_table=NULL;
 	struct plugin_table_desc* plugin_desc=NULL;
@@ -1661,14 +1668,13 @@ int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list,
 		{
 			continue;
 		}
-		ret=map_str2int(feather->map_tablename2id, rule_list[i].table_name, &table_id);
-		if(ret<0)
+		p_table=Maat_table_get_desc_by_name(feather->table_mgr, rule_list[i].table_name);
+		if(!p_table||p_table->table_type!=TABLE_TYPE_PLUGIN)
 		{
 			continue;
 		}
-		p_table=feather->p_table_info[table_id];
 		plugin_desc= &(p_table->plugin);
-		if(p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->n_foreign==0)
+		if(plugin_desc->n_foreign==0)
 		{
 			continue;
 		}
@@ -1722,25 +1728,44 @@ void _get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int
 	redisReply* reply=NULL;
 	struct serial_rule_t*p=NULL;
 	FILE* fp=NULL;
+	struct stat file_info;
+
 	for(i=0;i<rule_num;i++)
 	{
 		p=rule_list+i;
-		if(p->op==MAAT_OP_DEL||p->n_foreign==0)
+		if(p->n_foreign==0)
 		{
 			continue;
 		}
-		for(j=0; j<p->n_foreign; j++)
+		if(p->op==MAAT_OP_DEL)
 		{
-			if(p->f_keys[j].is_existed==1)
+			for(j=0; j<rule_list[i].n_foreign; j++)
 			{
-				continue;
+				ret=system_cmd_rm(rule_list[i].f_keys[j].filename);
+				if(ret==-1)
+				{
+					MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
+							"Foreign content file %s remove failed.",
+							rule_list[i].f_keys[j].filename);
+				}
+			}
+		}	
+		else
+		{
+			for(j=0; j<p->n_foreign; j++)
+			{
+				ret=stat(p->f_keys[i].filename, &file_info);
+				if(ret==0)
+				{
+					continue;
+				}
+				snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key);
+				ret=redisAppendCommand(ctx, redis_cmd);
+				track[key_num].rule_idx=i;
+				track[key_num].foreign_idx=j;
+				key_num++;
+				assert(ret==REDIS_OK);
 			}
-			snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key);
-			ret=redisAppendCommand(ctx, redis_cmd);
-			track[key_num].rule_idx=i;
-			track[key_num].foreign_idx=j;
-			key_num++;
-			assert(ret==REDIS_OK);
 		}
 	}
 	for(i=0;i<key_num;i++)
@@ -1799,15 +1824,16 @@ void redis_monitor_traverse(long        long version, struct source_redis_ctx* m
 		int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para
 		void (*finish)(void*),//u_para
 		void* u_para,
-		const unsigned char* dec_key,
+		const char* dec_key,
 		_Maat_feather_t* feather)
 {
-	int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0, valid_column=-1;
+	int table_id=0, i=0, rule_num=0, empty_value_num=0, valid_column=-1;
 	int ret=0;
 	struct serial_rule_t* rule_list=NULL;
 	int update_type=CM_UPDATE_TYPE_INC;
 	long long new_version=0;
 	enum MAAT_TABLE_TYPE table_type;
+	struct Maat_table_desc* table_desc=NULL;
 	const struct plugin_table_desc* plugin_desc=NULL;
 	void* logger=feather->logger;

@@ -1889,17 +1915,18 @@ void redis_monitor_traverse(long        long version, struct source_redis_ctx* m
 		if(rule_list[i].table_line==NULL||rule_list[i].with_error==1)
 		{
 			continue;
-		}
-		ret=map_str2int(feather->map_tablename2id,rule_list[i].table_name,&table_id);
+		}
+		table_id=Maat_table_get_id_by_name(feather->table_mgr, rule_list[i].table_name);
 		if(table_id<0)//Unrecognized table.
 		{
 			continue;
-		}
+		}
 		table_type=Maat_table_get_type_by_id(feather->table_mgr, table_id);
 		if(rule_list[i].op==MAAT_OP_DEL)
 		{
 			if(table_type==TABLE_TYPE_PLUGIN)
-			{
+			{
+				table_desc=Maat_table_get_by_id(feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
 				plugin_desc=&(table_desc->plugin);
 				valid_column=plugin_desc->valid_flag_column;
 			}
@@ -1919,33 +1946,6 @@ void redis_monitor_traverse(long        long version, struct source_redis_ctx* m
 		{
 			rewrite_table_line_with_foreign(rule_list+i);
 		}
-		update(rule_list[i].table_name,rule_list[i].table_line,u_para);
-		if(rule_list[i].n_foreign&&rule_list[i].op==MAAT_OP_DEL)
-		{
-			
-			for(j=0; j<rule_list[i].n_foreign; j++)
-			{
-				if(feather->foreign_cont_linger==0)
-				{
-					ret=system_cmd_rm(rule_list[i].f_keys[j].filename);
-					if(ret==-1)
-					{
-						MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
-								"Foreign content file %s remove failed.",
-								rule_list[i].f_keys[j].filename);
-					}
-				}
-				else if(feather->foreign_cont_linger>0)
-				{
-					garbage_bagging_with_timeout(GARBAGE_FOREIGN_FILE, rule_list[i].f_keys[j].filename, feather->foreign_cont_linger, feather->garbage_q);
-					rule_list[i].f_keys[j].filename=NULL;//transfer owner to garbage collection.
-				}
-				else
-				{
-					//Less than 0, don't delete.
-				}
-			}
-			
 		update(rule_list[i].table_name,rule_list[i].table_line,u_para);
 	}
 	finish(u_para);
@@ -2102,11 +2102,15 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
 	{
 		return -1;
 	}
+	server_time=redis_server_time(write_ctx);
+	if(!server_time)
+	{
+		return -1;
 	}
 	s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),line_num);
 	for(i=0;i<line_num;i++)
-	{
-		ret=map_str2int(_feather->map_tablename2id, line_rule[i]->table_name, &table_id);
+	{
+		table_id=Maat_table_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name);
 		if(table_id<0)
 		{
 			MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command 
@@ -2115,9 +2119,8 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
 					, line_rule[i]->table_name);
 			ret=-1;
 			goto error_out;
-		}
-		p_table=_feather->p_table_info[table_id];
-		plugin_desc=&(p_table->plugin);
+		}
+		p_table=Maat_table_get_by_id(_feather->table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
 		if(!p_table)
 		{
 			MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command 
@@ -2126,6 +2129,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_line_t** line_ru
 					, line_rule[i]->table_name);
 			ret=-1;
 			goto error_out;
+		}
 		plugin_desc=&(p_table->plugin);
 		if(op==MAAT_OP_ADD)
 		{
@@ -2361,7 +2365,7 @@ int Maat_cmd_commit(Maat_feather_t feather)
 int Maat_cmd_commit(Maat_feather_t feather)
 {
 	_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
-
+
 	int ret=-1, i=0;
 	int new_region_num=0,new_group_num=0;
 	int serial_rule_num=0,serial_rule_idx=0;
@@ -2373,7 +2377,7 @@ int Maat_cmd_commit(Maat_feather_t feather)

 	struct serial_rule_t* s_rule=NULL;
 	if(_feather->input_mode!=SOURCE_REDIS)
-	{
+	{
 		return ret;
 	}
 	if(_feather->cmd_q_cnt==0)
@@ -2391,14 +2395,16 @@ int Maat_cmd_commit(Maat_feather_t feather)
 		serial_rule_num+=calculate_serial_rule_num(p, &new_region_num, &new_group_num);
 		p=p->next;
 	}
-	_feather->server_time=redis_server_time(write_ctx);
+	_feather->server_time=redis_server_time(write_ctx);
+	if(!_feather->server_time)
+	{
+		goto error_out;
 	}
 	if(_feather->AUTO_NUMBERING_ON==1)
 	{
 		data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num); 
 		if(data_reply->type!=REDIS_REPLY_INTEGER)
 		{
-			freeReplyObject(data_reply);
 			freeReplyObject(data_reply);
 			goto error_out;
 		}
@@ -2408,7 +2414,6 @@ int Maat_cmd_commit(Maat_feather_t feather)
 		data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_group_id_var, new_group_num); 
 		if(data_reply->type!=REDIS_REPLY_INTEGER)
 		{
-			freeReplyObject(data_reply);
 			freeReplyObject(data_reply);
 			goto error_out;
 		}
@@ -2611,6 +2616,10 @@ static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAA
 static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAAT_OPERATION op, int id, const char* table_name, const char* line)
 {
 	redisContext* write_ctx=get_redis_ctx_for_write(_feather);
+	_feather->server_time=redis_server_time(write_ctx);
+	if(!_feather->server_time)
+	{
+		return -1;
 	}
 	struct serial_rule_t s_rule;
 	set_serial_rule(&s_rule, op, id, 0, table_name, line, 0);
--- a/src/entry/Maat_rule.cpp
+++ b/src/entry/Maat_rule.cpp
--- a/src/entry/Maat_stat.cpp
+++ b/src/entry/Maat_stat.cpp
@@ -1,4 +1,5 @@
 #include "Maat_rule_internal.h"
+#include "Maat_table.h"
 #include "alignment_int64.h"
 #include <time.h>
 #include <MESA/field_stat.h>
@@ -121,10 +122,11 @@ void maat_stat_init(struct _Maat_feather_t* feather)
 					FS_STYLE_COLUMN,
 					FS_CALC_SPEED,
 					"hit_rate");
-	feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT,"Sum");
-	for(i=0;i<MAX_TABLE_NUM;i++)
+	feather->total_stat_id=FS_register(feather->stat_handle, FS_STYLE_LINE, FS_CALC_CURRENT, "Sum");
+	size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr);
+	for(i=0; i<(int)max_table_num; i++)
 	{
-		p_table=feather->p_table_info[i];
+		p_table=Maat_table_get_by_id_raw(feather->table_mgr, i);
 		if(p_table==NULL||p_table->table_type==TABLE_TYPE_PLUGIN
 						||p_table->table_type==TABLE_TYPE_GROUP
 						||p_table->table_type==TABLE_TYPE_COMPILE)
@@ -185,7 +187,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)

 	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_VERSION], 0,FS_OP_SET,feather->maat_version);
 	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_THRED_NUM], 0,FS_OP_SET,active_thread_num);
-	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,feather->table_cnt);
+	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_TABLE_NUM], 0,FS_OP_SET,Maat_table_manager_get_count(feather->table_mgr));
+
 	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_OUTER_MID_NUM], 0,FS_OP_SET,outer_mid_cnt);
 	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_INNER_MID_NUM], 0,FS_OP_SET,inner_mid_cnt);
 	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_NOT_GROUP_HIT], 0,FS_OP_SET,not_grp_hit_cnt);
@@ -197,7 +200,8 @@ void maat_stat_output(struct _Maat_feather_t* feather)
 	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);
 	feather->update_err_cnt=0;
 	feather->iconv_err_cnt=0;
-	for(i=0;i<MAX_TABLE_NUM;i++)
+	size_t max_table_num=Maat_table_manager_get_size(feather->table_mgr);
+	for(i=0; i<(int)max_table_num; i++)
 	{
 		table_stream_num=0;
 		table_scan_cnt=0;
@@ -205,12 +209,13 @@ void maat_stat_output(struct _Maat_feather_t* feather)
 		table_scan_cpu_time=0;
 		table_hit_cnt=0;
 		table_regex_ipv6_num=0;
-		p_table=feather->p_table_info[i];
+		
+		p_table=Maat_table_get_by_id_raw(feather->table_mgr, i);
 		if(p_table==NULL)
 		{
 			continue;
 		}
-		table_rt=feather->scanner->table_rt[i];
+		table_rt=Maat_table_runtime_get(feather->scanner->table_rt_mgr, i);
 		switch(p_table->table_type)
 		{
 			case TABLE_TYPE_PLUGIN:
--- a/src/entry/Maat_table.cpp
+++ b/src/entry/Maat_table.cpp
@@ -0,0 +1,784 @@
+#include "Maat_table.h"
+#include "map_str2int.h"
+#include "Maat_utils.h"
+#include "cJSON.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <MESA/MESA_handle_logger.h>
+
+#define	MAX_TABLE_NUM			256
+
+struct Maat_table_manager
+{
+	struct Maat_table_desc* p_table_info[MAX_TABLE_NUM];
+	size_t table_cnt;
+	MESA_htable_handle map_tablename2id;
+	int active_plugin_table_num;
+	int is_last_plugin_table_updating;
+};
+
+int read_expr_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map)
+{
+	int j=0,ret[4]={0};
+	char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0};
+	char *token=NULL,*sub_token=NULL,*saveptr;
+	struct expr_table_desc* p=&(table->expr);
+	sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id)
+											,table->table_name[0]
+											,table_type
+											,src_charset
+											,dst_charset
+											,merge
+											,&(p->cross_cache_size)
+											,quick_str_scan);
+	memset(ret,0,sizeof(ret));
+	ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type));
+	ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset));
+	ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge));
+	if(strlen(quick_str_scan)>0)
+	{
+		ret[3]=map_str2int(string2int_map,str_tolower(quick_str_scan),&(p->quick_expr_switch));
+	}
+	memset(quick_str_scan,0,sizeof(quick_str_scan));
+
+	for(j=0;j<4;j++)
+	{
+		if(ret[j]<0)
+		{
+			return -1;
+		}
+	}
+	j=0;
+	for (token = dst_charset; ; token= NULL)
+	{
+		sub_token= strtok_r(token,"/", &saveptr);
+		if (sub_token == NULL)
+			break;
+		ret[3]=map_str2int(string2int_map,str_tolower(sub_token),(int*)&(p->dst_charset[j]));
+		if(ret[3]>0)
+		{
+			if(p->dst_charset[j]==p->src_charset)
+			{
+				p->src_charset_in_dst=TRUE;
+			}
+			j++;
+		}
+		else
+		{
+			return -1;
+		}
+		
+	}
+	return 0;
+}
+int read_virtual_table_info(const char* line, struct Maat_table_desc* table, MESA_htable_handle string2int_map)
+{
+	int ret=0;
+	char table_type[16];
+	ret=sscanf(line, "%d\t%s\t%s\t%s", &(table->table_id),
+									table->table_name[0],
+									table_type,
+									table->virtual_table.real_table_name);
+	if(ret!=4)
+	{
+		return -1;
+	}	
+	ret=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type));
+	if(ret<0)
+	{
+		return -1;
+	}
+	return 0;
+}
+Maat_table_desc* table_info_new(void)
+{
+	struct Maat_table_desc*p=ALLOC(struct Maat_table_desc, 1);
+	p->conj_cnt=1;
+	return p;
+}
+void table_info_free(struct Maat_table_desc*p)
+{
+	free(p);
+	return;
+}
+int _read_integer_arrary(char* string, int *array, int size)
+{
+	char *token=NULL,*sub_token=NULL,*saveptr;
+	int i=0;
+	for (token = string, i=0; i<size ; token= NULL, i++)
+	{
+		sub_token= strtok_r(token,",", &saveptr);
+		if (sub_token == NULL)
+			break;
+		sscanf(sub_token, "%d", array+i);
+	}
+	return i;
+}
+#define COLUMN_PLUGIN_DESCR_JSON	4
+int read_plugin_table_description(const char* line, struct Maat_table_desc* p)
+{
+	int i=0,ret=0;
+	size_t offset=0, len=0;
+	cJSON* json=NULL, *tmp=NULL, *array_item=NULL;
+	char* copy_line=NULL, *plug_info=NULL;
+	struct plugin_table_desc* plugin_desc=&(p->plugin);
+	copy_line=_maat_strdup(line);
+	ret=get_column_pos(copy_line, COLUMN_PLUGIN_DESCR_JSON, &offset, &len);
+	if(i<0)
+	{
+		goto error_out;
+	}
+	if(offset+len<strlen(copy_line))
+	{
+		copy_line[offset+len+1]='\0';
+	}
+	plug_info=copy_line+offset;
+
+	if(NULL==strchr(plug_info,'{'))//For old version compatible.
+	{
+		ret=sscanf(plug_info, "%d", &(plugin_desc->valid_flag_column));
+		if(ret==0||ret==EOF)
+		{
+			plugin_desc->valid_flag_column=-1;
+		}
+		free(copy_line);
+		return 0;
+	}
+	json=cJSON_Parse(plug_info);
+	if(!json)
+	{
+		goto error_out;
+	}
+	tmp=cJSON_GetObjectItem(json, "key");
+	if(tmp!=NULL)
+	{
+		assert(tmp->type==cJSON_Number);
+		plugin_desc->key_column=tmp->valueint;
+	}
+	tmp=cJSON_GetObjectItem(json, "valid");
+	if(tmp!=NULL)
+	{
+		assert(tmp->type==cJSON_Number);
+		plugin_desc->valid_flag_column=tmp->valueint;
+	}
+	tmp=cJSON_GetObjectItem(json, "tag");
+	if(tmp!=NULL)
+	{
+		assert(tmp->type==cJSON_Number);
+		plugin_desc->rule_tag_column=tmp->valueint;
+	}
+	tmp=cJSON_GetObjectItem(json, "estimate_size");
+	if(tmp!=NULL)
+	{
+		assert(tmp->type==cJSON_Number);
+		plugin_desc->estimate_size=tmp->valueint;
+	}
+	tmp=cJSON_GetObjectItem(json, "foreign");
+	if(tmp!=NULL)
+	{
+		if(tmp->type==cJSON_String)
+		{
+			plugin_desc->n_foreign=_read_integer_arrary(tmp->valuestring, plugin_desc->foreign_columns, MAX_FOREIGN_CLMN_NUM);
+		}
+		else if(tmp->type==cJSON_Array)
+		{
+			plugin_desc->n_foreign= cJSON_GetArraySize(tmp);
+			for(i=0;i<plugin_desc->n_foreign; i++)
+			{
+				array_item=cJSON_GetArrayItem(tmp, i);
+				assert(array_item->type==cJSON_Number);
+				plugin_desc->foreign_columns[i]=array_item->valueint;
+			}
+		}
+	}
+	cJSON_Delete(json);
+
+	free(copy_line);
+	return 0;
+error_out:
+	free(copy_line);
+	return -1;
+}
+
+static int Maat_table_build_map(struct Maat_table_manager* table_mgr, void* logger)
+{
+	struct Maat_table_desc** p_table_info=table_mgr->p_table_info;
+	size_t n_table=MAX_TABLE_NUM;
+
+	MESA_htable_handle map_tablename2id=map_create();
+	size_t i=0;
+	int j=0, ret=0;
+	for(i=0;i<n_table;i++)
+	{
+		if(p_table_info[i]==NULL)
+		{
+			continue;
+		}
+		
+		switch(p_table_info[i]->table_type)
+		{
+			case TABLE_TYPE_VIRTUAL:
+				ret=map_str2int(map_tablename2id, p_table_info[i]->virtual_table.real_table_name, &(p_table_info[i]->virtual_table.real_table_id));
+				if(ret<0)
+				{
+					MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+									"Undefined real table %s, virtual table %s of table id %d.",
+									p_table_info[i]->virtual_table.real_table_name,
+									p_table_info[i]->table_name[j],
+									p_table_info[i]->table_id);
+					goto failed;
+				}
+				break;
+			default:
+				break;
+		}
+
+		
+		for(j=0; j<p_table_info[i]->conj_cnt; j++)
+		{
+			ret=map_register(map_tablename2id, p_table_info[i]->table_name[j], p_table_info[i]->table_id);
+			if(ret<0)
+			{
+				MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+								"Duplicate table %s of table id %d",
+								p_table_info[i]->table_name[j],
+								p_table_info[i]->table_id);
+				continue;
+			}
+		}
+		
+	}
+	table_mgr->map_tablename2id=map_tablename2id;
+	return 0;
+failed:
+	map_destroy(map_tablename2id);
+	return -1;
+}
+
+void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr)
+{
+	size_t i=0;
+	for(i=0;i<MAX_TABLE_NUM;i++)
+	{
+		if(table_mgr->p_table_info[i]==NULL)
+		{
+			continue;
+		}
+		table_info_free(table_mgr->p_table_info[i]);
+		table_mgr->p_table_info[i]=NULL;
+	}
+	MESA_htable_destroy(table_mgr->map_tablename2id, free);
+	free(table_mgr);
+	return;
+}
+
+struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger)
+{
+	struct Maat_table_manager* table_mgr=NULL;
+	FILE*fp=NULL;
+	char line[MAX_TABLE_LINE_SIZE];
+	int i=0, ret=0;
+	char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0};
+	MESA_htable_handle string2int_map=NULL;;
+	struct Maat_table_desc*p=NULL;
+	struct Maat_table_desc*conj_table=NULL;
+	fp=fopen(table_info_path,"r");
+	if(fp==NULL)
+	{
+		fprintf(stderr,"Maat read table info %s error.\n",table_info_path);
+		MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+								"Maat read table info %s failed: %s.\n", table_info_path, strerror(errno));
+		return NULL;
+	}
+	table_mgr=ALLOC(struct Maat_table_manager, 1);
+	struct Maat_table_desc** p_table_info=table_mgr->p_table_info;
+	size_t n_table=MAX_TABLE_NUM;
+
+	string2int_map=map_create();
+	map_register(string2int_map,"expr", TABLE_TYPE_EXPR);
+	map_register(string2int_map,"ip", TABLE_TYPE_IP);	
+	map_register(string2int_map,"ip_plus", TABLE_TYPE_IP_PLUS);
+	map_register(string2int_map,"compile", TABLE_TYPE_COMPILE);
+	map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
+	map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL);
+	map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
+	map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS);
+	map_register(string2int_map,"group", TABLE_TYPE_GROUP);
+	map_register(string2int_map,"similar", TABLE_TYPE_SIMILARITY);
+	map_register(string2int_map,"virtual", TABLE_TYPE_VIRTUAL);
+	map_register(string2int_map,"quickoff", 0);
+	map_register(string2int_map,"quickon", 1);
+	map_register(string2int_map,"escape", USER_REGION_ENCODE_ESCAPE);
+//	map_register(string2int_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet
+
+	const char** charset_name_list=charset_get_all_name();
+	for(i=0;i<MAX_CHARSET_NUM;i++)
+	{
+		if(strlen(charset_name_list[i])>0)
+		{
+			map_register(string2int_map, charset_name_list[i], i);
+		}
+		else
+		{
+			break;
+		}
+	}
+
+	map_register(string2int_map,"yes", 1);
+	map_register(string2int_map,"no", 0);
+	
+
+	i=0;
+	while(NULL!=fgets(line,sizeof(line),fp))
+	{
+		i++;
+
+		if(line[0]=='#'||line[0]==' '||line[0]=='\t'||strlen(line)<4)
+		{
+			continue;
+		}
+		p=table_info_new();
+
+		ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id)
+											,p->table_name[0]
+											,table_type_str
+											,not_care);
+		if(ret<3)
+		{
+			MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+									"Maat read table info %s line %d error: not enough column.",table_info_path,i);
+			continue;
+		}
+		ret=map_str2int(string2int_map,str_tolower(table_type_str),(int*)&(p->table_type));
+		if(ret<0)
+		{
+			MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+						"Maat read table info %s line %d error:invalid table type.",table_info_path,i);
+			goto invalid_table;
+		}
+		switch(p->table_type)
+		{
+			case TABLE_TYPE_EXPR:
+			case TABLE_TYPE_EXPR_PLUS:
+				ret=read_expr_table_info(line, p, string2int_map);
+				if(ret<0)
+				{
+					fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i);
+					MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+									"Maat read table info %s line %d error:illegal column.",table_info_path,i);
+					goto invalid_table;
+				}
+				break;
+			case TABLE_TYPE_PLUGIN:
+				ret=read_plugin_table_description(line, p);
+				if(ret<0)
+				{
+					fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i);
+					MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+									"Maat read table info %s line %d error:illegal plugin info.",table_info_path,i);
+					goto invalid_table;
+				}
+				break;
+			case TABLE_TYPE_VIRTUAL:
+				ret=read_virtual_table_info(line, p, string2int_map);
+				if(ret<0)
+				{
+					fprintf(stderr,"Maat read table info %s line %d error:illegal virtual info.\n",table_info_path,i);
+					MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+									"Maat read table info %s line %d error:illegal virtual info.",table_info_path,i);
+					goto invalid_table;
+				}
+				break;
+			case TABLE_TYPE_COMPILE:	
+				ret=sscanf(not_care,"%[a-z0-9]",tmp_str);
+				if(ret>0)
+				{	
+					ret=map_str2int(string2int_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding));
+				}
+				if(ret!=1)
+				{
+					p->compile.user_region_encoding=USER_REGION_ENCODE_NONE;
+				}
+			default:
+				break;
+		}
+
+		if((unsigned int)p->table_id>=n_table)
+		{
+			fprintf(stderr,"Maat read table info %s:%d error: table id %uh > %zu.\n",table_info_path,i,p->table_id,n_table);
+			MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+				"Maat read table info %s line %d error: table id %uh > %d.\n",table_info_path,i,p->table_id,n_table);
+
+			goto invalid_table;
+		}
+		if(p_table_info[p->table_id]!=NULL)//duplicate table_id,means conjunction table;
+		{
+			conj_table=p_table_info[p->table_id];
+			if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM)
+			{
+				MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+								"Maat read table info %s line %d error:reach tableid %d conjunction upper limit."
+								,table_info_path,i,p->table_id);
+				goto invalid_table;
+			}
+			memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN);
+			conj_table->conj_cnt++;
+			MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
+								"Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)."
+								,table_info_path,i,p->table_name[0]
+								,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt);
+			//use goto to free the conjunctioned table_info
+			goto invalid_table;
+		}
+
+		p_table_info[p->table_id]=p;
+		table_mgr->table_cnt++;
+		continue;
+invalid_table:
+		table_info_free(p);
+		p=NULL;
+	}
+	fclose(fp);
+	ret=Maat_table_build_map(table_mgr, logger);
+	if(ret<0)
+	{
+		return NULL;
+	}
+	map_destroy(string2int_map);
+	return table_mgr;
+}
+size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr)
+{
+	return MAX_TABLE_NUM;
+}
+size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr)
+{
+	return table_mgr->table_cnt;
+}
+int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
+{
+	int i=0;
+	for(i=0; i< MAX_TABLE_NUM; i++)
+	{
+		if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_COMPILE)
+		{
+			strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz);
+			return 1;
+		}
+	}
+	return 0;
+}
+int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz)
+{
+	int i=0;
+	for(i=0; i< MAX_TABLE_NUM; i++)
+	{
+		if(table_mgr->p_table_info[i] && table_mgr->p_table_info[i]->table_type==TABLE_TYPE_GROUP)
+		{
+			strncpy(buff, table_mgr->p_table_info[i]->table_name[0], sz);
+			return 1;
+		}
+	}
+	return 0;
+}
+const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id)
+{
+	if(table_id>MAX_TABLE_NUM)
+	{
+		return NULL;
+	}
+	if(table_mgr->p_table_info[table_id])
+	{
+		return table_mgr->p_table_info[table_id]->table_name[0];
+	}
+	return NULL;
+}
+enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id)
+{
+	if(table_id>MAX_TABLE_NUM)
+	{
+		return TABLE_TYPE_INVALID;
+	}
+	if(table_mgr->p_table_info[table_id])
+	{
+		return table_mgr->p_table_info[table_id]->table_type;
+	}
+	return TABLE_TYPE_INVALID;
+
+}
+struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id)
+{
+	if(table_id>MAX_TABLE_NUM)
+	{
+		return NULL;
+	}
+
+	return table_mgr->p_table_info[table_id];
+}
+
+struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id)
+{
+
+	struct Maat_table_desc **p_table_info=table_mgr->p_table_info;
+	size_t n_table=MAX_TABLE_NUM;
+
+	struct Maat_table_desc *p_table=NULL, *p_real_table=NULL;
+	if((unsigned int) table_id>n_table)
+	{
+		return NULL;
+	}
+	if(p_table_info[table_id]==NULL)
+	{
+		return NULL;
+	}
+	p_table=p_table_info[table_id];
+	if(p_table==NULL)
+	{
+		return NULL;
+	}
+	if(p_table->table_type==TABLE_TYPE_VIRTUAL)
+	{
+		p_real_table=p_table_info[p_table->virtual_table.real_table_id];
+		*virutal_table_id=table_id;
+	}
+	else
+	{
+		p_real_table=p_table;
+		if(virutal_table_id) *virutal_table_id=0;
+	}
+	if(p_real_table->table_type!=expect_type)
+	{
+		if((expect_type==TABLE_TYPE_EXPR && p_real_table->table_type!=TABLE_TYPE_EXPR_PLUS)||
+			(expect_type==TABLE_TYPE_IP && p_real_table->table_type!=TABLE_TYPE_IP_PLUS))
+		{
+			return NULL;
+		}
+	}
+	return p_real_table;
+}
+int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
+{
+	int table_id=-1,ret=0;
+	ret=map_str2int(table_mgr->map_tablename2id, table_name, &table_id);
+	if(ret>0)
+	{
+		return table_id;
+	}
+	else
+	{
+		return -1;
+	}
+}
+int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
+									int table_id,
+									Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
+									Maat_update_callback_t *update,//table line ,u_para
+									Maat_finish_callback_t *finish,//u_para
+									void* u_para)
+{
+	int idx=0;
+	struct Maat_table_desc *p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
+	struct plugin_table_desc *plugin_desc=&(p_table->plugin);
+	if(p_table==NULL)
+	{
+		return -1;
+	}
+
+	idx=plugin_desc->cb_plug_cnt;
+	if(idx==MAX_PLUGIN_PER_TABLE)
+	{
+		return -1;
+	}
+	plugin_desc->cb_plug_cnt++;
+	plugin_desc->cb_plug[idx].start=start;
+	plugin_desc->cb_plug[idx].update=update;
+	plugin_desc->cb_plug[idx].finish=finish;
+	plugin_desc->cb_plug[idx].u_para=u_para;
+	return 1;
+}
+
+struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx)
+{
+	int table_id=-1;
+	struct Maat_table_desc *p_table=NULL;
+
+	table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name);
+	if(table_id<0)
+	{
+		return NULL;
+	}
+	p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL);
+	if(!p_table)
+	{
+		return NULL;
+	}
+	if(idx<p_table->compile.ex_data_num)
+	{
+		return	p_table->compile.ex_desc+idx;
+	}
+	return NULL;
+
+}
+int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
+										 Maat_rule_EX_new_func_t *new_func,
+										 Maat_rule_EX_free_func_t* free_func,
+										 Maat_rule_EX_dup_func_t* dup_func,
+										 long argl, void *argp)
+{
+	int table_id=-1;
+	struct Maat_table_desc *p_table=NULL;
+	table_id=Maat_table_get_id_by_name(table_mgr, compile_table_name);
+	if(table_id<0)
+	{
+		return -1;
+	}
+	p_table=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_COMPILE, NULL);
+	if(!p_table)
+	{
+		return -1;
+	}
+	int idx=-1;
+
+	struct compile_table_desc* compile_desc=&(p_table->compile);
+	if(compile_desc->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
+	{
+		return -1;
+	}
+	idx=compile_desc->ex_data_num;
+	compile_desc->ex_desc[idx].idx=idx;
+	compile_desc->ex_desc[idx].table_id=table_id;
+	compile_desc->ex_desc[idx].argl=argl;
+	compile_desc->ex_desc[idx].argp=argp;
+	compile_desc->ex_desc[idx].new_func=new_func;
+	compile_desc->ex_desc[idx].free_func=free_func;
+	compile_desc->ex_desc[idx].dup_func=dup_func;
+
+	compile_desc->ex_data_num++;
+
+	return idx;
+} 
+int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id,
+										Maat_plugin_EX_new_func_t* new_func,
+										Maat_plugin_EX_free_func_t* free_func,
+										Maat_plugin_EX_dup_func_t* dup_func,
+										Maat_plugin_EX_key2index_func_t* key2index_func,
+										long argl, void *argp)
+
+{
+	struct Maat_table_desc *table_desc=NULL;;
+	table_desc=Maat_table_get_by_id(table_mgr, table_id, TABLE_TYPE_PLUGIN, NULL);
+	struct plugin_table_desc* plugin_desc=&(table_desc->plugin);
+	
+	if(plugin_desc->have_exdata
+		|| plugin_desc->key_column==0 || plugin_desc->valid_flag_column==0)
+	{
+		return -1;
+	}
+	plugin_desc->ex_desc.new_func=new_func;
+	plugin_desc->ex_desc.free_func=free_func;
+	plugin_desc->ex_desc.dup_func=dup_func;
+	plugin_desc->ex_desc.key2index_func=key2index_func;//Set but not used.
+	plugin_desc->ex_desc.argl=argl;
+	plugin_desc->ex_desc.argp=argp;
+	plugin_desc->have_exdata=1;
+	return 0;
+}
+void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type)
+{
+	table_mgr->active_plugin_table_num=0;
+	int i=0, j=0;
+	struct Maat_table_desc* p_table=NULL;
+	struct plugin_table_desc* plugin_desc=NULL;
+
+	for(i=0; i<MAX_TABLE_NUM; i++)
+	{
+		p_table=table_mgr->p_table_info[i];
+		plugin_desc=&(p_table->plugin);
+		if(p_table==NULL||p_table->table_type!=TABLE_TYPE_PLUGIN||plugin_desc->cb_plug_cnt==0)
+		{
+			continue;
+		}
+
+		table_mgr->active_plugin_table_num++;
+		
+		for(j=0;j<plugin_desc->cb_plug_cnt;j++)
+		{
+			if(plugin_desc->cb_plug[j].start!=NULL)
+			{
+				plugin_desc->cb_plug[j].start(update_type, plugin_desc->cb_plug[j].u_para);
+			}
+		}
+	}
+
+}
+void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr)
+{
+	int i=0, j=0;
+	struct Maat_table_desc* p_table=NULL;
+	struct plugin_table_desc* plugin_desc=NULL;
+
+	int call_plugin_table_cnt=0;
+	for(i=0;i<MAX_TABLE_NUM;i++)
+	{
+		p_table=table_mgr->p_table_info[i];
+		if(p_table==NULL)
+		{
+			continue;
+		}
+		switch(p_table->table_type)
+		{
+			case TABLE_TYPE_PLUGIN:
+				plugin_desc=&(p_table->plugin);				
+				call_plugin_table_cnt++;
+				if(call_plugin_table_cnt==table_mgr->active_plugin_table_num)
+				{
+					table_mgr->is_last_plugin_table_updating=1;
+				}
+				for(j=0;j<plugin_desc->cb_plug_cnt;j++)
+				{
+					if(plugin_desc->cb_plug[j].finish!=NULL)
+					{
+						plugin_desc->cb_plug[j].finish(plugin_desc->cb_plug[j].u_para);
+					}
+				}
+				table_mgr->is_last_plugin_table_updating=0;
+				break;
+			default:
+				break;
+		}		
+	}
+	table_mgr->active_plugin_table_num=0;
+	return;
+}
+int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr)
+{
+	return table_mgr->is_last_plugin_table_updating;
+}
+struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name)
+{
+	struct Maat_table_desc * p_table=NULL;
+	int table_id=0;
+	table_id=Maat_table_get_id_by_name(table_mgr, table_name);
+	if(table_id<0)
+	{
+		return NULL;
+	}
+	p_table=table_mgr->p_table_info[table_id];
+	return p_table;
+}
+void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name)
+{
+	int i=0;
+	for(i=0; i<p_table->conj_cnt; i++)
+	{
+		if(0==strcmp(p_table->table_name[i], table_name))
+		{
+			p_table->updating_name=i;
+		}
+	}
+	assert(i<=p_table->conj_cnt);
+}
+
--- a/src/entry/Maat_table_runtime.cpp
+++ b/src/entry/Maat_table_runtime.cpp
@@ -0,0 +1,427 @@
+#include "Maat_table_runtime.h"
+#include "Maat_rule.h"
+#include "Maat_utils.h"
+
+#include <MESA/MESA_handle_logger.h>
+#include <MESA/MESA_list_queue.h>
+#include <assert.h>
+
+struct wrap_plugin_EX_data
+{
+	MAAT_RULE_EX_DATA exdata;
+	const struct Maat_table_desc* ref_plugin_table;
+};
+void wrap_plugin_EX_data_free(void *data)
+{
+	struct wrap_plugin_EX_data* wrap_data=(struct wrap_plugin_EX_data*)data;
+	const struct plugin_table_ex_data_desc* ex_desc= &(wrap_data->ref_plugin_table->plugin.ex_desc);
+	ex_desc->free_func(wrap_data->ref_plugin_table->table_id, &(wrap_data->exdata), ex_desc->argl, ex_desc->argp);
+	wrap_data->ref_plugin_table=NULL;
+	free(wrap_data);
+	return;
+}
+MESA_htable_handle wrap_plugin_EX_hash_new(long long estimate_size, Maat_plugin_EX_key2index_func_t * key2index)
+{
+	MESA_htable_handle key2ex_hash=NULL;
+	unsigned int slot_size=1;
+	while(estimate_size!=0)
+	{
+		estimate_size=estimate_size>>1;
+		slot_size*=2;
+	}
+	if(slot_size==1)
+	{
+		slot_size=4096;
+	}
+
+	MESA_htable_create_args_t hargs;
+	memset(&hargs,0,sizeof(hargs));
+	hargs.thread_safe=8;
+	hargs.hash_slot_size = slot_size;
+	hargs.max_elem_num = 0;
+	hargs.eliminate_type = HASH_ELIMINATE_ALGO_FIFO;
+	hargs.expire_time = 0;
+	hargs.key_comp = NULL;
+	hargs.key2index = NULL;	//Not supported yet.
+	hargs.recursive = 1;
+	hargs.data_free = wrap_plugin_EX_data_free;
+	hargs.data_expire_with_condition = NULL;
+	key2ex_hash=MESA_htable_create(&hargs, sizeof(hargs));
+	MESA_htable_print_crtl(key2ex_hash, 0);
+	return key2ex_hash;
+}
+
+int plugin_EX_data_free(const struct Maat_table_desc* plugin_table, const char* line,
+								MESA_htable_handle key2ex_hash, void *logger)
+{
+	size_t key_offset=0, key_len=0;
+	const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);	
+	int ret=0;
+	ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
+	if(ret<0)
+	{
+		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+								"Plugin EX data del error: cannot find column %d of %s",
+								plugin_desc->key_column, line);
+		return -1;
+	}
+	ret=MESA_htable_del(key2ex_hash, (const unsigned char*)line+key_offset, key_len, NULL);
+	if(ret<0)
+	{
+		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+								"Plugin EX data del error: no such key %.*s of %s",
+								key_len, line+key_offset, line);
+		return -1;
+	}	
+	return 0;
+}
+
+int plugin_EX_data_new(const struct Maat_table_desc* plugin_table, const char* line,
+								MESA_htable_handle key2ex_hash, void *logger)
+{
+	char* key=NULL;
+	size_t key_offset=0, key_len=0;
+	MAAT_RULE_EX_DATA exdata=NULL;
+	struct wrap_plugin_EX_data* wrap_data=NULL;
+	const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
+	int ret=0;
+	ret=get_column_pos(line, plugin_desc->key_column, &key_offset, &key_len);
+	if(ret<0)
+	{
+		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+								"Plugin EX data add error: cannot find column %d of %s",
+								plugin_desc->key_column, line);
+		return -1;
+	}
+	key=ALLOC(char, key_len+1);
+	memcpy(key, line+key_offset, key_len);
+	plugin_desc->ex_desc.new_func(plugin_table->table_id, key, line, &exdata, 
+								plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
+	wrap_data=ALLOC(struct wrap_plugin_EX_data, 1);
+	wrap_data->exdata=exdata;
+	wrap_data->ref_plugin_table=plugin_table;
+	ret=MESA_htable_add(key2ex_hash, (const unsigned char*)line+key_offset, key_len, wrap_data);
+	free(key);
+	if(ret<0)
+	{
+		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+								"Plugin EX data add error: duplicated key %.*s of %s",
+								key_len, line+key_offset, line);
+		wrap_plugin_EX_data_free(wrap_data);
+		return -1;
+	}
+	return 0;
+}
+
+struct Maat_table_runtime_manager
+{
+	struct Maat_table_runtime** table_rt;
+	size_t n_table_rt;
+};
+static GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest,
+						short cfds_lvl, void* tag)
+{
+	GIE_digest_t* rule=(GIE_digest_t*)calloc(sizeof(GIE_digest_t),1);
+	int digest_len=0;
+	rule->id=id;
+	rule->operation=op;
+	if(digest!=NULL)
+	{
+		digest_len=strlen(digest);
+		rule->sfh=(char*)calloc(sizeof(char),digest_len+1);
+		memcpy(rule->sfh,digest,digest_len);
+
+	}
+	rule->sfh_length=digest_len;
+	rule->cfds_lvl=cfds_lvl;
+	rule->tag=tag;
+	return rule;
+}
+static void destroy_digest_rule(GIE_digest_t*rule)
+{
+	if(rule->sfh!=NULL)
+	{
+		free(rule->sfh);
+		rule->sfh=NULL;
+	}
+	free(rule);
+	rule=NULL;
+	return;
+}
+
+static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_desc* table_desc, int max_thread_num)
+{
+
+	struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1);
+	table_rt->table_type=table_desc->table_type;
+	switch(table_desc->table_type)
+	{
+		case TABLE_TYPE_DIGEST:
+		case TABLE_TYPE_SIMILARITY:
+			table_rt->similar.update_q=MESA_lqueue_create(0,0);
+			break;
+		case TABLE_TYPE_PLUGIN:
+			table_rt->plugin.cache_lines=dynamic_array_create(1, 1024);
+			if(table_desc->plugin.have_exdata)
+			{
+				table_rt->plugin.key2ex_hash=wrap_plugin_EX_hash_new(table_desc->plugin.estimate_size, 
+										table_desc->plugin.ex_desc.key2index_func);
+			}
+			break;
+		default:
+			break;
+	}
+
+	table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num);
+	table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
+	table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num);
+	table_rt->stream_num=alignment_int64_array_alloc(max_thread_num);
+	table_rt->hit_cnt=alignment_int64_array_alloc(max_thread_num);
+	return table_rt;
+}
+static void table_runtime_free(struct Maat_table_runtime* p)
+{
+	long q_cnt=0,data_size=0;
+	int i=0;
+	UNUSED int q_ret=0;
+
+	GIE_digest_t* digest_rule=NULL;
+	if(p==NULL)
+	{
+		return;
+	}
+	switch(p->table_type)
+	{
+		case TABLE_TYPE_DIGEST:
+		case TABLE_TYPE_SIMILARITY:
+			if(p->similar.gie_handle!=NULL)
+			{
+				GIE_destory(p->similar.gie_handle);
+			}
+			if(p->similar.update_q!=NULL)
+			{
+				q_cnt=MESA_lqueue_get_count(p->similar.update_q);
+				for(i=0;i<q_cnt;i++)
+				{
+					data_size=sizeof(GIE_digest_t*);
+					q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(p->similar.update_q,&digest_rule,&data_size);
+					assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
+					destroy_digest_rule(digest_rule);
+				}
+				MESA_lqueue_destroy(p->similar.update_q, lqueue_destroy_cb, NULL);				
+			}
+			break;
+		case TABLE_TYPE_PLUGIN:
+			dynamic_array_destroy(p->plugin.cache_lines, free);
+			p->plugin.cache_lines=NULL;
+			if(p->plugin.key2ex_hash!=NULL)
+			{
+				MESA_htable_destroy(p->plugin.key2ex_hash, NULL);
+			}
+		default:
+			break;
+	}
+
+	alignment_int64_array_free(p->scan_cnt);
+	alignment_int64_array_free(p->scan_cpu_time);
+	alignment_int64_array_free(p->input_bytes);
+	alignment_int64_array_free(p->stream_num);
+	alignment_int64_array_free(p->hit_cnt);
+	free(p);
+	return;
+}
+
+struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num)
+{	
+	const struct Maat_table_desc* table_desc=NULL;
+	struct Maat_table_runtime* table_rt=NULL;
+	struct Maat_table_runtime_manager* table_rt_mgr=ALLOC(struct Maat_table_runtime_manager, 1);
+	size_t i=0;
+	table_rt_mgr->n_table_rt=Maat_table_manager_get_size(table_manager);
+	table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt);
+	for(i=0; i<table_rt_mgr->n_table_rt; i++)
+	{
+		table_desc=Maat_table_get_by_id_raw(table_manager, i);
+		if(!table_desc)
+		{
+			continue;
+		}
+		table_rt=table_runtime_new(table_desc, max_thread_num);
+		table_rt_mgr->table_rt[i]=table_rt;
+	}
+	return table_rt_mgr;
+}
+void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr)
+{
+	size_t i=0;
+	for(i=0; i<table_rt_mgr->n_table_rt; i++)
+	{
+		table_runtime_free(table_rt_mgr->table_rt[i]);
+		table_rt_mgr->table_rt[i]=NULL;
+	}
+	free(table_rt_mgr->table_rt);
+	table_rt_mgr->table_rt=NULL;
+	free(table_rt_mgr);
+}
+struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id)
+{
+	assert(table_id<(int)table_rt_mgr->n_table_rt);
+	return table_rt_mgr->table_rt[table_id];
+}
+long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt)
+{
+	struct plugin_runtime* plugin_rt=&(table_rt->plugin);
+	return plugin_rt->cache_line_num;
+}
+const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line)
+{
+	const char* line=NULL;
+	struct plugin_runtime* plugin_rt=&(table_rt->plugin);
+
+	line=(const char*)dynamic_array_read(plugin_rt->cache_lines, Nth_line);
+	return line;
+}
+MESA_htable_handle plugin_EX_htable_new(const struct Maat_table_desc* plugin_table,
+								struct dynamic_array_t* lines, size_t line_cnt, void* logger)
+{
+	MESA_htable_handle key2ex_hash=NULL;
+	size_t i=0;
+	const char* line=NULL;
+	const struct plugin_table_desc* plugin_desc= &(plugin_table->plugin);
+
+	key2ex_hash=wrap_plugin_EX_hash_new(plugin_desc->estimate_size, plugin_desc->ex_desc.key2index_func);
+
+	for(i=0; i< line_cnt; i++)
+	{
+		line=(const char*)dynamic_array_read(lines, i);
+		plugin_EX_data_new(plugin_table, line, key2ex_hash, logger);
+	}
+	return key2ex_hash;
+}
+int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger)
+{
+	assert(table_rt->plugin.key2ex_hash==NULL);
+	if(table_rt->plugin.key2ex_hash)
+	{
+		return -1;
+	}
+	table_rt->plugin.key2ex_hash=plugin_EX_htable_new(table_desc, table_rt->plugin.cache_lines, 
+													table_rt->plugin.cache_line_num, logger);
+	return 0;
+}
+MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key)
+{
+	struct plugin_table_desc* plugin_desc=NULL;
+	struct wrap_plugin_EX_data* wrap_data=NULL;
+	MAAT_RULE_EX_DATA exdata=NULL;
+
+	plugin_desc=&(table_desc->plugin);
+	if(!plugin_desc->have_exdata)
+	{	
+		assert(0);
+		return NULL;
+	}
+	wrap_data=(struct wrap_plugin_EX_data*)MESA_htable_search(table_rt->plugin.key2ex_hash,
+													(const unsigned char*)key, strlen(key));
+	if(wrap_data!=NULL)
+	{
+		plugin_desc->ex_desc.dup_func(table_desc->table_id, &(exdata), &(wrap_data->exdata), 
+			plugin_desc->ex_desc.argl, plugin_desc->ex_desc.argp);
+	}
+	return exdata;
+
+}
+
+void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag)
+{
+	GIE_digest_t* digest_rule=NULL;
+	char *dup_digest=_maat_strdup(digest);
+
+	if(table_rt->table_type==TABLE_TYPE_SIMILARITY)
+	{
+		dup_digest=str_unescape(dup_digest);
+	}
+	digest_rule=create_digest_rule(expr_id, GIE_INSERT_OPT,
+								dup_digest,
+								confidence_degree,
+								tag);
+	MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*));
+	return;
+}
+void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id)
+{
+	GIE_digest_t* digest_rule=NULL;
+	digest_rule=create_digest_rule(expr_id, GIE_DELETE_OPT	//del digest
+							,NULL
+							,0
+							,NULL);
+	MESA_lqueue_join_tail(table_rt->similar.update_q,&digest_rule, sizeof(void*));
+	return;
+}
+int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
+{
+	long i=0,data_size=0;
+	int ret=0;
+	GIE_digest_t* digest_rule=NULL;
+	GIE_digest_t** update_array=NULL;
+	UNUSED	MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK; 
+	
+
+	GIE_create_para_t para;
+	para.gram_value=7;
+	para.position_accuracy=10;
+
+	const long q_cnt=MESA_lqueue_get_count(table_rt->similar.update_q);
+	if(q_cnt==0)
+	{
+		return 0;
+	}
+	if(table_rt->similar.gie_handle==NULL)
+	{
+		if(table_rt->table_type==TABLE_TYPE_SIMILARITY)
+		{
+			para.ED_reexamine=1;
+			para.format=GIE_INPUT_FORMAT_PLAIN;
+		}
+		else
+		{
+			para.ED_reexamine=0;
+			para.format=GIE_INPUT_FORMAT_SFH;
+		}
+		table_rt->similar.gie_handle=GIE_create(&para);
+	}
+
+	update_array=(GIE_digest_t** )calloc(sizeof(GIE_digest_t*),q_cnt);
+	for(i=0;i<q_cnt;i++)
+	{
+		data_size=sizeof(void*);
+		q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(table_rt->similar.update_q, &digest_rule, &data_size);
+		assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
+		update_array[i]=digest_rule;
+		digest_rule=NULL;
+	}
+	ret=GIE_update(table_rt->similar.gie_handle, update_array, (int)q_cnt);
+	for(i=0;i<q_cnt;i++)
+	{
+		if(update_array[i]->operation==GIE_INSERT_OPT)
+		{
+			table_rt->origin_rule_num++;
+		}
+		else
+		{
+			table_rt->origin_rule_num--;
+		}
+		destroy_digest_rule(update_array[i]);
+		update_array[i]=NULL;
+	}
+	free(update_array);
+	update_array=NULL;
+	
+	if(ret!=(int)q_cnt)
+	{
+		return -1;
+	}
+	return q_cnt;
+}
+
+
--- a/src/entry/Maat_utils.cpp
+++ b/src/entry/Maat_utils.cpp
@@ -1,8 +1,10 @@
 #include <string.h>
-#include <stdio.h>
 #include <ctype.h>
 #include <stdlib.h>
+#include <assert.h>
 #include <openssl/md5.h>
+#include <sys/stat.h>
+#include <openssl/evp.h>

 #include "Maat_utils.h"
 pid_t gettid()
@@ -164,6 +166,17 @@ char* str_unescape(char* s)
 	s[j]='\0';
 	return s;
 }
+size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t n_src)
+{
+	if(*n_dest<offset+n_src)
+	{
+		*n_dest=(offset+n_src)*2;
+		*dest=realloc(*dest, sizeof(char)*(*n_dest));
+	}
+	memcpy((char*)*dest+offset, src, n_src);
+	return n_src;
+}
+
 int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len)
 {
 	const char* seps=" \t";
@@ -199,16 +212,21 @@ int system_cmd_mkdir(const char* path)
 int system_cmd_mv(const char* src_file,const char*dst_file)
 {
 	char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
-	snprintf(cmd,sizeof(cmd), "mv %s %s", src_file, dst_file);
+	snprintf(cmd, sizeof(cmd), "mv %s %s", src_file, dst_file);
 	return  system(cmd);
 }
 int system_cmd_cp(const char* src_file,const char*dst_file)
 {
 	char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
-	snprintf(cmd,sizeof(cmd), "cp -f %s %s", src_file, dst_file);
+	snprintf(cmd, sizeof(cmd), "cp -f %s %s", src_file, dst_file);
+	return  system(cmd);
+}
+int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password)
+{
+	char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
+	snprintf(cmd,sizeof(cmd), "openssl enc -e -aes-256-cbc -k %s -p -nosalt -in %s -out %s", password, src_file, dst_file);
 	return  system(cmd);
 }
-
 int system_cmd_rm(const char* src_file)
 {
 	char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
@@ -241,4 +259,135 @@ char* md5_file(const char* filename, char* md5string)
 	free(file_buff);
 	return md5string;
 }
+const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
+							"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8", "windows-1251", ""};
+
+const char** charset_get_all_name(void)
+{
+	return CHARSET_STRING;
+}
+const char* charset_get_name(enum MAAT_CHARSET charset)
+{
+	return CHARSET_STRING[charset];
+}
+int lqueue_destroy_cb(void *data, long data_len, void *arg)
+{
+	assert(0);
+	return 0;
+}
+
+int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz)
+{
+	int ret=0, out_blk_len=0;
+	int out_buff_len=0, out_buff_offset=0;
+	EVP_CIPHER_CTX *ctx;
+
+	unsigned char cipher_key[EVP_MAX_KEY_LENGTH];
+	unsigned char cipher_iv[EVP_MAX_IV_LENGTH];
+	memset(cipher_key,0,sizeof(cipher_key));
+	memset(cipher_iv,0,sizeof(cipher_iv));
+
+	const EVP_CIPHER *cipher;
+	const EVP_MD *dgst=NULL;
+	const unsigned char *salt=NULL;
+
+	OpenSSL_add_all_algorithms();
+	cipher=EVP_get_cipherbyname(algorithm);
+	if(cipher==NULL)
+	{
+		snprintf(err_str, err_str_sz, "Cipher %s is not supported.", algorithm);
+		return 0;
+	}
+	dgst=EVP_get_digestbyname("md5");
+	if(dgst==NULL)
+	{
+		snprintf(err_str, err_str_sz, "Get MD5 object failed.");
+		return 0;
+	}
+	ret=EVP_BytesToKey(cipher, dgst, salt, (unsigned char*)key, strlen((const char*)key), 1, cipher_key,  cipher_iv);
+	if(ret==0)
+	{
+		snprintf(err_str, err_str_sz, "Key and IV generatioin failed.");
+		return 0;
+	}
+     /* Don't set key or IV right away; we want to check lengths */
+	ctx = EVP_CIPHER_CTX_new();
+	EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, do_encrypt);
+	OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16==0);
+	OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16);
+	 
+	/* Now we can set key and IV */
+	//It should be set to 1 for encryption, 0 for decryption and -1 to leave the value unchanged (the actual value of 'enc' being supplied in a previous call).
+	EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, -1);
+	out_buff_len=inlen+EVP_CIPHER_block_size(cipher)-1;
+	*pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char));
+	if (!EVP_CipherUpdate(ctx, *pp_out+out_buff_offset, &out_blk_len, inbuf, inlen))
+	{
+		snprintf(err_str, err_str_sz, "EVP_CipherUpdate failed.");
+		EVP_CIPHER_CTX_free(ctx);
+		goto error_out;
+	}
+	out_buff_offset+=out_blk_len;
+	if (!EVP_CipherFinal_ex(ctx, *pp_out+out_buff_offset, &out_blk_len))
+	{
+		snprintf(err_str, err_str_sz, "EVP_CipherFinal_ex failed. Maybe password is wrong?");
+		EVP_CIPHER_CTX_free(ctx);
+		goto error_out;
+	}
+	out_buff_offset+=out_blk_len;
+	EVP_CIPHER_CTX_free(ctx);
+	*out_sz=out_buff_offset;
+	return 0;	
+	
+error_out:	
+	free(*pp_out);
+	*pp_out=NULL;
+	return -1;
+}
+int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz)
+{
+	int ret=0;
+	FILE* fp=NULL;		
+	struct stat fstat_buf;
+	size_t read_size=0;
+	
+	ret=stat(file_name, &fstat_buf);
+	if(ret!=0)
+	{
+		return -1;
+	}
+	fp=fopen(file_name, "r");
+	if(fp==NULL)
+	{
+		return -1;
+	}
+	*out_sz=fstat_buf.st_size;
+	*pp_out=(unsigned char*)calloc(1, *out_sz+1);
+	read_size=fread(*pp_out, 1, *out_sz, fp);
+	if(read_size!= *out_sz)
+	{
+		free(*pp_out);
+		pp_out=NULL;
+		return -1;
+	}	
+	
+	fclose(fp);
+	fp=NULL;
+	return 0;
+}
+int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz)
+{
+	int ret=0;
+	size_t file_sz=0;
+	unsigned char* file_buff=NULL;
+	ret=load_file_to_memory(file_name, &file_buff, &file_sz);
+	if(ret<0)
+	{
+		return -1;
+	}
+	ret=crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz);
+	free(file_buff);
+	file_buff=NULL;
+	return ret;
+}

--- a/src/entry/config_monitor.cpp
+++ b/src/entry/config_monitor.cpp
@@ -28,101 +28,19 @@ struct cm_table_info_t
 	int cfg_num;
 	char encryp_algorithm[MAX_CONFIG_FN_LEN];
 };
-int decrypt_open(FILE* in,const unsigned char* key, const char* algorithm,unsigned char**pp_out,void *logger)
-{
-
-	unsigned char inbuf[MAX_CONFIG_LINE];
-	int inlen, out_blk_len=0;
-	int out_buff_len=0,buff_offset=0;
-	EVP_CIPHER_CTX *ctx;
-
-	unsigned char cipher_key[EVP_MAX_KEY_LENGTH];
-	unsigned char cipher_iv[EVP_MAX_IV_LENGTH];
-	memset(cipher_key,0,sizeof(cipher_key));
-	memset(cipher_iv,0,sizeof(cipher_iv));
-
-	const EVP_CIPHER *cipher;
-	const EVP_MD *dgst=NULL;
-	const unsigned char *salt=NULL;
-	int ret=0;
-
-	OpenSSL_add_all_algorithms();
-	cipher=EVP_get_cipherbyname(algorithm);
-	if(cipher==NULL)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"Cipher %s is not supported.",algorithm);
-		return 0;
-	}
-	dgst=EVP_get_digestbyname("md5");
-	if(dgst==NULL)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"Get MD5 object failed.");
-		return 0;
-	}
-	ret=EVP_BytesToKey(cipher,dgst,salt,key,strlen((const char*)key),1,cipher_key,cipher_iv);
-	if(ret==0)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"Key and IV generatioin failed.");
-		return 0;
-	}
-     /* Don't set key or IV right away; we want to check lengths */
-	ctx = EVP_CIPHER_CTX_new();
-	EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL,0);
-	OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) == 16);
-	OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16);
-	 
-	/* Now we can set key and IV */
-	EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, 0);
-	out_buff_len=16*1024;
-	*pp_out=(unsigned char*)malloc(out_buff_len*sizeof(unsigned char));
-	for (;;) 
-	{
-		inlen = fread(inbuf, 1, MAX_CONFIG_LINE, in);
-		if (inlen <= 0)
-			break;
-		
-		if(out_buff_len-buff_offset<inlen+EVP_CIPHER_block_size(cipher)-1)
-		{
-			out_buff_len*=2;
-			*pp_out=(unsigned char*)realloc(*pp_out,out_buff_len);
-		}
-		out_blk_len=out_buff_len-buff_offset;
-		if (!EVP_CipherUpdate(ctx, *pp_out+buff_offset, &out_blk_len, inbuf, inlen))
-		{
-			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"EVP_CipherUpdate failed.");
-			EVP_CIPHER_CTX_free(ctx);
-			goto error_out;
-		}
-		buff_offset+=out_blk_len;
-
-	}
-	if (!EVP_CipherFinal_ex(ctx, *pp_out+buff_offset, &out_blk_len))
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"EVP_CipherFinal_ex failed. Maybe password is wrong?");
-		EVP_CIPHER_CTX_free(ctx);
-		goto error_out;
-	}
-	buff_offset+=out_blk_len;
-	EVP_CIPHER_CTX_free(ctx);
-	return buff_offset;
-error_out:
-	free(*pp_out);
-	*pp_out=NULL;
-	return 0;
-}
-char* read_nxt_line_from_buff(const unsigned char* buff, int buff_size, int* offset, char*line ,int line_size)
+char* read_nxt_line_from_buff(const char* buff, size_t buff_size, size_t* offset, char*line, int line_size)
 {
 	int this_offset=0;
-	const unsigned char* p;
+	const char* p;
 	//search for CRLF, aka '\r', '\n' or "\r\n"
-	p=(const unsigned char*)memchr(buff+*offset,'\r',buff_size-*offset);
+	p=(const char*)memchr(buff+*offset,'\r',buff_size-*offset);
 	if(p==NULL)
 	{
-		p=(const unsigned char*)memchr(buff+*offset,'\n',buff_size-*offset);
+		p=(const char*)memchr(buff+*offset,'\n',buff_size-*offset);
 	}
 	else
 	{
-		if(p-buff<buff_size-1&&*(p+1)=='\n')
+		if((size_t)(p-buff)<buff_size-1 && *(p+1)=='\n')
 		{
 			p++;
 		}
@@ -136,8 +54,9 @@ char* read_nxt_line_from_buff(const unsigned char* buff, int buff_size, int* off
 		p=buff+buff_size;
 	}
 	this_offset=p-(buff+*offset);
-	memcpy(line,buff+*offset,MIN(this_offset,line_size));
+	memcpy(line,buff+*offset, MIN(this_offset,line_size-1));
 	*offset+=this_offset;
+	line[MIN(this_offset,line_size-1)]='\0';
 	return line;
 }
 //replacement of glibc scandir, to adapt dictator malloc wrap
@@ -358,60 +277,56 @@ int cm_read_cfg_index_file(const char* path,struct cm_table_info_t* idx,int size
 int cm_read_table_file(struct cm_table_info_t* index,
 							int (*update)(const char*,const char*,void*),
 							void* u_para,
-							const unsigned char* key,
+							const char* key,
 							void* logger)
 {
 	int cfg_num=0,i=0;
 	int ret=0;
+	char error_string[MAX_CONFIG_FN_LEN];
 	char line[MAX_CONFIG_LINE]={0},*ret_str=NULL;
-	unsigned char* decrypt_buff=NULL;
-	int decrypt_len=0,do_decrypt=0,decrypt_offset=0;
-	FILE*fp=fopen(index->cfg_path,"r");
-	if(fp==NULL)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error,open %s failed.",index->cfg_path);
-		return -1;
-	}
+	char* table_file_buff=NULL;
+	size_t file_sz=0, file_offset=0;
+
 	if(strlen(index->encryp_algorithm)>0)
 	{
 		if(key==NULL||strlen((const char*)key)==0)
 		{
 			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, no key to decrypt %s.",index->cfg_path);
-			fclose(fp);
 			return -1;
 		}
-		decrypt_len=decrypt_open(fp, key,index->encryp_algorithm, &decrypt_buff,logger);
-		if(decrypt_len==0)
+		ret=decrypt_open(index->cfg_path, key, index->encryp_algorithm, (unsigned char**)&table_file_buff, &file_sz, error_string, sizeof(error_string));
+		if(ret<0)
 		{
-			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"update error, %s decrypt failed.",index->cfg_path);
-			fclose(fp);
+			MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, decrypt %s failed: %s",
+																				index->cfg_path, error_string);
 			return -1;
 		}
-		read_nxt_line_from_buff(decrypt_buff, decrypt_len, &decrypt_offset, line, sizeof(line));
-		sscanf(line,"%d\n",&cfg_num);
-		do_decrypt=1;
 	}
 	else
 	{
-		fscanf(fp,"%d\n",&cfg_num);
+		ret=load_file_to_memory(index->cfg_path, (unsigned char**)&table_file_buff, &file_sz);
+		if(ret<0)
+		{
+			MESA_handle_runtime_log(logger, RLOG_LV_FATAL, module_config_monitor, "update error, open %s failed: %s",
+																				index->cfg_path, error_string);
+			return -1;
+		}
+
 	}
+	read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
+	sscanf(line, "%d\n", &cfg_num);
+
 	if(cfg_num!=index->cfg_num)
 	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,"file %s config num not matched",index->cfg_path);
-		fclose(fp);
+		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor, "file %s config num not matched", index->cfg_path);
 		return -1;
 	}
 	for(i=0;i<cfg_num;i++)
 	{
 		line[sizeof(line)-1]='\0';
-		if(do_decrypt==1)
-		{
-			ret_str=read_nxt_line_from_buff(decrypt_buff, decrypt_len, &decrypt_offset, line, sizeof(line));
-		}
-		else
-		{
-			ret_str=fgets(line,sizeof(line),fp);
-		}
+
+		ret_str=read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line));
+		
 		if(ret_str==NULL)
 		{
 			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor ,
@@ -432,11 +347,9 @@ int cm_read_table_file(struct cm_table_info_t* index,
 			break;
 		}
 	}
-	fclose(fp);
-	if(decrypt_buff!=NULL)
-	{
-		free(decrypt_buff);
-	}
+
+	free(table_file_buff);
+	
 	return 0;
 }
 const char* path2filename(const char*path)
@@ -456,7 +369,7 @@ void config_monitor_traverse(long long version,const char*idx_dir,
 							int (*update)(const char*, const char*, void*),//table name ,line ,u_para
 							void (*finish)(void*),//u_para
 							void* u_para,
-							const unsigned char* dec_key,
+							const char* dec_key,
 							void* logger)

 {
@@ -492,7 +405,7 @@ void config_monitor_traverse(long long version,const char*idx_dir,
 			}
 			for(j=0;j<table_num;j++)
 			{
-				cm_read_table_file(table_array+j,update,u_para,dec_key,logger);
+				cm_read_table_file(table_array+j, update, u_para, dec_key, logger);
 			}
 			if(finish!=NULL)
 			{
--- a/src/entry/json2iris.cpp
+++ b/src/entry/json2iris.cpp
@@ -12,7 +12,7 @@
 #include "cJSON.h"
 #include "hiredis.h"
 #include "map_str2int.h"
-#include "Maat_table_description.h"
+#include "Maat_table.h"
 #include "Maat_rule_internal.h"
 #include "Maat_utils.h"

@@ -31,6 +31,10 @@ struct iris_table_t
 	char table_name[MAX_PATH_LINE];
 	char table_path[MAX_PATH_LINE];
 	int line_count;
+	enum MAAT_TABLE_TYPE table_type;
+	void* buff;
+	size_t write_pos;
+	size_t buff_sz;
 };
 struct iris_description_t
 {
@@ -47,6 +51,9 @@ struct iris_description_t
 	MESA_htable_handle iris_table_map;
 	MESA_htable_handle str2int_map;
 	redisContext *redis_write_ctx;
+	char* encrypt_key;
+	char* encrypt_algo;
+	FILE* idx_fp;
 };
 struct traslate_command_t
 {
@@ -58,21 +65,28 @@ struct traslate_command_t
 	const char* default_string;
 	int default_int;
 };
-struct iris_table_t* query_table_info(iris_description_t* p_iris,const char* table_name)
+struct iris_table_t* query_table_info(iris_description_t* p_iris, const char* table_name, enum MAAT_TABLE_TYPE table_type)
 {
 	struct iris_table_t* table_info=NULL;
 	table_info=(struct iris_table_t*)MESA_htable_search(p_iris->iris_table_map, (const unsigned char*)table_name,strlen(table_name));
 	if(table_info==NULL)
 	{
-		table_info=(struct iris_table_t*)calloc(sizeof(struct iris_table_t),1);
+		table_info=ALLOC(struct iris_table_t, 1);
 		table_info->line_count=0;
-		memcpy(table_info->table_name,table_name,MIN(sizeof(table_info->table_name)-1, strlen(table_name)));
-		snprintf(table_info->table_path,sizeof(table_info->table_path),"%s/%s.local",p_iris->tmp_iris_dir,table_info->table_name);
-		MESA_htable_add(p_iris->iris_table_map,(const unsigned char*)table_info->table_name,strlen(table_info->table_name),table_info);
+		table_info->table_type=table_type;
+		memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name)-1, strlen(table_name)));
+		snprintf(table_info->table_path,sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name);
+		MESA_htable_add(p_iris->iris_table_map, (const unsigned char*)table_info->table_name, strlen(table_info->table_name), table_info);
 	}
 	return table_info;
 }
-
+void free_iris_table_info(void* p)
+{
+	struct iris_table_t* table=(struct iris_table_t*)p;
+	free(table->buff);
+	table->buff=NULL;
+	free(table);
+}
 static int get_group_seq(struct iris_description_t* iris_cfg)
 {
 	redisReply* data_reply=NULL;
@@ -108,7 +122,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg)
 	return sequence;
 }

-int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger)
+int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_key, const char* encrypt_algo, const char*compile_tn,const char* group_tn, redisContext *redis_write_ctx, struct iris_description_t *iris_cfg, void * logger)
 {
 	memset(iris_cfg,0,sizeof(struct iris_description_t));
 	snprintf(iris_cfg->tmp_iris_dir,sizeof(iris_cfg->tmp_iris_dir),"%s_iris_tmp",json_file);
@@ -131,7 +145,8 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,
 	
 	iris_cfg->group_name_map=MESA_htable_create(&hargs, sizeof(hargs));
 	MESA_htable_print_crtl(iris_cfg->group_name_map, 0);
-	
+
+	hargs.data_free = free_iris_table_info;
 	iris_cfg->iris_table_map=MESA_htable_create(&hargs, sizeof(hargs));
 	MESA_htable_print_crtl(iris_cfg->iris_table_map, 0);

@@ -172,8 +187,14 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,
 	map_register(iris_cfg->str2int_map, "hexbin",1);
 	map_register(iris_cfg->str2int_map, "case plain",2);

-	iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn);
-	iris_cfg->group_table=query_table_info(iris_cfg, group_tn);
+	iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE);
+	iris_cfg->group_table=query_table_info(iris_cfg, group_tn, TABLE_TYPE_GROUP);
+	
+	if(encrypt_key && encrypt_algo)
+	{
+		iris_cfg->encrypt_key=_maat_strdup(encrypt_key);
+		iris_cfg->encrypt_algo=_maat_strdup(encrypt_algo);
+	}

 	return 0;
 }
@@ -188,6 +209,8 @@ void clear_iris_descriptor(struct iris_description_t *iris_cfg)
 		MESA_htable_destroy(iris_cfg->iris_table_map, NULL);
 	}
 	map_destroy(iris_cfg->str2int_map);
+	free(iris_cfg->encrypt_algo);
+	free(iris_cfg->encrypt_key);
 	return;
 }
 int create_tmp_dir(struct iris_description_t *p)
@@ -231,14 +254,13 @@ int set_file_rulenum(const char* path,int rulenum,void* logger)
 	fclose(fp);
 	return 0;
 }
-int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_command_t*cmd,int cmd_cnt,const char* path,void* logger)
+int direct_write_rule(cJSON* json, MESA_htable_handle str2int, struct traslate_command_t*cmd, int cmd_cnt, struct iris_table_t* table, void* logger)
 {
 	int i=0,ret=-1;
 	cJSON* item=NULL;
 	cJSON dummy;
 	char *p=NULL;
 	int int_value=0;
-	FILE* fp=NULL;
 	for(i=0;i<cmd_cnt;i++)
 	{
 		item=cJSON_GetObjectItem(json,cmd[i].json_string);
@@ -255,6 +277,7 @@ int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_com
 		{
 			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
 									"%s not defined or wrong format.",cmd[i].json_string);
+			ret=-1;
 			goto error_out;
 		}
 		if(cmd[i].str2int_flag==1)
@@ -266,10 +289,11 @@ int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_com
 				MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
 										"%s's value %s is not valid format.",cmd[i].json_string,p);
 				free(p);
+				ret=-1;
 				goto error_out;
 			}
 			cmd[i].json_value=(char*)malloc(21);/* 2^64+1 can be represented in 21 chars. */
-			snprintf(cmd[i].json_value,21,"%d",int_value);
+			snprintf(cmd[i].json_value,21, "%d", int_value);

 		}
 		else
@@ -280,8 +304,8 @@ int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_com
 					cmd[i].json_value=cJSON_Print(item);
 					break;
 				case cJSON_String:
-					cmd[i].json_value=(char*)calloc(strlen(item->valuestring)+1,1);
-					memcpy(cmd[i].json_value,item->valuestring,strlen(item->valuestring));
+					cmd[i].json_value=ALLOC(char, strlen(item->valuestring)+1);
+					memcpy(cmd[i].json_value, item->valuestring, strlen(item->valuestring));
 					break;
 				default://impossible ,already checked
 					assert(0);
@@ -289,30 +313,16 @@ int direct_write_rule(cJSON* json,MESA_htable_handle str2int,struct traslate_com
 			}
 		}
 	}
-	fp=fopen(path,"a");
-	if(fp==NULL)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-										"fopen %s error %s.",path,strerror(errno));
-		goto error_out;
-	}
 	for(i=0;i<cmd_cnt;i++)
 	{
-		fprintf(fp,"%s\t",cmd[i].json_value);
+		table->write_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, cmd[i].json_value, strlen(cmd[i].json_value));
+		table->write_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1);
 	}
-	fprintf(fp,"\n");
-	fclose(fp);
-
-	for(i=0;i<cmd_cnt;i++)
-	{
-		if(cmd[i].json_value!=NULL)
-		{
-			free(cmd[i].json_value);
-		}
-	}
-	return 0;
+	table->write_pos+=memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1);
+	table->line_count++;
+	ret=0;
 	
-error_out:
+error_out:	
 	for(i=0;i<cmd_cnt;i++)
 	{
 		if(cmd[i].json_value!=NULL)
@@ -320,9 +330,10 @@ error_out:
 			free(cmd[i].json_value);
 		}
 	}
-	return -1;
+	return ret;
+
 }
-int write_ip_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger)
+int write_ip_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger)
 {
 	struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
 	int cmd_cnt=0;
@@ -406,9 +417,9 @@ int write_ip_line(cJSON *region_json, struct iris_description_t *p_iris, const c
 	json_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;
 	
-	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
+	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger);
 }
-int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger)
+int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger)
 {
 	struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
 	int cmd_cnt=0;
@@ -516,11 +527,11 @@ int write_ip_plus_line(cJSON *region_json, struct iris_description_t *p_iris, co
 	json_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;

-	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
+	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger);

 }

-int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger)
+int write_expr_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger)
 {
 	struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
 	int cmd_cnt=0;
@@ -534,7 +545,7 @@ int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const c
 	json_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;
 	
-	if(table_type==TABLE_TYPE_EXPR_PLUS)
+	if(table->table_type==TABLE_TYPE_EXPR_PLUS)
 	{
 		json_cmd[cmd_cnt].json_string="district";
 		json_cmd[cmd_cnt].json_type=cJSON_String;
@@ -564,10 +575,10 @@ int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const c
 	json_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;
 	
-	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
+	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger);

 }
-int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger)
+int write_intval_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger)
 {
 	struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
 	int cmd_cnt=0;
@@ -593,10 +604,10 @@ int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const
 	json_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;
 	
-	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
+	return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table, logger);

 }
-int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger)
+int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger)
 {
 	struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
 	int cmd_cnt=0;
@@ -626,10 +637,10 @@ int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, con
 	json_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;
 	
-	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
+	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger);

 }
-int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger)
+int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, struct iris_table_t* table, void * logger)
 {
 	struct traslate_command_t json_cmd[MAX_COLUMN_NUM];
 	int cmd_cnt=0;
@@ -655,28 +666,27 @@ int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, co
 	json_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;
 	
-	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
+	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt, table, logger);

 }

-int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger)
+int write_plugin_line(cJSON* plug_table_json, int sequence, iris_description_t* p_iris, void* logger)
 {
-	cJSON* item=NULL,*table_content=NULL,*each_line=NULL;
+	cJSON* item=NULL,*table_content=NULL, *each_line=NULL;
 	struct iris_table_t* table_info=NULL;
-	const char* table_name=NULL,*line_content=NULL;
-	int ret=0,i=0,line_cnt=0;
-	FILE*fp=NULL;
+	const char* table_name=NULL, *line_content=NULL;
+	int i=0, line_cnt=0;
 	
-	item=cJSON_GetObjectItem(plug_table_json,"table_name");
+	item=cJSON_GetObjectItem(plug_table_json, "table_name");
 	if(item==NULL||item->type!=cJSON_String)
 	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-								"The %d plugin_table's table_name not defined or format error.",sequence);
+		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json,
+								"The %d plugin_table's table_name not defined or format error.", sequence);
 		return -1;
 	}
-	table_name= item->valuestring;
-	table_info=query_table_info(p_iris, table_name);
-	table_content=cJSON_GetObjectItem(plug_table_json,"table_content");
+	table_name=item->valuestring;
+	table_info=query_table_info(p_iris, table_name, TABLE_TYPE_PLUGIN);
+	table_content=cJSON_GetObjectItem(plug_table_json, "table_content");
 	if(table_content==NULL||table_content->type!=cJSON_Array)
 	{
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
@@ -685,21 +695,6 @@ int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_
 		return -1;
 	}
 	line_cnt=cJSON_GetArraySize(table_content);
-	if(table_info->line_count==0)
-	{
-		ret=set_file_rulenum(table_info->table_path,0,logger);
-		if(ret<0)
-		{
-			return -1;
-		}
-	}
-	fp=fopen(table_info->table_path,"a");
-	if(fp==NULL)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-										"fopen %s error %s.",table_info->table_path,strerror(errno));
-		return -1;
-	}

 	for(i=0;i<line_cnt;i++)
 	{
@@ -711,14 +706,13 @@ int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_
 			continue;
 		}
 		line_content=each_line->valuestring;
-		fprintf(fp,"%s\n",line_content);
+		table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), line_content, strlen(line_content));
+		table_info->write_pos+=memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), "\n", 1);
 		table_info->line_count++;
 	}
-	fclose(fp);
-	set_file_rulenum(table_info->table_path,table_info->line_count,logger);
 	return 0;
 }
-int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_description_t* p_iris,void* logger)
+int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_description_t* p_iris, void* logger)
 {
 	cJSON* item=NULL,*table_content=NULL;
 	int ret=0;
@@ -727,15 +721,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
 	enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_EXPR;
 	struct iris_table_t* table_info=NULL;
 	
-	item=cJSON_GetObjectItem(region_json,"table_name");
-	if(item==NULL||item->type!=cJSON_String)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-								"compile rule %d's region table_name not defined or format error.",compile_id);
-		return -1;
-	}
-	table_name=item->valuestring;
-	table_info=query_table_info( p_iris, table_name);
+
 	item=cJSON_GetObjectItem(region_json,"table_type");
 	if(item==NULL||item->type!=cJSON_String)
 	{
@@ -745,7 +731,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
 		return -1;
 	}
 	table_type_str=item->valuestring;
-	ret=map_str2int(p_iris->str2int_map,table_type_str,(int*)&(table_type));
+	ret=map_str2int(p_iris->str2int_map, table_type_str, (int*)&(table_type));
 	if(ret!=1)
 	{
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
@@ -753,6 +739,16 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
 								,compile_id,table_name,table_type_str);
 		return -1;
 	}
+	item=cJSON_GetObjectItem(region_json,"table_name");
+	if(item==NULL||item->type!=cJSON_String)
+	{
+		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
+								"compile rule %d's region table_name not defined or format error.",compile_id);
+		return -1;
+	}
+	table_name=item->valuestring;
+	table_info=query_table_info(p_iris, table_name, table_type);
+
 	table_content=cJSON_GetObjectItem(region_json,"table_content");
 	if(table_content==NULL||table_content->type!=cJSON_Object)
 	{
@@ -761,14 +757,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
 								,compile_id,table_name);
 		return -1;
 	}
-	if(table_info->line_count==0)
-	{
-		ret=set_file_rulenum(table_info->table_path,0,logger);
-		if(ret<0)
-		{
-			return -1;
-		}
-	}
+
 	region_id=get_region_seq(p_iris);
 	cJSON_AddNumberToObject(table_content, "region_id", region_id);
 	cJSON_AddNumberToObject(table_content, "group_id", group_id);
@@ -778,32 +767,27 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri
 	{
 		case TABLE_TYPE_EXPR:
 		case TABLE_TYPE_EXPR_PLUS:
-			ret=write_expr_line(table_content, p_iris, table_info->table_path,table_type, logger);
+			ret=write_expr_line(table_content, p_iris, table_info, logger);
 			break;
 		case TABLE_TYPE_IP:
-			ret=write_ip_line(table_content, p_iris, table_info->table_path, logger);
+			ret=write_ip_line(table_content, p_iris, table_info, logger);
 			break;
 		case TABLE_TYPE_IP_PLUS:
-			write_ip_plus_line(table_content, p_iris, table_info->table_path, logger);
+			write_ip_plus_line(table_content, p_iris, table_info, logger);
 			break;
 		case TABLE_TYPE_INTERVAL:
-			ret=write_intval_line(table_content, p_iris, table_info->table_path, logger);
+			ret=write_intval_line(table_content, p_iris, table_info, logger);
 			break;
 		case TABLE_TYPE_DIGEST:
-			ret=write_digest_line(table_content, p_iris, table_info->table_path, logger);
+			ret=write_digest_line(table_content, p_iris, table_info, logger);
 			break;
 		case TABLE_TYPE_SIMILARITY:
-			write_similar_line(table_content, p_iris,table_info->table_path, logger);
+			ret=write_similar_line(table_content, p_iris, table_info, logger);
 			break;
 		default:
 			assert(0);
 			break;
 	}
-	if(ret>=0)
-	{
-		table_info->line_count++;
-		set_file_rulenum(table_info->table_path,table_info->line_count,logger);
-	}
 	return ret;
 }

@@ -860,18 +844,10 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void *
 	}
 	else
 	{
-		table_info=query_table_info(p_iris,item->valuestring);
+		table_info=query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE);
 	}
 	
-	if(table_info->line_count==0)
-	{
-		ret=set_file_rulenum(table_info->table_path, 0,logger);
-		if(ret<0)
-		{
-			return -1;
-		}
-	}
-	ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info->table_path,logger);
+	ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info, logger);
 	if(ret<0)
 	{
 		return -1;
@@ -884,66 +860,73 @@ int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void *
 		return -1;
 	}
 	compile_id=item->valueint;
-	table_info->line_count++;
-	set_file_rulenum(table_info->table_path,table_info->line_count,logger);
 	return compile_id;
 }
-int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, struct iris_description_t *p_iris, void * logger)
+int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, const char* virtual_table, struct iris_description_t *p_iris, void * logger)
 {
-	FILE*fp=NULL;
-	int ret=0;
-
-	if(p_iris->group_table->line_count==0)
-	{
-		ret=set_file_rulenum(p_iris->group_table->table_path,0,logger);
-		if(ret<0)
-		{
-			return -1;
-		}
-	}
-	fp=fopen(p_iris->group_table->table_path,"a");
-	if(fp==NULL)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-										"fopen %s error %s.",p_iris->group_table->table_path,strerror(errno));
-		return -1;
-	}
-	fprintf(fp,"%d\t%d\t1\t%d\t%d\n",group_id, parent_id, group_not_flag, parent_type);
-	fclose(fp);
-	p_iris->group_table->line_count++;
-	ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger);
+	char buff[1024*4];
+	struct iris_table_t* table=p_iris->group_table;
+	snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%d\t%s\n", group_id, parent_id, group_not_flag, parent_type, virtual_table);
+	table->write_pos+=memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff));
+	table->line_count++;
 	return 0;
 }
 void table_idx_write_cb(const uchar * key, uint size, void * data, void * user)
 {
-	struct iris_table_t* p_table=(struct iris_table_t*)data;
-	FILE* fp=(FILE*)user;
-	fprintf(fp,"%s\t%d\t%s\n",p_table->table_name,p_table->line_count,p_table->table_path);
+	struct iris_description_t *p_iris=(struct iris_description_t *)user;
+	struct iris_table_t* table=(struct iris_table_t*)data;
+	FILE* table_fp=NULL;
+	char line_cnt_str[32], err_str[256];
+	snprintf(line_cnt_str, sizeof(line_cnt_str), "%010d\n", table->line_count);
+
+	int ret=0;
+	size_t table_file_sz=strlen(line_cnt_str)+table->write_pos;
+	unsigned char* buff=ALLOC(unsigned char, table_file_sz);
+	unsigned char* encrypt_buff=NULL;
+	size_t encrypt_buff_sz=0;
+	memcpy(buff, line_cnt_str, strlen(line_cnt_str));
+	memcpy(buff+strlen(line_cnt_str), table->buff, table->write_pos);
+	table_fp=fopen(table->table_path, "w");
+	if(p_iris->encrypt_key)
+	{
+		ret=crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key, p_iris->encrypt_algo, 1, err_str, sizeof(err_str));
+		assert(ret==0);
+		fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp);
+		fprintf(p_iris->idx_fp,"%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo);
+	}
+	else
+	{
+		fwrite(buff, table_file_sz, 1, table_fp);		
+		fprintf(p_iris->idx_fp,"%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path);
+	}
+	fclose(table_fp);
+	free(buff);
+	buff=NULL;
+	
+	
 }
 int write_index_file(struct iris_description_t *p_iris,void* logger)
 {
-	FILE*fp=NULL;
-	fp=fopen(p_iris->index_path,"w");
-	if(fp==NULL)
+	p_iris->idx_fp=fopen(p_iris->index_path,"w");
+	if(p_iris->idx_fp==NULL)
 	{
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-									"index file %s fopen error %s.",p_iris->index_path,strerror(errno));
+									"index file %s fopen error %s.",p_iris->index_path, strerror(errno));
 		return -1;
 	}
-	MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, fp);
-	fclose(fp);
+	MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, p_iris);
+	fclose(p_iris->idx_fp);
+	p_iris->idx_fp=NULL;
 	return 0;
 }
 int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger)
 {
 	const char* _str_parent_type[2]={"compile", "group"};
-	int i=0;
-	int sub_group_cnt=0, region_cnt=0;
 	int ret=0;
 	int group_not_flag=0;
 	cJSON *region_json=NULL, *item=NULL;
 	cJSON *sub_groups=NULL, *region_rule=NULL;
-	const char* group_name=NULL;
+	const char* group_name=NULL, *virtual_table=NULL;
 	struct group_info_t *group_info=NULL;
 	struct group_info_t untitled_group;

@@ -956,6 +939,16 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
 	{
 		group_name=item->valuestring;
 	}
+
+	item=cJSON_GetObjectItem(group_json, "virtual_table");
+	if(item==NULL||item->type!=cJSON_String)
+	{
+		virtual_table="null";
+	}
+	else
+	{
+		virtual_table=item->valuestring;
+	}
 	item=cJSON_GetObjectItem(group_json,"not_flag");
 	if(item==NULL||item->type!=cJSON_Number)
 	{
@@ -982,10 +975,10 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
 		{
 			group_info=ALLOC(struct group_info_t, 1);
 			group_info->group_id=get_group_seq(p_iris);
-			MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info);
+			MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info);
 		}
 	}	
-	ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, p_iris,  logger);
+	ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, virtual_table, p_iris, logger);
 	if(ret<0)
 	{
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
@@ -995,10 +988,8 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
 	region_json=cJSON_GetObjectItem(group_json,"regions");
 	if(region_json!=NULL)
 	{
-		region_cnt=cJSON_GetArraySize(region_json);
-		for(i=0; i<region_cnt; i++)
+		cJSON_ArrayForEach(region_rule, region_json)
 		{
-			region_rule=cJSON_GetArrayItem(region_json,i);
 			ret=write_region_rule(region_rule, tracking_compile_id,	group_info->group_id, p_iris, logger);
 			if(ret<0)
 			{
@@ -1012,10 +1003,9 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
 	if(sub_groups!=NULL)
 	{
 		//recursively
-		sub_group_cnt=cJSON_GetArraySize(sub_groups);
-		for(i=0; i<sub_group_cnt; i++)
+		
+		cJSON_ArrayForEach(item, sub_groups)
 		{
-			item=cJSON_GetArrayItem(sub_groups, i);	
 			ret=write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, p_iris, logger);
 			if(ret<0)
 			{
@@ -1027,18 +1017,16 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
 }
 int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
 {
-	int i=0,j=0;
-	int compile_id=-1, compile_cnt=0, group_cnt=0, plug_table_cnt=0;
+	int i=0;
+	int compile_id=-1, compile_cnt=0, group_cnt=0;
 	int ret=0;
 	cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL;
 	cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL;
 	plug_tables=cJSON_GetObjectItem(json,"plugin_table");
 	if(NULL!=plug_tables)
 	{
-		plug_table_cnt=cJSON_GetArraySize(plug_tables);
-		for(i=0;i<plug_table_cnt;i++)
+		cJSON_ArrayForEach(each_plug_table, plug_tables)
 		{
-			each_plug_table=cJSON_GetArrayItem(plug_tables,i);
 			write_plugin_line(each_plug_table, i, p_iris, logger);
 		}
 	}
@@ -1057,9 +1045,8 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
 									"have no rules.");
 		return -1;
 	}
-	for(i=0;i<compile_cnt;i++)
+	cJSON_ArrayForEach(compile_rule, c_rules)
 	{
-		compile_rule=cJSON_GetArrayItem(c_rules,i);
 		compile_id=write_compile_line(compile_rule,p_iris, logger);
 		if(compile_id<0)
 		{
@@ -1081,9 +1068,8 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
 										"compile rule %d have no groups.",compile_id);
 			return -1;
 		}
-		for(j=0;j<group_cnt;j++)
+		cJSON_ArrayForEach(group_rule, g_rules)
 		{
-			group_rule=cJSON_GetArrayItem(g_rules,j);
 			ret=write_group_rule(group_rule, compile_id, PARENT_TYPE_COMPILE, compile_id, p_iris, logger);
 			if(ret<0)
 			{
@@ -1091,7 +1077,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
 			}
 		}
 	}
-	ret=write_index_file(p_iris,logger);
+	ret=write_index_file(p_iris, logger);
 	if(ret<0)
 	{
 		return -1;
@@ -1099,62 +1085,30 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
 	return 0;
 }
 // redis_write_ctx is used by maat_redis_tool to write json to redis.
-int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger)
+int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger)
 {
-	FILE* json_fp=NULL;
 	cJSON *json=NULL, *tmp_obj=NULL;
-	struct stat fstat_buf;
 	int ret=-1;
-	char* json_buff=NULL;
-	unsigned long json_file_size=0,read_size=0;
 	struct iris_description_t iris_cfg;
-	memset(&iris_cfg,0,sizeof(iris_cfg));
-	ret=stat(json_file, &fstat_buf);
-	if(ret!=0)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-								"fstat file %s error.",json_file);
-		goto error_out;
-	}
-
-	json_fp=fopen(json_file,"r");
-	if(json_fp==NULL)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-								"fopen file %s error %s.",json_file,strerror(errno));
-		goto error_out;
-	}
-	
-
-	json_file_size=fstat_buf.st_size;
-	json_buff=(char*)calloc(1,json_file_size+1);
-	read_size=fread(json_buff,1,json_file_size,json_fp);
-	if(read_size!=json_file_size)
-	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-								"fread file %s error.",json_file);
-		goto error_out;
-	}
-
-	
+	memset(&iris_cfg, 0, sizeof(iris_cfg));
 	json=cJSON_Parse(json_buff);
 	if (!json)
 	{
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr());
 		goto error_out;
 	}
-	tmp_obj=cJSON_GetObjectItem(json,"compile_table");
+	tmp_obj=cJSON_GetObjectItem(json, "compile_table");
 	if(tmp_obj)
 	{
 		compile_tn=tmp_obj->valuestring;
 	}

-	tmp_obj=cJSON_GetObjectItem(json,"group_table");
+	tmp_obj=cJSON_GetObjectItem(json, "group_table");
 	if(tmp_obj)
 	{
 		group_tn=tmp_obj->valuestring;
 	}
-	ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger);
+	ret=set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tn, group_tn, redis_write_ctx, &iris_cfg, logger);
 	if(ret<0)
 	{
 		goto error_out;
@@ -1162,31 +1116,24 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r
 	ret=create_tmp_dir(&iris_cfg);
 	if(ret<0)
 	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-								"create tmp folder %s error",iris_cfg.tmp_iris_dir);
+		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_json,
+								"create tmp folder %s error", iris_cfg.tmp_iris_dir);
 		goto error_out;
 	}
-	ret=write_iris(json,&iris_cfg,logger);
+	ret=write_iris(json, &iris_cfg, logger);
 	if(ret<0)
 	{
 		goto error_out;
 	}
-	memcpy(iris_dir_buf,iris_cfg.tmp_iris_index_dir,MIN(strlen(iris_cfg.tmp_iris_index_dir)+1,(unsigned int)buf_len));
+	memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir)+1, (unsigned int)buf_len));

 	cJSON_Delete(json);
-	fclose(json_fp);
-	free(json_buff);
 	clear_iris_descriptor(&iris_cfg);
 	return 0;	

 	
 error_out:
 	cJSON_Delete(json);
-	if(json_fp!=NULL)
-	{
-		fclose(json_fp);
-	}
-	free(json_buff);
 	clear_iris_descriptor(&iris_cfg);
 	return -1;
 }
--- a/src/inc_internal/Maat_limits.h
+++ b/src/inc_internal/Maat_limits.h
@@ -0,0 +1,11 @@
+#define	MAX_TABLE_LINE_SIZE		(1024*16)
+#define	MAX_EXPR_KEYLEN			1024
+#define	MAX_DISTRICT_LEN			64
+
+#define	MAX_SCANNER_HIT_NUM			4096
+#define	MAX_GROUP_CACHE		128
+
+#define	MAX_FAILED_NUM		128
+
+#define	MAX_MAAT_STAT_NUM	64
+
--- a/src/inc_internal/Maat_rule_internal.h
+++ b/src/inc_internal/Maat_rule_internal.h
@@ -1,9 +1,10 @@
-#ifndef H_MAAT_RULE_INTERNAL_H_INCLUDE
-#define H_MAAT_RULE_INTERNAL_H_INCLUDE
+#pragma once

 #include "Maat_rule.h"
 #include "Maat_command.h"
-#include "Maat_table_description.h"
+#include "Maat_limits.h"
+#include "Maat_table.h"
+#include "Maat_table_runtime.h"

 #include <MESA/MESA_htable.h>
 #include <MESA/MESA_list_queue.h>
@@ -20,24 +21,13 @@
 #include <iconv.h>
 #include <openssl/md5.h>

-extern const char *maat_module;


 #define mr_region_id_var	"SEQUENCE_REGION"
 #define mr_group_id_var		"SEQUENCE_GROUP"


-#define	MAX_TABLE_NUM			256
-#define	MAX_TABLE_LINE_SIZE		(1024*16)
-#define	MAX_EXPR_KEYLEN			1024
-#define	MAX_DISTRICT_LEN			64

-#define	MAX_SCANNER_HIT_NUM			64
-#define	MAX_GROUP_CACHE		128
-
-#define	MAX_FAILED_NUM		128
-
-#define	MAX_MAAT_STAT_NUM	64


 typedef	void*	rule_scanner_t;
@@ -114,6 +104,7 @@ struct db_group_rule_t
 	int is_valid;
 	int not_flag;
 	int parent_type;	//PARENT_TYPE_**, 0:compile, 1: group.
+	int virtual_table_id;
 };
 struct op_expr_t
 {
@@ -141,6 +132,7 @@ struct Maat_compile_group_relation
 	long long magic_num;
 	struct Maat_compile_rule *compile;
 	dynamic_array_t *groups;	//element is struct Maat_group_inner*
+	int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR];
 	char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
 	int compile_id;//equal to compile->m_rule.config_id
 	int group_boundary;
@@ -159,9 +151,9 @@ struct Maat_group_inner
 	int ref_by_parent_cnt;
 	int ref_by_children_cnt;
 	char* group_name;
-	int has_compile_neighbors;
+	int ref_by_compile_cnt;
 	int vertex_id;
-	int top_group_cnt;
+	size_t top_group_cnt;
 	long long* top_groups;
 	dynamic_array_t *regions;
 	pthread_mutex_t mutex;
@@ -208,11 +200,12 @@ struct iconv_handle_t
 struct _stream_para_t
 {
 	struct _Maat_feather_t* feather;
+	const struct Maat_table_desc* p_real_table;
+	int virtual_table_id;
 	int version;
 	int thread_num;
 	int max_cross_size;
 	int caching_size;
-	unsigned short table_id;
 	char do_merge;
 	char do_expr:4;
 	char do_regex:4;
@@ -225,54 +218,7 @@ struct _stream_para_t
 	pthread_mutex_t fuzzy_mutex;
 	unsigned char query_point[8];
 };
-struct similar_runtime
-{
-	GIE_handle_t*	gie_handle;
-	MESA_lqueue_head update_q;
-};

-struct plugin_runtime
-{
-	dynamic_array_t *cache_lines;
-	long long cache_line_num;
-	long long acc_line_num;
-	long long cache_size;
-	MESA_htable_handle key2ex_hash;
-};
-struct expr_runtime
-{
-	long long expr_rule_cnt;	//expr_type=0,1,3
-	long long regex_rule_cnt;	//expr_type=2
-};
-struct ip_runtime
-{
-	long long ipv4_rule_cnt;
-	long long ipv6_rule_cnt;
-
-};
-struct group_runtime
-{
-	long long not_flag_group;
-};
-struct Maat_table_runtime
-{
-	enum MAAT_TABLE_TYPE table_type;
-	long origin_rule_num;
-	union
-	{	
-		struct similar_runtime similar;	//for digest and similarity
-		struct plugin_runtime plugin;		
-		struct expr_runtime expr;
-		struct ip_runtime ip;
-		struct group_runtime group;		
-		void * other;
-	};
-	mcore_long_t	scan_cnt;
-	mcore_long_t	scan_cpu_time;	//nano
-	mcore_long_t	input_bytes;
-	mcore_long_t	stream_num;
-	mcore_long_t	hit_cnt;
-};
 struct rule_tag
 {
 	char* tag_name;
@@ -287,9 +233,10 @@ struct Maat_scanner
 	size_t gie_update_q_size;
 	size_t to_update_group_cnt;
 	size_t to_update_compile_cnt;
-	
-	struct Maat_table_runtime* table_rt[MAX_TABLE_NUM];

+	struct Maat_table_runtime_manager*  table_rt_mgr;
+	size_t max_table_num;
+	
 	MESA_htable_handle region_hash;	//key: region_id, value: struct region_group_relation*
 	MESA_htable_handle exprid_hash;	//key: expr_id, value: int array_idx of Maat_group_inner->regions;
 	MESA_htable_handle group_hash;	//key: group_id, value: struct Maat_group_inner*
@@ -301,9 +248,12 @@ struct Maat_scanner
 	MESA_htable_handle vertex_id2group;

 	igraph_t group_graph;
+	igraph_integer_t group_graph_vcount;
+	igraph_vector_t dfs_vids;
+
 	int grp_vertex_id_generator;
 	int most_popular_sub_group;
-	long long max_presented_top_group_cnt;
+	unsigned long long max_presented_top_group_cnt;
 	
 	unsigned int district_num;
 	unsigned int cfg_num;
@@ -312,6 +262,7 @@ struct Maat_scanner
 	MESA_lqueue_head region_update_q;
 	struct bool_matcher * bool_matcher_expr_compiler;
 	scan_result_t *region_rslt_buff;
+	GIE_result_t* gie_rslt_buff;
 	void* logger_ref;
 	MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q

@@ -335,7 +286,7 @@ struct source_json_ctx
 	char json_file[MAX_TABLE_NAME_LEN];
 	char iris_file[MAX_TABLE_NAME_LEN];
 	char effective_json_md5[MD5_DIGEST_LENGTH*2+1];
-	time_t last_md5_time;
+	struct timespec last_md5_time;
 };
 struct source_redis_ctx
 {
@@ -351,9 +302,9 @@ struct _Maat_feather_t
 	struct Maat_scanner *scanner;
 	struct Maat_scanner *update_tmp_scanner;
 	MESA_lqueue_head garbage_q;
-	int table_cnt;
+	struct Maat_table_manager* table_mgr;
+	
 	int DEFERRED_LOAD_ON;
-	int GROUP_MODE_ON;
 	int REDIS_MODE_ON;
 	enum data_source input_mode;
 	union
@@ -368,8 +319,7 @@ struct _Maat_feather_t
 	int cumulative_update_off;
 	int stat_on;
 	int perf_on;
-	struct Maat_table_desc *p_table_info[MAX_TABLE_NUM];
-	MESA_htable_handle map_tablename2id;
+	
 	void* logger;
 	long long maat_version;
 	long long last_full_version;
@@ -382,7 +332,8 @@ struct _Maat_feather_t
 	char compile_tn[MAX_TABLE_NAME_LEN];
 	char group_tn[MAX_TABLE_NAME_LEN];
 	pthread_mutex_t background_update_mutex;
-	unsigned char decrypt_key[MAX_TABLE_NAME_LEN];
+	char decrypt_key[MAX_TABLE_NAME_LEN];
+	char decrypt_algo[MAX_TABLE_NAME_LEN];
 	pthread_t cfg_mon_t;

 	int AUTO_NUMBERING_ON;
@@ -399,11 +350,8 @@ struct _Maat_feather_t
 	int n_tags;
 	
 	char foreign_cont_dir[MAX_TABLE_NAME_LEN];
-	int foreign_cont_linger;
 //internal states
 	long long new_version;
-	int active_plugin_table_num;
-	int is_last_plugin_table_updating;

 //for scanner independent stat>>>>
 	int backgroud_update_enabled;
@@ -448,7 +396,6 @@ struct _maat_garbage_t
 struct foreign_key
 {
 	int column;
-	int is_existed;
 	char* key;
 	char* filename;
 };
@@ -469,7 +416,6 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
 void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
 void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
 void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
-int read_table_description(struct Maat_table_desc** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
 void maat_start_cb(long long new_version,int update_type,void*u_para);
 int maat_update_cb(const char* table_name,const char* line,void *u_para);
 void maat_finish_cb(void* u_para);
@@ -504,17 +450,14 @@ void empty_serial_rules(struct serial_rule_t* rule);
 int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger);
 long long redis_server_time(redisContext* ctx);
 redisContext * connect_redis(const char*redis_ip, int redis_port, int redis_db, void* logger);
-char* md5_file(const char* filename, char* md5string);

+int load_maat_json_file(_Maat_feather_t* feather, const char* maat_json_fn, char* err_str, size_t err_str_sz);
 void redis_monitor_traverse(long        long version, struct source_redis_ctx* mr_ctx,
 		void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
 		int (*update)(const char* ,const char*,void* ),//table name ,line ,u_para
 		void (*finish)(void*),//u_para
 		void* u_para,
-		const unsigned char* dec_key,
+		const char* dec_key,
 		_Maat_feather_t* feather);


-
-#endif
-
--- a/src/inc_internal/Maat_table.h
+++ b/src/inc_internal/Maat_table.h
@@ -0,0 +1,158 @@
+#pragma once
+#include <MESA/MESA_htable.h>
+#include "Maat_rule.h"
+#include "Maat_limits.h"
+
+#define	MAX_COMPILE_EX_DATA_NUM		2
+#define	MAX_FOREIGN_CLMN_NUM		8
+#define	MAX_PLUGIN_PER_TABLE		32
+#define	MAX_CHARSET_NUM				__CHARSET_MAX
+#define	MAX_CONJUNCTION_TABLE_NUM	8
+#define	MAX_TABLE_NAME_LEN			256	
+
+
+enum USER_REGION_ENCODE
+{
+	USER_REGION_ENCODE_NONE=0,
+	USER_REGION_ENCODE_ESCAPE,
+	USER_REGION_ENCODE_BASE64
+};
+
+enum MAAT_TABLE_TYPE
+{	
+	TABLE_TYPE_INVALID=-1,
+	TABLE_TYPE_EXPR=0,
+	TABLE_TYPE_IP,
+	TABLE_TYPE_IP_PLUS,
+	TABLE_TYPE_INTERVAL,
+	TABLE_TYPE_DIGEST,
+	TABLE_TYPE_EXPR_PLUS,
+	TABLE_TYPE_SIMILARITY,
+	TABLE_TYPE_VIRTUAL,
+	TABLE_TYPE_GROUP,
+	TABLE_TYPE_COMPILE,
+	TABLE_TYPE_PLUGIN
+};
+
+struct compile_ex_data_idx
+{
+	Maat_rule_EX_new_func_t *new_func;
+	Maat_rule_EX_free_func_t* free_func;
+	Maat_rule_EX_dup_func_t* dup_func;
+	long argl;
+	void *argp;
+	int idx;
+	int table_id;
+};
+struct compile_table_desc
+{
+	enum USER_REGION_ENCODE user_region_encoding;
+	int ex_data_num;
+	struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
+};
+
+struct expr_table_desc
+{
+	enum MAAT_CHARSET src_charset;
+	enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
+	int src_charset_in_dst;
+	int do_charset_merge;
+	int cross_cache_size;
+	int quick_expr_switch;//obsolete since 20190401
+	long long iconv_err_cnt;
+};
+struct virtual_table_desc
+{
+	int real_table_id;
+	char real_table_name[MAX_TABLE_NAME_LEN];
+};
+struct plugin_table_callback_desc
+{
+	Maat_start_callback_t 	*start;
+	Maat_update_callback_t	*update;
+	Maat_finish_callback_t	*finish;
+	void* u_para;
+};
+struct plugin_table_ex_data_desc
+{
+	Maat_plugin_EX_new_func_t* new_func;
+	Maat_plugin_EX_free_func_t* free_func;
+	Maat_plugin_EX_dup_func_t* dup_func;
+	Maat_plugin_EX_key2index_func_t* key2index_func;
+	long argl;
+	void *argp;
+};
+struct plugin_table_desc
+{
+	int key_column;
+	int valid_flag_column;
+	int rule_tag_column;
+	int n_foreign;
+	int foreign_columns[MAX_FOREIGN_CLMN_NUM];
+	int cb_plug_cnt;
+	int have_exdata;
+	long long estimate_size;
+	struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];	
+	struct plugin_table_ex_data_desc ex_desc;
+};
+
+struct Maat_table_desc
+{
+	int table_id;
+	int conj_cnt;
+	int updating_name;
+	char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
+	enum MAAT_TABLE_TYPE table_type;
+	union
+	{
+		struct compile_table_desc compile;		
+		struct expr_table_desc expr;
+		struct plugin_table_desc plugin;		
+		struct virtual_table_desc virtual_table;;
+		void* others;//group, ip, interval and digest don't have sperate description info.
+	};	
+//for stat>>>>>>>>
+	unsigned long long udpate_err_cnt;
+	unsigned long long unmatch_tag_cnt;
+	int stat_line_id;
+};
+struct Maat_table_manager;
+struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path, void* logger);
+void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr);
+size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr);
+size_t Maat_table_manager_get_count(struct Maat_table_manager* table_mgr);
+
+struct Maat_table_desc * Maat_table_get_by_id(struct Maat_table_manager* table_mgr, int table_id, enum MAAT_TABLE_TYPE expect_type, int* virutal_table_id);
+struct Maat_table_desc * Maat_table_get_by_id_raw(struct Maat_table_manager* table_mgr, int table_id);
+
+int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
+int Maat_table_add_callback_func(struct Maat_table_manager* table_mgr,
+									int table_id,
+									Maat_start_callback_t *start,//MAAT_RULE_UPDATE_TYPE_*,u_para
+									Maat_update_callback_t *update,//table line ,u_para
+									Maat_finish_callback_t *finish,//u_para
+									void* u_para);
+int Maat_table_get_compile_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
+int Maat_table_get_group_table_name(struct Maat_table_manager* table_mgr, char* buff, size_t sz);
+const char* Maat_table_get_name_by_id(struct Maat_table_manager* table_mgr, int table_id);
+enum MAAT_TABLE_TYPE Maat_table_get_type_by_id(struct Maat_table_manager* table_mgr, int table_id);
+
+int Maat_table_new_compile_rule_ex_index(struct Maat_table_manager* table_mgr, const char* compile_table_name,
+										 Maat_rule_EX_new_func_t *new_func,
+										 Maat_rule_EX_free_func_t* free_func,
+										 Maat_rule_EX_dup_func_t* dup_func,
+										 long argl, void *argp);
+struct compile_ex_data_idx* Maat_table_get_compile_rule_ex_desc(struct Maat_table_manager* table_mgr, const char* compile_table_name, int idx);
+int Maat_table_plugin_new_ex_index(struct Maat_table_manager* table_mgr, int table_id,
+										Maat_plugin_EX_new_func_t* new_func,
+										Maat_plugin_EX_free_func_t* free_func,
+										Maat_plugin_EX_dup_func_t* dup_func,
+										Maat_plugin_EX_key2index_func_t* key2index_func,
+										long argl, void *argp);
+void Maat_table_manager_all_plugin_cb_start(struct Maat_table_manager* table_mgr, int update_type);
+void Maat_table_manager_all_plugin_cb_finish(struct Maat_table_manager* table_mgr);
+
+int Maat_table_manager_is_last_plugin_table_updating(struct Maat_table_manager* table_mgr);
+struct Maat_table_desc* Maat_table_get_desc_by_name(struct Maat_table_manager* table_mgr, const char* table_name);
+void Maat_table_set_updating_name(struct Maat_table_desc* p_table, const char* table_name);
+
--- a/src/inc_internal/Maat_table_description.h
+++ b/src/inc_internal/Maat_table_description.h
@@ -1,114 +0,0 @@
-#pragma once
-#include "Maat_rule.h"
-#define	MAX_COMPILE_EX_DATA_NUM		2
-#define	MAX_FOREIGN_CLMN_NUM		8
-#define	MAX_PLUGIN_PER_TABLE		32
-#define	MAX_CHARSET_NUM				16
-#define	MAX_CONJUNCTION_TABLE_NUM	8
-#define	MAX_TABLE_NAME_LEN			256	
-
-enum USER_REGION_ENCODE
-{
-	USER_REGION_ENCODE_NONE=0,
-	USER_REGION_ENCODE_ESCAPE,
-	USER_REGION_ENCODE_BASE64
-};
-
-enum MAAT_TABLE_TYPE
-{	
-	TABLE_TYPE_EXPR=0,
-	TABLE_TYPE_IP,
-	TABLE_TYPE_IP_PLUS,
-	TABLE_TYPE_INTERVAL,
-	TABLE_TYPE_DIGEST,
-	TABLE_TYPE_EXPR_PLUS,
-	TABLE_TYPE_SIMILARITY,
-	TABLE_TYPE_GROUP,
-	TABLE_TYPE_COMPILE,
-	TABLE_TYPE_PLUGIN
-
-};
-
-struct compile_ex_data_idx
-{
-	Maat_rule_EX_new_func_t *new_func;
-	Maat_rule_EX_free_func_t* free_func;
-	Maat_rule_EX_dup_func_t* dup_func;
-	long argl;
-	void *argp;
-	int idx;
-	int table_id;
-};
-struct compile_table_desc
-{
-	enum USER_REGION_ENCODE user_region_encoding;
-	int ex_data_num;
-	struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
-};
-
-struct plugin_table_callback_desc
-{
-	Maat_start_callback_t 	*start;
-	Maat_update_callback_t	*update;
-	Maat_finish_callback_t	*finish;
-	void* u_para;
-};
-struct plugin_table_ex_data_desc
-{
-	Maat_plugin_EX_new_func_t* new_func;
-	Maat_plugin_EX_free_func_t* free_func;
-	Maat_plugin_EX_dup_func_t* dup_func;
-	Maat_plugin_EX_key2index_func_t* key2index_func;
-	long argl;
-	void *argp;
-};
-struct plugin_table_desc
-{
-	int key_column;
-	int valid_flag_column;
-	int rule_tag_column;
-	int n_foreign;
-	int foreign_columns[MAX_FOREIGN_CLMN_NUM];
-	int cb_plug_cnt;
-	int have_exdata;
-	long long estimate_size;
-	struct plugin_table_callback_desc cb_plug[MAX_PLUGIN_PER_TABLE];	
-	struct plugin_table_ex_data_desc ex_desc;
-};
-struct expr_table_desc
-{
-	enum MAAT_CHARSET src_charset;
-	enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
-	int src_charset_in_dst;
-	int do_charset_merge;
-	int cross_cache_size;
-	int quick_expr_switch;//obsolete since 20190401
-	long long iconv_err_cnt;
-};
-struct ip_table_desc
-{
-	int ipv4_rule_cnt;
-	int ipv6_rule_cnt;
-};
-
-struct Maat_table_desc
-{
-	int table_id;
-	int conj_cnt;
-	int updating_name;
-	char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
-	enum MAAT_TABLE_TYPE table_type;
-	union
-	{
-		struct compile_table_desc compile;		
-		struct expr_table_desc expr;
-		struct ip_table_desc ip;		
-		struct plugin_table_desc plugin;
-		void* others;//group, interval and digest don't have sperate description info.
-	};	
-//for stat>>>>>>>>
-	unsigned long long udpate_err_cnt;
-	unsigned long long unmatch_tag_cnt;
-	int stat_line_id;
-};
-
--- a/src/inc_internal/Maat_table_runtime.h
+++ b/src/inc_internal/Maat_table_runtime.h
@@ -0,0 +1,69 @@
+#include "Maat_table.h"
+#include "gram_index_engine.h"
+#include "alignment_int64.h"
+#include "dynamic_array.h"
+#include <MESA/MESA_htable.h>
+#include <MESA/MESA_list_queue.h>
+
+struct similar_runtime
+{
+	GIE_handle_t*	gie_handle;
+	MESA_lqueue_head update_q;
+};
+
+struct plugin_runtime
+{
+	dynamic_array_t *cache_lines;
+	long long cache_line_num;
+	long long acc_line_num;
+	long long cache_size;
+	MESA_htable_handle key2ex_hash;
+};
+struct expr_runtime
+{
+	long long expr_rule_cnt;	//expr_type=0,1,3
+	long long regex_rule_cnt;	//expr_type=2
+};
+struct ip_runtime
+{
+	long long ipv4_rule_cnt;
+	long long ipv6_rule_cnt;
+
+};
+struct group_runtime
+{
+	long long not_flag_group;
+};
+struct Maat_table_runtime
+{
+	enum MAAT_TABLE_TYPE table_type;
+	long origin_rule_num;
+	union
+	{	
+		struct similar_runtime similar;	//for digest and similarity
+		struct plugin_runtime plugin;		
+		struct expr_runtime expr;
+		struct ip_runtime ip;
+		struct group_runtime group;		
+		void * other;
+	};
+	mcore_long_t	scan_cnt;
+	mcore_long_t	scan_cpu_time;	//nano
+	mcore_long_t	input_bytes;
+	mcore_long_t	stream_num;
+	mcore_long_t	hit_cnt;
+};
+struct Maat_table_runtime_manager;
+struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num);
+void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr);
+struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id);
+long long Maat_table_runtime_plugin_cached_line_count(struct Maat_table_runtime* table_rt);
+const char* Maat_table_runtime_plugin_get_cached_line(struct Maat_table_runtime* table_rt, long long Nth_line);
+
+int Maat_table_runtime_plugin_new_ex_idx(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, void* logger);
+MAAT_PLUGIN_EX_DATA Maat_table_runtime_plugin_get_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_desc* table_desc, const char* key);
+void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag);
+void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id);
+int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt);
+
+
--- a/src/inc_internal/Maat_utils.h
+++ b/src/inc_internal/Maat_utils.h
@@ -1,6 +1,8 @@
 #pragma once
+#include "Maat_rule.h"
 #include <arpa/inet.h>
 #include <unistd.h>
+#include <stdio.h>
 #include <sys/syscall.h>
 #include <sys/types.h>//fstat
 #include <sys/types.h>//fstat
@@ -45,7 +47,7 @@
 #define UNUSED	__attribute__((unused))
 const char*	module_name_str(const char*name);
 #define maat_module	(module_name_str("MAAT_Frame"))
-
+#define	TO_RELATION_ID(vid, gid)	((unsigned long long)vid<<32|gid)
 char* _maat_strdup(const char* s);
 char* str_unescape(char* s);
 inline void ipv6_ntoh(unsigned int *v6_addr)
@@ -62,12 +64,23 @@ char* str_tolower(char* string);
 char *strtok_r_esc(char *s, const char delim, char **save_ptr);
 char *str_unescape_and(char*s);
 char* str_unescape(char* s);
+size_t memcat(void**dest, size_t offset, size_t *n_dest, const void* src, size_t n_src);
+
 pid_t gettid(void);
 int system_cmd_mkdir(const char* path);
 int system_cmd_rm(const char* src_file);
-int system_cmd_mv(const char* src_file,const char*dst_file);
-int system_cmd_cp(const char* src_file,const char*dst_file);
+int system_cmd_mv(const char* src_file, const char*dst_file);
+int system_cmd_cp(const char* src_file, const char*dst_file);
+int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password);
+
 char* md5_file(const char* filename, char* md5string);
 int get_column_pos(const char* line, int column_seq, size_t *offset, size_t *len);
+const char** charset_get_all_name(void);
+const char* charset_get_name(enum MAAT_CHARSET charset);
+int lqueue_destroy_cb(void *data, long data_len, void *arg);
+int decrypt_open(const char* file_name, const char* key, const char* algorithm, unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz);
+int load_file_to_memory(const char* file_name, unsigned char**pp_out, size_t *out_sz);
+//do_encrypt: 1 for encryption, 0 for decryption.
+int crypt_memory(const unsigned char* inbuf, size_t inlen, unsigned char** pp_out, size_t *out_sz, const char* key, const char* algorithm, int do_encrypt, char* err_str, size_t err_str_sz);


--- a/src/inc_internal/config_monitor.h
+++ b/src/inc_internal/config_monitor.h
@@ -9,7 +9,7 @@ void config_monitor_traverse(long long version,const char*idx_dir,
 							int (*update)(const char*, const char*, void*),//table name ,line ,u_para
 							void (*finish)(void*),//u_para
 							void* u_para,
-							const unsigned char* dec_key,
+							const char* dec_key,
 							void* logger);
 #endif

--- a/src/inc_internal/json2iris.h
+++ b/src/inc_internal/json2iris.h
@@ -1,6 +1,6 @@
 #ifndef H_MAAT_JSON2IRIS_H_INCLUDE
 #define H_MAAT_JSON2IRIS_H_INCLUDE
-int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger);
-int set_file_rulenum(const char* path,int rulenum,void* logger);
+int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, const char* group_tn, redisContext *redis_write_ctx, char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo, void* logger);
+int set_file_rulenum(const char* path, int rulenum, void* logger);
 #endif

--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -7,6 +7,9 @@ target_link_libraries(test_igraph igraph-static)
 add_executable(test_maatframe test_maatframe.cpp)
 target_link_libraries(test_maatframe maat_frame_shared gtest)

+add_executable(perf_test_maatframe perf_test_maatframe.cpp)
+target_link_libraries(perf_test_maatframe maat_frame_shared gtest)
+
 configure_file(table_info.conf table_info.conf COPYONLY)
 configure_file(t2_tableinfo.conf t2_tableinfo.conf COPYONLY)
 configure_file(maat_json.json maat_json.json COPYONLY)
--- a/test/maat_json.json
+++ b/test/maat_json.json
@@ -196,7 +196,7 @@
                    "group_name": "Untitled",
                    "regions": [
                        {
-                            "table_name": "HTTP_REGION",
+                            "table_name": "HTTP_SIGNATURE",
                            "table_type": "expr_plus",
                            "table_content": {
                            		"district": "HTTP\\bURL",
@@ -298,7 +298,7 @@
            "is_valid": "yes",
            "groups": [
                {
-                    "group_name": "Untitled",
+                    "group_name": "TakeMeHome",
                    "regions": [
                        {
                            "table_name": "KEYWORDS_TABLE",
@@ -1027,7 +1027,7 @@
                    "group_name": "Untitled",
                    "regions": [
                        {
-                            "table_name": "HTTP_REGION",
+                            "table_name": "HTTP_SIGNATURE",
                            "table_type": "expr_plus",
                            "table_content": {
                            		"district": "Content-Type",
@@ -1141,8 +1141,167 @@
                    ],
                    "not_flag" : 0
                }
-            ]
-        }
+        		]
+  		  },
+		    {
+		        "compile_id": 160,
+		        "service": 0,
+		        "action": 0,
+		        "do_blacklist": 0,
+		        "do_log": 0,
+		        "effective_rage": 0,
+		        "user_region": "VirtualWithPhysical",
+		        "is_valid": "yes",
+		        "groups": [
+		            {
+		            		"group_name":"TakeMeHome",
+		            		"virtual_table":"HTTP_RESPONSE_KEYWORDS",
+		            		"not_flag" : 0                    
+		            },
+		            {
+		                "not_flag":0,
+		                "regions": [
+		                    {
+		                        "table_name": "HTTP_URL",
+		                        "table_type": "string",
+		                        "table_content": {
+		                            "keywords": "https://blog.csdn.net/littlefang/article/details/8213058",
+		                            "expr_type": "none",
+		                            "match_method": "sub",
+		                            "format": "uncase plain"
+		                        }
+		                    }
+		                ]
+		            }               
+		        ]
+		    },
+				{
+		        "compile_id": 161,
+		        "service": 0,
+		        "action": 0,
+		        "do_blacklist": 0,
+		        "do_log": 0,
+		        "effective_rage": 0,
+		        "user_region": "virtual_table_test_temp",
+		        "is_valid": "yes",
+		        "groups": [
+		            {
+		            		"group_name":"vt_grp_http_sig1",
+		                "not_flag":0,
+		                "regions": [
+		                    {
+		                        "table_name": "HTTP_SIGNATURE",
+		                        "table_type": "expr_plus",
+		                        "table_content": {
+                            		"district": "User-Agent",
+                                "keywords": "Chrome/78.0.3904.108",
+                                "expr_type": "none",
+                                "match_method": "sub",
+                                "format": "uncase plain"
+                            }
+		                    }
+		                ]
+		            },
+		            {
+		            		"group_name":"vt_grp_http_sig2",
+		                "not_flag":0,
+		                "regions": [
+		                    {
+		                        "table_name": "HTTP_SIGNATURE",
+		                        "table_type": "expr_plus",
+		                        "table_content": {
+                            		"district": "Cookie",
+                                "keywords": "uid=12345678",
+                                "expr_type": "none",
+                                "match_method": "sub",
+                                "format": "uncase plain"
+                            }
+		                    },
+		                    {
+		                        "table_name": "HTTP_SIGNATURE",
+		                        "table_type": "expr_plus",
+		                        "table_content": {
+                            		"district": "Cookie",
+                                "keywords": "sessionid=888888",
+                                "expr_type": "none",
+                                "match_method": "sub",
+                                "format": "uncase plain"
+                            }
+		                    }	                    
+		                ]
+		            }      
+		        ]
+		    },
+		    {
+		        "compile_id": 162,
+		        "service": 0,
+		        "action": 0,
+		        "do_blacklist": 0,
+		        "do_log": 0,
+		        "effective_rage": 0,
+		        "user_region": "VirtualWithVirtual",
+		        "is_valid": "yes",
+		        "groups": [
+		            {
+		            		"group_name":"vt_grp_http_sig1",
+		            		"virtual_table":"HTTP_REQUEST_HEADER",
+		                "not_flag":0                   
+		            },
+		            {
+		            		"group_name":"vt_grp_http_sig2",
+		            		"virtual_table":"HTTP_RESPONSE_HEADER",
+		                "not_flag":0                  
+		            }             
+		        ]
+		    },
+		    {
+		        "compile_id": 163,
+		        "service": 0,
+		        "action": 0,
+		        "do_blacklist": 0,
+		        "do_log": 0,
+		        "effective_rage": 0,
+		        "user_region": "OneGroupInTwoVirtual",
+		        "is_valid": "yes",
+		        "groups": [
+		            {
+		            		"group_name":"vt_grp_http_sig2",
+		            		"virtual_table":"HTTP_REQUEST_HEADER",
+		                "not_flag":0                   
+		            },
+		            {
+		            		"group_name":"vt_grp_http_sig2",
+		            		"virtual_table":"HTTP_RESPONSE_HEADER",
+		                "not_flag":0                  
+		            }             
+		        ]
+            },
+            {
+                "compile_id": 164,
+                "service": 1,
+                "action": 1,
+                "do_blacklist": 1,
+                "do_log": 1,
+                "user_region": "CharsetWindows1251",
+                "is_valid": "yes",
+                "groups": [
+                    {
+                        "group_name": "Untitled",
+                        "regions": [
+                            {
+                                "table_name": "KEYWORDS_TABLE",
+                                "table_type": "string",
+                                "table_content": {
+                                    "keywords": ">ЗАО\\b«Севергазвтоматика\\bАйС»<",
+                                    "expr_type": "none",
+                                    "match_method": "sub",
+                                    "format": "uncase plain"
+                                }
+                            }
+                        ]
+                    }
+                ]
+            }
    ],
    "plugin_table": [
        {
--- a/test/perf_test_maatframe.cpp
+++ b/test/perf_test_maatframe.cpp
@@ -0,0 +1,203 @@
+#include "Maat_rule.h"
+#include "stream_fuzzy_hash.h"
+#include "Maat_command.h"
+#include <MESA/MESA_handle_logger.h>
+#include <gtest/gtest.h>
+int test_add_expr_command_copy(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords)
+{
+	struct Maat_cmd_t* cmd=NULL;
+	struct Maat_rule_t rule;
+	char huge_serv_def[1024*2];
+	memset(huge_serv_def,'s',sizeof(huge_serv_def));
+	struct Maat_region_t region;
+	int group_num=1,ret=0;
+	memset(&rule,0,sizeof(rule));
+	rule.config_id=config_id;
+	strcpy(rule.service_defined,"maat_command");
+	//MUST acqire by function, because Maat_cmd_t has some hidden members.
+	cmd=Maat_create_cmd(&rule, group_num);
+	cmd->expire_after=timeout;
+	cmd->label_id=label_id;
+	memset(&region,0,sizeof(region));
+	region.region_type=REGION_EXPR;
+	region.table_name=region_table;
+	region.expr_rule.district=NULL;
+	region.expr_rule.keywords=keywords;
+	region.expr_rule.expr_type=EXPR_TYPE_AND;
+	region.expr_rule.match_method=MATCH_METHOD_SUB;
+	region.expr_rule.hex_bin=UNCASE_PLAIN;
+	Maat_cmd_set_opt(cmd, MAAT_RULE_SERV_DEFINE, huge_serv_def, sizeof(huge_serv_def));
+	Maat_add_region2cmd(cmd, 0, &region);
+	//use pipeline model.
+	ret=Maat_cmd_append(feather, cmd, MAAT_OP_ADD);
+	if(ret<0)
+	{
+		printf("Add Maat command %d failed.\n",rule.config_id);
+		Maat_free_cmd(cmd);
+		return 0;
+	}
+	//cmd has been saved in feather, so free cmd before commit is allowed.
+	Maat_free_cmd(cmd);
+	return 0;
+
+}
+void wait_for_cmd_effective_copy(Maat_feather_t feather, long long version_before)
+{
+	long long	version_after=version_before;
+	int is_updating=1;
+	long long wating_us=0, sleep_us=1000*100;
+	while(is_updating||version_before==version_after)
+	{
+		Maat_read_state(feather,MAAT_STATE_IN_UPDATING, &is_updating, sizeof(is_updating));
+		Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after));
+
+		usleep(sleep_us);//waiting for commands go into effect
+		wating_us+=sleep_us;
+	}
+//	printf("wait for %lld ms\n", wating_us/1000);
+}
+
+class MaatCMDPerfTest : public testing::Test 
+{
+
+protected:
+
+
+	static void SetUpTestCase()
+	{
+		const char* test_maat_redis_ip="127.0.0.1";
+		unsigned short test_maat_redis_port=6379;
+		int g_iThreadNum=4;
+		const char* table_info_path="./table_info.conf";
+		int scan_interval_ms=500;
+		int effective_interval_ms=0;
+
+	  	logger=MESA_create_runtime_log_handle("test_maat_redis.log",0);
+
+		_shared_feather=Maat_feather(g_iThreadNum, table_info_path, logger);
+		Maat_set_feather_opt(_shared_feather,MAAT_OPT_INSTANCE_NAME,"perf", strlen("perf")+1);
+		Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_IP, test_maat_redis_ip, strlen(test_maat_redis_ip)+1);
+		Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_PORT, &test_maat_redis_port, sizeof(test_maat_redis_port));
+		Maat_set_feather_opt(_shared_feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms));
+		//Set a short intevral for testing.
+		Maat_set_feather_opt(_shared_feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms));
+		
+		const char* foregin_dir="./foreign_files/";	
+		Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR, foregin_dir, strlen(foregin_dir)+1);
+	
+		Maat_cmd_flushDB(_shared_feather);
+		Maat_initiate_feather(_shared_feather);
+	}
+	static void TearDownTestCase()
+	{
+		Maat_burn_feather(_shared_feather);
+		MESA_destroy_runtime_log_handle(logger);
+
+	}	
+ 	// Some expensive resource shared by all tests.
+	static Maat_feather_t _shared_feather;
+	static void *logger;
+};
+Maat_feather_t MaatCMDPerfTest::_shared_feather;
+
+void* MaatCMDPerfTest::logger;
+
+//Following tests must be coded/tested at last, for they stalled the maat update thread and interrupt other tests.
+TEST_F(MaatCMDPerfTest, SetExpr200K)
+{
+	const int CMD_EXPR_NUM=200*1000;
+	const char* table_name="HTTP_URL";
+	
+	const char* keywords1="Hiredis";
+	const char* keywords2="C Client";
+	char escape_buff1[256],escape_buff2[256];
+	char keywords[256];
+
+	int label_id=5210, config_id=0,ret=0, output_id_cnt=0;
+	Maat_feather_t feather=MaatCMDPerfTest::_shared_feather;
+	long long version_before=0;
+	ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
+	
+	Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1);
+	Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2);
+	snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2);
+	
+	config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", CMD_EXPR_NUM);
+	int i=0;
+	for(i=0; i<CMD_EXPR_NUM;i++)
+	{
+		test_add_expr_command_copy(feather,table_name,config_id-i, 0, label_id, keywords);
+	}
+	ret=Maat_cmd_commit(feather);
+	EXPECT_TRUE(ret>=0);
+	wait_for_cmd_effective_copy(feather, version_before);
+	struct Maat_cmd_t* cmd=NULL;
+	struct Maat_rule_t rule;
+	memset(&rule,0,sizeof(rule));
+	int *output_ids=(int*)malloc(sizeof(int)*CMD_EXPR_NUM);
+	output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, CMD_EXPR_NUM);
+	EXPECT_EQ(output_id_cnt, CMD_EXPR_NUM);
+	for(i=0; i<CMD_EXPR_NUM;i++)
+	{
+		memset(&rule,0,sizeof(rule));
+		rule.config_id=output_ids[i];
+		cmd=Maat_create_cmd(&rule, 0);
+		ret=Maat_cmd_append(feather, cmd, MAAT_OP_DEL);
+		EXPECT_EQ(ret, 0);
+		Maat_free_cmd(cmd);
+	}
+	ret=Maat_cmd_commit(feather);
+	
+	EXPECT_EQ(ret, CMD_EXPR_NUM);
+	free(output_ids);
+}
+
+TEST_F(MaatCMDPerfTest, SetLinesOneMillion)
+{
+	const int TEST_CMD_LINE_NUM=1000*1000;
+	Maat_feather_t feather=MaatCMDPerfTest::_shared_feather;
+
+	struct Maat_line_t **p_line=(struct Maat_line_t **)calloc(sizeof(struct Maat_line_t *), TEST_CMD_LINE_NUM);
+	struct Maat_line_t *line_rule=(struct Maat_line_t *)calloc(sizeof(struct Maat_line_t), TEST_CMD_LINE_NUM);
+	int i=0;
+	const char* line="1\t192.168.0.1\t4444444444\t1";
+	int seq=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", TEST_CMD_LINE_NUM);
+	for(i=0;i<TEST_CMD_LINE_NUM;i++)
+	{
+		line_rule[i].label_id=0;
+		line_rule[i].rule_id=seq-i;
+		line_rule[i].table_name="QD_ENTRY_INFO";
+		line_rule[i].table_line=line;
+//		asprintf(&(line_rule[i].table_line),"1\t192.168.0.1\t%d\t1",100+i);
+		line_rule[i].expire_after=0;
+		p_line[i]=line_rule+i;
+	}
+	
+	long long version_before=0;
+	Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
+
+	Maat_cmd_set_lines(feather,(const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
+	
+	wait_for_cmd_effective_copy(feather, version_before);
+	
+	for(i=0;i<TEST_CMD_LINE_NUM;i++)
+	{
+		line_rule[i].table_line=NULL;
+	}
+	Maat_cmd_set_lines(feather, (const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
+	free(p_line);
+	free(line_rule);
+
+	return;
+}
+
+int main(int argc, char ** argv)
+{
+	int ret=0;
+	::testing::InitGoogleTest(&argc, argv);
+
+	ret=RUN_ALL_TESTS();
+	return ret;
+}
+
+
--- a/test/table_info.conf
+++ b/test/table_info.conf
@@ -13,18 +13,21 @@
 #For plugin table. The first column's id is 1. 0 as not speicified.
 #id	name	type	column_define
 #
+#For virtual Table
+#id	name	type real_table_name
+#
 #For expr/expr_plus Table
 #id	name	type	src_charset	dst_charset	do_merge cross_cache quick_mode
 0	COMPILE	compile	escape	--
 1	GROUP	group	--
 2	HTTP_URL	expr	UTF8	GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8	yes	128	quickoff
 2	HTTP_HOST	expr	UTF8	GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8	yes	128	quickoff
-3	KEYWORDS_TABLE	expr	UTF8	GBK/BIG5/UNICODE/UTF8/unicode_ascii_esc/unicode_ascii_aligned/unicode_ncr_dec/unicode_ncr_hex	yes	0
+3	KEYWORDS_TABLE	expr	UTF8	GBK/BIG5/UNICODE/UTF8/unicode_ascii_esc/unicode_ascii_aligned/unicode_ncr_dec/unicode_ncr_hex/windows-1251	yes	0
 4	IP_CONFIG	ip	--
 5	CONTENT_SIZE	intval	--
 6	QD_ENTRY_INFO	plugin	4	--
 7	FILE_DIGEST	digest	--
-8	HTTP_REGION	expr_plus	GBK	GBK	yes	0
+8	HTTP_SIGNATURE	expr_plus	GBK	GBK	yes	0
 9	SIM_URL	similar	--
 10	IMAGE_FP	expr	UTF8	UTF8	yes	128	quickoff
 11	TEST_EFFECTIVE_RANGE_TABLE	plugin	{"valid":4,"tag":5}	--
@@ -35,4 +38,7 @@
 16	APP_PAYLOAD	expr_plus	UTF8	UTF8	yes	0	quickoff
 17	TROJAN_PAYLOAD	expr	UTF8	UTF8	yes	0	quickoff
 18	MAIL_ADDR	expr	UTF8	UTF8	yes	0	quickoff
-19	IP_PLUS_CONFIG	ip_plus	--
+19	IP_PLUS_CONFIG	ip_plus	--
+20	HTTP_RESPONSE_KEYWORDS	virtual	KEYWORDS_TABLE	--
+21	HTTP_REQUEST_HEADER	virtual	HTTP_SIGNATURE	--
+22	HTTP_RESPONSE_HEADER	virtual	HTTP_SIGNATURE	--
--- a/test/test_igraph.cpp
+++ b/test/test_igraph.cpp
@@ -19,7 +19,7 @@ int main() {
 	igraph_integer_t edge_num_before=0, edge_num_after=0;
 	int v[10];
 	int i=0;
-	for(i=0; i<sizeof(v)/sizeof(int); i++)
+	for(i=0; (unsigned int)i<sizeof(v)/sizeof(int); i++)
 	{
 		v[i]=i;
 		igraph_add_vertices(&g, 1, NULL);	//Add 1 vertice.
--- a/test/test_maatframe.cpp
+++ b/test/test_maatframe.cpp
@@ -32,6 +32,7 @@ extern int my_scandir(const char *dir, struct dirent ***namelist,
              int(*compar)(const void *, const void *));
 extern char* md5_file(const char* filename, char* md5string);
 extern int system_cmd_cp(const char* src_file,const char*dst_file);
+extern int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password);
 Maat_feather_t g_feather=NULL;
 void *g_logger=NULL;
 int g_iThreadNum=4;
@@ -54,7 +55,7 @@ void wait_for_cmd_effective(Maat_feather_t feather, long long version_before)
 //	printf("wait for %lld ms\n", wating_us/1000);
 }

-void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old)
+void scan_with_old_or_new_cfg(Maat_feather_t feather, int is_old)
 {
 	const char* hit_old_data="Hello world! I'm eve.";
 	const char* hit_new_data="Maat was borned in MESA.";
@@ -72,7 +73,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old)
 	ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_old_data, strlen(hit_old_data), 
 						&result,NULL, 1,
 						&mid, 0);
-	if(hit_old)
+	if(is_old)
 	{
 		EXPECT_EQ(ret, 1);
 		EXPECT_TRUE(result.config_id==1);
@@ -87,7 +88,7 @@ void scan_with_old_or_new_cfg(Maat_feather_t feather, int hit_old)
 	ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_new_data, strlen(hit_new_data), 
 						&result,NULL, 1,
 						&mid, 0);
-	if(!hit_old)
+	if(!is_old)
 	{
 		EXPECT_EQ(ret, 1);
 		EXPECT_TRUE(result.config_id==2);
@@ -105,6 +106,7 @@ const char* watched_json="./json_update/maat.json";
 const char* old_json="./json_update/old.json";
 const char* new_json="./json_update/new.json";
 const char* corrupted_json="./json_update/corrupted.json";
+const char* json_decrypt_key="himaat!";

 class JSONUpdate : public testing::Test 
 {
@@ -112,10 +114,13 @@ class JSONUpdate : public testing::Test
 protected:
 	static void SetUpTestCase()
 	{	
-		system_cmd_cp(old_json, watched_json);
+		
+		system_cmd_encrypt(old_json, watched_json, json_decrypt_key);
+		
 		_shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger);
+		Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, json_decrypt_key, strlen(json_decrypt_key)+1);
 		Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_FILE_PATH, watched_json, strlen(watched_json)+1);		
-		Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms));
+		Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms));

 		Maat_initiate_feather(_shared_feather_j);

@@ -137,11 +142,12 @@ TEST_F(JSONUpdate, OldCfg)
 }
 TEST_F(JSONUpdate, NewCfg)
 {
-	system_cmd_cp(corrupted_json, watched_json);
+	system_cmd_encrypt(corrupted_json, watched_json, json_decrypt_key);
 	sleep(2);
 	scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1);
-	system_cmd_cp(new_json, watched_json);
-	sleep(2);
+
+	system_cmd_encrypt(new_json, watched_json, json_decrypt_key);
+	sleep(5);
 	scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0);
 }

@@ -357,7 +363,7 @@ TEST(StringScan,   ExprPlus)
 	int found_pos[4];
 	const char* region_name="HTTP URL";
 	const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567";
-	table_id=Maat_table_register(g_feather, "HTTP_REGION");
+	table_id=Maat_table_register(g_feather, "HTTP_SIGNATURE");
 	ASSERT_GT(table_id, 0);
 	scan_status_t mid=NULL;
 	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_data, strlen(scan_data), 
@@ -377,6 +383,7 @@ TEST(StringScan,   ExprPlus)
 	return;
 }

+
 TEST(StringScan, ExprPlusWithOffset)
 {
 	int ret=0, table_id=0;
@@ -437,7 +444,7 @@ TEST(StringScan, ExprPlusWithHex)
 	const char* scan_data="text/html; charset=UTF-8";	
 	const char* region_name="Content-Type";
 	int found_pos[4];
-	table_id=Maat_table_register(g_feather, "HTTP_REGION");
+	table_id=Maat_table_register(g_feather, "HTTP_SIGNATURE");
 	ASSERT_GT(table_id, 0);
 	scan_status_t mid=NULL;
 	ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name, strlen(region_name));
@@ -653,6 +660,54 @@ TEST(NOTLogic, OneRegion)
 	Maat_clean_status(&mid);

 }
+
+TEST(StringScan,   CharsetWindows1251)
+{
+	int table_id=0,ret=0;
+	int read_size=0,pass_flag=0;
+	struct Maat_rule_t result[4];
+	scan_status_t mid=NULL;
+	//const char* fn="./testdata/mesa_logo.jpg";
+	const char* table_name="KEYWORDS_TABLE";
+	const char* fn="./testdata/charsetWindows1251.txt";
+	FILE* fp=fopen(fn,"r");
+	ASSERT_FALSE(fp==NULL);
+
+	char scan_data[4096]={0};
+	table_id=Maat_table_register(g_feather,table_name);
+	ASSERT_GT(table_id, 0);
+	
+
+	struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10);
+	stream_para_t sp=Maat_stream_scan_string_start(g_feather,table_id,0);
+	int detail_ret=0;
+	
+	ASSERT_FALSE(sp==NULL);
+
+	while(0==feof(fp))
+	{
+		read_size=fread(scan_data,1,sizeof(scan_data),fp);
+		ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data,read_size
+							,result,4,hit_detail,10
+							,&detail_ret,&mid);
+		if(ret>0)
+		{
+			pass_flag=1;
+			break;
+		}
+	}
+	EXPECT_EQ(pass_flag, 1);
+	EXPECT_EQ(result[0].config_id, 164);
+	
+	Maat_stream_scan_string_end(&sp);
+	free(hit_detail);
+	fclose(fp);
+	Maat_clean_status(&mid);
+	return;
+
+}
+
+
 TEST(NOTLogic, ScanNotAtLast)
 {
 	const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-144.";
@@ -1462,6 +1517,126 @@ TEST(ScanResult, LongerServiceDefine)
 	free(buff);
 	return;
 }
+TEST(VirtualTable, VirtualWithPhysical)
+{
+#define TestVirtualTable1
+	int ret=0, table_id=0;
+	const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me.";
+	const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058";
+	
+	struct Maat_rule_t result[4];
+	memset(result, 0, sizeof(result));
+	
+	scan_status_t mid=NULL;
+
+	table_id=Maat_table_register(g_feather, "HTTP_URL");	
+	ASSERT_GT(table_id, 0);
+	
+	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_url, strlen(http_url), 
+						result, NULL, 4, &mid, 0);
+
+	EXPECT_EQ(ret, -2);
+	
+
+	table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_KEYWORDS");
+	ASSERT_GT(table_id, 0);
+	
+	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_content, strlen(http_content), 
+						result, NULL, 4, &mid, 0);
+
+	EXPECT_EQ(ret, 1);
+	EXPECT_EQ(result[0].config_id, 160);	
+
+	const char* should_not_hit="2018-10-05 is a keywords of table KEYWORDS_TABLE. Should not hit.";
+
+	mid=NULL;
+	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, should_not_hit, strlen(should_not_hit), 
+						result, NULL, 4, &mid, 0);
+	EXPECT_EQ(ret, -2);
+	Maat_clean_status(&mid);
+
+	return;
+}
+TEST(VirtualTable, VirtualWithVirtual)
+{
+#define TestVirtualTable2
+	int ret=0, table_id=0;
+	const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
+	const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;";
+	
+	struct Maat_rule_t result[4];
+	memset(result, 0, sizeof(result));
+	
+	scan_status_t mid=NULL;
+
+	table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER");	
+	ASSERT_GT(table_id, 0);
+	
+	ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "User-Agent", strlen("User-Agent"));
+	ASSERT_EQ(ret, 0);
+	
+	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_req_hdr_ua, strlen(http_req_hdr_ua), 
+						result, NULL, 4, &mid, 0);
+
+	EXPECT_EQ(ret, -2);
+	
+
+	table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER");
+	ASSERT_GT(table_id, 0);
+
+	ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie"));
+	ASSERT_EQ(ret, 0);
+
+	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), 
+						result, NULL, 4, &mid, 0);
+
+	EXPECT_EQ(ret, 1);
+	EXPECT_EQ(result[0].config_id, 162);	
+
+	Maat_clean_status(&mid);
+
+	return;
+}
+TEST(VirtualTable, OneGroupInTwoVirtual)
+{
+#define TestVirtualTable3
+	int ret=0, table_id=0;
+	const char* http_resp_hdr_cookie="sessionid=888888;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;";
+	
+	struct Maat_rule_t result[4];
+	memset(result, 0, sizeof(result));
+	
+	scan_status_t mid=NULL;
+
+	table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER");	
+	ASSERT_GT(table_id, 0);
+	
+	ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie"));
+	ASSERT_EQ(ret, 0);
+	
+	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), 
+						result, NULL, 4, &mid, 0);
+
+	EXPECT_EQ(ret, -2);
+	
+
+	table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER");
+	ASSERT_GT(table_id, 0);
+
+	ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie"));
+	ASSERT_EQ(ret, 0);
+
+	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), 
+						result, NULL, 4, &mid, 0);
+
+	EXPECT_EQ(ret, 1);
+	EXPECT_EQ(result[0].config_id, 163);	
+
+	Maat_clean_status(&mid);
+
+	return;
+}
+

 class MaatFileTest : public testing::Test 
 {
@@ -1589,8 +1764,6 @@ protected:
 		
 		const char* foregin_dir="./foreign_files/";	
 		Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR,foregin_dir, strlen(foregin_dir)+1);
-		linger_timeout=2;
-		Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_LINGER,&linger_timeout, sizeof(linger_timeout));

 		Maat_cmd_flushDB(_shared_feather);
 		Maat_initiate_feather(_shared_feather);
@@ -2174,6 +2347,123 @@ TEST_F(MaatCmdTest,    RefGroup)
 	EXPECT_EQ(result[0].config_id, compile1.config_id);
 	Maat_clean_status(&mid);

+}
+#define MaatCmdTest_VirtualTable
+TEST_F(MaatCmdTest,   VirtualTable)
+{
+	Maat_feather_t feather=MaatCmdTest::_shared_feather;
+	const char* group_table_name="GROUP";
+	const char* compile_table_name="COMPILE";
+	const char* region_table_name="HTTP_SIGNATURE";
+
+	struct Maat_rule_t compile1;
+	struct Maat_group_t group1, group2;
+	struct Maat_region_t region1, region2;
+
+	memset(&compile1, 0, sizeof(compile1));
+	compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
+	Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 2);
+
+	
+	//group1->compile1
+	memset(&group1, 0, sizeof(group1));
+	group1.group_id=Maat_cmd_get_new_group_id(feather);
+	group1.table_name=group_table_name;
+	group1.virtual_table_name="HTTP_REQUEST_HEADER";
+	group1.parent_id=compile1.config_id;
+	group1.parent_type=PARENT_TYPE_COMPILE;
+	Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
+
+
+	/*region1->group1->compile1
+	*/
+	memset(&region1, 0, sizeof(region1));
+	region1.region_id=Maat_cmd_get_new_region_id(feather);
+	region1.region_type=REGION_EXPR;
+	region1.table_name=region_table_name;
+	region1.expr_rule.district="User-Agent";
+	region1.expr_rule.keywords="AppleWebKit";
+	region1.expr_rule.expr_type=EXPR_TYPE_STRING;
+	Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region1, group1.group_id);
+
+
+	//group2->compile1
+	memset(&group2, 0, sizeof(group2));
+	group2.group_id=Maat_cmd_get_new_group_id(feather);
+	group2.table_name=group_table_name;
+	group2.virtual_table_name="HTTP_RESPONSE_HEADER";
+	group2.parent_id=compile1.config_id;
+	group2.parent_type=PARENT_TYPE_COMPILE;
+	Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2);
+
+	//region2->group2
+	memset(&region2, 0, sizeof(region2));
+	region2.region_id=Maat_cmd_get_new_region_id(feather);
+	region2.region_type=REGION_EXPR;
+	region2.table_name=region_table_name;
+	region2.expr_rule.district="Cookie";
+	region2.expr_rule.keywords="uid=12345678;";
+	region2.expr_rule.expr_type=EXPR_TYPE_STRING;
+	Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region2, group2.group_id);
+
+	sleep(1);
+
+	int ret=0, table_id=0;
+	const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
+	const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;";
+	
+	struct Maat_rule_t result[4];
+	memset(result, 0, sizeof(result));
+	
+	scan_status_t mid=NULL;
+
+	table_id=Maat_table_register(feather, "HTTP_REQUEST_HEADER");	
+	ASSERT_GT(table_id, 0);
+	
+	ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, "User-Agent", strlen("User-Agent"));
+	ASSERT_EQ(ret, 0);
+	
+	ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, http_req_hdr_ua, strlen(http_req_hdr_ua), 
+						result, NULL, 4, &mid, 0);
+
+	EXPECT_EQ(ret, -2);
+	
+
+	table_id=Maat_table_register(feather, "HTTP_RESPONSE_HEADER");
+	ASSERT_GT(table_id, 0);
+
+	ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie"));
+	ASSERT_EQ(ret, 0);
+
+	ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), 
+						result, NULL, 4, &mid, 0);
+
+	EXPECT_EQ(ret, 1);
+	EXPECT_EQ(result[0].config_id, compile1.config_id);	
+
+	Maat_clean_status(&mid);
+
+	//Delete group1
+	Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
+	Maat_command_raw_set_compile(feather, MAAT_OP_DEL, &compile1, compile_table_name, NULL, 2);	
+	Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1);	
+
+	sleep(1);
+
+	table_id=Maat_table_register(feather, "HTTP_RESPONSE_HEADER");
+	ASSERT_GT(table_id, 0);
+
+	ret=Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie"));
+	ASSERT_EQ(ret, 0);
+
+	ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), 
+						result, NULL, 4, &mid, 0);
+
+	EXPECT_EQ(ret, 1);
+	EXPECT_EQ(result[0].config_id, compile1.config_id);	
+	Maat_clean_status(&mid);
+	return;
+
 }

 TEST_F(MaatCmdTest, SetLines)
@@ -2211,97 +2501,7 @@ TEST_F(MaatCmdTest, SetLines)

 	return;
 }
-/*
-//Following tests must be coded/tested at last, for they stalled the maat update thread and interrupt other tests.
-TEST_F(MaatCmdTest, SetExprOneMillion)
-{
-	const int CMD_EXPR_NUM=1000*1000;
-	const char* table_name="HTTP_URL";
-	
-	const char* keywords1="Hiredis";
-	const char* keywords2="C Client";
-	char escape_buff1[256],escape_buff2[256];
-	char keywords[256];

-	int label_id=5210, config_id=0,ret=0, output_id_cnt=0;
-	Maat_feather_t feather=MaatCmdTest::_shared_feather;
-	long long version_before=0;
-	ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
-	
-	Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1);
-	Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2);
-	snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2);
-	
-	config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", CMD_EXPR_NUM);
-	int i=0;
-	for(i=0; i<CMD_EXPR_NUM;i++)
-	{
-		test_add_expr_command(feather,table_name,config_id-i, 0, label_id, keywords);
-	}
-	ret=Maat_cmd_commit(feather);
-	EXPECT_TRUE(ret>=0);
-	wait_for_cmd_effective(feather, version_before);
-	struct Maat_cmd_t* cmd=NULL;
-	struct Maat_rule_t rule;
-	memset(&rule,0,sizeof(rule));
-	int *output_ids=(int*)malloc(sizeof(int)*CMD_EXPR_NUM);
-	output_id_cnt=Maat_cmd_select(feather,label_id, output_ids, CMD_EXPR_NUM);
-	EXPECT_EQ(output_id_cnt, CMD_EXPR_NUM);
-	for(i=0; i<CMD_EXPR_NUM;i++)
-	{
-		memset(&rule,0,sizeof(rule));
-		rule.config_id=output_ids[i];
-		cmd=Maat_create_cmd(&rule, 0);
-		ret=Maat_cmd_append(feather, cmd, MAAT_OP_DEL);
-		EXPECT_EQ(ret, 0);
-		Maat_free_cmd(cmd);
-	}
-	ret=Maat_cmd_commit(feather);
-	
-	EXPECT_EQ(ret, CMD_EXPR_NUM);
-	free(output_ids);
-}
-
-TEST_F(MaatCmdTest, SetLinesOneMillion)
-{
-	const int TEST_CMD_LINE_NUM=1000*1000;
-	Maat_feather_t feather=MaatCmdTest::_shared_feather;
-
-	struct Maat_line_t **p_line=(struct Maat_line_t **)calloc(sizeof(struct Maat_line_t *), TEST_CMD_LINE_NUM);
-	struct Maat_line_t *line_rule=(struct Maat_line_t *)calloc(sizeof(struct Maat_line_t), TEST_CMD_LINE_NUM);
-	int i=0;
-	const char* line="1\t192.168.0.1\t4444444444\t1";
-	int seq=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", TEST_CMD_LINE_NUM);
-	for(i=0;i<TEST_CMD_LINE_NUM;i++)
-	{
-		line_rule[i].label_id=0;
-		line_rule[i].rule_id=seq-i;
-		line_rule[i].table_name="QD_ENTRY_INFO";
-		line_rule[i].table_line=line;
-//		asprintf(&(line_rule[i].table_line),"1\t192.168.0.1\t%d\t1",100+i);
-		line_rule[i].expire_after=0;
-		p_line[i]=line_rule+i;
-	}
-	
-	long long version_before=0;
-	Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
-
-	Maat_cmd_set_lines(feather,(const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
-	
-	wait_for_cmd_effective(feather, version_before);
-	
-	for(i=0;i<TEST_CMD_LINE_NUM;i++)
-	{
-		line_rule[i].table_line=NULL;
-	}
-	Maat_cmd_set_lines(feather, (const struct Maat_line_t **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
-	free(p_line);
-	free(line_rule);
-
-	return;
-}
-
-*/
 int g_test_update_paused=0;
 void pause_update_test_entry_cb(int table_id,const char* table_line,void* u_para)
 {
--- a/test/testdata/charsetWindows1251.txt
+++ b/test/testdata/charsetWindows1251.txt
@@ -0,0 +1,48 @@
+<EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>!','JS_CORE_WINDOW_AUTH':'<27><><EFBFBD><EFBFBD><EFBFBD>','JS_CORE_IMAGE_FULL':'<27><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>'});</script>
+<script type="text/javascript">(window.BX||top.BX).message({'LANGUAGE_ID':'ru','FORMAT_DATE':'DD.MM.YYYY','FORMAT_DATETIME':'DD.MM.YYYY HH:MI:SS','COOKIE_PREFIX':'BITRIX_SM','USER_ID':'','SERVER_TIME':'1578340589','SERVER_TZ_OFFSET':'18000','USER_TZ_OFFSET':'0','USER_TZ_AUTO':'Y','bitrix_sessid':'fadf964e9f5bb819f212e5abf5ffb255','SITE_ID':'s1'});</script>
+
+
+<script type="text/javascript" src="/bitrix/cache/js/s1/web20/kernel_main/kernel_main.js?1402043622360126"></script>
+<script type="text/javascript" src="/bitrix/js/main/rsasecurity.js?136876011925044"></script>
+<script type="text/javascript">BX.setCSSList(['/bitrix/js/main/core/css/core.css','/bitrix/js/main/core/css/core_popup.css','/bitrix/js/main/core/css/core_date.css','/bitrix/js/main/core/css/core.css','/bitrix/js/main/core/css/core_popup.css','/bitrix/js/main/core/css/core_date.css']); </script>
+<script type="text/javascript">BX.setJSList(['/bitrix/js/main/core/core.js','/bitrix/js/main/core/core_ajax.js','/bitrix/js/main/session.js','/bitrix/js/main/json/json2.min.js','/bitrix/js/main/core/core_ls.js','/bitrix/js/main/core/core_window.js','/bitrix/js/main/utils.js','/bitrix/js/main/core/core_popup.js','/bitrix/js/main/core/core_date.js','/bitrix/js/main/dd.js']); </script>
+
+<script type="text/javascript">
+bxSession.Expand(1440, 'fadf964e9f5bb819f212e5abf5ffb255', false, '35a74b06af8f9ea55ffbda20075b0894');
+</script>
+<script>new Image().src='http://www.sgaice.ru/bitrix/spread.php?s=QklUUklYX1NNX0dVRVNUX0lEATY4MTg5NQExNjA5NDQ0NTg5AS8BAQECQklUUklYX1NNX0xBU1RfVklTSVQBMDcuMDEuMjAyMCAwMDo1NjoyOQExNjA5NDQ0NTg5AS8BAQEC&k=71d3b79b44f9716b27b47feab4a206cf';
+</script>
+
+<script type="text/javascript" src="/bitrix/cache/js/s1/web20/template_1e341eb2f86845c7519566374f51d35a/template_1e341eb2f86845c7519566374f51d35a_368c1a68876fd1c32b307a10695f3654.js?14010848191120"></script>
+<script type="text/javascript" src="/bitrix/js/imgzoom/thumb.js"></script>
+<meta name="google-site-verification" content="gL_64SaiDgQcX5z-pvPZmBJ-exN-wS6KZNoDMcPtYtM" />
+<title><3E><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><>ѻ</title>
+</head>
+
+<body>
+
+
+<div id="maintop">
+    <table align="left1" width="100%" border="0" cellpadding="0" cellspacing="0">
+    <tr><td valign="top">
+		
+<script type="text/javascript">
+top.BX.defer(top.rsasec_form_bind)({'formid':'system_auth_form6zOUGO','key':{'M':'HazQxsgvQCIFPf30iHR40R22fp7P9YLPXFhQu6uus68RZxf2IpMo9v0KDpxkgg43WXaZaXrTRvjg1e2126IOo66vH5bphkMP/69MSPlEoaXYzWjTokd+Yzy30WR6HEOyB9tJwADGyjysqoE4+jUfHZQv2JMaVZS0U4SHWOUPwNU=','E':'AQAB','chunk':'128'},'rsa_rand':'5e1390ed8a8e19.17355178','params':['USER_PASSWORD']});
+</script>
+
+
+
+<div id="login-form-window">
+
+<a href="" onclick="return CloseLoginForm()" style="float:right;"><3E><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD></a>
+
+<form method="post" target="_top" action="/index.php?login=yes">
+			<input type="hidden" name="backurl" value="/index.php" />
+			<input type="hidden" name="AUTH_FORM" value="Y" />
+	<input type="hidden" name="TYPE" value="AUTH" />
+
+	<table width="95%">
+			<tr>
+				<td colspan="2">
+				<09><><EFBFBD><EFBFBD><EFBFBD>:<br />
+				<input type="text" name="USER_LOGIN" maxlength="50" value="
--- a/tools/maat_debug_tool.cpp
+++ b/tools/maat_debug_tool.cpp
@@ -2,16 +2,42 @@
 #include "Maat_command.h"
 #include <MESA/MESA_handle_logger.h>

+#include <MESA/stream.h>
 #include <stdio.h>     /* for printf */
 #include <stdlib.h>    /* for exit */
 #include <getopt.h>
 #include <sys/stat.h>
+#include <arpa/inet.h>

+void debug_maat_result_print(const char* table_name, int scan_ret, struct Maat_rule_t* result)
+{
+	printf("Scan table %s ", table_name);
+	if(scan_ret==-1)
+	{
+		printf("error.\n");
+	}
+	else if(scan_ret==-2)
+	{
+		printf("hits group, but not compile.\n");
+	}
+	else if(scan_ret==0)
+	{
+		printf("not hit.");
+	}
+	else
+	{
+		printf("hits ");
+		for(int i=0; i<scan_ret; i++)
+		{
+			printf("%d, ", result[i].config_id);
+		}
+	}
+	printf("\n");

+}
 int debug_maat_str_scan(Maat_feather_t feather, const char* table_name, const char* district, char* data, size_t sz)
 {
 	int table_id=0,ret=0;
-	int i=0;
 	int scan_result_max=64;
 	struct Maat_rule_t result[scan_result_max];
 	scan_status_t mid=NULL;
@@ -35,33 +61,38 @@ int debug_maat_str_scan(Maat_feather_t feather, const char* table_name, const ch
 	ret=Maat_full_scan_string(feather, table_id, maat_charset, data, sz, 
 						result, NULL, scan_result_max,
 						&mid, 0);
-	printf("Scan table %s ", table_name);
-	if(ret==-1)
-	{
-		printf("error.\n");
-	}
-	else if(ret==-2)
-	{
-		printf("hits group, but not compile.\n");
-	}
-	else if(ret==0)
-	{
-		printf("not hit.");
-	}
-	else
-	{
-		printf("hits ");
-		for(i=0; i<ret; i++)
-		{
-			printf("%d, ", result[i].config_id);
-		}
-	}
-	printf("\n");
+	debug_maat_result_print(table_name, ret, result);

 	free(hit_detail);
 	Maat_clean_status(&mid);
 	return 0;
 }
+int debug_maat_ip_scan(Maat_feather_t feather, const char* table_name, const char* sip)
+{
+	int table_id=0,ret=0;
+	int scan_result_max=64;
+
+	struct Maat_rule_t result[scan_result_max];
+	scan_status_t mid=NULL;
+	struct ipaddr ipv4_addr;
+	struct stream_tuple4_v4 v4_addr;
+	ipv4_addr.addrtype=ADDR_TYPE_IPV4;
+	inet_pton(AF_INET, sip, &(v4_addr.saddr));
+	v4_addr.source=htons(5210);
+	inet_pton(AF_INET, "127,0.0.1", &(v4_addr.daddr));
+	v4_addr.dest=htons(7400);
+	ipv4_addr.v4=&v4_addr;
+
+
+	table_id=Maat_table_register(feather, table_name);
+	ret=Maat_scan_proto_addr(feather, table_id, &ipv4_addr, 6, result, scan_result_max, &mid, 0);
+	
+	debug_maat_result_print(table_name, ret, result);
+
+	Maat_clean_status(&mid);
+	return 0;
+}
+
 enum tool_arg_type{
 	ARG_TABLE_INFO=0,
 	ARG_TABLE_NAME,
@@ -70,8 +101,10 @@ enum tool_arg_type{
 	ARG_SCAN_DISTRICT,
 	ARG_SCAN_CHARSET,
 	ARG_INPUT_JSON,
+	ARG_INPUT_FULL_INDEX,
 	ARG_DECRYPT_KEY,
 	ARG_ACCEPT_TAGS,
+	ARG_SCAN_IPv4,
 	__ARG_MAX
 };

@@ -87,21 +120,25 @@ int main(int argc, char ** argv)
 	{
 		return 0;
 	}
+	static struct option long_options[]={
+		{"table-info",		required_argument, 0, 0},
+		{"table-name",		required_argument, 0, 0},
+		{"scan-file",		optional_argument, 0, 0},
+		{"scan-string", optional_argument, 0, 0},
+		{"scan-district", optional_argument, 0, 0},
+		{"scan-charset",	optional_argument, 0, 0},
+		{"maat-json",		required_argument, 0, 0},
+		{"full-index",	required_argument, 0, 0},
+		{"decrypt-key", optional_argument, 0, 0},
+		{"accept-tags", optional_argument, 0, 0},
+		{"scan-ipv4", required_argument, 0, 0},	
+		{0, 0, 0, 0}
+	};
+
 	int c=0;
 	while (1) {
 		int option_index = 0;
-		static struct option long_options[]={
-			{"table-info",		required_argument, 0, 0},
-			{"table-name",		required_argument, 0, 0},
-			{"scan-file",		optional_argument, 0, 0},
-			{"scan-string",	optional_argument, 0, 0},
-			{"scan-district", optional_argument, 0, 0},
-			{"scan-charset",	optional_argument, 0, 0},
-			{"maat-json",		required_argument, 0, 0},
-			{"decrypt-key",	optional_argument, 0, 0},
-			{"accept-tags",	optional_argument, 0, 0},
-			{0, 0, 0, 0}
-		};
+

 		c = getopt_long(argc, argv, "ab", long_options, &option_index);
 		if (c == -1)
@@ -133,10 +170,29 @@ int main(int argc, char ** argv)
 	{
 		Maat_set_feather_opt(feather, MAAT_OPT_ACCEPT_TAGS, arg_value[ARG_ACCEPT_TAGS], strlen(arg_value[ARG_ACCEPT_TAGS])+1);
 	}
-	ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, arg_value[ARG_INPUT_JSON], strlen(arg_value[ARG_INPUT_JSON])+1);
-	if(ret!=0)
+	if(strlen(arg_value[ARG_INPUT_JSON])>0)
 	{
-		printf("Read %s failed, invalid maat json.\n", arg_value[ARG_INPUT_JSON]);
+		ret=Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, arg_value[ARG_INPUT_JSON], strlen(arg_value[ARG_INPUT_JSON])+1);
+		if(ret!=0)
+		{
+			printf("Read %s failed, invalid maat json.\n", arg_value[ARG_INPUT_JSON]);
+			ret=-1;
+			goto clean_up;
+		}
+	}
+	else if(strlen(arg_value[ARG_INPUT_FULL_INDEX]))
+	{
+		ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, arg_value[ARG_INPUT_FULL_INDEX], strlen(arg_value[ARG_INPUT_FULL_INDEX])+1);
+		if(ret!=0)
+		{
+			printf("Set %s failed, invalid maat json.\n", arg_value[ARG_INPUT_FULL_INDEX]);
+			ret=-1;
+			goto clean_up;
+		}
+	}
+	else
+	{
+		printf("Error: One of --%s and --%s should be specified.\n", long_options[ARG_INPUT_JSON].name, long_options[ARG_INPUT_FULL_INDEX].name);
 		ret=-1;
 		goto clean_up;
 	}
@@ -165,6 +221,10 @@ int main(int argc, char ** argv)
 		debug_maat_str_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_DISTRICT], file_buff, file_size);
 		free(file_buff);
 	}
+	if(strlen(arg_value[ARG_SCAN_IPv4])>0)
+	{
+		debug_maat_ip_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_IPv4]);
+	}
 clean_up:
 	Maat_burn_feather(feather);
 	MESA_destroy_runtime_log_handle(g_logger);
--- a/tools/maat_redis_tool.cpp
+++ b/tools/maat_redis_tool.cpp
@@ -1,4 +1,5 @@
 #include "Maat_rule.h"
+#include "Maat_utils.h"
 #include "Maat_command.h"
 #include "Maat_rule_internal.h"
 #include "cJSON.h"
@@ -238,7 +239,7 @@ int main(int argc, char * argv[])
 	unsigned long json_file_size=0,read_size=0;
 	long long desired_version=0;
 	char* json_buff=NULL;
-
+	size_t json_buff_sz=0;
 	while((oc=getopt(argc,argv,"h:p:n:d:v:f:j:t:"))!=-1)
 	{
 		switch(oc)
@@ -325,8 +326,13 @@ int main(int argc, char * argv[])
 		read_rule_from_redis(ctx,desired_version,dump_dir, NULL);
 	}
 	else if(model==WORK_MODE_JSON)
-	{		
-		ret=json2iris(json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL);
+	{
+		ret=load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz);
+		if(ret<0)
+		{
+			printf("open %s failed.\n", json_file);
+		}
+		ret=json2iris(json_buff, json_file, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, NULL);
 		if(ret<0)
 		{
 			printf("Invalid json format.\n");
@@ -336,6 +342,10 @@ int main(int argc, char * argv[])
 		printf("Serialize %s to %d lines, write temp file to %s .\n",json_file, total_line_cnt, tmp_iris_path);
 		struct serial_rule_t * s_rule=(struct serial_rule_t *)calloc(sizeof(struct serial_rule_t),total_line_cnt);
 		server_time=redis_server_time(ctx);
+		if(!server_time)
+		{
+			printf("Get Redis Time failed.\n");
+		}
 		if(timeout>0)
 		{
 			absolute_expire_time=server_time+timeout;