支持内容外键，即某一列指向redis中的一个key，将其变成文件路径。

inc/Maat_rule.h

@@ -141,10 +141,10 @@ enum MAAT_INIT_OPT
 	MAAT_OPT_FULL_CFG_DIR,			//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default.
 	MAAT_OPT_INC_CFG_DIR,			//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default.
 	MAAT_OPT_JSON_FILE_PATH,		//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1.DEFAULT: no default.
-	MAAT_OPT_STAT_ON,				//VALUE is  NULL,SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
+	MAAT_OPT_STAT_ON,				//VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
-	MAAT_OPT_PERF_ON,				//VALUE is NULL,SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
+	MAAT_OPT_PERF_ON,				//VALUE is NULL, SIZE is 0. MAAT_OPT_STAT_FILE_PATH must be set. Default: stat OFF.
 	MAAT_OPT_STAT_FILE_PATH,		//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. DEFAULT: no default.
-	MAAT_OPT_SCAN_DETAIL,			//VALUE is interger *, SIZE=sizeof(int). 0: not return any detail;1: return  hit pos, not include regex grouping;
+	MAAT_OPT_SCAN_DETAIL,			//VALUE is interger *, SIZE=sizeof(int). 0: not return any detail;1: return  hit pos, not include regex grouping.
 									//  2 return hit pos and regex grouping pos;DEFAULT:0
 	MAAT_OPT_INSTANCE_NAME,			//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1, no more than 11 bytes.DEFAULT: MAAT_$tableinfo_path$.
 	MAAT_OPT_DECRYPT_KEY,			//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. No DEFAULT.
@@ -157,8 +157,10 @@ enum MAAT_INIT_OPT
 	MAAT_OPT_LOAD_VERSION_FROM,		//VALUE is a long long, SIZE=sizeof(long long). Default: Load the Latest. Only valid in redis mode, and maybe failed for too old. 
 									//This option also disables background update.
 	MAAT_OPT_ENABLE_UPDATE,			//VALUE is interger, SIZE=sizeof(int). 1: Enabled, 0:Disabled.  DEFAULT: Backgroud update is enabled. Runtime setting is allowed.
-	MAAT_OPT_ACCEPT_TAGS			//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]}
+	MAAT_OPT_ACCEPT_TAGS,			//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Format is a JSON, e.g.{"tags":[{"tag":"location","value":"Beijing/ChaoYang/Huayan/22A"},{"tag":"isp","value":"telecom"}]}
-};
+	MAAT_OPT_FOREIGN_CONT_DIR,		//VALUE is a const char*, MUST end with '\0', SIZE= strlen(string+'\0')+1. Specifies a local diretory to store foreign content. Default: []table_info_path]_files
+ 	MAAT_OPT_FOREIGN_CONT_LINGER		//VALUE is interger *, SIZE=sizeof(int). Greater than 0: delete after VALUE seconds; 0: delete foreign content right after the notification callbacks;  Less than 0: NEVER delete. Default: 0.
+ };
 //return -1 if failed, return 0 on success;
 int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT  type,const void* value,int size);
 enum MAAT_STATE_OPT

src/CMakeLists.txt

@@ -9,7 +9,7 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$
 message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}")
 
 add_definitions(-fPIC)
-set(MAAT_SRC entry/cJSON.c  entry/config_monitor.cpp  entry/dynamic_array.cpp  entry/gram_index_engine.c  entry/interval_index.c  entry/json2iris.cpp  entry/Maat_api.cpp  entry/Maat_command.cpp  entry/Maat_rule.cpp  entry/Maat_stat.cpp  entry/map_str2int.cpp  entry/rbtree.c  entry/stream_fuzzy_hash.c  entry/UniversalBoolMatch.cpp)
+set(MAAT_SRC entry/cJSON.c  entry/config_monitor.cpp  entry/dynamic_array.cpp  entry/gram_index_engine.c  entry/interval_index.c  entry/json2iris.cpp entry/Maat_utils.cpp entry/Maat_api.cpp  entry/Maat_command.cpp  entry/Maat_rule.cpp  entry/Maat_stat.cpp  entry/map_str2int.cpp  entry/rbtree.c  entry/stream_fuzzy_hash.c  entry/UniversalBoolMatch.cpp)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../inc/)
 include_directories(/opt/MESA/include/MESA/)
 

src/entry/Maat_api.cpp

@@ -8,6 +8,7 @@
 #include "UniversalBoolMatch.h"
 #include "Maat_rule.h"
 #include "Maat_rule_internal.h"
+#include "Maat_utils.h"
 #include "dynamic_array.h"
 #include "aligment_int64.h"
 #include "config_monitor.h"
@@ -486,6 +487,8 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
 	feather->connect_timeout.tv_sec=0;
 	feather->connect_timeout.tv_usec=100*1000; // 100 ms
 	feather->backgroud_update_enabled=1;
+	feather->foreign_cont_linger=0;
+	snprintf(feather->foreign_cont_dir, sizeof(feather->foreign_cont_dir), "%s_files", table_info_path);
 	pthread_mutex_init(&(feather->backgroud_update_mutex),NULL);
 	pthread_mutex_init(&(feather->redis_write_lock),NULL);
 	snprintf(feather->table_info_fn,sizeof(feather->table_info_fn),"%s",table_info_path);
@@ -673,6 +676,16 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
 				return -1;
 			}
 			break;
+		case MAAT_OPT_FOREIGN_CONT_DIR:
+			strncpy(_feather->foreign_cont_dir, (char*)value, sizeof(_feather->foreign_cont_dir));
+			if(_feather->foreign_cont_dir[size-1]=='/')
+			{
+				_feather->foreign_cont_dir[size-1]='\0';
+			}
+			break;
+		case MAAT_OPT_FOREIGN_CONT_LINGER:
+			_feather->foreign_cont_linger=*(int*)value;
+			break;
 		default:
 			return -1;
 	}
@@ -904,7 +917,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
 	//plugin table register blocks background update.
 	pthread_mutex_lock(&(_feather->backgroud_update_mutex));
 	idx=p_table->cb_info->cb_plug_cnt;
-	if(idx==MAX_PLUGING_NUM)
+	if(idx==MAX_PLUGIN_PER_TABLE)
 	{
 		pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
 		return -1;

src/entry/Maat_command.cpp

@@ -1,6 +1,7 @@
 #include "Maat_command.h"
 #include "Maat_rule.h"
 #include "Maat_rule_internal.h"
+#include "Maat_utils.h"
 #include "config_monitor.h"
 #include "map_str2int.h"
 #include "hiredis.h"
@@ -292,7 +293,15 @@ void empty_serial_rules(struct serial_rule_t* rule)
 	if(rule->table_line!=NULL)
 	{
 		free(rule->table_line);
-		rule->table_line=NULL;
+	}
+	if(rule->n_foreign>0)
+	{
+		for(int i=0; i<rule->n_foreign; i++)
+		{
+			free(rule->f_keys[i].filename);
+			free(rule->f_keys[i].key);
+		}
+		free(rule->f_keys);
 	}
 	memset(rule,0,sizeof(struct serial_rule_t));
 	return;
@@ -1145,7 +1154,6 @@ void _exec_serial_rule(redisContext* ctx,	long long version, struct serial_rule_
 	return;
 }
 
-#define	MAX_REDIS_OP_PER_SRULE	8
 int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_rule_num, long long server_time, void* logger)
 {
 	int max_redis_batch=1*1024,batch_cnt=0;
@@ -1154,6 +1162,7 @@ int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,int serial_r
 	unsigned int i=0;
 
 	unsigned int multi_cmd_cnt=0;
+	const int MAX_REDIS_OP_PER_SRULE=8;
 	unsigned int max_multi_cmd_num=MAX_REDIS_OP_PER_SRULE*serial_rule_num+2;// 2 for operation in _exec_serial_rule_end()
 	
 	struct expected_reply_t *expected_reply=(struct expected_reply_t*)calloc(sizeof(struct expected_reply_t), max_multi_cmd_num);
@@ -1371,6 +1380,208 @@ void cleanup_update_status(redisContext *ctx, void *logger)
 			,entry_num);
 
 }
+const char* find_Nth_column(const char* line, int Nth, int* column_len)
+{
+	int i=0, j=0;
+	int start=0, end=0;
+	for(i=0;i<(int)strlen(line);i++)
+	{
+		if(line[i]==' '||line[i]=='\t')
+		{
+			j++;
+		}
+		if(j==Nth-1)
+		{
+			start=i+1;
+		}
+		if(j==Nth)
+		{
+			end=i+1;
+			break;
+		}
+	}
+	if(start==0)
+	{
+		return NULL;
+	}
+	if(end==0)
+	{
+		end=i;
+	}
+	*column_len=end-start;
+	return line+start;
+}
+char* get_foreign_cont_filename(const char* table_name, int rule_id, const char* foreign_key, const char* dir)
+{
+	char* filename=NULL;
+	char buffer[512];
+	snprintf(buffer, sizeof(buffer),"%s/%s/%d%s",dir, table_name, rule_id, foreign_key);
+	filename=(char*)calloc(sizeof(char), strlen(buffer)+1);
+	memcpy(filename, buffer, strlen(buffer));
+	return filename;
+}
+void rewrite_table_line_with_foreign(struct serial_rule_t*p)
+{
+	int origin_column_size=0;
+	const char* origin_column=NULL, *pos_origin_line=NULL;
+	char* pos_rewrite_line=NULL;
+	char* rewrite_line=NULL;
+	size_t fn_size=0;
+	int i=0;
+	for(i=0; i<p->n_foreign; i++)
+	{
+		fn_size+=strlen(p->f_keys[i].filename);
+	}
+	
+	rewrite_line=(char*)calloc(sizeof(char), strlen(p->table_line)+fn_size);
+	pos_origin_line=p->table_line;
+	pos_rewrite_line=rewrite_line;
+	
+	for(i=0; i<p->n_foreign; i++)
+	{
+		origin_column=find_Nth_column(p->table_line, p->f_keys[i].column, &origin_column_size);
+		strncat(pos_rewrite_line, pos_origin_line, origin_column-pos_origin_line);
+		pos_rewrite_line+=origin_column-pos_origin_line;
+		pos_origin_line+=origin_column_size;
+		
+		strncat(pos_rewrite_line, p->f_keys[i].filename, strlen(p->f_keys[i].filename));
+		pos_rewrite_line+=strlen(p->f_keys[i].filename);
+	}
+	strncat(pos_rewrite_line, pos_origin_line, strlen(p->table_line)-(pos_origin_line-p->table_line));
+	pos_rewrite_line+=strlen(p->f_keys[i].filename); 
+	
+	free(p->table_line);
+	p->table_line=rewrite_line;
+	return;
+}
+int get_foreign_keys(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger)
+{
+	int ret=0, table_id=0, i=0, j=0;
+	int foregin_key_size=0;
+	int rule_with_foreign_key=0;
+	const char* foreign_source_prefix="redis://";
+	const char* foreign_key_prefix="__FILE_";
+	const char* p_foregin=NULL;
+	struct _Maat_table_info_t* p_table=NULL;
+	for(i=0; i<rule_num; i++)
+	{
+		if(rule_list[i].table_line==NULL)
+		{
+			continue;
+		}
+		ret=map_str2int(feather->map_tablename2id, rule_list[i].table_name, &table_id);
+		if(ret<0)
+		{
+			continue;
+		}
+		p_table=feather->p_table_info[table_id];
+		if(p_table->table_type!=TABLE_TYPE_PLUGIN||p_table->n_foreign==0)
+		{
+			continue;
+		}
+		rule_list[i].n_foreign=p_table->n_foreign;
+		rule_list[i].f_keys=(struct foreign_key*)calloc(sizeof(struct foreign_key), rule_list[i].n_foreign);
+		for(j=0;j<p_table->n_foreign;j++)
+		{
+			p_foregin=find_Nth_column(rule_list[i].table_line, p_table->foreign_columns[j], &foregin_key_size);
+			if(p_foregin==NULL)
+			{
+				MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
+										, "Get %s,%d foreign keys failed: No %dth column."
+										, rule_list[i].table_name, rule_list[i].rule_id, p_table->foreign_columns[j]);
+				continue;
+			}
+			if(0!=strncasecmp(p_foregin, foreign_source_prefix, strlen(foreign_source_prefix)))
+			{
+				MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
+										,"Get %s,%d foreign key failed: Invalid source prefix %s."
+										, rule_list[i].table_name, rule_list[i].rule_id, p_foregin);
+				continue;
+			}
+			rule_list[i].f_keys[j].column=p_table->foreign_columns[j];
+			foregin_key_size=foregin_key_size+1-strlen(foreign_source_prefix);
+			p_foregin+=strlen(foreign_source_prefix);
+			if(0!=strncasecmp(p_foregin, foreign_key_prefix, strlen(foreign_key_prefix)))
+			{
+				MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor
+									,"%s,%d foreign key prefix %s is discouraged."
+									, rule_list[i].table_name, rule_list[i].rule_id, p_foregin);
+			}
+			rule_list[i].f_keys[j].key=(char*)calloc(sizeof(char),foregin_key_size);
+			memcpy(rule_list[i].f_keys[j].key, p_foregin, foregin_key_size);
+
+			rule_list[i].f_keys[j].filename=get_foreign_cont_filename(rule_list[i].table_name, rule_list[i].rule_id, p_foregin, dir);
+		}
+		rule_with_foreign_key++;
+	}
+	return rule_with_foreign_key;
+}
+
+void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, void *logger)
+{
+	int i=0, j=0, ret=0;
+	int key_num=0;
+	
+	char redis_cmd[256];
+	redisReply* reply=NULL;
+	struct serial_rule_t*p=NULL;
+	char** filenames[MAX_FOREIGN_CLMN_NUM];
+	FILE* fp=NULL;
+	for(i=0;i<rule_num;i++)
+	{
+		p=rule_list+i;
+		if(p->op==MAAT_OP_DEL||p->n_foreign==0)
+		{
+			continue;
+		}
+		for(j=0; j<p->n_foreign; j++)
+		{
+			snprintf(redis_cmd,sizeof(redis_cmd),"GET %s", p->f_keys[j].key);
+			ret=redisAppendCommand(ctx, redis_cmd);
+			key_num++;
+			assert(ret==REDIS_OK);
+		}
+	}
+	for(i=0;i<rule_num;i++)
+	{
+		p=rule_list+i;
+		if(p->op==MAAT_OP_DEL||p->n_foreign==0)
+		{
+			continue;
+		}
+		for(j=0; j<p->n_foreign; j++)
+		{
+			ret=_wrap_redisGetReply(ctx,&reply);
+			if(reply->type!=REDIS_REPLY_STRING)
+			{
+				MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor
+						,"Get %s,%d foreign key %s content failed."
+						,rule_list[i].table_name
+						,rule_list[i].rule_id
+						,p->f_keys[j].key);
+				continue;
+			}
+			else
+			{
+				fp=fopen(p->f_keys[j].filename, "w");
+				if(fp==NULL)
+				{
+					MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor
+											, "Write foreign content failed: fopen %s error."
+											, filenames[j]);
+				}
+				else
+				{
+					fwrite(reply->str, 1, reply->len, fp);
+					fclose(fp);
+					fp=NULL;
+				}
+			}
+			freeReplyObject(reply);
+		}
+	}
+	return;
+}
 
 void redis_monitor_traverse(long        long version,redisContext *c,
 		void (*start)(long long,int ,void*),//vesion,CM_UPDATE_TYPE_*,u_para
@@ -1380,7 +1591,7 @@ void redis_monitor_traverse(long        long version,redisContext *c,
 		const unsigned char* dec_key,
 		_Maat_feather_t* feather)
 {
-	int table_id=0,i=0,rule_num=0,empty_value_num=0;
+	int table_id=0,i=0, j=0, rule_num=0,empty_value_num=0;
 	int ret=0;
 	struct serial_rule_t* rule_list=NULL;
 	int update_type=CM_UPDATE_TYPE_INC;
@@ -1432,6 +1643,11 @@ void redis_monitor_traverse(long        long version,redisContext *c,
 		{
 
 			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor,"%d of %d rules are empty.",empty_value_num,rule_num);
+		}
+		ret=get_foreign_keys(c, rule_list, rule_num, feather, feather->foreign_cont_dir, logger);
+		if(ret>0)
+		{
+			get_foreign_conts(c, rule_list, rule_num, logger);
 		}
 	}
 	start(new_version,update_type,u_para);
@@ -1458,11 +1674,41 @@ void redis_monitor_traverse(long        long version,redisContext *c,
 										,rule_list[i].table_line);
 				continue;
 			}
+		}
+		if(rule_list[i].n_foreign>0)
+		{
+			rewrite_table_line_with_foreign(rule_list+i);
 		}
-		update(rule_list[i].table_name,rule_list[i].table_line,u_para);
+		update(rule_list[i].table_name,rule_list[i].table_line,u_para);
+		if(rule_list[i].n_foreign&&rule_list[i].op==MAAT_OP_DEL)
+		{
+			
+			for(j=0; j<rule_list[i].n_foreign; j++)
+			{
+				if(feather->foreign_cont_linger==0)
+				{
+					ret=system_cmd_rm(rule_list[i].f_keys[j].filename);
+					if(ret==-1)
+					{
+						MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
+								"Foreign content file %s remove failed.",
+								rule_list[i].f_keys[j].filename);
+					}
+				}
+				else if(feather->foreign_cont_linger>0)
+				{
+					garbage_bagging_with_timeout(GARBAGE_FOREIGN_FILE, rule_list[i].f_keys[j].filename, feather->foreign_cont_linger, feather->garbage_q);
+					rule_list[i].f_keys[j].filename=NULL;//transfer owner to garbage collection.
+				}
+				else
+				{
+					//Less than 0, don't delete.
+				}
+			}
+			
 		}
 	}
-	finish(u_para);
+	finish(u_para);
 
 clean_up:
 	for(i=0;i<rule_num;i++)

src/entry/Maat_rule.cpp

@@ -19,6 +19,7 @@
 
 #include "Maat_rule.h"
 #include "Maat_rule_internal.h"
+#include "Maat_utils.h"
 #include "json2iris.h"
 #include "cJSON.h"
 #include "dynamic_array.h"
@@ -35,57 +36,7 @@ int MAAT_FRAME_VERSION_2_2_20180921=1;
 
 const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
 							"unicode_ascii_esc","unicode_ascii_aligned","unicode_ncr_dec","unicode_ncr_hex","url_encode_gb2312","url_encode_utf8",""};
-pid_t gettid()
-{
-	return syscall(SYS_gettid);
-}
-const char*	module_name_str(const char*name)
-{
-	static __thread char module[64];
-	snprintf(module,sizeof(module),"%s(%d)", name, gettid());
-	return module;
-}
 
-int converHextoint(char srctmp)
-{
-	if(isdigit(srctmp))
-	{
-		return srctmp-'0';
-	}
-	else
-	{
-		char temp=toupper(srctmp);
-		temp=temp-'A'+10;
-		return temp;
-	}
-}
-int  hex2bin(char *hex,int hex_len,char *binary,int size)
-{
-	int i=0;
-	int resultlen=0;
-	int high,low;
-	for(i=0;i<hex_len&&size>resultlen; i+=2,resultlen++)
-	{
-		high=converHextoint(hex[i]);
-		low=converHextoint(hex[i+1]);
-		binary[resultlen]=high*16+low;
-	}
-	size=resultlen;
-	binary[resultlen]='\0';
-	return resultlen;
-}
-//functioned as strdup, for dictator compatible.
-char* _maat_strdup(const char* s)
-{
-	char*d=NULL;
-	if(s==NULL)
-	{
-		return NULL;
-	}
-	d=(char*)malloc(strlen(s)+1);
-	memcpy(d,s,strlen(s)+1);
-	return d;
-}
 int is_valid_expr_type(enum MAAT_EXPR_TYPE expr_type)
 {
 	switch(expr_type)
@@ -320,113 +271,6 @@ error_out:
 	
 	return ret;
 }
-char* str_tolower(char* string)
-{
-	int i=0;
-	for(i=0;i<(int)strlen(string);i++)
-	{
-		string[i]=(char)tolower(string[i]);
-	}
-	return string;
-}
-char * strchr_esc(char* s,const char delim)
-{
-	char *token;
-	if(s==NULL)
-		return NULL;
-	for(token=s;*token!='\0';token++)
-	{
-		if(*token=='\\')
-		{
-			token++;
-			continue;
-		}
-		if(*token==delim)
-			break;
-	}
-	if (*token == '\0')
-	{ 
-		return NULL;
-	}
-	else
-	{
-		return token;
-	}
-}
-char *strtok_r_esc(char *s, const char delim, char **save_ptr) {  
-    char *token;  
-  
-    if (s == NULL) s = *save_ptr;  
-  
-    /* Scan leading delimiters.  */
-	token=strchr_esc(s,delim);
-	if(token==NULL)
-	{
-		*save_ptr=token;
-		return s;
-	}
-    /* Find the end of the token.  */  
-	*token='\0';
-	token++;
-	*save_ptr=token;
-
-    return s;  
-}   
-char *str_unescape_and(char*s)
-{
-	int i=0,j=0;
-	for(i=0,j=0;i<(int)strlen(s);i++)
-	{
-		if(s[i]=='\\'&&s[i+1]=='&')
-		{
-			s[j]='&';
-			i++;
-			j++;
-		}
-		else{
-			s[j]=s[i];
-			j++;
-		}
-	}
-	s[j]='\0';
-	return s;
-}
-char* str_unescape(char* s)
-{
-	int i=0,j=0;
-	int len=strlen(s);
-	for(i=0,j=0;i<len;i++)
-	{
-		if(s[i]=='\\')
-		{
-			switch(s[i+1])
-			{
-				case '&':
-					s[j]='&';
-					break;
-				case 'b':
-					s[j]=' ';//space,0x20;
-					break;
-				case '\\':
-					s[j]='\\';
-					break;
-				default:
-					s[j]=s[i];
-					i--;	//undo the followed i++
-					break;
-			}
-			i++;
-			j++;
-		}
-		else
-		{
-			s[j]=s[i];
-			j++;
-		}
-	}
-	s[j]='\0';
-	return s;
-}
 char* Maat_str_escape(char* dst,int size,const char*src)
 {
 	int i=0,j=0;
@@ -460,6 +304,7 @@ char* Maat_str_escape(char* dst,int size,const char*src)
 	dst[j]='\0';
 	return dst;
 }
+
 int cnt_maskbits(struct in6_addr mask)
 {
 	unsigned int i=0;
@@ -689,7 +534,7 @@ void destroy_table_info(struct _Maat_table_info_t*p)
 	free(p);
 	return;
 }
-int read_expr_table_info(const char* line,int line_num,struct _Maat_table_info_t* p,MESA_htable_handle string2int_map)
+int read_expr_table_info(const char* line, struct _Maat_table_info_t* p, MESA_htable_handle string2int_map)
 {
 	int j=0,ret[4]={0};
 	char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0};
@@ -743,6 +588,91 @@ int read_expr_table_info(const char* line,int line_num,struct _Maat_table_info_t
 	}
 	return 0;
 }
+int _read_integer_arrary(char* string, int *array, int size)
+{
+	char *token=NULL,*sub_token=NULL,*saveptr;
+	int i=0;
+	for (token = string, i=0; i<size ; token= NULL, i++)
+	{
+		sub_token= strtok_r(token,",", &saveptr);
+		if (sub_token == NULL)
+			break;
+		sscanf(sub_token, "%d", array+i);
+	}
+	return i;
+}
+int read_plugin_table_info(const char* line, struct _Maat_table_info_t* p)
+{
+	int i=0,ret=0;
+	size_t j=0;
+	char* copy_line=NULL, *plug_info=NULL;
+	char *token=NULL,*sub_token=NULL,*saveptr;
+	const char* plug_info_prefix[]={"valid","tag","foreign"};
+	const int VALID_PREFIX_IDX=0, TAG_PREFIX_IDX=1, FOREIGN_PREFIX_IDX=2;
+	p->valid_flag_column=-1;
+	copy_line=_maat_strdup(line);
+	for (token = copy_line, i=0; ; token= NULL, i++)
+	{
+		sub_token= strtok_r(token,"\t", &saveptr);
+		if (sub_token == NULL)
+			break;
+		if(i==3)
+		{
+			break;
+		}
+	
+	}
+	if(i<3)
+	{
+		goto error_out;
+	}
+	plug_info=sub_token;
+
+	if(strlen(plug_info)<4)//For old version compatible.
+	{
+		ret=sscanf(plug_info, "%d", &(p->valid_flag_column));
+		if(ret==0||ret==EOF)
+		{
+			p->valid_flag_column=-1;
+		}
+		free(copy_line);
+		return 0;
+	}
+
+	for (token = plug_info; ; token= NULL)
+	{
+		sub_token= strtok_r(token,";", &saveptr);
+		if (sub_token == NULL)
+			break;
+		for(j=0; j< sizeof(plug_info_prefix)/sizeof(const char*); j++)
+		{
+			if(0==strncasecmp(sub_token, plug_info_prefix[j], strlen(plug_info_prefix[j])))
+			{
+				break;
+			}
+		}
+		switch(j)
+		{
+			case VALID_PREFIX_IDX:
+				sscanf(sub_token+strlen(plug_info_prefix[j])+1,"%d",&(p->valid_flag_column));
+				break;
+			case TAG_PREFIX_IDX:
+				sscanf(sub_token+strlen(plug_info_prefix[j])+1,"%d",&(p->rule_tag_column));
+				break;
+			case FOREIGN_PREFIX_IDX:
+				p->n_foreign=_read_integer_arrary(sub_token+strlen(plug_info_prefix[j])+1, p->foreign_columns, MAX_FOREIGN_CLMN_NUM);
+				break;
+			default:
+				break;
+		}
+
+	}	
+	free(copy_line);
+	return 0;
+error_out:
+	free(copy_line);
+	return -1;
+}
 int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger)
 {
 	FILE*fp=NULL;
@@ -821,22 +751,25 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
 		{
 			case TABLE_TYPE_EXPR:
 			case TABLE_TYPE_EXPR_PLUS:
-				ret=read_expr_table_info(line, i, p, string2int_map);
+				ret=read_expr_table_info(line, p, string2int_map);
 				if(ret<0)
 				{
-					fprintf(stderr,"Maat read table info %s line %d error:unknown column.\n",table_info_path,i);
+					fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i);
 					MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
-									"Maat read table info %s line %d error:unknown column.",table_info_path,i);
+									"Maat read table info %s line %d error:illegal column.",table_info_path,i);
 					goto error_jump;
 				}
 				break;
 			case TABLE_TYPE_PLUGIN:
 				p->cb_info=(struct _plugin_table_info*)calloc(sizeof(struct _plugin_table_info),1);
 				p->cb_info->cache_lines=dynamic_array_create(1024,1024);
-				ret=sscanf(not_care,"%d\t%d\t%[0-9,]",&(p->valid_flag_column), &p->rule_tag_column, tmp_str);
+				ret=read_plugin_table_info(line, p);
-				if(ret==0||ret==EOF)
+				if(ret<0)
 				{
-					p->valid_flag_column=-1;
+					fprintf(stderr,"Maat read table info %s line %d error:illegal plugin info.\n",table_info_path,i);
+					MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+									"Maat read table info %s line %d error:illegal plugin info.",table_info_path,i);
+					goto error_jump;
 				}
 				break;
 			case TABLE_TYPE_COMPILE:	
@@ -2997,7 +2930,7 @@ error_out:
 	free(digest_rule);
 	digest_rule=NULL;		
 }
-void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q)
+void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q)
 {
 	if(p==NULL)
 	{
@@ -3008,25 +2941,40 @@ void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbag
 	bag->type=type;
 	bag->create_time=time(NULL);
 	bag->ok_times=0;
+	bag->expire_after=timeout;
 	MESA_lqueue_join_tail(garbage_q,&bag,sizeof(void*));
 	return;
 }
+void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q)
+{
+	garbage_bagging_with_timeout(type, p, -1, garbage_q);
+	return;
+}
 void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
 {
 	MESA_queue_errno_t q_ret=MESA_QUEUE_RET_OK;
 	_maat_garbage_t* bag=NULL;
 	long data_size=0;
 	const long q_cnt=MESA_lqueue_get_count(garbage_q);
-	int i=0,bury_cnt=0;
+	int i=0,bury_cnt=0, ret=0;
 	long long ref_cnt=0;
 	int have_timeout=0;
+	int override_timeout=0;
 	time_t now=time(NULL);
 	for(i=0;i<q_cnt;i++)
 	{
 		data_size=sizeof(void*);
 		q_ret=(MESA_queue_errno_t)MESA_lqueue_get_head(garbage_q,&bag,&data_size);
 		assert(data_size==sizeof(void*)&&q_ret==MESA_QUEUE_RET_OK);
-		if(now-bag->create_time<timeout)
+		if(bag->expire_after<0)
+		{
+			override_timeout=timeout;
+		}
+		else
+		{
+			override_timeout=bag->expire_after;
+		}
+		if(now-bag->create_time<override_timeout)
 		{
 			MESA_lqueue_join_tail(garbage_q,&bag,sizeof(void*));
 			continue;
@@ -3052,7 +3000,6 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
 					MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
 							"scanner %p version %d force destroyed,ref_cnt %lld.",
 							bag->scanner,bag->scanner->version,ref_cnt);
-					
 				}
 				destroy_maat_scanner(bag->scanner);
 				break;
@@ -3062,6 +3009,15 @@ void garbage_bury(MESA_lqueue_head garbage_q,int timeout,void *logger)
 			case GARBAGE_MAP_STR2INT:
 				map_destroy(bag->str2int_map);
 				break;
+			case GARBAGE_FOREIGN_FILE:
+				ret=system_cmd_rm(bag->filename);
+				if(ret==-1)
+				{
+					MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module,
+							"Foreign content file %s remove failed.",
+							bag->filename);
+				}
+				break;
 			default:
 				assert(0);
 		}

src/entry/Maat_utils.cpp

@@ -0,0 +1,186 @@
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <stdlib.h>
+
+#include "Maat_utils.h"
+pid_t gettid()
+{
+	return syscall(SYS_gettid);
+}
+const char*	module_name_str(const char*name)
+{
+	static __thread char module[64];
+	snprintf(module,sizeof(module),"%s(%d)", name, gettid());
+	return module;
+}
+
+int converHextoint(char srctmp)
+{
+	if(isdigit(srctmp))
+	{
+		return srctmp-'0';
+	}
+	else
+	{
+		char temp=toupper(srctmp);
+		temp=temp-'A'+10;
+		return temp;
+	}
+}
+int  hex2bin(char *hex,int hex_len,char *binary,int size)
+{
+	int i=0;
+	int resultlen=0;
+	int high,low;
+	for(i=0;i<hex_len&&size>resultlen; i+=2,resultlen++)
+	{
+		high=converHextoint(hex[i]);
+		low=converHextoint(hex[i+1]);
+		binary[resultlen]=high*16+low;
+	}
+	size=resultlen;
+	binary[resultlen]='\0';
+	return resultlen;
+}
+//functioned as strdup, for dictator compatible.
+char* _maat_strdup(const char* s)
+{
+	char*d=NULL;
+	if(s==NULL)
+	{
+		return NULL;
+	}
+	d=(char*)malloc(strlen(s)+1);
+	memcpy(d,s,strlen(s)+1);
+	return d;
+}
+char* str_tolower(char* string)
+{
+	int i=0;
+	for(i=0;i<(int)strlen(string);i++)
+	{
+		string[i]=(char)tolower(string[i]);
+	}
+	return string;
+}
+char * strchr_esc(char* s,const char delim)
+{
+	char *token;
+	if(s==NULL)
+		return NULL;
+	for(token=s;*token!='\0';token++)
+	{
+		if(*token=='\\')
+		{
+			token++;
+			continue;
+		}
+		if(*token==delim)
+			break;
+	}
+	if (*token == '\0')
+	{ 
+		return NULL;
+	}
+	else
+	{
+		return token;
+	}
+}
+char *strtok_r_esc(char *s, const char delim, char **save_ptr)
+{  
+    char *token;  
+  
+    if (s == NULL) s = *save_ptr;  
+  
+    /* Scan leading delimiters.  */
+	token=strchr_esc(s,delim);
+	if(token==NULL)
+	{
+		*save_ptr=token;
+		return s;
+	}
+    /* Find the end of the token.  */  
+	*token='\0';
+	token++;
+	*save_ptr=token;
+
+    return s;  
+}   
+char *str_unescape_and(char*s)
+{
+	int i=0,j=0;
+	for(i=0,j=0;i<(int)strlen(s);i++)
+	{
+		if(s[i]=='\\'&&s[i+1]=='&')
+		{
+			s[j]='&';
+			i++;
+			j++;
+		}
+		else{
+			s[j]=s[i];
+			j++;
+		}
+	}
+	s[j]='\0';
+	return s;
+}
+char* str_unescape(char* s)
+{
+	int i=0,j=0;
+	int len=strlen(s);
+	for(i=0,j=0;i<len;i++)
+	{
+		if(s[i]=='\\')
+		{
+			switch(s[i+1])
+			{
+				case '&':
+					s[j]='&';
+					break;
+				case 'b':
+					s[j]=' ';//space,0x20;
+					break;
+				case '\\':
+					s[j]='\\';
+					break;
+				default:
+					s[j]=s[i];
+					i--;	//undo the followed i++
+					break;
+			}
+			i++;
+			j++;
+		}
+		else
+		{
+			s[j]=s[i];
+			j++;
+		}
+	}
+	s[j]='\0';
+	return s;
+}
+#define MAX_SYSTEM_CMD_LEN	512
+
+int system_cmd_mkdir(const char* path)
+{
+	char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
+	snprintf(cmd, sizeof(cmd),"mkdir -p %s", path);
+	return  system(cmd) ;
+}
+int system_cmd_mv(const char* src_file,const char*dst_file)
+{
+	char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
+	snprintf(cmd,sizeof(cmd), "mv %s %s", src_file, dst_file);
+	return  system(cmd);
+}
+int system_cmd_rm(const char* src_file)
+{
+	char cmd[MAX_SYSTEM_CMD_LEN] = { 0 };
+	snprintf(cmd,sizeof(cmd), "rm %s -f", src_file);
+	return  system(cmd);
+}
+

src/entry/config_monitor.cpp

@@ -1,6 +1,6 @@
 #include "MESA_handle_logger.h"
 #include "config_monitor.h"
-#include "Maat_rule_internal.h"
+#include "Maat_utils.h"
 #include <dirent.h> 
 #include <stdlib.h>
 #include <stdio.h>
@@ -18,13 +18,7 @@
 #define	MAX_CONFIG_FN_LEN	256
 #define	MAX_CONFIG_LINE		(1024*16)
 
-#ifndef MAX
-#define MAX(a, b)  	(((a) > (b)) ? (a) : (b))
-#endif
 
-#ifndef MIN
-#define MIN(a, b)  	(((a) < (b)) ? (a) : (b))
-#endif
 //#define	USING_DICTATOR	1
 extern "C" void __real_free(void*p);
 struct cm_table_info_t

src/entry/json2iris.cpp

@@ -13,6 +13,8 @@
 #include "hiredis.h"
 #include "map_str2int.h"
 #include "Maat_rule_internal.h"
+#include "Maat_utils.h"
+
 #define maat_json (module_name_str("MAAT_JSON"))
 
 const char* untitled_group_name="Untitled";

src/inc_internal/Maat_rule_internal.h

@@ -43,8 +43,8 @@ typedef int atomic_t;
 #define	MAX_TABLE_LINE_SIZE		(1024*16)
 #define	MAX_EXPR_KEYLEN			1024
 #define	MAX_DISTRICT_LEN			64
-#define	MAX_PLUGING_NUM			32
+#define	MAX_PLUGIN_PER_TABLE		32
-
+#define	MAX_FOREIGN_CLMN_NUM		8
 #define	MAX_SCANNER_HIT_NUM			64
 
 #define	MAX_GROUP_CACHE		128
@@ -210,7 +210,7 @@ struct _callback_plugin
 struct _plugin_table_info
 {
 	int cb_plug_cnt;
-	struct _callback_plugin  cb_plug[MAX_PLUGING_NUM];
+	struct _callback_plugin  cb_plug[MAX_PLUGIN_PER_TABLE];
 	dynamic_array_t *cache_lines;
 	int cache_line_num;
 	int acc_line_num;
@@ -262,6 +262,8 @@ struct _Maat_table_info_t
 	struct _plugin_table_info *cb_info;
 	int valid_flag_column;		//for plugin table
 	int rule_tag_column;		//for plugin table;
+	int foreign_columns[MAX_FOREIGN_CLMN_NUM];			//for plugin table;
+	int n_foreign;
 	int user_region_encoding;	//for compile table, USER_REGION_ENCODE_xx
 //for stat>>>>>>>>
 	unsigned long long udpate_err_cnt;
@@ -298,7 +300,8 @@ enum maat_garbage_type
 	GARBAGE_GROUP_RULE,
 	GARBAGE_COMPILE_RULE,
 	GARBAGE_BOOL_MATCHER,
-	GARBAGE_MAP_STR2INT
+	GARBAGE_MAP_STR2INT,
+	GARBAGE_FOREIGN_FILE
 };
 struct iconv_handle_t
 {
@@ -409,6 +412,8 @@ struct _Maat_feather_t
 
 	struct rule_tag *accept_tags;
 	int n_tags;
+	char foreign_cont_dir[MAX_TABLE_NAME_LEN];
+	int foreign_cont_linger;
 //internal states
 	long long new_version;
 	int active_plugin_table_num;
@@ -441,6 +446,7 @@ struct _maat_garbage_t
 {
 	enum maat_garbage_type type;
 	time_t create_time;
+	int expire_after;
 	int ok_times;
 	union
 	{
@@ -450,8 +456,15 @@ struct _maat_garbage_t
 		void* bool_matcher;
 		void * raw;
 		MESA_htable_handle str2int_map;
+		char* filename;
 	};
 };
+struct foreign_key
+{
+	int column;
+	char* key;
+	char* filename;
+};
 struct serial_rule_t		//rm= Redis Maat
 {
 	enum MAAT_OPERATION op;//0: delete, 1: add.
@@ -460,9 +473,12 @@ struct serial_rule_t		//rm= Redis Maat
 	long long timeout;			// absolute unix time.
 	char table_name[256];
 	char* table_line;
+	int n_foreign;
+	struct foreign_key* f_keys;
 };
 int parse_accept_tag(const char* value, struct rule_tag** result, void* logger);
 void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
+void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
 void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
 void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set);
 int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* table_info_path,int max_thread_num,void* logger);
@@ -471,15 +487,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para);
 void maat_finish_cb(void* u_para);
 void *thread_rule_monitor(void *arg);
 unsigned int make_sub_type(unsigned short table_id,enum MAAT_CHARSET charset,int do_charset_merge);
-inline void ipv6_ntoh(unsigned int *v6_addr)
+
-{
-	unsigned int i=0;
-	for(i=0;i<4;i++)
-	{
-		v6_addr[i]=ntohl(v6_addr[i]);
-	}
-	return;
-}
 void * HASH_fetch_by_id(MESA_htable_handle hash,int id);
 int HASH_add_by_id(MESA_htable_handle hash,int id,void*data);
 int HASH_delete_by_id(MESA_htable_handle hash,int id);
@@ -487,8 +495,7 @@ void maat_read_full_config(_Maat_feather_t* _feather);
 void maat_stat_init(struct _Maat_feather_t* feather);
 void maat_stat_table(struct _Maat_table_info_t* p_table,int scan_len,struct timespec* start, struct timespec* end,int thread_num);
 void maat_stat_output(struct _Maat_feather_t* feather);
-char* _maat_strdup(const char* s);
+
-char* str_unescape(char* s);
 redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...);
 int get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, long long* new_version, struct serial_rule_t** list,int *update_type, void* logger, int cumulative_off);
 int get_maat_redis_value(redisContext *c,struct serial_rule_t* rule_list,int rule_num,void* logger,int print_process);
@@ -505,8 +512,6 @@ void redis_monitor_traverse(long        long version,redisContext *c,
 		const unsigned char* dec_key,
 		_Maat_feather_t* feather);
 
-const char*	module_name_str(const char*name);
-#define maat_module	(module_name_str("MAAT_Frame"))
 
 
 #endif

src/inc_internal/Maat_utils.h

@@ -0,0 +1,43 @@
+#pragma once
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+
+#define ALLOC(type, number) ((type *)calloc(sizeof(type), number))
+#define	FREE(p)	{free(*p);*p=NULL;}
+
+#define	ATOMIC_INC(x) 	__atomic_fetch_add(x,1,__ATOMIC_RELAXED)
+#define	ATOMIC_READ(x)	__atomic_fetch_add(x,0,__ATOMIC_RELAXED)
+
+
+#ifndef MAX
+#define MAX(a, b)  	(((a) > (b)) ? (a) : (b))
+#endif
+
+#ifndef MIN
+#define MIN(a, b)  	(((a) < (b)) ? (a) : (b))
+#endif
+
+const char*	module_name_str(const char*name);
+#define maat_module	(module_name_str("MAAT_Frame"))
+
+char* _maat_strdup(const char* s);
+char* str_unescape(char* s);
+inline void ipv6_ntoh(unsigned int *v6_addr)
+{
+	unsigned int i=0;
+	for(i=0;i<4;i++)
+	{
+		v6_addr[i]=ntohl(v6_addr[i]);
+	}
+	return;
+}
+int  hex2bin(char *hex,int hex_len,char *binary,int size);
+char* str_tolower(char* string);
+char *strtok_r_esc(char *s, const char delim, char **save_ptr);
+char *str_unescape_and(char*s);
+char* str_unescape(char* s);
+pid_t gettid(void);
+int system_cmd_mkdir(const char* path);
+int system_cmd_rm(const char* src_file);
+

test/table_info.conf

@@ -7,10 +7,10 @@
 #do_merege [yes/no]
 #cross cache [number]
 #quick mode [quickon/quickoff], default [quickoff]
-#For ip/intval/digest/compile/group
+#For ip/intval/digest/compile/group table.
 #id	name	type
 #
-#For plugin table
+#For plugin table. The first column's id is 1. 0 as not speicified.
 #id	name	type	valid_column tag_column
 #
 #For expr/expr_plus Table
@@ -27,4 +27,5 @@
 8	HTTP_REGION	expr_plus	GBK	GBK	no	0
 9	SIM_URL	similar	--
 10	IMAGE_FP	expr	UTF8	UTF8	yes	128	quickoff
-11	TEST_EFFECTIVE_RANGE_TABLE	plugin	4	5	--
+11	TEST_EFFECTIVE_RANGE_TABLE	plugin	valid_pos=4;tag_pos=5;	--
+12	TEST_EXETERN_LINK	plugin valid_pos=3;foreign_pos=6,7;