Command增删功能调试通过。

src/entry/Maat_api.cpp

@@ -601,6 +601,10 @@ int Maat_initiate_feather(Maat_feather_t feather)
 	if(strlen(_feather->redis_ip)>0&&_feather->redis_port!=0)
 	{
 		_feather->REDIS_MODE_ON=1;
+		MESA_handle_runtime_log(_feather->logger,RLOG_LV_INFO,maat_module ,
+							"Maat initiate from Redis %s:%hu"
+							,_feather->redis_ip
+							,_feather->redis_port);
 		_feather->redis_read_ctx=redisConnectWithTimeout(_feather->redis_ip,_feather->redis_port,_feather->connect_timeout);
 		if(_feather->redis_read_ctx==NULL)
 		{
@@ -638,10 +642,21 @@ int Maat_initiate_feather(Maat_feather_t feather)
 								_feather->logger);
 	}
 	if(_feather->update_tmp_scanner==NULL)
+	{
+		if(_feather->REDIS_MODE_ON==1)
+		{
+			MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module ,
+							"At initiation: no avilable rule in redis in %s:%hu"
+							,_feather->redis_ip
+							,_feather->redis_port);
+
+		}
+		else
 		{
 			MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_module ,
 							"At initiation: no valid index file in %s",_feather->full_dir);
 		}
+	}
 	_feather->scanner=_feather->update_tmp_scanner;
 	_feather->update_tmp_scanner=NULL;
 	_feather->still_working=1;

src/entry/Maat_command.cpp

@@ -13,7 +13,7 @@ const char* maat_redis_monitor="MAAT_REDIS_MONITOR";
 const char* maat_redis_command="MAAT_REDIS_COMMAND";
 
 const char* rm_key_prefix[2]={"OBSOLETE_RULE","EFFECTIVE_RULE"};
-
+const char* rm_status_key="MAAT_UPDATE_STATUS";
 const int MAAT_REDIS_SYNC_TIME=30*60;
 
 struct serial_rule_t		//rm= Redis Maat
@@ -32,6 +32,39 @@ struct _Maat_cmd_inner_t
 	int region_size[MAX_EXPR_ITEM_NUM];
 	struct _Maat_cmd_inner_t* next;
 };
+int _wrap_redisGetReply(redisContext *c, redisReply **reply)
+{
+	return redisGetReply(c, (void **)reply);
+}
+redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...)
+{
+    va_list ap;
+    void *reply = NULL;
+    va_start(ap,format);
+    reply = redisvCommand(c,format,ap);
+    va_end(ap);
+    return (redisReply *)reply;
+}
+long long read_redis_integer(const redisReply* reply)
+{
+	switch(reply->type)
+	{
+		case REDIS_REPLY_INTEGER:
+			return reply->integer;
+			break;
+		case REDIS_REPLY_ARRAY:
+			assert(reply->element[0]->type==REDIS_REPLY_INTEGER);
+			return reply->element[0]->integer;
+			break;
+		case REDIS_REPLY_STRING:
+			return atoll(reply->str);
+			break;
+		default:
+			assert(0);
+			break;
+	}	
+	return 0;
+}
 enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p)
 {
 	enum MAAT_TABLE_TYPE ret=TABLE_TYPE_IP;
@@ -107,11 +140,12 @@ void invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
 		{
 			j++;
 		}
-		if(j==offset)
+		if(j==offset-1)
 		{
 			break;
 		}
 	}
+	i++;
 	assert(i<strlen(line));
 	assert(line[i]=='1');
 	line[i]='0';
@@ -242,8 +276,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
 			return 0;
 		}	
 	}
-	assert(reply->type==REDIS_REPLY_STRING);
+	version_in_redis=read_redis_integer(reply);
-	version_in_redis=atoll(reply->str);
 	assert(version_in_redis>=version);
 	freeReplyObject(reply);
 	if(version_in_redis==version)
@@ -260,17 +293,17 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
 	
 	//Returns all the elements in the sorted set at key with a score that version < score <= version_in_redis.
 	//The elements are considered to be ordered from low to high scores(version).
-	reply=(redisReply*)redisCommand(c, "ZRANGE MAAT_UPDATE_STATUS (%d %d",version,version_in_redis);
+	reply=(redisReply*)redisCommand(c, "ZRANGEBYSCORE %s (%d %d",rm_status_key,version,version_in_redis);
 	if(reply==NULL)
 	{
 		__redis_strerror_r(errno,err_buff,sizeof(err_buff));
 		MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
-								"GET MAAT_UPDATE_STATUS failed %s.",err_buff);
+								"GET %s failed %s.",rm_status_key,err_buff);
 		return 0;
 	}	
 	assert(reply->type==REDIS_REPLY_ARRAY);
-	tmp_reply=(redisReply*)redisCommand(c, "ZRANK MAAT_UPDATE_STATUS %s",reply->element[0]->str);
+	tmp_reply=(redisReply*)redisCommand(c, "ZSCORE %s %s",rm_status_key,reply->element[0]->str);
-	nearest_rule_version=tmp_reply->integer;
+	nearest_rule_version=read_redis_integer(tmp_reply);
 	freeReplyObject(tmp_reply);
 	tmp_reply=NULL;
 	if(nearest_rule_version!=version+1)
@@ -289,7 +322,7 @@ int get_rm_key_list(unsigned int version,redisContext *c,struct serial_rule_t**
 	for(i=0;i<reply->elements;i++)
 	{
 		assert(reply->element[i]->type==REDIS_REPLY_STRING);
-		ret=sscanf(reply->element[i]->str,"%[^:]:%[^,],%d",op_str,s_rule[i].table_name,&(s_rule[i].rule_id));
+		ret=sscanf(reply->element[i]->str,"%[^,],%[^,],%d",op_str,s_rule[i].table_name,&(s_rule[i].rule_id));
 		assert(ret==3);
 		if(strncmp(op_str,"ADD",strlen("ADD"))==0)
 		{
@@ -328,19 +361,7 @@ FULL_UPDATE:
 	reply=NULL;
 	return i;
 }
-int _wrap_redisGetReply(redisContext *c, redisReply **reply)
+
-{
-	return redisGetReply(c, (void **)reply);
-}
-redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...)
-{
-    va_list ap;
-    void *reply = NULL;
-    va_start(ap,format);
-    reply = redisvCommand(c,format,ap);
-    va_end(ap);
-    return (redisReply *)reply;
-}
 int calculate_serial_rule_num(struct _Maat_cmd_inner_t* _cmd,int * new_region_cnt, int* new_group_cnt)
 {
 	int serial_num=0;
@@ -359,6 +380,9 @@ int calculate_serial_rule_num(struct _Maat_cmd_inner_t* _cmd,int * new_region_cn
 			*new_region_cnt+=cmd->groups[i].region_num;
 			(*new_group_cnt)++;
 		}
+		//for MAAT_OP_DEL, if the group's refcnt>0, group->region_num=0.
+		//so it's OK to add.
+		serial_num+=cmd->groups[i].region_num;
 	}
 	return serial_num;
 }
@@ -449,7 +473,7 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st
 		set_serial_rule(list+rule_num,MAAT_OP_DEL,cmd->compile.config_id,feather->compile_tn,NULL);
 	}
 	rule_num++;
-	for(i=0;cmd->group_num;i++)
+	for(i=0;i<cmd->group_num;i++)
 	{		
 		p_group=&(cmd->groups[i]);
 		if(op==MAAT_OP_ADD)
@@ -489,20 +513,19 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st
 			rule_num++;
 		}
 	}
-	assert(rule_num<size);
+	assert(rule_num<=size);
 	return rule_num;
 }
 int mr_transaction_success(redisReply* data_reply)
 {
-	unsigned int i=0;
+	if(data_reply->type==REDIS_REPLY_NIL)
-	for(i=0;i<data_reply->elements;i++)
-	{
-		if(data_reply->element[i]->type==REDIS_REPLY_NIL)
 	{
 		return 0;
 	}
-	}
+	else
+	{
 		return 1;
+	}
 }
 int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd)
 {
@@ -512,6 +535,18 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd)
 	struct Maat_group_t* p_group=NULL;
 	struct Maat_region_t* p_region=NULL;
 	enum MAAT_TABLE_TYPE table_type;
+	
+	struct _Maat_compile_inner_t *compile_rule=NULL;
+	if(feather->scanner!=NULL)
+	{
+		compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
+		if(compile_rule!=NULL)
+		{
+			MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module 
+								,"Maat rule %d already exisits.",cmd->compile.config_id);
+			return -1;
+		}
+	}
 	for(i=0;i<cmd->group_num;i++)
 	{
 		p_group=&(cmd->groups[i]);
@@ -529,7 +564,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd)
 				return -1;
 			}
 			table_type=type_region2table(p_region);
-			if(table_type!=feather->p_table_info[i]->table_type)
+			if(table_type!=feather->p_table_info[table_id]->table_type)
 			{
 				MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module 
 									,"Table %s not support region type %d of Maat_command_t[%d]->group[%d]->region[%d]."
@@ -539,7 +574,7 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_command_t* cmd)
 				return -1;
 			}
 			free((char*)p_region->table_name);
-			p_region->table_name=_maat_strdup(feather->p_table_info[i]->table_name[0]);
+			p_region->table_name=_maat_strdup(feather->p_table_info[table_id]->table_name[0]);
 		}
 	}
 	return 0;
@@ -671,7 +706,14 @@ struct Maat_command_t* Maat_create_comand(const struct Maat_rule_t* rule, int gr
 	memcpy(&(_cmd->cmd.compile),rule,sizeof(_cmd->cmd.compile));
 	_cmd->ref_cnt=1;
 	_cmd->cmd.group_num=group_num;
+	if(group_num>0)
+	{
 		_cmd->cmd.groups=(struct Maat_group_t*)calloc(sizeof(struct Maat_group_t),group_num);
+	}
+	else
+	{
+		_cmd->cmd.groups=NULL;
+	}
 	for(i=0;i<group_num;i++)
 	{
 		_cmd->cmd.groups[i].regions=(struct Maat_region_t*)calloc(sizeof(struct Maat_region_t),1);
@@ -795,13 +837,21 @@ int Maat_commit_command(Maat_feather_t feather)
 	long long maat_redis_version=0;
 	int new_region_num=0,new_group_num=0;
 	int serial_rule_num=0,serial_rule_idx=0;
+	int redis_transaction_failed=1;
 	struct _Maat_cmd_inner_t* p=NULL,*n=NULL;
 
 	redisContext* ctx=NULL;
 	redisReply* data_reply=NULL;
 
 	struct serial_rule_t* s_rule=NULL;
-
+	if(_feather->REDIS_MODE_ON==0)
+	{
+		return -1;
+	}
+	if(_feather->cmd_q_cnt==0)
+	{
+		return 0;
+	}
 	if(_feather->redis_write_ctx==NULL)
 	{
 		_feather->redis_write_ctx=redisConnectWithTimeout(_feather->redis_ip, _feather->redis_port,_feather->connect_timeout);
@@ -815,11 +865,10 @@ int Maat_commit_command(Maat_feather_t feather)
 		}
 	}
 	ctx=_feather->redis_write_ctx;
-
 	for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++)
 	{
-		p=p->next;
 		serial_rule_num+=calculate_serial_rule_num(p, &new_region_num, &new_group_num);
+		p=p->next;
 	}
 
 	data_reply=_wrap_redisCommand(ctx,"INCRBY SEQUENCE_REGION %d",new_region_num); 
@@ -836,34 +885,36 @@ int Maat_commit_command(Maat_feather_t feather)
 
 	for(i=0,p=_feather->cmd_qhead;i<_feather->cmd_q_cnt;i++)
 	{
-		p=p->next;
 		serial_rule_idx+=build_serial_rule(_feather,p,s_rule, serial_rule_num-serial_rule_idx);
+		p=p->next;
 	}
 	assert(serial_rule_idx==serial_rule_num);
-	
+	redis_transaction_failed=1;
-	while(1)
+	while(redis_transaction_failed)
 	{
 		data_reply=_wrap_redisCommand(ctx, "WATCH MAAT_VERSION");
 		freeReplyObject(data_reply);
 		data_reply=_wrap_redisCommand(ctx, "GET MAAT_VERSION");
-		freeReplyObject(data_reply);
+		maat_redis_version=read_redis_integer(data_reply);
-		maat_redis_version=atoll(data_reply->str);
 		maat_redis_version++;
+		freeReplyObject(data_reply);
   		data_reply=_wrap_redisCommand(ctx,"MULTI");
 		freeReplyObject(data_reply);
+		append_cmd_cnt=0;
 		for(i=0;i<serial_rule_num;i++)
 		{
 			if(s_rule[i].op==MAAT_OP_ADD)
 			{
-				redisAppendCommand(ctx,"SET %s:%s,%d \"%s\"",rm_key_prefix[MAAT_OP_ADD]
+				redisAppendCommand(ctx,"SET %s:%s,%d %s",rm_key_prefix[MAAT_OP_ADD]
 													,s_rule[i].table_name
 													,s_rule[i].rule_id
 													,s_rule[i].table_line);
 				append_cmd_cnt++;
 				//NX: Don't update already exisiting elements. Always add new elements.
-				redisAppendCommand(ctx,"ZADD NX ADD,%s,%d %d",s_rule[i].table_name
+				redisAppendCommand(ctx,"ZADD %s NX %d ADD,%s,%d",rm_status_key
-													,s_rule[i].rule_id
+													,maat_redis_version
-													,maat_redis_version);
+													,s_rule[i].table_name
+													,s_rule[i].rule_id);
 				append_cmd_cnt++;
 			}
 			else
@@ -884,9 +935,10 @@ int Maat_commit_command(Maat_feather_t feather)
 														,MAAT_REDIS_SYNC_TIME);
 				append_cmd_cnt++;
 				//NX: Don't update already exisiting elements. Always add new elements.
-				redisAppendCommand(ctx,"ZADD NX DEL,%s,%d %d",s_rule[i].table_name
+				redisAppendCommand(ctx,"ZADD %s NX %d DEL,%s,%d",rm_status_key
-														,s_rule[i].rule_id
+														,maat_redis_version
-														,maat_redis_version);
+														,s_rule[i].table_name
+														,s_rule[i].rule_id);
 				append_cmd_cnt++;
 			}
 		}
@@ -894,20 +946,20 @@ int Maat_commit_command(Maat_feather_t feather)
 		append_cmd_cnt++;
 		redisAppendCommand(ctx,"EXEC");
 		append_cmd_cnt++;
+		redis_transaction_failed=0;
 		for(i=0;i<append_cmd_cnt;i++)
 		{
 			_wrap_redisGetReply(ctx, &data_reply);
-			if(1==mr_transaction_success(data_reply))
+			if(0==mr_transaction_success(data_reply))
 			{
-				freeReplyObject(data_reply);
+				redis_transaction_failed=1;
-				break;
 			}
-			else
+			freeReplyObject(data_reply);
+		}
+		if(redis_transaction_failed==1)
 		{
 			retry++;
 			assert(retry<5);			
-				freeReplyObject(data_reply);
-			}
 		}
 	}
 	_feather->cmd_acc_num+=_feather->cmd_q_cnt;

src/entry/config_monitor.cpp

@@ -100,7 +100,7 @@ int decrypt_open(FILE* in,const unsigned char* key, const char* algorithm,unsign
 	}
 	if (!EVP_CipherFinal_ex(ctx, *pp_out+buff_offset, &out_blk_len))
 	{
-		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"EVP_CipherFinal_ex failed.");
+		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor,"EVP_CipherFinal_ex failed. Maybe password is wrong?");
 		EVP_CIPHER_CTX_free(ctx);
 		goto error_out;
 	}

test/maat_redis_reset.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-host="127.0.0.1"
-port="6379"
-echo "Reset Redis For Maat Starting"
-redis-cli -h $host -p $port GET MAAT_VERSION
-redis-cli -h $host -p $port FLUSHALL
-redis-cli -h $host -p $port SET MAAT_VERSION 0
-redis-cli -h $host -p $port SET SEQUENCE_REGION 0
-redis-cli -h $host -p $port SET SEQUENCE_GROUP 0
-

test/maat_test.cpp

@@ -471,6 +471,7 @@ int test_command(Maat_feather_t feather)
 	struct Maat_rule_t result;
 	struct Maat_region_t region;
 	int group_num=1,ret=0;
+	memset(&rule,0,sizeof(rule));
 	rule.config_id=201;
 	strcpy(rule.service_defined,"maat_command");
 	//MUST acqire by function, because Maat_command_t has some hidden members.
@@ -479,17 +480,28 @@ int test_command(Maat_feather_t feather)
 	region.region_type=REGION_EXPR;
 	region.table_name=table_name;
 	region.expr_rule.district=NULL;
-	region.expr_rule.keywords="Hiredis&C\bClient";
+	region.expr_rule.keywords="Hiredis&C\\bClient";
 	region.expr_rule.expr_type=EXPR_TYPE_AND;
 	region.expr_rule.match_method=MATCH_METHOD_SUB;
 	region.expr_rule.hex_bin=UNCASE_PLAIN;
 	Maat_cmd_add_region(cmd, 0, &region);
 	//use pipeline model.
-	Maat_append_command(feather, cmd, MAAT_OP_ADD);
+	ret=Maat_append_command(feather, cmd, MAAT_OP_ADD);
+	if(ret<0)
+	{
+		printf("Add Maat command %d failed.\n",rule.config_id);
+		Maat_free_command(cmd);
+		return 0;
+	}
 	//cmd has been saved in feather, so free before commit is allowed.
 	Maat_free_command(cmd);
-	Maat_commit_command(feather);
+	ret=Maat_commit_command(feather);
-	sleep(3);//waiting for commands go into effect
+	if(ret<0)
+	{
+		printf("Commit Maat command %d failed.\n",rule.config_id);
+		return 0;
+	}
+	sleep(1);//waiting for commands go into effect
 	table_id=Maat_table_register(feather,table_name);
 	ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), 
 						&result,NULL, 1,
@@ -497,11 +509,36 @@ int test_command(Maat_feather_t feather)
 
 	if(ret>0&&result.config_id==rule.config_id)
 	{
-		printf("Test Maat command success %s\n",print_maat_result(&result,ret));
+		printf("Test Maat add command Success %s\n",print_maat_result(&result,ret));
 	}
 	else
 	{
-		printf("Test Maat command failed.\n");
+		printf("Test Maat add command failed.\n");
+	}
+	Maat_clean_status(&mid);
+	
+	memset(&rule,0,sizeof(rule));
+	rule.config_id=201;
+	cmd=Maat_create_comand(&rule, 0);
+	ret=Maat_command(feather, cmd, MAAT_OP_DEL);
+	if(ret<0)
+	{
+		printf("Delete Maat command %d failed.\n",rule.config_id);
+		Maat_free_command(cmd);
+		return 0;
+	}
+	Maat_free_command(cmd);
+	sleep(1);//waiting for commands go into effect
+	ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), 
+						&result,NULL, 1,
+						&mid, 0);
+	if(ret>0)
+	{
+		printf("Test Maat delete command failed\n");
+	}
+	else
+	{
+		printf("Test Maat delete command Success.\n");
 	}
 	Maat_clean_status(&mid);
 	return 0;
@@ -518,11 +555,12 @@ int main(int argc,char* argv[])
 	const char* log_file="./test.log";
 	const char* stat_file="./scan_staus.log";
 	const char* decrypt_key="mesa2017wy";
+	int scan_interval_ms=10;
 	const char* redis_ip="127.0.0.1";
 	unsigned short redis_port=6379;
 	int scan_detail=0;
 	scan_status_t mid=NULL;
-	int wait_second=4;
+	int wait_second=400;
 	void *logger=MESA_create_runtime_log_handle(log_file,0);
 
 	feather=Maat_feather(g_iThreadNum, table_info_path, logger);
@@ -544,6 +582,7 @@ int main(int argc,char* argv[])
 	{
 		Maat_set_feather_opt(feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1);
 	}
+	Maat_set_feather_opt(feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms));
 	Maat_set_feather_opt(feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1);
 	Maat_set_feather_opt(feather, MAAT_OPT_STAT_ON, NULL, 0);
 	Maat_set_feather_opt(feather, MAAT_OPT_PERF_ON, NULL, 0);

test/reset_redis4maat.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+host="127.0.0.1"
+port="6379"
+echo "Reseting Redis For Maat..."
+redis-cli -h $host -p $port GET MAAT_VERSION
+redis-cli -h $host -p $port FLUSHALL
+redis-cli -h $host -p $port SET MAAT_VERSION 1
+redis-cli -h $host -p $port SET SEQUENCE_REGION 1
+redis-cli -h $host -p $port SET SEQUENCE_GROUP 1
+

test/table_info.conf

@@ -6,13 +6,13 @@
 #dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/'
 #do_merege yes or no
 #id	name	type	src_charset	dst_charset	do_merge
-0	COMPILE	compile	GBK	GBK	no	0
+0	COMPILE	compile	--
-1	GROUP	group	GBK	GBK	no	0
+1	GROUP	group	--
 2	HTTP_URL	expr	UTF8	GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8	yes	128	quickoff
 2	HTTP_HOST	expr	UTF8	GBK/BIG5/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8	yes	128	quickoff
 3	KEYWORDS_TABLE	expr	UTF8	GBK/BIG5/UNICODE/UTF8/unicode_ascii_esc/unicode_ascii_aligned/unicode_ncr_dec/unicode_ncr_hex	yes	0
-4	IP_CONFIG	ip	GBK	GBK	no	0
+4	IP_CONFIG	ip	--
-5	CONTENT_SIZE	intval	GBK	GBK	no	0
+5	CONTENT_SIZE	intval	--
-6	QD_ENTRY_INFO	plugin	GBK	GBK	no	0
+6	QD_ENTRY_INFO	plugin	4
-7	FILE_DIGEST	digest	GBK	GBK	no	0
+7	FILE_DIGEST	digest	--
 8	HTTP_REGION	expr_plus	GBK	GBK	no	0