1、增加Maat_command_raw_set_xx系列函数，可以操作sub-group、分组复用，增加sub-group增删的测试用例。 2、fix #13。

inc/Maat_command.h

@@ -10,7 +10,11 @@ enum MAAT_OPERATION
 	MAAT_OP_ADD,
 	MAAT_OP_RENEW_TIMEOUT	//Rule expire time is changed to now+cmd->expire_after
 };
- 
+enum MAAT_GROUP_RELATION
+{
+	PARENT_TYPE_COMPILE=0,
+	PARENT_TYPE_GROUP
+};
 enum MAAT_REGION_TYPE
 {
 	REGION_EXPR,
@@ -104,8 +108,12 @@ struct Maat_region_t
 };
 struct Maat_group_t
 {
-	int region_num;
+	const char* table_name;
 	int group_id;	//If MAAT_OPT_CMD_AUTO_NUMBERING==1, maat will assigned one. Or users must assign a unique number.
+	int parent_id;
+	int not_flag;
+	enum MAAT_GROUP_RELATION parent_type;
+	int region_num;
 	struct Maat_region_t *regions;	
 };
 struct Maat_cmd_t
@@ -170,5 +178,14 @@ void Maat_cmd_key_free(struct Maat_cmd_key**keys, int number);
 int Maat_cmd_key_select(Maat_feather_t feather, int label_id, struct Maat_cmd_key** keys);
 int Maat_cmd_select(Maat_feather_t feather, int label_id, int * output_ids, unsigned int size);
 int Maat_cmd_flushDB(Maat_feather_t feather);
+
+int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num);
+int Maat_command_raw_set_region(Maat_feather_t feather,  enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id);
+int Maat_command_raw_set_group(Maat_feather_t feather,  enum MAAT_OPERATION op, const struct Maat_group_t* group);
+
+int Maat_cmd_get_new_group_id(Maat_feather_t feather);
+int Maat_cmd_get_new_region_id(Maat_feather_t feather);
+
+
 #endif
 

src/entry/Maat_api.cpp

@@ -181,7 +181,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
 		{
 			continue;
 		}
-		if(group_rule->compile_shortcut!=NULL&&group_rule->parent_ref_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
+		if(group_rule->compile_shortcut!=NULL&&group_rule->ref_by_parent_cnt==1&&shortcut_avilable_cnt<MAX_SCANNER_HIT_NUM)
 		{			
 			relation_arrary[shortcut_avilable_cnt]=(struct Maat_compile_group_relation*)(group_rule->compile_shortcut);
 			shortcut_avilable_cnt++;
@@ -662,12 +662,12 @@ int Maat_set_feather_opt(Maat_feather_t feather,enum MAAT_INIT_OPT type,const vo
 			break;
 		case MAAT_OPT_JSON_FILE_PATH:
 			assert(_feather->input_mode==SOURCE_NONE);
-			ret=json2iris((const char*)value
+			ret=json2iris((const char*)value,
-				,_feather->compile_tn,_feather->group_tn
+					_feather->compile_tn,_feather->group_tn,
-				,NULL
+					NULL,
-				,_feather->json_ctx.iris_file
+					_feather->json_ctx.iris_file,
-				,sizeof(_feather->json_ctx.iris_file)
+					sizeof(_feather->json_ctx.iris_file),
-				,_feather->logger);
+					_feather->logger);
 			if(ret<0)
 			{
 				return -1;

src/entry/Maat_command.cpp

@@ -242,13 +242,39 @@ int invalidate_line(char* line, enum MAAT_TABLE_TYPE type,int valid_column_seq)
 	line[i]='0';
 	return 0;
 }
-void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int size)
+void serialize_group(const struct Maat_group_t* p_group, enum MAAT_OPERATION op, char* buff, size_t sz)
 {
-	UNUSED int ret=0;
+	if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
+	snprintf(buff, sz, "%d\t%d\t%d\t%d\t%d", p_group->group_id, 
+									p_group->parent_id,
+									op,
+									p_group->not_flag,
+									p_group->parent_type);
+	return;
+}
+void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz)
+{
+	if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
+	const char* service_define=huge_service_defined?huge_service_defined:(strlen(p_m_rule->service_defined)?p_m_rule->service_defined:"null");
+
+	snprintf(buff, sz,"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t%d\t%d", 
+		p_m_rule->config_id,
+		p_m_rule->service_id,
+		p_m_rule->action,
+		p_m_rule->do_blacklist,
+		p_m_rule->do_log,
+		service_define,
+		op,
+		group_num);
+	return;
+}
+void serialize_region(const struct Maat_region_t* p, int group_id, char* buff, size_t sz)
+{
+	UNUSED size_t ret=0;
 	switch(p->region_type)
 	{
 		case REGION_IP:
-			ret=snprintf(buff,size,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1"
+			ret=snprintf(buff,sz,"%d\t%d\t%d\t%s\t%s\t%hu\t%hu\t%s\t%s\t%hu\t%hu\t%d\t%d\t1"
 					,p->region_id
 					,group_id
 					,p->ip_rule.addr_type
@@ -266,7 +292,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
 		case REGION_EXPR:
 			if(p->expr_rule.district==NULL)
 			{
-				ret=snprintf(buff,size,"%d\t%d\t%s\t%d\t%d\t%d\t1"
+				ret=snprintf(buff,sz,"%d\t%d\t%s\t%d\t%d\t%d\t1"
 						,p->region_id
 						,group_id
 						,p->expr_rule.keywords
@@ -276,7 +302,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
 			}
 			else	//expr_plus
 			{
-				ret=snprintf(buff,size,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1"
+				ret=snprintf(buff,sz,"%d\t%d\t%s\t%s\t%d\t%d\t%d\t1"
 						,p->region_id
 						,group_id
 						,p->expr_rule.keywords
@@ -287,14 +313,14 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
 			}
 			break;
 		case REGION_INTERVAL:
-			ret=snprintf(buff,size,"%d\t%d\t%u\t%u\t1"
+			ret=snprintf(buff,sz,"%d\t%d\t%u\t%u\t1"
 					,p->region_id
 					,group_id
 					,p->interval_rule.low_boundary
 					,p->interval_rule.up_boundary);
 			break;
 		case REGION_DIGEST:
-			ret=snprintf(buff,size,"%d\t%d\t%llu\t%s\t%hd\t1"
+			ret=snprintf(buff,sz,"%d\t%d\t%llu\t%s\t%hd\t1"
 					,p->region_id
 					,group_id
 					,p->digest_rule.orgin_len
@@ -302,7 +328,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
 					,p->digest_rule.confidence_degree);
 			break;
 		case REGION_SIMILARITY:
-			ret=snprintf(buff,size,"%d\t%d\t%s\t%hd\t1"
+			ret=snprintf(buff,sz,"%d\t%d\t%s\t%hd\t1"
 					,p->region_id
 					,group_id
 					,p->similarity_rule.target
@@ -311,7 +337,7 @@ void serialize_region(const struct Maat_region_t* p,int group_id, char* buff,int
 		default:
 			assert(0);
 	}
-	assert(ret<size);
+	assert(ret<sz);
 	return;
 }
 void empty_serial_rules(struct serial_rule_t* rule)
@@ -332,15 +358,15 @@ void empty_serial_rules(struct serial_rule_t* rule)
 	memset(rule,0,sizeof(struct serial_rule_t));
 	return;
 }
-void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout)
+void set_serial_rule(struct serial_rule_t* rule, enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout)
 {
+	memset(rule, 0, sizeof(struct serial_rule_t));
 	rule->op=op;
 	rule->rule_id=rule_id;
 	rule->label_id=label_id;
 	rule->timeout=timeout;
 	assert(strlen(table_name)<sizeof(rule->table_name));
-	memset(rule->table_name, 0, sizeof(rule->table_name));	
+	strncpy(rule->table_name, table_name, sizeof(rule->table_name));
-	memcpy(rule->table_name,table_name,strlen(table_name));
 	if(line!=NULL)
 	{
 		rule->table_line=_maat_strdup(line);
@@ -875,7 +901,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
 		group_cmd=&(cmd->groups[grp_idx]);
 		group_cmd->group_id=group_inner->group_id;
 
-		if(group_inner->parent_ref_cnt>1)
+		if(group_inner->ref_by_parent_cnt>1)
 		{
 			continue;
 		}
@@ -900,7 +926,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
 	return 0;
 }
 
-int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,struct serial_rule_t* list, int size)
+int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd, struct serial_rule_t* list, int size)
 {
 	struct Maat_group_t* p_group=NULL;
 	struct Maat_region_t* p_region=NULL;
@@ -918,19 +944,12 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st
 	}
 	if(op==MAAT_OP_ADD)
 	{
-		snprintf(line,sizeof(line),"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t1\t%d",p_m_rule->config_id
+		serialize_compile(p_m_rule, _cmd->huge_service_defined, cmd->group_num, MAAT_OP_ADD, line, sizeof(line));
-				,p_m_rule->service_id
+		set_serial_rule(list+rule_num, MAAT_OP_ADD, cmd->compile.config_id, cmd->label_id, feather->compile_tn, line, timeout);
-				,p_m_rule->action
-				,p_m_rule->do_blacklist
-				,p_m_rule->do_log
-				,(_cmd->huge_service_defined!=NULL)?(_cmd->huge_service_defined):(p_m_rule->service_defined)
-				,cmd->group_num);
-		set_serial_rule(list+rule_num,MAAT_OP_ADD,cmd->compile.config_id,cmd->label_id,feather->compile_tn,line,timeout);
-
 	}
 	else
 	{
-		set_serial_rule(list+rule_num,op,cmd->compile.config_id,cmd->label_id,feather->compile_tn,NULL,timeout);
+		set_serial_rule(list+rule_num, op, cmd->compile.config_id, cmd->label_id, feather->compile_tn, NULL, timeout);
 	}
 	rule_num++;
 	for(i=0;i<cmd->group_num;i++)
@@ -943,9 +962,10 @@ int build_serial_rule(_Maat_feather_t *feather,struct _Maat_cmd_inner_t* _cmd,st
 				p_group->group_id=feather->base_grp_seq;
 				feather->base_grp_seq++;
 			}
-			snprintf(line,sizeof(line),"%d\t%d\t1",p_group->group_id
+			p_group->parent_id=p_m_rule->config_id;
-					,p_m_rule->config_id);
+			p_group->parent_type=PARENT_TYPE_COMPILE;
-			set_serial_rule(list+rule_num,MAAT_OP_ADD,p_group->group_id,0,feather->group_tn,line,timeout);
+			serialize_group(p_group, MAAT_OP_ADD, line, sizeof(line));
+			set_serial_rule(list+rule_num, MAAT_OP_ADD, p_group->group_id, 0, feather->group_tn, line, timeout);
 		}
 		else
 		{
@@ -2374,7 +2394,7 @@ int Maat_cmd_commit(Maat_feather_t feather)
 	_feather->server_time=redis_server_time(write_ctx);
 
 	if(_feather->AUTO_NUMBERING_ON==1)
-	{
+	{
 		data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_region_id_var, new_region_num); 
 		if(data_reply->type!=REDIS_REPLY_INTEGER)
 		{
@@ -2384,7 +2404,7 @@ int Maat_cmd_commit(Maat_feather_t feather)
 		}
 		_feather->base_rgn_seq=data_reply->integer-new_region_num;
 		freeReplyObject(data_reply);
-
+
 		data_reply=_wrap_redisCommand(write_ctx,"INCRBY %s %d", mr_group_id_var, new_group_num); 
 		if(data_reply->type!=REDIS_REPLY_INTEGER)
 		{
@@ -2448,6 +2468,19 @@ long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment)
 	}
 	freeReplyObject(data_reply);
 	return result;		
+}
+int Maat_cmd_get_new_group_id(Maat_feather_t feather)
+{
+	int group_id=0;
+	group_id=(int) Maat_cmd_incrby(feather, mr_group_id_var, 1);
+	return group_id;
+}
+int Maat_cmd_get_new_region_id(Maat_feather_t feather)
+{
+	int region_id=0;
+	region_id=(int) Maat_cmd_incrby(feather, mr_region_id_var, 1);
+	return region_id;
+}
 
 void Maat_cmd_key_free(struct Maat_cmd_key**keys, int size)
 {
@@ -2550,9 +2583,9 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
 	redisAppendCommand(ctx,"SET MAAT_VERSION %lld",maat_redis_version);
 	append_cmd_cnt++;
 	redisAppendCommand(ctx,"SET MAAT_PRE_VER %lld",maat_redis_version);
-	append_cmd_cnt++;
+	append_cmd_cnt++;
 	redisAppendCommand(ctx,"SET %s 1", mr_region_id_var);
-	append_cmd_cnt++;
+	append_cmd_cnt++;
 	redisAppendCommand(ctx,"SET %s 1", mr_group_id_var);
 	append_cmd_cnt++;
 	redisAppendCommand(ctx,"EXEC");
@@ -2574,6 +2607,41 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
 				);
 	}
 	return redis_transaction_success;
+}
+static int _Maat_command_set_one_line(struct _Maat_feather_t* _feather, enum MAAT_OPERATION op, int id, const char* table_name, const char* line)
+{
+	redisContext* write_ctx=get_redis_ctx_for_write(_feather);
+	_feather->server_time=redis_server_time(write_ctx);
+	struct serial_rule_t s_rule;
+	set_serial_rule(&s_rule, op, id, 0, table_name, line, 0);
+	int transaction_success=0;
+	transaction_success=exec_serial_rule(write_ctx, &s_rule, 1, _feather->server_time, _feather->logger);
+	empty_serial_rules(&s_rule);
+	return transaction_success;
+}
+int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int group_num)
+{
+	struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
+	char line[MAX_TABLE_LINE_SIZE];
+	serialize_compile(compile, huge_service_defined, group_num, op, line, sizeof(line));
+	int ret=_Maat_command_set_one_line(_feather, op, compile->config_id, table_name, line);
+	return ret;
+}
+int Maat_command_raw_set_region(Maat_feather_t feather,  enum MAAT_OPERATION op, const struct Maat_region_t* region, int group_id)
+{
+	struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
+	char line[MAX_TABLE_LINE_SIZE];
+	serialize_region(region, group_id, line, sizeof(line));
+	int ret=_Maat_command_set_one_line(_feather, op, region->region_id, region->table_name, line);
+	return ret;
+}
+int Maat_command_raw_set_group(Maat_feather_t feather,  enum MAAT_OPERATION op, const struct Maat_group_t* group)
+{
+	struct _Maat_feather_t* _feather=(struct _Maat_feather_t*)feather;
+	char line[MAX_TABLE_LINE_SIZE];
+	serialize_group(group, op, line, sizeof(line));
+	int ret=_Maat_command_set_one_line(_feather, op, group->group_id*1024*1204+group->parent_id, group->table_name, line);
+	return ret;
 }
 int Maat_cmd_flushDB(Maat_feather_t feather)
 {

src/entry/Maat_rule.cpp

@@ -814,14 +814,14 @@ int read_table_description(struct Maat_table_desc** p_table_info,int num,const c
 			conj_table=p_table_info[p->table_id];
 			if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM)
 			{
-				MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module,
+				MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
 								"Maat read table info %s line %d error:reach tableid %d conjunction upper limit."
 								,table_info_path,i,p->table_id);
 				goto invalid_table;
 			}
 			memcpy(conj_table->table_name[conj_table->conj_cnt],p->table_name[0],MAX_TABLE_NAME_LEN);
 			conj_table->conj_cnt++;
-			MESA_handle_runtime_log(logger, RLOG_LV_INFO,maat_module,
+			MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module,
 								"Maat read table info %s:%d:conjunction %s with %s (id=%d,total=%d)."
 								,table_info_path,i,p->table_name[0]
 								,conj_table->table_name[0],conj_table->table_id,conj_table->conj_cnt);
@@ -847,7 +847,7 @@ struct Maat_group_inner* create_group_rule(int group_id, int table_id, struct Ma
 	group->group_id=group_id;
 	group->region_cnt=0;
 	group->region_boundary=0;
-	group->parent_ref_cnt=0;
+	group->ref_by_parent_cnt=0;
 	group->regions=dynamic_array_create(1,8);
 	group->compile_shortcut=NULL;
 	group->table_id=table_id;
@@ -868,7 +868,7 @@ void _destroy_group_rule(struct Maat_group_inner* group)
 	group->region_cnt=0;
 	group->region_boundary=0;
 	group->regions=NULL;
-	group->parent_ref_cnt=0;
+	group->ref_by_parent_cnt=0;
 	group->group_id=-1;
 	group->table_id=-1;
 	free(group->group_name);
@@ -879,8 +879,18 @@ void _destroy_group_rule(struct Maat_group_inner* group)
 	free(group);
 
 }
+size_t print_igraph_vector(igraph_vector_t *v, char* buff, size_t sz) {
+  long int i;
+  int printed=0;
+  for (i=0; i<igraph_vector_size(v); i++) {
+    printed+=snprintf(buff+printed, sz-printed, " %li", (long int) VECTOR(*v)[i]);
+  }
+  return printed;
+}
+
 #define	DESTROY_GROUP_BY_REGION	0
 #define	DESTROY_GROUP_BY_PARENT	1
+#define	DESTROY_GROUP_BY_CHILD	2
 void destroy_group_rule(struct Maat_group_inner* group_rule, int by_whom, struct Maat_scanner_t* scanner)
 {
 	switch(by_whom)
@@ -888,16 +898,32 @@ void destroy_group_rule(struct Maat_group_inner* group_rule, int by_whom, struct
 		case	DESTROY_GROUP_BY_REGION:
 			break;
 		case	DESTROY_GROUP_BY_PARENT:
-			group_rule->parent_ref_cnt--;
+			group_rule->ref_by_parent_cnt--;
+			break;
+		case	DESTROY_GROUP_BY_CHILD:
+			group_rule->ref_by_children_cnt--;
 			break;
 		default:
 			assert(0);
 			break;
 	}
-	if(group_rule->parent_ref_cnt==0&&group_rule->region_cnt==0)
+	igraph_vector_t v; 
+	char buff[4096];
+	if(group_rule->ref_by_parent_cnt==0&&group_rule->ref_by_children_cnt==0&&group_rule->region_cnt==0)
 	{
 		HASH_delete_by_id(scanner->group_hash, group_rule->group_id);
-		HASH_delete_by_id(scanner->vertex_id2group, group_rule->vertex_id);		
+		HASH_delete_by_id(scanner->vertex_id2group, group_rule->vertex_id);	
+		igraph_vector_init(&v, 8);
+		igraph_neighbors(&scanner->group_graph, &v, group_rule->vertex_id, IGRAPH_ALL);
+		if(igraph_vector_size(&v)>0)
+		{
+			print_igraph_vector(&v, buff, sizeof(buff));
+			MESA_handle_runtime_log(scanner->logger_ref, RLOG_LV_FATAL, maat_module,
+							"Del group %d exception, still reached by %s.",
+							group_rule->vertex_id, buff);
+			assert(0);
+		}
+		igraph_vector_destroy(&v);
 		//Calling _destroy_group_rule on garbage collection to free memory.
 		garbage_bagging(GARBAGE_GROUP_RULE, group_rule, scanner->tomb_ref);
 
@@ -1413,6 +1439,7 @@ struct Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t
 		rulescan_set_param(scanner->region,RULESCAN_REGEX_GROUP,NULL,0);
 	}
 	scanner->tomb_ref=feather->garbage_q;
+	scanner->logger_ref=feather->logger;
 	scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num);
 
 	for(i=0;i<MAX_TABLE_NUM;i++)
@@ -1859,7 +1886,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa
 	}
 	relation->group_cnt++;
 	//member group->compile_shortcut may set to NULL and compile rule pointer repeatly,until rule build finish.
-	if(a_rule_group->parent_ref_cnt==1&&relation->group_cnt==1)
+	if(a_rule_group->ref_by_parent_cnt==1&&relation->group_cnt==1)
 	{
 		a_rule_group->compile_shortcut=relation;
 	}
@@ -1867,7 +1894,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa
 	{
 		a_rule_group->compile_shortcut=NULL;
 	}
-	a_rule_group->parent_ref_cnt++;
+	a_rule_group->ref_by_parent_cnt++;
 	//update group's shortcut when compile has more than one group.
 	if(relation->group_cnt!=1)
 	{
@@ -1900,7 +1927,7 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati
 		}
 		if(group_rule->group_id==group_id)
 		{
-			group_rule->parent_ref_cnt--;
+			group_rule->ref_by_parent_cnt--;
 			dynamic_array_write(relation->groups,i,NULL);
 			if(relation->not_flag[i]==1)
 			{
@@ -2387,6 +2414,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
 	struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
 	struct Maat_compile_group_relation*compile_rule=NULL;
 	int ret=0;
+	igraph_integer_t edge_id;
 	group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
 	if(group_rule==NULL)
 	{
@@ -2400,7 +2428,19 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
 		{
 			parent_group=create_group_rule(db_group_rule->parent_id, table->table_id, scanner);
 		}
-		group_rule->parent_ref_cnt++;		
+		group_rule->ref_by_parent_cnt++;
+		parent_group->ref_by_children_cnt++;
+		ret=igraph_get_eid(&scanner->group_graph, &edge_id, group_rule->vertex_id, parent_group->vertex_id, IGRAPH_DIRECTED, /*error*/ 0);
+		if(edge_id>0)//if the edge was not found and error is false, then -1 will be assigned to eid.
+		{
+			MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
+								"update error, add sub group: %s %d to group %d error, sub group exist.",
+																		table->table_name[table->updating_name],
+																		db_group_rule->group_id,
+																		db_group_rule->parent_id);
+			return -1;
+		}
+		//igraph allow add multiple edges between two vetex, igraph_delete_edges removes one edge per call.
 		igraph_add_edge(&scanner->group_graph, group_rule->vertex_id, parent_group->vertex_id);
 	}
 	else
@@ -2422,7 +2462,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
 			return -1;
 		}
 	}
-	return 0;
+	return 1;
 }
 int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner_t *scanner, void* logger)
 {
@@ -2430,6 +2470,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
 	struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
 	igraph_es_t es;
 	int ret=0;
+	igraph_integer_t edge_num_before=0, edge_num_after=0;
 	
 	
 	if(db_group_rule->parent_type==PARENT_TYPE_GROUP)
@@ -2454,15 +2495,21 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
 																		db_group_rule->parent_id);
 			return 0;
 		}
+		edge_num_before=igraph_ecount(&scanner->group_graph);
 		// The edges between the given pairs of vertices will be included in the edge selection.
 		//The vertex pairs must be given as the arguments of the function call, the third argument 
 		//is the first vertex of the first edge, the fourth argument is the second vertex of the 
 		//first edge, the fifth is the first vertex of the second edge and so on. The last element
 		//of the argument list must be -1 to denote the end of the argument list.
 		//https://igraph.org/c/doc/igraph-Iterators.html#igraph_es_pairs_small
-		igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id, -1);
+		ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, group_rule->vertex_id, parent_group->vertex_id, -1);
+		assert(ret==IGRAPH_SUCCESS);
+		// ignore no such edge to abort().
+		igraph_set_error_handler(igraph_error_handler_ignore);
 		ret=igraph_delete_edges(&scanner->group_graph, es);
-		if(ret!=IGRAPH_SUCCESS)
+		edge_num_after=igraph_ecount(&scanner->group_graph);
+		
+		if(ret!=IGRAPH_SUCCESS||edge_num_before-edge_num_after!=1)
 		{
 			MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module ,
 								"update error, delete %s group %d from parent group %d error, not such relation before.",
@@ -2471,6 +2518,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
 																		db_group_rule->parent_id);
 		}
 		igraph_es_destroy(&es);
+		destroy_group_rule(parent_group, DESTROY_GROUP_BY_CHILD, scanner);
 	}
 	else
 	{
@@ -2478,7 +2526,7 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
 		if(relation==NULL)
 		{
 			MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
-								"update error, delete %s group %d error : compile id %d does not exisit.",
+								"update error, delete %s group %d form compile %d error, compile does not exist.",
 																		table->table_name[table->updating_name],
 																		db_group_rule->group_id,
 																		db_group_rule->parent_id);
@@ -2488,13 +2536,13 @@ int del_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro
 		if(group_rule==NULL)
 		{
 			MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
-								"update error, delete %s group rule error : group id %d not in compile id %d."
+								"update error, delete %s group %d from compile %d error, target group does not in compile.",
-																		,table->table_name[table->updating_name]
+																		table->table_name[table->updating_name],
-																		,db_group_rule->group_id
+																		db_group_rule->group_id,
-																		,db_group_rule->parent_id);
+																		db_group_rule->parent_id);
 			return 0;
 		}
-		if(relation->group_cnt==0)
+		if(relation->group_cnt==0 && relation->compile==NULL)
 		{
 			destroy_compile_group_relation(relation, scanner);
 		}
@@ -2531,7 +2579,7 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_
 	if(cg_relation==NULL || cg_relation->compile==NULL)
 	{
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
-							"update error, delete %s compile rule error : config id %d does not exist."
+							"update error, delete %s compile rule error : compile id %d does not exist."
 																	,table->table_name[table->updating_name]
 																	,compile_id);
 		return -1;
@@ -2542,7 +2590,7 @@ int del_compile_rule(struct Maat_table_desc* table, int compile_id, struct Maat_
 	cg_relation->compile=NULL;
 	pthread_rwlock_unlock(&(cg_relation->rwlock));
 
-	if(cg_relation->group_cnt==0)
+	if(cg_relation->group_cnt==0&&cg_relation->compile==NULL)
 	{
 		destroy_compile_group_relation(cg_relation, scanner);
 	}
@@ -3880,10 +3928,11 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
 	{
 		scanner=feather->scanner;
 	}
+	MESA_handle_runtime_log(feather->logger, RLOG_LV_DEBUG, maat_module, "Maat table %s input: %s", table_name, line);
 	ret=map_str2int(feather->map_tablename2id,table_name,&table_id);
 	if(ret<0)
 	{
-		MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module ,"update warning,unknown table name %s",table_name);
+		MESA_handle_runtime_log(feather->logger, RLOG_LV_INFO, maat_module ,"update warning, unknown table name %s",table_name);
 		return -1;	
 	}
 	p_table=feather->p_table_info[table_id];

src/entry/json2iris.cpp

@@ -83,7 +83,7 @@ static int get_group_seq(struct iris_description_t* iris_cfg)
 	}
 	else
 	{
-		data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_GROUP 1");
+		data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_group_id_var);
 		sequence=(int)data_reply->integer-1;
 		freeReplyObject(data_reply);
 	}
@@ -100,7 +100,7 @@ static int get_region_seq(struct iris_description_t* iris_cfg)
 	}
 	else
 	{
-		data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY SEQUENCE_REGION 1");
+		data_reply=_wrap_redisCommand(iris_cfg->redis_write_ctx,"INCRBY %s 1", mr_region_id_var);
 		sequence=(int)data_reply->integer-1;
 		freeReplyObject(data_reply);
 	}
@@ -1098,6 +1098,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
 	}
 	return 0;
 }
+// redis_write_ctx is used by maat_redis_tool to write json to redis.
 int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,redisContext *redis_write_ctx,char* iris_dir_buf,int buf_len,void* logger)
 {
 	FILE* json_fp=NULL;

src/inc_internal/Maat_rule_internal.h

@@ -23,6 +23,8 @@
 extern const char *maat_module;
 
 
+#define mr_region_id_var	"SEQUENCE_REGION"
+#define mr_group_id_var		"SEQUENCE_GROUP"
 
 
 #define	MAX_TABLE_NUM			256
@@ -104,8 +106,7 @@ struct Maat_compile_rule
 	const struct Maat_table_desc* ref_table;
 	MAAT_RULE_EX_DATA* ads;
 };
-#define PARENT_TYPE_COMPILE	0
+
-#define PARENT_TYPE_GROUP	1
 struct db_group_rule_t
 {
 	int group_id;
@@ -140,7 +141,8 @@ struct Maat_group_inner
 	int table_id;
 	int region_boundary;
 	int region_cnt;
-	int parent_ref_cnt;
+	int ref_by_parent_cnt;
+	int ref_by_children_cnt;
 	char* group_name;
 	int has_compile_neighbors;
 	int vertex_id;
@@ -303,6 +305,7 @@ struct Maat_scanner_t
 	MESA_lqueue_head region_update_q;
 	struct bool_matcher * bool_matcher_expr_compiler;
 	scan_result_t *region_rslt_buff;
+	void* logger_ref;
 	MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q
 
 	int max_thread_num;

test/CMakeLists.txt

@@ -1,6 +1,9 @@
 add_executable(maat_demo maat_demo.cpp)
 target_link_libraries(maat_demo maat_frame_shared)
 
+add_executable(test_igraph test_igraph.cpp)
+target_link_libraries(test_igraph igraph-static)
+
 add_executable(test_maatframe test_maatframe.cpp)
 target_link_libraries(test_maatframe maat_frame_shared gtest)
 

test/test_igraph.cpp

@@ -0,0 +1,62 @@
+#include "igraph/igraph.h"
+#include <assert.h>
+
+void print_vector(igraph_vector_t *v, FILE *f) {
+  long int i;
+  for (i=0; i<igraph_vector_size(v); i++) {
+    fprintf(f, " %li", (long int) VECTOR(*v)[i]);
+  }
+  fprintf(f, "\n");
+}
+
+int main() {
+
+	igraph_t g;
+	int ret;
+	ret=igraph_empty(&g, 0, IGRAPH_DIRECTED);
+
+	igraph_es_t es;
+	igraph_integer_t edge_num_before=0, edge_num_after=0;
+	int v[10];
+	int i=0;
+	for(i=0; i<sizeof(v)/sizeof(int); i++)
+	{
+		v[i]=i;
+		igraph_add_vertices(&g, 1, NULL);	//Add 1 vertice.
+	}
+	igraph_add_edge(&g, v[0], v[1]);
+	igraph_add_edge(&g, v[0], v[1]);
+
+	igraph_add_edge(&g, v[2], v[3]);
+
+	int edge_id=0;
+	ret=igraph_get_eid(&g, &edge_id, v[2], v[3], IGRAPH_DIRECTED, 0);
+	assert(edge_id>0);
+
+	ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[0], v[1], -1);
+	assert(ret==IGRAPH_SUCCESS);
+	edge_num_before=igraph_ecount(&g);
+	ret=igraph_delete_edges(&g, es);
+	edge_num_after=igraph_ecount(&g);
+	assert(edge_num_before-edge_num_after==1);
+	assert(ret==IGRAPH_SUCCESS);
+	igraph_es_destroy(&es);
+
+	ret=igraph_es_pairs_small(&es, IGRAPH_DIRECTED, v[3], v[4], -1);
+	assert(ret==IGRAPH_SUCCESS);
+	edge_num_before=igraph_ecount(&g);
+	/* error test, no such edge to delete */
+	igraph_set_error_handler(igraph_error_handler_ignore);
+
+	ret=igraph_delete_edges(&g, es);
+	edge_num_after=igraph_ecount(&g);
+	assert(edge_num_before=edge_num_after);
+	assert(ret!=IGRAPH_SUCCESS);
+	igraph_es_destroy(&es);
+
+
+	igraph_destroy(&g);
+
+  return 0;
+}
+

test/test_maatframe.cpp

@@ -1475,6 +1475,7 @@ protected:
 	static int linger_timeout;
 };
 Maat_feather_t MaatCmdTest::_shared_feather;
+
 void* MaatCmdTest::logger;
 
 int MaatCmdTest::linger_timeout;
@@ -1760,7 +1761,7 @@ TEST_F(MaatCmdTest,  ReturnRuleIDWithDescendingOrder)
 
 
 	int table_id=0;
-	table_id=Maat_table_register(feather,table_name);
+	table_id=Maat_table_register(feather, table_name);
 	ASSERT_GT(table_id, 0);
 
 	int i=0, repeat_times=4;
@@ -1792,8 +1793,121 @@ TEST_F(MaatCmdTest,  ReturnRuleIDWithDescendingOrder)
 	}
 	return;
 }
+#define MaatCmdTest_SubGroup	0
+TEST_F(MaatCmdTest,    SubGroup)
+{
 
+/*  compile1->group1--group2->region1
+                    \
+                     \group3->region2
 
+    compile2->group1
+*/
+	const char* table_name="HTTP_URL";
+	const char* group_table_name="GROUP";
+	const char* compile_table_name="COMPILE";
+	const char* scan_data1="www.v2ex.com/t/573028#程序员的核心竞争力是什么";
+	const char* keyword1="程序员&核心竞争力";
+
+	const char* scan_data2="https://ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770";
+//	const char* keyword2="ask.leju.com/bj/detail/12189672562229248/?bi=tg&type=sina-pc&pos=index-dbtlwzl&wt_campaign=M_5CE750003F393&wt_source=PDPS_514ACACFD9E770";
+	const char* keyword2="ask.leju.com/b&/detail/12189672562229248/?&?bi=tg\\&type=sina-pc\\&&\\&pos=index-dbtlwzl\\&&\\&type=sina-pc\\&pos=index-dbtlwzl\\&";
+	Maat_feather_t feather=MaatCmdTest::_shared_feather;
+	int table_id=0;
+	table_id=Maat_table_register(feather, table_name);
+	struct Maat_rule_t compile1, compile2;
+	struct Maat_group_t group1, group2, group3;
+	struct Maat_region_t region1,region2;
+	memset(&compile1, 0, sizeof(compile1));
+	compile1.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
+	Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile1, compile_table_name, NULL, 1);
+
+	memset(&compile2, 0, sizeof(compile2));
+	compile2.config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
+	Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile2, compile_table_name, NULL, 1);
+
+	
+	memset(&group1, 0, sizeof(group1));
+	group1.group_id=Maat_cmd_get_new_group_id(feather);
+	group1.table_name=group_table_name;
+	group1.parent_id=compile1.config_id;
+	group1.parent_type=PARENT_TYPE_COMPILE;
+	Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
+
+	group1.parent_id=compile2.config_id;
+	Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
+	
+	memset(&group2, 0, sizeof(group2));
+	group2.group_id=Maat_cmd_get_new_group_id(feather);
+	group2.table_name=group_table_name;
+	group2.parent_id=group1.group_id;
+	group2.parent_type=PARENT_TYPE_GROUP;
+	Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group2);
+
+	memset(&region1, 0, sizeof(region1));
+	region1.region_id=Maat_cmd_get_new_region_id(feather);
+	region1.region_type=REGION_EXPR;
+	region1.table_name=table_name;
+	region1.expr_rule.keywords=keyword1;
+	region1.expr_rule.expr_type=EXPR_TYPE_AND;
+	Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region1, group2.group_id);
+	sleep(1);
+	struct Maat_rule_t result[4];
+	memset(&result, 0, sizeof(result));
+	
+	scan_status_t mid=NULL;
+	int ret=0;
+	ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), 
+						result, NULL, 4,
+						&mid, 0);
+	EXPECT_EQ(ret, 2);
+	EXPECT_EQ(result[0].config_id, compile2.config_id);
+	EXPECT_EQ(result[1].config_id, compile1.config_id);
+	Maat_clean_status(&mid);
+
+	Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
+	sleep(1);
+	ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), 
+						result, NULL, 4,
+						&mid, 0);
+	EXPECT_EQ(ret, 1);
+	EXPECT_EQ(result[0].config_id, compile1.config_id);
+	Maat_clean_status(&mid);
+
+	group1.parent_id=compile1.config_id;
+	Maat_command_raw_set_group(feather, MAAT_OP_DEL, &group1);
+	sleep(1);
+	ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1), 
+						result, NULL, 4,
+						&mid, 0);
+	EXPECT_EQ(ret, -2);
+	Maat_clean_status(&mid);
+
+	memset(&group3, 0, sizeof(group3));
+	group3.group_id=Maat_cmd_get_new_group_id(feather);
+	group3.parent_id=group1.group_id;
+	group3.parent_type=PARENT_TYPE_GROUP;
+	group3.table_name=group_table_name;
+	Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group3);
+
+	memset(&region2, 0, sizeof(region2));
+	region2.region_id=Maat_cmd_get_new_region_id(feather);
+	region2.region_type=REGION_EXPR;
+	region2.table_name=table_name;
+	region2.expr_rule.keywords=keyword2;
+	region2.expr_rule.expr_type=EXPR_TYPE_AND;
+	Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region2, group3.group_id);
+
+	Maat_command_raw_set_group(feather, MAAT_OP_ADD, &group1);
+	sleep(1);
+	ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, scan_data2, strlen(scan_data2), 
+						result, NULL, 4,
+						&mid, 0);
+	EXPECT_EQ(ret, 1);
+	EXPECT_EQ(result[0].config_id, compile1.config_id);	
+	Maat_clean_status(&mid);
+
+}
 TEST_F(MaatCmdTest, SetLines)
 {
 	const int TEST_CMD_LINE_NUM=4;