通过原有功能的单元测试。

src/entry/Maat_api.cpp

@@ -549,13 +549,12 @@ Maat_feather_t Maat_feather(int max_thread_num,const char* table_info_path,void*
 	feather->logger=logger;
 	feather->scan_thread_num=max_thread_num;
 	
-	feather->garbage_q=MESA_lqueue_create(0,0);
 	feather->effect_interval_ms=60*1000;
 	feather->scan_interval_ms=1*1000;
 	feather->rule_scan_type=2;
 	feather->thread_call_cnt=alignment_int64_array_alloc(max_thread_num);
 	feather->outer_mid_cnt=alignment_int64_array_alloc(max_thread_num);
-	feather-> compile_mid_cnt=alignment_int64_array_alloc(max_thread_num);
+	feather->compile_mid_cnt=alignment_int64_array_alloc(max_thread_num);
 	feather->hit_cnt=alignment_int64_array_alloc(max_thread_num);
 	feather->not_grp_hit_cnt=alignment_int64_array_alloc(max_thread_num);
 	feather->maat_version=0;

src/entry/Maat_command.cpp

@@ -219,7 +219,8 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_
 		case	TABLE_TYPE_EXPR_PLUS:
 			column_seq=8;
 			break;
-		case	TABLE_TYPE_GROUP:
+		case	TABLE_TYPE_GROUP2COMPILE:
+		case	TABLE_TYPE_GROUP2GROUP:
 			column_seq=3;
 			break;
 		default:
@@ -262,12 +263,12 @@ void serialize_group2compile(enum MAAT_OPERATION op, const struct Maat_cmd_group
 								g2c->Nth_clause==0?1:g2c->Nth_clause);
 	return;
 }
-void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int group_num, enum MAAT_OPERATION op, char* buff, size_t sz)
+void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_service_defined, int clause_num, enum MAAT_OPERATION op, char* buff, size_t sz)
 {
 	if(op==MAAT_OP_RENEW_TIMEOUT) op=MAAT_OP_ADD;
 	const char* service_define=huge_service_defined?huge_service_defined:(strlen(p_m_rule->service_defined)?p_m_rule->service_defined:"null");
 
-	snprintf(buff, sz,"%d\t%d\t%hhd\t%hhd\t%hhd\t0\t%s\t%d\t%d", 
+	snprintf(buff, sz, "%d\t%d\t%hhu\t%hhu\t%hhu\t0\t%s\t%d\t%d", 
 		p_m_rule->config_id,
 		p_m_rule->service_id,
 		p_m_rule->action,
@@ -275,7 +276,7 @@ void serialize_compile(const struct Maat_rule_t* p_m_rule, const char* huge_serv
 		p_m_rule->do_log,
 		service_define,
 		op,
-		group_num);
+		clause_num);
 	return;
 }
 void serialize_region(const struct Maat_cmd_region* p, int group_id, char* buff, size_t sz)
@@ -1807,7 +1808,65 @@ void _maat_empty_region(struct Maat_region_t* p)
 	memset(p,0,sizeof(struct Maat_region_t));
 	return;
 
+}
+int Maat_cmd_raw_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, int line_num ,enum MAAT_OPERATION op)
+{
+	int i=0;
+	_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
+	int ret=0, table_id=0,success_cnt=0;
+	struct serial_rule_t *s_rule=NULL;
+	long long server_time=0,absolute_expire_time=0;
+	redisContext* write_ctx=get_redis_ctx_for_write(_feather);
+	if(write_ctx==NULL)
+	{
+		return -1;
+	}
+	server_time=redis_server_time(write_ctx);
+	if(!server_time)
+	{
+		return -1;
+	}
+	s_rule=ALLOC(struct serial_rule_t, line_num);
+	for(i=0;i<line_num;i++)
+	{
+		table_id=Maat_table_get_id_by_name(_feather->table_mgr, line_rule[i]->table_name);
+		if(table_id<0)
+		{
+			MESA_handle_runtime_log(_feather->logger, RLOG_LV_FATAL, maat_command,
+					"Command raw set line id %d failed: unknown table %s.",
+					line_rule[i]->rule_id,
+					line_rule[i]->table_name);
+			ret=-1;
+			goto error_out;
+		}
+		if(op==MAAT_OP_RENEW_TIMEOUT)
+		{
+			assert(line_rule[i]->expire_after>0);
+		}
+		if(line_rule[i]->expire_after>0)
+		{
+			absolute_expire_time=server_time+line_rule[i]->expire_after;
+		}
+		set_serial_rule(s_rule+i, op,line_rule[i]->rule_id,line_rule[i]->label_id, line_rule[i]->table_name,
+						line_rule[i]->table_line, absolute_expire_time);
+	}
+	success_cnt=exec_serial_rule(write_ctx,s_rule, line_num,server_time,_feather->logger);
+	if(success_cnt<0||success_cnt!=line_num)//error
+	{
+		ret=-1;
+		goto error_out;
+	}
+	ret=success_cnt;
 	_feather->line_cmd_acc_num+=success_cnt;
+
+error_out:
+	for(i=0;i<line_num;i++)
+	{
+		empty_serial_rules(s_rule+i);
+	}
+	free(s_rule);
+	return ret;
+
 }
 int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_rule, int line_num ,enum MAAT_OPERATION op)
 {
@@ -2142,6 +2201,7 @@ int redis_flush_DB(redisContext* ctx, int db_index, void* logger)
 }
 int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num,  int label_id, int expire_after)
 {
+	struct Maat_cmd_line line_cmd;
 	const struct Maat_cmd_line *p=NULL;
 	char line[MAX_TABLE_LINE_SIZE];
 	serialize_compile(compile, huge_service_defined, clause_num, op, line, sizeof(line));
@@ -2150,12 +2210,15 @@ int Maat_command_raw_set_compile(Maat_feather_t feather, enum MAAT_OPERATION op,
 	line_cmd.rule_id=compile->config_id;
 	line_cmd.table_line=line;
 	line_cmd.label_id=label_id;
-	line_cmd.expire_after=expire_after;
+	line_cmd.expire_after=expire_after;
+	p=&line_cmd;
 	int ret=Maat_cmd_raw_set_lines(feather, &p, 1, op);
 	return ret;
 }
 int Maat_command_raw_set_region(Maat_feather_t feather,  enum MAAT_OPERATION op, const struct Maat_cmd_region* region, int group_id)
 {
+	struct Maat_cmd_line line_cmd;
+	const struct Maat_cmd_line *p=NULL;
 
 	char line[MAX_TABLE_LINE_SIZE];
 
@@ -2163,12 +2226,15 @@ int Maat_command_raw_set_region(Maat_feather_t feather,  enum MAAT_OPERATION op,
 	memset(&line_cmd, 0, sizeof(line_cmd));
 	line_cmd.table_name=region->table_name;
 	line_cmd.rule_id=region->region_id;
-	line_cmd.table_line=line;
+	line_cmd.table_line=line;
+	p=&line_cmd;
 	int ret=Maat_cmd_raw_set_lines(feather, &p, 1, op);
 	return ret;
 }
 int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2compile* g2c)
 {
+	struct Maat_cmd_line line_cmd;
+	const struct Maat_cmd_line *p=NULL;
 
 	char line[MAX_TABLE_LINE_SIZE];
 
@@ -2177,12 +2243,14 @@ int Maat_command_raw_set_group2compile(Maat_feather_t feather, enum MAAT_OPERATI
 	memset(&line_cmd, 0, sizeof(line_cmd));
 	line_cmd.table_name=g2c->table_name;
 	line_cmd.rule_id=g2c->compile_id*1024*1024+g2c->group_id;
-	line_cmd.table_line=line;
+	line_cmd.table_line=line;
+	p=&line_cmd;
 	int ret=Maat_cmd_raw_set_lines(feather, &p, 1, op);
 	return ret;
 }
 int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g)
 {
+	struct Maat_cmd_line line_cmd;
 	const struct Maat_cmd_line *p=NULL;
 	char line[MAX_TABLE_LINE_SIZE];
 
@@ -2191,7 +2259,8 @@ int Maat_command_raw_set_group2group(Maat_feather_t feather, enum MAAT_OPERATION
 	memset(&line_cmd, 0, sizeof(line_cmd));
 	line_cmd.table_name=g2g->table_name;
 	line_cmd.rule_id=g2g->superior_group_id*1024*1024+g2g->group_id;
-	line_cmd.table_line=line;
+	line_cmd.table_line=line;
+	p=&line_cmd;
 	int ret=Maat_cmd_raw_set_lines(feather, &p, 1, op);
 	return ret;
 }

src/entry/Maat_ex_data.cpp

@@ -71,6 +71,7 @@ void EX_data_rt_free(struct EX_data_rt* p)
 		p->cache_rows=NULL;
 	}
 	MESA_htable_destroy(p->key2ex_hash, NULL);
+	free(p);
 	return;
 }
 void EX_data_rt_cache_row(struct EX_data_rt* p, const char* row)

src/entry/Maat_garbage_collection.cpp

@@ -38,11 +38,16 @@ void Maat_garbage_bin_free(struct Maat_garbage_bin* bin)
 		p->garbage_free(p->garbage);
 		TAILQ_REMOVE(&bin->garbage_q, p, entries);
 		free(p);
+		bin->bag_cnt--;
 	}
 
 	free(bin);
 	return;
 }
+size_t  Maat_garbage_bin_get_size(struct Maat_garbage_bin* bin)
+{
+	return bin->bag_cnt;
+}
 void Maat_garbage_bagging(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *))
 {
 	struct Maat_garbage_bag* bag=ALLOC(	struct Maat_garbage_bag, 1);

src/entry/Maat_hierarchy.cpp

@@ -209,7 +209,7 @@ static struct Maat_hierarchy_compile* Maat_hierarchy_compile_new(struct Maat_hie
 static void Maat_hierarchy_compile_free(struct Maat_hierarchy* hier, struct Maat_hierarchy_compile* compile)
 {
 	HASH_DEL(hier->hash_compile_by_id, compile);
-	if(hier->compile_user_data_free)
+	if(hier->compile_user_data_free && compile->user_data)
 	{
 		hier->compile_user_data_free(compile->user_data);
 	}
@@ -234,7 +234,7 @@ static void Maat_hierarchy_region_free(struct Maat_hierarchy* hier, struct Maat_
 {
 	HASH_DELETE(hh, hier->hash_region_by_id, region);
 	region->ref_parent_group->ref_by_region_cnt--;
-	if(region->user_data)
+	if(hier->region_user_data_free && region->user_data)
 	{
 		hier->region_user_data_free(region->user_data);
 		region->user_data=NULL;
@@ -281,14 +281,6 @@ void Maat_hierarchy_free(struct Maat_hierarchy* hier)
 		Maat_hierarchy_compile_free(hier, compile);
 	}
 	assert(hier->hash_compile_by_id==NULL);
-
-	HASH_CLEAR(hh_vertex_id, hier->hash_group_by_vertex);//No need group memory clean up.
-	HASH_ITER(hh_group_id, hier->hash_group_by_id, group, tmp_group)
-	{
-		HASH_DELETE(hh_group_id, hier->hash_group_by_id, group);
-		_group_vertex_free(group);
-	}
-	assert(hier->hash_group_by_id==NULL);	
 	
 	HASH_ITER(hh, hier->hash_literal_by_id, literal, tmp_literal)
 	{
@@ -301,9 +293,19 @@ void Maat_hierarchy_free(struct Maat_hierarchy* hier)
 		Maat_hierarchy_region_free(hier, region);
 	}
 
+
+	//Free group as the last.
+	HASH_CLEAR(hh_vertex_id, hier->hash_group_by_vertex);//No need group memory clean up.
+	HASH_ITER(hh_group_id, hier->hash_group_by_id, group, tmp_group)
+	{
+		HASH_DELETE(hh_group_id, hier->hash_group_by_id, group);
+		_group_vertex_free(group);
+	}
+	assert(hier->hash_group_by_id==NULL);	
 	
 	igraph_destroy(&hier->group_graph);
-	
+	bool_matcher_free(hier->bm);
+	hier->bm=NULL;
 	pthread_rwlock_unlock(&hier->rwlock);
 	free(hier);
 }
@@ -352,7 +354,7 @@ int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id)
 	
 	pthread_rwlock_wrlock(&hier->rwlock);
 	HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile);
-	if(!compile)
+	if(compile)
 	{
 		Maat_hierarchy_compile_free(hier, compile);
 		ret=0;
@@ -368,7 +370,7 @@ int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id)
 	pthread_rwlock_unlock(&hier->rwlock);
 	return ret;
 }
-static void* Maat_hier_get_user_data(struct Maat_hierarchy* hier, int compile_id, int is_dettach)
+static void* Maat_hier_compile_get_user_data(struct Maat_hierarchy* hier, int compile_id, int is_dettach)
 {
 	struct Maat_hierarchy_compile* compile=NULL;
 	void* ret=NULL;
@@ -390,13 +392,13 @@ static void* Maat_hier_get_user_data(struct Maat_hierarchy* hier, int compile_id
 void* Maat_hierarchy_compile_dettach_user_data(struct Maat_hierarchy* hier, int compile_id)
 {
 	void* user_data=NULL;
-	user_data=Maat_hier_get_user_data(hier, compile_id, 1);
+	user_data=Maat_hier_compile_get_user_data(hier, compile_id, 1);
 	return user_data;
 }
 void* Maat_hierarchy_compile_read_user_data(struct Maat_hierarchy* hier, int compile_id)
 {
 	void* user_data=NULL;
-	user_data=Maat_hier_get_user_data(hier, compile_id, 0);
+	user_data=Maat_hier_compile_get_user_data(hier, compile_id, 0);
 	return user_data;
 
 }

src/entry/Maat_rule.cpp

@@ -2596,12 +2596,12 @@ void *thread_rule_monitor(void *arg)
 		}
 	}
 	
-	Maat_table_manager_destroy(feather->table_mgr);
 	destroy_maat_scanner(feather->scanner);
 	Maat_garbage_bin_free(feather->garbage_bin);
+	Maat_table_manager_destroy(feather->table_mgr);//Table manager MUST be freed at last.
 	
 	alignment_int64_array_free(feather->thread_call_cnt);
-	alignment_int64_array_free(feather-> compile_mid_cnt);
+	alignment_int64_array_free(feather->compile_mid_cnt);
 	alignment_int64_array_free(feather->outer_mid_cnt);
 	alignment_int64_array_free(feather->hit_cnt);
 	alignment_int64_array_free(feather->not_grp_hit_cnt);

src/entry/Maat_stat.cpp

@@ -1,5 +1,6 @@
 #include "Maat_rule_internal.h"
 #include "Maat_table.h"
+#include "Maat_garbage_collection.h"
 #include "alignment_int64.h"
 #include <time.h>
 #include <MESA/field_stat.h>
@@ -196,7 +197,7 @@ void maat_stat_output(struct _Maat_feather_t* feather)
 	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_Q_SIZE], 0,FS_OP_SET,feather->cmd_q_cnt);
 	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_CMD_LINE_NUM], 0,FS_OP_SET,feather->line_cmd_acc_num);
 
-	value=MESA_lqueue_get_count(feather->garbage_q);
+	value=Maat_garbage_bin_get_size(feather->garbage_bin);
 	FS_operate(feather->stat_handle, feather->fs_status_id[STATUS_GARBAGE_QSIZE], 0,FS_OP_SET,value);
 	feather->update_err_cnt=0;
 	feather->iconv_err_cnt=0;

src/entry/Maat_table_runtime.cpp

@@ -343,6 +343,7 @@ void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr
 								confidence_degree,
 								tag);
 	MESA_lqueue_join_tail(table_rt->similar.update_q, &digest_rule, sizeof(void*));
+	free(dup_digest);
 	return;
 }
 void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr_id)

src/entry/json2iris.cpp

@@ -952,6 +952,7 @@ void table_idx_write_cb(const uchar * key, uint size, void * data, void * user)
 		assert(ret==0);
 		fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp);
 		fprintf(p_iris->idx_fp,"%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo);
+		free(encrypt_buff);
 	}
 	else
 	{
@@ -1149,7 +1150,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
 			parent_group=group_info_read(p_iris->group_name_map, parent_group_name);
 			if(parent_group==NULL)
 			{
-				parent_group=group_info_add_unsafe(p_iris, p_iris->group_name_map, item->string);
+				parent_group=group_info_add_unsafe(p_iris, p_iris->group_name_map, parent_group_name);
 			}
 			ret=write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris, logger);
 			if(ret<0)

src/entry/map_str2int.cpp

@@ -100,6 +100,7 @@ void duplicate_func(const uchar * key, uint size, void * data, void * user)
 	memcpy(string,key,size);
 	ret=map_register(target,string,value);
 	assert(ret>0);
+	free(string);
 }
 MESA_htable_handle map_duplicate(MESA_htable_handle origin_map)
 {

src/inc_internal/Maat_garbage_collection.h

@@ -5,4 +5,5 @@ struct Maat_garbage_bin* Maat_garbage_bin_new(int default_timeout);
 void Maat_garbage_bin_free(struct Maat_garbage_bin* bin);
 void Maat_garbage_bagging(struct Maat_garbage_bin* bin, void* garbage, void (* func)(void *));
 void Maat_garbage_collect_routine(struct Maat_garbage_bin* bin);
+size_t  Maat_garbage_bin_get_size(struct Maat_garbage_bin* bin);
 

src/inc_internal/Maat_rule_internal.h

@@ -262,7 +262,6 @@ struct _Maat_feather_t
 {
 	struct Maat_scanner *scanner;
 	struct Maat_scanner *update_tmp_scanner;
-	MESA_lqueue_head garbage_q;
 	struct Maat_garbage_bin * garbage_bin;
 	struct Maat_table_manager* table_mgr;