在编译配置级别支持合取范式，编译配置最多8个子句（Clause）组成，子句内的分组是“或”运算，子句之间是“与”运算。

2020-05-30 20:56:49 +08:00
parent 48881faafe
commit 1179e9c76a
8 changed files with 259 additions and 110 deletions
--- a/src/entry/Maat_api.cpp
+++ b/src/entry/Maat_api.cpp
@@ -104,7 +104,7 @@ struct compile_sort_para
 	int compile_id;
 	void* user;
 };
-static void compile_sort_para_set(struct compile_sort_para* para, const struct Maat_compile_group_relation* compile_relation, void* user)
+static void compile_sort_para_set(struct compile_sort_para* para, const struct Maat_compile_inner* compile_relation, void* user)
 {
 	para->compile_id=compile_relation->compile_id;
 	para->evaluation_order=compile_relation->compile->evaluation_order;
@@ -144,8 +144,8 @@ static int compile_sort_para_compare(const struct compile_sort_para* a, const st
 }
 static int compare_compile_inner(const void *a, const void *b)
 {
-	const struct Maat_compile_group_relation *ra=*(const struct Maat_compile_group_relation **)a;
-	const struct Maat_compile_group_relation *rb=*(const struct Maat_compile_group_relation **)b;
+	const struct Maat_compile_inner *ra=*(const struct Maat_compile_inner **)a;
+	const struct Maat_compile_inner *rb=*(const struct Maat_compile_inner **)b;
 	
 	struct compile_sort_para sa, sb;
 	compile_sort_para_set(&sa, ra, NULL);
@@ -166,12 +166,12 @@ size_t Maat_rule_sort_by_evaluation_order(Maat_feather_t feather, struct Maat_ru
 	
 	memcpy(copy_rule_array, rule_array, sizeof(struct Maat_rule_t)*n_rule);

-	struct Maat_compile_group_relation *p=NULL;
+	struct Maat_compile_inner *p=NULL;

 	size_t i=0, j=0;
 	for(i=0; i<n_rule; i++)
 	{
-		p=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule_array[i].config_id);
+		p=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule_array[i].config_id);
 		if(p && 0==pthread_rwlock_tryrdlock(&(p->rwlock)))//rule maybe already deleted.
 		{
 			compile_sort_para_set(sort_para+i, p, copy_rule_array+i);
@@ -302,7 +302,7 @@ void scan_hit_status_update_by_group(struct scan_hit_status* hit_status, struct
 	}
 	return;
 }
-size_t scan_hit_status_update_by_compile(struct scan_hit_status* hit_status, struct Maat_compile_group_relation* compile_rule, int Nth_scan)
+size_t scan_hit_status_update_by_compile(struct scan_hit_status* hit_status, struct Maat_compile_inner* compile_rule, int Nth_scan)
 {
 	size_t i=0, j=0;
 	struct Maat_hit_path_inner* p=NULL, *q=NULL;
@@ -410,8 +410,8 @@ int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, const
 	size_t r_in_c_cnt=0;
 	struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
 	struct Maat_group_inner* group_rule=NULL;
-	struct Maat_compile_group_relation* relation_array[MAX_SCANNER_HIT_NUM];
-	struct Maat_compile_group_relation* relation=NULL;
+	struct Maat_compile_inner* relation_array[MAX_SCANNER_HIT_NUM];
+	struct Maat_compile_inner* relation=NULL;
 	int virtual_table_id=0;
 	const unsigned long long* hit_group_ids=NULL;
 	size_t hit_group_id_cnt=0;
@@ -452,7 +452,7 @@ int region_compile(_Maat_feather_t*feather, struct scan_hit_status *_mid, const
 	}
 	if(scan_ret>1)
 	{
-		qsort(relation_array, scan_ret, sizeof(struct Maat_compile_group_relation*),
+		qsort(relation_array, scan_ret, sizeof(struct Maat_compile_inner*),
 			  compare_compile_inner);
 	}
 	for(i=0; i<(unsigned int)scan_ret&&result_cnt<size; i++)
@@ -1245,7 +1245,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
 void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
 {
 	struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
-	struct Maat_compile_group_relation *relation=(struct Maat_compile_group_relation *)data;
+	struct Maat_compile_inner *relation=(struct Maat_compile_inner *)data;
 	struct Maat_compile_rule* compile_rule=relation->compile;
 	MAAT_RULE_EX_DATA ad=NULL;
 	
@@ -1301,13 +1301,13 @@ int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table
 MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
 {
 	struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
-	struct Maat_compile_group_relation *relation=NULL;
+	struct Maat_compile_inner *relation=NULL;
 	struct Maat_compile_rule* compile=NULL;
 	const struct compile_table_schema* compile_desc=NULL;
 	const struct compile_ex_data_idx* ex_desc=NULL;
 	MAAT_RULE_EX_DATA ad=NULL;
 	
-	relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
+	relation=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
 	if(relation==NULL||relation->compile==NULL)
 	{
 		return NULL;
@@ -2478,11 +2478,11 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
 {
 	int ret=0;
 	struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
-	struct Maat_compile_group_relation *compile_inner=NULL;
+	struct Maat_compile_inner *compile_inner=NULL;
 	switch(type)
 	{
 		case	MAAT_RULE_SERV_DEFINE:
-			compile_inner=(struct Maat_compile_group_relation *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
+			compile_inner=(struct Maat_compile_inner *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
 			if(compile_inner==NULL)
 			{
 				ret=0;
--- a/src/entry/Maat_command.cpp
+++ b/src/entry/Maat_command.cpp
@@ -873,7 +873,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
 	struct Maat_group_t* group_cmd=NULL;
 	struct Maat_region_t* region_cmd=NULL;

-	struct Maat_compile_group_relation *relation=NULL;
+	struct Maat_compile_inner *relation=NULL;
 	struct Maat_group_inner* group_inner=NULL;
 	struct Maat_region_inner* region_inner=NULL;
 	void* logger=feather->logger;
@@ -885,7 +885,7 @@ int reconstruct_cmd(struct _Maat_feather_t *feather, struct _Maat_cmd_inner_t* _
 				,"MAAT not ready.");
 		return -1;
 	}
-	relation=(struct Maat_compile_group_relation *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
+	relation=(struct Maat_compile_inner *)HASH_fetch_by_id(feather->scanner->compile_hash, config_id);
 	//Operation on relation is thread safe, no immediate memory free when delete a compile rule or a scanner.
 	//In another words, if the relation is accessable from compile means, its was valid in at least 10 seconds (garbage bury).
 	if(relation==NULL)
@@ -1389,10 +1389,10 @@ int fix_table_name(_Maat_feather_t* feather,struct Maat_cmd_t* cmd)
 	struct Maat_region_t* p_region=NULL;
 	enum MAAT_TABLE_TYPE table_type;

-	struct Maat_compile_group_relation *compile_rule=NULL;
+	struct Maat_compile_inner *compile_rule=NULL;
 	if(feather->scanner!=NULL)
 	{
-		compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
+		compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(feather->scanner->compile_hash, cmd->compile.config_id);
 		if(compile_rule!=NULL)
 		{
 			MESA_handle_runtime_log(feather->logger,RLOG_LV_FATAL,maat_module 
--- a/src/entry/Maat_rule.cpp
+++ b/src/entry/Maat_rule.cpp
@@ -624,7 +624,7 @@ void destroy_group_rule(struct Maat_group_inner* group_rule, int by_whom, struct

 	}
 }
-void make_group_set(struct Maat_compile_group_relation* relation, struct bool_expr* a_set, unsigned char *has_not)
+void make_group_set(struct Maat_compile_inner* relation, struct bool_expr* a_set, unsigned char *has_not)
 {
 	int i=0,j=0;
 	a_set->user_tag=relation;
@@ -657,23 +657,23 @@ struct compile_walker
 void walk_compile_hash(const uchar * key, uint size, void * data, void * user)
 {
 	struct bool_expr* one_set=NULL;
-	struct Maat_compile_group_relation* relation=(struct Maat_compile_group_relation*)data;
+	struct Maat_compile_inner* compile_inner=(struct Maat_compile_inner*)data;
 	struct compile_walker* walker=(struct compile_walker*)user;
 	unsigned char has_not_flag=0;
 	MESA_lqueue_head update_q=walker->update_q;
-	if(relation->compile==NULL)
+	if(compile_inner->compile==NULL)
 	{
 		return;
 	}
 	//make sure compile rule's each group has loadded.
-	if((relation->group_cnt==relation->compile->declared_grp_num
-		|| relation->compile->declared_grp_num==0)//for compatible old version
-		&& relation->group_cnt>0
-		&& relation->group_cnt!=relation->not_group_cnt)
+	if((compile_inner->group_cnt==compile_inner->compile->declared_grp_num
+		|| compile_inner->compile->declared_grp_num==0)//for compatible old version
+		&& compile_inner->group_cnt>0
+		&& compile_inner->group_cnt!=compile_inner->not_group_cnt)
 	{
 		one_set=ALLOC(struct bool_expr, 1);
 		//reading compile rule is safe in update thread, mutex lock called when modified
-		make_group_set(relation, one_set, &has_not_flag);
+		make_group_set(compile_inner, one_set, &has_not_flag);
 		if(has_not_flag)
 		{
 			walker->compile_has_not_flag++;
@@ -788,10 +788,10 @@ void destroy_compile_rule(struct Maat_compile_rule* compile_rule)
 	free(compile_rule);
 	return;
 }
-struct Maat_compile_group_relation * create_compile_group_relation(int compile_id, struct Maat_scanner *scanner)
+struct Maat_compile_inner * create_compile_group_relation(int compile_id, struct Maat_scanner *scanner)
 {
 	int ret=0;
-	struct Maat_compile_group_relation* p=ALLOC(struct Maat_compile_group_relation, 1);
+	struct Maat_compile_inner* p=ALLOC(struct Maat_compile_inner, 1);
 	p->magic_num=COMPILE_RELATION_MAGIC;
 	p->compile_id=compile_id;
 	p->group_cnt=0;
@@ -803,7 +803,7 @@ struct Maat_compile_group_relation * create_compile_group_relation(int compile_i
 	return p;
 }

-void _destroy_compile_group_relation(struct Maat_compile_group_relation * cg_relation)
+void _destroy_compile_group_relation(struct Maat_compile_inner * cg_relation)
 {
 	assert(cg_relation->magic_num==COMPILE_RELATION_MAGIC);
 	pthread_rwlock_wrlock(&(cg_relation->rwlock));
@@ -815,7 +815,7 @@ void _destroy_compile_group_relation(struct Maat_compile_group_relation * cg_rel

 	free(cg_relation);
 }
-void destroy_compile_group_relation(struct Maat_compile_group_relation * p, struct Maat_scanner *scanner)
+void destroy_compile_group_relation(struct Maat_compile_inner * p, struct Maat_scanner *scanner)
 {
 	int i=0;
 	UNUSED struct Maat_group_inner* p_group=NULL;
@@ -1278,12 +1278,6 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue,
 	free(to_update_expr);
 }

-struct region_group_relation
-{
-	int region_id;
-	int group_id;
-	int array_idx;
-};
 int region_group_relation_add(MESA_htable_handle region_hash, int region_id, int group_id, int array_idx)
 {
 	struct region_group_relation* relation=ALLOC(struct region_group_relation, 1);
@@ -1423,67 +1417,68 @@ unsigned int del_region_from_group(struct Maat_group_inner* group,int region_id,
 	return i;
 }

-int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag)
+int add_group_to_compile(struct Maat_compile_inner* compile_inner, struct Maat_group_inner* a_rule_group, int virual_table_id, int not_flag)
 {
 	int i=0,ret=-1;
 	int write_pos=-1;
 	struct Maat_group_inner* p=NULL;
 	
-	pthread_rwlock_wrlock(&(relation->rwlock));
-	if(relation->compile!=NULL
-		&& relation->group_cnt>=relation->compile->declared_grp_num
-		&& relation->compile->declared_grp_num!=0)
+	pthread_rwlock_wrlock(&(compile_inner->rwlock));
+	if(compile_inner->compile!=NULL
+		&& compile_inner->group_cnt>=compile_inner->compile->declared_grp_num
+		&& compile_inner->compile->declared_grp_num!=0)
 	{
 		ret=-1;
 		goto error_out;
 	}

-	for(i=0;i<relation->group_boundary;i++)
+	for(i=0;i<compile_inner->group_boundary;i++)
 	{
-		p=(struct Maat_group_inner*)dynamic_array_read(relation->groups,i);
+		p=(struct Maat_group_inner*)dynamic_array_read(compile_inner->groups,i);
 		if(p==NULL)
 		{
 			write_pos=i;
 		}
 		else
 		{
-			if(p->group_id==a_rule_group->group_id && relation->virtual_table_id[i]==virual_table_id)//duplicate group
+			if(p->group_id==a_rule_group->group_id && compile_inner->virtual_table_id[i]==virual_table_id)//duplicate group
 			{
 				ret=-1;
 				goto error_out;
 			}
 		}
 	}
-	if(write_pos<0&&relation->group_boundary==MAX_EXPR_ITEM_NUM)
+	if(write_pos<0&&compile_inner->group_boundary==MAX_EXPR_ITEM_NUM)
 	{
 		ret=-1;
 		goto error_out;
 	}
 	if(write_pos<0)
 	{
-		write_pos=relation->group_boundary;
-		relation->group_boundary++;
+		write_pos=compile_inner->group_boundary;
+		compile_inner->group_boundary++;
 	}
-	dynamic_array_write(relation->groups, write_pos, a_rule_group);
+	dynamic_array_write(compile_inner->groups, write_pos, a_rule_group);
 	if(not_flag)
 	{
-		relation->not_flag[write_pos]=1;
-		relation->not_group_cnt++;
+		compile_inner->not_flag[write_pos]=1;
+		compile_inner->not_group_cnt++;
 	}
 	else
 	{
-		relation->not_flag[write_pos]=0;
+		compile_inner->not_flag[write_pos]=0;
 	}
-	relation->virtual_table_id[write_pos]=virual_table_id;
-	relation->group_cnt++;
+	compile_inner->virtual_table_id[write_pos]=virual_table_id;
+	compile_inner->group_cnt++;
 	a_rule_group->ref_by_parent_cnt++;
+	a_rule_group->ref_by_compile_cnt++;
 	ret=1;
 error_out:
-	pthread_rwlock_unlock(&(relation->rwlock));
+	pthread_rwlock_unlock(&(compile_inner->rwlock));

 	return ret;
 }
-struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relation*relation, int group_id, int virual_table_id)
+struct Maat_group_inner* del_group_from_compile(struct Maat_compile_inner*relation, int group_id, int virual_table_id)
 {
 	int i=0;
 	struct Maat_group_inner* group_rule=NULL;
@@ -1513,6 +1508,7 @@ struct Maat_group_inner* del_group_from_compile(struct Maat_compile_group_relati
 		}
 	}
 	pthread_rwlock_unlock(&(relation->rwlock));
+	group_rule->ref_by_compile_cnt--;
 	return group_rule;
 }

@@ -1981,7 +1977,7 @@ int del_region_rule(struct Maat_table_schema* table,int region_id,int group_id,i
 int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger)
 {
 	struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
-	struct Maat_compile_group_relation*compile_rule=NULL;
+	struct Maat_compile_inner*compile_rule=NULL;
 	int ret=0;
 	igraph_integer_t edge_id;
 	group_rule=(struct Maat_group_inner*)HASH_fetch_by_id(scanner->group_hash, db_group_rule->group_id);
@@ -2014,8 +2010,7 @@ int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
 	}
 	else
 	{
-		group_rule->ref_by_compile_cnt++;
-		compile_rule=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id);
+		compile_rule=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id);
 		if(compile_rule==NULL)
 		{
 			compile_rule=create_compile_group_relation(db_group_rule->parent_id, scanner);
@@ -2036,7 +2031,7 @@ int add_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
 }
 int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_group_rule, struct Maat_scanner *scanner, void* logger)
 {
-	struct Maat_compile_group_relation* relation=NULL;
+	struct Maat_compile_inner* relation=NULL;
 	struct Maat_group_inner* group_rule=NULL, *parent_group=NULL;
 	igraph_es_t es;
 	int ret=0;
@@ -2092,7 +2087,7 @@ int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
 	}
 	else
 	{
-		relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id);
+		relation=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, db_group_rule->parent_id);
 		if(relation==NULL)
 		{
 			MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
@@ -2116,7 +2111,7 @@ int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
 		{
 			destroy_compile_group_relation(relation, scanner);
 		}
-		group_rule->ref_by_compile_cnt--;
+		
 	}
 	destroy_group_rule(group_rule, DESTROY_GROUP_BY_PARENT, scanner);
 	scanner->to_update_group_cnt++;
@@ -2124,10 +2119,10 @@ int del_group_rule(struct Maat_table_schema* table, struct db_group_rule_t* db_g
 }
 int add_compile_rule(struct Maat_table_schema* table, struct Maat_compile_rule* db_compile_rule, struct Maat_scanner *scanner, void* logger)
 {
-	struct Maat_compile_group_relation *cg_relation=NULL;
+	struct Maat_compile_inner *cg_relation=NULL;
 	struct Maat_rule_head *p_maat_rule_head=&(db_compile_rule->head);

-	cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
+	cg_relation=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
 	if(cg_relation==NULL)
 	{
 		cg_relation=create_compile_group_relation(p_maat_rule_head->config_id, scanner);
@@ -2146,8 +2141,8 @@ int add_compile_rule(struct Maat_table_schema* table, struct Maat_compile_rule*
 }
 int del_compile_rule(struct Maat_table_schema* table, int compile_id, struct Maat_scanner *scanner, void* logger)
 {
-	struct Maat_compile_group_relation *cg_relation=NULL;
-	cg_relation=(struct Maat_compile_group_relation*)HASH_fetch_by_id(scanner->compile_hash, compile_id);
+	struct Maat_compile_inner *cg_relation=NULL;
+	cg_relation=(struct Maat_compile_inner*)HASH_fetch_by_id(scanner->compile_hash, compile_id);
 	if(cg_relation==NULL || cg_relation->compile==NULL)
 	{
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
--- a/src/entry/json2iris.cpp
+++ b/src/entry/json2iris.cpp
@@ -183,6 +183,7 @@ int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_k
 	map_register(iris_cfg->str2int_map, "left",2);
 	map_register(iris_cfg->str2int_map, "prefix",2);
 	map_register(iris_cfg->str2int_map, "complete",3);
+	map_register(iris_cfg->str2int_map, "exact",3);

 	map_register(iris_cfg->str2int_map, "uncase plain",0);
 	map_register(iris_cfg->str2int_map, "hexbin",1);
@@ -934,6 +935,28 @@ int write_index_file(struct iris_description_t *p_iris,void* logger)
 	p_iris->idx_fp=NULL;
 	return 0;
 }
+static struct group_info_t* group_info_read(MESA_htable_handle table, const char* group_name)
+{
+	return (struct group_info_t*)MESA_htable_search(table, (const unsigned char*)group_name, strlen(group_name));
+}
+static struct group_info_t* group_info_add_unsafe(struct iris_description_t* p_iris, MESA_htable_handle table, const char* group_name)
+{
+	static struct group_info_t untitled_group;
+	struct group_info_t *group_info=NULL;
+	if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name)))
+	{
+		group_info=&untitled_group;
+		group_info->group_id=get_group_seq(p_iris);
+	}
+	else
+	{
+		group_info=ALLOC(struct group_info_t, 1);
+		group_info->group_id=get_group_seq(p_iris);
+		strncpy(group_info->group_name, group_name, sizeof(group_info->group_name));
+		MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info);
+	}
+	return group_info;
+}
 int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger)
 {
 	const char* _str_parent_type[2]={"compile", "group"};
@@ -943,7 +966,6 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
 	cJSON *sub_groups=NULL, *region_rule=NULL;
 	const char* group_name=NULL, *virtual_table=NULL;
 	struct group_info_t *group_info=NULL;
-	struct group_info_t untitled_group;

 	item=cJSON_GetObjectItem(group_json, "group_name");
 	if(item==NULL||item->type!=cJSON_String)
@@ -977,22 +999,10 @@ int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int trac
 	{
 		group_not_flag=0;
 	}
-
-	group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name));
+	group_info=group_info_read(p_iris->group_name_map, group_name);
 	if(group_info==NULL)//exist group name, regions and sub groups will be ommit.
 	{
-		if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name)))
-		{
-			group_info=&untitled_group;
-			group_info->group_id=get_group_seq(p_iris);
-		}
-		else
-		{
-			group_info=ALLOC(struct group_info_t, 1);
-			group_info->group_id=get_group_seq(p_iris);
-			strncpy(group_info->group_name, group_name, sizeof(group_info->group_name));
-			MESA_htable_add(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name), group_info);
-		}
+		group_info=group_info_add_unsafe(p_iris, p_iris->group_name_map, group_name);
 		region_json=cJSON_GetObjectItem(group_json,"regions");
 		if(region_json!=NULL)
 		{
@@ -1043,7 +1053,10 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
 	int compile_id=-1, compile_cnt=0, group_cnt=0;
 	int ret=0;
 	cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL;
-	cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL;
+	cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL, *item=NULL;
+	static struct group_info_t* parent_group=NULL;
+	const char* parent_group_name=NULL;
+
 	plug_tables=cJSON_GetObjectItem(json,"plugin_table");
 	if(NULL!=plug_tables)
 	{
@@ -1052,6 +1065,34 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
 			write_plugin_line(each_plug_table, i, p_iris, logger);
 		}
 	}
+	g_rules=cJSON_GetObjectItem(json, "groups");//sub-group to group
+	if(g_rules!=NULL)
+	{
+		cJSON_ArrayForEach(group_rule, g_rules)
+		{
+			item=cJSON_GetObjectItem(group_rule, "parent_group");
+			if(item==NULL || item->type!=cJSON_String)
+			{
+				parent_group_name=untitled_group_name;
+				
+			}
+			else
+			{
+				parent_group_name=item->string;
+			}
+			parent_group=group_info_read(p_iris->group_name_map, parent_group_name);
+			if(parent_group==NULL)
+			{
+				parent_group=group_info_add_unsafe(p_iris, p_iris->group_name_map, item->string);
+			}
+			ret=write_group_rule(group_rule, parent_group->group_id, PARENT_TYPE_GROUP, 0, p_iris, logger);
+			if(ret<0)
+			{
+				return -1;
+			}
+		}
+		
+	}
 	c_rules=cJSON_GetObjectItem(json,"rules");
 	if(c_rules==NULL)
 	{
--- a/src/inc_internal/Maat_rule_internal.h
+++ b/src/inc_internal/Maat_rule_internal.h
@@ -108,6 +108,7 @@ struct db_group_rule_t
 	int not_flag;
 	int parent_type;	//PARENT_TYPE_**, 0:compile, 1: group.
 	int virtual_table_id;
+	int clause_id;
 };
 struct op_expr_t
 {
@@ -119,6 +120,13 @@ struct op_expr_t
 	int rule_type;
 };

+struct region_group_relation
+{
+	int region_id;
+	int group_id;
+	int array_idx;
+};
+
 struct Maat_region_inner
 {
 	int region_id;
@@ -129,22 +137,6 @@ struct Maat_region_inner
 	int expr_id_ub;
 	enum MAAT_TABLE_TYPE table_type;
 };
-#define	COMPILE_RELATION_MAGIC	0x1a2b3c4d
-struct Maat_compile_group_relation
-{
-	long long magic_num;
-	struct Maat_compile_rule *compile;
-	dynamic_array_t *groups;	//element is struct Maat_group_inner*
-	int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR];
-	char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
-	int compile_id;//equal to compile->m_rule.config_id
-	int group_boundary;
-	int group_cnt;
- 	int not_group_cnt;
-	pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
-
-};
-
 struct Maat_group_inner
 {
 	int group_id;
@@ -162,6 +154,35 @@ struct Maat_group_inner
 	pthread_mutex_t mutex;
 };

+#define	COMPILE_RELATION_MAGIC	0x1a2b3c4d
+struct Maat_group_reference
+{
+	struct Maat_group_inner* ref_group;
+	char not_flag;
+	int virtual_table_id;
+	int clause_id;
+	TAILQ_ENTRY(Maat_group_reference) entries;
+};
+TAILQ_HEAD(ref_group_q, Maat_group_reference);
+
+struct Maat_compile_inner
+{
+	long long magic_num;
+	struct Maat_compile_rule *compile;
+	struct ref_group_q ref_group_qhead;
+	
+	dynamic_array_t *groups;	//element is struct Maat_group_inner*
+	int virtual_table_id[MAX_ITEMS_PER_BOOL_EXPR];
+	char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
+	int compile_id;//equal to compile->m_rule.config_id
+	int group_boundary;
+	int group_cnt;
+ 	int not_group_cnt;
+	pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
+
+};
+
+
 struct _compile_result_t
 {
 	int compile_id;
@@ -253,10 +274,14 @@ struct Maat_scanner
 	struct Maat_table_runtime_manager*  table_rt_mgr;
 	size_t max_table_num;
 	
-	MESA_htable_handle region_hash;	//key: region_id, value: struct region_group_relation*
+	//Access in both UPDATE thread and SCAN thread
 	MESA_htable_handle exprid_hash;	//key: expr_id, value: int array_idx of Maat_group_inner->regions;
+	MESA_htable_handle compile_hash;//key: compile_id, value: struct Maat_compile_inner *
+	MESA_htable_handle clause_hash;	//key: global_clause_id, value: 
+		
+	//Access in UPDATE thread ONLY.	
+	MESA_htable_handle region_hash;	//key: region_id, value: struct region_group_relation*
 	MESA_htable_handle group_hash;	//key: group_id, value: struct Maat_group_inner*
-	MESA_htable_handle compile_hash;//key: compile_id, value: struct Maat_compile_group_relation *


 	MESA_htable_handle district_map;
@@ -402,7 +427,7 @@ struct _maat_garbage_t
 		struct Maat_scanner* scanner;
 		struct Maat_group_inner* group_rule;
 		struct Maat_compile_rule* compile_rule;
-		struct Maat_compile_group_relation * compile_group_relation;
+		struct Maat_compile_inner * compile_group_relation;
 		struct bool_matcher* bool_matcher;
 		struct ip_matcher* a_ip_matcher;
 		void * raw;
@@ -432,7 +457,7 @@ int parse_accept_tag(const char* value, struct rule_tag** result, void* logger);
 void garbage_bagging(enum maat_garbage_type type,void *p,MESA_lqueue_head garbage_q);
 void garbage_bagging_with_timeout(enum maat_garbage_type type,void *p, int timeout, MESA_lqueue_head garbage_q);
 void garbage_bury(MESA_lqueue_head garbage_q,void *logger);
-void make_group_set(struct Maat_compile_group_relation* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
+void make_group_set(struct Maat_compile_inner* compile_rule, struct bool_expr* a_set, unsigned char *has_not);
 void maat_start_cb(long long new_version,int update_type,void*u_para);
 int maat_update_cb(const char* table_name,const char* line,void *u_para);
 void maat_finish_cb(void* u_para);
--- a/test/maat_json.json
+++ b/test/maat_json.json
@@ -1,6 +1,42 @@
 {
    "compile_table": "COMPILE",
    "group_table": "GROUP",
+    "groups": [		
+			{
+				"group_name": "ASN1234",
+				"parent_group": "SRC_IP_ASN_1234",
+        "virtual_table":"SOURCE_IP_ASN",
+        "regions": [
+          {
+            "table_name": "AS_NUMBER",
+            "table_type": "string",
+            "table_content": {
+                "keywords": "AS1234",
+                "expr_type": "none",
+                "match_method": "exact",
+                "format": "uncase plain"
+            }
+          }
+        ]
+      },
+      {
+          "group_name": "ASN2345",
+          "parent_group": "DEST_IP_ASN_2345",
+          "virtual_table":"DESTINATION_IP_ASN",
+          "regions": [
+              {
+                  "table_name": "AS_NUMBER",
+                  "table_type": "string",
+                  "table_content": {
+                      "keywords": "AS2345",
+                      "expr_type": "none",
+                      "match_method": "exact",
+                      "format": "uncase plain"
+                  }
+              }
+          ]
+      }
+    ],
    "rules": [
        {
            "compile_id": 123,
@@ -1718,6 +1754,25 @@
 		                "not_flag":0                   
 		            }          
 		        ]
+		    },
+        {
+            "compile_id": 178,
+            "service": 1,
+            "action": 1,
+            "do_blacklist": 1,
+            "do_log": 1,
+            "user_region": "non-top-object-as-virtual",
+            "is_valid": "yes",
+            "groups": [
+		            {
+		            		"group_name":"SRC_IP_ASN_1234",
+		                "not_flag":0                   
+		            },          	
+		            {
+		            		"group_name":"DEST_IP_ASN_2345",
+		                "not_flag":0                   
+		            }
+            ]
        }
    ],
    "plugin_table": [
--- a/test/table_info.conf
+++ b/test/table_info.conf
@@ -28,15 +28,15 @@
 7	FILE_DIGEST	digest	--
 8	HTTP_SIGNATURE	expr_plus	GBK	GBK	yes	0
 9	SIM_URL	similar	--
-10	IMAGE_FP	expr	UTF8	UTF8	yes	128	quickoff
+10	IMAGE_FP	expr	UTF8	UTF8	yes	128
 11	TEST_EFFECTIVE_RANGE_TABLE	plugin	{"valid":4,"tag":5}	--
 12	TEST_FOREIGN_KEY	plugin {"valid":4,"foreign":[6,8],"tag":3}	--
 13	COMPILE_ALIAS	compile	escape	--
 14	TEST_PLUGIN_EXDATA_TABLE	plugin {"key":2,"valid":4,"tag":5,"estimate_size":1024}	--
 15	IR_INTERCEPT_IP plugin  {"valid":14,"tag":18}
-16	APP_PAYLOAD	expr_plus	UTF8	UTF8	yes	0	quickoff
-17	TROJAN_PAYLOAD	expr	UTF8	UTF8	yes	0	quickoff
-18	MAIL_ADDR	expr	UTF8	UTF8	yes	0	quickoff
+16	APP_PAYLOAD	expr_plus	UTF8	UTF8	yes	0
+17	TROJAN_PAYLOAD	expr	UTF8	UTF8	yes	0
+18	MAIL_ADDR	expr	UTF8	UTF8	yes	0
 19	IP_PLUS_CONFIG	ip_plus	--
 20	HTTP_RESPONSE_KEYWORDS	virtual	KEYWORDS_TABLE	--
 21	HTTP_REQUEST_HEADER	virtual	HTTP_SIGNATURE	--
@@ -49,3 +49,6 @@
 26	COMPOSITION_IP_SESSION	virtual	IP_PLUS_CONFIG	--
 27	COMPOSITION_IP	composition	{"source":"COMPOSITION_IP_SOURCE","destination":"COMPOSITION_IP_DESTINATION","session":"COMPOSITION_IP_SESSION"}
 28	TEST_IP_PLUGIN_WITH_EXDATA	ip_plugin {"row_id":1,"ip_type":2,"start_ip":3,"end_ip":4,"valid":6}	--
+29	AS_NUMBER	expr	UTF8	UTF8	yes	0
+30	SOURCE_IP_ASN	virtual	AS_NUMBER	--
+31	DESTINATION_IP_ASN	virtual	AS_NUMBER	--
--- a/test/test_maatframe.cpp
+++ b/test/test_maatframe.cpp
@@ -1732,7 +1732,7 @@ TEST(ScanResult, LongerServiceDefine)
 }
 TEST(VirtualTable, VirtualWithPhysical)
 {
-#define TestVirtualTable1
+#define VT_VirtualWithPhysical
 	int ret=0, table_id=0;
 	const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me.";
 	const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058";
@@ -1772,7 +1772,7 @@ TEST(VirtualTable, VirtualWithPhysical)
 }
 TEST(VirtualTable, VirtualWithVirtual)
 {
-#define TestVirtualTable2
+#define VT_VirtualWithVirtual
 	int ret=0, table_id=0;
 	const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
 	const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;";
@@ -1812,7 +1812,7 @@ TEST(VirtualTable, VirtualWithVirtual)
 }
 TEST(VirtualTable, OneGroupInTwoVirtual)
 {
-#define TestVirtualTable3
+#define VT_OneGroupInTwoVirtual
 	int ret=0, table_id=0;
 	const char* http_resp_hdr_cookie="sessionid=888888;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;";
 	
@@ -1849,6 +1849,36 @@ TEST(VirtualTable, OneGroupInTwoVirtual)

 	return;
 }
+TEST(VirtualTable, VirtualAsNonTopObject)
+{
+#define VT_AsNonTopObject
+	int ret=0, table_id=0;
+
+	const char* src_asn="AS1234", *dst_asn="AS2345";
+	
+	struct Maat_rule_t result[4];
+	memset(result, 0, sizeof(result));
+	
+	scan_status_t mid=NULL;
+
+	table_id=Maat_table_register(g_feather, "SOURCE_IP_ASN");	
+	ASSERT_GT(table_id, 0);
+	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, src_asn, strlen(src_asn), 
+						result, NULL, 4, &mid, 0);
+	EXPECT_EQ(ret, -2);
+	
+
+	table_id=Maat_table_register(g_feather, "DESTINATION_IP_ASN");	
+	ASSERT_GT(table_id, 0);
+	ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, dst_asn, strlen(dst_asn), 
+						result, NULL, 4, &mid, 0);
+	EXPECT_EQ(ret, 1);
+	EXPECT_EQ(result[0].config_id, 178);	
+
+	Maat_clean_status(&mid);
+
+	return;
+}


 class MaatFileTest : public testing::Test