Merge branch 'feature-rule-extra-data-index' into 'master'

Feature rule extra data index

See merge request MESA_framework/maat!11

inc/Maat_rule.h

@@ -1,12 +1,12 @@
 
 /*
-*****************Maat Network Flow Rule Manage Framework********
+*****************Maat Deep Packet Inspection Policy Framework********
 *	Maat is the Goddess of truth and justice in ancient Egyptian concept.
 *	Her feather was the measure that determined whether the souls (considered 
 *	to reside in the heart) of the departed would reach the paradise of afterlife
 *	successfully.
-*	Author: zhengchao@iie.ac.cn,MESA
+*	Author: zhengchao@iie.ac.cn, MESA
-*	Version 2018-09-25 foreign key and rule tags.
+*	Version 2018-11-06 Maat Rule Extra Data.
 *	NOTE: MUST compile with G++
 *	All right reserved by Institute of Infomation Engineering,Chinese Academic of Science 2014~2018
 *********************************************************
@@ -181,6 +181,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
 									Maat_finish_callback_t *finish,//u_para
 									void* u_para);
 
+
 enum MAAT_SCAN_OPT
 {
 	MAAT_SET_SCAN_DISTRICT=1,		//VALUE is a const char*,SIZE= strlen(string).DEFAULT: no default.
@@ -238,10 +239,31 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
 						,scan_status_t* mid,int thread_num);
 
 void Maat_clean_status(scan_status_t* mid);
+
+typedef void*  MAAT_RULE_EX_DATA;
+// The idx parameter is the index: this will be the same value returned by Maat_rule_get_ex_new_index() when the functions were initially registered.
+// Finally the argl and argp parameters are the values originally passed to the same corresponding parameters when Maat_rule_get_ex_new_index() was called.
+typedef void Maat_rule_EX_new_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
+											MAAT_RULE_EX_DATA* ad, long argl, void *argp);
+typedef void Maat_rule_EX_free_func_t(int idx, const struct Maat_rule_t* rule, const char* srv_def_large,
+											MAAT_RULE_EX_DATA* ad, long argl, void *argp);
+typedef void Maat_rule_EX_dup_func_t(int idx,	MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp);
+
+int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
+										 Maat_rule_EX_new_func_t* new_func,
+										 Maat_rule_EX_free_func_t* free_func,
+										 Maat_rule_EX_dup_func_t* dup_func,
+										 long argl, void *argp);
+//returned data is duplicated by dup_func of Maat_rule_get_ex_new_index,  caller is responsible to free the data.
+MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx);
+
+
 enum MAAT_RULE_OPT
 {
 	MAAT_RULE_SERV_DEFINE	//VALUE is a char* buffer,SIZE= buffer size. 
 };
 int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum MAAT_RULE_OPT type, void* value, int size);
+
+
 #endif	//	H_MAAT_RULE_H_INCLUDE
 

src/entry/Maat_api.cpp

@@ -99,6 +99,13 @@ int pickup_hit_region_from_compile(universal_bool_expr_t *compile_hit,const unsi
 	}
 	return k;
 }
+void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len)
+{
+	memcpy(rule, rule_head, sizeof(struct _head_Maat_rule_t));
+	memcpy(rule->service_defined, srv_def, MIN(srv_def_len,MAX_SERVICE_DEFINE_LEN));
+	return;
+}
+
 int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num)
 {
 
@@ -155,25 +162,25 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
 	for(i=0;i<scan_ret&&result_cnt<size;i++)
 	{
 		_mi_rule=array_mi_rule[i];
-		if(_mi_rule==NULL||_mi_rule->db_c_rule==NULL)
+		if(_mi_rule==NULL)
 		{
 			continue;
 		}
 		
 		if(0==pthread_rwlock_tryrdlock(&(_mi_rule->rwlock)))
 		{
-			make_group_set(_mi_rule,&(rs_result[result_cnt].group_set));
+			if(_mi_rule->is_valid==1)
-			r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt,
-										region_pos, MAX_SCANNER_HIT_NUM);
-			if(r_in_c_cnt>0)//compile config hitted becasue of new reigon
 			{
-				memcpy(&(result[result_cnt]),&(_mi_rule->db_c_rule->m_rule_head),sizeof(struct _head_Maat_rule_t));
+				make_group_set(_mi_rule,&(rs_result[result_cnt].group_set));
-				memcpy(result[result_cnt].service_defined
+				r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set),_mid->cur_hit_id,_mid->cur_hit_cnt,
-						,_mi_rule->db_c_rule->service_defined
+											region_pos, MAX_SCANNER_HIT_NUM);
-						,MIN(_mi_rule->db_c_rule->m_rule_head.serv_def_len,MAX_SERVICE_DEFINE_LEN));
+				if(r_in_c_cnt>0)//compile config hitted becasue of new reigon
-				
+				{
-				rs_result[result_cnt].compile_id=_mi_rule->compile_id;
+					fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head),
-				result_cnt++;
+								_mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len);
+					rs_result[result_cnt].compile_id=_mi_rule->compile_id;
+					result_cnt++;
+				}
 			}
 			pthread_rwlock_unlock(&(_mi_rule->rwlock));
 		}		
@@ -880,7 +887,7 @@ error_out:
 
 void Maat_burn_feather(Maat_feather_t feather)
 {
-	_Maat_feather_t* _feather=(_Maat_feather_t*)feather;
+	struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
 	_feather->still_working=0;//destroy will proceed in thread_rule_monitor
 	return;
 }
@@ -888,7 +895,7 @@ int Maat_table_register(Maat_feather_t feather,const char* table_name)
 {
 	struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
 	int table_id=-1,ret=0;
-	ret=map_str2int(_feather->map_tablename2id, table_name,&table_id);
+	ret=map_str2int(_feather->map_tablename2id, table_name, &table_id);
 	if(ret>0)
 	{
 		return table_id;
@@ -906,7 +913,7 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
 {
 	struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
 	int idx=0,i=0;
-	_Maat_table_info_t *p_table=_feather->p_table_info[table_id];
+	struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
 	const char* lines=NULL;
 	if(p_table==NULL)
 	{
@@ -952,6 +959,88 @@ int Maat_table_callback_register(Maat_feather_t feather,short table_id,
 	pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
 	return 1;
 }
+
+void rule_ex_data_new_cb(const uchar * key, uint size, void * data, void * user)
+{
+	struct compile_ex_data_idx *ex_desc=(struct compile_ex_data_idx*)user;
+	struct _Maat_compile_inner_t *compile_inner=(struct _Maat_compile_inner_t *)data;
+	MAAT_RULE_EX_DATA ad=NULL;
+	
+	if(compile_inner->ref_table==NULL || compile_inner->ref_table->table_id!=ex_desc->table_id || compile_inner->db_c_rule==NULL)
+	{
+		return;
+	}
+	pthread_rwlock_wrlock(&(compile_inner->rwlock));
+	ad=rule_ex_data_new(&(compile_inner->db_c_rule->m_rule_head),
+						compile_inner->db_c_rule->service_defined,
+						ex_desc);
+	compile_inner->ads[ex_desc->idx]=ad;
+	pthread_rwlock_unlock(&(compile_inner->rwlock));
+	return;
+}
+int Maat_rule_get_ex_new_index(Maat_feather_t feather, const char* compile_table_name,
+										 Maat_rule_EX_new_func_t *new_func,
+										 Maat_rule_EX_free_func_t* free_func,
+										 Maat_rule_EX_dup_func_t* dup_func,
+										 long argl, void *argp)
+{
+	struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
+	int table_id=-1,ret=0, idx=-1;
+	ret=map_str2int(_feather->map_tablename2id, compile_table_name, &table_id);
+	if(ret<0)
+	{
+		return -1;
+	}
+	struct _Maat_table_info_t *p_table=_feather->p_table_info[table_id];
+	if(p_table->table_type!=TABLE_TYPE_COMPILE || new_func==NULL || free_func==NULL)
+	{
+		return -1;
+	}
+
+	pthread_mutex_lock(&(_feather->backgroud_update_mutex));
+	if(p_table->ex_data_num==MAX_COMPILE_EX_DATA_NUM)
+	{
+		ret=-1;
+		goto failed;
+	}
+	idx=p_table->ex_data_num;
+	p_table->ex_desc[idx].idx=idx;
+	p_table->ex_desc[idx].table_id=table_id;
+	p_table->ex_desc[idx].argl=argl;
+	p_table->ex_desc[idx].argp=argp;
+	p_table->ex_desc[idx].new_func=new_func;
+	p_table->ex_desc[idx].free_func=free_func;
+	p_table->ex_desc[idx].dup_func=dup_func;
+
+	p_table->ex_data_num++;
+	if(_feather->scanner!=NULL)
+	{
+		MESA_htable_iterate(_feather->scanner->compile_hash, rule_ex_data_new_cb, p_table->ex_desc+idx);
+	}
+failed:
+	pthread_mutex_unlock(&(_feather->backgroud_update_mutex));
+	return idx;
+
+}
+MAAT_RULE_EX_DATA Maat_rule_get_ex_data(Maat_feather_t feather, const struct Maat_rule_t* rule, int idx)
+{
+	struct _Maat_feather_t *_feather=(struct _Maat_feather_t *)feather;
+	struct _Maat_compile_inner_t *compile_inner=NULL;
+	const struct compile_ex_data_idx* ex_desc=NULL;	
+	MAAT_RULE_EX_DATA ad=NULL;
+	
+	compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
+	if(compile_inner==NULL)
+	{
+		return NULL;
+	}
+	pthread_rwlock_rdlock(&(compile_inner->rwlock));
+	assert(idx<compile_inner->ref_table->ex_data_num);	
+	ex_desc=compile_inner->ref_table->ex_desc+idx;
+	ex_desc->dup_func(ex_desc->idx, &ad, compile_inner->ads+idx, ex_desc->argl,ex_desc->argp);
+	pthread_rwlock_unlock(&(compile_inner->rwlock));		
+	return ad;
+}
 int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
 						,enum MAAT_CHARSET charset,const char* data,int data_len
 						,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
@@ -1781,17 +1870,17 @@ int Maat_read_rule(Maat_feather_t feather, const struct Maat_rule_t* rule, enum
 	{
 		case	MAAT_RULE_SERV_DEFINE:
 			compile_inner=(struct _Maat_compile_inner_t *)HASH_fetch_by_id(_feather->scanner->compile_hash, rule->config_id);
-			pthread_rwlock_rdlock(&(compile_inner->rwlock));
 			if(compile_inner==NULL)
 			{
 				ret=0;
 			}
 			else
 			{
+				pthread_rwlock_rdlock(&(compile_inner->rwlock));
 				ret=MIN(size,compile_inner->db_c_rule->m_rule_head.serv_def_len);
 				memcpy(value,compile_inner->db_c_rule->service_defined,ret);
+				pthread_rwlock_unlock(&(compile_inner->rwlock));
 			}
-			pthread_rwlock_unlock(&(compile_inner->rwlock));		
 			break;
 		default:
 			ret=-1;

src/entry/Maat_command.cpp

@@ -1532,14 +1532,14 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n
 			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
 									, "Get %s,%d foreign keys failed: No %dth column."
 									, p_rule->table_name, p_rule->rule_id, foreign_columns[i]);
-			continue;
+			break;
 		}
 		if(0!=strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix)))
 		{
 			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_redis_monitor
 									,"Get %s,%d foreign key failed: Invalid source prefix %s."
 									, p_rule->table_name, p_rule->rule_id, p_foreign);
-			continue;
+			break;
 		}
 		p_rule->f_keys[i].column=foreign_columns[i];
 		foreign_key_size=foreign_key_size-strlen(foreign_source_prefix);
@@ -1560,7 +1560,13 @@ void _get_foregin_keys(struct serial_rule_t* p_rule, int* foreign_columns, int n
 			p_rule->f_keys[i].is_existed=1;
 		}
 	}
-	return;
+	if(i!=n_foreign)
+	{
+		p_rule->n_foreign=0;
+		free(p_rule->f_keys);
+		p_rule->f_keys=NULL;
+		p_rule->with_error=1;
+	}
 }
 int get_foreign_keys_define(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, _Maat_feather_t* feather, const char* dir,void *logger)
 {
@@ -1784,7 +1790,7 @@ void redis_monitor_traverse(long        long version,redisContext *c,
 	MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_redis_monitor,"Start %s update: %lld->%lld (%d entries).",
 								update_type==CM_UPDATE_TYPE_INC?"INC":"FULL",version,new_version,rule_num);
 	for(i=0;i<rule_num;i++)
-	{
+	{
 		if(rule_list[i].table_line==NULL||rule_list[i].with_error==1)
 		{
 			continue;

src/entry/Maat_rule.cpp

@@ -504,6 +504,23 @@ int HASH_delete_by_id(MESA_htable_handle hash,int id)
 	ret=MESA_htable_del(hash,(unsigned char*)&id, sizeof(id), NULL);
 	return ret;
 }
+MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc)
+{
+	MAAT_RULE_EX_DATA ad=NULL;
+	struct Maat_rule_t rule;
+	fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
+	ex_desc->new_func(ex_desc->idx, &rule, srv_def, &ad, ex_desc->argl,ex_desc->argp);
+	return ad;
+}
+void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc)
+{
+	struct Maat_rule_t rule;
+	fill_maat_rule(&rule, rule_head, srv_def, strlen(srv_def)+1);
+	ex_desc->free_func(ex_desc->idx, &rule, srv_def, ad, ex_desc->argl,ex_desc->argp);
+	return;
+}
+
+
 _Maat_table_info_t* create_table_info(int max_thread_num)
 {
 	struct _Maat_table_info_t*p=NULL;
@@ -832,7 +849,7 @@ struct _Maat_group_inner_t* create_group_rule(int group_id)
 	pthread_mutex_init(&(group->mutex), NULL);
 	return group;
 }
-void force_destroy_group_rule(struct _Maat_group_inner_t* group)
+void _destroy_group_rule(struct _Maat_group_inner_t* group)
 {
 	dynamic_array_destroy(group->regions,free);
 	group->region_cnt=0;
@@ -854,7 +871,7 @@ void destroy_group_rule(struct _Maat_group_inner_t* group)
 	{
 		return;
 	}
-	force_destroy_group_rule(group);
+	_destroy_group_rule(group);
 }
 void make_group_set(const struct _Maat_compile_inner_t* compile_rule,universal_bool_expr_t* a_set)
 {
@@ -963,22 +980,39 @@ struct _Maat_compile_inner_t * create_compile_rule(int compile_id)
 	p=(struct _Maat_compile_inner_t*)calloc(sizeof(struct _Maat_compile_inner_t),1);
 	p->compile_id=compile_id;
 	p->group_cnt=0;
-	p->table_id=0;
 	p->group_boundary=1;
 	p->groups=dynamic_array_create(1, 1);
+	p->ads=ALLOC(MAAT_RULE_EX_DATA, MAX_COMPILE_EX_DATA_NUM);
 	pthread_rwlock_init(&(p->rwlock), NULL);
 	return p;
 }
-void force_destroy_compile_rule(struct _Maat_compile_inner_t * p)
+void _destroy_compile_rule(struct _Maat_compile_inner_t * compile_rule)
 {
-	p->compile_id=-1;
+	const struct _Maat_table_info_t* table=compile_rule->ref_table;
-	dynamic_array_destroy(p->groups,NULL);
+	struct db_compile_rule_t* db_compile_rule=compile_rule->db_c_rule;
-	if(p->db_c_rule!=NULL)
+	int i=0;
+	compile_rule->compile_id=-1;
+	dynamic_array_destroy(compile_rule->groups,NULL);
+
+	
+	pthread_rwlock_wrlock(&(compile_rule->rwlock));
+	if(db_compile_rule!=NULL)
 	{
-		free(p->db_c_rule);
+		free(db_compile_rule->service_defined);
+		free(db_compile_rule);
+		compile_rule->db_c_rule=NULL;
 	}
-	pthread_rwlock_destroy(&(p->rwlock));
+	for(i=0; i<table->ex_data_num; i++)
-	free(p);
+	{
+		rule_ex_data_free(&(db_compile_rule->m_rule_head), db_compile_rule->service_defined, compile_rule->ads+i, table->ex_desc+i);
+		compile_rule->ads[i]=NULL;
+	}
+	
+	free(compile_rule->ads);
+	pthread_rwlock_unlock(&(compile_rule->rwlock));
+
+	pthread_rwlock_destroy(&(compile_rule->rwlock));
+	free(compile_rule);
 }
 void destroy_compile_rule(struct _Maat_compile_inner_t * p)
 {
@@ -990,7 +1024,7 @@ void destroy_compile_rule(struct _Maat_compile_inner_t * p)
 		p_group=(struct _Maat_compile_inner_t*)dynamic_array_read(p->groups,i);
 		assert(p_group==NULL);
 	}
-	force_destroy_compile_rule(p);
+	_destroy_compile_rule(p);
 }
 scan_rule_t* create_rs_str_rule(unsigned int sub_type,enum MAAT_MATCH_METHOD match_method,int is_case_sensitive,const char* string,int len,int l_offset,int r_offset)
 {
@@ -1265,8 +1299,8 @@ void destroy_maat_scanner(struct _Maat_scanner_t*scanner)
 		return;
 	}
 	rulescan_destroy(scanner->region);
-	MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))force_destroy_compile_rule);
+	MESA_htable_destroy(scanner->compile_hash,(void (*)(void*))_destroy_compile_rule);
-	MESA_htable_destroy(scanner->group_hash, (void (*)(void*))force_destroy_group_rule);
+	MESA_htable_destroy(scanner->group_hash, (void (*)(void*))_destroy_group_rule);
 	MESA_htable_destroy(scanner->region_hash, NULL);
 	map_destroy(scanner->district_map);
 	scanner->district_map=NULL;
@@ -2254,18 +2288,31 @@ int add_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
 {
 	struct _Maat_compile_inner_t *compile_rule=NULL;
 	struct _head_Maat_rule_t *p_maat_rule_head=&(db_compile_rule->m_rule_head);
+	int i=0;
 	compile_rule=(struct _Maat_compile_inner_t*)HASH_fetch_by_id(scanner->compile_hash, p_maat_rule_head->config_id);
 	if(compile_rule==NULL)
 	{
 		compile_rule=create_compile_rule(p_maat_rule_head->config_id);
-		compile_rule->table_id=table->table_id;
 		HASH_add_by_id(scanner->compile_hash,p_maat_rule_head->config_id,compile_rule);
 	}
-	if(compile_rule->db_c_rule!=NULL)//duplicate config
+	else
 	{
-		return -1;
+		if(compile_rule->db_c_rule!=NULL)//duplicate config
+		{
+			return -1;
+		}
 	}
+
+	pthread_rwlock_wrlock(&(compile_rule->rwlock));		
+	compile_rule->ref_table=table;
 	compile_rule->db_c_rule=db_compile_rule;
+	for(i=0; i<table->ex_data_num; i++)
+	{
+		compile_rule->ads[i]=rule_ex_data_new(p_maat_rule_head, db_compile_rule->service_defined, table->ex_desc+i);
+	}
+	compile_rule->is_valid=1;
+	pthread_rwlock_unlock(&(compile_rule->rwlock));
+
 	return 0;
 	
 }
@@ -2282,12 +2329,7 @@ int del_compile_rule(struct _Maat_table_info_t* table,struct db_compile_rule_t*
 		return -1;
 	}
 	pthread_rwlock_wrlock(&(compile_rule->rwlock));
-	if(compile_rule->db_c_rule!=NULL)
+	compile_rule->is_valid=0;
-	{
-		free(compile_rule->db_c_rule->service_defined);
-		free(compile_rule->db_c_rule);
-		compile_rule->db_c_rule=NULL;
-	}
 	pthread_rwlock_unlock(&(compile_rule->rwlock));
 
 	if(compile_rule->group_cnt==0)
@@ -2304,7 +2346,14 @@ void update_group_rule(struct _Maat_table_info_t* table,const char* table_line,s
 	ret=sscanf(table_line,"%d\t%d\t%d",&(db_group_rule.group_id)
 										,&(db_group_rule.compile_id)
 										,&(db_group_rule.is_valid));
-	assert(ret==3);
+	if(ret!=3)
+	{
+		MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module ,
+							"update error,invalid format of group table %s:%s"
+							,table->table_name[table->updating_name], table_line);	
+		table->udpate_err_cnt++;
+		return;
+	}
 	if(db_group_rule.is_valid==FALSE)
 	{
 		del_group_rule(table, &db_group_rule,scanner,logger);

src/entry/config_monitor.cpp

@@ -234,10 +234,10 @@ int  get_new_idx_path(long long current_version,const char*file_dir,void* logger
 		{
 			continue;
 		}
-		if(strlen(namelist[i]->d_name)>32)
+		if(strlen(namelist[i]->d_name)>42)
 		{
 			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor 
-										,"config file %s filename too long,should like full_config_index.0000000001"
+										,"config file %s filename too long,should like full_config_index.00000000000000000001"
 										,namelist[i]->d_name);
 			continue;
 		}
@@ -245,7 +245,7 @@ int  get_new_idx_path(long long current_version,const char*file_dir,void* logger
 		if(sscanf_ret!=2)
 		{
 			MESA_handle_runtime_log(logger,RLOG_LV_FATAL,module_config_monitor 
-										,"config file %s filename error,should like full_config_index.0000000001"
+										,"config file %s filename error,should like full_config_index.00000000000000000001"
 										,namelist[i]->d_name);
 			continue;
 		}

src/entry/json2iris.cpp

@@ -40,8 +40,8 @@ struct iris_description_t
 	char tmp_iris_index_dir[MAX_PATH_LINE];
 	char index_path[MAX_PATH_LINE];
 
-	struct iris_table_t group_table;
+	struct iris_table_t* group_table;
-	struct iris_table_t compile_table;
+	struct iris_table_t* compile_table;
 	MESA_htable_handle group_name_map;
 	MESA_htable_handle iris_table_map;
 	MESA_htable_handle str2int_map;
@@ -57,6 +57,21 @@ struct traslate_command_t
 	const char* default_string;
 	int default_int;
 };
+struct iris_table_t* query_table_info(iris_description_t* p_iris,const char* table_name)
+{
+	struct iris_table_t* table_info=NULL;
+	table_info=(struct iris_table_t*)MESA_htable_search(p_iris->iris_table_map, (const unsigned char*)table_name,strlen(table_name));
+	if(table_info==NULL)
+	{
+		table_info=(struct iris_table_t*)calloc(sizeof(struct iris_table_t),1);
+		table_info->line_count=0;
+		memcpy(table_info->table_name,table_name,MIN(sizeof(table_info->table_name)-1, strlen(table_name)));
+		snprintf(table_info->table_path,sizeof(table_info->table_path),"%s/%s.local",p_iris->tmp_iris_dir,table_info->table_name);
+		MESA_htable_add(p_iris->iris_table_map,(const unsigned char*)table_info->table_name,strlen(table_info->table_name),table_info);
+	}
+	return table_info;
+}
+
 static int get_group_seq(struct iris_description_t* iris_cfg)
 {
 	redisReply* data_reply=NULL;
@@ -99,15 +114,6 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,
 	snprintf(iris_cfg->tmp_iris_index_dir,sizeof(iris_cfg->tmp_iris_index_dir),"%s_iris_tmp/index",json_file);
 	snprintf(iris_cfg->index_path,sizeof(iris_cfg->index_path),"%s/full_config_index.%010d",iris_cfg->tmp_iris_index_dir,json_version);
 
-
-	memcpy(iris_cfg->compile_table.table_name,compile_tn,strlen(compile_tn));
-	snprintf(iris_cfg->compile_table.table_path,sizeof(iris_cfg->compile_table.table_path),
-			"%s/%s.local",iris_cfg->tmp_iris_dir,iris_cfg->compile_table.table_name);
-
-	memcpy(iris_cfg->group_table.table_name,group_tn,strlen(group_tn));
-	snprintf(iris_cfg->group_table.table_path,sizeof(iris_cfg->group_table.table_name),
-			"%s/%s.local",iris_cfg->tmp_iris_dir,iris_cfg->group_table.table_name);
-
 	iris_cfg->redis_write_ctx=redis_write_ctx;
 	MESA_htable_create_args_t hargs;
 	memset(&hargs,0,sizeof(hargs));
@@ -162,6 +168,9 @@ int set_iris_descriptor(const char* json_file,cJSON *json,const char*compile_tn,
 	map_register(iris_cfg->str2int_map, "hexbin",1);
 	map_register(iris_cfg->str2int_map, "case plain",2);
 
+	iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn);
+	iris_cfg->group_table=query_table_info(iris_cfg, group_tn);
+
 	return 0;
 }
 void clear_iris_descriptor(struct iris_description_t *iris_cfg)
@@ -174,6 +183,7 @@ void clear_iris_descriptor(struct iris_description_t *iris_cfg)
 	{
 		MESA_htable_destroy(iris_cfg->iris_table_map, NULL);
 	}
+	map_destroy(iris_cfg->str2int_map);
 	return;
 }
 int create_tmp_dir(struct iris_description_t *p)
@@ -532,20 +542,7 @@ int write_similar_rule(cJSON *region_json,struct iris_description_t *p_iris,cons
 	return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger);
 
 }
-struct iris_table_t* query_table_info(iris_description_t* p_iris,const char* table_name)
+
-{
-	struct iris_table_t* table_info=NULL;
-	table_info=(struct iris_table_t*)MESA_htable_search(p_iris->iris_table_map, (const unsigned char*)table_name,strlen(table_name));
-	if(table_info==NULL)
-	{
-		table_info=(struct iris_table_t*)calloc(sizeof(struct iris_table_t),1);
-		table_info->line_count=0;
-		memcpy(table_info->table_name,table_name,MIN(sizeof(table_info->table_name)-1, strlen(table_name)));
-		snprintf(table_info->table_path,sizeof(table_info->table_path),"%s/%s.local",p_iris->tmp_iris_dir,table_info->table_name);
-		MESA_htable_add(p_iris->iris_table_map,(const unsigned char*)table_info->table_name,strlen(table_info->table_name),table_info);
-	}
-	return table_info;
-}
 int write_plugin_table(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger)
 {
 	cJSON* item=NULL,*table_content=NULL,*each_line=NULL;
@@ -695,8 +692,12 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l
 {
 	int compile_id=-1,cmd_cnt=0,ret=-1;
 	cJSON* item=NULL;
+	struct iris_table_t* table_info=NULL;
+	
 	struct traslate_command_t compile_cmd[MAX_COLUMN_NUM];
 	memset(compile_cmd,0,sizeof(compile_cmd));
+
+
 	compile_cmd[cmd_cnt].json_string="compile_id";
 	compile_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;
@@ -733,16 +734,25 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l
 	compile_cmd[cmd_cnt].str2int_flag=1;
 	cmd_cnt++;
 
-	
+	item=cJSON_GetObjectItem(compile,"table_name");
-	if(p_iris->compile_table.line_count==0)
+	if(item==NULL||item->type!=cJSON_String)
 	{
-		ret=set_file_rulenum(p_iris->compile_table.table_path, 0,logger);
+		table_info=p_iris->compile_table;
+	}
+	else
+	{
+		table_info=query_table_info(p_iris,item->valuestring);
+	}
+	
+	if(table_info->line_count==0)
+	{
+		ret=set_file_rulenum(table_info->table_path, 0,logger);
 		if(ret<0)
 		{
 			return -1;
 		}
 	}
-	ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, p_iris->compile_table.table_path,logger);
+	ret=direct_write_rule(compile, p_iris->str2int_map,compile_cmd,cmd_cnt, table_info->table_path,logger);
 	if(ret<0)
 	{
 		return -1;
@@ -755,8 +765,8 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l
 		return -1;
 	}
 	compile_id=item->valueint;
-	p_iris->compile_table.line_count++;
+	table_info->line_count++;
-	set_file_rulenum(p_iris->compile_table.table_path,p_iris->compile_table.line_count,logger);
+	set_file_rulenum(table_info->table_path,table_info->line_count,logger);
 	return compile_id;
 }
 int write_group_rule(int compile_id ,int group_id,struct iris_description_t *p_iris,void * logger)
@@ -764,25 +774,25 @@ int write_group_rule(int compile_id ,int group_id,struct iris_description_t *p_i
 	FILE*fp=NULL;
 	int ret=0;
 
-	if(p_iris->group_table.line_count==0)
+	if(p_iris->group_table->line_count==0)
 	{
-		ret=set_file_rulenum(p_iris->group_table.table_path,0,logger);
+		ret=set_file_rulenum(p_iris->group_table->table_path,0,logger);
 		if(ret<0)
 		{
 			return -1;
 		}
 	}
-	fp=fopen(p_iris->group_table.table_path,"a");
+	fp=fopen(p_iris->group_table->table_path,"a");
 	if(fp==NULL)
 	{
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,
-										"fopen %s error %s.",p_iris->group_table.table_path,strerror(errno));
+										"fopen %s error %s.",p_iris->group_table->table_path,strerror(errno));
 		return -1;
 	}
 	fprintf(fp,"%d\t%d\t1\n",group_id,compile_id);
 	fclose(fp);
-	p_iris->group_table.line_count++;
+	p_iris->group_table->line_count++;
-	ret=set_file_rulenum(p_iris->group_table.table_path,p_iris->group_table.line_count,logger);
+	ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger);
 	return 0;
 }
 void table_idx_write_cb(const uchar * key, uint size, void * data, void * user)
@@ -801,12 +811,6 @@ int write_index_file(struct iris_description_t *p_iris,void* logger)
 									"index file %s fopen error %s.",p_iris->index_path,strerror(errno));
 		return -1;
 	}
-	fprintf(fp,"%s\t%d\t%s\n",p_iris->compile_table.table_name
-						,p_iris->compile_table.line_count
-						,p_iris->compile_table.table_path);
-	fprintf(fp,"%s\t%d\t%s\n",p_iris->group_table.table_name
-						,p_iris->group_table.line_count
-						,p_iris->group_table.table_path);
 	MESA_htable_iterate(p_iris->iris_table_map, table_idx_write_cb, fp);
 	fclose(fp);
 	return 0;
@@ -876,8 +880,6 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger)
 			item=cJSON_GetObjectItem(group_rule,"group_name");
 			if(item==NULL||item->type!=cJSON_String)
 			{
-				MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_json,
-											"compile rule %d's group has no name,regard as Untitled.",compile_id);
 				group_name=untitled_group_name;
 			}
 			else
@@ -984,7 +986,7 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r
 	
 
 	json_file_size=fstat_buf.st_size;
-	json_buff=(char*)calloc(1,json_file_size);
+	json_buff=(char*)calloc(1,json_file_size+1);
 	read_size=fread(json_buff,1,json_file_size,json_fp);
 	if(read_size!=json_file_size)
 	{
@@ -1000,14 +1002,15 @@ int json2iris(const char* json_file,const char*compile_tn,const char* group_tn,r
 		MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json,"Error before: %-200.200s",cJSON_GetErrorPtr());
 		goto error_out;
 	}
-	if(compile_tn==NULL)
+	tmp_obj=cJSON_GetObjectItem(json,"compile_table");
+	if(tmp_obj)
 	{
-		tmp_obj=cJSON_GetObjectItem(json,"compile_table");
 		compile_tn=tmp_obj->valuestring;
 	}
-	if(group_tn==NULL)
+
+	tmp_obj=cJSON_GetObjectItem(json,"group_table");
+	if(tmp_obj)
 	{
-		tmp_obj=cJSON_GetObjectItem(json,"group_table");
 		group_tn=tmp_obj->valuestring;
 	}
 	ret=set_iris_descriptor(json_file,json,compile_tn,group_tn,redis_write_ctx,&iris_cfg,logger);

src/entry/map_str2int.cpp

@@ -95,7 +95,7 @@ void duplicate_func(const uchar * key, uint size, void * data, void * user)
 {
 	MESA_htable_handle target=(MESA_htable_handle)user;
 	int value=*(int*)data;
-	int ret=0;
+	__attribute__((unused))	int ret=0;
 	char* string=(char*)calloc(sizeof(char),(size+1));
 	memcpy(string,key,size);
 	ret=map_register(target,string,value);
@@ -103,7 +103,7 @@ void duplicate_func(const uchar * key, uint size, void * data, void * user)
 }
 MESA_htable_handle map_duplicate(MESA_htable_handle origin_map)
 {
-	int ret=0;
+	__attribute__((unused))	int ret=0;
 	MESA_htable_handle target=NULL;
 	target=map_create();
 	ret=MESA_htable_iterate(origin_map, duplicate_func, target);

src/inc_internal/Maat_rule_internal.h

@@ -46,7 +46,7 @@ typedef int atomic_t;
 #define	MAX_PLUGIN_PER_TABLE		32
 #define	MAX_FOREIGN_CLMN_NUM		8
 #define	MAX_SCANNER_HIT_NUM			64
-
+#define	MAX_COMPILE_EX_DATA_NUM		8
 #define	MAX_GROUP_CACHE		128
 
 #define	MAX_FAILED_NUM		128
@@ -84,6 +84,67 @@ enum MAAT_TABLE_TYPE
 	TABLE_TYPE_PLUGIN
 
 };
+enum USER_REGION_ENCODE
+{
+	USER_REGION_ENCODE_NONE=0,
+	USER_REGION_ENCODE_ESCAPE,
+	USER_REGION_ENCODE_BASE64
+};
+struct compile_ex_data_idx
+{
+	Maat_rule_EX_new_func_t *new_func;
+	Maat_rule_EX_free_func_t* free_func;
+	Maat_rule_EX_dup_func_t* dup_func;
+	long argl;
+	void *argp;
+	int idx;
+	int table_id;
+};
+struct _Maat_table_info_t
+{
+	unsigned short table_id;
+	unsigned short conj_cnt;
+	unsigned short updating_name;
+	char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
+	enum MAAT_TABLE_TYPE table_type;
+	enum MAAT_CHARSET src_charset;
+	enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
+	int src_charset_in_dst;
+	int do_charset_merge;
+	int cfg_num;
+	int cross_cache_size;
+	int quick_expr_switch;
+	union
+	{
+		int expr_rule_cnt;	//expr_type=0,1,3
+		int ipv4_rule_cnt;
+	};
+	union
+	{
+		int regex_rule_cnt;	//expr_type=2
+		int ipv6_rule_cnt;
+	};
+	struct _plugin_table_info *cb_info;
+	int valid_flag_column;		//for plugin table
+	int rule_tag_column;		//for plugin table;
+	int foreign_columns[MAX_FOREIGN_CLMN_NUM];			//for plugin table;
+	int n_foreign;
+
+	//for compile table
+	enum USER_REGION_ENCODE user_region_encoding;
+	int ex_data_num;
+	struct compile_ex_data_idx ex_desc[MAX_COMPILE_EX_DATA_NUM];
+//for stat>>>>>>>>
+	unsigned long long udpate_err_cnt;
+	unsigned long long unmatch_tag_cnt;
+	unsigned long long iconv_err_cnt;
+	int stat_line_id;
+	mcore_long_t	scan_cnt;
+	mcore_long_t	scan_cpu_time;	//nano
+	mcore_long_t	input_bytes;
+	mcore_long_t	stream_num;
+	mcore_long_t	hit_cnt;
+};
 
 struct db_str_rule_t
 {
@@ -188,10 +249,12 @@ struct _Maat_compile_inner_t
 {
 	struct db_compile_rule_t *db_c_rule;
 	dynamic_array_t *groups;
+	int is_valid;
 	int compile_id;//equal to db_c_rule->m_rule.config_id
-	int table_id;
+	const struct _Maat_table_info_t* ref_table;
 	int group_boundary;
 	int group_cnt;
+	MAAT_RULE_EX_DATA* ads;
 	pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
 };
 struct _compile_result_t
@@ -231,51 +294,8 @@ struct _region_stat_t
 		int ipv6_rule_cnt;
 	};
 };
-#define	USER_REGION_ENCODE_NONE		0
-#define	USER_REGION_ENCODE_ESCAPE	1
-#define	USER_REGION_ENCODE_BASE64	2
 
-struct _Maat_table_info_t
+
-{
-	unsigned short table_id;
-	unsigned short conj_cnt;
-	unsigned short updating_name;
-	char table_name[MAX_CONJUNCTION_TABLE_NUM][MAX_TABLE_NAME_LEN];
-	enum MAAT_TABLE_TYPE table_type;
-	enum MAAT_CHARSET src_charset;
-	enum MAAT_CHARSET dst_charset[MAX_CHARSET_NUM];
-	int src_charset_in_dst;
-	int do_charset_merge;
-	int cfg_num;
-	int cross_cache_size;
-	int quick_expr_switch;
-	union
-	{
-		int expr_rule_cnt;	//expr_type=0,1,3
-		int ipv4_rule_cnt;
-	};
-	union
-	{
-		int regex_rule_cnt;	//expr_type=2
-		int ipv6_rule_cnt;
-	};
-	struct _plugin_table_info *cb_info;
-	int valid_flag_column;		//for plugin table
-	int rule_tag_column;		//for plugin table;
-	int foreign_columns[MAX_FOREIGN_CLMN_NUM];			//for plugin table;
-	int n_foreign;
-	int user_region_encoding;	//for compile table, USER_REGION_ENCODE_xx
-//for stat>>>>>>>>
-	unsigned long long udpate_err_cnt;
-	unsigned long long unmatch_tag_cnt;
-	unsigned long long iconv_err_cnt;
-	int stat_line_id;
-	mcore_long_t	scan_cnt;
-	mcore_long_t	scan_cpu_time;	//nano
-	mcore_long_t	input_bytes;
-	mcore_long_t	stream_num;
-	mcore_long_t	hit_cnt;
-};
 
 struct _INNER_scan_status_t
 {
@@ -472,6 +492,7 @@ struct serial_rule_t		//rm= Redis Maat
 	enum MAAT_OPERATION op;//0: delete, 1: add.
 	long rule_id;
 	int label_id;
+	char with_error;
 	long long timeout;			// absolute unix time.
 	char table_name[256];
 	char* table_line;
@@ -505,6 +526,10 @@ int get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule_t* rule_lis
 void get_foreign_conts(redisContext *ctx, struct serial_rule_t* rule_list, int rule_num, int print_fn, void *logger);
 void rewrite_table_line_with_foreign(struct serial_rule_t*p);
 
+void fill_maat_rule(struct Maat_rule_t *rule, const struct _head_Maat_rule_t* rule_head, const char* srv_def, int srv_def_len);
+MAAT_RULE_EX_DATA rule_ex_data_new(const struct _head_Maat_rule_t * rule_head, const char* srv_def, const struct compile_ex_data_idx* ex_desc);
+void rule_ex_data_free(const struct _head_Maat_rule_t * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
+
 
 void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);
 void empty_serial_rules(struct serial_rule_t* rule);

src/inc_internal/cJSON.h

@@ -1,19 +1,19 @@
 /*
-  Copyright (c) 2009 Dave Gamble
+  Copyright (c) 2009-2017 Dave Gamble and cJSON contributors
- 
+
   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:
- 
+
   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.
- 
+
   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE EXPR_TYPE_AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
@@ -28,113 +28,247 @@ extern "C"
 {
 #endif
 
+/* project version */
+#define CJSON_VERSION_MAJOR 1
+#define CJSON_VERSION_MINOR 7
+#define CJSON_VERSION_PATCH 7
+
+#include <stddef.h>
+
 /* cJSON Types: */
-#define cJSON_False 0
+#define cJSON_Invalid (0)
-#define cJSON_True 1
+#define cJSON_False  (1 << 0)
-#define cJSON_NULL 2
+#define cJSON_True   (1 << 1)
-#define cJSON_Number 3
+#define cJSON_NULL   (1 << 2)
-#define cJSON_String 4
+#define cJSON_Number (1 << 3)
-#define cJSON_Array 5
+#define cJSON_String (1 << 4)
-#define cJSON_Object 6
+#define cJSON_Array  (1 << 5)
-	
+#define cJSON_Object (1 << 6)
+#define cJSON_Raw    (1 << 7) /* raw json */
+
 #define cJSON_IsReference 256
+#define cJSON_StringIsConst 512
 
 /* The cJSON structure: */
-typedef struct cJSON {
+typedef struct cJSON
-	struct cJSON *next,*prev;	/* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */
+{
-	struct cJSON *child;		/* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */
+    /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */
+    struct cJSON *next;
+    struct cJSON *prev;
+    /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */
+    struct cJSON *child;
 
-	int type;					/* The type of the item, as above. */
+    /* The type of the item, as above. */
+    int type;
 
-	char *valuestring;			/* The item's string, if type==cJSON_String */
+    /* The item's string, if type==cJSON_String  and type == cJSON_Raw */
-	int valueint;				/* The item's number, if type==cJSON_Number */
+    char *valuestring;
-	double valuedouble;			/* The item's number, if type==cJSON_Number */
+    /* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */
+    int valueint;
+    /* The item's number, if type==cJSON_Number */
+    double valuedouble;
 
-	char *string;				/* The item's name string, if this item is the child of, or is in the list of subitems of an object. */
+    /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */
+    char *string;
 } cJSON;
 
-typedef struct cJSON_Hooks {
+typedef struct cJSON_Hooks
+{
       void *(*malloc_fn)(size_t sz);
       void (*free_fn)(void *ptr);
 } cJSON_Hooks;
 
+typedef int cJSON_bool;
+
+#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32))
+#define __WINDOWS__
+#endif
+#ifdef __WINDOWS__
+
+/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention.  For windows you have 2 define options:
+
+CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols
+CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default)
+CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol
+
+For *nix builds that support visibility attribute, you can define similar behavior by
+
+setting default visibility to hidden by adding
+-fvisibility=hidden (for gcc)
+or
+-xldscope=hidden (for sun cc)
+to CFLAGS
+
+then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does
+
+*/
+
+/* export symbols by default, this is necessary for copy pasting the C and header file */
+#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS)
+#define CJSON_EXPORT_SYMBOLS
+#endif
+
+#if defined(CJSON_HIDE_SYMBOLS)
+#define CJSON_PUBLIC(type)   type __stdcall
+#elif defined(CJSON_EXPORT_SYMBOLS)
+#define CJSON_PUBLIC(type)   __declspec(dllexport) type __stdcall
+#elif defined(CJSON_IMPORT_SYMBOLS)
+#define CJSON_PUBLIC(type)   __declspec(dllimport) type __stdcall
+#endif
+#else /* !WIN32 */
+#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY)
+#define CJSON_PUBLIC(type)   __attribute__((visibility("default"))) type
+#else
+#define CJSON_PUBLIC(type) type
+#endif
+#endif
+
+/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them.
+ * This is to prevent stack overflows. */
+#ifndef CJSON_NESTING_LIMIT
+#define CJSON_NESTING_LIMIT 1000
+#endif
+
+/* returns the version of cJSON as a string */
+CJSON_PUBLIC(const char*) cJSON_Version(void);
+
 /* Supply malloc, realloc and free functions to cJSON */
-extern void cJSON_InitHooks(cJSON_Hooks* hooks);
+CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks);
 
+/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */
+/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */
+CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value);
+/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */
+/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */
+CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated);
 
-/* Supply a block of JSON, and this returns a cJSON object you can interrogate. Call cJSON_Delete when finished. */
+/* Render a cJSON entity to text for transfer/storage. */
-extern cJSON *cJSON_Parse(const char *value);
+CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item);
-/* Render a cJSON entity to text for transfer/storage. Free the char* when finished. */
+/* Render a cJSON entity to text for transfer/storage without any formatting. */
-extern char  *cJSON_Print(cJSON *item);
+CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item);
-/* Render a cJSON entity to text for transfer/storage without any formatting. Free the char* when finished. */
+/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */
-extern char  *cJSON_PrintUnformatted(cJSON *item);
+CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt);
+/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */
+/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */
+CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format);
 /* Delete a cJSON entity and all subentities. */
-extern void   cJSON_Delete(cJSON *c);
+CJSON_PUBLIC(void) cJSON_Delete(cJSON *c);
 
 /* Returns the number of items in an array (or object). */
-extern int	  cJSON_GetArraySize(cJSON *array);
+CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array);
-/* Retrieve item number "item" from array "array". Returns NULL if unsuccessful. */
+/* Retrieve item number "index" from array "array". Returns NULL if unsuccessful. */
-extern cJSON *cJSON_GetArrayItem(cJSON *array,int item);
+CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index);
 /* Get item "string" from object. Case insensitive. */
-extern cJSON *cJSON_GetObjectItem(cJSON *object,const char *string);
+CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string);
-
+CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string);
+CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string);
 /* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */
-extern const char *cJSON_GetErrorPtr(void);
+CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void);
-	
+
+/* Check if the item is a string and return its valuestring */
+CJSON_PUBLIC(char *) cJSON_GetStringValue(cJSON *item);
+
+/* These functions check the type of an item */
+CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item);
+CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item);
+CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item);
+CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item);
+CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item);
+CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item);
+CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item);
+CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item);
+CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item);
+CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item);
+
 /* These calls create a cJSON item of the appropriate type. */
-extern cJSON *cJSON_CreateNull(void);
+CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void);
-extern cJSON *cJSON_CreateTrue(void);
+CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void);
-extern cJSON *cJSON_CreateFalse(void);
+CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void);
-extern cJSON *cJSON_CreateBool(int b);
+CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean);
-extern cJSON *cJSON_CreateNumber(double num);
+CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num);
-extern cJSON *cJSON_CreateString(const char *string);
+CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string);
-extern cJSON *cJSON_CreateArray(void);
+/* raw json */
-extern cJSON *cJSON_CreateObject(void);
+CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw);
+CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void);
+CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void);
+
+/* Create a string where valuestring references a string so
+ * it will not be freed by cJSON_Delete */
+CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string);
+/* Create an object/arrray that only references it's elements so
+ * they will not be freed by cJSON_Delete */
+CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child);
+CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child);
 
 /* These utilities create an Array of count items. */
-extern cJSON *cJSON_CreateIntArray(const int *numbers,int count);
+CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count);
-extern cJSON *cJSON_CreateFloatArray(const float *numbers,int count);
+CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count);
-extern cJSON *cJSON_CreateDoubleArray(const double *numbers,int count);
+CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count);
-extern cJSON *cJSON_CreateStringArray(const char **strings,int count);
+CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char **strings, int count);
 
 /* Append item to the specified array/object. */
-extern void cJSON_AddItemToArray(cJSON *array, cJSON *item);
+CJSON_PUBLIC(void) cJSON_AddItemToArray(cJSON *array, cJSON *item);
-extern void	cJSON_AddItemToObject(cJSON *object,const char *string,cJSON *item);
+CJSON_PUBLIC(void) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item);
+/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object.
+ * WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before
+ * writing to `item->string` */
+CJSON_PUBLIC(void) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item);
 /* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */
-extern void cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item);
+CJSON_PUBLIC(void) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item);
-extern void	cJSON_AddItemReferenceToObject(cJSON *object,const char *string,cJSON *item);
+CJSON_PUBLIC(void) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item);
 
 /* Remove/Detatch items from Arrays/Objects. */
-extern cJSON *cJSON_DetachItemFromArray(cJSON *array,int which);
+CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item);
-extern void   cJSON_DeleteItemFromArray(cJSON *array,int which);
+CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which);
-extern cJSON *cJSON_DetachItemFromObject(cJSON *object,const char *string);
+CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which);
-extern void   cJSON_DeleteItemFromObject(cJSON *object,const char *string);
+CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string);
-	
+CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string);
+CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string);
+CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string);
+
 /* Update array items. */
-extern void cJSON_ReplaceItemInArray(cJSON *array,int which,cJSON *newitem);
+CJSON_PUBLIC(void) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */
-extern void cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem);
+CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement);
+CJSON_PUBLIC(void) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem);
+CJSON_PUBLIC(void) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem);
+CJSON_PUBLIC(void) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem);
 
 /* Duplicate a cJSON item */
-extern cJSON *cJSON_Duplicate(cJSON *item,int recurse);
+CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse);
 /* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will
 need to be released. With recurse!=0, it will duplicate any children connected to the item.
 The item->next and ->prev pointers are always zero on return from Duplicate. */
+/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal.
+ * case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */
+CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive);
 
-/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */
-extern cJSON *cJSON_ParseWithOpts(const char *value,const char **return_parse_end,int require_null_terminated);
 
-extern void cJSON_Minify(char *json);
+CJSON_PUBLIC(void) cJSON_Minify(char *json);
 
-/* Macros for creating things quickly. */
+/* Helper functions for creating and adding items to an object at the same time.
-#define cJSON_AddNullToObject(object,name)		cJSON_AddItemToObject(object, name, cJSON_CreateNull())
+ * They return the added item or NULL on failure. */
-#define cJSON_AddTrueToObject(object,name)		cJSON_AddItemToObject(object, name, cJSON_CreateTrue())
+CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name);
-#define cJSON_AddFalseToObject(object,name)		cJSON_AddItemToObject(object, name, cJSON_CreateFalse())
+CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name);
-#define cJSON_AddBoolToObject(object,name,b)	cJSON_AddItemToObject(object, name, cJSON_CreateBool(b))
+CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name);
-#define cJSON_AddNumberToObject(object,name,n)	cJSON_AddItemToObject(object, name, cJSON_CreateNumber(n))
+CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean);
-#define cJSON_AddStringToObject(object,name,s)	cJSON_AddItemToObject(object, name, cJSON_CreateString(s))
+CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number);
+CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string);
+CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw);
+CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name);
+CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name);
 
 /* When assigning an integer value, it needs to be propagated to valuedouble too. */
-#define cJSON_SetIntValue(object,val)			((object)?(object)->valueint=(object)->valuedouble=(val):(val))
+#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number))
+/* helper for the cJSON_SetNumberValue macro */
+CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number);
+#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number))
+
+/* Macro for iterating over an array or object */
+#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next)
+
+/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */
+CJSON_PUBLIC(void *) cJSON_malloc(size_t size);
+CJSON_PUBLIC(void) cJSON_free(void *object);
 
 #ifdef __cplusplus
 }

test/maat_demo.cpp

@@ -58,9 +58,16 @@ void Maat_read_entry_finish_cb(void* u_para)
 	long long version=0;
 	int ret=0,is_last_updating_table=0;
 	ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
-	assert(ret==0);
+	if(ret!=0)
+	{
+		assert(0);
+	}
 	ret=Maat_read_state(feather,MAAT_STATE_LAST_UPDATING_TABLE, &is_last_updating_table, sizeof(is_last_updating_table));
-	assert(ret==0);
+	if(ret!=0)
+	{
+		assert(0);
+	}
+
 	//printf("Maat Version %lld at plugin finish callback, is_last_update=%d.\n",version,is_last_updating_table);
 
 	return;
@@ -674,7 +681,7 @@ void test_set_cmd_line(Maat_feather_t feather)
 	const struct Maat_line_t *p_line[TEST_CMD_LINE_NUM];
 	struct Maat_line_t line_rule[TEST_CMD_LINE_NUM];
 	char table_line[TEST_CMD_LINE_NUM][128];
-	int ret=0,i=0;
+	int i=0;
 	memset(&line_rule,0,sizeof(line_rule));
 	for(i=0;i<TEST_CMD_LINE_NUM;i++)
 	{
@@ -688,16 +695,14 @@ void test_set_cmd_line(Maat_feather_t feather)
 	}
 	
 
-	ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
+	Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
-	assert(ret>0);
 	usleep(WAIT_FOR_EFFECTIVE_US);
 	
 	for(i=0;i<TEST_CMD_LINE_NUM;i++)
 	{
 		line_rule[i].table_line=NULL;
 	}
-	ret=Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
+	Maat_cmd_set_lines(feather, p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
-	assert(ret>0);
 
 
 	return;
@@ -707,12 +712,10 @@ void history_matter_cb(int table_id,const char* table_line,void* u_para)
 }
 void test_load_specific_version(Maat_feather_t feather, const char* redis_ip, unsigned short redis_port, int redis_db,void*logger)
 {
-	int ret=0;
 	long long version=0;
 	const char *table_name="HISTORY_MATTER_CB";
 	test_plugin_table(feather,table_name,NULL,history_matter_cb,NULL,NULL,logger);
-	ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
+	Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version));
-	assert(ret==0);	
 	printf("Maat Version %lld.\n",version);
 }
 int test_add_expr_command(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords)

test/maat_json.json

@@ -521,6 +521,33 @@
                     ]
                 }
             ]
+        },
+        {
+            "compile_id": 141,
+            "service": 1,
+            "action": 1,
+            "do_blacklist": 1,
+            "do_log": 1,
+            "user_region": "Something:I\\bhave\\ba\\bname,7799",
+            "table_name":"COMPILE_ALIAS",
+            "is_valid": "yes",
+            "groups": [
+                {
+                    "group_name": "Untitled",
+                    "regions": [
+                        {
+                            "table_name": "HTTP_URL",
+                            "table_type": "string",
+                            "table_content": {
+                                "keywords": "i.ytimg.com",
+                                "expr_type": "none",
+                                "match_method": "sub",
+                                "format": "uncase plain"
+                            }
+                        }
+                    ]
+                }
+            ]
         }
     ],
     "plugin_table": [

test/right_result.txt

@@ -1,35 +0,0 @@
-Load entry id 101 SUCCESS.
-hit 1 rules
-ipv4 scan hit compile rule id 123.
-hit current region,but not hit compile rule.
-ipv6 scan hit region.
-digest scan hit 127.
-digest scan hit 127.
-digest scan hit 127.
-digest scan hit 127.
-digest scan hit 127.
-digest scan hit 127.
-Hit expr_plus rule 128.
-URL encode scan utf8 url hit 1 rules, hit ruleid=129 
-URL encode scan gb2312 url hit 1 rules, hit ruleid=129 
-test_unicode_esc processing ./testdata_uni2ascii/original_Uygur_webpage.html
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 2 rules, hit ruleid=130 131 
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131 
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131 
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131 
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131 
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131 
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131 
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131 
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131 
-UNI2ASCII file ./testdata_uni2ascii/original_Uygur_webpage.html,hit 1 rules, hit ruleid=131 
-test_unicode_esc processing ./testdata_uni2ascii/original_uy.txt
-UNI2ASCII file ./testdata_uni2ascii/original_uy.txt,hit 2 rules, hit ruleid=130 131 
-test_unicode_esc processing ./testdata_uni2ascii/qq_mail_https.txt
-UNI2ASCII file ./testdata_uni2ascii/qq_mail_https.txt,hit 2 rules, hit ruleid=130 131 
-UNI2ASCII file ./testdata_uni2ascii/qq_mail_https.txt,hit 1 rules, hit ruleid=130 
-test_unicode_esc processing ./testdata_uni2ascii/sina_read_mail.txt
-UNI2ASCII file ./testdata_uni2ascii/sina_read_mail.txt,hit 2 rules, hit ruleid=130 131 
-test_unicode_esc processing ./testdata_uni2ascii/sohu_mail_unicode.txt
-UNI2ASCII file ./testdata_uni2ascii/sohu_mail_unicode.txt,hit 2 rules, hit ruleid=130 131 
-UNI2ASCII file ./testdata_uni2ascii/sohu_mail_unicode.txt,hit 2 rules, hit ruleid=130 131 

test/table_info.conf

@@ -28,4 +28,5 @@
 9	SIM_URL	similar	--
 10	IMAGE_FP	expr	UTF8	UTF8	yes	128	quickoff
 11	TEST_EFFECTIVE_RANGE_TABLE	plugin	{"valid":4,"tag":5}	--
-12	TEST_FOREIGN_KEY	plugin {"valid":4,"foreign":"6,8","tag":3}	--
+12	TEST_FOREIGN_KEY	plugin {"valid":4,"foreign":"6,8","tag":3}	--
+13	COMPILE_ALIAS	compile	escape	--

test/test_maatframe.cpp

@@ -18,6 +18,7 @@
 #include <unistd.h>
 #include <dirent.h>
 #include <openssl/md5.h>
+#include <pthread.h>
 
 #include <gtest/gtest.h>
 const char* test_maat_redis_ip="127.0.0.1";
@@ -540,6 +541,96 @@ TEST(RuleTags, Compile)
 	return;
 }
 
+struct rule_ex_param
+{
+	int ref_cnt;
+	char name[128];
+	int id;
+	pthread_mutex_t lock;
+};
+
+void ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, 
+												MAAT_RULE_EX_DATA* ad, long argl, void *argp)
+{
+	int *counter=(int*)argp;
+	*ad=NULL;
+	ASSERT_GT(rule->serv_def_len, 4);
+
+	struct rule_ex_param* param=(struct rule_ex_param*)calloc(sizeof(struct rule_ex_param), 1);
+	
+	param->ref_cnt=1;
+	pthread_mutex_init(&(param->lock), NULL);
+	
+	sscanf(srv_def_large,"%*[^:]:%[^,],%d",param->name,&(param->id));
+	(*counter)++;
+	*ad=param;
+	return;
+}
+void ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp)
+{
+	if(*ad==NULL)
+	{
+		return;
+	}
+	struct rule_ex_param* param=(struct rule_ex_param*)*ad;
+	pthread_mutex_lock(&(param->lock));
+	param->ref_cnt--;
+	if(param->ref_cnt>0)
+	{	
+		pthread_mutex_unlock(&(param->lock));
+		return;
+	}
+	free(param);
+	return;
+}
+void ex_param_dup(int idx,	MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp)
+{
+	struct rule_ex_param* from_param=*((struct rule_ex_param**)from);
+	pthread_mutex_lock(&(from_param->lock));
+	from_param->ref_cnt++;
+	pthread_mutex_unlock(&(from_param->lock));
+	*((struct rule_ex_param**)to)=from_param;
+	return;
+}
+
+
+
+TEST(EX_DATA_INDEX, all)
+{
+#define EX_data_index
+
+	int ret=0;
+	int table_id=0, ex_data_counter=0;
+	scan_status_t mid=NULL;
+	struct Maat_rule_t result[4];
+	const char* url="i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg";
+	const char* table_name="HTTP_URL";
+	const char* expect_name="I have a name";
+	table_id=Maat_table_register(g_feather,table_name);
+	ASSERT_GT(table_id, 0);
+	
+	int ex_param_idx=Maat_rule_get_ex_new_index(g_feather, "COMPILE_ALIAS", 
+													ex_param_new, ex_param_free, ex_param_dup,
+													0, &ex_data_counter);
+	ASSERT_TRUE(ex_param_idx>=0);
+	EXPECT_EQ(ex_data_counter, 1);
+	
+	ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url, strlen(url), 
+						result,NULL, 4,
+						&mid, 0);
+	
+	EXPECT_EQ(ret, 1);
+	void *ex_data=Maat_rule_get_ex_data(g_feather, result, ex_param_idx);
+	ASSERT_TRUE(ex_data!=NULL);
+	struct rule_ex_param* param=(struct rule_ex_param*)ex_data;
+	EXPECT_EQ(param->id, 7799);
+	EXPECT_EQ(strcmp(param->name, expect_name),0);
+	ex_param_free(0, NULL, NULL, &ex_data, 0, NULL);
+	Maat_clean_status(&mid);
+	return;
+}
+
+
 TEST(StreamFuzzyHash, Pure)
 {
 	
@@ -736,7 +827,6 @@ TEST_F(MaatFileTest, StreamFiles)
 		{
 			hit_cnt++;
 		}
-		printf("Stream Scan %s, ret=%d.\n",file_path,ret);
 		fclose(fp);
 		free(buff);
 		buff=NULL;