Plugin表更新后，不需要进行Hierarchy的重建。

src/entry/Maat_hierarchy.cpp

@@ -170,6 +170,7 @@ struct Maat_hierarchy
 	pthread_rwlock_t rwlock;
 	pthread_mutex_t mutex;
 	time_t version;				//After full update, clause id may indicate a different clause. Comparing hier->version and mid->hier_ver can prevent false positive match.
+	int changed_flag;
 	
 	struct Maat_hierarchy_compile* hash_compile_by_id;		//key: compile_id, value: struct Maat_hierarchy_compile*.
 	void (* compile_user_data_free)(void *compile_ud);
@@ -479,6 +480,7 @@ int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int
 	struct Maat_hierarchy_compile* compile=NULL;
 	
 	pthread_rwlock_wrlock(&hier->rwlock);
+	hier->changed_flag=1;
 	HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile);
 	if(!compile)
 	{
@@ -511,6 +513,7 @@ int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id)
 	int ret=0;
 	
 	pthread_rwlock_wrlock(&hier->rwlock);
+	hier->changed_flag=1;
 	HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile);
 	if(compile)
 	{
@@ -664,6 +667,7 @@ int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_i
 	struct Maat_hierarchy_compile* compile=NULL;
 	
 	pthread_rwlock_wrlock(&hier->rwlock);
+	hier->changed_flag=1;
 	HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
 	if(!group)
 	{
@@ -700,6 +704,7 @@ int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int gr
 	int ret=0;
 	
 	pthread_rwlock_wrlock(&hier->rwlock);
+	hier->changed_flag=1;
 	HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
 	if(!group)
 	{
@@ -746,6 +751,7 @@ int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id,
 	struct Maat_hierarchy_group* group=NULL, *superior_group=NULL;
 
 	pthread_rwlock_wrlock(&hier->rwlock);
+	hier->changed_flag=1;
 	HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
 	if(!group)
 	{
@@ -782,7 +788,7 @@ int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int grou
 	struct Maat_hierarchy_group* group=NULL, *superior_group=NULL;
 
 	//No hash write operation, LOCK protection is unnecessary.
-	
+	hier->changed_flag=1;
 	HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
 	if(group==NULL)
 	{
@@ -838,6 +844,7 @@ int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id
 	int ret=0;
 	
 	pthread_rwlock_wrlock(&hier->rwlock);
+	hier->changed_flag=1;
 	HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
 	if(!group)
 	{
@@ -867,6 +874,7 @@ void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int r
 	struct Maat_hierarchy_region* region=NULL;
 	void* ret=NULL;
 	pthread_rwlock_wrlock(&hier->rwlock);
+	hier->changed_flag=1;
 	HASH_FIND_INT(hier->hash_region_by_id, &region_id, region);
 	if(region)
 	{
@@ -881,6 +889,7 @@ int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int gro
 	struct Maat_hierarchy_group* group=NULL;
 	struct Maat_hierarchy_region* region=NULL;
 	pthread_rwlock_wrlock(&hier->rwlock);
+	hier->changed_flag=1;
 	HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
 	if(!group)
 	{
@@ -1209,6 +1218,10 @@ int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier)
 	struct region2clause_value* new_region2clause_hash=NULL, *old_region2clause_hash=NULL;
 
 	//Read hier from update thread is OK.
+	if(!hier->changed_flag)
+	{
+		return ret;
+	}
 	ret=Maat_hierarchy_build_top_groups(hier);
 	new_bm=Maat_hierarchy_build_bool_matcher(hier);
 	new_region2clause_hash=Maat_hierarchy_build_region2clause_hash(hier);
@@ -1220,6 +1233,7 @@ int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier)
 
 	hier->bm=new_bm;
 	hier->hash_region2clause=new_region2clause_hash;
+	hier->changed_flag=0;
 	pthread_rwlock_unlock(&hier->rwlock);
 
 	Maat_garbage_bagging(hier->ref_garbage_bin, old_bm, (void (*)(void*))bool_matcher_free);	

src/entry/Maat_rule.cpp

@@ -57,7 +57,7 @@ extern "C"
 }
 #endif
 
-int MAAT_FRAME_VERSION_3_4_3_20210727=1;
+int MAAT_FRAME_VERSION_3_4_4_20210805=1;
 
 int is_valid_table_name(const char* str)
 {
@@ -1414,7 +1414,6 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r
 	Maat_region_inner_add_expr_id(u_para, expr_id);
 
 	Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, u_para);
-	scanner->gie_update_q_size++;
 	return 0;
 }
 
@@ -1456,7 +1455,6 @@ int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id
 			assert(region->expr_id_cnt==1);
 			table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id);
 			Maat_table_runtime_digest_del(table_rt, region->expr_id_lb);
-			maat_scanner->gie_update_q_size++;
 			break;
 		default:
 			assert(0);
@@ -1521,7 +1519,6 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl
 			{
 				table_rt->group2compile.not_flag_group--;
 			}
-			scanner->to_update_group_cnt++;
 		}
 	}
 	else
@@ -1534,7 +1531,6 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl
 			{
 				table_rt->group2compile.not_flag_group++;
 			}
-			scanner->to_update_group_cnt++;
 		}
 	}
 	return;
@@ -1563,7 +1559,6 @@ void update_group2group_rule(struct Maat_table_schema* table, const char* table_
 		{
 			table_rt->origin_rule_num--;
 			assert(table_rt->origin_rule_num>=0);
-			scanner->to_update_group_cnt++;
 		}
 	}
 	else
@@ -1572,7 +1567,6 @@ void update_group2group_rule(struct Maat_table_schema* table, const char* table_
 		if(ret==0)
 		{
 			table_rt->origin_rule_num++;
-			scanner->to_update_group_cnt++;
 		}
 	}
 
@@ -2101,7 +2095,6 @@ void update_compile_rule(struct Maat_table_schema* table,const char* table_line
 			assert(ret==0);
 			table_rt->origin_rule_num--;
 			Maat_garbage_bagging(scanner->ref_garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule);
-			scanner->to_update_compile_cnt++;
 		}
 		else
 		{
@@ -2115,7 +2108,6 @@ void update_compile_rule(struct Maat_table_schema* table,const char* table_line
 		if(ret==0)
 		{
 			table_rt->origin_rule_num++;
-			scanner->to_update_compile_cnt++;
 		}
 		else
 		{
@@ -2317,7 +2309,7 @@ void update_xx_plugin_table(struct Maat_table_schema* table_schema, const char*
 	{
 		Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger);
 	}
-	scanner->xx_plugin_update_q_size++;
+	scanner->xx_plugin_rule_to_update_cnt++;
 	return;
 }
 
@@ -2402,10 +2394,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
 		Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))map_destroy);
 	}
 	scanner->last_update_time=time(NULL);
-	scanner->gie_update_q_size=0;
+	scanner->xx_plugin_rule_to_update_cnt=0;
-	scanner->to_update_group_cnt=0;
-	scanner->to_update_compile_cnt=0;
-	scanner->xx_plugin_update_q_size=0;
 	return;
 
 }
@@ -2451,7 +2440,7 @@ long long scanner_rule_num(struct Maat_scanner *scanner)
 void maat_finish_cb(void* u_para)
 {
 	struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; 
-	long expr_wait_q_cnt=0;
+	long rulescan_wait_q_cnt=0;
 
 	Maat_table_manager_all_plugin_cb_finish(feather->table_mgr);
 
@@ -2469,8 +2458,8 @@ void maat_finish_cb(void* u_para)
 	{
 		feather->scanner->cfg_num=scanner_rule_num(feather->scanner);
 		feather->scanner->version=feather->maat_version;
-		expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
+		rulescan_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
-		feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->xx_plugin_update_q_size;
+		feather->postpone_q_size=rulescan_wait_q_cnt;
 		if(time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000)
 		{
 			do_scanner_update(feather->scanner,
@@ -2563,7 +2552,7 @@ void *thread_rule_monitor(void *arg)
 {
 	struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg;
 	struct Maat_scanner* old_scanner=NULL;
-	long expr_wait_q_cnt=0;
+	long rulescan_update_wait_q_cnt=0;
 	int scan_dir_cnt=0;
 	int ret=0;
 	char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0};
@@ -2695,9 +2684,9 @@ void *thread_rule_monitor(void *arg)
 			}
 			if(feather->scanner!=NULL)
 			{
-				expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
+				rulescan_update_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
-				feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size;
+				feather->postpone_q_size=rulescan_update_wait_q_cnt;
-				total_wait_rule_cnt=feather->postpone_q_size+feather->scanner->to_update_compile_cnt+feather->scanner->to_update_group_cnt;
+				total_wait_rule_cnt=feather->postpone_q_size+feather->scanner->xx_plugin_rule_to_update_cnt;
 				if(total_wait_rule_cnt>0&&time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000)
 				{
 					do_scanner_update(feather->scanner,

src/inc_internal/Maat_rule_internal.h

@@ -214,11 +214,7 @@ struct Maat_scanner
 	time_t last_update_time;
 	mcore_long_t ref_cnt;
 	rule_scanner_t region;
-	size_t gie_update_q_size;
+	size_t xx_plugin_rule_to_update_cnt;
-	size_t xx_plugin_update_q_size;
-	size_t to_update_group_cnt;
-	size_t to_update_compile_cnt;
-
 	struct Maat_table_runtime_manager*  table_rt_mgr;
 	size_t max_table_num;
 

test/perf_test_maatframe.cpp

@@ -289,6 +289,7 @@ protected:
 		const char* test_maat_redis_ip="127.0.0.1";
 		unsigned short test_maat_redis_port=6379;
 		const char* table_info_path="./table_info.conf";
+		const char* stat_file="./perf_cmd_test.fs2";
 		int scan_interval_ms=500;
 		int effective_interval_ms=0;
 
@@ -305,6 +306,10 @@ protected:
 		const char* foregin_dir="./foreign_files/";	
 		Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR, foregin_dir, strlen(foregin_dir)+1);
 		
+		Maat_set_feather_opt(_shared_feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1);
+		Maat_set_feather_opt(_shared_feather, MAAT_OPT_STAT_ON, NULL, 0);
+		Maat_set_feather_opt(_shared_feather, MAAT_OPT_PERF_ON, NULL, 0);
+	
 		Maat_cmd_flushDB(_shared_feather);
 		Maat_initiate_feather(_shared_feather);
 	}
@@ -513,7 +518,7 @@ TEST_F(MaatCMDPerfTest, HighMatchRateOnMultiThread)
 	}
 	scan_per_second=scan_count*1000/time_elapse_ms;
 	EXPECT_GT(scan_per_second, 800*1000);
-	printf("High Match on Multi-Thread speed %lld lookups/s\n", scan_per_second);
+	printf("High match rate on %d-threads speed %lld lookups/s/thread\n", global_thread_num, scan_per_second);
 }
 
 #define IP_PLUGIN_EX_DATA
@@ -543,16 +548,18 @@ void ip_plugin_EX_new_cb(int table_id, const char* key, const char* table_line,
 void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
 {
 	struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad);
-	u->ref_cnt--;
+	
-	if(u->ref_cnt>0) return;
+	if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
+	{
 		free(u->buffer);
 		free(u);
 		*ad=NULL;
 	}
+}
 void ip_plugin_EX_dup_cb(int table_id,	MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
 {
 	struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from);
-	u->ref_cnt++;
+	__sync_add_and_fetch(&(u->ref_cnt), 1);
 	*to=u;
 }
 
@@ -702,7 +709,7 @@ void* fqdn_plugin_update_thread(void *arg)
 {
 	struct thread_para* para=(struct thread_para*)arg;
 	Maat_feather_t feather=para->feather;
-	const int CMD_EXPR_NUM=10;
+	const int CMD_EXPR_NUM=20;
 	int i=0;
 	struct Maat_cmd_line line_rule;
 	char line_buff[1024], fqdn_buff[256];
@@ -729,13 +736,13 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread)
 	Maat_feather_t feather=MaatCMDPerfTest::_shared_feather;
 	int* is_all_hit=NULL;
 	int ret=0;
-	size_t i=0, j=0;
+	int i=0, j=0;
 	int table_id=0, fqdn_plugin_ex_data_counter=0;
 	const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA";
 
 #define FIXED_LINE_CNT 5
-	size_t randomed_line_cnt=100*1000;
+	int randomed_line_cnt=100*1000;
-	size_t total_line_cnt=FIXED_LINE_CNT+randomed_line_cnt;
+	int total_line_cnt=FIXED_LINE_CNT+randomed_line_cnt;
 	const struct Maat_cmd_line **p_line=(const struct Maat_cmd_line**)calloc(sizeof(struct Maat_cmd_line*), total_line_cnt);
 	struct Maat_cmd_line *line_rule=(struct Maat_cmd_line*)calloc(sizeof(struct Maat_cmd_line), total_line_cnt);
 	const char* fixed_table_line[FIXED_LINE_CNT]={
@@ -805,8 +812,8 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread)
 	{
 		perf_fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
 	}
-	size_t global_thread_num=4;
+	int global_thread_num=4;
-	unsigned long long time_elapse_ms=0, scan_count=0, scan_per_second=0;
+	unsigned long long time_elapse_ms=0, scan_count=0, scan_per_second_per_thread=0;
 	pthread_t threads[global_thread_num+1]; 
 	struct thread_para thread_para[global_thread_num+1];
 	for(i=0; i<global_thread_num+1; i++)
@@ -814,7 +821,7 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread)
 		thread_para[i].feather=feather;
 		thread_para[i].thread_id=i;
 		thread_para[i].table_name=table_name;
-		thread_para[i].test_count=1000*1000;
+		thread_para[i].test_count=20*1000*1000;
 		thread_para[i].time_elapse_ms=0;
 		if(i<global_thread_num)
 		{
@@ -836,12 +843,9 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread)
 		*is_all_hit=0;
 		free(is_all_hit);
 	}
-	scan_per_second=scan_count*1000/time_elapse_ms;
+	scan_per_second_per_thread=scan_count*1000/time_elapse_ms;
-	EXPECT_GT(scan_per_second, 800*1000);
+	EXPECT_GT(scan_per_second_per_thread, 800*1000);
-	printf("FQDN Plugin on Multi-Thread speed %lld lookups/s\n", scan_per_second);
+	printf("FQDN Plugin on %d-threads speed %lld lookups/s/thread.\n", global_thread_num, scan_per_second_per_thread);	
-	
-//	printf("ready to sleep\n");
-//	sleep(300);
 
 	return;
 }

test/test_maatframe.cpp

@@ -266,15 +266,17 @@ void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void
 {
 	struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad);
 	u->ref_cnt--;
-	if(u->ref_cnt>0) return;
+	if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
+	{
 		free(u->buffer);
 		free(u);
 		*ad=NULL;
 	}
+}
 void ip_plugin_EX_dup_cb(int table_id,	MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
 {
 	struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from);
-	u->ref_cnt++;
+	__sync_add_and_fetch(&(u->ref_cnt), 1);
 	*to=u;
 }
 
@@ -348,15 +350,16 @@ void fqdn_plugin_EX_new_cb(int table_id, const char* key, const char* table_line
 void fqdn_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
 {
 	struct perf_fqdn_plugin_ud* u=(struct perf_fqdn_plugin_ud*)(*ad);
-	u->ref_cnt--;
+	if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
-	if(u->ref_cnt>0) return;
+	{
 		free(u);
 		*ad=NULL;
 	}
+}
 void fqdn_plugin_EX_dup_cb(int table_id,	MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
 {
 	struct perf_fqdn_plugin_ud* u=(struct perf_fqdn_plugin_ud*)(*from);
-	u->ref_cnt++;
+	__sync_add_and_fetch(&(u->ref_cnt), 1);
 	*to=u;
 }
 TEST(FQDN_Plugin_Table, EX_DATA)
@@ -3372,15 +3375,16 @@ void plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAA
 void plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
 {
 	struct user_info* u=(struct user_info*)(*ad);
-	u->ref_cnt--;
+	if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
-	if(u->ref_cnt>0) return;
+	{
 		free(u);
 		*ad=NULL;
 	}
+}
 void plugin_EX_dup_cb(int table_id,	MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
 {
 	struct user_info* u=(struct user_info*)(*from);
-	u->ref_cnt++;
+	__sync_add_and_fetch(&(u->ref_cnt), 1);
 	*to=u;
 }