gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Plugin表更新后，不需要进行Hierarchy的重建。

Browse Source
This commit is contained in:
zhengchao
2021-08-05 13:51:17 +08:00
parent 0cb6a59d31
commit 50934de91d
5 changed files with 74 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/entry/Maat_hierarchy.cpp
View File

@@ -170,6 +170,7 @@ struct Maat_hierarchy
pthread_rwlock_t rwlock; pthread_rwlock_t rwlock;
pthread_mutex_t mutex; pthread_mutex_t mutex;
time_t version; //After full update, clause id may indicate a different clause. Comparing hier->version and mid->hier_ver can prevent false positive match. time_t version; //After full update, clause id may indicate a different clause. Comparing hier->version and mid->hier_ver can prevent false positive match.
int changed_flag;
struct Maat_hierarchy_compile* hash_compile_by_id; //key: compile_id, value: struct Maat_hierarchy_compile*. struct Maat_hierarchy_compile* hash_compile_by_id; //key: compile_id, value: struct Maat_hierarchy_compile*.
void (* compile_user_data_free)(void *compile_ud); void (* compile_user_data_free)(void *compile_ud);
@@ -479,6 +480,7 @@ int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int
struct Maat_hierarchy_compile* compile=NULL; struct Maat_hierarchy_compile* compile=NULL;
pthread_rwlock_wrlock(&hier->rwlock); pthread_rwlock_wrlock(&hier->rwlock);
hier->changed_flag=1;
HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile);
if(!compile) if(!compile)
{ {
@@ -511,6 +513,7 @@ int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id)
int ret=0; int ret=0;
pthread_rwlock_wrlock(&hier->rwlock); pthread_rwlock_wrlock(&hier->rwlock);
hier->changed_flag=1;
HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile);
if(compile) if(compile)
{ {
@@ -664,6 +667,7 @@ int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_i
struct Maat_hierarchy_compile* compile=NULL; struct Maat_hierarchy_compile* compile=NULL;
pthread_rwlock_wrlock(&hier->rwlock); pthread_rwlock_wrlock(&hier->rwlock);
hier->changed_flag=1;
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(!group) if(!group)
{ {
@@ -700,6 +704,7 @@ int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int gr
int ret=0; int ret=0;
pthread_rwlock_wrlock(&hier->rwlock); pthread_rwlock_wrlock(&hier->rwlock);
hier->changed_flag=1;
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(!group) if(!group)
{ {
@@ -746,6 +751,7 @@ int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id,
struct Maat_hierarchy_group* group=NULL, *superior_group=NULL; struct Maat_hierarchy_group* group=NULL, *superior_group=NULL;
pthread_rwlock_wrlock(&hier->rwlock); pthread_rwlock_wrlock(&hier->rwlock);
hier->changed_flag=1;
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(!group) if(!group)
{ {
@@ -782,7 +788,7 @@ int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int grou
struct Maat_hierarchy_group* group=NULL, *superior_group=NULL; struct Maat_hierarchy_group* group=NULL, *superior_group=NULL;
//No hash write operation, LOCK protection is unnecessary. //No hash write operation, LOCK protection is unnecessary.
hier->changed_flag=1;
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(group==NULL) if(group==NULL)
{ {
@@ -838,6 +844,7 @@ int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id
int ret=0; int ret=0;
pthread_rwlock_wrlock(&hier->rwlock); pthread_rwlock_wrlock(&hier->rwlock);
hier->changed_flag=1;
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(!group) if(!group)
{ {
@@ -867,6 +874,7 @@ void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int r
struct Maat_hierarchy_region* region=NULL; struct Maat_hierarchy_region* region=NULL;
void* ret=NULL; void* ret=NULL;
pthread_rwlock_wrlock(&hier->rwlock); pthread_rwlock_wrlock(&hier->rwlock);
hier->changed_flag=1;
HASH_FIND_INT(hier->hash_region_by_id, &region_id, region); HASH_FIND_INT(hier->hash_region_by_id, &region_id, region);
if(region) if(region)
{ {
@@ -881,6 +889,7 @@ int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int gro
struct Maat_hierarchy_group* group=NULL; struct Maat_hierarchy_group* group=NULL;
struct Maat_hierarchy_region* region=NULL; struct Maat_hierarchy_region* region=NULL;
pthread_rwlock_wrlock(&hier->rwlock); pthread_rwlock_wrlock(&hier->rwlock);
hier->changed_flag=1;
HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group);
if(!group) if(!group)
{ {
@@ -1209,6 +1218,10 @@ int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier)
struct region2clause_value* new_region2clause_hash=NULL, *old_region2clause_hash=NULL; struct region2clause_value* new_region2clause_hash=NULL, *old_region2clause_hash=NULL;
//Read hier from update thread is OK. //Read hier from update thread is OK.
if(!hier->changed_flag)
{
return ret;
}
ret=Maat_hierarchy_build_top_groups(hier); ret=Maat_hierarchy_build_top_groups(hier);
new_bm=Maat_hierarchy_build_bool_matcher(hier); new_bm=Maat_hierarchy_build_bool_matcher(hier);
new_region2clause_hash=Maat_hierarchy_build_region2clause_hash(hier); new_region2clause_hash=Maat_hierarchy_build_region2clause_hash(hier);
@@ -1220,6 +1233,7 @@ int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier)
hier->bm=new_bm; hier->bm=new_bm;
hier->hash_region2clause=new_region2clause_hash; hier->hash_region2clause=new_region2clause_hash;
hier->changed_flag=0;
pthread_rwlock_unlock(&hier->rwlock); pthread_rwlock_unlock(&hier->rwlock);
Maat_garbage_bagging(hier->ref_garbage_bin, old_bm, (void (*)(void*))bool_matcher_free); Maat_garbage_bagging(hier->ref_garbage_bin, old_bm, (void (*)(void*))bool_matcher_free);

31
src/entry/Maat_rule.cpp
View File

@@ -57,7 +57,7 @@ extern "C"
} }
#endif #endif
int MAAT_FRAME_VERSION_3_4_3_20210727=1; int MAAT_FRAME_VERSION_3_4_4_20210805=1;
int is_valid_table_name(const char* str) int is_valid_table_name(const char* str)
{ {
@@ -1414,7 +1414,6 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r
Maat_region_inner_add_expr_id(u_para, expr_id); Maat_region_inner_add_expr_id(u_para, expr_id);
Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, u_para); Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, u_para);
scanner->gie_update_q_size++;
return 0; return 0;
} }
@@ -1456,7 +1455,6 @@ int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id
assert(region->expr_id_cnt==1); assert(region->expr_id_cnt==1);
table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id); table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id);
Maat_table_runtime_digest_del(table_rt, region->expr_id_lb); Maat_table_runtime_digest_del(table_rt, region->expr_id_lb);
maat_scanner->gie_update_q_size++;
break; break;
default: default:
assert(0); assert(0);
@@ -1521,7 +1519,6 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl
{ {
table_rt->group2compile.not_flag_group--; table_rt->group2compile.not_flag_group--;
} }
scanner->to_update_group_cnt++;
} }
} }
else else
@@ -1534,7 +1531,6 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl
{ {
table_rt->group2compile.not_flag_group++; table_rt->group2compile.not_flag_group++;
} }
scanner->to_update_group_cnt++;
} }
} }
return; return;
@@ -1563,7 +1559,6 @@ void update_group2group_rule(struct Maat_table_schema* table, const char* table_
{ {
table_rt->origin_rule_num--; table_rt->origin_rule_num--;
assert(table_rt->origin_rule_num>=0); assert(table_rt->origin_rule_num>=0);
scanner->to_update_group_cnt++;
} }
} }
else else
@@ -1572,7 +1567,6 @@ void update_group2group_rule(struct Maat_table_schema* table, const char* table_
if(ret==0) if(ret==0)
{ {
table_rt->origin_rule_num++; table_rt->origin_rule_num++;
scanner->to_update_group_cnt++;
} }
} }
@@ -2101,7 +2095,6 @@ void update_compile_rule(struct Maat_table_schema* table,const char* table_line
assert(ret==0); assert(ret==0);
table_rt->origin_rule_num--; table_rt->origin_rule_num--;
Maat_garbage_bagging(scanner->ref_garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule); Maat_garbage_bagging(scanner->ref_garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule);
scanner->to_update_compile_cnt++;
} }
else else
{ {
@@ -2115,7 +2108,6 @@ void update_compile_rule(struct Maat_table_schema* table,const char* table_line
if(ret==0) if(ret==0)
{ {
table_rt->origin_rule_num++; table_rt->origin_rule_num++;
scanner->to_update_compile_cnt++;
} }
else else
{ {
@@ -2317,7 +2309,7 @@ void update_xx_plugin_table(struct Maat_table_schema* table_schema, const char*
{ {
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger); Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger);
} }
scanner->xx_plugin_update_q_size++; scanner->xx_plugin_rule_to_update_cnt++;
return; return;
} }
@@ -2402,10 +2394,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))map_destroy); Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))map_destroy);
} }
scanner->last_update_time=time(NULL); scanner->last_update_time=time(NULL);
scanner->gie_update_q_size=0; scanner->xx_plugin_rule_to_update_cnt=0;
scanner->to_update_group_cnt=0;
scanner->to_update_compile_cnt=0;
scanner->xx_plugin_update_q_size=0;
return; return;
} }
@@ -2451,7 +2440,7 @@ long long scanner_rule_num(struct Maat_scanner *scanner)
void maat_finish_cb(void* u_para) void maat_finish_cb(void* u_para)
{ {
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para;
long expr_wait_q_cnt=0; long rulescan_wait_q_cnt=0;
Maat_table_manager_all_plugin_cb_finish(feather->table_mgr); Maat_table_manager_all_plugin_cb_finish(feather->table_mgr);
@@ -2469,8 +2458,8 @@ void maat_finish_cb(void* u_para)
{ {
feather->scanner->cfg_num=scanner_rule_num(feather->scanner); feather->scanner->cfg_num=scanner_rule_num(feather->scanner);
feather->scanner->version=feather->maat_version; feather->scanner->version=feather->maat_version;
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); rulescan_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->xx_plugin_update_q_size; feather->postpone_q_size=rulescan_wait_q_cnt;
if(time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000) if(time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000)
{ {
do_scanner_update(feather->scanner, do_scanner_update(feather->scanner,
@@ -2563,7 +2552,7 @@ void *thread_rule_monitor(void *arg)
{ {
struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg; struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg;
struct Maat_scanner* old_scanner=NULL; struct Maat_scanner* old_scanner=NULL;
long expr_wait_q_cnt=0; long rulescan_update_wait_q_cnt=0;
int scan_dir_cnt=0; int scan_dir_cnt=0;
int ret=0; int ret=0;
char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0}; char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0};
@@ -2695,9 +2684,9 @@ void *thread_rule_monitor(void *arg)
} }
if(feather->scanner!=NULL) if(feather->scanner!=NULL)
{ {
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); rulescan_update_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size; feather->postpone_q_size=rulescan_update_wait_q_cnt;
total_wait_rule_cnt=feather->postpone_q_size+feather->scanner->to_update_compile_cnt+feather->scanner->to_update_group_cnt; total_wait_rule_cnt=feather->postpone_q_size+feather->scanner->xx_plugin_rule_to_update_cnt;
if(total_wait_rule_cnt>0&&time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000) if(total_wait_rule_cnt>0&&time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000)
{ {
do_scanner_update(feather->scanner, do_scanner_update(feather->scanner,

6
src/inc_internal/Maat_rule_internal.h
View File

@@ -214,11 +214,7 @@ struct Maat_scanner
time_t last_update_time; time_t last_update_time;
mcore_long_t ref_cnt; mcore_long_t ref_cnt;
rule_scanner_t region; rule_scanner_t region;
size_t gie_update_q_size; size_t xx_plugin_rule_to_update_cnt;
size_t xx_plugin_update_q_size;
size_t to_update_group_cnt;
size_t to_update_compile_cnt;
struct Maat_table_runtime_manager* table_rt_mgr; struct Maat_table_runtime_manager* table_rt_mgr;
size_t max_table_num; size_t max_table_num;

46
test/perf_test_maatframe.cpp
View File

@@ -289,22 +289,27 @@ protected:
const char* test_maat_redis_ip="127.0.0.1"; const char* test_maat_redis_ip="127.0.0.1";
unsigned short test_maat_redis_port=6379; unsigned short test_maat_redis_port=6379;
const char* table_info_path="./table_info.conf"; const char* table_info_path="./table_info.conf";
const char* stat_file="./perf_cmd_test.fs2";
int scan_interval_ms=500; int scan_interval_ms=500;
int effective_interval_ms=0; int effective_interval_ms=0;
logger=MESA_create_runtime_log_handle("maat_perf_test.log",0); logger=MESA_create_runtime_log_handle("maat_perf_test.log",0);
_shared_feather=Maat_feather(global_thread_num, table_info_path, logger); _shared_feather=Maat_feather(global_thread_num, table_info_path, logger);
Maat_set_feather_opt(_shared_feather,MAAT_OPT_INSTANCE_NAME,"cmdperf", strlen("cmdperf")+1); Maat_set_feather_opt(_shared_feather,MAAT_OPT_INSTANCE_NAME, "cmdperf", strlen("cmdperf")+1);
Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_IP, test_maat_redis_ip, strlen(test_maat_redis_ip)+1); Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_IP, test_maat_redis_ip, strlen(test_maat_redis_ip)+1);
Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_PORT, &test_maat_redis_port, sizeof(test_maat_redis_port)); Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_PORT, &test_maat_redis_port, sizeof(test_maat_redis_port));
Maat_set_feather_opt(_shared_feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); Maat_set_feather_opt(_shared_feather, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms));
//Set a short intevral for testing. //Set a short intevral for testing.
Maat_set_feather_opt(_shared_feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms)); Maat_set_feather_opt(_shared_feather, MAAT_OPT_EFFECT_INVERVAL_MS, &effective_interval_ms, sizeof(effective_interval_ms));
const char* foregin_dir="./foreign_files/"; const char* foregin_dir="./foreign_files/";
Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR, foregin_dir, strlen(foregin_dir)+1); Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR, foregin_dir, strlen(foregin_dir)+1);
Maat_set_feather_opt(_shared_feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1);
Maat_set_feather_opt(_shared_feather, MAAT_OPT_STAT_ON, NULL, 0);
Maat_set_feather_opt(_shared_feather, MAAT_OPT_PERF_ON, NULL, 0);
Maat_cmd_flushDB(_shared_feather); Maat_cmd_flushDB(_shared_feather);
Maat_initiate_feather(_shared_feather); Maat_initiate_feather(_shared_feather);
} }
@@ -513,7 +518,7 @@ TEST_F(MaatCMDPerfTest, HighMatchRateOnMultiThread)
} }
scan_per_second=scan_count*1000/time_elapse_ms; scan_per_second=scan_count*1000/time_elapse_ms;
EXPECT_GT(scan_per_second, 800*1000); EXPECT_GT(scan_per_second, 800*1000);
printf("High Match on Multi-Thread speed %lld lookups/s\n", scan_per_second); printf("High match rate on %d-threads speed %lld lookups/s/thread\n", global_thread_num, scan_per_second);
} }
#define IP_PLUGIN_EX_DATA #define IP_PLUGIN_EX_DATA
@@ -543,16 +548,18 @@ void ip_plugin_EX_new_cb(int table_id, const char* key, const char* table_line,
void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{ {
struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad); struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad);
u->ref_cnt--;
if(u->ref_cnt>0) return; if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
{
free(u->buffer); free(u->buffer);
free(u); free(u);
*ad=NULL; *ad=NULL;
}
} }
void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{ {
struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from); struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from);
u->ref_cnt++; __sync_add_and_fetch(&(u->ref_cnt), 1);
*to=u; *to=u;
} }
@@ -702,7 +709,7 @@ void* fqdn_plugin_update_thread(void *arg)
{ {
struct thread_para* para=(struct thread_para*)arg; struct thread_para* para=(struct thread_para*)arg;
Maat_feather_t feather=para->feather; Maat_feather_t feather=para->feather;
const int CMD_EXPR_NUM=10; const int CMD_EXPR_NUM=20;
int i=0; int i=0;
struct Maat_cmd_line line_rule; struct Maat_cmd_line line_rule;
char line_buff[1024], fqdn_buff[256]; char line_buff[1024], fqdn_buff[256];
@@ -729,13 +736,13 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread)
Maat_feather_t feather=MaatCMDPerfTest::_shared_feather; Maat_feather_t feather=MaatCMDPerfTest::_shared_feather;
int* is_all_hit=NULL; int* is_all_hit=NULL;
int ret=0; int ret=0;
size_t i=0, j=0; int i=0, j=0;
int table_id=0, fqdn_plugin_ex_data_counter=0; int table_id=0, fqdn_plugin_ex_data_counter=0;
const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA"; const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA";
#define FIXED_LINE_CNT 5 #define FIXED_LINE_CNT 5
size_t randomed_line_cnt=100*1000; int randomed_line_cnt=100*1000;
size_t total_line_cnt=FIXED_LINE_CNT+randomed_line_cnt; int total_line_cnt=FIXED_LINE_CNT+randomed_line_cnt;
const struct Maat_cmd_line **p_line=(const struct Maat_cmd_line**)calloc(sizeof(struct Maat_cmd_line*), total_line_cnt); const struct Maat_cmd_line **p_line=(const struct Maat_cmd_line**)calloc(sizeof(struct Maat_cmd_line*), total_line_cnt);
struct Maat_cmd_line *line_rule=(struct Maat_cmd_line*)calloc(sizeof(struct Maat_cmd_line), total_line_cnt); struct Maat_cmd_line *line_rule=(struct Maat_cmd_line*)calloc(sizeof(struct Maat_cmd_line), total_line_cnt);
const char* fixed_table_line[FIXED_LINE_CNT]={ const char* fixed_table_line[FIXED_LINE_CNT]={
@@ -805,8 +812,8 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread)
{ {
perf_fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL); perf_fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL);
} }
size_t global_thread_num=4; int global_thread_num=4;
unsigned long long time_elapse_ms=0, scan_count=0, scan_per_second=0; unsigned long long time_elapse_ms=0, scan_count=0, scan_per_second_per_thread=0;
pthread_t threads[global_thread_num+1]; pthread_t threads[global_thread_num+1];
struct thread_para thread_para[global_thread_num+1]; struct thread_para thread_para[global_thread_num+1];
for(i=0; i<global_thread_num+1; i++) for(i=0; i<global_thread_num+1; i++)
@@ -814,7 +821,7 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread)
thread_para[i].feather=feather; thread_para[i].feather=feather;
thread_para[i].thread_id=i; thread_para[i].thread_id=i;
thread_para[i].table_name=table_name; thread_para[i].table_name=table_name;
thread_para[i].test_count=1000*1000; thread_para[i].test_count=20*1000*1000;
thread_para[i].time_elapse_ms=0; thread_para[i].time_elapse_ms=0;
if(i<global_thread_num) if(i<global_thread_num)
{ {
@@ -836,12 +843,9 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread)
*is_all_hit=0; *is_all_hit=0;
free(is_all_hit); free(is_all_hit);
} }
scan_per_second=scan_count*1000/time_elapse_ms; scan_per_second_per_thread=scan_count*1000/time_elapse_ms;
EXPECT_GT(scan_per_second, 800*1000); EXPECT_GT(scan_per_second_per_thread, 800*1000);
printf("FQDN Plugin on Multi-Thread speed %lld lookups/s\n", scan_per_second); printf("FQDN Plugin on %d-threads speed %lld lookups/s/thread.\n", global_thread_num, scan_per_second_per_thread);
// printf("ready to sleep\n");
// sleep(300);
return; return;
} }
@@ -858,7 +862,7 @@ protected:
logger=MESA_create_runtime_log_handle("file_perf_test.log",0); logger=MESA_create_runtime_log_handle("file_perf_test.log",0);
const char* table_info="./tsg_tableinfo.conf"; const char* table_info="./tsg_tableinfo.conf";
_shared_feather_f=Maat_feather(global_thread_num, table_info, logger); _shared_feather_f=Maat_feather(global_thread_num, table_info, logger);
Maat_set_feather_opt(_shared_feather_f,MAAT_OPT_INSTANCE_NAME,"files", strlen("files")+1); Maat_set_feather_opt(_shared_feather_f, MAAT_OPT_INSTANCE_NAME, "files", strlen("files")+1);
Maat_set_feather_opt(_shared_feather_f, MAAT_OPT_FULL_CFG_DIR, rule_folder, strlen(rule_folder)+1); Maat_set_feather_opt(_shared_feather_f, MAAT_OPT_FULL_CFG_DIR, rule_folder, strlen(rule_folder)+1);
Maat_set_feather_opt(_shared_feather_f, MAAT_OPT_INC_CFG_DIR, rule_folder, strlen(rule_folder)+1); Maat_set_feather_opt(_shared_feather_f, MAAT_OPT_INC_CFG_DIR, rule_folder, strlen(rule_folder)+1);

20
test/test_maatframe.cpp
View File

@@ -266,15 +266,17 @@ void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void
{ {
struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad); struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad);
u->ref_cnt--; u->ref_cnt--;
if(u->ref_cnt>0) return; if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
{
free(u->buffer); free(u->buffer);
free(u); free(u);
*ad=NULL; *ad=NULL;
}
} }
void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{ {
struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from); struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from);
u->ref_cnt++; __sync_add_and_fetch(&(u->ref_cnt), 1);
*to=u; *to=u;
} }
@@ -348,15 +350,16 @@ void fqdn_plugin_EX_new_cb(int table_id, const char* key, const char* table_line
void fqdn_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) void fqdn_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{ {
struct perf_fqdn_plugin_ud* u=(struct perf_fqdn_plugin_ud*)(*ad); struct perf_fqdn_plugin_ud* u=(struct perf_fqdn_plugin_ud*)(*ad);
u->ref_cnt--; if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
if(u->ref_cnt>0) return; {
free(u); free(u);
*ad=NULL; *ad=NULL;
}
} }
void fqdn_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) void fqdn_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{ {
struct perf_fqdn_plugin_ud* u=(struct perf_fqdn_plugin_ud*)(*from); struct perf_fqdn_plugin_ud* u=(struct perf_fqdn_plugin_ud*)(*from);
u->ref_cnt++; __sync_add_and_fetch(&(u->ref_cnt), 1);
*to=u; *to=u;
} }
TEST(FQDN_Plugin_Table, EX_DATA) TEST(FQDN_Plugin_Table, EX_DATA)
@@ -3372,15 +3375,16 @@ void plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAA
void plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) void plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{ {
struct user_info* u=(struct user_info*)(*ad); struct user_info* u=(struct user_info*)(*ad);
u->ref_cnt--; if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
if(u->ref_cnt>0) return; {
free(u); free(u);
*ad=NULL; *ad=NULL;
}
} }
void plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) void plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp)
{ {
struct user_info* u=(struct user_info*)(*from); struct user_info* u=(struct user_info*)(*from);
u->ref_cnt++; __sync_add_and_fetch(&(u->ref_cnt), 1);
*to=u; *to=u;
} }