diff --git a/src/entry/Maat_hierarchy.cpp b/src/entry/Maat_hierarchy.cpp index ca6da37..4f5282e 100644 --- a/src/entry/Maat_hierarchy.cpp +++ b/src/entry/Maat_hierarchy.cpp @@ -170,7 +170,8 @@ struct Maat_hierarchy pthread_rwlock_t rwlock; pthread_mutex_t mutex; time_t version; //After full update, clause id may indicate a different clause. Comparing hier->version and mid->hier_ver can prevent false positive match. - + int changed_flag; + struct Maat_hierarchy_compile* hash_compile_by_id; //key: compile_id, value: struct Maat_hierarchy_compile*. void (* compile_user_data_free)(void *compile_ud); @@ -479,6 +480,7 @@ int Maat_hierarchy_compile_add(struct Maat_hierarchy* hier, int compile_id, int struct Maat_hierarchy_compile* compile=NULL; pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag=1; HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); if(!compile) { @@ -511,6 +513,7 @@ int Maat_hierarchy_compile_remove(struct Maat_hierarchy * hier, int compile_id) int ret=0; pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag=1; HASH_FIND_INT(hier->hash_compile_by_id, &compile_id, compile); if(compile) { @@ -664,6 +667,7 @@ int Maat_hierarchy_add_group_to_compile(struct Maat_hierarchy* hier, int group_i struct Maat_hierarchy_compile* compile=NULL; pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag=1; HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(!group) { @@ -700,6 +704,7 @@ int Maat_hierarchy_remove_group_from_compile(struct Maat_hierarchy* hier, int gr int ret=0; pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag=1; HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(!group) { @@ -746,6 +751,7 @@ int Maat_hierarchy_add_group_to_group(struct Maat_hierarchy* hier, int group_id, struct Maat_hierarchy_group* group=NULL, *superior_group=NULL; pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag=1; HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(!group) { @@ -782,7 +788,7 @@ int Maat_hierarchy_remove_group_from_group(struct Maat_hierarchy* hier, int grou struct Maat_hierarchy_group* group=NULL, *superior_group=NULL; //No hash write operation, LOCK protection is unnecessary. - + hier->changed_flag=1; HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(group==NULL) { @@ -838,6 +844,7 @@ int Maat_hierarchy_add_region_to_group(struct Maat_hierarchy* hier, int group_id int ret=0; pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag=1; HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(!group) { @@ -867,6 +874,7 @@ void* Maat_hierarchy_region_dettach_user_data(struct Maat_hierarchy* hier, int r struct Maat_hierarchy_region* region=NULL; void* ret=NULL; pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag=1; HASH_FIND_INT(hier->hash_region_by_id, ®ion_id, region); if(region) { @@ -881,6 +889,7 @@ int Maat_hierarchy_remove_region_from_group(struct Maat_hierarchy* hier, int gro struct Maat_hierarchy_group* group=NULL; struct Maat_hierarchy_region* region=NULL; pthread_rwlock_wrlock(&hier->rwlock); + hier->changed_flag=1; HASH_FIND(hh_group_id, hier->hash_group_by_id, &group_id, sizeof(group_id), group); if(!group) { @@ -1209,6 +1218,10 @@ int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier) struct region2clause_value* new_region2clause_hash=NULL, *old_region2clause_hash=NULL; //Read hier from update thread is OK. + if(!hier->changed_flag) + { + return ret; + } ret=Maat_hierarchy_build_top_groups(hier); new_bm=Maat_hierarchy_build_bool_matcher(hier); new_region2clause_hash=Maat_hierarchy_build_region2clause_hash(hier); @@ -1220,6 +1233,7 @@ int Maat_hierarchy_rebuild(struct Maat_hierarchy* hier) hier->bm=new_bm; hier->hash_region2clause=new_region2clause_hash; + hier->changed_flag=0; pthread_rwlock_unlock(&hier->rwlock); Maat_garbage_bagging(hier->ref_garbage_bin, old_bm, (void (*)(void*))bool_matcher_free); diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 783be7b..ed2e1ea 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -57,7 +57,7 @@ extern "C" } #endif -int MAAT_FRAME_VERSION_3_4_3_20210727=1; +int MAAT_FRAME_VERSION_3_4_4_20210805=1; int is_valid_table_name(const char* str) { @@ -1414,7 +1414,6 @@ int add_digest_rule(struct Maat_table_schema* table, struct db_digest_rule* db_r Maat_region_inner_add_expr_id(u_para, expr_id); Maat_table_runtime_digest_add(table_rt, expr_id, db_rule->digest_string, db_rule->confidence_degree, u_para); - scanner->gie_update_q_size++; return 0; } @@ -1456,7 +1455,6 @@ int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id assert(region->expr_id_cnt==1); table_rt=Maat_table_runtime_get(maat_scanner->table_rt_mgr, table->table_id); Maat_table_runtime_digest_del(table_rt, region->expr_id_lb); - maat_scanner->gie_update_q_size++; break; default: assert(0); @@ -1521,7 +1519,6 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl { table_rt->group2compile.not_flag_group--; } - scanner->to_update_group_cnt++; } } else @@ -1534,7 +1531,6 @@ void update_group2compile_rule(struct Maat_table_schema* table, const char* tabl { table_rt->group2compile.not_flag_group++; } - scanner->to_update_group_cnt++; } } return; @@ -1563,7 +1559,6 @@ void update_group2group_rule(struct Maat_table_schema* table, const char* table_ { table_rt->origin_rule_num--; assert(table_rt->origin_rule_num>=0); - scanner->to_update_group_cnt++; } } else @@ -1572,7 +1567,6 @@ void update_group2group_rule(struct Maat_table_schema* table, const char* table_ if(ret==0) { table_rt->origin_rule_num++; - scanner->to_update_group_cnt++; } } @@ -2101,7 +2095,6 @@ void update_compile_rule(struct Maat_table_schema* table,const char* table_line assert(ret==0); table_rt->origin_rule_num--; Maat_garbage_bagging(scanner->ref_garbage_bin, p_compile, (void (*)(void*))destroy_compile_rule); - scanner->to_update_compile_cnt++; } else { @@ -2115,7 +2108,6 @@ void update_compile_rule(struct Maat_table_schema* table,const char* table_line if(ret==0) { table_rt->origin_rule_num++; - scanner->to_update_compile_cnt++; } else { @@ -2317,7 +2309,7 @@ void update_xx_plugin_table(struct Maat_table_schema* table_schema, const char* { Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, table_row, logger); } - scanner->xx_plugin_update_q_size++; + scanner->xx_plugin_rule_to_update_cnt++; return; } @@ -2402,10 +2394,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))map_destroy); } scanner->last_update_time=time(NULL); - scanner->gie_update_q_size=0; - scanner->to_update_group_cnt=0; - scanner->to_update_compile_cnt=0; - scanner->xx_plugin_update_q_size=0; + scanner->xx_plugin_rule_to_update_cnt=0; return; } @@ -2451,7 +2440,7 @@ long long scanner_rule_num(struct Maat_scanner *scanner) void maat_finish_cb(void* u_para) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)u_para; - long expr_wait_q_cnt=0; + long rulescan_wait_q_cnt=0; Maat_table_manager_all_plugin_cb_finish(feather->table_mgr); @@ -2469,8 +2458,8 @@ void maat_finish_cb(void* u_para) { feather->scanner->cfg_num=scanner_rule_num(feather->scanner); feather->scanner->version=feather->maat_version; - expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); - feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->xx_plugin_update_q_size; + rulescan_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); + feather->postpone_q_size=rulescan_wait_q_cnt; if(time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000) { do_scanner_update(feather->scanner, @@ -2563,7 +2552,7 @@ void *thread_rule_monitor(void *arg) { struct _Maat_feather_t *feather=(struct _Maat_feather_t *)arg; struct Maat_scanner* old_scanner=NULL; - long expr_wait_q_cnt=0; + long rulescan_update_wait_q_cnt=0; int scan_dir_cnt=0; int ret=0; char md5_tmp[MD5_DIGEST_LENGTH*2+1]={0}; @@ -2695,9 +2684,9 @@ void *thread_rule_monitor(void *arg) } if(feather->scanner!=NULL) { - expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); - feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size; - total_wait_rule_cnt=feather->postpone_q_size+feather->scanner->to_update_compile_cnt+feather->scanner->to_update_group_cnt; + rulescan_update_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q); + feather->postpone_q_size=rulescan_update_wait_q_cnt; + total_wait_rule_cnt=feather->postpone_q_size+feather->scanner->xx_plugin_rule_to_update_cnt; if(total_wait_rule_cnt>0&&time(NULL)-feather->scanner->last_update_time>=feather->rule_effect_interval_ms/1000) { do_scanner_update(feather->scanner, diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 1c3c511..1c3b60c 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -214,11 +214,7 @@ struct Maat_scanner time_t last_update_time; mcore_long_t ref_cnt; rule_scanner_t region; - size_t gie_update_q_size; - size_t xx_plugin_update_q_size; - size_t to_update_group_cnt; - size_t to_update_compile_cnt; - + size_t xx_plugin_rule_to_update_cnt; struct Maat_table_runtime_manager* table_rt_mgr; size_t max_table_num; diff --git a/test/perf_test_maatframe.cpp b/test/perf_test_maatframe.cpp index 6608161..e7d7721 100644 --- a/test/perf_test_maatframe.cpp +++ b/test/perf_test_maatframe.cpp @@ -289,21 +289,26 @@ protected: const char* test_maat_redis_ip="127.0.0.1"; unsigned short test_maat_redis_port=6379; const char* table_info_path="./table_info.conf"; + const char* stat_file="./perf_cmd_test.fs2"; int scan_interval_ms=500; int effective_interval_ms=0; logger=MESA_create_runtime_log_handle("maat_perf_test.log",0); _shared_feather=Maat_feather(global_thread_num, table_info_path, logger); - Maat_set_feather_opt(_shared_feather,MAAT_OPT_INSTANCE_NAME,"cmdperf", strlen("cmdperf")+1); + Maat_set_feather_opt(_shared_feather,MAAT_OPT_INSTANCE_NAME, "cmdperf", strlen("cmdperf")+1); Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_IP, test_maat_redis_ip, strlen(test_maat_redis_ip)+1); Maat_set_feather_opt(_shared_feather, MAAT_OPT_REDIS_PORT, &test_maat_redis_port, sizeof(test_maat_redis_port)); - Maat_set_feather_opt(_shared_feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms)); //Set a short intevral for testing. - Maat_set_feather_opt(_shared_feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms)); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_EFFECT_INVERVAL_MS, &effective_interval_ms, sizeof(effective_interval_ms)); const char* foregin_dir="./foreign_files/"; Maat_set_feather_opt(_shared_feather, MAAT_OPT_FOREIGN_CONT_DIR, foregin_dir, strlen(foregin_dir)+1); + + Maat_set_feather_opt(_shared_feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_STAT_ON, NULL, 0); + Maat_set_feather_opt(_shared_feather, MAAT_OPT_PERF_ON, NULL, 0); Maat_cmd_flushDB(_shared_feather); Maat_initiate_feather(_shared_feather); @@ -513,7 +518,7 @@ TEST_F(MaatCMDPerfTest, HighMatchRateOnMultiThread) } scan_per_second=scan_count*1000/time_elapse_ms; EXPECT_GT(scan_per_second, 800*1000); - printf("High Match on Multi-Thread speed %lld lookups/s\n", scan_per_second); + printf("High match rate on %d-threads speed %lld lookups/s/thread\n", global_thread_num, scan_per_second); } #define IP_PLUGIN_EX_DATA @@ -543,16 +548,18 @@ void ip_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) { struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad); - u->ref_cnt--; - if(u->ref_cnt>0) return; - free(u->buffer); - free(u); - *ad=NULL; + + if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) + { + free(u->buffer); + free(u); + *ad=NULL; + } } void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) { struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from); - u->ref_cnt++; + __sync_add_and_fetch(&(u->ref_cnt), 1); *to=u; } @@ -702,7 +709,7 @@ void* fqdn_plugin_update_thread(void *arg) { struct thread_para* para=(struct thread_para*)arg; Maat_feather_t feather=para->feather; - const int CMD_EXPR_NUM=10; + const int CMD_EXPR_NUM=20; int i=0; struct Maat_cmd_line line_rule; char line_buff[1024], fqdn_buff[256]; @@ -729,13 +736,13 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread) Maat_feather_t feather=MaatCMDPerfTest::_shared_feather; int* is_all_hit=NULL; int ret=0; - size_t i=0, j=0; + int i=0, j=0; int table_id=0, fqdn_plugin_ex_data_counter=0; const char* table_name="TEST_FQDN_PLUGIN_WITH_EXDATA"; #define FIXED_LINE_CNT 5 - size_t randomed_line_cnt=100*1000; - size_t total_line_cnt=FIXED_LINE_CNT+randomed_line_cnt; + int randomed_line_cnt=100*1000; + int total_line_cnt=FIXED_LINE_CNT+randomed_line_cnt; const struct Maat_cmd_line **p_line=(const struct Maat_cmd_line**)calloc(sizeof(struct Maat_cmd_line*), total_line_cnt); struct Maat_cmd_line *line_rule=(struct Maat_cmd_line*)calloc(sizeof(struct Maat_cmd_line), total_line_cnt); const char* fixed_table_line[FIXED_LINE_CNT]={ @@ -805,8 +812,8 @@ TEST_F(MaatCMDPerfTest, FQDNPluginOnMultiThread) { perf_fqdn_plugin_EX_free_cb(0, (void**)&(result[i]), 0, NULL); } - size_t global_thread_num=4; - unsigned long long time_elapse_ms=0, scan_count=0, scan_per_second=0; + int global_thread_num=4; + unsigned long long time_elapse_ms=0, scan_count=0, scan_per_second_per_thread=0; pthread_t threads[global_thread_num+1]; struct thread_para thread_para[global_thread_num+1]; for(i=0; iref_cnt--; - if(u->ref_cnt>0) return; - free(u->buffer); - free(u); - *ad=NULL; + if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) + { + free(u->buffer); + free(u); + *ad=NULL; + } } void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) { struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from); - u->ref_cnt++; + __sync_add_and_fetch(&(u->ref_cnt), 1); *to=u; } @@ -348,15 +350,16 @@ void fqdn_plugin_EX_new_cb(int table_id, const char* key, const char* table_line void fqdn_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) { struct perf_fqdn_plugin_ud* u=(struct perf_fqdn_plugin_ud*)(*ad); - u->ref_cnt--; - if(u->ref_cnt>0) return; - free(u); - *ad=NULL; + if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) + { + free(u); + *ad=NULL; + } } void fqdn_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) { struct perf_fqdn_plugin_ud* u=(struct perf_fqdn_plugin_ud*)(*from); - u->ref_cnt++; + __sync_add_and_fetch(&(u->ref_cnt), 1); *to=u; } TEST(FQDN_Plugin_Table, EX_DATA) @@ -3372,15 +3375,16 @@ void plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAA void plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) { struct user_info* u=(struct user_info*)(*ad); - u->ref_cnt--; - if(u->ref_cnt>0) return; - free(u); - *ad=NULL; + if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) + { + free(u); + *ad=NULL; + } } void plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) { struct user_info* u=(struct user_info*)(*from); - u->ref_cnt++; + __sync_add_and_fetch(&(u->ref_cnt), 1); *to=u; }