diff --git a/src/entry/Maat_ex_data.cpp b/src/entry/Maat_ex_data.cpp index a98645b..eff004d 100644 --- a/src/entry/Maat_ex_data.cpp +++ b/src/entry/Maat_ex_data.cpp @@ -1,6 +1,7 @@ #include "Maat_ex_data.h" #include "Maat_table.h" #include "Maat_utils.h" +#include "Maat_garbage_collection.h" #include "uthash/uthash.h" #include "uthash/utarray.h" @@ -26,12 +27,13 @@ struct EX_data_rt UT_array *cache_rows; size_t cache_row_num; size_t cache_size; - + pthread_rwlock_t rwlock; struct EX_data_container* hash_key2ex; const struct EX_data_schema* ex_schema; int table_id; - void (* user_data_free)(void *user_data); + void (* user_data_free)(void *user_data); + struct Maat_garbage_bin* bin; }; @@ -64,6 +66,8 @@ struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t utarray_new(p->cache_rows, &ut_cache_row_icd); p->table_id=table_id; p->user_data_free=user_data_free; + pthread_rwlock_init(&p->rwlock, NULL); + p->bin=Maat_garbage_bin_new(0); return p; }; size_t EX_data_rt_get_cached_row_num(struct EX_data_rt* ex_rt) @@ -90,6 +94,8 @@ void EX_data_rt_free(struct EX_data_rt* p) p->cache_rows=NULL; p->cache_row_num=0; } + pthread_rwlock_destroy(&p->rwlock); + Maat_garbage_bin_free(p->bin); free(p); return; } @@ -125,43 +131,59 @@ int EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt, MAAT_RULE_EX_DATA ex_data=NULL; const struct EX_data_schema* ex_schema=ex_rt->ex_schema; struct EX_data_container* ex_container=NULL, *tmp=NULL; + int ret=0; + pthread_rwlock_wrlock(&ex_rt->rwlock); + HASH_FIND(hh, ex_rt->hash_key2ex, key, key_len, tmp); - if(tmp!=NULL) + if(tmp==NULL) + { + ex_container=ALLOC(struct EX_data_container, 1); + ex_container->key=ALLOC(char, key_len+1); + memcpy(ex_container->key, key, key_len); + ex_schema->new_func(ex_rt->table_id, ex_container->key, row, &ex_data, + ex_schema->argl, ex_schema->argp); + ex_container->ex_data=ex_data; + ex_container->rt=ex_rt; + ex_container->user_data=user_data; + + ex_container->key_len=key_len; + HASH_ADD_KEYPTR(hh, ex_rt->hash_key2ex, ex_container->key, ex_container->key_len, ex_container); + ret=0; + } + else { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "EX data add error: duplicated key %.*s of %s", key_len, key, row); - return -1; + ret=-1; } - ex_container=ALLOC(struct EX_data_container, 1); - ex_container->key=ALLOC(char, key_len+1); - memcpy(ex_container->key, key, key_len); - ex_schema->new_func(ex_rt->table_id, ex_container->key, row, &ex_data, - ex_schema->argl, ex_schema->argp); - ex_container->ex_data=ex_data; - ex_container->rt=ex_rt; - ex_container->user_data=user_data; - ex_container->key_len=key_len; - HASH_ADD_KEYPTR(hh, ex_rt->hash_key2ex, ex_container->key, ex_container->key_len, ex_container); - return 0; + pthread_rwlock_unlock(&ex_rt->rwlock); + return ret; } int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len, void *logger) { struct EX_data_container* exc=NULL; + int ret=0; + pthread_rwlock_wrlock(&ex_rt->rwlock); HASH_FIND(hh, ex_rt->hash_key2ex, key, key_len, exc); - if(exc==NULL) + if(exc) + { + HASH_DELETE(hh, ex_rt->hash_key2ex, exc); + Maat_garbage_bagging(ex_rt->bin, exc, (void (*)(void*))EX_data_container_free); + ret=0; + } + else { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "EX data del error: no such key %.*s of %s", key_len, key, row); - return -1; + ret=-1; } - HASH_DELETE(hh, ex_rt->hash_key2ex, exc); - EX_data_container_free(exc); - return 0; + pthread_rwlock_unlock(&ex_rt->rwlock); + return ret; } MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len) @@ -174,12 +196,14 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const assert(0); return NULL; } + pthread_rwlock_rdlock(&ex_rt->rwlock); HASH_FIND(hh, ex_rt->hash_key2ex, key, key_len, container); if(container!=NULL) { ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(container->ex_data), ex_rt->ex_schema->argl, ex_rt->ex_schema->argp); } + pthread_rwlock_unlock(&ex_rt->rwlock); return ex_data; } MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container) @@ -192,15 +216,18 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array) { - size_t ex_data_cnt=HASH_COUNT(ex_rt->hash_key2ex), i=0; + size_t ex_data_cnt=0, i=0; struct EX_data_container* ex_container=NULL, *tmp=NULL; - + + pthread_rwlock_rdlock(&ex_rt->rwlock); + ex_data_cnt=HASH_COUNT(ex_rt->hash_key2ex); *ex_container_array=ALLOC(struct EX_data_container*, ex_data_cnt); HASH_ITER(hh, ex_rt->hash_key2ex, ex_container, tmp) { (*ex_container_array)[i]=ex_container; i++; } + pthread_rwlock_unlock(&ex_rt->rwlock); return ex_data_cnt; } void* EX_data_container_get_user_data(struct EX_data_container* ex_container) @@ -209,5 +236,14 @@ void* EX_data_container_get_user_data(struct EX_data_container* ex_container) } size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt) { - return HASH_COUNT(ex_rt->hash_key2ex); + size_t count=0; + pthread_rwlock_rdlock(&ex_rt->rwlock); + count=HASH_COUNT(ex_rt->hash_key2ex); + pthread_rwlock_unlock(&ex_rt->rwlock); + return count; } +void EX_data_rt_garbage_collection(struct EX_data_rt* ex_rt) +{ + Maat_garbage_collect_routine(ex_rt->bin); +} + diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 481b556..0aeff39 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -57,7 +57,7 @@ extern "C" } #endif -int MAAT_FRAME_VERSION_3_1_12_20210308=1; +int MAAT_FRAME_VERSION_3_1_16_20210320=1; int is_valid_table_name(const char* str) { @@ -2369,6 +2369,9 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* "GIE_update error."); } break; + case TABLE_TYPE_PLUGIN: + Maat_table_runtime_plugin_garbage_collection(table_rt); + break; case TABLE_TYPE_IP_PLUGIN: ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt); if(ret) diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp index 6928371..2db3add 100644 --- a/src/entry/Maat_table_runtime.cpp +++ b/src/entry/Maat_table_runtime.cpp @@ -7,30 +7,6 @@ #include #include - -int plugin_EX_data_free(const char* row, int key_column, - MESA_htable_handle key2ex_hash, void *logger) -{ - size_t key_offset=0, key_len=0; - int ret=0; - ret=get_column_pos(row, key_column, &key_offset, &key_len); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "plugin/ip_plugin EX data del error: cannot find column %d of %s", - key_column, row); - return -1; - } - ret=MESA_htable_del(key2ex_hash, (const unsigned char*)row+key_offset, key_len, NULL); - if(ret<0) - { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "plugin/ip_plugin EX data del error: no such key %.*s of %s", - key_len, row+key_offset, row); - return -1; - } - return 0; -} struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row) { struct ip_rule* range_rule=ALLOC(struct ip_rule, 1); @@ -146,25 +122,6 @@ void _notype_fqdn_rule_free(void* p) fqdn_rule_free((struct FQDN_rule*)p); return; } -struct xx_plugin_ex_free_wrapper -{ - struct EX_data_rt* ex_data_rt; - char* row; - size_t key_offset; - size_t key_len; - void* logger; -}; -void xx_plugin_ex_data_wrapper_free(void* ex_data) -{ - struct xx_plugin_ex_free_wrapper* wrapper=(struct xx_plugin_ex_free_wrapper*)ex_data; - EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger); - free(wrapper->row); - wrapper->key_offset=0; - wrapper->key_len=0; - free(wrapper); - return; -} - static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num) { @@ -193,7 +150,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche { EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema); } - table_rt->ip_plugin.bin=Maat_garbage_bin_new(0); break; case TABLE_TYPE_FQDN_PLUGIN: table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id, @@ -203,7 +159,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche { EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema); } - table_rt->fqdn_plugin.bin=Maat_garbage_bin_new(0); break; default: break; @@ -251,13 +206,11 @@ static void table_runtime_free(struct Maat_table_runtime* p) case TABLE_TYPE_IP_PLUGIN: ip_matcher_free(p->ip_plugin.ip_matcher); - Maat_garbage_bin_free(p->ip_plugin.bin); EX_data_rt_free(p->ip_plugin.ex_data_rt); assert(p->ip_plugin.new_ip_matcher==NULL); break; case TABLE_TYPE_FQDN_PLUGIN: FQDN_engine_free(p->fqdn_plugin.fqdn_engine); - Maat_garbage_bin_free(p->fqdn_plugin.bin); EX_data_rt_free(p->fqdn_plugin.ex_data_rt); assert(p->fqdn_plugin.new_fqdn_engine==NULL); break; @@ -395,7 +348,10 @@ void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, stru return; } - +void Maat_table_runtime_plugin_garbage_collection(struct Maat_table_runtime* table_rt) +{ + EX_data_rt_garbage_collection(table_rt->plugin.ex_data_rt); +}; void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag) { GIE_digest_t* digest_rule=NULL; @@ -434,7 +390,6 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, size_t fqdn_offset=0, fqdn_len=0; struct FQDN_rule* fqdn_rule=NULL; int ret=0; - struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL; if(fqdn_plugin_schema->have_exdata) { ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); @@ -477,18 +432,12 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, } else { - wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1); - wrapper_for_free->row=_maat_strdup(row); - wrapper_for_free->ex_data_rt=fqdn_plugin_rt->ex_data_rt; - wrapper_for_free->key_len=row_id_len; - wrapper_for_free->key_offset=row_id_offset; - wrapper_for_free->logger=logger; - Maat_garbage_bagging(fqdn_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free); + EX_data_rt_delete_by_row(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger); } } else { - EX_data_rt_cache_row_put(fqdn_plugin_rt->ex_data_rt, row); + EX_data_rt_cache_row_put(fqdn_plugin_rt->ex_data_rt, row); } fqdn_plugin_rt->changed_flag=1; return; @@ -500,12 +449,10 @@ int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runti struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin; assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); struct EX_data_container **exc_array=NULL; - Maat_garbage_collect_routine(fqdn_rt->bin); struct FQDN_rule* rules=NULL; size_t rule_cnt=0, i=0; if(!fqdn_rt->changed_flag) { - assert(0==Maat_garbage_bin_get_size(fqdn_rt->bin)); return 0; } @@ -532,6 +479,7 @@ struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_r table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine; assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); table_rt->fqdn_plugin.new_fqdn_engine=NULL; + EX_data_rt_garbage_collection(table_rt->fqdn_plugin.ex_data_rt); table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt); return old_one; } @@ -649,10 +597,8 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN); if(!ip_plugin->changed_flag) { - assert(0==Maat_garbage_bin_get_size(ip_plugin->bin)); return 0; } - Maat_garbage_collect_routine(ip_plugin->bin); rule_cnt=EX_data_rt_list_all_ex_container(ip_plugin->ex_data_rt, &exc_array); rules=ALLOC(struct ip_rule, rule_cnt); for(i=0; itable_type==TABLE_TYPE_IP_PLUGIN); table_rt->ip_plugin.new_ip_matcher=NULL; table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt); + EX_data_rt_garbage_collection(table_rt->ip_plugin.ex_data_rt); return old_one; } @@ -694,7 +641,6 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s size_t row_id_offset=0, row_id_len=0; struct ip_rule* ip_rule=NULL; int ret=0; - struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL; if(ip_plugin_schema->have_exdata) { ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len); @@ -728,19 +674,14 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s } else { - wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1); - wrapper_for_free->row=_maat_strdup(row); - wrapper_for_free->ex_data_rt=ip_plugin_rt->ex_data_rt; - wrapper_for_free->key_len=row_id_len; - wrapper_for_free->key_offset=row_id_offset; - wrapper_for_free->logger=logger; - Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free); + + EX_data_rt_delete_by_row(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger); free(ip_rule); } } else { - EX_data_rt_cache_row_put(ip_plugin_rt->ex_data_rt, row); + EX_data_rt_cache_row_put(ip_plugin_rt->ex_data_rt, row); } ip_plugin_rt->changed_flag=1; return; diff --git a/src/inc_internal/Maat_ex_data.h b/src/inc_internal/Maat_ex_data.h index 17230ab..412eb7c 100644 --- a/src/inc_internal/Maat_ex_data.h +++ b/src/inc_internal/Maat_ex_data.h @@ -24,4 +24,6 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array); void* EX_data_container_get_user_data(struct EX_data_container* ex_container); size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt); +void EX_data_rt_garbage_collection(struct EX_data_rt* ex_rt); + diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h index 0ac2846..4871c11 100644 --- a/src/inc_internal/Maat_table_runtime.h +++ b/src/inc_internal/Maat_table_runtime.h @@ -1,6 +1,5 @@ #include "Maat_table.h" #include "Maat_ex_data.h" -#include "Maat_garbage_collection.h" #include "IPMatcher.h" #include "gram_index_engine.h" @@ -19,8 +18,7 @@ struct fqdn_plugin_runtime { struct FQDN_engine* fqdn_engine; struct FQDN_engine* new_fqdn_engine; - struct EX_data_rt* ex_data_rt; //for fqdn_plugin ONLY - struct Maat_garbage_bin* bin; + struct EX_data_rt* ex_data_rt; int changed_flag; }; struct plugin_runtime @@ -34,7 +32,6 @@ struct ip_plugin_runtime struct EX_data_rt* ex_data_rt; struct ip_matcher* ip_matcher; struct ip_matcher* new_ip_matcher; - struct Maat_garbage_bin* bin; int changed_flag; }; struct expr_runtime @@ -88,6 +85,8 @@ void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt); void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); +void Maat_table_runtime_plugin_garbage_collection(struct Maat_table_runtime* table_rt); + void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger); int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);