gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复bug：ip_plugin和fqdn_plugin的扫描线程可能访问到无效的EX_data_container，导致#26； plugin的扫描线程和更新线程发生线程不安全访问EX_data_rt中的uthash。

Browse Source
This commit is contained in:
郑超
2021-03-20 08:09:52 +00:00
parent 1d8f35ca50
commit f82454a310
5 changed files with 79 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
src/entry/Maat_ex_data.cpp
View File

@@ -1,6 +1,7 @@
#include "Maat_ex_data.h"
#include "Maat_table.h"
#include "Maat_utils.h"
#include "Maat_garbage_collection.h"
#include "uthash/uthash.h"
#include "uthash/utarray.h"
@@ -26,12 +27,13 @@ struct EX_data_rt
UT_array *cache_rows;
size_t cache_row_num;
size_t cache_size;
pthread_rwlock_t rwlock;
struct EX_data_container* hash_key2ex;
const struct EX_data_schema* ex_schema;
int table_id;
void (* user_data_free)(void *user_data);
struct Maat_garbage_bin* bin;
};
@@ -64,6 +66,8 @@ struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t
utarray_new(p->cache_rows, &ut_cache_row_icd);
p->table_id=table_id;
p->user_data_free=user_data_free;
pthread_rwlock_init(&p->rwlock, NULL);
p->bin=Maat_garbage_bin_new(0);
return p;
};
size_t EX_data_rt_get_cached_row_num(struct EX_data_rt* ex_rt)
@@ -90,6 +94,8 @@ void EX_data_rt_free(struct EX_data_rt* p)
p->cache_rows=NULL;
p->cache_row_num=0;
}
pthread_rwlock_destroy(&p->rwlock);
Maat_garbage_bin_free(p->bin);
free(p);
return;
}
@@ -125,15 +131,12 @@ int EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt,
MAAT_RULE_EX_DATA ex_data=NULL;
const struct EX_data_schema* ex_schema=ex_rt->ex_schema;
struct EX_data_container* ex_container=NULL, *tmp=NULL;
HASH_FIND(hh, ex_rt->hash_key2ex, key, key_len, tmp);
if(tmp!=NULL)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"EX data add error: duplicated key %.*s of %s",
key_len, key, row);
int ret=0;
pthread_rwlock_wrlock(&ex_rt->rwlock);
return -1;
}
HASH_FIND(hh, ex_rt->hash_key2ex, key, key_len, tmp);
if(tmp==NULL)
{
ex_container=ALLOC(struct EX_data_container, 1);
ex_container->key=ALLOC(char, key_len+1);
memcpy(ex_container->key, key, key_len);
@@ -145,23 +148,42 @@ int EX_data_rt_row2EX_data(struct EX_data_rt* ex_rt,
ex_container->key_len=key_len;
HASH_ADD_KEYPTR(hh, ex_rt->hash_key2ex, ex_container->key, ex_container->key_len, ex_container);
return 0;
ret=0;
}
else
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"EX data add error: duplicated key %.*s of %s",
key_len, key, row);
ret=-1;
}
pthread_rwlock_unlock(&ex_rt->rwlock);
return ret;
}
int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const char* key, size_t key_len,
void *logger)
{
struct EX_data_container* exc=NULL;
int ret=0;
pthread_rwlock_wrlock(&ex_rt->rwlock);
HASH_FIND(hh, ex_rt->hash_key2ex, key, key_len, exc);
if(exc==NULL)
if(exc)
{
HASH_DELETE(hh, ex_rt->hash_key2ex, exc);
Maat_garbage_bagging(ex_rt->bin, exc, (void (*)(void*))EX_data_container_free);
ret=0;
}
else
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"EX data del error: no such key %.*s of %s",
key_len, key, row);
return -1;
ret=-1;
}
HASH_DELETE(hh, ex_rt->hash_key2ex, exc);
EX_data_container_free(exc);
return 0;
pthread_rwlock_unlock(&ex_rt->rwlock);
return ret;
}
MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const char* key, size_t key_len)
@@ -174,12 +196,14 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_key(struct EX_data_rt* ex_rt, const
assert(0);
return NULL;
}
pthread_rwlock_rdlock(&ex_rt->rwlock);
HASH_FIND(hh, ex_rt->hash_key2ex, key, key_len, container);
if(container!=NULL)
{
ex_rt->ex_schema->dup_func(ex_rt->table_id, &(ex_data), &(container->ex_data),
ex_rt->ex_schema->argl, ex_rt->ex_schema->argp);
}
pthread_rwlock_unlock(&ex_rt->rwlock);
return ex_data;
}
MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt, struct EX_data_container* container)
@@ -192,15 +216,18 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt,
size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array)
{
size_t ex_data_cnt=HASH_COUNT(ex_rt->hash_key2ex), i=0;
size_t ex_data_cnt=0, i=0;
struct EX_data_container* ex_container=NULL, *tmp=NULL;
pthread_rwlock_rdlock(&ex_rt->rwlock);
ex_data_cnt=HASH_COUNT(ex_rt->hash_key2ex);
*ex_container_array=ALLOC(struct EX_data_container*, ex_data_cnt);
HASH_ITER(hh, ex_rt->hash_key2ex, ex_container, tmp)
{
(*ex_container_array)[i]=ex_container;
i++;
}
pthread_rwlock_unlock(&ex_rt->rwlock);
return ex_data_cnt;
}
void* EX_data_container_get_user_data(struct EX_data_container* ex_container)
@@ -209,5 +236,14 @@ void* EX_data_container_get_user_data(struct EX_data_container* ex_container)
}
size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt)
{
return HASH_COUNT(ex_rt->hash_key2ex);
size_t count=0;
pthread_rwlock_rdlock(&ex_rt->rwlock);
count=HASH_COUNT(ex_rt->hash_key2ex);
pthread_rwlock_unlock(&ex_rt->rwlock);
return count;
}
void EX_data_rt_garbage_collection(struct EX_data_rt* ex_rt)
{
Maat_garbage_collect_routine(ex_rt->bin);
}

5
src/entry/Maat_rule.cpp
View File

@@ -57,7 +57,7 @@ extern "C"
}
#endif
int MAAT_FRAME_VERSION_3_1_12_20210308=1;
int MAAT_FRAME_VERSION_3_1_16_20210320=1;
int is_valid_table_name(const char* str)
{
@@ -2369,6 +2369,9 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
"GIE_update error.");
}
break;
case TABLE_TYPE_PLUGIN:
Maat_table_runtime_plugin_garbage_collection(table_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);
if(ret)

77
src/entry/Maat_table_runtime.cpp
View File

@@ -7,30 +7,6 @@
#include <assert.h>
#include <sys/queue.h>
int plugin_EX_data_free(const char* row, int key_column,
MESA_htable_handle key2ex_hash, void *logger)
{
size_t key_offset=0, key_len=0;
int ret=0;
ret=get_column_pos(row, key_column, &key_offset, &key_len);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"plugin/ip_plugin EX data del error: cannot find column %d of %s",
key_column, row);
return -1;
}
ret=MESA_htable_del(key2ex_hash, (const unsigned char*)row+key_offset, key_len, NULL);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"plugin/ip_plugin EX data del error: no such key %.*s of %s",
key_len, row+key_offset, row);
return -1;
}
return 0;
}
struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row)
{
struct ip_rule* range_rule=ALLOC(struct ip_rule, 1);
@@ -146,25 +122,6 @@ void _notype_fqdn_rule_free(void* p)
fqdn_rule_free((struct FQDN_rule*)p);
return;
}
struct xx_plugin_ex_free_wrapper
{
struct EX_data_rt* ex_data_rt;
char* row;
size_t key_offset;
size_t key_len;
void* logger;
};
void xx_plugin_ex_data_wrapper_free(void* ex_data)
{
struct xx_plugin_ex_free_wrapper* wrapper=(struct xx_plugin_ex_free_wrapper*)ex_data;
EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger);
free(wrapper->row);
wrapper->key_offset=0;
wrapper->key_len=0;
free(wrapper);
return;
}
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num)
{
@@ -193,7 +150,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
{
EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema);
}
table_rt->ip_plugin.bin=Maat_garbage_bin_new(0);
break;
case TABLE_TYPE_FQDN_PLUGIN:
table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
@@ -203,7 +159,6 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
{
EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
}
table_rt->fqdn_plugin.bin=Maat_garbage_bin_new(0);
break;
default:
break;
@@ -251,13 +206,11 @@ static void table_runtime_free(struct Maat_table_runtime* p)
case TABLE_TYPE_IP_PLUGIN:
ip_matcher_free(p->ip_plugin.ip_matcher);
Maat_garbage_bin_free(p->ip_plugin.bin);
EX_data_rt_free(p->ip_plugin.ex_data_rt);
assert(p->ip_plugin.new_ip_matcher==NULL);
break;
case TABLE_TYPE_FQDN_PLUGIN:
FQDN_engine_free(p->fqdn_plugin.fqdn_engine);
Maat_garbage_bin_free(p->fqdn_plugin.bin);
EX_data_rt_free(p->fqdn_plugin.ex_data_rt);
assert(p->fqdn_plugin.new_fqdn_engine==NULL);
break;
@@ -395,7 +348,10 @@ void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, stru
return;
}
void Maat_table_runtime_plugin_garbage_collection(struct Maat_table_runtime* table_rt)
{
EX_data_rt_garbage_collection(table_rt->plugin.ex_data_rt);
};
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag)
{
GIE_digest_t* digest_rule=NULL;
@@ -434,7 +390,6 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt,
size_t fqdn_offset=0, fqdn_len=0;
struct FQDN_rule* fqdn_rule=NULL;
int ret=0;
struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL;
if(fqdn_plugin_schema->have_exdata)
{
ret=Maat_helper_read_column(row, fqdn_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
@@ -477,13 +432,7 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt,
}
else
{
wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1);
wrapper_for_free->row=_maat_strdup(row);
wrapper_for_free->ex_data_rt=fqdn_plugin_rt->ex_data_rt;
wrapper_for_free->key_len=row_id_len;
wrapper_for_free->key_offset=row_id_offset;
wrapper_for_free->logger=logger;
Maat_garbage_bagging(fqdn_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free);
EX_data_rt_delete_by_row(fqdn_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger);
}
}
else
@@ -500,12 +449,10 @@ int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runti
struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin;
assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
struct EX_data_container **exc_array=NULL;
Maat_garbage_collect_routine(fqdn_rt->bin);
struct FQDN_rule* rules=NULL;
size_t rule_cnt=0, i=0;
if(!fqdn_rt->changed_flag)
{
assert(0==Maat_garbage_bin_get_size(fqdn_rt->bin));
return 0;
}
@@ -532,6 +479,7 @@ struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_r
table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine;
assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
table_rt->fqdn_plugin.new_fqdn_engine=NULL;
EX_data_rt_garbage_collection(table_rt->fqdn_plugin.ex_data_rt);
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt);
return old_one;
}
@@ -649,10 +597,8 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime*
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
if(!ip_plugin->changed_flag)
{
assert(0==Maat_garbage_bin_get_size(ip_plugin->bin));
return 0;
}
Maat_garbage_collect_routine(ip_plugin->bin);
rule_cnt=EX_data_rt_list_all_ex_container(ip_plugin->ex_data_rt, &exc_array);
rules=ALLOC(struct ip_rule, rule_cnt);
for(i=0; i<rule_cnt; i++)
@@ -683,6 +629,7 @@ struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_run
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
table_rt->ip_plugin.new_ip_matcher=NULL;
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt);
EX_data_rt_garbage_collection(table_rt->ip_plugin.ex_data_rt);
return old_one;
}
@@ -694,7 +641,6 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
size_t row_id_offset=0, row_id_len=0;
struct ip_rule* ip_rule=NULL;
int ret=0;
struct xx_plugin_ex_free_wrapper* wrapper_for_free=NULL;
if(ip_plugin_schema->have_exdata)
{
ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
@@ -728,13 +674,8 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
}
else
{
wrapper_for_free=ALLOC(struct xx_plugin_ex_free_wrapper, 1);
wrapper_for_free->row=_maat_strdup(row);
wrapper_for_free->ex_data_rt=ip_plugin_rt->ex_data_rt;
wrapper_for_free->key_len=row_id_len;
wrapper_for_free->key_offset=row_id_offset;
wrapper_for_free->logger=logger;
Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, xx_plugin_ex_data_wrapper_free);
EX_data_rt_delete_by_row(ip_plugin_rt->ex_data_rt, row, row+row_id_offset, row_id_len, logger);
free(ip_rule);
}
}

2
src/inc_internal/Maat_ex_data.h
View File

@@ -24,4 +24,6 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt,
size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array);
void* EX_data_container_get_user_data(struct EX_data_container* ex_container);
size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt);
void EX_data_rt_garbage_collection(struct EX_data_rt* ex_rt);

7
src/inc_internal/Maat_table_runtime.h
View File

@@ -1,6 +1,5 @@
#include "Maat_table.h"
#include "Maat_ex_data.h"
#include "Maat_garbage_collection.h"
#include "IPMatcher.h"
#include "gram_index_engine.h"
@@ -19,8 +18,7 @@ struct fqdn_plugin_runtime
{
struct FQDN_engine* fqdn_engine;
struct FQDN_engine* new_fqdn_engine;
struct EX_data_rt* ex_data_rt; //for fqdn_plugin ONLY
struct Maat_garbage_bin* bin;
struct EX_data_rt* ex_data_rt;
int changed_flag;
};
struct plugin_runtime
@@ -34,7 +32,6 @@ struct ip_plugin_runtime
struct EX_data_rt* ex_data_rt;
struct ip_matcher* ip_matcher;
struct ip_matcher* new_ip_matcher;
struct Maat_garbage_bin* bin;
int changed_flag;
};
struct expr_runtime
@@ -88,6 +85,8 @@ void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
void Maat_table_runtime_plugin_garbage_collection(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);