gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

删除IP Plugin表项时，未能及时更新ip_matcher，也未能正确同步uthash和ip_matcher的状态，导致ip_matcher返回了已被删除的ex_data。解决方案：

Browse Source 
- 在IP Plugin的table runtime中增加垃圾回收队列，延迟删除EX_data，并延后ip_matcher在扫描线程的生效时机。
- 在scanner中增加ip_plugin_update_q_size，在IP Plugin的table runtime中增加changed_flag，以判断ip_matcher是否需要更新
This commit is contained in:
zhengchao
2020-08-19 22:57:37 +08:00
parent 5931b445ff
commit a44e14f82d
7 changed files with 172 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/entry/Maat_command.cpp
View File

@@ -1901,22 +1901,36 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_
, line_rule[i]->table_name);
ret=-1;
goto error_out;
}
}
p_table=Maat_table_get_by_id_raw(_feather->table_mgr, table_id);
if(!p_table)
{
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
,"Command set line id %d failed: table %s is not a plugin table."
, line_rule[i]->rule_id
{
ret=-1;
goto error_out;
}
}
int valid_flag_column=0;
switch(p_table->table_type)
{
case TABLE_TYPE_PLUGIN:
valid_flag_column=p_table->plugin.valid_flag_column;
plugin_desc=&(p_table->plugin);
break;
case TABLE_TYPE_IP_PLUGIN:
valid_flag_column=p_table->ip_plugin.valid_flag_column;
break;
default:
MESA_handle_runtime_log(_feather->logger,RLOG_LV_FATAL,maat_command
,"Command set line id %d failed: table %s is not a plugin or ip_plugin table."
, line_rule[i]->rule_id
, line_rule[i]->table_name);
ret=-1;
goto error_out;
}
if(op==MAAT_OP_ADD)
{
ret=get_valid_flag_offset(line_rule[i]->table_line
, p_table->table_type
, p_table->table_type
, valid_flag_column);
if(ret<0||
(op==MAAT_OP_ADD&&line_rule[i]->table_line[ret]!='1'))
@@ -1935,7 +1949,7 @@ int Maat_cmd_set_lines(Maat_feather_t feather,const struct Maat_cmd_line** line_
if(line_rule[i]->expire_after>0)
{
absolute_expire_time=server_time+line_rule[i]->expire_after;
}
}
if(plugin_desc && plugin_desc->n_foreign>0)
{
for(j=0;j<plugin_desc->n_foreign;j++)

2
src/entry/Maat_garbage_collection.cpp
View File

@@ -67,7 +67,7 @@ void Maat_garbage_collect_routine(struct Maat_garbage_bin* bin)
for(p=TAILQ_FIRST(&bin->garbage_q); p!=NULL; p=tmp)
{
tmp=TAILQ_NEXT(p, entries);
if(now-p->create_time>p->timeout)
if(now-p->create_time>p->timeout || p->timeout==0)
{
p->garbage_free(p->garbage);
TAILQ_REMOVE(&bin->garbage_q, p, entries);

15
src/entry/Maat_rule.cpp
View File

@@ -34,7 +34,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_3_0_20200731=1;
int MAAT_FRAME_VERSION_3_0_20200819=1;
int is_valid_table_name(const char* str)
{
@@ -2119,7 +2119,6 @@ void update_digest_rule(struct Maat_table_schema* table, const char* table_line,
}
}
error_out:
digest_rule->digest_string=NULL;
free(digest_rule);
@@ -2218,6 +2217,7 @@ void update_ip_plugin_table(struct Maat_table_schema* table_schema, const char*
}
}
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, table_row, logger);
scanner->ip_plugin_update_q_size++;
return;
}
@@ -2269,9 +2269,12 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
}
break;
case TABLE_TYPE_IP_PLUGIN:
Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt);
old_ip_matcher=Maat_table_runtime_dettach_old_ip_matcher(table_rt);
Maat_garbage_bagging(scanner->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);
old_ip_matcher=Maat_table_runtime_apply_new_ip_matcher(table_rt);
if(old_ip_matcher)
{
Maat_garbage_bagging(scanner->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
}
break;
default:
break;
@@ -2353,7 +2356,7 @@ void maat_finish_cb(void* u_para)
feather->scanner->cfg_num=scanner_rule_num(feather->scanner);
feather->scanner->version=feather->maat_version;
expr_wait_q_cnt=MESA_lqueue_get_count(feather->scanner->region_update_q);
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size;
feather->postpone_q_size=expr_wait_q_cnt+feather->scanner->gie_update_q_size+feather->scanner->ip_plugin_update_q_size;
if(time(NULL)-feather->scanner->last_update_time>=feather->effect_interval_ms/1000)
{
do_scanner_update(feather->scanner,

67
src/entry/Maat_table_runtime.cpp
View File

@@ -144,6 +144,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
{
EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema);
}
table_rt->ip_plugin.bin=Maat_garbage_bin_new(0);
break;
default:
break;
@@ -191,8 +192,9 @@ static void table_runtime_free(struct Maat_table_runtime* p)
case TABLE_TYPE_IP_PLUGIN:
ip_matcher_free(p->ip_plugin.ip_matcher);
EX_data_rt_free(p->ip_plugin.ex_data_rt);
assert(p->ip_plugin.old_ip_matcher==NULL);
Maat_garbage_bin_free(p->ip_plugin.bin);
EX_data_rt_free(p->ip_plugin.ex_data_rt);
assert(p->ip_plugin.new_ip_matcher==NULL);
break;
case TABLE_TYPE_PLUGIN:
EX_data_rt_free(p->plugin.ex_data_rt);
@@ -422,15 +424,22 @@ int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
return q_cnt;
}
int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt)
int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt)
{
struct ip_matcher* new_ip_matcher=NULL;
size_t rule_cnt=0;
size_t i=0, mem_use=0;
struct ip_rule *rules=NULL;
struct EX_data_container **exc_array=NULL;
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
rule_cnt=EX_data_rt_list_all_ex_container(table_rt->ip_plugin.ex_data_rt, &exc_array);
struct ip_plugin_runtime *ip_plugin=&(table_rt->ip_plugin);
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
if(!ip_plugin->changed_flag)
{
assert(0==Maat_garbage_bin_get_size(ip_plugin->bin));
return 0;
}
Maat_garbage_collect_routine(ip_plugin->bin);
rule_cnt=EX_data_rt_list_all_ex_container(ip_plugin->ex_data_rt, &exc_array);
rules=ALLOC(struct ip_rule, rule_cnt);
for(i=0; i<rule_cnt; i++)
{
@@ -441,21 +450,41 @@ int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* t
if(rule_cnt>0)
{
new_ip_matcher=ip_matcher_new(rules, rule_cnt, &mem_use);
table_rt->ip_plugin.old_ip_matcher=table_rt->ip_plugin.ip_matcher;
table_rt->ip_plugin.ip_matcher=new_ip_matcher;
assert(ip_plugin->new_ip_matcher==NULL);
ip_plugin->new_ip_matcher=new_ip_matcher;
}
free(rules);
free(exc_array);
exc_array=NULL;
ip_plugin->changed_flag=0;
return 0;
}
struct ip_matcher* Maat_table_runtime_dettach_old_ip_matcher(struct Maat_table_runtime* table_rt)
struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt)
{
struct ip_matcher* old_one=table_rt->ip_plugin.old_ip_matcher;
struct ip_matcher* old_one=table_rt->ip_plugin.ip_matcher;
table_rt->ip_plugin.ip_matcher=table_rt->ip_plugin.new_ip_matcher;
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
table_rt->ip_plugin.old_ip_matcher=NULL;
table_rt->ip_plugin.new_ip_matcher=NULL;
return old_one;
}
struct ip_plugin_ex_free_wrapper
{
struct EX_data_rt* ex_data_rt;
char* row;
size_t key_offset;
size_t key_len;
void* logger;
};
void ip_plugin_ex_data_wrapper_free(void* ex_data)
{
struct ip_plugin_ex_free_wrapper* wrapper=(struct ip_plugin_ex_free_wrapper*)ex_data;
EX_data_rt_delete_by_row(wrapper->ex_data_rt, wrapper->row, wrapper->row + wrapper->key_offset, wrapper->key_len, wrapper->logger);
free(wrapper->row);
wrapper->key_offset=0;
wrapper->key_len=0;
free(wrapper);
return;
}
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{
struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
@@ -463,8 +492,8 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
size_t is_valid_offset=0, valid_len=0;
size_t key_offset=0, key_len=0;
struct ip_rule* ip_rule=NULL;
int ret=0;
int ret=0;
struct ip_plugin_ex_free_wrapper* wrapper_for_free=NULL;
if(ip_plugin_schema->have_exdata)
{
ret=Maat_helper_read_column(row, ip_plugin_schema->valid_flag_column, &is_valid_offset, &valid_len);
@@ -493,17 +522,25 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
}
if(atoi(row+is_valid_offset)==1)//add
{
EX_data_rt_row2EX_data(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, ip_rule, logger);
}
else
{
EX_data_rt_delete_by_row(ip_plugin_rt->ex_data_rt, row, row+key_offset, key_len, logger);
wrapper_for_free=ALLOC(struct ip_plugin_ex_free_wrapper, 1);
wrapper_for_free->row=_maat_strdup(row);
wrapper_for_free->ex_data_rt=ip_plugin_rt->ex_data_rt;
wrapper_for_free->key_len=key_len;
wrapper_for_free->key_offset=key_offset;
wrapper_for_free->logger=logger;
Maat_garbage_bagging(ip_plugin_rt->bin, wrapper_for_free, ip_plugin_ex_data_wrapper_free);
}
}
else
{
EX_data_rt_cache_row_put(ip_plugin_rt->ex_data_rt, row);
}
ip_plugin_rt->changed_flag=1;
return;
}
int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger)
@@ -518,8 +555,8 @@ int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* tab
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger);
}
EX_data_rt_clear_row_cache(ip_plugin_rt->ex_data_rt);
Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt);
Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);
Maat_table_runtime_apply_new_ip_matcher(table_rt);//returned NULL.
return 0;
}