gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

因使用MAAT_SET_SCAN_NO_COUNT后，在Group移除region id时会出现误命中，移除对MAAT_SET_SCAN_NO_COUNT的支持。 回调表使用feather中的垃圾回收队列，便于观察队列长度。

Browse Source
This commit is contained in:
zhengchao
2021-07-21 14:48:08 +08:00
parent 72ffbd9809
commit 0037646ef6
10 changed files with 54 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/entry/Maat_api.cpp
View File

@@ -76,16 +76,9 @@ int insert_set_id(unsigned long long **set, size_t* size, size_t cnt, unsigned l
}
}
void scan_staus_count_inc(struct _OUTER_scan_status_t* _mid)
static void scan_staus_count_inc(struct _OUTER_scan_status_t* _mid)
{
if(_mid->is_no_count_scan)
{
_mid->is_no_count_scan=0;
}
else
{
_mid->scan_cnt++;
}
_mid->scan_cnt++;
return;
}
@@ -2481,10 +2474,6 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA
assert(_mid->is_last_region==0);
_mid->is_last_region=1;
break;
case MAAT_SET_SCAN_NO_COUNT:
assert(_mid->is_no_count_scan==0);
_mid->is_no_count_scan=1;
break;
default:
_feather->scan_err_cnt++;
return -1;

14
src/entry/Maat_ex_data.cpp
View File

@@ -33,7 +33,7 @@ struct EX_data_rt
const struct EX_data_schema* ex_schema;
int table_id;
void (* user_data_free)(void *user_data);
struct Maat_garbage_bin* bin;
struct Maat_garbage_bin* ref_bin;
};
@@ -59,7 +59,7 @@ void cache_row_free(void*p)
}
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data), struct Maat_garbage_bin* bin)
{
struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1);
p->hash_key2ex=NULL;
@@ -67,7 +67,7 @@ struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t
p->table_id=table_id;
p->user_data_free=user_data_free;
pthread_rwlock_init(&p->rwlock, NULL);
p->bin=Maat_garbage_bin_new(0);
p->ref_bin=bin;
return p;
};
size_t EX_data_rt_get_cached_row_num(struct EX_data_rt* ex_rt)
@@ -95,7 +95,7 @@ void EX_data_rt_free(struct EX_data_rt* p)
p->cache_row_num=0;
}
pthread_rwlock_destroy(&p->rwlock);
Maat_garbage_bin_free(p->bin);
p->ref_bin=NULL;
free(p);
return;
}
@@ -172,7 +172,7 @@ int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const ch
if(exc)
{
HASH_DELETE(hh, ex_rt->hash_key2ex, exc);
Maat_garbage_bagging(ex_rt->bin, exc, (void (*)(void*))EX_data_container_free);
Maat_garbage_bagging(ex_rt->ref_bin, exc, (void (*)(void*))EX_data_container_free);
ret=0;
}
else
@@ -242,8 +242,4 @@ size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt)
pthread_rwlock_unlock(&ex_rt->rwlock);
return count;
}
void EX_data_rt_garbage_collection(struct EX_data_rt* ex_rt)
{
Maat_garbage_collect_routine(ex_rt->bin);
}

5
src/entry/Maat_hierarchy.cpp
View File

@@ -72,11 +72,13 @@ struct Maat_hierarchy_internal_hit_path
};
static int Maat_hierarchy_hit_path_add(UT_array* hit_paths, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result)
{
struct Maat_hierarchy_internal_hit_path new_path, *tmp_path=NULL;
struct Maat_hierarchy_internal_hit_path new_path;
new_path.region_id=region_id;
new_path.Nth_hit_region=Nth_region_result;
new_path.Nth_scan=Nth_scan;
new_path.virtual_table_id=virtual_table_id;
/*
struct Maat_hierarchy_internal_hit_path *tmp_path=NULL;
size_t i=0, num=utarray_len(hit_paths);
for(i=0; i<num; i--)
{
@@ -95,6 +97,7 @@ static int Maat_hierarchy_hit_path_add(UT_array* hit_paths, int region_id, int v
}
}
}
*/
utarray_push_back(hit_paths, &new_path);
return 1;
}

3
src/entry/Maat_rule.cpp
View File

@@ -790,7 +790,7 @@ struct Maat_scanner* create_maat_scanner(unsigned int version, _Maat_feather_t *
scanner->logger_ref=feather->logger;
scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num);
scanner->gie_rslt_buff=ALLOC(GIE_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num);
scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num);
scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num, feather->garbage_bin);
scanner->max_table_num=Maat_table_manager_get_size(feather->table_mgr);
return scanner;
}
@@ -2369,7 +2369,6 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
}
break;
case TABLE_TYPE_PLUGIN:
Maat_table_runtime_plugin_garbage_collection(table_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);

24
src/entry/Maat_table_runtime.cpp
View File

@@ -58,6 +58,7 @@ struct Maat_table_runtime_manager
{
struct Maat_table_runtime** table_rt;
size_t n_table_rt;
struct Maat_garbage_bin* ref_bin;
};
static GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest,
short cfds_lvl, void* tag)
@@ -122,7 +123,7 @@ void _notype_fqdn_rule_free(void* p)
fqdn_rule_free((struct FQDN_rule*)p);
return;
}
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num)
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num, struct Maat_garbage_bin* bin)
{
struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1);
@@ -136,7 +137,8 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
case TABLE_TYPE_PLUGIN:
table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->plugin.ex_schema.key2index_func,
NULL);
NULL,
bin);
if(table_schema->plugin.have_exdata)
{
EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema);
@@ -145,7 +147,8 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->ip_plugin.ex_schema.key2index_func,
free);
free,
bin);
if(table_schema->ip_plugin.have_exdata)
{
EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema);
@@ -154,7 +157,8 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
case TABLE_TYPE_FQDN_PLUGIN:
table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->fqdn_plugin.ex_schema.key2index_func,
_notype_fqdn_rule_free);
_notype_fqdn_rule_free,
bin);
if(table_schema->fqdn_plugin.have_exdata)
{
EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
@@ -230,12 +234,13 @@ static void table_runtime_free(struct Maat_table_runtime* p)
return;
}
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num)
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin)
{
const struct Maat_table_schema* table_desc=NULL;
struct Maat_table_runtime* table_rt=NULL;
struct Maat_table_runtime_manager* table_rt_mgr=ALLOC(struct Maat_table_runtime_manager, 1);
size_t i=0;
table_rt_mgr->ref_bin=bin;
table_rt_mgr->n_table_rt=Maat_table_manager_get_size(table_manager);
table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt);
for(i=0; i<table_rt_mgr->n_table_rt; i++)
@@ -245,7 +250,7 @@ struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat
{
continue;
}
table_rt=table_runtime_new(table_desc, max_thread_num);
table_rt=table_runtime_new(table_desc, max_thread_num, table_rt_mgr->ref_bin);
table_rt_mgr->table_rt[i]=table_rt;
}
return table_rt_mgr;
@@ -260,6 +265,7 @@ void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_m
}
free(table_rt_mgr->table_rt);
table_rt_mgr->table_rt=NULL;
table_rt_mgr->ref_bin=NULL;
free(table_rt_mgr);
}
struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id)
@@ -348,10 +354,6 @@ void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, stru
return;
}
void Maat_table_runtime_plugin_garbage_collection(struct Maat_table_runtime* table_rt)
{
EX_data_rt_garbage_collection(table_rt->plugin.ex_data_rt);
};
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag)
{
GIE_digest_t* digest_rule=NULL;
@@ -479,7 +481,6 @@ struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_r
table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine;
assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
table_rt->fqdn_plugin.new_fqdn_engine=NULL;
EX_data_rt_garbage_collection(table_rt->fqdn_plugin.ex_data_rt);
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt);
return old_one;
}
@@ -629,7 +630,6 @@ struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_run
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
table_rt->ip_plugin.new_ip_matcher=NULL;
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt);
EX_data_rt_garbage_collection(table_rt->ip_plugin.ex_data_rt);
return old_one;
}