gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

因使用MAAT_SET_SCAN_NO_COUNT后，在Group移除region id时会出现误命中，移除对MAAT_SET_SCAN_NO_COUNT的支持。 回调表使用feather中的垃圾回收队列，便于观察队列长度。

Browse Source
This commit is contained in:
zhengchao
2021-07-21 14:48:08 +08:00
parent 72ffbd9809
commit 0037646ef6
10 changed files with 54 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
inc/Maat_rule.h
View File

@@ -198,9 +198,8 @@ enum MAAT_SCAN_OPT
{
MAAT_SET_SCAN_DISTRICT=1, //VALUE is a const char*, SIZE= strlen(string). DEFAULT: no default.
MAAT_SET_SCAN_LAST_REGION, //VALUE is NULL, SIZE=0. This option indicates that the follow scan is the last region of current scan combination.
MAAT_GET_SCAN_HIT_PATH, //VALUE is struct Maat_hit_path_t*, an array of struct Maat_hit_path_t, SIZE= sizeof(struct Maat_hit_path_t)*N,
MAAT_GET_SCAN_HIT_PATH //VALUE is struct Maat_hit_path_t*, an array of struct Maat_hit_path_t, SIZE= sizeof(struct Maat_hit_path_t)*N,
//Maat_get_scan_status returns actual got number.
MAAT_SET_SCAN_NO_COUNT //VALUE is NULL, SIZE=0. This option indicates taht follow scan is a duplication of previous scan.
};
//return 0 if success, return -1 when failed;
int Maat_set_scan_status(Maat_feather_t feather, scan_status_t* mid, enum MAAT_SCAN_OPT type, const void* value, int size);

13
src/entry/Maat_api.cpp
View File

@@ -76,16 +76,9 @@ int insert_set_id(unsigned long long **set, size_t* size, size_t cnt, unsigned l
}
}
void scan_staus_count_inc(struct _OUTER_scan_status_t* _mid)
static void scan_staus_count_inc(struct _OUTER_scan_status_t* _mid)
{
if(_mid->is_no_count_scan)
{
_mid->is_no_count_scan=0;
}
else
{
_mid->scan_cnt++;
}
return;
}
@@ -2481,10 +2474,6 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA
assert(_mid->is_last_region==0);
_mid->is_last_region=1;
break;
case MAAT_SET_SCAN_NO_COUNT:
assert(_mid->is_no_count_scan==0);
_mid->is_no_count_scan=1;
break;
default:
_feather->scan_err_cnt++;
return -1;

14
src/entry/Maat_ex_data.cpp
View File

@@ -33,7 +33,7 @@ struct EX_data_rt
const struct EX_data_schema* ex_schema;
int table_id;
void (* user_data_free)(void *user_data);
struct Maat_garbage_bin* bin;
struct Maat_garbage_bin* ref_bin;
};
@@ -59,7 +59,7 @@ void cache_row_free(void*p)
}
UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free};
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data))
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data), struct Maat_garbage_bin* bin)
{
struct EX_data_rt* p=ALLOC(struct EX_data_rt, 1);
p->hash_key2ex=NULL;
@@ -67,7 +67,7 @@ struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t
p->table_id=table_id;
p->user_data_free=user_data_free;
pthread_rwlock_init(&p->rwlock, NULL);
p->bin=Maat_garbage_bin_new(0);
p->ref_bin=bin;
return p;
};
size_t EX_data_rt_get_cached_row_num(struct EX_data_rt* ex_rt)
@@ -95,7 +95,7 @@ void EX_data_rt_free(struct EX_data_rt* p)
p->cache_row_num=0;
}
pthread_rwlock_destroy(&p->rwlock);
Maat_garbage_bin_free(p->bin);
p->ref_bin=NULL;
free(p);
return;
}
@@ -172,7 +172,7 @@ int EX_data_rt_delete_by_row(struct EX_data_rt* ex_rt, const char* row, const ch
if(exc)
{
HASH_DELETE(hh, ex_rt->hash_key2ex, exc);
Maat_garbage_bagging(ex_rt->bin, exc, (void (*)(void*))EX_data_container_free);
Maat_garbage_bagging(ex_rt->ref_bin, exc, (void (*)(void*))EX_data_container_free);
ret=0;
}
else
@@ -242,8 +242,4 @@ size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt)
pthread_rwlock_unlock(&ex_rt->rwlock);
return count;
}
void EX_data_rt_garbage_collection(struct EX_data_rt* ex_rt)
{
Maat_garbage_collect_routine(ex_rt->bin);
}

5
src/entry/Maat_hierarchy.cpp
View File

@@ -72,11 +72,13 @@ struct Maat_hierarchy_internal_hit_path
};
static int Maat_hierarchy_hit_path_add(UT_array* hit_paths, int region_id, int virtual_table_id, int Nth_scan, int Nth_region_result)
{
struct Maat_hierarchy_internal_hit_path new_path, *tmp_path=NULL;
struct Maat_hierarchy_internal_hit_path new_path;
new_path.region_id=region_id;
new_path.Nth_hit_region=Nth_region_result;
new_path.Nth_scan=Nth_scan;
new_path.virtual_table_id=virtual_table_id;
/*
struct Maat_hierarchy_internal_hit_path *tmp_path=NULL;
size_t i=0, num=utarray_len(hit_paths);
for(i=0; i<num; i--)
{
@@ -95,6 +97,7 @@ static int Maat_hierarchy_hit_path_add(UT_array* hit_paths, int region_id, int v
}
}
}
*/
utarray_push_back(hit_paths, &new_path);
return 1;
}

3
src/entry/Maat_rule.cpp
View File

@@ -790,7 +790,7 @@ struct Maat_scanner* create_maat_scanner(unsigned int version, _Maat_feather_t *
scanner->logger_ref=feather->logger;
scanner->region_rslt_buff=ALLOC(scan_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num);
scanner->gie_rslt_buff=ALLOC(GIE_result_t, MAX_SCANNER_HIT_NUM*scan_thread_num);
scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num);
scanner->table_rt_mgr=Maat_table_runtime_manager_create(feather->table_mgr, feather->scan_thread_num, feather->garbage_bin);
scanner->max_table_num=Maat_table_manager_get_size(feather->table_mgr);
return scanner;
}
@@ -2369,7 +2369,6 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
}
break;
case TABLE_TYPE_PLUGIN:
Maat_table_runtime_plugin_garbage_collection(table_rt);
break;
case TABLE_TYPE_IP_PLUGIN:
ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);

24
src/entry/Maat_table_runtime.cpp
View File

@@ -58,6 +58,7 @@ struct Maat_table_runtime_manager
{
struct Maat_table_runtime** table_rt;
size_t n_table_rt;
struct Maat_garbage_bin* ref_bin;
};
static GIE_digest_t* create_digest_rule(unsigned int id, enum GIE_operation op,const char* digest,
short cfds_lvl, void* tag)
@@ -122,7 +123,7 @@ void _notype_fqdn_rule_free(void* p)
fqdn_rule_free((struct FQDN_rule*)p);
return;
}
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num)
static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_schema* table_schema, int max_thread_num, struct Maat_garbage_bin* bin)
{
struct Maat_table_runtime* table_rt= ALLOC(struct Maat_table_runtime, 1);
@@ -136,7 +137,8 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
case TABLE_TYPE_PLUGIN:
table_rt->plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->plugin.ex_schema.key2index_func,
NULL);
NULL,
bin);
if(table_schema->plugin.have_exdata)
{
EX_data_rt_set_schema(table_rt->plugin.ex_data_rt, &table_schema->plugin.ex_schema);
@@ -145,7 +147,8 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
case TABLE_TYPE_IP_PLUGIN:
table_rt->ip_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->ip_plugin.ex_schema.key2index_func,
free);
free,
bin);
if(table_schema->ip_plugin.have_exdata)
{
EX_data_rt_set_schema(table_rt->ip_plugin.ex_data_rt, &table_schema->ip_plugin.ex_schema);
@@ -154,7 +157,8 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
case TABLE_TYPE_FQDN_PLUGIN:
table_rt->fqdn_plugin.ex_data_rt=EX_data_rt_new(table_schema->table_id,
table_schema->fqdn_plugin.ex_schema.key2index_func,
_notype_fqdn_rule_free);
_notype_fqdn_rule_free,
bin);
if(table_schema->fqdn_plugin.have_exdata)
{
EX_data_rt_set_schema(table_rt->fqdn_plugin.ex_data_rt, &table_schema->fqdn_plugin.ex_schema);
@@ -230,12 +234,13 @@ static void table_runtime_free(struct Maat_table_runtime* p)
return;
}
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num)
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin)
{
const struct Maat_table_schema* table_desc=NULL;
struct Maat_table_runtime* table_rt=NULL;
struct Maat_table_runtime_manager* table_rt_mgr=ALLOC(struct Maat_table_runtime_manager, 1);
size_t i=0;
table_rt_mgr->ref_bin=bin;
table_rt_mgr->n_table_rt=Maat_table_manager_get_size(table_manager);
table_rt_mgr->table_rt=ALLOC(struct Maat_table_runtime*, table_rt_mgr->n_table_rt);
for(i=0; i<table_rt_mgr->n_table_rt; i++)
@@ -245,7 +250,7 @@ struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat
{
continue;
}
table_rt=table_runtime_new(table_desc, max_thread_num);
table_rt=table_runtime_new(table_desc, max_thread_num, table_rt_mgr->ref_bin);
table_rt_mgr->table_rt[i]=table_rt;
}
return table_rt_mgr;
@@ -260,6 +265,7 @@ void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_m
}
free(table_rt_mgr->table_rt);
table_rt_mgr->table_rt=NULL;
table_rt_mgr->ref_bin=NULL;
free(table_rt_mgr);
}
struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id)
@@ -348,10 +354,6 @@ void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, stru
return;
}
void Maat_table_runtime_plugin_garbage_collection(struct Maat_table_runtime* table_rt)
{
EX_data_rt_garbage_collection(table_rt->plugin.ex_data_rt);
};
void Maat_table_runtime_digest_add(struct Maat_table_runtime* table_rt, int expr_id, const char* digest, short confidence_degree, void* tag)
{
GIE_digest_t* digest_rule=NULL;
@@ -479,7 +481,6 @@ struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_r
table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine;
assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
table_rt->fqdn_plugin.new_fqdn_engine=NULL;
EX_data_rt_garbage_collection(table_rt->fqdn_plugin.ex_data_rt);
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt);
return old_one;
}
@@ -629,7 +630,6 @@ struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_run
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
table_rt->ip_plugin.new_ip_matcher=NULL;
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt);
EX_data_rt_garbage_collection(table_rt->ip_plugin.ex_data_rt);
return old_one;
}

5
src/inc_internal/Maat_ex_data.h
View File

@@ -1,9 +1,9 @@
#include "Maat_rule.h"
#include "Maat_garbage_collection.h"
struct EX_data_rt;
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data));
struct EX_data_rt* EX_data_rt_new(int table_id, Maat_plugin_EX_key2index_func_t * key2index, void (* user_data_free)(void *user_data), struct Maat_garbage_bin* bin);
void EX_data_rt_free(struct EX_data_rt* p);
void EX_data_rt_set_schema(struct EX_data_rt* p, const struct EX_data_schema* schema);
void EX_data_rt_cache_row_put(struct EX_data_rt* p, const char* row);
@@ -24,6 +24,5 @@ MAAT_RULE_EX_DATA EX_data_rt_get_EX_data_by_container(struct EX_data_rt* ex_rt,
size_t EX_data_rt_list_all_ex_container(struct EX_data_rt* ex_rt, struct EX_data_container*** ex_container_array);
void* EX_data_container_get_user_data(struct EX_data_container* ex_container);
size_t EX_data_rt_get_ex_container_count(struct EX_data_rt* ex_rt);
void EX_data_rt_garbage_collection(struct EX_data_rt* ex_rt);

1
src/inc_internal/Maat_rule_internal.h
View File

@@ -169,7 +169,6 @@ struct _OUTER_scan_status_t
unsigned short thread_num;
unsigned char is_set_district;
unsigned char is_last_region;
unsigned char is_no_count_scan;
int district_id;
int scan_cnt;
struct Maat_hierarchy_compile_mid* compile_mid;

3
src/inc_internal/Maat_table_runtime.h
View File

@@ -71,7 +71,7 @@ struct Maat_table_runtime
mcore_long_t hit_cnt;
};
struct Maat_table_runtime_manager;
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num);
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin);
void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr);
struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id);
@@ -85,7 +85,6 @@ void Maat_table_runtime_digest_del(struct Maat_table_runtime* table_rt, int expr
int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
void Maat_table_runtime_plugin_garbage_collection(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);

34
test/test_maatframe.cpp
View File

@@ -4083,8 +4083,8 @@ TEST_F(MaatCmdTest, SameScanStatusWhenClauseUpdate_TSG6419)
Maat_clean_status(&mid);
}
#define ScanStatus_Set_No_Count
TEST_F(MaatCmdTest, ScanStatusSetNoCount)
#define Hierarchy_GroupEdit
TEST_F(MaatCmdTest, GroupEdit)
{
Maat_feather_t feather=MaatCmdTest::_shared_feather;
@@ -4148,7 +4148,7 @@ TEST_F(MaatCmdTest, ScanStatusSetNoCount)
sleep(1);
int table_id=0, ret=0, i=0;
int table_id=0, ret=0;
struct Maat_rule_t result[4];
scan_status_t mid=NULL;
struct ipaddr ipv4_addr;
@@ -4164,6 +4164,7 @@ TEST_F(MaatCmdTest, ScanStatusSetNoCount)
table_id=Maat_table_register(feather, app_id_table_name);
ret=Maat_scan_intval(feather, table_id, scan_app_id, result, 4, &mid, 0);
EXPECT_EQ(ret, 0);
Maat_clean_status(&mid);
//region11->group11--clause1-->compile1
@@ -4180,14 +4181,13 @@ TEST_F(MaatCmdTest, ScanStatusSetNoCount)
sleep(1);
table_id=Maat_table_register(feather, ip_table_name);
ret=Maat_scan_proto_addr(feather,table_id, &ipv4_addr, 6, result, 4, &mid,0);
table_id=Maat_table_register(feather, app_id_table_name);
ret=Maat_scan_intval(feather, table_id, scan_app_id, result, 4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, compile1.config_id);
for(i=0; i<100; i++)
{
Maat_set_scan_status(feather, &mid, MAAT_SET_SCAN_NO_COUNT, NULL, 0);
ret=Maat_scan_intval(feather, table_id, scan_app_id, result, 4, &mid, 0);
}
struct Maat_hit_path_t hit_path[128];
memset(hit_path, 0, sizeof(hit_path));
@@ -4197,6 +4197,24 @@ TEST_F(MaatCmdTest, ScanStatusSetNoCount)
Maat_clean_status(&mid);
//region11->group11--clause1-->compile1
// /
//region21->group21--clause2---/
Maat_command_raw_set_region(feather, MAAT_OP_DEL, &region22, group21.group_id);
sleep(1);
memset(result, 0, sizeof(result));
table_id=Maat_table_register(feather, ip_table_name);
ret=Maat_scan_proto_addr(feather,table_id, &ipv4_addr, 6, result, 4, &mid,0);
EXPECT_EQ(ret, -2);
table_id=Maat_table_register(feather, app_id_table_name);
ret=Maat_scan_intval(feather, table_id, scan_app_id, result, 4, &mid, 0);
EXPECT_EQ(ret, 0);
Maat_clean_status(&mid);
}
TEST_F(MaatCmdTest, CompileDelete_TSG6548)