gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#17 命中包含“非”规则的编译配置时，如果不是最后一次扫描（未设置MAAT_SET_SCAN_LAST_REGION），则只记录状态，不算命中。当设置MAAT_SET_SCAN_LAST_REGION后，无论是否命中域配置，都做region_compile。

Browse Source
This commit is contained in:
zhengchao
2019-01-08 21:25:08 +06:00
parent 9c0a0786ab
commit deb41ea520
5 changed files with 196 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
src/entry/Maat_api.cpp
View File

@@ -53,6 +53,40 @@ inline void DEC_SCANNER_REF(Maat_scanner_t*scanner,int thread_num)
alignment_int64_array_add(scanner->ref_cnt, thread_num, -1);
return;
}
void Maat_clean_status(scan_status_t* mid)
{
struct _OUTER_scan_status_t* _mid=NULL;
if(*mid==NULL)
{
return;
}
_mid=(struct _OUTER_scan_status_t*)(*mid);
alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL)
{
free(_mid->inner->hitted_group_id);
free(_mid->inner);
alignment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1);
}
_mid->feather=NULL;
free(_mid);
*mid=NULL;
return;
}
inline int scan_status_should_compile_NOT(struct _OUTER_scan_status_t* _mid)
{
if( _mid &&
_mid->is_last_region==1 &&
_mid->inner &&
_mid->inner->not_grp_compile_hitted_flag)
{
return 1;
}
else
{
return 0;
}
}
//return 1 if insert a unique id
//return 0 if id is duplicated
@@ -123,7 +157,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
size_t r_in_c_cnt=0;
int shortcut_avilable_cnt=0;
unsigned char has_not_flag=0;
struct bool_matcher* bm=feather->scanner->bool_macher_expr_compiler;
struct bool_matcher* bm=feather->scanner->bool_matcher_expr_compiler;
struct Maat_group_inner_t* group_rule=NULL;
struct Maat_compile_inner_t* array_mi_rule[MAX_SCANNER_HIT_NUM];
struct Maat_compile_inner_t* _mi_rule=NULL;
@@ -149,7 +183,8 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
group_rule->group_id);
_mid->hit_group_cnt+=ret;
}
if(shortcut_avilable_cnt==region_hit_num||shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM)
if((region_hit_num>0 &&shortcut_avilable_cnt==region_hit_num) ||
shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM)
{
//short cut for rules contains one group
scan_ret=shortcut_avilable_cnt;
@@ -180,12 +215,19 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
if(0==pthread_rwlock_tryrdlock(&(_mi_rule->rwlock)))
{
if(_mi_rule->is_valid==1 && !(_mi_rule->not_group_cnt>0 && !is_last_region))
if(_mi_rule->is_valid==1)
{
if(_mi_rule->not_group_cnt>0 && !is_last_region)
{
_mid->not_grp_compile_hitted_flag=1;
}
else
{
make_group_set(_mi_rule, &(rs_result[result_cnt].group_set), &has_not_flag);
r_in_c_cnt=pickup_hit_region_from_compile(&(rs_result[result_cnt].group_set), _mid->cur_hit_id, _mid->cur_hit_cnt,
region_pos, MAX_SCANNER_HIT_NUM);
if(r_in_c_cnt>0)//compile config hitted becasue of new reigon
if(r_in_c_cnt>0 || //compile config hitted becasue of new reigon
_mid->cur_hit_cnt==0) //or ever hit compile has not group
{
fill_maat_rule(&(result[result_cnt]), &(_mi_rule->db_c_rule->m_rule_head),
_mi_rule->db_c_rule->service_defined ,_mi_rule->db_c_rule->m_rule_head.serv_def_len);
@@ -193,6 +235,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int
result_cnt++;
}
}
}
pthread_rwlock_unlock(&(_mi_rule->rwlock));
}
}
@@ -1364,9 +1407,12 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
hit_region_cnt=match_district(_mid,region_result,hit_region_cnt);
}
if(hit_region_cnt>0 || scan_status_should_compile_NOT(_mid))
{
if(hit_region_cnt>0)
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1);
}
_mid=grab_mid(mid,_feather,thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -1379,7 +1425,7 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
{
_mid->is_last_region=2;
}
if(hit_detail!=NULL&&_feather->rule_scan_type!=0)
if(hit_region_cnt>0&&hit_detail!=NULL&&_feather->rule_scan_type!=0)
{
*detail_ret=fill_region_hit_detail(data,_mid->inner,
region_result,hit_region_cnt,
@@ -1464,9 +1510,12 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
else if(region_ret>0)
else if(region_ret>0 || scan_status_should_compile_NOT(_mid))
{
if(region_ret>0)
{
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
}
_mid=grab_mid(mid, _feather, thread_num, 1);
compile_ret=region_compile(_feather,_mid->inner,
_mid->is_last_region,
@@ -2235,26 +2284,6 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
return compile_ret;
}
void Maat_clean_status(scan_status_t* mid)
{
struct _OUTER_scan_status_t* _mid=NULL;
if(*mid==NULL)
{
return;
}
_mid=(struct _OUTER_scan_status_t*)(*mid);
alignment_int64_array_add(_mid->feather->outer_mid_cnt,_mid->thread_num,-1);
if(_mid->inner!=NULL)
{
free(_mid->inner->hitted_group_id);
free(_mid->inner);
alignment_int64_array_add(_mid->feather->inner_mid_cnt,_mid->thread_num,-1);
}
_mid->feather=NULL;
free(_mid);
*mid=NULL;
return;
}
int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* value,int size)
{
struct _Maat_feather_t* _feather=(_Maat_feather_t*)feather;
@@ -2299,6 +2328,7 @@ int Maat_read_state(Maat_feather_t feather,enum MAAT_STATE_OPT type, void* valu
}
return 0;
}
int Maat_helper_read_column(const char* line, int Nth_column, size_t *column_offset, size_t *column_len)
{
return get_column_pos(line, Nth_column, column_offset, column_len);

9
src/entry/Maat_rule.cpp
View File

@@ -917,7 +917,8 @@ void walk_compile_hash(const uchar * key, uint size, void * data, void * user)
//make sure compile rule's each group has loadded.
if((compile_rule->group_cnt==compile_rule->db_c_rule->declare_grp_num
||compile_rule->db_c_rule->declare_grp_num==0)//for compatible old version
&&compile_rule->group_cnt>0)
&&compile_rule->group_cnt>0
&&compile_rule->group_cnt!=compile_rule->not_group_cnt)
{
one_set=ALLOC(struct bool_expr, 1);
//reading compile rule is safe in update thread, mutex lock called when modified
@@ -1403,7 +1404,7 @@ void destroy_maat_scanner(struct Maat_scanner_t*scanner)
map_destroy(scanner->district_map);
scanner->district_map=NULL;
assert(scanner->tmp_district_map==NULL);
destroy_bool_matcher(scanner->bool_macher_expr_compiler);
destroy_bool_matcher(scanner->bool_matcher_expr_compiler);
q_cnt=MESA_lqueue_get_count(scanner->region_update_q);
for(i=0;i<q_cnt;i++)
{
@@ -3309,10 +3310,10 @@ void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q
tmp1=create_bool_matcher(scanner->compile_hash,
scan_thread_num,
logger);
tmp2=scanner->bool_macher_expr_compiler;
tmp2=scanner->bool_matcher_expr_compiler;
//assume pinter = operation is thread safe
scanner->bool_macher_expr_compiler=tmp1;
scanner->bool_matcher_expr_compiler=tmp1;
if(tmp2!=NULL)
{
garbage_bagging(GARBAGE_BOOL_MATCHER, tmp2, garbage_q);

5
src/inc_internal/Maat_rule_internal.h
View File

@@ -147,12 +147,12 @@ struct Maat_compile_inner_t
struct db_compile_rule_t *db_c_rule;
dynamic_array_t *groups; //element is struct Maat_group_inner_t*
char not_flag[MAX_ITEMS_PER_BOOL_EXPR];
char not_group_cnt;
char is_valid;
int compile_id;//equal to db_c_rule->m_rule.config_id
const struct Maat_table_desc* ref_table;
int group_boundary;
int group_cnt;
int not_group_cnt;
MAAT_RULE_EX_DATA* ads;
pthread_rwlock_t rwlock;//reading compile rule is safe in update thread, rwlock lock called when delete or scan thread read
};
@@ -168,6 +168,7 @@ struct _INNER_scan_status_t
size_t hit_group_size;
unsigned long long cur_hit_id[MAX_SCANNER_HIT_NUM];
unsigned long long *hitted_group_id;
char not_grp_compile_hitted_flag;
};
struct _OUTER_scan_status_t
{
@@ -285,7 +286,7 @@ struct Maat_scanner_t
unsigned int exprid_generator;
unsigned int dedup_expr_num;
MESA_lqueue_head region_update_q;
struct bool_matcher * bool_macher_expr_compiler;
struct bool_matcher * bool_matcher_expr_compiler;
scan_result_t *region_rslt_buff;
MESA_lqueue_head tomb_ref;//reference of g_feather->garbage_q

47
test/maat_json.json
View File

@@ -580,7 +580,7 @@
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "NOT-logic-test",
"user_region": "NOT-logic-test1",
"is_valid": "yes",
"groups": [
{
@@ -590,7 +590,7 @@
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "must-contained-string-11111",
"keywords": "must-contained-string-of-rule-143",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
@@ -605,7 +605,48 @@
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "must-not-contained-string-22222",
"keywords": "must-not-contained-string-of-rule-143",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
}
]
},
{
"compile_id": 144,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "NOT-logic-test2",
"is_valid": "yes",
"groups": [
{
"not_flag":0,
"regions": [
{
"table_name": "HTTP_URL",
"table_type": "string",
"table_content": {
"keywords": "must-contained-string-of-rule-144",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"
}
}
]
},
{
"not_flag":1,
"regions": [
{
"table_name": "KEYWORDS_TABLE",
"table_type": "string",
"table_content": {
"keywords": "must-not-contained-string-of-rule-144",
"expr_type": "none",
"match_method": "sub",
"format": "uncase plain"

89
test/test_maatframe.cpp
View File

@@ -104,7 +104,7 @@ const char* old_json="./json_update/old.json";
const char* new_json="./json_update/new.json";
const char* corrupted_json="./json_update/corrupted.json";
class MaatJSONTest : public testing::Test
class JSONUpdate : public testing::Test
{
protected:
@@ -126,21 +126,21 @@ protected:
static Maat_feather_t _shared_feather_j;
static void *logger;
};
Maat_feather_t MaatJSONTest::_shared_feather_j;
Maat_feather_t JSONUpdate::_shared_feather_j;
TEST_F(MaatJSONTest, OldCfg)
TEST_F(JSONUpdate, OldCfg)
{
scan_with_old_or_new_cfg(MaatJSONTest::_shared_feather_j, 1);
scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1);
}
TEST_F(MaatJSONTest, NewCfg)
TEST_F(JSONUpdate, NewCfg)
{
system_cmd_cp(corrupted_json, watched_json);
sleep(2);
scan_with_old_or_new_cfg(MaatJSONTest::_shared_feather_j, 1);
scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1);
system_cmd_cp(new_json, watched_json);
sleep(2);
scan_with_old_or_new_cfg(MaatJSONTest::_shared_feather_j, 0);
scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0);
}
void Maat_read_entry_start_cb(int update_type,void* u_para)
@@ -289,10 +289,10 @@ TEST(IPScan, IPv6)
Maat_clean_status(&mid);
return;
}
TEST(Boolmatcher, NotFlagGroup)
TEST(NOTLogic, OneRegion)
{
const char* string_should_hit="This string ONLY contains must-contained-string-11111.";
const char* string_should_not_hit="This string contains both must-contained-string-11111 and must-not-contained-string-22222.";
const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-143.";
const char* string_should_not_hit="This string contains both must-contained-string-of-rule-143 and must-not-contained-string-of-rule-143.";
int ret=0;
int table_id=0;
@@ -317,6 +317,65 @@ TEST(Boolmatcher, NotFlagGroup)
EXPECT_EQ(ret, -2);
Maat_clean_status(&mid);
}
TEST(NOTLogic, ScanNotAtLast)
{
const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-144.";
const char* string_should_not_hit="This string contains both must-contained-string-of-rule-144 and must-not-contained-string-of-rule-144.";
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* hit_table_name="HTTP_URL", *not_hit_table_name="KEYWORDS_TABLE";
scan_status_t mid=NULL;
table_id=Maat_table_register(g_feather,hit_table_name);
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result,found_pos, 4, &mid, 0);
EXPECT_GE(ret, -2);
table_id=Maat_table_register(g_feather,not_hit_table_name);
ASSERT_GT(table_id, 0);
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, -2);
Maat_clean_status(&mid);
}
TEST(NOTLogic, ScanIrrelavantAtLast)
{
const char* string_should_hit="This string ONLY contains must-contained-string-of-rule-144.";
const char* string_should_not_hit="This string contains both must-contained-string-of-rule-144 and must-not-contained-string-of-rule-144.";
const char* string_irrelevant="This string contiains nothing to hit.";
int ret=0;
int table_id=0;
struct Maat_rule_t result[4];
int found_pos[4];
const char* hit_table_name="HTTP_URL", *not_hit_table_name="KEYWORDS_TABLE";
scan_status_t mid=NULL;
table_id=Maat_table_register(g_feather,hit_table_name);
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_hit, strlen(string_should_hit),
result,found_pos, 4, &mid, 0);
EXPECT_GE(ret, -2);
/*
table_id=Maat_table_register(g_feather,not_hit_table_name);
ASSERT_GT(table_id, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_should_not_hit, strlen(string_should_not_hit),
result,found_pos, 4, &mid, 0);
EXPECT_GE(ret, -2);*/
Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION, NULL, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, string_irrelevant, strlen(string_irrelevant),
result,found_pos, 4, &mid, 0);
EXPECT_EQ(ret, 1);
Maat_clean_status(&mid);
}
@@ -1790,16 +1849,16 @@ int main(int argc, char ** argv)
int scan_detail=0;
::testing::InitGoogleTest(&argc, argv);
g_logger=MESA_create_runtime_log_handle(log_file,0);
g_logger=MESA_create_runtime_log_handle(log_file, 0);
g_feather=Maat_feather(g_iThreadNum, table_info_path, g_logger);
Maat_set_feather_opt(g_feather,MAAT_OPT_INSTANCE_NAME,"demo", strlen("demo")+1);
Maat_set_feather_opt(g_feather,MAAT_OPT_DECRYPT_KEY,decrypt_key, strlen(decrypt_key)+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_INSTANCE_NAME, "demo", strlen("demo")+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_DECRYPT_KEY, decrypt_key, strlen(decrypt_key)+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms));
Maat_set_feather_opt(g_feather, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms));
//Set a short intevral for testing.
Maat_set_feather_opt(g_feather, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms));
Maat_set_feather_opt(g_feather, MAAT_OPT_EFFECT_INVERVAL_MS, &effective_interval_ms, sizeof(effective_interval_ms));
Maat_set_feather_opt(g_feather, MAAT_OPT_STAT_FILE_PATH, stat_file, strlen(stat_file)+1);
Maat_set_feather_opt(g_feather, MAAT_OPT_STAT_ON, NULL, 0);