gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复expr_plus和expr混合扫描时，expr漏命中的问题。

Browse Source
This commit is contained in:
zhengchao
2021-10-12 09:55:53 +03:00
parent 2eb2a4aca9
commit 51b26e3a54
4 changed files with 66 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/entry/Maat_api.cpp
View File

@@ -1362,7 +1362,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num ,struct Maat_rule_t*result,int rule_num,struct Maat_hit_detail_t *hit_detail,int detail_num
,int* detail_ret,scan_status_t* mid,int thread_num) ,int* detail_ret,scan_status_t* mid,int thread_num)
{ {
int region_ret=0,compile_ret=0; int region_ret=0, compile_ret=0;
int district_id=DISTRICT_ANY;
size_t hit_region_cnt=0; size_t hit_region_cnt=0;
unsigned int sub_type=0; unsigned int sub_type=0;
int virtual_table_id=0; int virtual_table_id=0;
@@ -1458,7 +1459,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1); alignment_int64_array_add(table_rt->hit_cnt, thread_num, 1);
} }
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt, _mid->district_id, if(hit_region_cnt>0&&p_table->table_type==TABLE_TYPE_EXPR_PLUS)
{
district_id=_mid->district_id;
}
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, hit_region_cnt, district_id,
_mid->is_last_scan, virtual_table_id, _mid->scan_cnt); _mid->is_last_scan, virtual_table_id, _mid->scan_cnt);
if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid)) if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid))
@@ -1516,6 +1521,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
,scan_status_t *mid,int thread_num) ,scan_status_t *mid,int thread_num)
{ {
int region_ret=0,compile_ret=0; int region_ret=0,compile_ret=0;
int district_id=DISTRICT_ANY;
struct _OUTER_scan_status_t* _mid=NULL; struct _OUTER_scan_status_t* _mid=NULL;
scan_data_t intval_scan_data; scan_data_t intval_scan_data;
scan_result_t *region_result=NULL; scan_result_t *region_result=NULL;
@@ -1578,7 +1584,12 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
alignment_int64_array_add(table_rt->hit_cnt, thread_num,1); alignment_int64_array_add(table_rt->hit_cnt, thread_num,1);
} }
_mid=grab_mid(mid, _feather, thread_num, 1); _mid=grab_mid(mid, _feather, thread_num, 1);
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret, _mid->district_id, if(region_ret>0&&p_table->table_type==TABLE_TYPE_INTERVAL_PLUS)
{
district_id=_mid->district_id;
}
scan_region_hit_wraper_build_with_rulescan(&region_hit_wraper, region_result, region_ret, district_id,
_mid->is_last_scan, virtual_table_id, _mid->scan_cnt); _mid->is_last_scan, virtual_table_id, _mid->scan_cnt);
if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid)) if(region_hit_wraper.n_wrapped_region>0 || scan_status_should_compile_NOT(_mid))

2
src/entry/Maat_rule.cpp
View File

@@ -57,7 +57,7 @@ extern "C"
} }
#endif #endif
int MAAT_FRAME_VERSION_3_4_10_20211011=1; int MAAT_FRAME_VERSION_3_4_11_20211012=1;
int is_valid_table_name(const char* str) int is_valid_table_name(const char* str)
{ {

74
test/maat_json.json
View File

@@ -8,7 +8,7 @@
"regions": [ "regions": [
{ {
"table_name": "AS_NUMBER", "table_name": "AS_NUMBER",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "AS1234", "keywords": "AS1234",
"expr_type": "none", "expr_type": "none",
@@ -23,7 +23,7 @@
"regions": [ "regions": [
{ {
"table_name": "AS_NUMBER", "table_name": "AS_NUMBER",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "AS2345", "keywords": "AS2345",
"expr_type": "none", "expr_type": "none",
@@ -232,7 +232,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "abckkk&123", "keywords": "abckkk&123",
"expr_type": "and", "expr_type": "and",
@@ -284,7 +284,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "action=search\\&query=(.*)", "keywords": "action=search\\&query=(.*)",
"expr_type": "regex", "expr_type": "regex",
@@ -310,7 +310,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "should_not_hit_any_rule", "keywords": "should_not_hit_any_rule",
"expr_type": "none", "expr_type": "none",
@@ -401,7 +401,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "C#中国", "keywords": "C#中国",
"expr_type": "none", "expr_type": "none",
@@ -427,7 +427,7 @@
"regions": [ "regions": [
{ {
"table_name": "KEYWORDS_TABLE", "table_name": "KEYWORDS_TABLE",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "2010&يىلىدىكى", "keywords": "2010&يىلىدىكى",
"expr_type": "and", "expr_type": "and",
@@ -453,7 +453,7 @@
"regions": [ "regions": [
{ {
"table_name": "KEYWORDS_TABLE", "table_name": "KEYWORDS_TABLE",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "سىياسىي", "keywords": "سىياسىي",
"expr_type": "and", "expr_type": "and",
@@ -479,7 +479,7 @@
"regions": [ "regions": [
{ {
"table_name": "KEYWORDS_TABLE", "table_name": "KEYWORDS_TABLE",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "Take\\bme\\bHome&Batman\\", "keywords": "Take\\bme\\bHome&Batman\\",
"expr_type": "and", "expr_type": "and",
@@ -505,7 +505,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_HOST", "table_name": "HTTP_HOST",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "www.3300av.com", "keywords": "www.3300av.com",
"expr_type": "none", "expr_type": "none",
@@ -531,7 +531,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "novel&27122.txt", "keywords": "novel&27122.txt",
"expr_type": "and", "expr_type": "and",
@@ -581,7 +581,7 @@
"regions": [ "regions": [
{ {
"table_name": "IMAGE_FP", "table_name": "IMAGE_FP",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "4362-4458:323031333A30333A30372032333A35363A313000323031333A30333A30372032333A35363A3130000000FFE20C584943435F50524F46494C4500010100000C484C696E6F021000006D6E74725247422058595A2007CE00020009000600310000", "keywords": "4362-4458:323031333A30333A30372032333A35363A313000323031333A30333A30372032333A35363A3130000000FFE20C584943435F50524F46494C4500010100000C484C696E6F021000006D6E74725247422058595A2007CE00020009000600310000",
"expr_type": "offset", "expr_type": "offset",
@@ -607,7 +607,7 @@
"regions": [ "regions": [
{ {
"table_name": "IMAGE_FP", "table_name": "IMAGE_FP",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "19339-19467:6CB2CB2F2028474C994991CCFC65CCA5E3B6FF001673985D157358610CACC674EE64CC27B5721CCDABD9CCA7C8E9F7BB1F54A930A6034D50F92711F5B2DACCB0715D2E6873CE5CE431DC701A194C260E9DB78CC89F2C84745869AB88349A3AE0412AB59D9ABA84EDEFFF0057FA4DA66D333698B5AD6F844DA2226D1CADAD5E44", "keywords": "19339-19467:6CB2CB2F2028474C994991CCFC65CCA5E3B6FF001673985D157358610CACC674EE64CC27B5721CCDABD9CCA7C8E9F7BB1F54A930A6034D50F92711F5B2DACCB0715D2E6873CE5CE431DC701A194C260E9DB78CC89F2C84745869AB88349A3AE0412AB59D9ABA84EDEFFF0057FA4DA66D333698B5AD6F844DA2226D1CADAD5E44",
"expr_type": "offset", "expr_type": "offset",
@@ -634,7 +634,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "should&hit&aaa", "keywords": "should&hit&aaa",
"expr_type": "and", "expr_type": "and",
@@ -661,7 +661,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "should&hit&bbb", "keywords": "should&hit&bbb",
"expr_type": "and", "expr_type": "and",
@@ -687,7 +687,7 @@
"regions": [ "regions": [
{ {
"table_name": "KEYWORDS_TABLE", "table_name": "KEYWORDS_TABLE",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "2018-10-05", "keywords": "2018-10-05",
"expr_type": "none", "expr_type": "none",
@@ -714,7 +714,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "i.ytimg.com", "keywords": "i.ytimg.com",
"expr_type": "none", "expr_type": "none",
@@ -739,7 +739,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": ",IgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE", "keywords": ",IgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE",
"expr_type": "none", "expr_type": "none",
@@ -765,7 +765,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-contained-string-of-rule-143", "keywords": "must-contained-string-of-rule-143",
"expr_type": "none", "expr_type": "none",
@@ -780,7 +780,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-not-contained-string-of-rule-143", "keywords": "must-not-contained-string-of-rule-143",
"expr_type": "none", "expr_type": "none",
@@ -806,7 +806,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-contained-string-of-rule-144", "keywords": "must-contained-string-of-rule-144",
"expr_type": "none", "expr_type": "none",
@@ -821,7 +821,7 @@
"regions": [ "regions": [
{ {
"table_name": "KEYWORDS_TABLE", "table_name": "KEYWORDS_TABLE",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-not-contained-string-of-rule-144", "keywords": "must-not-contained-string-of-rule-144",
"expr_type": "none", "expr_type": "none",
@@ -847,7 +847,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-contained-string-of-rule-145", "keywords": "must-contained-string-of-rule-145",
"expr_type": "none", "expr_type": "none",
@@ -876,7 +876,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "Cookie:\\s&head", "keywords": "Cookie:\\s&head",
"expr_type": "regex", "expr_type": "regex",
@@ -901,7 +901,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "googlevideo.com/videoplayback&mn=sn-35153iuxa-5a56%2Csn-n8v7znz7", "keywords": "googlevideo.com/videoplayback&mn=sn-35153iuxa-5a56%2Csn-n8v7znz7",
"expr_type": "and", "expr_type": "and",
@@ -954,7 +954,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "^((?!.*\\binstagram\\b)).*\\.fbcdn\\.net$", "keywords": "^((?!.*\\binstagram\\b)).*\\.fbcdn\\.net$",
"expr_type": "regex", "expr_type": "regex",
@@ -1340,7 +1340,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "https://blog.csdn.net/littlefang/article/details/8213058", "keywords": "https://blog.csdn.net/littlefang/article/details/8213058",
"expr_type": "none", "expr_type": "none",
@@ -1467,7 +1467,7 @@
"regions": [ "regions": [
{ {
"table_name": "KEYWORDS_TABLE", "table_name": "KEYWORDS_TABLE",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": ">ЗАО\\b«Севергазвтоматика\\bАйС»<", "keywords": ">ЗАО\\b«Севергазвтоматика\\bАйС»<",
"expr_type": "none", "expr_type": "none",
@@ -1494,7 +1494,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "cavemancircus.com/", "keywords": "cavemancircus.com/",
"expr_type": "none", "expr_type": "none",
@@ -1536,7 +1536,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "2019/12/27/pretty-girls-6", "keywords": "2019/12/27/pretty-girls-6",
"expr_type": "none", "expr_type": "none",
@@ -1563,7 +1563,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "2019/12/27", "keywords": "2019/12/27",
"expr_type": "none", "expr_type": "none",
@@ -1590,7 +1590,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "2019/12/27", "keywords": "2019/12/27",
"expr_type": "none", "expr_type": "none",
@@ -2012,7 +2012,7 @@
"regions": [ "regions": [
{ {
"table_name": "KEYWORDS_TABLE", "table_name": "KEYWORDS_TABLE",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "string1&string2&string3&string4&string5&string6&string7&string8", "keywords": "string1&string2&string3&string4&string5&string6&string7&string8",
"expr_type": "and", "expr_type": "and",
@@ -2038,7 +2038,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "^(?=.*/rain/a/TWF2021042600418000)(?!new.qq.com).*", "keywords": "^(?=.*/rain/a/TWF2021042600418000)(?!new.qq.com).*",
"expr_type": "regex", "expr_type": "regex",
@@ -2121,7 +2121,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-not-contained-string-of-rule-186", "keywords": "must-not-contained-string-of-rule-186",
"expr_type": "none", "expr_type": "none",
@@ -2173,7 +2173,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-not-contained-string-of-rule-187", "keywords": "must-not-contained-string-of-rule-187",
"expr_type": "none", "expr_type": "none",
@@ -2225,7 +2225,7 @@
"regions": [ "regions": [
{ {
"table_name": "HTTP_URL", "table_name": "HTTP_URL",
"table_type": "string", "table_type": "expr",
"table_content": { "table_content": {
"keywords": "must-not-contained-string-of-rule-188", "keywords": "must-not-contained-string-of-rule-188",
"expr_type": "none", "expr_type": "none",

18
test/test_maatframe.cpp
View File

@@ -265,7 +265,6 @@ void ip_plugin_EX_new_cb(int table_id, const char* key, const char* table_line,
void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp)
{ {
struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad); struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad);
u->ref_cnt--;
if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0)) if ((__sync_sub_and_fetch(&u->ref_cnt, 1) == 0))
{ {
free(u->buffer); free(u->buffer);
@@ -672,7 +671,7 @@ TEST(StringScan, ShouldNotHitExprPlus)
ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name, strlen(region_name)); ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name, strlen(region_name));
EXPECT_EQ(ret, 0); EXPECT_EQ(ret, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, (char*)udp_payload_not_hit, sizeof(udp_payload_not_hit), ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, (char*)udp_payload_not_hit, sizeof(udp_payload_not_hit),
result, NULL, 4, result, NULL, 4,
&mid, 0); &mid, 0);
EXPECT_EQ(ret, -2); EXPECT_EQ(ret, -2);
@@ -686,7 +685,8 @@ TEST(StringScan, ExprPlusWithHex)
{ {
int table_id=0,ret=0; int table_id=0,ret=0;
struct Maat_rule_t result[4]; struct Maat_rule_t result[4];
const char* scan_data="text/html; charset=UTF-8"; const char* scan_data1="text/html; charset=UTF-8";
const char* scan_data2="Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char* region_name1="Content-Type"; const char* region_name1="Content-Type";
const char* region_name2="User-Agent"; const char* region_name2="User-Agent";
int found_pos[4]; int found_pos[4];
@@ -695,7 +695,7 @@ TEST(StringScan, ExprPlusWithHex)
scan_status_t mid=NULL; scan_status_t mid=NULL;
ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name1, strlen(region_name1)); ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name1, strlen(region_name1));
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, scan_data1, strlen(scan_data1),
result, found_pos, 4, result, found_pos, 4,
&mid, 0); &mid, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
@@ -703,11 +703,19 @@ TEST(StringScan, ExprPlusWithHex)
ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name2, strlen(region_name2)); ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name2, strlen(region_name2));
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_data1, strlen(scan_data1),
result, found_pos, 4, result, found_pos, 4,
&mid, 0); &mid, 0);
EXPECT_EQ(ret, -2); EXPECT_EQ(ret, -2);
table_id=Maat_table_register(g_feather, "KEYWORDS_TABLE");
ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, scan_data2, strlen(scan_data2),
result, found_pos, 4,
&mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 132);
Maat_clean_status(&mid); Maat_clean_status(&mid);
return; return;