diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index c3be724..0f88fa3 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -33,7 +33,7 @@ #include "stream_fuzzy_hash.h" #include "gram_index_engine.h" -int MAAT_FRAME_VERSION_2_8_20191121=1; +int MAAT_FRAME_VERSION_2_8_20191126=1; int is_valid_table_name(const char* str) { @@ -1445,7 +1445,7 @@ int add_group_to_compile(struct Maat_compile_group_relation*relation, struct Maa } else { - if(p->group_id==a_rule_group->group_id)//duplicate group + if(p->group_id==a_rule_group->group_id && relation->virtual_table_id[i]==virual_table_id)//duplicate group { ret=-1; goto error_out; @@ -2021,7 +2021,7 @@ int add_group_rule(struct Maat_table_desc* table, struct db_group_rule_t* db_gro if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "update error,add group: %s %d to compile rule %d error, compile rule is full or duplicate group.", + "update error, add group: %s %d to compile rule %d error, compile rule is full or duplicate group.", table->table_name[table->updating_name], db_group_rule->group_id, db_group_rule->parent_id); @@ -2232,7 +2232,7 @@ void update_group_rule(struct Maat_table_desc* table,const char* table_line,stru ret=add_group_rule(table,&db_group_rule, scanner, logger); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , + MESA_handle_runtime_log(logger,RLOG_LV_FATAL, maat_module, "duplicate config of group table %s group_id %d compile_id %d.", table->table_name[0], db_group_rule.group_id, db_group_rule.parent_id); @@ -2271,8 +2271,8 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc ,&(maat_str_rule->is_valid)); if(ret!=7) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config: invalid format of expr table %s:%s",table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config: invalid format of expr table %s:%s", table->table_name[table->updating_name], table_line); free(maat_str_rule); maat_str_rule=NULL; table->udpate_err_cnt++; @@ -2290,8 +2290,8 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc ,&(maat_str_rule->is_valid)); if(ret!=8) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config: invalid format of expr_plus table %s:%s",table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config: invalid format of expr_plus table %s:%s", table->table_name[table->updating_name], table_line); free(maat_str_rule); maat_str_rule=NULL; table->udpate_err_cnt++; @@ -2317,30 +2317,30 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc maat_str_rule->is_case_sensitive=TRUE; break; default: - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config %d:update error,invalid hexbin value of expr table %s:%s" - ,maat_str_rule->region_id - ,table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config %d:update error,invalid hexbin value of expr table %s:%s", + maat_str_rule->region_id, + table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; goto error_out; } if(!is_valid_match_method(maat_str_rule->match_method)) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config %d:update error,invalid match method=%d in expr table %s:%s" - ,maat_str_rule->region_id - ,maat_str_rule->match_method - ,table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config %d:update error,invalid match method=%d in expr table %s:%s", + maat_str_rule->region_id, + maat_str_rule->match_method, + table->table_name[table->updating_name],table_line); table->udpate_err_cnt++; goto error_out; } if(!is_valid_expr_type(maat_str_rule->expr_type)) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "abandon config %d:update error,invalid expr type=%d in expr table %s:%s" - ,maat_str_rule->region_id - ,maat_str_rule->expr_type - ,table->table_name[table->updating_name],table_line); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "abandon config %d:update error,invalid expr type=%d in expr table %s:%s", + maat_str_rule->region_id, + maat_str_rule->expr_type, + table->table_name[table->updating_name], table_line); table->udpate_err_cnt++; goto error_out; } @@ -2365,9 +2365,8 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc { rule_type=RULETYPE_STR; } - ret=del_region_rule(table - ,maat_str_rule->region_id,maat_str_rule->group_id,rule_type - ,scanner, logger); + ret=del_region_rule(table, maat_str_rule->region_id, maat_str_rule->group_id, rule_type, + scanner, logger); if(ret<0) { table->udpate_err_cnt++; @@ -2382,18 +2381,18 @@ void update_expr_rule(struct Maat_table_desc* table,const char* table_line,struc if(maat_str_rule->expr_type==EXPR_TYPE_AND &&maat_str_rule->match_method!=MATCH_METHOD_SUB) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "table %s region cfg %d is EXPR_TYPE_AND,but match method is not MATCH_METHOD_SUB,force fixed.", - table->table_name[table->updating_name],maat_str_rule->region_id); + table->table_name[table->updating_name], maat_str_rule->region_id); maat_str_rule->match_method=MATCH_METHOD_SUB; } - ret=add_expr_rule(table, maat_str_rule,scanner, logger); + ret=add_expr_rule(table, maat_str_rule, scanner, logger); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_module , - "duplicate config of expr table %s region_id=%d" - ,table->table_name[table->updating_name],maat_str_rule->region_id); + MESA_handle_runtime_log(logger, RLOG_LV_INFO, maat_module, + "duplicate config of expr table %s region_id=%d", + table->table_name[table->updating_name], maat_str_rule->region_id); table->udpate_err_cnt++; } else diff --git a/test/maat_json.json b/test/maat_json.json index 47da2f5..df4c290 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -196,7 +196,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_REGION", + "table_name": "HTTP_SIGNATURE", "table_type": "expr_plus", "table_content": { "district": "HTTP\\bURL", @@ -1027,7 +1027,7 @@ "group_name": "Untitled", "regions": [ { - "table_name": "HTTP_REGION", + "table_name": "HTTP_SIGNATURE", "table_type": "expr_plus", "table_content": { "district": "Content-Type", @@ -1150,7 +1150,7 @@ "do_blacklist": 0, "do_log": 0, "effective_rage": 0, - "user_region": "Virtual", + "user_region": "VirtualWithPhysical", "is_valid": "yes", "groups": [ { @@ -1174,6 +1174,107 @@ ] } ] + }, + { + "compile_id": 161, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "virtual_table_test_temp", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig1", + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "User-Agent", + "keywords": "Chrome/78.0.3904.108", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "group_name":"vt_grp_http_sig2", + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "Cookie", + "keywords": "uid=12345678", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + }, + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "Cookie", + "keywords": "sessionid=888888", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 162, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "VirtualWithVirtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig1", + "virtual_table":"HTTP_REQUEST_HEADER", + "not_flag":0 + }, + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_RESPONSE_HEADER", + "not_flag":0 + } + ] + }, + { + "compile_id": 163, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "OneGroupInTwoVirtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_REQUEST_HEADER", + "not_flag":0 + }, + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_RESPONSE_HEADER", + "not_flag":0 + } + ] } ], "plugin_table": [ diff --git a/test/table_info.conf b/test/table_info.conf index e960a64..013dfeb 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -27,7 +27,7 @@ 5 CONTENT_SIZE intval -- 6 QD_ENTRY_INFO plugin 4 -- 7 FILE_DIGEST digest -- -8 HTTP_REGION expr_plus GBK GBK yes 0 +8 HTTP_SIGNATURE expr_plus GBK GBK yes 0 9 SIM_URL similar -- 10 IMAGE_FP expr UTF8 UTF8 yes 128 quickoff 11 TEST_EFFECTIVE_RANGE_TABLE plugin {"valid":4,"tag":5} -- @@ -39,4 +39,6 @@ 17 TROJAN_PAYLOAD expr UTF8 UTF8 yes 0 quickoff 18 MAIL_ADDR expr UTF8 UTF8 yes 0 quickoff 19 IP_PLUS_CONFIG ip_plus -- -20 HTTP_RESPONSE_KEYWORDS virtual KEYWORDS_TABLE -- \ No newline at end of file +20 HTTP_RESPONSE_KEYWORDS virtual KEYWORDS_TABLE -- +21 HTTP_REQUEST_HEADER virtual HTTP_SIGNATURE -- +22 HTTP_RESPONSE_HEADER virtual HTTP_SIGNATURE -- \ No newline at end of file diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 0c117aa..5682666 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -357,7 +357,7 @@ TEST(StringScan, ExprPlus) int found_pos[4]; const char* region_name="HTTP URL"; const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567"; - table_id=Maat_table_register(g_feather, "HTTP_REGION"); + table_id=Maat_table_register(g_feather, "HTTP_SIGNATURE"); ASSERT_GT(table_id, 0); scan_status_t mid=NULL; ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_data, strlen(scan_data), @@ -437,7 +437,7 @@ TEST(StringScan, ExprPlusWithHex) const char* scan_data="text/html; charset=UTF-8"; const char* region_name="Content-Type"; int found_pos[4]; - table_id=Maat_table_register(g_feather, "HTTP_REGION"); + table_id=Maat_table_register(g_feather, "HTTP_SIGNATURE"); ASSERT_GT(table_id, 0); scan_status_t mid=NULL; ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name, strlen(region_name)); @@ -1462,9 +1462,9 @@ TEST(ScanResult, LongerServiceDefine) free(buff); return; } -TEST(VirtualTable, Test1) +TEST(VirtualTable, VirtualWithPhysical) { -#define TestVirtualTable +#define TestVirtualTable1 int ret=0, table_id=0; const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me."; const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058"; @@ -1502,6 +1502,85 @@ TEST(VirtualTable, Test1) return; } +TEST(VirtualTable, VirtualWithVirtual) +{ +#define TestVirtualTable2 + int ret=0, table_id=0; + const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; + const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "User-Agent", strlen("User-Agent")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_req_hdr_ua, strlen(http_req_hdr_ua), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 162); + + Maat_clean_status(&mid); + + return; +} +TEST(VirtualTable, OneGroupInTwoVirtual) +{ +#define TestVirtualTable3 + int ret=0, table_id=0; + const char* http_resp_hdr_cookie="sessionid=888888;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; + + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + + table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, -2); + + + table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER"); + ASSERT_GT(table_id, 0); + + ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); + ASSERT_EQ(ret, 0); + + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), + result, NULL, 4, &mid, 0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 163); + + Maat_clean_status(&mid); + + return; +} class MaatFileTest : public testing::Test