#include "Maat_rule.h" #include "stream_fuzzy_hash.h" #include "Maat_command.h" #include #include #include #include //inet_addr #include //inet_addr #include //inet_addr #include #include //fstat #include #include #include #include #include //fstat #include //fstat #include #include #include #include #include const char* test_maat_redis_ip="127.0.0.1"; unsigned short test_maat_redis_port=6379; const char* json_path="./maat_json.json"; const char* ful_cfg_dir="./rule/full/index/"; const char* inc_cfg_dir="./rule/inc/index/"; #define WAIT_FOR_EFFECTIVE_US 1*1000*1000 extern int my_scandir(const char *dir, struct dirent ***namelist, int(*filter)(const struct dirent *), int(*compar)(const void *, const void *)); extern char* md5_file(const char* filename, char* md5string); extern int system_cmd_cp(const char* src_file,const char*dst_file); extern int system_cmd_encrypt(const char* src_file, const char* dst_file, const char* password); Maat_feather_t g_feather=NULL; void *g_logger=NULL; int g_iThreadNum=4; const char* table_info_path="./table_info.conf"; int scan_interval_ms=500; int effective_interval_ms=0; void wait_for_cmd_effective(Maat_feather_t feather, long long version_before) { long long version_after=version_before; int is_updating=1; long long wating_us=0, sleep_us=1000*100; while(is_updating||version_before==version_after) { Maat_read_state(feather,MAAT_STATE_IN_UPDATING, &is_updating, sizeof(is_updating)); Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after)); usleep(sleep_us);//waiting for commands go into effect wating_us+=sleep_us; } // printf("wait for %lld ms\n", wating_us/1000); } void ipv4_addr_set(struct ipaddr *ipv4_addr, struct stream_tuple4_v4* v4_addr, const char* src_ip, unsigned short sport, const char* dest_ip, unsigned short dport) { ipv4_addr->addrtype=ADDR_TYPE_IPV4; inet_pton(AF_INET, src_ip, &(v4_addr->saddr)); v4_addr->source=htons(sport); inet_pton(AF_INET, dest_ip, &(v4_addr->daddr)); v4_addr->dest=htons(dport); ipv4_addr->v4=v4_addr; return; } void scan_with_old_or_new_cfg(Maat_feather_t feather, int is_old) { const char* hit_old_data="Hello world! I'm eve."; const char* hit_new_data="Maat was borned in MESA."; const char* table_name="HTTP_URL"; scan_status_t mid=NULL; int table_id=0, ret=0; struct Maat_rule_t result; table_id=Maat_table_register(feather,table_name); ASSERT_GT(table_id, 0); memset(&result, 0, sizeof(result)); ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, hit_old_data, strlen(hit_old_data), &result,NULL, 1, &mid, 0); if(is_old) { EXPECT_EQ(ret, 1); EXPECT_TRUE(result.config_id==1); } else { EXPECT_EQ(ret, 0); } Maat_clean_status(&mid); memset(&result, 0, sizeof(result)); ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, hit_new_data, strlen(hit_new_data), &result,NULL, 1, &mid, 0); if(!is_old) { EXPECT_EQ(ret, 1); EXPECT_EQ(result.config_id, 2); } else { EXPECT_EQ(ret, 0); } Maat_clean_status(&mid); } const char* watched_json="./json_update/maat.json"; const char* old_json="./json_update/old.json"; const char* new_json="./json_update/new.json"; const char* corrupted_json="./json_update/corrupted.json"; const char* json_decrypt_key="himaat!"; class JSONUpdate : public testing::Test { protected: static void SetUpTestCase() { system_cmd_encrypt(old_json, watched_json, json_decrypt_key); _shared_feather_j=Maat_feather(g_iThreadNum, table_info_path, g_logger); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_DECRYPT_KEY, json_decrypt_key, strlen(json_decrypt_key)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_JSON_FILE_PATH, watched_json, strlen(watched_json)+1); Maat_set_feather_opt(_shared_feather_j, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms)); Maat_initiate_feather(_shared_feather_j); } static void TearDownTestCase() { Maat_burn_feather(_shared_feather_j); } // Some expensive resource shared by all tests. static Maat_feather_t _shared_feather_j; static void *logger; }; Maat_feather_t JSONUpdate::_shared_feather_j; TEST_F(JSONUpdate, OldCfg) { scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1); } TEST_F(JSONUpdate, NewCfg) { system_cmd_encrypt(corrupted_json, watched_json, json_decrypt_key); sleep(2); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 1); system_cmd_encrypt(new_json, watched_json, json_decrypt_key); sleep(5); scan_with_old_or_new_cfg(JSONUpdate::_shared_feather_j, 0); } void Maat_read_entry_start_cb(int update_type,void* u_para) { return; } void Maat_read_entry_cb(int table_id,const char* table_line,void* u_para) { char ip_str[16]={0}; int entry_id=-1,seq=-1; unsigned int ip_uint=0; int is_valid=0; unsigned int local_ip_nr=16820416;//192.168.0.1 sscanf(table_line,"%d\t%s\t%d\t%d",&seq,ip_str,&entry_id,&is_valid); inet_pton(AF_INET,ip_str,&ip_uint); if(local_ip_nr==ip_uint) { if(is_valid==1) { //printf("Load entry id %d success.\n",entry_id); EXPECT_EQ(entry_id, 101); } else { //printf("Offload entry id %d success.\n",entry_id); } } return; } void Maat_read_entry_finish_cb(void* u_para) { Maat_feather_t feather=u_para; long long version=0; int ret=0,is_last_updating_table=0; ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version, sizeof(version)); EXPECT_EQ(ret, 0); ret=Maat_read_state(feather,MAAT_STATE_LAST_UPDATING_TABLE, &is_last_updating_table, sizeof(is_last_updating_table)); EXPECT_EQ(ret, 0); //printf("Maat Version %lld at plugin finish callback, is_last_update=%d.\n",version,is_last_updating_table); return; } void test_plugin_table(Maat_feather_t feather,const char* table_name, Maat_start_callback_t *start,Maat_update_callback_t *update,Maat_finish_callback_t *finish, void *u_para, void* logger) { int table_id=0,ret=0; table_id=Maat_table_register(feather,table_name); ASSERT_GT(table_id, 0); ret=Maat_table_callback_register(feather, table_id, start, update, finish, u_para); ASSERT_GT(ret, 0); } #define Plugin_callback TEST(PluginTable, Callback) { test_plugin_table(g_feather, "QD_ENTRY_INFO", Maat_read_entry_start_cb, Maat_read_entry_cb, Maat_read_entry_finish_cb, g_feather, g_logger); } #define IP_PLUGIN_EX_DATA struct ip_plugin_ud { int rule_id; char* buffer; int ref_cnt; }; void ip_plugin_EX_new_cb(int table_id, const char* key, const char* table_line, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) { int *counter=(int *)argp, ret=0; size_t column_offset=0, column_len=0; struct ip_plugin_ud* ud=(struct ip_plugin_ud*)calloc(sizeof(struct ip_plugin_ud), 1); ret=Maat_helper_read_column(table_line, 1, &column_offset, &column_len); EXPECT_EQ(ret, 0); ud->rule_id=atoi(table_line+column_offset); ret=Maat_helper_read_column(table_line, 5, &column_offset, &column_len); EXPECT_EQ(ret, 0); ud->buffer=(char*)calloc(sizeof(char), column_len+1); strncpy(ud->buffer, table_line+column_offset, column_len); ud->ref_cnt=1; *ad=ud; (*counter)++; return; } void ip_plugin_EX_free_cb(int table_id, MAAT_PLUGIN_EX_DATA* ad, long argl, void *argp) { struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*ad); u->ref_cnt--; if(u->ref_cnt>0) return; free(u->buffer); free(u); *ad=NULL; } void ip_plugin_EX_dup_cb(int table_id, MAAT_PLUGIN_EX_DATA *to, MAAT_PLUGIN_EX_DATA *from, long argl, void *argp) { struct ip_plugin_ud* u=(struct ip_plugin_ud*)(*from); u->ref_cnt++; *to=u; } TEST(IP_Plugin_Table, EX_DATA) { int ip_plugin_ex_data_counter=0, i=0; const char* table_name="TEST_IP_PLUGIN_WITH_EXDATA"; int table_id=0, ret=0; table_id=Maat_table_register(g_feather, table_name); ASSERT_GT(table_id, 0); ret=Maat_ip_plugin_EX_register(g_feather, table_id, ip_plugin_EX_new_cb, ip_plugin_EX_free_cb, ip_plugin_EX_dup_cb, 0, &ip_plugin_ex_data_counter); ASSERT_TRUE(ret>=0); EXPECT_EQ(ip_plugin_ex_data_counter, 4); struct ip_address ipv4, ipv6; struct ip_plugin_ud* result[4]; ipv4.ip_type=4; inet_pton(AF_INET, "192.168.30.100", &(ipv4.ipv4)); ret=Maat_ip_plugin_get_EX_data(g_feather, table_id, &ipv4, (void**)result, 4); ASSERT_EQ(ret, 2); EXPECT_EQ(result[0]->rule_id, 101); EXPECT_EQ(result[1]->rule_id, 102); for(i=0; irule_id, 104); EXPECT_EQ(result[1]->rule_id, 103); for(i=0; i0) { pass_flag=1; break; } } EXPECT_EQ(pass_flag, 1); EXPECT_EQ(result[0].config_id, 164); Maat_stream_scan_string_end(&sp); free(hit_detail); fclose(fp); Maat_clean_status(&mid); return; } #define TEST_IPSCAN TEST(IPScan, IPv4_mask) { int table_id=0,ret=0; const char* table_name="IP_CONFIG"; struct Maat_rule_t result[4]; scan_status_t mid=NULL; struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; ipv4_addr_set(&ipv4_addr, &v4_addr, "10.0.6.205", 50001, "10.0.6.201", 80); const char* scan_data="http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; table_id=Maat_table_register(g_feather,"HTTP_URL"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), result,NULL, 4, &mid, 0); EXPECT_GE(ret, 1); table_id=Maat_table_register(g_feather,table_name); EXPECT_GT(table_id, 0); ret=Maat_scan_proto_addr(g_feather,table_id,&ipv4_addr,6,result,4, &mid,0); EXPECT_GT(ret, 0); Maat_clean_status(&mid); return; } TEST(IPScan, IPv6_mask) { int table_id=0,ret=0; struct Maat_rule_t result[4]; struct ipaddr ipv6_addr; struct stream_tuple4_v6 v6_addr; scan_status_t mid=NULL; ipv6_addr.addrtype=ADDR_TYPE_IPV6; inet_pton(AF_INET6,"2001:da8:205:1::101",&(v6_addr.saddr)); v6_addr.source=htons(50001); inet_pton(AF_INET6,"2001:da8:205:1::102",&(v6_addr.daddr)); v6_addr.dest=htons(80); ipv6_addr.v6=&v6_addr; const char* table_name="IP_CONFIG"; table_id=Maat_table_register(g_feather,table_name); EXPECT_GT(table_id, 0); //for improving performance. Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0); ret=Maat_scan_proto_addr(g_feather,table_id,&ipv6_addr,6,result,4, &mid,0); EXPECT_EQ(ret, -2); Maat_clean_status(&mid); return; } TEST(IPScan, IPv4_range) { int table_id=0,ret=0; const char* table_name="IP_PLUS_CONFIG"; struct Maat_rule_t result[4]; scan_status_t mid=NULL; struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; ipv4_addr.addrtype=ADDR_TYPE_IPV4; inet_pton(AF_INET, "10.0.7.106", &(v4_addr.saddr)); v4_addr.source=htons(5000); inet_pton(AF_INET, "123.56.104.254", &(v4_addr.daddr)); v4_addr.dest=htons(7400); ipv4_addr.v4=&v4_addr; table_id=Maat_table_register(g_feather, table_name); EXPECT_GT(table_id, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 154); Maat_clean_status(&mid); return; } TEST(IPScan, IPv6_range) { int table_id=0,ret=0; struct Maat_rule_t result[4]; struct ipaddr ipv6_addr; struct stream_tuple4_v6 v6_addr; scan_status_t mid=NULL; ipv6_addr.addrtype=ADDR_TYPE_IPV6; inet_pton(AF_INET6,"1001:da8:205:1::151",&(v6_addr.saddr)); v6_addr.source=htons(5204);//5200~5299? inet_pton(AF_INET6,"3001:da8:205:1::901",&(v6_addr.daddr)); v6_addr.dest=htons(80);//any ipv6_addr.v6=&v6_addr; const char* table_name="IP_PLUS_CONFIG"; table_id=Maat_table_register(g_feather,table_name); EXPECT_GT(table_id, 0); //for improving performance. Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv6_addr, 6, result,4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 155); Maat_clean_status(&mid); return; } TEST(IPScan, IPv4_CIDR) { int table_id=0,ret=0; const char* table_name="IP_PLUS_CONFIG"; struct Maat_rule_t result[4]; scan_status_t mid=NULL; struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; ipv4_addr.addrtype=ADDR_TYPE_IPV4; inet_pton(AF_INET, "192.168.0.1", &(v4_addr.saddr)); v4_addr.source=htons(5210); inet_pton(AF_INET, "10.0.6.210", &(v4_addr.daddr)); v4_addr.dest=htons(7400); ipv4_addr.v4=&v4_addr; table_id=Maat_table_register(g_feather, table_name); EXPECT_GT(table_id, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 158); Maat_clean_status(&mid); return; } TEST(IPScan, IPv6_CIDR) { int table_id=0,ret=0; struct Maat_rule_t result[4]; struct ipaddr ipv6_addr; struct stream_tuple4_v6 v6_addr; scan_status_t mid=NULL; ipv6_addr.addrtype=ADDR_TYPE_IPV6; inet_pton(AF_INET6,"2001:db8::00fe",&(v6_addr.saddr)); v6_addr.source=htons(5204);//5200~5299? inet_pton(AF_INET6,"2001:4860:4860::8888",&(v6_addr.daddr)); v6_addr.dest=htons(80);//any ipv6_addr.v6=&v6_addr; const char* table_name="IP_PLUS_CONFIG"; table_id=Maat_table_register(g_feather,table_name); EXPECT_GT(table_id, 0); //for improving performance. Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_LAST_REGION,NULL, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv6_addr, 6, result,4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 159); Maat_clean_status(&mid); return; } TEST(IPScan, IPv4_Port) { int table_id=0,ret=0; const char* table_name="IP_PLUS_CONFIG"; struct Maat_rule_t result[4]; scan_status_t mid=NULL; struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; ipv4_addr_set(&ipv4_addr, &v4_addr, "192.168.244.1", 20304, "0.0.0.0", 0); table_id=Maat_table_register(g_feather, table_name); EXPECT_GT(table_id, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 169); Maat_clean_status(&mid); return; } TEST(IPScan, IPv4_virtual) { int table_id=0,ret=0; const char* table_name="VIRTUAL_IP_PLUS_TABLE"; struct Maat_rule_t result[4]; scan_status_t mid=NULL; struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; ipv4_addr_set(&ipv4_addr, &v4_addr, "192.168.40.10", 443, "192.168.231.46", 25705); table_id=Maat_table_register(g_feather, table_name); EXPECT_GT(table_id, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 172); Maat_clean_status(&mid); return; } TEST(IPScan, IPv4_composition) { int table_id=0,ret=0; const char* table_name="COMPOSITION_IP"; struct Maat_rule_t result[4]; scan_status_t mid=NULL; struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; ipv4_addr_set(&ipv4_addr, &v4_addr, "192.168.40.11", 443, "192.168.231.47", 25715); table_id=Maat_table_register(g_feather, table_name); EXPECT_GT(table_id, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0); EXPECT_EQ(ret, 2); EXPECT_EQ(result[0].config_id, 177); EXPECT_EQ(result[1].config_id, 175); struct Maat_hit_path_t hit_path[128]; int n_read=0; n_read=Maat_get_scan_status(g_feather, &mid, MAAT_GET_SCAN_HIT_PATH, hit_path, sizeof(hit_path)); int i=0, c175=0, c177=0; for(i=0; i0) { hit_cnt++; } } fclose(fp); Maat_stream_scan_digest_end(&sp); EXPECT_GE(hit_cnt, 1); Maat_clean_status(&mid); return; } TEST(StringScan, GBKEncodedURL) { const char* url_gb2312="www.baidu.com/?wd=C%23%D6%D0%B9%FA"; int table_id=0,ret=0; struct Maat_rule_t result[4]; int found_pos[4]; const char* table_name="HTTP_URL"; scan_status_t mid=NULL; table_id=Maat_table_register(g_feather,table_name); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_gb2312, strlen(url_gb2312), result,found_pos, 4, &mid, 0); EXPECT_GE(ret, 1); Maat_clean_status(&mid); return; } TEST(StringScan, UTF8EncodedURL) { const char* url_utf8="www.google.com/?q=C%23%E4%B8%AD%E5%9B%BD"; const char* url_utf8_qs="googlevideo.com/qs?nh=%2CIgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE"; const char* url_utf8_long="https://r1---sn-35153iuxa-5a56.googlevideo.com/videoplayback?clen=14143675&sparams=clen%2Cdur%2Cei%2Cgir%2Cid%2Cinitcwndbps%2Cip%2Cipbits%2Citag%2Ckeepalive%2Clmt%2Cmime%2Cmm%2Cmn%2Cms%2Cmv%2Cpl%2Crequiressl%2Csource%2Cexpire&initcwndbps=11060000&ipbits=0&c=WEB&lmt=1545866423716040&source=youtube&dur=995.221&requiressl=yes&txp=5511222&mime=audio%2Fwebm&gir=yes&signature=D1BAE8D85C01B7761609D5143FEC67B33EA35A3E.BDBFB0F6E657EC8D5E9D42015378B885CA87364E&key=yt6&itag=251&expire=1548352912&fvip=1&ei=MKlJXJjUMY6p7QSx2p3gDA&ms=au%2Crdu&mt=1548331263&mv=m&pl=24&ip=178.89.4.220&keepalive=yes&id=o-AMlnrhwQnnM19Tz8lVnzpUT7bbAsrbPPZBl9M1Sx2sw2&mm=31%2C29&mn=sn-35153iuxa-5a56%2Csn-n8v7znz7&alr=yes&cpn=nCaBFGicSajzXMjH&cver=2.20190119&range=266995-528469&rn=17&rbuf=16106"; int table_id=0,ret=0; struct Maat_rule_t result[4]; int found_pos[4]; const char* table_name="HTTP_URL"; scan_status_t mid=NULL; table_id=Maat_table_register(g_feather,table_name); ASSERT_GT(table_id, 0); memset(result, 0, sizeof(result)); ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_utf8, strlen(url_utf8), result,found_pos, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 129); Maat_clean_status(&mid); memset(result, 0, sizeof(result)); ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_utf8_qs, strlen(url_utf8_qs), result,found_pos, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 142); Maat_clean_status(&mid); memset(result, 0, sizeof(result)); ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url_utf8_long, strlen(url_utf8_long), result,found_pos, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 147); Maat_clean_status(&mid); return; } TEST(StringScan, UnicodeEscape) { const char* test_data_dir="./testdata_uni2ascii"; struct dirent **namelist; FILE* fp=NULL; char file_path[256]={0}; char buff[4096]; size_t read_len=0; int table_id=0,ret=0; struct Maat_rule_t result[4]; stream_para_t sp=NULL; int n=0,i=0, hit_cnt=0; const char* table_name="KEYWORDS_TABLE"; scan_status_t mid=NULL; table_id=Maat_table_register(g_feather,table_name); ASSERT_GT(table_id, 0); n = my_scandir(test_data_dir, &namelist, NULL, (int (*)(const void*, const void*))alphasort); ASSERT_GT(n, 0); for(i=0;id_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0)) { continue; } snprintf(file_path,sizeof(file_path),"%s/%s",test_data_dir,namelist[i]->d_name); fp=fopen(file_path,"rb"); if(fp==NULL) { printf("fopen %s error.\n",file_path);; continue; } sp=Maat_stream_scan_string_start(g_feather,table_id,0); ASSERT_FALSE(sp==NULL); read_len=fread(buff,1,sizeof(buff),fp); while(read_len>0) { ret=Maat_stream_scan_string(&sp,CHARSET_NONE,buff,read_len ,result, NULL, 4, &mid); read_len=fread(buff,1,sizeof(buff),fp); if(ret>0) { hit_cnt++; } } Maat_stream_scan_string_end(&sp); fclose(fp); EXPECT_GT(hit_cnt, 0); EXPECT_GE(result[0].config_id, 130);//130, 131 Maat_clean_status(&mid); } for(i=0;i0) { pass_flag=1; break; } } EXPECT_EQ(pass_flag, 1); EXPECT_EQ(result[0].config_id, 136); Maat_stream_scan_string_end(&sp); free(hit_detail); fclose(fp); Maat_clean_status(&mid); return; } TEST(StringScan, OffsetChunk64) { test_offset_str_scan_with_chunk(64); return; } TEST(StringScan, OffsetChunk1460) { test_offset_str_scan_with_chunk(1460); return; } #define StringScan_StreamScanUTF8 1 TEST(StringScan, StreamScanUTF8) { //Bug report: https://mantis.mesalab.cn/view.php?id=712 int table_id=0,ret=0; int read_size=0,pass_flag=0; struct Maat_rule_t result[4]; scan_status_t mid=NULL; const char* table_name="TROJAN_PAYLOAD"; const char* fn="./testdata/jd.com.html"; FILE* fp=fopen(fn,"r"); ASSERT_FALSE(fp==NULL); char scan_data[2048]; table_id=Maat_table_register(g_feather,table_name); ASSERT_GT(table_id, 0); struct Maat_hit_detail_t *hit_detail=(struct Maat_hit_detail_t *)malloc(sizeof(struct Maat_hit_detail_t)*10); stream_para_t sp=Maat_stream_scan_string_start(g_feather,table_id,0); int detail_ret=0; ASSERT_FALSE(sp==NULL); while(0==feof(fp)) { read_size=fread(scan_data,1,sizeof(scan_data),fp); ret=Maat_stream_scan_string_detail(&sp,CHARSET_NONE,scan_data,read_size ,result,4,hit_detail,10 ,&detail_ret,&mid); if(ret>0) { pass_flag=1; break; } } EXPECT_EQ(pass_flag, 1); EXPECT_EQ(result[0].config_id, 157); Maat_stream_scan_string_end(&sp); free(hit_detail); fclose(fp); Maat_clean_status(&mid); return; } void accept_tags_entry_cb(int table_id,const char* table_line,void* u_para) { int* callback_times=(int*)u_para; char status[32]={0}; int entry_id=-1,seq=-1; int is_valid=0; sscanf(table_line,"%d\t%s\t%d\t%d",&seq,status,&entry_id,&is_valid); EXPECT_STREQ(status ,"SUCCESS"); (*callback_times)++; return; } TEST(Policy, PluginRuleTags1) { #define RuleTags_Plugin int ret=0; int table_id=Maat_table_register(g_feather,"TEST_EFFECTIVE_RANGE_TABLE"); ASSERT_GT(table_id, 0); int callback_times=0; ret=Maat_table_callback_register(g_feather, table_id, NULL, accept_tags_entry_cb, NULL, &callback_times); ASSERT_GE(ret, 0); EXPECT_EQ(callback_times, 5); return; } void accept_tags_entry2_cb(int table_id,const char* table_line,void* u_para) { int* callback_times=(int*)u_para; (*callback_times)++; return; } TEST(Policy, PluginRuleTags2) { #define RuleTags_Plugin2 int ret=0; int table_id=Maat_table_register(g_feather,"IR_INTERCEPT_IP"); ASSERT_GT(table_id, 0); int callback_times=0; ret=Maat_table_callback_register(g_feather, table_id, NULL, accept_tags_entry2_cb, NULL, &callback_times); ASSERT_GE(ret, 0); EXPECT_EQ(callback_times, 2); return; } TEST(Policy, CompileRuleTags) { #define RuleTags_Compile int ret=0; int table_id=0; scan_status_t mid=NULL; struct Maat_rule_t result[4]; const char* should_hit="string bbb should hit"; const char* should_not_hit="string aaa should not hit"; const char* table_name="HTTP_URL"; table_id=Maat_table_register(g_feather,table_name); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, should_not_hit, strlen(should_not_hit), result,NULL, 4, &mid, 0); EXPECT_EQ(ret, -2); ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, should_hit, strlen(should_hit), result,NULL, 4, &mid, 0); EXPECT_EQ(ret, 1); Maat_clean_status(&mid); return; } struct rule_ex_param { int ref_cnt; char name[128]; int id; pthread_mutex_t lock; }; void compile_ex_param_new(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) { int *counter=(int*)argp; *ad=NULL; ASSERT_GT(rule->serv_def_len, 4); struct rule_ex_param* param=(struct rule_ex_param*)calloc(sizeof(struct rule_ex_param), 1); param->ref_cnt=1; pthread_mutex_init(&(param->lock), NULL); sscanf(srv_def_large,"%*[^:]:%[^,],%d",param->name,&(param->id)); (*counter)++; *ad=param; return; } void compile_ex_param_free(int idx, const struct Maat_rule_t* rule, const char* srv_def_large, MAAT_RULE_EX_DATA* ad, long argl, void *argp) { if(*ad==NULL) { return; } struct rule_ex_param* param=(struct rule_ex_param*)*ad; pthread_mutex_lock(&(param->lock)); param->ref_cnt--; if(param->ref_cnt>0) { pthread_mutex_unlock(&(param->lock)); return; } free(param); return; } void compile_ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *from, long argl, void *argp) { struct rule_ex_param* from_param=*((struct rule_ex_param**)from); pthread_mutex_lock(&(from_param->lock)); from_param->ref_cnt++; pthread_mutex_unlock(&(from_param->lock)); *((struct rule_ex_param**)to)=from_param; return; } TEST(Policy, CompileEXData) { #define rule_EX_data_index int ret=0; int table_id=0, ex_data_counter=0; scan_status_t mid=NULL; struct Maat_rule_t result[4]; const char* url="i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; const char* table_name="HTTP_URL"; const char* expect_name="I have a name"; table_id=Maat_table_register(g_feather, table_name); ASSERT_GT(table_id, 0); int ex_param_idx=Maat_rule_get_ex_new_index(g_feather, "COMPILE_ALIAS", compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, 0, &ex_data_counter); ASSERT_TRUE(ex_param_idx>=0); EXPECT_EQ(ex_data_counter, 1); ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, url, strlen(url), result,NULL, 4, &mid, 0); EXPECT_EQ(ret, 1); void *ex_data=Maat_rule_get_ex_data(g_feather, result, ex_param_idx); ASSERT_TRUE(ex_data!=NULL); struct rule_ex_param* param=(struct rule_ex_param*)ex_data; EXPECT_EQ(param->id, 7799); EXPECT_EQ(strcmp(param->name, expect_name),0); compile_ex_param_free(0, NULL, NULL, &ex_data, 0, NULL); Maat_clean_status(&mid); return; } TEST(Policy, SubGroup) { #define TestSubGroup int ret=0, table_id=0; const char* scan_string="ceshi6@mailhost.cn"; struct Maat_rule_t result[4]; memset(result, 0, sizeof(result)); scan_status_t mid=NULL; struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; ipv4_addr.addrtype=ADDR_TYPE_IPV4; inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr)); v4_addr.source=htons(50001); inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr)); v4_addr.dest=htons(80); ipv4_addr.v4=&v4_addr; table_id=Maat_table_register(g_feather,"MAIL_ADDR"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_string, strlen(scan_string), result,NULL, 4, &mid, 0); EXPECT_EQ(ret, -2); table_id=Maat_table_register(g_feather, "IP_CONFIG"); ASSERT_GT(table_id, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid,0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 153); Maat_clean_status(&mid); return; } TEST(Policy, EvaluationOrder) { #define EvaluationOrder int ret=0, table_id=0; size_t i=0; const char* url="cavemancircus.com/2019/12/27/pretty-girls-6/"; struct Maat_rule_t result[4]; memset(result, 0, sizeof(result)); scan_status_t mid=NULL; table_id=Maat_table_register(g_feather, "HTTP_URL"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, url, strlen(url), result+i, NULL, 4-i, &mid, 0); EXPECT_EQ(ret, 3); EXPECT_EQ(result[i].config_id, 166); EXPECT_EQ(result[i+1].config_id, 167); EXPECT_EQ(result[i+2].config_id, 168); i+=ret; struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; ipv4_addr.addrtype=ADDR_TYPE_IPV4; inet_pton(AF_INET,"192.168.23.23",&(v4_addr.saddr)); v4_addr.source=htons(50001); inet_pton(AF_INET,"172.0.6.233",&(v4_addr.daddr)); v4_addr.dest=htons(80); ipv4_addr.v4=&v4_addr; table_id=Maat_table_register(g_feather, "IP_PLUS_CONFIG"); ASSERT_GT(table_id, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result+i, 4-i, &mid,0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[i].config_id, 165); i+=ret; ret=Maat_rule_sort_by_evaluation_order(g_feather, result, i); EXPECT_EQ(ret, i); EXPECT_EQ(result[0].config_id, 165); EXPECT_EQ(result[1].config_id, 166); EXPECT_EQ(result[2].config_id, 167); EXPECT_EQ(result[3].config_id, 168); Maat_clean_status(&mid); } TEST(StreamFuzzyHash, Pure) { #define StreamFuzzyHash_Pure const size_t FILE_CHUNK_SIZE=4096; char * file_buff=NULL,*sfh_ordered=NULL,*sfh_unorder=NULL; int read_size=0,ret=0,chunk_num=0,i=0,idx=0; unsigned long long *offset=NULL; unsigned long long file_size=0,tmp=0,hash_length=0; const char* filename="./testdata/digest_test.data"; FILE* fp=fopen(filename,"r"); sfh_instance_t * fhandle = NULL; struct stat file_info; ret=stat(filename, &file_info); ASSERT_TRUE(ret==0); file_size=file_info.st_size; file_buff=(char*)malloc(file_size); ret=fread(file_buff,1,file_size,fp); ASSERT_TRUE((unsigned long long)ret==file_size); chunk_num=file_size/FILE_CHUNK_SIZE; if(file_size%FILE_CHUNK_SIZE==0) { chunk_num=file_size/FILE_CHUNK_SIZE; } else { chunk_num=file_size/FILE_CHUNK_SIZE+1; } offset=(unsigned long long*)malloc(sizeof(unsigned long long)*chunk_num); for(i=0;ifile_size) { read_size=file_size-offset[i]; } else { read_size=FILE_CHUNK_SIZE; } SFH_feed(fhandle,file_buff+offset[i],read_size,offset[i]); } hash_length = SFH_status(fhandle, HASH_LENGTH); sfh_unorder=(char*)malloc(hash_length); SFH_digest(fhandle, sfh_unorder, hash_length); //printf("%s %u %lf %s\n",path,digest_fstat.st_size,file_entropy,digest_result_buff); SFH_release(fhandle); EXPECT_STREQ(sfh_ordered, sfh_unorder); fclose(fp); free(file_buff); free(sfh_ordered); free(sfh_unorder); free(offset); return; } TEST(ScanResult, LongerServiceDefine) { int ret=0; int table_id=0; scan_status_t mid=NULL; struct Maat_rule_t result[4]; const char* scan_data="soq is using table conjunction function.http://www.3300av.com/novel/27122.txt"; const char* table_name="HTTP_URL"; table_id=Maat_table_register(g_feather,table_name); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, 2); EXPECT_EQ(result[1].config_id, 133); EXPECT_GT(result[1].serv_def_len, 128); char* buff=(char*)malloc(sizeof(char)*result[1].serv_def_len); ret=Maat_read_rule(g_feather, result+1, MAAT_RULE_SERV_DEFINE, buff, result[1].serv_def_len); EXPECT_EQ(ret, result[1].serv_def_len); Maat_clean_status(&mid); free(buff); return; } TEST(Hierarchy, VirtualWithPhysical) { #define Hierarchy_VirtualWithPhysical int ret=0, table_id=0; const char* http_content="Batman\\:Take me Home.Superman/:Fine,stay with me."; const char* http_url="https://blog.csdn.net/littlefang/article/details/8213058"; struct Maat_rule_t result[4]; memset(result, 0, sizeof(result)); scan_status_t mid=NULL; table_id=Maat_table_register(g_feather, "HTTP_URL"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_url, strlen(http_url), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, -2); table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_KEYWORDS"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_content, strlen(http_content), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 160); Maat_clean_status(&mid); const char* should_not_hit="2018-10-05 is a keywords of table KEYWORDS_TABLE. Should not hit."; mid=NULL; ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, should_not_hit, strlen(should_not_hit), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, -2); Maat_clean_status(&mid); return; } TEST(Hierarchy, VirtualWithVirtual) { #define Hierarchy_VirtualWithVirtual int ret=0, table_id=0; const char* http_req_hdr_ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; const char* http_resp_hdr_cookie="uid=12345678;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; struct Maat_rule_t result[4]; memset(result, 0, sizeof(result)); scan_status_t mid=NULL; table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER"); ASSERT_GT(table_id, 0); ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "User-Agent", strlen("User-Agent")); ASSERT_EQ(ret, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_req_hdr_ua, strlen(http_req_hdr_ua), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, -2); table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER"); ASSERT_GT(table_id, 0); ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); ASSERT_EQ(ret, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 162); Maat_clean_status(&mid); return; } TEST(Hierarchy, OneGroupInTwoVirtual) { #define Hierarchy_OneGroupInTwoVirtual int ret=0, table_id=0; const char* http_resp_hdr_cookie="sessionid=888888;BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; sugstore=1;"; struct Maat_rule_t result[4]; memset(result, 0, sizeof(result)); scan_status_t mid=NULL; table_id=Maat_table_register(g_feather, "HTTP_REQUEST_HEADER"); ASSERT_GT(table_id, 0); ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); ASSERT_EQ(ret, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, -2); table_id=Maat_table_register(g_feather, "HTTP_RESPONSE_HEADER"); ASSERT_GT(table_id, 0); ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "Cookie", strlen("Cookie")); ASSERT_EQ(ret, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, http_resp_hdr_cookie, strlen(http_resp_hdr_cookie), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 163); Maat_clean_status(&mid); return; } TEST(Hierarchy, TwoVirtualInOneClause) { #define Hierarchy_TwoVirtualInOneClause int ret=0, table_id=0; struct ipaddr ipv4_addr; struct stream_tuple4_v4 v4_addr; const char* src_asn="AS1234", *dst_asn="AS2345"; const char* my_county="Greece.Sparta"; struct Maat_rule_t result[4]; memset(result, 0, sizeof(result)); scan_status_t mid=NULL; //-------------------------------------- // Source ASN & Dest ASN //-------------------------------------- table_id=Maat_table_register(g_feather, "SOURCE_IP_ASN"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, src_asn, strlen(src_asn), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, -2); table_id=Maat_table_register(g_feather, "DESTINATION_IP_ASN"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, dst_asn, strlen(dst_asn), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 178); Maat_clean_status(&mid); //-------------------------------------- // Source IP & Dest ASN //-------------------------------------- ipv4_addr_set(&ipv4_addr, &v4_addr, "192.168.40.88", 8888, "47.92.55.81", 443); table_id=Maat_table_register(g_feather, "IP_CONFIG"); ASSERT_GT(table_id, 0); ret=Maat_scan_proto_addr(g_feather, table_id, &ipv4_addr, 6, result, 4, &mid, 0); EXPECT_EQ(ret, -2); table_id=Maat_table_register(g_feather, "DESTINATION_IP_ASN"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, dst_asn, strlen(dst_asn), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 178); Maat_clean_status(&mid); //-------------------------------------- // Source Geo & Dest ASN //-------------------------------------- table_id=Maat_table_register(g_feather, "SOURCE_IP_GEO"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, my_county, strlen(my_county), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, -2); table_id=Maat_table_register(g_feather, "DESTINATION_IP_ASN"); ASSERT_GT(table_id, 0); ret=Maat_full_scan_string(g_feather, table_id, CHARSET_UTF8, dst_asn, strlen(dst_asn), result, NULL, 4, &mid, 0); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0].config_id, 178); Maat_clean_status(&mid); return; } class MaatFileTest : public testing::Test { protected: static void SetUpTestCase() { const char* rule_folder="./ntcrule/full/index"; logger=MESA_create_runtime_log_handle("test_maat_file.log",0); const char* table_info="./file_test_tableinfo.conf"; _shared_feather_f=Maat_feather(g_iThreadNum, table_info, logger); Maat_set_feather_opt(_shared_feather_f,MAAT_OPT_INSTANCE_NAME,"files", strlen("files")+1); Maat_set_feather_opt(_shared_feather_f, MAAT_OPT_FULL_CFG_DIR, rule_folder, strlen(rule_folder)+1); Maat_set_feather_opt(_shared_feather_f, MAAT_OPT_INC_CFG_DIR, rule_folder, strlen(rule_folder)+1); Maat_set_feather_opt(_shared_feather_f, MAAT_OPT_SCANDIR_INTERVAL_MS,&scan_interval_ms, sizeof(scan_interval_ms)); //Set a short intevral for testing. Maat_set_feather_opt(_shared_feather_f, MAAT_OPT_EFFECT_INVERVAL_MS,&effective_interval_ms, sizeof(effective_interval_ms)); Maat_initiate_feather(_shared_feather_f); } static void TearDownTestCase() { Maat_burn_feather(_shared_feather_f); MESA_destroy_runtime_log_handle(logger); } // Some expensive resource shared by all tests. static Maat_feather_t _shared_feather_f; static void *logger; }; Maat_feather_t MaatFileTest::_shared_feather_f; void* MaatFileTest::logger; TEST_F(MaatFileTest, StreamFiles) { #define StreamScan_StreamFiles Maat_feather_t feather=MaatFileTest::_shared_feather_f; const char* test_data_dir="./test_streamfiles"; struct dirent **namelist; FILE* fp=NULL; char file_path[256]={0}; char *buff; size_t read_len=0; int table_id=0,ret=0; struct Maat_rule_t result[4]; stream_para_t sp=NULL; int n=0,i=0, hit_cnt=0; const char* table_name="NTC_HTTP_REQ_BODY"; scan_status_t mid=NULL; table_id=Maat_table_register(feather,table_name); ASSERT_GT(table_id, 0); n = my_scandir(test_data_dir, &namelist, NULL, (int (*)(const void*, const void*))alphasort); ASSERT_GT(n, 0); sp=Maat_stream_scan_string_start(feather,table_id,0); ASSERT_FALSE(sp==NULL); struct stat file_info; size_t file_size=0; for(i=0;id_name, ".") == 0) || (strcmp(namelist[i]->d_name, "..") == 0)) { continue; } snprintf(file_path,sizeof(file_path),"%s/%s",test_data_dir,namelist[i]->d_name); ret=stat(file_path, &file_info); ASSERT_TRUE(ret==0); file_size=file_info.st_size; buff=(char*)malloc(file_size); fp=fopen(file_path,"rb"); if(fp==NULL) { printf("fopen %s error.\n",file_path); continue; } read_len=fread(buff,1,file_size,fp); ret=Maat_stream_scan_string(&sp,CHARSET_NONE,buff,read_len ,result, NULL, 4, &mid); read_len=fread(buff,1,sizeof(buff),fp); if(ret>0) { hit_cnt++; } fclose(fp); free(buff); buff=NULL; } Maat_clean_status(&mid); Maat_stream_scan_string_end(&sp); EXPECT_GT(hit_cnt, 0); for(i=0;i