zhangshuo1-domain-classification/DomainDeal.cpp

// DomainDeal.cpp : 此文件包含 "main" 函数。程序执行将在此处开始并结束。
//

#include "Common.h"
#include "Data.h"


AssertOperator<> g_asst(cout, AssertOption::thrw_log);

int main(int argc, char *argv[])
{
	
	//判断参数
	if(argc<2) {
		cout <<"input filename as arg" <<endl;
		return -1;
	}

	//读取文件
	ifstream ifs(argv[1]);
	if(!ifs.is_open()) {
		cout <<"cant open file" <<endl;
		return -1;
	}

	cout <<"read metadata start\n";
	//读取元数据
	AcMachine<char, string> mtCdn;
	TrieTree<char, string> mtUrl;
	BinReadFile brf(true, true);
	g_asst(brf.Open(CDN_FILE)
		&& (brf >>mtCdn)
		&& brf.Close(true),
		"cant read cdn data\n");
	g_asst(brf.Open(URL_FILE)
		&& (brf >>mtUrl)
		&& brf.Close(true),
		"cant read url data\n");
	cout <<"read metadata success\n";

	//组织输出结构
	std::map<string, Partition> mapPar;
	ofstream ofs(STATIS_FILE);

	//排除行
	cout <<"deal dns start\n";
	string strHead;
	long long cntLine= 0, cntForm= 0, cntValid= 0;
	constexpr long long limLine = -1;
	constexpr bool bOutHead = false;
	for(int i=0; i!=1; ++i) {
		string str;
		std::getline(ifs, str);
		if(bOutHead)
			strHead <<str <<"\n";
		++ cntLine;
	}
	//按行读取
	for(string strLine; std::getline(ifs, strLine)
		&& (limLine<=0 || cntLine<limLine); ++cntLine)
	{
		//间隔输出中间结果
		if(cntLine%OUTPUT_INTERVAL==0) {
			Statistic(ofs, mapPar, cntValid);
			cout <<"\r" <<cntLine <<" lines" <<flush;
		}
		//解析字段
		istringstream iss(strLine);
		vector<string> vec;
		for(int i=0; i!=e_end; ++i) {
			vec.emplace_back();
			if(!std::getline(iss, vec.back(), '\t')) {
				break;
			}
		}
		if(vec.size()!=e_end)
			continue;
		++ cntForm;
		//处理域名
		if(!FormDomain(vec[e_domain]))
			continue;
		++ cntValid;
		//查找CDN
		string &strOut = vec[e_domain];
		string *res;
		if((res= mtUrl.Judge(vec[e_domain].rbegin(), vec[e_domain].rend())))
			AddMap(mapPar, strHead, *res, strOut);
		else if((res= FastFindCdn(mtCdn, vec[e_domain])))
			AddMap(mapPar, strHead, SPCDN_NAME, strOut);
		else
			AddMap(mapPar, strHead, OTHER_NAME, strOut);
	}
	cout <<"\ndeal dns success\n";
	cout <<"cntLine: " <<cntLine <<"\n"
		<<"cntForm: " <<cntForm <<"\n"
		<<"cntValid: " <<cntValid <<"\n";

	//最终统计
	Statistic(ofs, mapPar, cntValid);
}