gfwleak/web-sketch-webskt-query-agent
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加ICP调用第三方API chinaz 查询并存储结果到数据库，批量查询，文件查询，文件查询导出 CN-664

Browse Source
This commit is contained in:
zhanghongqing
2023-01-29 16:36:18 +08:00
parent bdf48f2b4f
commit 2a54823160
42 changed files with 43588 additions and 1350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/com/mesasoft/cn/interceptor/WebInterceptor.java
View File

@@ -28,10 +28,11 @@ public class WebInterceptor implements HandlerInterceptor {
UserServiceImpl userService;
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws
Exception {
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
String url = request.getServletPath();
InterceptorLevel level = InterceptorLevel.NONE;
if (handler instanceof HandlerMethod) {
AuthInterceptor interceptor = ((HandlerMethod) handler).getMethodAnnotation(AuthInterceptor.class);
//注解到类上面的注解，无法直接获取，只能通过扫描
@@ -58,16 +59,14 @@ public class WebInterceptor implements HandlerInterceptor {
//读取token，自动登录
Cookie cookie = HttpUtils.getCookie(ValueConsts.TOKEN_STRING, request.getCookies());
if (Checker.isNotNull(cookie)) {
user = userService.login(ValueConsts.EMPTY_STRING, ValueConsts.EMPTY_STRING, cookie.getValue(),
response);
user = userService.login(ValueConsts.EMPTY_STRING, ValueConsts.EMPTY_STRING, cookie.getValue(), response);
if (Checker.isNotNull(user)) {
request.getSession().setAttribute(ValueConsts.USER_STRING, user);
}
}
}
if (level != InterceptorLevel.NONE) {
boolean isRedirect = Checker.isNull(user) || (level == InterceptorLevel.ADMIN && user.getPermission() <
2) || (level == InterceptorLevel.SYSTEM && user.getPermission() < 3);
boolean isRedirect = Checker.isNull(user) || (level == InterceptorLevel.ADMIN && user.getPermission() < 2) || (level == InterceptorLevel.SYSTEM && user.getPermission() < 3);
if (isRedirect) {
response.setStatus(401);
response.sendRedirect(DefaultValues.SIGNIN_PAGE);

3
src/main/java/com/mesasoft/cn/service/impl/UserServiceImpl.java
View File

@@ -73,8 +73,7 @@ public class UserServiceImpl implements IUserService {
if (Checker.isNotEmpty(token) && WebSketchApplication.tokens.containsKey(token)) {
user = userDAO.getUserById(WebSketchApplication.tokens.get(token));
if (Checker.isNotNull(response)) {
Cookie cookie = new Cookie(ValueConsts.TOKEN_STRING, TokenConfig.generateToken(token, user.getId
()));
Cookie cookie = new Cookie(ValueConsts.TOKEN_STRING, TokenConfig.generateToken(token, user.getId ()));
cookie.setMaxAge(30 * 24 * 60 * 60);
response.addCookie(cookie);
}

30
src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java
View File

@@ -7,8 +7,8 @@ import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.BrightCloudConfig;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import com.mesasoft.cn.sketch.util.FileUtils;
import com.mesasoft.cn.sketch.util.SpringContextUtil;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.*;
import java.net.HttpURLConnection;
@@ -21,17 +21,15 @@ import java.util.*;
* @version 1.0
* @date 2021/2/22 2:37 下午
*/
public class BrightCloud {
private static final Log logger = Log.get();
private final HashMap<Integer, List<String>> catId2Info = new HashMap<>();
private HttpURLConnection con;
@Autowired
private BrightCloudConfig bcConfig;
private static BrightCloudConfig bcConfig = (BrightCloudConfig) SpringContextUtil.getBean("brightCloudConfig");
public List<DomainCategory> getBrightCloudDomains(List<String> domains) {
public List<DomainCategory> getBrightCloudDomainCategory(List<String> domains) {
JSONObject queryResults = getQueryResults(domains);
return responseSparse(queryResults);
}
@@ -129,6 +127,28 @@ public class BrightCloud {
return domainFiles;
}
// json响应内容解析
public List<DomainCategory> responseSparseTest() {
List<DomainCategory> domainFiles = new ArrayList<>();
for (int i = 0; i < 2; i++) {
// json处理
String fqdn = "www.websketch.com" + i;
domainFiles.add(new DomainCategory(
fqdn,
"brightcloud",
true,
ValidationUtils.getMatchPattern(fqdn),
1,
"",
1,
"1",
"1",
1,
true));
}
return domainFiles;
}
private String getRepLevel(Integer repScore) {
String level = null; //用str存放数据
if (repScore > 80) {

66
src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java
View File

@@ -4,7 +4,9 @@ import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.ChinazConfig;
import com.mesasoft.cn.sketch.entity.domain.DomainICP;
import com.mesasoft.cn.sketch.entity.domain.DomainWhois;
import com.mesasoft.cn.sketch.util.SpringContextUtil;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
@@ -13,6 +15,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.BufferedReader;
import java.io.InputStream;
@@ -30,24 +33,46 @@ import java.util.concurrent.TimeUnit;
* @LastEditTime: 2021-12-29 17:05:45
* @LastEditors: chenxu
*/
public class ChinaZ {
private static final Log logger = Log.get();
@Autowired
private ChinazConfig chinazConfig;
private static final int REQUEST_MAX_BATCH = 50;
private static final String BATCH_SEPARATOR = "|";
private static final String CHINAZ = "chinaz";
private static final int SUFFIX_MATCHING = 1;
private static final int EXACTLY_MATCHING = 2;
public List<DomainWhois> getChinaZDomains(List<String> objectList) {
List<JSONObject> queryResults = getQueryResults(objectList);
private static ChinazConfig chinazConfig = (ChinazConfig) SpringContextUtil.getBean("chinazConfig");
public List<DomainWhois> getDomainsWhois(List<String> domainList) {
List<JSONObject> queryResults = getQueryResults(domainList);
return responseSparse(queryResults);
}
public List<DomainICP> getDomainsICP(List<String> domainList) {
//查询ICP
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("key", chinazConfig.getKey());
List<DomainICP> queryResults = new ArrayList<>();
logger.info("query chainz icp domain size {}", domainList.size());
for (String domain : domainList) {
params.put("domain", domain);
JSONObject r = doPost(chinazConfig.getICPSingle(), params);
DomainICP domainICP = icpInfoResolve(r, domain);
queryResults.add(domainICP);
}
return queryResults;
}
/**
* @Description: 站长之家单查询
* @Param : 域名
* @Return: 查询结果
*/
public JSONObject getQueryResult(String domain) {
String urlString = chinazConfig.getUrlSingle();
String urlString = chinazConfig.getWhoisSingle();
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("key", chinazConfig.getKey());
params.put("domain", domain);
@@ -65,7 +90,7 @@ public class ChinaZ {
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("key", chinazConfig.getKey());
params.put("domain", s);
JSONObject r = doPost(chinazConfig.getUrlSingle(), params);
JSONObject r = doPost(chinazConfig.getWhoisSingle(), params);
whoisInfoList.add(whoisInfoResolve(r, s));
}
return whoisInfoList;
@@ -83,7 +108,7 @@ public class ChinaZ {
String fqdn = record.getString("domain_name");
String domainName = record.getString("domain_host");
Integer matchPattern = fqdn.equals(domainName) ? 1 : 2;
Integer matchPattern = fqdn.equals(domainName) ? EXACTLY_MATCHING : SUFFIX_MATCHING;
String source = "chinaz";
// json处理
@@ -148,6 +173,28 @@ public class ChinaZ {
return whoisInfo;
}
public DomainICP icpInfoResolve(JSONObject jsonRes, String queryDomain) {
JSONObject res = jsonRes.getJSONObject("Result");
DomainICP icp = new DomainICP();
if (jsonRes.get("StateCode").equals(1)) {
icp.setFqdn(queryDomain);
icp.setSource(CHINAZ);
icp.setMatch_pattern(queryDomain.equals(res.getString("MainPage")) ? EXACTLY_MATCHING : SUFFIX_MATCHING);
icp.setQuery_success(jsonRes.getBoolean("StateCode"));
// icp.setIs_accessible();
icp.setIcp_owner(res.getString("Owner"));
icp.setIcp_company_name(res.getString("CompanyName"));
icp.setIcp_company_type(res.getString("CompanyType"));
icp.setIcp_main_page(res.getString("MainPage"));
icp.setIcp_site_license(res.getString("SiteLicense"));
icp.setIcp_site_name(res.getString("SiteName"));
icp.setIcp_verify_time(res.getString("VerifyTime"));
} else {
icp.setQuery_success(jsonRes.getBoolean("StateCode"));
}
return icp;
}
/**
* @Description: 构造批量查询需要的URL
* @Param : 待查询域名
@@ -191,7 +238,7 @@ public class ChinaZ {
*/
public String batchRequest_step1(String domainsString) {
String TaskID = "";
String urlString = chinazConfig.getUrlBatch();
String urlString = chinazConfig.getWhoisBatch();
Map<String, String> params = new LinkedHashMap<String, String>();
if (!Objects.equals(domainsString, "overflow")) {
@@ -212,7 +259,7 @@ public class ChinaZ {
* @Return: 查询结果
*/
public JSONObject batchRequest_step2(String TaskID) {
String urlString = chinazConfig.getUrlBatch();
String urlString = chinazConfig.getWhoisBatch();
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("taskid", TaskID);
@@ -300,7 +347,6 @@ public class ChinaZ {
InputStream inStream = httpResponse.getEntity().getContent();
//对放回数据进行处理
BufferedReader reader = new BufferedReader(new InputStreamReader(inStream, StandardCharsets.UTF_8));
StringBuilder strber = new StringBuilder();
StringBuilder sbf = new StringBuilder();
String strRead = null;
while ((strRead = reader.readLine()) != null) {

5
src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java
View File

@@ -1,8 +1,11 @@
package com.mesasoft.cn.sketch.config;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.resource.ResourceUtil;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@@ -12,8 +15,6 @@ public class AppConfig {
// Mariadb
public static final String DATABASE = ConfigUtils.getStringProperty("database");
public static final String DOMAIN_CATE_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.category");
public static final String DOMAIN_WHOIS_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.whois");
@Value("${db.query.batch.size}")
private Integer dbQueryBatchSize;

19
src/main/java/com/mesasoft/cn/sketch/config/ChinazConfig.java
View File

@@ -14,4 +14,23 @@ public class ChinazConfig {
private String key;
private int maximumQueryNum;
private String usereportFilePath;
public String getWhoisSingle() {
return getUrlSingle() + "/whois";
}
public String getWhoisBatch() {
return getUrlBatch() + "/whois";
}
public String getICPSingle() {
return getUrlSingle() + "/icp";
}
public String getICPBatch() {
return getUrlBatch() + "/icp";
}
}

153
src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java
View File

@@ -1,21 +1,33 @@
package com.mesasoft.cn.sketch.controller;
import cn.hutool.core.date.DateUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONArray;
import com.mesasoft.cn.sketch.entity.ReturnT;
import com.mesasoft.cn.sketch.entity.SketchAuth;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import com.mesasoft.cn.sketch.service.DomainService;
import com.mesasoft.cn.sketch.util.SktUtil;
import com.mesasoft.cn.util.ControllerUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
import java.util.Map;
@RestController
@RequestMapping("/v1/domain/")
public class DomainController {
private Log logger = Log.get();
@Resource
private DomainService domainService;
@GetMapping(value = "info")
@ResponseBody
public ReturnT info(SketchAuth sketchAuth) {
@@ -24,21 +36,148 @@ public class DomainController {
@GetMapping(value = "getcategoryinfo")
@ResponseBody
public ReturnT getCategoryInfo(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
public ReturnT getCategoryInfo1(HttpServletRequest request, SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
JSONArray categoryInfo = domainService.getCategoryInfo(domain, "", true);
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getCategoryInfo(domain, SktUtil.getRequestUser(request));
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), categoryInfo);
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@PostMapping(value = "getcategoryinfo")
@ResponseBody
public ReturnT getCategoryInfo2(HttpServletRequest request, SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getCategoryInfo(domain, SktUtil.getRequestUser(request));
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@GetMapping(value = "getwhoisinfo")
@ResponseBody
public ReturnT getWhoisInfo(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
public ReturnT getWhoisInfo1(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
JSONArray whoisInfo = domainService.getWhoisInfo(domain, true);
return ReturnT.succeed(sketchAuth.getRequestid(), whoisInfo);
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getWhoisInfo(domain);
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@PostMapping(value = "getwhoisinfo")
@ResponseBody
public ReturnT getWhoisInfo2(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getWhoisInfo(domain);
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@GetMapping(value = "geticpinfo")
@ResponseBody
public ReturnT getICPInfo1(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getICPInfo(domain);
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@PostMapping(value = "geticpinfo")
@ResponseBody
public ReturnT getICPInfo2(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getICPInfo(domain);
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
/**
* 文件查询
*
* @param sketchAuth
* @param request
* @return
*/
@PostMapping(value = "/getcategoryinfobyfile")
public void getCategoryInfo(HttpServletResponse response, SketchAuth sketchAuth, MultipartHttpServletRequest request) {
try {
List<DomainInfo> categoryInfoByFile = domainService.getCategoryInfoByFile(request, SktUtil.getRequestUser(request));
ControllerUtils.loadFile(response, "domaincategory-" + DateUtil.date() + ".json", JSONArray.toJSONString(categoryInfoByFile, true));
} catch (Exception e) {
logger.error(e);
}
}
/**
* 文件查询whois
*
* @param sketchAuth
* @param request
* @return curl -o ./domaintest.json --request POST 'http://IP:8196/v1/domain/getcategoryinfobyfile' --form 'file=@"/home/test/domain.txt"' --form 'omeid=1'
*/
@PostMapping(value = "/getwhoisinfobyfile")
public void getWhoisInfo(HttpServletResponse response, SketchAuth sketchAuth, MultipartHttpServletRequest request) throws Exception {
try {
List<DomainInfo> whoisInfoByFile = domainService.getWhoisInfoByFile(request);
ControllerUtils.loadFile(response, "domainwhois-" + DateUtil.date() + ".json", JSONArray.toJSONString(whoisInfoByFile, true));
} catch (Exception e) {
logger.error(e);
}
}
/**
* 文件查询whois
*
* @param sketchAuth
* @param request
* @return curl -o ./domaintest.json --request POST 'http://IP:8196/v1/domain/getcategoryinfobyfile' --form 'file=@"/home/test/domain.txt"' --form 'omeid=1'
*/
@PostMapping(value = "/geticpinfobyfile")
public void getICPInfo(HttpServletResponse response, SketchAuth sketchAuth, MultipartHttpServletRequest request) throws Exception {
try {
List<DomainInfo> domainInfo = domainService.getICPInfoByFile(request);
ControllerUtils.loadFile(response, "domainicp-" + DateUtil.date() + ".json", JSONArray.toJSONString(domainInfo, true));
} catch (Exception e) {
logger.error(e);
}
}
/**
* 本地文件上传
* 查询大量数据
*/
@GetMapping(value = "/getdomaininfobyfile")
public ReturnT getDomainInfoByFile(HttpServletRequest request, String srcFile, String queryType, SketchAuth sketchAuth) {
try {
Map domainInfoByFile = domainService.getDomainInfoByFile(SktUtil.getRequestUser(request), srcFile, queryType);
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfoByFile);
} catch (Exception e) {
logger.error(e);
return ReturnT.failed(sketchAuth.getRequestid(), e.getMessage());
}
}
}

49
src/main/java/com/mesasoft/cn/sketch/controller/GlobalException.java Normal file
View File

@@ -0,0 +1,49 @@
package com.mesasoft.cn.sketch.controller;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.mesasoft.cn.entity.Result;
import com.mesasoft.cn.entity.ResultEntity;
import com.mesasoft.cn.enums.StatusEnum;
import com.mesasoft.cn.exception.BusinessException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.context.request.async.AsyncRequestTimeoutException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@RestControllerAdvice
public class GlobalException {
private static final Log log = LogFactory.get();
@ExceptionHandler(AsyncRequestTimeoutException.class) //捕获特定异常
public void handleAsyncRequestTimeoutException(AsyncRequestTimeoutException e, HttpServletRequest request) {
log.info("Handle Async Request Timeout Exception");
}
@ExceptionHandler(Exception.class)
public ResultEntity handleException(Exception e, HttpServletRequest request, HttpServletResponse response) {
response.setStatus(StatusEnum.FAIL.getStatus());
String message = e.getMessage() + (e.getCause() != null ? e.getCause().getMessage() : "");
log.error("message:{}, stackTrace:{}", message, getStackTrace(e));
return Result.fail(e.getMessage());
}
@ExceptionHandler({BusinessException.class})
public ResultEntity handleBusinessException(BusinessException e, HttpServletRequest request, HttpServletResponse response) {
response.setStatus(e.getStatus());
String message = (e.getMessage() != null ? e.getMessage() : e.getMessage()) + " " + (e.getCause() != null ? e.getCause().getMessage() : "");
log.error("message:{}.stackTrace:{}", message, getStackTrace(e));
return Result.fail(e.getStatus(), e.getCode(), message);
}
private String getStackTrace(Exception e) {
return ObjectUtil.isNotNull(e.getStackTrace()) ? e.getStackTrace()[0].toString() : "";
}
}

54
src/main/java/com/mesasoft/cn/sketch/controller/SketchDomainController.java
View File

@@ -1,54 +0,0 @@
package com.mesasoft.cn.sketch.controller;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.service.IFileManagerService;
import io.swagger.annotations.Api;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
/**
* @description:
* @author: zhq
* @create: 2022-03-10
**/
@RestController
@RequestMapping("/sketch/domain")
@Api(value = "/sketch/domain", description = "文件相关操作")
@Slf4j
public class SketchDomainController {
private final IFileManagerService fileManagerService;
private final HttpServletRequest request;
private JSONObject jsonObject;
@Value("${sketch.path.admin}")
private String pathAdmin;
@Value("${sketch.path.user}")
private String pathUser;
@Autowired
public SketchDomainController(HttpServletRequest request, IFileManagerService fileManagerService, JSONObject jsonObject) {
this.fileManagerService = fileManagerService;
this.jsonObject = jsonObject;
this.request = request;
}
@AuthInterceptor(InterceptorLevel.USER)
@RequestMapping(value = "/list", method = RequestMethod.GET)
public String list(String searchPath) {
log.info("search path :", searchPath);
return jsonObject.toJSONString();
}
}

58
src/main/java/com/mesasoft/cn/sketch/controller/SketchFileController.java
View File

@@ -2,9 +2,10 @@ package com.mesasoft.cn.sketch.controller;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.service.IFileManagerService;
import com.mesasoft.cn.sketch.util.FileUtils;
import com.mesasoft.cn.sketch.util.SktUtil;
import com.mesasoft.cn.util.ControllerUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.ArrayUtils;
@@ -21,6 +22,7 @@ import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.util.Map;
@@ -39,10 +41,8 @@ public class SketchFileController {
private final HttpServletRequest request;
private JSONObject jsonObject;
@Value("${sketch.path.admin}")
private String pathAdmin;
@Value("${sketch.path.user}")
private String pathUser;
@Value("${sketch.home.path}")
private String sketchHomePath;
@Autowired
public SketchFileController(HttpServletRequest request, IFileManagerService fileManagerService, JSONObject jsonObject) {
@@ -55,39 +55,49 @@ public class SketchFileController {
@RequestMapping(value = "/list", method = RequestMethod.GET)
public String list(String searchPath) {
log.info("search path :", searchPath);
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
log.info("search path : {}", searchPath);
JSONObject json = new JSONObject();
String rootPath = "";
if ("system".equals(user.getUsername())) {
rootPath = pathAdmin;
} else {
rootPath = pathUser;
}
searchPath = searchPath.replace("\\\\", "");
searchPath = searchPath.replace("//", "");
if (StringUtils.isNotBlank(searchPath) && !rootPath.contains(searchPath)) {
json.fluentPut("path", searchPath);
} else {
json.fluentPut("path", rootPath);
}
json.fluentPut("path", getAbsolutePath(searchPath));
log.info("current path : {}", json.getString("path"));
//返回结果
jsonObject.put("path", json.getString("path"));
jsonObject.put("path", searchPath);
jsonObject.put("result", fileManagerService.list(json));
return jsonObject.toJSONString();
}
@RequestMapping(value = "/upload", method = RequestMethod.POST)
public String upload(String destination, MultipartHttpServletRequest request) {
String absolutePath = getAbsolutePath(destination);
log.info("file upload {}", absolutePath);
Map<String, MultipartFile> fileMap = request.getFileMap();
MultipartFile[] files = ArrayUtils.mapToArray(fileMap, MultipartFile.class);
jsonObject.put("result", fileManagerService.upload(destination, files));
jsonObject.put("result", fileManagerService.upload(absolutePath, files));
return jsonObject.toJSONString();
}
@RequestMapping(value = "/download", method = RequestMethod.GET)
public void download(HttpServletRequest request, HttpServletResponse response, String path) throws IOException, ClassNotFoundException {
ControllerUtils.loadResource2(response, path, ValueConsts.TRUE);
log.info("search path : {}", path);
String absolutePath = getAbsolutePath(path);
log.info("down load absolute file path : {} ", absolutePath);
ControllerUtils.loadResource2(response, absolutePath, ValueConsts.TRUE);
}
private String getAbsolutePath(String searchPath) {
String requestUser = SktUtil.getRequestUser(request);
String absolutePath = StringUtils.join(sketchHomePath, File.separator, requestUser, File.separator);
if ("system".equals(requestUser) || "default".equals(requestUser)) {
absolutePath = StringUtils.join(sketchHomePath, File.separator);
}
searchPath = FileUtils.formatPath(searchPath);
if (StringUtils.isNotBlank(searchPath) && !absolutePath.contains(searchPath)) {
absolutePath = StringUtils.join(absolutePath, searchPath, File.separator);
}
log.info(" absolute path : {}", absolutePath);
return absolutePath;
}
}

340
src/main/java/com/mesasoft/cn/sketch/dao/DomainCategorySchema.java
View File

@@ -1,340 +0,0 @@
package com.mesasoft.cn.sketch.dao;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
@Deprecated
public class DomainCategorySchema {
private static final String dataBase = AppConfig.DATABASE;
private static final String tableName = AppConfig.DOMAIN_CATE_TABLENAME;
private String fqdn;
private String source;
private Boolean query_success;
private Integer match_pattern;
private Integer reputation_score;
private String reputation_level;
private Integer category_id;
private String category_name;
private String category_group;
private Integer category_conf;
private Boolean is_a1_cat;
private Integer status_code = 0;
private String submit_user ="";
// category schema
public DomainCategorySchema(String fqdn,
String source,
Boolean query_success,
Integer match_pattern,
Integer reputation_score,
String reputationLevel,
Integer categoryId,
String categoryName,
String categoryGroup,
Integer categoryConf,
Boolean isA1Cat,Integer statusCode, String submitUser) {
this.fqdn = fqdn;
this.source = source; // 默认应为为brightcloud
this.query_success = query_success;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.reputation_score = reputation_score;
this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel);
this.category_id = categoryId;
this.category_name = ConfigUtils.getEffectiveString(categoryName);
this.category_group = ConfigUtils.getEffectiveString(categoryGroup);
this.category_conf = categoryConf;
this.is_a1_cat = isA1Cat;
this.status_code = statusCode;
this.submit_user = submitUser;
}
public DomainCategorySchema(String fqdn,
String source,
Boolean query_success,
Integer match_pattern,
Integer reputation_score,
String reputationLevel,
Integer categoryId,
String categoryName,
String categoryGroup,
Integer categoryConf,
Boolean isA1Cat) {
this.fqdn = fqdn;
this.source = source; // 默认应为为brightcloud
this.query_success = query_success;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.reputation_score = reputation_score;
this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel);
this.category_id = categoryId;
this.category_name = ConfigUtils.getEffectiveString(categoryName);
this.category_group = ConfigUtils.getEffectiveString(categoryGroup);
this.category_conf = categoryConf;
this.is_a1_cat = isA1Cat;
}
public static void insertRecords(List<DomainCategorySchema> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainCategorySchema categoryFile : categoryFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + categoryFile.getKeys() + ") values" +
'(' + categoryFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public void updateRecords(List<DomainCategorySchema> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainCategorySchema categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainCategorySchema> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") and source = '" + source + "'";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}
public static List<DomainCategorySchema> rs2schema(ResultSet rs) throws SQLException {
List<DomainCategorySchema> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainCategorySchema(
rs.getString("fqdn"),
rs.getString("source"),
rs.getBoolean("query_success"),
rs.getInt("match_pattern"),
rs.getInt("reputation_score"),
rs.getString("reputation_level"),
rs.getInt("category_id"),
rs.getString("category_name"),
rs.getString("category_group"),
rs.getInt("category_conf"),
rs.getBoolean("is_a1_cat"),
rs.getInt("status_code"),
rs.getString("submit_user")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainCategorySchema schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("reputation_score", schema.getReputation_score());
jsonObject.put("reputation_level", schema.getReputation_level());
jsonObject.put("category_id", schema.getCategory_id());
jsonObject.put("category_group", schema.getCategory_group());
jsonObject.put("category_name", schema.getCategory_name());
jsonObject.put("category_conf", schema.getCategory_conf());
jsonObject.put("is_a1_cat", schema.getIs_a1_cat());
jsonObject.put("status_code", schema.getStatus_code());
jsonObject.put("submit_user", schema.getSubmit_user());
return jsonObject;
}
public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", " + reputation_score +
", '" + reputation_level + '\'' +
", " + category_id +
", '" + category_name + '\'' +
", '" + category_group + '\'' +
", " + category_conf +
", " + status_code +
", " + submit_user +
", " + is_a1_cat;
return resString.replace("'null'", "null");
}
public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", reputation_score" +
", reputation_level" +
", category_id" +
", category_name" +
", category_group" +
", category_conf" +
", status_code" +
", submit_user" +
", is_a1_cat";
return resString;
}
public String getKeyValues() {
String resString = "source='" + source + '\'' +
", query_success=" + query_success +
", match_pattern=" + match_pattern +
", reputation_score=" + reputation_score +
", reputation_level='" + reputation_level + '\'' +
", category_id=" + category_id +
", category_name='" + category_name + '\'' +
", category_group='" + category_group + '\'' +
", category_conf=" + category_conf +
", is_a1_cat=" + is_a1_cat;
return resString.replace("'null'", "null");
}
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public Boolean getQuery_success() {
return query_success;
}
public void setQuery_success(Boolean query_success) {
this.query_success = query_success;
}
public Integer getMatch_pattern() {
return match_pattern;
}
public void setMatch_pattern(Integer match_pattern) {
this.match_pattern = match_pattern;
}
public Integer getReputation_score() {
return reputation_score;
}
public void setReputation_score(Integer reputation_score) {
this.reputation_score = reputation_score;
}
public String getReputation_level() {
return reputation_level;
}
public void setReputation_level(String reputation_level) {
this.reputation_level = reputation_level;
}
public Integer getCategory_id() {
return category_id;
}
public void setCategory_id(Integer category_id) {
this.category_id = category_id;
}
public String getCategory_name() {
return category_name;
}
public void setCategory_name(String category_name) {
this.category_name = category_name;
}
public String getCategory_group() {
return category_group;
}
public void setCategory_group(String category_group) {
this.category_group = category_group;
}
public Integer getCategory_conf() {
return category_conf;
}
public void setCategory_conf(Integer category_conf) {
this.category_conf = category_conf;
}
public Boolean getIs_a1_cat() {
return is_a1_cat;
}
public void setIs_a1_cat(Boolean is_a1_cat) {
this.is_a1_cat = is_a1_cat;
}
public Integer getStatus_code() {
return status_code;
}
public void setStatus_code(Integer status_code) {
this.status_code = status_code;
}
public String getSubmit_user() {
return submit_user;
}
public void setSubmit_user(String submit_user) {
this.submit_user = submit_user;
}
}

5
src/main/java/com/mesasoft/cn/sketch/dao/DomainDao.java
View File

@@ -1,6 +1,7 @@
package com.mesasoft.cn.sketch.dao;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import com.mesasoft.cn.sketch.entity.domain.DomainICP;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import com.mesasoft.cn.sketch.entity.domain.DomainWhois;
import org.apache.ibatis.annotations.Param;
@@ -16,7 +17,11 @@ public interface DomainDao {
List<DomainInfo> getWhoisInfo(@Param("domain") List<String> domain);
List<DomainInfo> getICPInfo(@Param("domain") List<String> domain);
void insertCategory(DomainCategory categories);
void insertWhois(DomainWhois data);
void insertICP(DomainICP data);
}

423
src/main/java/com/mesasoft/cn/sketch/dao/DomainWhoisSchema.java
View File

@@ -1,423 +0,0 @@
package com.mesasoft.cn.sketch.dao;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
@Deprecated
public class DomainWhoisSchema {
private static final String dataBase = AppConfig.DATABASE;
private static final String tableName = AppConfig.DOMAIN_WHOIS_TABLENAME;
private String fqdn;
private String source;
private Boolean query_success;
private Integer match_pattern;
private String whois_domain;
private Timestamp whois_update_date;
private Timestamp whois_create_date;
private Timestamp whois_expire_date;
private String whois_email;
private String whois_ns;
private String whois_registrar_name;
private String whois_registrant_org;
private String whois_registrant_name;
private String whois_registrant_street;
private String whois_registrant_city;
private String whois_registrant_state;
private String whois_registrant_postcode;
private String whois_registrant_country;
private String whois_registrant_phone;
// category schema
public DomainWhoisSchema(String fqdn,
String source,
Integer match_pattern,
Boolean query_success,
String whoisDomain,
Date whoisUpdateDate,
Date whoisCreateDate,
Date whoisExpireDate,
String whoisEmail,
String whoisNs,
String whoisRegistrarName,
String whoisRegistrantOrg,
String whoisRegistrantName,
String whoisRegistrantStreet,
String whoisRegistrantCity,
String whoisRegistrantState,
String whoisRegistrantPostcode,
String whoisRegistrantCountry,
String whoisRegistrantPhone
) {
this.fqdn = fqdn;
this.source = source;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.query_success = query_success;
this.whois_domain = ConfigUtils.getEffectiveString(whoisDomain);
this.whois_update_date = whoisUpdateDate == null ? null : new Timestamp(whoisUpdateDate.getTime());
this.whois_create_date = whoisCreateDate == null ? null : new Timestamp(whoisCreateDate.getTime());
this.whois_expire_date = whoisExpireDate == null ? null : new Timestamp(whoisExpireDate.getTime());
this.whois_email = ConfigUtils.getEffectiveString(whoisEmail);
this.whois_ns = ConfigUtils.getEffectiveString(whoisNs);
this.whois_registrar_name = ConfigUtils.getEffectiveString(whoisRegistrarName);
this.whois_registrant_org = ConfigUtils.getEffectiveString(whoisRegistrantOrg);
this.whois_registrant_name = ConfigUtils.getEffectiveString(whoisRegistrantName);
this.whois_registrant_street = ConfigUtils.getEffectiveString(whoisRegistrantStreet);
this.whois_registrant_city = ConfigUtils.getEffectiveString(whoisRegistrantCity);
this.whois_registrant_state = ConfigUtils.getEffectiveString(whoisRegistrantState);
this.whois_registrant_postcode = ConfigUtils.getEffectiveString(whoisRegistrantPostcode);
this.whois_registrant_country = ConfigUtils.getEffectiveString(whoisRegistrantCountry);
this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone);
}
public static void insertRecords(List<DomainWhoisSchema> whoisFiles, MariaDbBase mariaDbBase) {
for (DomainWhoisSchema whoisFile : whoisFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static String insertSql(List<DomainWhoisSchema> whoisFiles) {
DomainWhoisSchema whoisFile = whoisFiles.get(0);
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
return resSql;
}
public static void updateRecords(List<DomainWhoisSchema> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainWhoisSchema categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainWhoisSchema> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") ";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}
public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", '" + whois_domain + '\'' +
", '" + whois_update_date + '\'' +
", '" + whois_create_date + '\'' +
", '" + whois_expire_date + '\'' +
", '" + whois_email + '\'' +
", '" + whois_ns + '\'' +
", '" + whois_registrar_name + '\'' +
", '" + whois_registrant_org + '\'' +
", '" + whois_registrant_name + '\'' +
", '" + whois_registrant_street + '\'' +
", '" + whois_registrant_city + '\'' +
", '" + whois_registrant_state + '\'' +
", '" + whois_registrant_postcode + '\'' +
", '" + whois_registrant_country + '\'' +
", '" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public static List<DomainWhoisSchema> rs2schema(ResultSet rs) throws SQLException {
List<DomainWhoisSchema> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainWhoisSchema(
rs.getString("fqdn"),
rs.getString("source"),
rs.getInt("match_pattern"),
rs.getBoolean("query_success"),
rs.getString("whois_domain"),
(Date) rs.getDate("whois_update_date"),
(Date) rs.getDate("whois_create_date"),
(Date) rs.getDate("whois_expire_date"),
rs.getString("whois_email"),
rs.getString("whois_ns"),
rs.getString("whois_registrar_name"),
rs.getString("whois_registrant_org"),
rs.getString("whois_registrant_name"),
rs.getString("whois_registrant_street"),
rs.getString("whois_registrant_city"),
rs.getString("whois_registrant_state"),
rs.getString("whois_registrant_postcode"),
rs.getString("whois_registrant_country"),
rs.getString("whois_registrant_phone")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainWhoisSchema schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("whois_domain", schema.getWhois_domain());
jsonObject.put("whois_update_date", schema.getWhois_update_date());
jsonObject.put("whois_create_date", schema.getWhois_create_date());
jsonObject.put("whois_expire_date", schema.getWhois_expire_date());
jsonObject.put("whois_email", schema.getWhois_email());
jsonObject.put("whois_ns", schema.getWhois_ns());
jsonObject.put("whois_registrar_name", schema.getWhois_registrar_name());
jsonObject.put("whois_registrant_org", schema.getWhois_registrant_org());
jsonObject.put("whois_registrant_name", schema.getWhois_registrant_name());
jsonObject.put("whois_registrant_street", schema.getWhois_registrant_street());
jsonObject.put("whois_registrant_city", schema.getWhois_registrant_city());
jsonObject.put("whois_registrant_state", schema.getWhois_registrant_state());
jsonObject.put("whois_registrant_postcode", schema.getWhois_registrant_postcode());
jsonObject.put("whois_registrant_country", schema.getWhois_registrant_country());
jsonObject.put("whois_registrant_phone", schema.getWhois_registrant_phone());
return jsonObject;
}
public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", whois_domain" +
", whois_update_date" +
", whois_create_date" +
", whois_expire_date" +
", whois_email" +
", whois_ns" +
", whois_registrar_name" +
", whois_registrant_org" +
", whois_registrant_name" +
", whois_registrant_street" +
", whois_registrant_city" +
", whois_registrant_state" +
", whois_registrant_postcode" +
", whois_registrant_country" +
", whois_registrant_phone";
return resString;
}
public String getKeyValues() {
String resString = "query_success=" + query_success +
", source='" + source + '\'' +
", match_pattern=" + match_pattern +
", whois_domain='" + whois_domain + '\'' +
", whois_update_date='" + whois_update_date + '\'' +
", whois_create_date='" + whois_create_date + '\'' +
", whois_expire_date='" + whois_expire_date + '\'' +
", whois_email='" + whois_email + '\'' +
", whois_ns='" + whois_ns + '\'' +
", whois_registrar_name='" + whois_registrar_name + '\'' +
", whois_registrant_org='" + whois_registrant_org + '\'' +
", whois_registrant_name='" + whois_registrant_name + '\'' +
", whois_registrant_street='" + whois_registrant_street + '\'' +
", whois_registrant_city='" + whois_registrant_city + '\'' +
", whois_registrant_state='" + whois_registrant_state + '\'' +
", whois_registrant_postcode='" + whois_registrant_postcode + '\'' +
", whois_registrant_country='" + whois_registrant_country + '\'' +
", whois_registrant_phone='" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
public Boolean getQuery_success() {
return query_success;
}
public void setQuery_success(Boolean query_success) {
this.query_success = query_success;
}
public Integer getMatch_pattern() {
return match_pattern;
}
public void setMatch_pattern(Integer match_pattern) {
this.match_pattern = match_pattern;
}
public String getWhois_domain() {
return whois_domain;
}
public void setWhois_domain(String whois_domain) {
this.whois_domain = whois_domain;
}
public Timestamp getWhois_update_date() {
return whois_update_date;
}
public void setWhois_update_date(Timestamp whois_update_date) {
this.whois_update_date = whois_update_date;
}
public Timestamp getWhois_create_date() {
return whois_create_date;
}
public void setWhois_create_date(Timestamp whois_create_date) {
this.whois_create_date = whois_create_date;
}
public Timestamp getWhois_expire_date() {
return whois_expire_date;
}
public void setWhois_expire_date(Timestamp whois_expire_date) {
this.whois_expire_date = whois_expire_date;
}
public String getWhois_email() {
return whois_email;
}
public void setWhois_email(String whois_email) {
this.whois_email = whois_email;
}
public String getWhois_ns() {
return whois_ns;
}
public void setWhois_ns(String whois_ns) {
this.whois_ns = whois_ns;
}
public String getWhois_registrar_name() {
return whois_registrar_name;
}
public void setWhois_registrar_name(String whois_registrar_name) {
this.whois_registrar_name = whois_registrar_name;
}
public String getWhois_registrant_org() {
return whois_registrant_org;
}
public void setWhois_registrant_org(String whois_registrant_org) {
this.whois_registrant_org = whois_registrant_org;
}
public String getWhois_registrant_name() {
return whois_registrant_name;
}
public void setWhois_registrant_name(String whois_registrant_name) {
this.whois_registrant_name = whois_registrant_name;
}
public String getWhois_registrant_street() {
return whois_registrant_street;
}
public void setWhois_registrant_street(String whois_registrant_street) {
this.whois_registrant_street = whois_registrant_street;
}
public String getWhois_registrant_city() {
return whois_registrant_city;
}
public void setWhois_registrant_city(String whois_registrant_city) {
this.whois_registrant_city = whois_registrant_city;
}
public String getWhois_registrant_state() {
return whois_registrant_state;
}
public void setWhois_registrant_state(String whois_registrant_state) {
this.whois_registrant_state = whois_registrant_state;
}
public String getWhois_registrant_postcode() {
return whois_registrant_postcode;
}
public void setWhois_registrant_postcode(String whois_registrant_postcode) {
this.whois_registrant_postcode = whois_registrant_postcode;
}
public String getWhois_registrant_country() {
return whois_registrant_country;
}
public void setWhois_registrant_country(String whois_registrant_country) {
this.whois_registrant_country = whois_registrant_country;
}
public String getWhois_registrant_phone() {
return whois_registrant_phone;
}
public void setWhois_registrant_phone(String whois_registrant_phone) {
this.whois_registrant_phone = whois_registrant_phone;
}
}

31
src/main/java/com/mesasoft/cn/sketch/dao/src/main/java/generator/DomainCategoryReputationDao.java
View File

@@ -1,31 +0,0 @@
/*
package generator;
import generator.DomainCategoryReputation;
import generator.DomainCategoryReputationExample;
import java.util.List;
import org.apache.ibatis.annotations.Param;
public interface DomainCategoryReputationDao {
long countByExample(DomainCategoryReputationExample example);
int deleteByExample(DomainCategoryReputationExample example);
int deleteByPrimaryKey(Integer id);
int insert(DomainCategoryReputation record);
int insertSelective(DomainCategoryReputation record);
List<DomainCategoryReputation> selectByExample(DomainCategoryReputationExample example);
DomainCategoryReputation selectByPrimaryKey(Integer id);
int updateByExampleSelective(@Param("record") DomainCategoryReputation record, @Param("example") DomainCategoryReputationExample example);
int updateByExample(@Param("record") DomainCategoryReputation record, @Param("example") DomainCategoryReputationExample example);
int updateByPrimaryKeySelective(DomainCategoryReputation record);
int updateByPrimaryKey(DomainCategoryReputation record);
}*/

18
src/main/java/com/mesasoft/cn/sketch/entity/ReturnT.java
View File

@@ -1,5 +1,6 @@
package com.mesasoft.cn.sketch.entity;
import com.mesasoft.cn.sketch.enums.ResultStatus;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@@ -13,8 +14,8 @@ import java.io.Serializable;
@AllArgsConstructor
public class ReturnT<T> implements Serializable {
public static final int SUCCESS_STATUS = 200;
public static final int FAIL_STATUS = 500;
public static final int SUCCESS_STATUS = ResultStatus.SUCCESS.getStatus();
public static final int FAIL_STATUS = ResultStatus.FAIL.getStatus();
private int status;
@@ -22,9 +23,20 @@ public class ReturnT<T> implements Serializable {
private String message;
private T data;
public static <T> ReturnT succeed(String requestid) {
return new ReturnT(SUCCESS_STATUS, requestid, ResultStatus.SUCCESS.getMessage(), null);
}
public static <T> ReturnT succeed(String requestid, T data) {
return new ReturnT(SUCCESS_STATUS, requestid, null, data);
return new ReturnT(SUCCESS_STATUS, requestid, ResultStatus.SUCCESS.getMessage(), data);
}
public static <T> ReturnT failed(String requestid) {
return new ReturnT(FAIL_STATUS, requestid, ResultStatus.FAIL.getMessage(), null);
}
public static <T> ReturnT failed(String requestid, String message) {

13
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java
View File

@@ -1,12 +1,18 @@
package com.mesasoft.cn.sketch.entity.domain;
import cn.hutool.core.date.DateUtil;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import lombok.ToString;
import org.springframework.format.annotation.DateTimeFormat;
import java.io.Serializable;
import java.util.Date;
@Data
public class DomainBase {
@ToString
public class DomainBase implements Serializable {
Integer id;
String fqdn;
@@ -16,9 +22,10 @@ public class DomainBase {
Integer status_code = 0;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd")
// @JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd")
Date create_time;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd")
// @JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd")
Date update_time;
}

2
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainCategory.java
View File

@@ -6,6 +6,7 @@ import com.mesasoft.cn.sketch.util.ValidationUtils;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -23,6 +24,7 @@ import java.util.List;
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class DomainCategory extends DomainBase{

49
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainICP.java Normal file
View File

@@ -0,0 +1,49 @@
package com.mesasoft.cn.sketch.entity.domain;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class DomainICP extends DomainBase {
private Boolean is_accessible;
private String icp_owner;
private String icp_company_name;
private String icp_company_type;
private String icp_main_page;
private String icp_site_license;
private String icp_site_name;
private String icp_verify_time;
public DomainICP(String fqdn, String source, Integer match_pattern, Boolean query_success,Boolean is_accessible, String icp_owner, String icp_company_name,
String icp_company_type,String icp_main_page, String icp_site_license, String icp_site_name, String icp_verify_time) {
this.fqdn = fqdn;
this.source = source;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.query_success = query_success;
this.is_accessible = is_accessible;
this.icp_owner = icp_owner;
this.icp_company_name = icp_company_name;
this.icp_company_type = icp_company_type;
this.icp_main_page = icp_main_page;
this.icp_site_license = icp_site_license;
this.icp_site_name = icp_site_name;
this.icp_verify_time = icp_verify_time;
}
}

6
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainInfo.java
View File

@@ -1,15 +1,13 @@
package com.mesasoft.cn.sketch.entity.domain;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.*;
import java.util.List;
@Data
@Builder
@AllArgsConstructor
@ToString
@NoArgsConstructor
public class DomainInfo<T> {

174
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java
View File

@@ -6,6 +6,7 @@ import com.mesasoft.cn.sketch.util.ValidationUtils;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.sql.Date;
import java.sql.ResultSet;
@@ -24,6 +25,7 @@ import java.util.List;
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class DomainWhois extends DomainBase{
@@ -95,177 +97,5 @@ public class DomainWhois extends DomainBase{
this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone);
}
/* public static void insertRecords(List<DomainWhois> whoisFiles, MariaDbBase mariaDbBase) {
for (DomainWhois whoisFile : whoisFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static String insertSql(List<DomainWhois> whoisFiles) {
DomainWhois whoisFile = whoisFiles.get(0);
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
return resSql;
}
public static void updateRecords(List<DomainWhois> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainWhois categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainWhois> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") ";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}*/
public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", '" + whois_domain + '\'' +
", '" + whois_update_date + '\'' +
", '" + whois_create_date + '\'' +
", '" + whois_expire_date + '\'' +
", '" + whois_email + '\'' +
", '" + whois_ns + '\'' +
", '" + whois_registrar_name + '\'' +
", '" + whois_registrant_org + '\'' +
", '" + whois_registrant_name + '\'' +
", '" + whois_registrant_street + '\'' +
", '" + whois_registrant_city + '\'' +
", '" + whois_registrant_state + '\'' +
", '" + whois_registrant_postcode + '\'' +
", '" + whois_registrant_country + '\'' +
", '" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public static List<DomainWhois> rs2schema(ResultSet rs) throws SQLException {
List<DomainWhois> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainWhois(
rs.getString("fqdn"),
rs.getString("source"),
rs.getInt("match_pattern"),
rs.getBoolean("query_success"),
rs.getString("whois_domain"),
(Date) rs.getDate("whois_update_date"),
(Date) rs.getDate("whois_create_date"),
(Date) rs.getDate("whois_expire_date"),
rs.getString("whois_email"),
rs.getString("whois_ns"),
rs.getString("whois_registrar_name"),
rs.getString("whois_registrant_org"),
rs.getString("whois_registrant_name"),
rs.getString("whois_registrant_street"),
rs.getString("whois_registrant_city"),
rs.getString("whois_registrant_state"),
rs.getString("whois_registrant_postcode"),
rs.getString("whois_registrant_country"),
rs.getString("whois_registrant_phone")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainWhois schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("whois_domain", schema.getWhois_domain());
jsonObject.put("whois_update_date", schema.getWhois_update_date());
jsonObject.put("whois_create_date", schema.getWhois_create_date());
jsonObject.put("whois_expire_date", schema.getWhois_expire_date());
jsonObject.put("whois_email", schema.getWhois_email());
jsonObject.put("whois_ns", schema.getWhois_ns());
jsonObject.put("whois_registrar_name", schema.getWhois_registrar_name());
jsonObject.put("whois_registrant_org", schema.getWhois_registrant_org());
jsonObject.put("whois_registrant_name", schema.getWhois_registrant_name());
jsonObject.put("whois_registrant_street", schema.getWhois_registrant_street());
jsonObject.put("whois_registrant_city", schema.getWhois_registrant_city());
jsonObject.put("whois_registrant_state", schema.getWhois_registrant_state());
jsonObject.put("whois_registrant_postcode", schema.getWhois_registrant_postcode());
jsonObject.put("whois_registrant_country", schema.getWhois_registrant_country());
jsonObject.put("whois_registrant_phone", schema.getWhois_registrant_phone());
return jsonObject;
}
/* public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", whois_domain" +
", whois_update_date" +
", whois_create_date" +
", whois_expire_date" +
", whois_email" +
", whois_ns" +
", whois_registrar_name" +
", whois_registrant_org" +
", whois_registrant_name" +
", whois_registrant_street" +
", whois_registrant_city" +
", whois_registrant_state" +
", whois_registrant_postcode" +
", whois_registrant_country" +
", whois_registrant_phone";
return resString;
}
public String getKeyValues() {
String resString = "query_success=" + query_success +
", source='" + source + '\'' +
", match_pattern=" + match_pattern +
", whois_domain='" + whois_domain + '\'' +
", whois_update_date='" + whois_update_date + '\'' +
", whois_create_date='" + whois_create_date + '\'' +
", whois_expire_date='" + whois_expire_date + '\'' +
", whois_email='" + whois_email + '\'' +
", whois_ns='" + whois_ns + '\'' +
", whois_registrar_name='" + whois_registrar_name + '\'' +
", whois_registrant_org='" + whois_registrant_org + '\'' +
", whois_registrant_name='" + whois_registrant_name + '\'' +
", whois_registrant_street='" + whois_registrant_street + '\'' +
", whois_registrant_city='" + whois_registrant_city + '\'' +
", whois_registrant_state='" + whois_registrant_state + '\'' +
", whois_registrant_postcode='" + whois_registrant_postcode + '\'' +
", whois_registrant_country='" + whois_registrant_country + '\'' +
", whois_registrant_phone='" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}*/
}

77
src/main/java/com/mesasoft/cn/sketch/enums/ResultStatus.java Normal file
View File

@@ -0,0 +1,77 @@
package com.mesasoft.cn.sketch.enums;
/**
* @Description: 返回码定义
* 规定:
* #1表示成功
* #1001～1999 区间表示参数错误
* #2001～2999 区间表示用户错误
* #3001～3999 区间表示接口异常
* @Date Create in 2019/7/22 19:28
*/
public enum ResultStatus {
/* 成功 */
SUCCESS(200, "success"),
/* 默认失败 */
COMMON_FAIL(400, "fail"),
FAIL(500, "request fail"),
/* 参数错误：1000～1999 */
PARAM_NOT_VALID(1001, "参数无效"),
PARAM_IS_BLANK(1002, "参数为空"),
PARAM_TYPE_ERROR(1003, "参数类型错误"),
PARAM_NOT_COMPLETE(1004, "参数缺失"),
/* 用户错误 */
USER_NOT_LOGIN(2001, "用户未登录"),
USER_ACCOUNT_EXPIRED(2002, "账号已过期"),
USER_CREDENTIALS_ERROR(2003, "密码错误"),
USER_CREDENTIALS_EXPIRED(2004, "密码过期"),
USER_ACCOUNT_DISABLE(2005, "账号不可用"),
USER_ACCOUNT_LOCKED(2006, "账号被锁定"),
USER_ACCOUNT_NOT_EXIST(2007, "账号不存在"),
USER_ACCOUNT_ALREADY_EXIST(2008, "账号已存在"),
USER_ACCOUNT_USE_BY_OTHERS(2009, "账号下线"),
/* 业务错误 */
NO_PERMISSION(3001, "没有权限");
private Integer status;
private String message;
ResultStatus(Integer status, String message) {
this.status = status;
this.message = message;
}
public Integer getStatus() {
return status;
}
public void setStatus(Integer status) {
this.status = status;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
/**
* 根据status获取message
*
* @param status
* @return
*/
public static String getMessageBystatus(Integer status) {
for (ResultStatus ele : values()) {
if (ele.getStatus().equals(status)) {
return ele.getMessage();
}
}
return null;
}
}

93
src/main/java/com/mesasoft/cn/sketch/service/DbService.java
View File

@@ -1,93 +0,0 @@
package com.mesasoft.cn.sketch.service;
import cn.hutool.log.Log;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.dao.DomainCategorySchema;
import com.mesasoft.cn.sketch.dao.MariaDbBase;
import java.util.List;
public class DbService {
private Log logger = Log.get();
private static final String dataBase = AppConfig.DATABASE;
private static final String tableName = AppConfig.DOMAIN_CATE_TABLENAME;
private MariaDbBase mariaDB;
public void execute(List<DomainCategorySchema> recordsFromBcApi) {
for (DomainCategorySchema categoryFile : recordsFromBcApi) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + categoryFile.getKeys() + ") values" +
'(' + categoryFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDB.writeSqlExecute(resSql);
}
}
/**
* @throws Exception
*/
/* public void exec(List<Map> data, String sql) throws Exception {
if (Objects.isNull(data)) {
logger.info(" data is empty ");
return;
}
Connection conn = null;
PreparedStatement pst = null;
try {
//创建数据库连接库对象
conn = MariaDBUtils.getConnection();
long start = System.currentTimeMillis();
conn.setAutoCommit(false);
pst = conn.prepareStatement(sql);
int count = 0;
int index = 1;
for (int i = 1; i <= data.size(); i++) {
for (Object val : data.get(i - 1).values()) {
if (val instanceof Long) {
pst.setLong((index++), Long.valueOf(String.valueOf(val)));
} else if (val instanceof Integer) {
pst.setInt((index++), Integer.valueOf(String.valueOf(val)));
} else if (val instanceof Boolean) {
pst.setBoolean((index++), Boolean.valueOf(String.valueOf(val)));
} else {
pst.setString((index++), String.valueOf(val));
}
}
index = 1;
pst.addBatch();
//1w提交一次
if (i % ApplicationConfig.DB_QUERY_BATCH_SIZE == 0) {
int[] ints = pst.executeBatch();
count = count + ints.length;
conn.commit();
pst.clearBatch();
}
}
int[] ints = pst.executeBatch();
count = count + ints.length;
conn.commit();
logger.info("sql {} , count {} , take {}", sql, tableName, count, (System.currentTimeMillis() - start) + "ms");
} catch (Exception e) {
if (conn != null) {
try {
conn.rollback();
} catch (SQLException e1) {
logger.error(e1);
}
}
logger.error(e);
throw e;
} finally {
IoUtil.close(pst);
IoUtil.close(conn);
}
}*/
}

22
src/main/java/com/mesasoft/cn/sketch/service/DomainService.java
View File

@@ -1,14 +1,30 @@
package com.mesasoft.cn.sketch.service;
import com.alibaba.fastjson.JSONArray;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import java.util.List;
import java.util.Map;
@Service
public interface DomainService {
JSONArray getCategoryInfo(List<String> domains, String username, boolean isLocal) throws Exception ;
List<DomainInfo> getCategoryInfo(List<String> domains, String username) throws Exception;
List<DomainInfo> getCategoryInfoByFile(MultipartHttpServletRequest request, String username) throws Exception;
List<DomainInfo> getWhoisInfo(List<String> domains) throws Exception;
List<DomainInfo> getWhoisInfoByFile(MultipartHttpServletRequest request) throws Exception;
List<DomainInfo> getICPInfo(List<String> domains) throws Exception;
List<DomainInfo> getICPInfoByFile(MultipartHttpServletRequest request) throws Exception;
Map getDomainInfoByFile(String requestUser, String srcFile, String queryType) throws Exception;
JSONArray getWhoisInfo(List<String> domains, boolean isLocal) throws Exception ;
}

264
src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java
View File

@@ -1,8 +1,11 @@
package com.mesasoft.cn.sketch.service.impl;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.Lists;
import com.mesasoft.cn.sketch.api.BrightCloud;
import com.mesasoft.cn.sketch.api.ChinaZ;
@@ -11,31 +14,41 @@ import com.mesasoft.cn.sketch.config.BrightCloudConfig;
import com.mesasoft.cn.sketch.config.ChinazConfig;
import com.mesasoft.cn.sketch.dao.DomainDao;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import com.mesasoft.cn.sketch.entity.domain.DomainICP;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import com.mesasoft.cn.sketch.entity.domain.DomainWhois;
import com.mesasoft.cn.sketch.service.DomainService;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import com.zhazhapan.util.ArrayUtils;
import org.apache.commons.io.IOUtils;
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.List;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class DomainServiceImpl implements DomainService {
private static final Log logger = Log.get();
@Autowired
@Resource
private BrightCloudConfig bcConfig;
@Autowired
@Resource
private ChinazConfig chinazConfig;
@Autowired
@Resource
private AppConfig appConfig;
@Autowired
@@ -43,12 +56,21 @@ public class DomainServiceImpl implements DomainService {
@Resource
SqlSessionFactory sqlSessionFactory;
@Value("${query.output.dir}")
private String queryOutputDir;
@Value("${query.readin.batch}")
private int queryReadinBatch;
private final boolean isLocal = true;
@Override
public JSONArray getCategoryInfo(List<String> domains, String username, boolean isLocal) throws Exception {
public List<DomainInfo> getCategoryInfo(List<String> domains, String username) throws Exception {
domains = ValidationUtils.getCheckedFqdns(domains);
logger.info("domain category query total size {}", domains.size());
JSONArray results = new JSONArray();
logger.info("domain category query total : {}", domains.size());
int queryNum = domains.size();
List<DomainInfo> results = new ArrayList<DomainInfo>();
//1. 查询本地数据库
List<List<String>> partitionDomains = Lists.partition(domains, appConfig.getDbQueryBatchSize());
List<String> dBDomains = new ArrayList<>();
@@ -64,20 +86,23 @@ public class DomainServiceImpl implements DomainService {
//2. 调用api查询
int apiResultNum = 0;
int failedQueryNum = 0;
int dbResultNum = dBDomains.size();
if (!isLocal && domains.size() > 0) {
List<DomainCategory> bcResults = new ArrayList<>();
//批量查询API
BrightCloud brightCloud = new BrightCloud();
List<List<String>> apiPartitions = Lists.partition(domains, bcConfig.getMaximumQueryNum());
for (List<String> partition : apiPartitions) {
List<DomainInfo> bcDomainInfos = new ArrayList<>();
List<DomainCategory> recordsFromBcApi = brightCloud.getBrightCloudDomains(partition);
List<DomainCategory> recordsFromBcApi = brightCloud.getBrightCloudDomainCategory(partition);
for (DomainCategory record : recordsFromBcApi) {
//查询成功的结果
if (record.getQuery_success().equals(true)) {
record.setSubmit_user(username);
record.setCreate_time(new Date());
record.setUpdate_time(new Date());
bcResults.add(record);
bcDomainInfos.add(DomainInfo.builder()
.domain(record.getFqdn())
@@ -89,27 +114,40 @@ public class DomainServiceImpl implements DomainService {
}
}
if (bcResults.size() > 0) {
insertBatchDomainCategory(bcResults);
insertBatchDomains(bcResults);
results.addAll(bcDomainInfos);
bcResults.clear();
bcDomainInfos.clear();
}
processLog("category", queryNum, apiResultNum, failedQueryNum, dbResultNum);
}
// 记录api调用次数
FileWriter fileWriter = new FileWriter(bcConfig.getUsereportFilePath(), true);
fileWriter.write(DateUtil.date() + "," + " domain category query brightCloud success record " + apiResultNum + ",fail query record " + failedQueryNum + "\n");
fileWriter.close();
logger.info("domain category query brightCloud success record " + apiResultNum + ",fail query record " + failedQueryNum);
logger.info("domain category query brightCloud success record : " + apiResultNum + ",fail query record : " + failedQueryNum);
}
return results;
}
@Override
public JSONArray getWhoisInfo(List<String> domains, boolean isLocal) throws Exception {
public List<DomainInfo> getCategoryInfoByFile(MultipartHttpServletRequest request, String username) throws Exception {
return getCategoryInfo(readDomainsByFile(request), username);
}
@Override
public List<DomainInfo> getWhoisInfoByFile(MultipartHttpServletRequest request) throws Exception {
return getWhoisInfo(readDomainsByFile(request));
}
@Override
public List<DomainInfo> getWhoisInfo(List<String> domains) throws Exception {
domains = ValidationUtils.getCheckedFqdns(domains);
logger.info("domain whois query total size {}", domains.size());
JSONArray results = new JSONArray();
int queryNum = domains.size();
ArrayList<DomainInfo> results = new ArrayList<DomainInfo>();
//1.查询本地数据库
List<String> dBDomains = new ArrayList<>();
List<List<String>> partitions = Lists.partition(domains, appConfig.getDbQueryBatchSize());
@@ -125,6 +163,7 @@ public class DomainServiceImpl implements DomainService {
//2. 调用api
int apiResultNum = 0;
int failedQueryNum = 0;
int dbResultNum = dBDomains.size();
if (!isLocal && domains.size() > 0) {
List<DomainWhois> chinazResults = new ArrayList<>();
@@ -133,10 +172,12 @@ public class DomainServiceImpl implements DomainService {
for (List<String> partition : apiPartitions) {
List<DomainInfo> chinazDomainInfos = new ArrayList<>();
List<DomainWhois> recordsFromApi = chinaz.getChinaZDomains(partition);
List<DomainWhois> recordsFromApi = chinaz.getDomainsWhois(partition);
for (DomainWhois record : recordsFromApi) {
if (record.getQuery_success().equals(true)) {
// 查询成功的结果
record.setCreate_time(new Date());
record.setUpdate_time(new Date());
chinazResults.add(record);
chinazDomainInfos.add(DomainInfo.builder()
.domain(record.getFqdn())
@@ -148,70 +189,199 @@ public class DomainServiceImpl implements DomainService {
}
}
if (chinazResults.size() > 0) {
insertBatchDomainWhois(chinazResults);
insertBatchDomains(chinazResults);
results.addAll(chinazDomainInfos);
chinazResults.clear();
chinazDomainInfos.clear();
}
processLog("whois", queryNum, apiResultNum, failedQueryNum, dbResultNum);
}
// 记录api调用次数
try {
FileWriter fileWriter = new FileWriter(chinazConfig.getUsereportFilePath(), true);
fileWriter.write(DateUtil.date() + "," + " domain whois query chinaz success record " + apiResultNum + ",fail query record " + failedQueryNum + "\n");
fileWriter.close();
logger.info("domain whois query chinaz success record " + apiResultNum + ",fail query record " + failedQueryNum);
} catch (IOException e) {
logger.error(e);
}
}
return results;
}
@Override
public List<DomainInfo> getICPInfo(List<String> domains) throws Exception {
domains = ValidationUtils.getCheckedFqdns(domains);
logger.info("domain ICP query total size {}", domains.size());
int queryNum = domains.size();
ArrayList<DomainInfo> results = new ArrayList<DomainInfo>();
//1.查询本地数据库
List<String> dBDomains = new ArrayList<>();
List<List<String>> partitions = Lists.partition(domains, appConfig.getDbQueryBatchSize());
for (List<String> domainICP : partitions) {
//批量查询
List<DomainInfo> dbRecords = domainDao.getICPInfo(domainICP);
results.addAll(dbRecords);
dbRecords.forEach(domainInfo -> dBDomains.add(domainInfo.getDomain()));
}
domains.removeAll(dBDomains);
logger.info("domain ICP query DB record {}", results.size());
if (!isLocal && domains.size() > 0) {
//2. 调用api
int apiResultNum = 0;
int failedQueryNum = 0;
int dbResultNum = dBDomains.size();
List<DomainICP> chinazResults = new ArrayList<>();
ChinaZ chinaz = new ChinaZ();
List<List<String>> apiPartitions = Lists.partition(domains, chinazConfig.getMaximumQueryNum());
for (List<String> partition : apiPartitions) {
List<DomainInfo> chinazDomainInfos = new ArrayList<>();
List<DomainICP> recordsFromApi = chinaz.getDomainsICP(partition);
for (DomainICP record : recordsFromApi) {
if (record.getQuery_success().equals(true)) {
// 查询成功的结果
record.setCreate_time(new Date());
record.setUpdate_time(new Date());
chinazResults.add(record);
chinazDomainInfos.add(DomainInfo.builder()
.domain(record.getFqdn())
.result(Lists.newArrayList(record))
.build());
apiResultNum += 1;
} else { // 查询失败的结果
failedQueryNum += 1;
}
}
if (chinazResults.size() > 0) {
insertBatchDomains(chinazResults);
results.addAll(chinazDomainInfos);
chinazResults.clear();
chinazDomainInfos.clear();
}
processLog("ICP", queryNum, apiResultNum, failedQueryNum, dbResultNum);
}
// 记录api调用次数
try {
FileWriter fileWriter = new FileWriter(chinazConfig.getUsereportFilePath(), true);
fileWriter.write(DateUtil.date() + "," + " domain ICP query chinaz success record " + apiResultNum + ",fail query record " + failedQueryNum + "\n");
fileWriter.close();
logger.info("domain ICP query chinaz success record " + apiResultNum + ",fail query record " + failedQueryNum);
} catch (IOException e) {
logger.error(e);
}
}
return results;
}
@Override
public List<DomainInfo> getICPInfoByFile(MultipartHttpServletRequest request) throws Exception {
return getICPInfo(readDomainsByFile(request));
}
private List<String> readDomainsByFile(MultipartHttpServletRequest request) {
List<String> domains = new ArrayList<>();
MultipartFile[] domainFiles = ArrayUtils.mapToArray(request.getFileMap(), MultipartFile.class);
for (MultipartFile domainFile : domainFiles) {
logger.info("domain query file ", domainFile.getOriginalFilename());
if (domainFile != null && !domainFile.isEmpty()) {
try {
List<String> lines = IOUtils.readLines(domainFile.getInputStream(), StandardCharsets.UTF_8);
domains.addAll(lines);
} catch (IOException e) {
logger.error("domain query file error {}", e);
}
}
}
return domains;
}
@Override
public Map getDomainInfoByFile(String requestUser, String srcFile, String queryType) throws Exception {
//输出文件
String fileName = String.join("-", FileUtil.getPrefix(srcFile), queryType, DateUtil.format(new Date(), "yyyyMMdd-HHmmss") + ".json");
File resultFile = FileUtil.touch(queryOutputDir + File.separator + fileName);
cn.hutool.core.io.file.FileWriter fileWriter = new cn.hutool.core.io.file.FileWriter(resultFile);
Map<String, Object> resultMsg = new HashMap<>();
resultMsg.put("result", resultFile.toString());
// 文件读取
List<String> readUtf8Lines = FileUtil.readUtf8Lines(srcFile);
logger.info("[File query]-" + srcFile + " ,queryType " + queryType + " query read in batch " + queryReadinBatch + ",domains size " + readUtf8Lines.size());
List<List<String>> partition = ListUtil.partition(readUtf8Lines, queryReadinBatch);
int total = 0;
for (List<String> domains : partition) {
List<DomainInfo> queryResults = new ArrayList<>();
switch (queryType) {
case "category":
queryResults = getCategoryInfo(domains, requestUser);
break;
case "whois":
queryResults = getWhoisInfo(domains);
break;
case "icp":
queryResults = getICPInfo(domains);
break;
}
if (ObjectUtil.isNotEmpty(queryResults)) {
total += queryResults.size();
List<String> prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x)).collect(Collectors.toList());
//List<String> prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x, true)).collect(Collectors.toList());
fileWriter.appendLines(prettyResults);
}
logger.info("[File query]- " + queryType + " query result size " + queryResults.size());
}
resultMsg.put("total", total);
logger.info("[File query]-" + srcFile + " Results saved in " + resultFile.getAbsolutePath());
return resultMsg;
}
/**
* 域名分类 批量入库
* 域名入库
*
* @param dataList
* @return
*/
public int insertBatchDomainCategory(List<DomainCategory> dataList) {
public <T> int insertBatchDomains(List<T> dataList) {
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
try {
DomainDao domainDaoBatch = sqlSession.getMapper(DomainDao.class);
dataList.stream().forEach(data -> domainDaoBatch.insertCategory(data));
sqlSession.commit();
sqlSession.clearCache();
logger.info("insert domain category data success ,data size {}", dataList.size());
dataList.stream().forEach(data -> {
if (data.getClass() == DomainCategory.class) {
domainDaoBatch.insertCategory((DomainCategory) data);
} else if (data.getClass() == DomainWhois.class) {
domainDaoBatch.insertWhois((DomainWhois) data);
} else if (data.getClass() == DomainICP.class) {
domainDaoBatch.insertICP((DomainICP) data);
}
});
sqlSession.commit();
logger.info("insert domain data success ,data size {}", dataList.size());
} catch (Exception e) {
logger.error("domainCategory insert error, {}", e);
sqlSession.rollback();
logger.error("domain insert error, {}", e);
return 0;
} finally {
sqlSession.clearCache();
sqlSession.close();
}
return dataList.size();
}
/**
* 域名whois入库
*
* @param dataList
* @return
*/
public int insertBatchDomainWhois(List<DomainWhois> dataList) {
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
try {
DomainDao domainDaoBatch = sqlSession.getMapper(DomainDao.class);
dataList.stream().forEach(data -> domainDaoBatch.insertWhois(data));
sqlSession.commit();
sqlSession.clearCache();
logger.info("insert domain whois data success ,data size {}", dataList.size());
} catch (Exception e) {
logger.error("domainWhois insert error, {}", e);
sqlSession.rollback();
return 0;
} finally {
sqlSession.close();
}
return dataList.size();
private void processLog(String logType, int queryNum, int apiResultNum, int failedQueryNum, int dbResultNum) {
logger.info("[domain " + logType + "]-"
+ "Query result: submit " + queryNum + " valid objects, "
+ dbResultNum + " (" + new DecimalFormat("##.0%").format((float) dbResultNum / queryNum) + ")" + " results from database,"
+ apiResultNum + " (" + new DecimalFormat("##.0%").format((float) apiResultNum / queryNum) + ")" + " results from api. "
+ failedQueryNum + " (" + new DecimalFormat("##.0%").format((float) failedQueryNum / queryNum) + ")" + " failed queries,");
}
}

66
src/main/java/com/mesasoft/cn/sketch/util/FileUtils.java
View File

@@ -14,36 +14,28 @@ import java.util.List;
public class FileUtils {
private static final Logger LOG = Logger.getLogger(FileUtils.class);
public static List<String> readTxtFileIntoStringArrList(String filePath)
{
public static List<String> readTxtFileIntoStringArrList(String filePath) {
List<String> list = new ArrayList<>();
try
{
try {
String encoding = "GBK";
File file = new File(filePath);
if (file.isFile() && file.exists())
{ // 判断文件是否存在
if (file.isFile() && file.exists()) { // 判断文件是否存在
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null)
{
while ((lineTxt = bufferedReader.readLine()) != null) {
if (!lineTxt.equals("")) {
list.add(lineTxt.trim());
}
}
bufferedReader.close();
read.close();
}
else
{
} else {
System.out.println("Can not find file: " + filePath);
}
}
catch (Exception e)
{
} catch (Exception e) {
System.out.println("Error occurred in Function 'readTxtFileIntoStringArrList'");
e.printStackTrace();
}
@@ -51,32 +43,31 @@ public class FileUtils {
return list;
}
public static List<String> getBatchLineReadIn(BufferedReader bufferedReader, int batchSize){
public static List<String> getBatchLineReadIn(BufferedReader bufferedReader, int batchSize) {
List<String> list = new ArrayList<>();
String lineTxt;
try{
while ((lineTxt = bufferedReader.readLine()) != null && list.size()<batchSize)
{
try {
while ((lineTxt = bufferedReader.readLine()) != null && list.size() < batchSize) {
if (!lineTxt.equals("")) {
list.add(lineTxt.trim());
}
}
} catch (IOException e){
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
public static void createFile(File filePath, String fileName){
public static void createFile(File filePath, String fileName) {
try {
File file = new File(filePath.toString() + "/" + fileName);
if (!filePath.exists()){
if (!filePath.exists()) {
filePath.mkdirs();
}
boolean isCreate = file.createNewFile();
if (isCreate){
if (isCreate) {
LOG.info("File " + fileName + " is created.");
}
} catch (IOException e) {
@@ -85,10 +76,10 @@ public class FileUtils {
}
public static void createFile(File file){
public static void createFile(File file) {
try {
boolean isCreate = file.createNewFile();
if (isCreate){
if (isCreate) {
LOG.info("File " + file + " is created.");
}
} catch (IOException e) {
@@ -121,9 +112,9 @@ public class FileUtils {
}
}
public static String getFileName(File file){
public static String getFileName(File file) {
String[] tmp = file.toString().split("/");
String fileName = tmp[tmp.length-1];
String fileName = tmp[tmp.length - 1];
return fileName;
}
@@ -142,14 +133,13 @@ public class FileUtils {
//执行cmd命令，获取返回结果
public static String execCMD(String command) {
StringBuilder sb =new StringBuilder();
StringBuilder sb = new StringBuilder();
try {
Process process=Runtime.getRuntime().exec(command);
BufferedReader bufferedReader=new BufferedReader(new InputStreamReader(process.getInputStream()));
Process process = Runtime.getRuntime().exec(command);
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while((line=bufferedReader.readLine())!=null)
{
sb.append(line+"\n");
while ((line = bufferedReader.readLine()) != null) {
sb.append(line + "\n");
}
} catch (Exception e) {
return e.toString();
@@ -157,9 +147,9 @@ public class FileUtils {
return sb.toString();
}
public static Long getFileLineNum(File file){
public static Long getFileLineNum(File file) {
Long num = 0L;
if (!file.exists()){
if (!file.exists()) {
LOG.error("File not exist: " + file.toString());
} else {
String res = FileUtils.execCMD("wc -l " + file.toString());
@@ -194,4 +184,12 @@ public class FileUtils {
}
return lengthDomain;
}
public static String formatPath(String path) {
path = path.replace("\\\\", "");
path = path.replace("//", "");
return path;
}
}

26
src/main/java/com/mesasoft/cn/sketch/util/SktUtil.java Normal file
View File

@@ -0,0 +1,26 @@
package com.mesasoft.cn.sketch.util;
import com.mesasoft.cn.entity.User;
import com.zhazhapan.modules.constant.ValueConsts;
import javax.servlet.http.HttpServletRequest;
public class SktUtil {
/**
* 获取当前请求用户
*
* @param request
* @return
*/
public static String getRequestUser(HttpServletRequest request) {
Object attribute = request.getSession().getAttribute(ValueConsts.USER_STRING);
User user = null;
if (attribute == null) {
return null;
} else {
user = (User) attribute;
}
return user.getUsername();
}
}

95
src/main/java/com/mesasoft/cn/sketch/util/SpringContextUtil.java Normal file
View File

@@ -0,0 +1,95 @@
package com.mesasoft.cn.sketch.util;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
import java.util.Map;
/**
* Spring上下文工具类
*
* @author dazzlzy
* @date 2018/4/25
*/
@Component
public class SpringContextUtil implements ApplicationContextAware {
private static ApplicationContext context = null;
private SpringContextUtil() {
super();
}
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
context = applicationContext;
}
/**
* 根据名称获取bean
*
* @param beanName bean的名字
* @return bean对象
*/
public static Object getBean(String beanName) {
return context.getBean(beanName);
}
/**
* 根据bean名称获取指定类型bean
*
* @param beanName bean名称
* @param clazz 返回的bean类型,若类型不匹配,将抛出异常
*/
public static <T> T getBean(String beanName, Class<T> clazz) {
return context.getBean(beanName, clazz);
}
/**
* 根据类型获取bean
*
* @param clazz bean类型
* @return 指定类型的bean
*/
public static <T> T getBean(Class<T> clazz) {
T t = null;
Map<String, T> map = context.getBeansOfType(clazz);
for (Map.Entry<String, T> entry : map.entrySet()) {
t = entry.getValue();
}
return t;
}
/**
* 是否包含bean
*
* @param beanName bean名字
* @return boolean，是否包含指定名字的bean
*/
public static boolean containsBean(String beanName) {
return context.containsBean(beanName);
}
/**
* 是否是单例
*
* @param beanName bean名字
* @return boolean， 是否是单例
*/
public static boolean isSingleton(String beanName) {
return context.isSingleton(beanName);
}
/**
* bean的类型
*
* @param beanName bean名字
* @return bean的类型
*/
public static Class getType(String beanName) {
return context.getType(beanName);
}
}

80
src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java
View File

@@ -1,10 +1,14 @@
package com.mesasoft.cn.sketch.util;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.resource.ResourceUtil;
import cn.hutool.log.Log;
import com.mesasoft.cn.sketch.config.AppConfig;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import sun.net.util.IPAddressUtil;
import javax.annotation.Resource;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -16,20 +20,26 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ValidationUtils {
private static final Logger LOG = Logger.getLogger(ValidationUtils.class);
private static final Log logger = Log.get();
/**
* 获取二级域名
**/
private static String tldFilePath;
@Value("${sketch.tld.file}")
public void setTldFilePath(String ttlFilePath){
this.tldFilePath= ttlFilePath;
// @Value("${sketch.tld.file}")
// public void setTldFilePath(String ttlFilePath) {
// this.tldFilePath = ttlFilePath;
// if (StringUtils.isBlank(ttlFilePath) || !FileUtil.exist(ttlFilePath)) {
// this.tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
// }
// }
static {
tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
}
public static String getSecDomain(String fqdnOrUrl){
/**
* 获取二级域名
**/
public static String getSecDomain(String fqdnOrUrl) {
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(tldFilePath);
try {
@@ -56,13 +66,12 @@ public class ValidationUtils {
}
}
// 右匹配为顶级域名
if (secDomain == null){
if (secDomain == null) {
secDomain = fqdnOrUrl;
}
return secDomain;
} catch (Exception e) {
LOG.error("urlDomain:" + fqdnOrUrl);
e.printStackTrace();
logger.error("urlDomain:" + fqdnOrUrl);
return "---no---return---";
}
}
@@ -75,7 +84,7 @@ public class ValidationUtils {
if (StringUtils.isNotBlank(secDomain) && !("---no---return---".equals(secDomain))) {
secDomainList.add(secDomain);
} else {
System.out.println(oriDomain);
logger.info(oriDomain);
}
}
return secDomainList;
@@ -95,8 +104,7 @@ public class ValidationUtils {
String encoding = "UTF-8";
File file = new File(filePath);
if (file.isFile() && file.exists()) {
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null) {
@@ -113,11 +121,10 @@ public class ValidationUtils {
}
read.close();
} else {
LOG.error("TopDomainUtils>=>readTopDomainFile filePath is wrong--->{" + filePath + "}<---");
logger.error("TopDomainUtils>=>readTopDomainFile filePath is wrong--->{" + filePath + "}<---");
}
} catch (Exception e) {
LOG.error("TopDomainUtils>=>readTopDomainFile get filePathData error--->{" + e + "}<---");
e.printStackTrace();
logger.error("TopDomainUtils>=>readTopDomainFile get filePathData error--->{" + e + "}<---");
}
return maps;
}
@@ -132,55 +139,54 @@ public class ValidationUtils {
return maps;
}
public static List<String> getChecked(List<String> objectList, String type){
if (type.equals("ip")){
public static List<String> getChecked(List<String> objectList, String type) {
if (type.equals("ip")) {
return getCheckedIps(objectList);
}
if (type.equals("domain")){
if (type.equals("domain")) {
return getCheckedFqdns(objectList);
}
LOG.error("Wrong type to be checked: " + type);
logger.error("Wrong type to be checked: " + type);
return objectList;
}
public static List<String> getCheckedFqdns(List<String> fqdns){
public static List<String> getCheckedFqdns(List<String> fqdns) {
List<String> res = new ArrayList<>();
for (String fqdn:fqdns){
for (String fqdn : fqdns) {
//去端口号
fqdn = fqdn.split(":")[0];
// 去重 & 校验
if (isValidDomain(fqdn) && !res.contains(fqdn)){
if (isValidDomain(fqdn) && !res.contains(fqdn)) {
res.add(fqdn.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + fqdn);
logger.debug("Bad or duplicated fqdn:" + fqdn);
}
}
return res;
}
public static List<String> getCheckedIps(List<String> ipList){
public static List<String> getCheckedIps(List<String> ipList) {
List<String> res = new ArrayList<>();
for (String ip:ipList){
for (String ip : ipList) {
//去端口号
ip = ip.split(":")[0];
// 去重 & 校验
if (isValidIp(ip) && !res.contains(ip)){
if (isValidIp(ip) && !res.contains(ip)) {
res.add(ip.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + ip);
logger.debug("Bad or duplicated fqdn:" + ip);
}
}
return res;
}
public static boolean isValidIp(String ip){
public static boolean isValidIp(String ip) {
boolean iPv4LiteralAddress = IPAddressUtil.isIPv4LiteralAddress(ip);
boolean iPv6LiteralAddress = IPAddressUtil.isIPv6LiteralAddress(ip);
return iPv4LiteralAddress || iPv6LiteralAddress;
}
private static boolean isValidDomain(String str)
{
private static boolean isValidDomain(String str) {
String regex = "^((?!-)[A-Za-z0-9-_]"
+ "{1,63}(?<!-)\\.)"
+ "+[A-Za-z]{2,6}";
@@ -193,15 +199,13 @@ public class ValidationUtils {
return m.matches();
}
public static Integer getMatchPattern(String fqdn){
public static Integer getMatchPattern(String fqdn) {
int match_pattern = 2;
if (fqdn.equals(getSecDomain(fqdn))){
if (fqdn.equals(getSecDomain(fqdn))) {
match_pattern = 1; // 二级域名-右匹配
}
return match_pattern;
}
}

31
src/main/java/com/mesasoft/cn/util/ControllerUtils.java
View File

@@ -1,8 +1,8 @@
package com.mesasoft.cn.util;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.text.UnicodeUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.zhazhapan.util.Checker;
@@ -10,7 +10,6 @@ import org.apache.commons.lang3.StringUtils;
import javax.activation.MimetypesFileTypeMap;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.FileInputStream;
@@ -53,7 +52,7 @@ public class ControllerUtils {
if (Checker.isNotEmpty(path)) {
File file = new File(path);
if (download) {
response.setContentType(getContentType(file)+";charset=UTF-8");
response.setContentType(getContentType(file) + ";charset=UTF-8");
setResponseFileName2(response, file.getName());
}
FileInputStream in = new FileInputStream(file);
@@ -71,12 +70,13 @@ public class ControllerUtils {
response.sendRedirect(DefaultValues.NOT_FOUND_PAGE);
}
}
public static void loadResource(HttpServletResponse response, String path, boolean download) throws IOException {
if (Checker.isNotEmpty(path)) {
File file = new File(path);
if (download) {
response.setContentType(getContentType(file));
setResponseFileName( response, file.getName());
setResponseFileName(response, file.getName());
response.setCharacterEncoding("UTF-8");
}
FileInputStream in = new FileInputStream(file);
@@ -94,10 +94,11 @@ public class ControllerUtils {
response.sendRedirect(DefaultValues.NOT_FOUND_PAGE);
}
}
public static String getContentType(File file) {
String defContentType = "application/octet-stream";
String fileName = file.getName();
String fileTyle=fileName.substring(fileName.lastIndexOf(".")+1,fileName.length());
String fileTyle = fileName.substring(fileName.lastIndexOf(".") + 1, fileName.length());
if (StringUtils.isNotBlank(fileTyle)) {
String type2 = Contants.CONTENT_TYPES.get(fileTyle);
if (StringUtils.isNotBlank(type2)) {
@@ -121,29 +122,26 @@ public class ControllerUtils {
public static void setResponseFileName2(HttpServletResponse response, String fileName) {
response.setHeader("Content-Disposition", "attachment;filename=" + UnicodeUtil.toUnicode(fileName));
}
public static void setResponseFileName( HttpServletResponse response, String fileName) throws
public static void setResponseFileName(HttpServletResponse response, String fileName) throws
UnsupportedEncodingException {
response.setHeader("Content-Disposition", "attachment;filename=" + new String(fileName.getBytes("UTF-8"),
"ISO-8859-1"));
}
public static void loadFile(HttpServletRequest request, HttpServletResponse response,String filePath,boolean download) throws IOException {
request.setCharacterEncoding("utf-8");
// 文件存储路径
// 从请求中获取文件名
File file=new File(filePath);
String fileName=file.getName();
public static void loadFile(HttpServletResponse response, String fileName, String data) throws IOException {
// 创建输出流对象
ServletOutputStream outputStream = response.getOutputStream();
//以字节数组的形式读取文件
byte[] bytes = FileUtil.readBytes(filePath);
//以字节数组的形式读取字符串
byte[] bytes = StrUtil.utf8Bytes(data);
// 设置返回内容格式
response.setContentType("application/octet-stream;charset=UTF-8");
String suffixName = StrUtil.subAfter(fileName, ".", true);
response.setContentType("application/" + suffixName + ";charset=UTF-8");
// 把文件名按UTF-8取出并按ISO8859-1编码，保证弹出窗口中的文件名中文不乱码
// 中文不要太多，最多支持17个中文，因为header有150个字节限制。
response.setHeader("filename", UnicodeUtil.toUnicode(fileName));
// 这一步一定要在读取文件之后进行，否则文件名会乱码，找不到文件
fileName = new String(fileName.getBytes("UTF-8"),"ISO-8859-1");
fileName = new String(fileName.getBytes("UTF-8"));
// 设置下载弹窗的文件名和格式（文件名要包括名字和文件格式）
response.setHeader("Content-Disposition", "attachment;filename=" + fileName);
// 返回数据到输出流对象中
@@ -152,4 +150,5 @@ public class ControllerUtils {
IoUtil.close(outputStream);
}
}

8
src/main/resources/META-INF/additional-spring-configuration-metadata.json Normal file
View File

@@ -0,0 +1,8 @@
{
"properties": [
{
"name": "bright-cloud.maximum.query.num",
"type": "java.lang.String",
"description": "Description for bright-cloud.maximum.query.num."
}
] }

25
src/main/resources/application-sketch.properties
View File

@@ -4,18 +4,16 @@ spring.datasource.username=root
spring.datasource.password=galaxy2019
################### application ####################
query.output.dir = /home/bigdata/domain/output_file
query.output.dir = c:\\test\\output_file
# offline\u8BFB\u53D6\u6279\u5904\u7406\u91CF
query.readin.batch = 10000
# offline读取批处理量
query.readin.batch = 1
# \u6253\u5370\u8FDB\u5EA6\u65E5\u5FD7\u7684\u67E5\u8BE2\u6761\u6570
# 打印进度日志的查询条数
query.log.file.line.interval = 10000
################### mariadb ########################
database = web_sketch_v2
tablename.domain.category = domain_category_reputation
tablename.domain.whois = domain_whois
db.query.batch.size = 10000
###################### api #########################
@@ -29,24 +27,23 @@ bright-cloud.method = POST
bright-cloud.queryType = getinfo
bright-cloud.isa1cat = 1
bright-cloud.isReputation = 1
#bright-cloud\u8FD4\u56DEjson\u683C\u5F0F
#bright-cloud返回json格式
bright-cloud.isxml = 0
# api\u5355\u6B21\u67E5\u8BE2url\u957F\u5EA6\u9650\u5236 API\u6700\u9AD8\u9650\u5236
# api单次查询url长度限制 API最高限制
bright-cloud.maximum-query-num = 100
bright-cloud.cateinfo-filepath = categoryinfo.json
bright-cloud.usereport-filepath = bright_cloud_query_count.csv
######### chinaz #########
chinaz.url-single = https://apidatav2.chinaz.com/single/whois
chinaz.url-batch = https://apidatav2.chinaz.com/batch/whois
chinaz.url-single = https://apidatav2.chinaz.com/single
chinaz.url-batch = https://apidatav2.chinaz.com/batch
chinaz.key = ffc9be4141bd49a093ed0185a54dc6a2
chinaz.maximum-query-num = 50
chinaz.usereport-filepath = chinaz_query_count.csv
###################### \u5176\u4ED6 #########################
#\u9876\u7EA7\u57DF\u540D
###################### 其他 #########################
#顶级域名
sketch.tld.file = public_suffix_list_only.dat
sketch.path.admin=C:\\test
sketch.path.user=C:\\test\\1\\
sketch.home.path=C:\\test

10
src/main/resources/application.properties
View File

@@ -31,20 +31,20 @@ spring.datasource.druid.stat-view-servlet.deny=
spring.jpa.show-sql=true
spring.servlet.multipart.max-file-size=1099511627776
spring.servlet.multipart.max-request-size=1099511627776
#\u70ED\u90E8\u7F72\u751F\u6548
#热部署生效
spring.devtools.restart.enabled=true
#\u9875\u9762\u70ED\u90E8\u7F72
#页面热部署
spring.thymeleaf.cache=false
spring.devtools.restart.additional-paths=src/main
#classpath\u76EE\u5F55\u4E0B\u7684WEB-INF\u6587\u4EF6\u5939\u5185\u5BB9\u4FEE\u6539\u4E0D\u91CD\u542F
#classpath目录下的WEB-INF文件夹内容修改不重启
spring.devtools.restart.exclude=WEB-INF/**
#spring.profiles.active=prod
#spring.mvc.favicon.enabled=false
swagger.enabled=true
swagger.title=\u7EBF\u4E0A\u6587\u4EF6\u7BA1\u7406\u7CFB\u7EDF
swagger.description=\u4E0A\u4F20\u3001\u4E0B\u8F7D\u3001\u5206\u4EAB\u3001\u8FDC\u7A0B\u6587\u4EF6\u7BA1\u7406
swagger.title=线上文件管理系统
swagger.description=上传、下载、分享、远程文件管理
swagger.version=1.2
swagger.license=MIT
swagger.licenseUrl=https://opensource.org/licenses/MIT

6
src/main/resources/assets/js/index.js
View File

@@ -565,12 +565,12 @@ function backPage() {
if(currentPath.endsWith("\\")){
currentPath = currentPath.substring(0, currentPath.lastIndexOf('\\'));
}
backPath = currentPath == '' ? '' : currentPath.substring(0, currentPath.lastIndexOf('\\'))+"\\";
backPath = currentPath == '' ? '' : currentPath.substring(0, currentPath.lastIndexOf('\\'));
} else {
if(currentPath.endsWith("/")){
currentPath = currentPath.substring(0, currentPath.lastIndexOf('\\'));
currentPath = currentPath.substring(0, currentPath.lastIndexOf('/'));
}
backPath = currentPath == '' ? '' : currentPath.substring(0, currentPath.lastIndexOf('/'))+"/";
backPath = currentPath == '' ? '' : currentPath.substring(0, currentPath.lastIndexOf('/'));
}
getFiles(backPath);
if ( $("#history_path").attr("value")==backPath){

431
src/main/resources/categoryinfo.json Normal file
View File

@@ -0,0 +1,431 @@
{
"status": 200,
"requestid": "12345",
"type": "url",
"results": [
{
"url": "getcatlist",
"queries": {
"getcatlist": {
"cats": [
{
"catid": 1,
"catname": "Real Estate",
"catgroup": "Productivity"
},
{
"catid": 2,
"catname": "Computer and Internet Security",
"catgroup": "Productivity"
},
{
"catid": 3,
"catname": "Financial Services",
"catgroup": "Privacy"
},
{
"catid": 4,
"catname": "Business and Economy",
"catgroup": "Productivity"
},
{
"catid": 5,
"catname": "Computer and Internet Info",
"catgroup": "Productivity"
},
{
"catid": 6,
"catname": "Auctions",
"catgroup": "Productivity"
},
{
"catid": 7,
"catname": "Shopping",
"catgroup": "Productivity"
},
{
"catid": 8,
"catname": "Cult and Occult",
"catgroup": "Sensitive"
},
{
"catid": 9,
"catname": "Travel",
"catgroup": "Productivity"
},
{
"catid": 10,
"catname": "Abused Drugs",
"catgroup": "Sensitive"
},
{
"catid": 11,
"catname": "Adult and Pornography",
"catgroup": "Sensitive"
},
{
"catid": 12,
"catname": "Home and Garden",
"catgroup": "Productivity"
},
{
"catid": 13,
"catname": "Military",
"catgroup": "Productivity"
},
{
"catid": 14,
"catname": "Social Networking",
"catgroup": "Productivity"
},
{
"catid": 15,
"catname": "Dead Sites",
"catgroup": "Misc"
},
{
"catid": 16,
"catname": "Individual Stock Advice and Tools",
"catgroup": "Productivity"
},
{
"catid": 17,
"catname": "Training and Tools",
"catgroup": "Productivity"
},
{
"catid": 18,
"catname": "Dating",
"catgroup": "Sensitive"
},
{
"catid": 19,
"catname": "Sex Education",
"catgroup": "Sensitive"
},
{
"catid": 20,
"catname": "Religion",
"catgroup": "Sensitive"
},
{
"catid": 21,
"catname": "Entertainment and Arts",
"catgroup": "Productivity"
},
{
"catid": 22,
"catname": "Personal sites and Blogs",
"catgroup": "Productivity"
},
{
"catid": 23,
"catname": "Legal",
"catgroup": "Privacy"
},
{
"catid": 24,
"catname": "Local Information",
"catgroup": "Productivity"
},
{
"catid": 25,
"catname": "Streaming Media",
"catgroup": "IT Resources"
},
{
"catid": 26,
"catname": "Job Search",
"catgroup": "Productivity"
},
{
"catid": 27,
"catname": "Gambling",
"catgroup": "Sensitive"
},
{
"catid": 28,
"catname": "Translation",
"catgroup": "Sensitive"
},
{
"catid": 29,
"catname": "Reference and Research",
"catgroup": "Productivity"
},
{
"catid": 30,
"catname": "Shareware and Freeware",
"catgroup": "IT Resources"
},
{
"catid": 31,
"catname": "Peer to Peer",
"catgroup": "IT Resources"
},
{
"catid": 32,
"catname": "Marijuana",
"catgroup": "Sensitive"
},
{
"catid": 33,
"catname": "Hacking",
"catgroup": "Sensitive"
},
{
"catid": 34,
"catname": "Games",
"catgroup": "Productivity"
},
{
"catid": 35,
"catname": "Philosophy and Political Advocacy",
"catgroup": "Productivity"
},
{
"catid": 36,
"catname": "Weapons",
"catgroup": "Sensitive"
},
{
"catid": 37,
"catname": "Pay to Surf",
"catgroup": "Productivity"
},
{
"catid": 38,
"catname": "Hunting and Fishing",
"catgroup": "Productivity"
},
{
"catid": 39,
"catname": "Society",
"catgroup": "Productivity"
},
{
"catid": 40,
"catname": "Educational Institutions",
"catgroup": "Productivity"
},
{
"catid": 41,
"catname": "Online Greeting Cards",
"catgroup": "IT Resources"
},
{
"catid": 42,
"catname": "Sports",
"catgroup": "Productivity"
},
{
"catid": 43,
"catname": "Swimsuits and Intimate Apparel",
"catgroup": "Sensitive"
},
{
"catid": 44,
"catname": "Questionable",
"catgroup": "Sensitive"
},
{
"catid": 45,
"catname": "Kids",
"catgroup": "Productivity"
},
{
"catid": 46,
"catname": "Hate and Racism",
"catgroup": "Sensitive"
},
{
"catid": 47,
"catname": "Personal Storage",
"catgroup": "IT Resources"
},
{
"catid": 48,
"catname": "Violence",
"catgroup": "Sensitive"
},
{
"catid": 49,
"catname": "Keyloggers and Monitoring",
"catgroup": "Security"
},
{
"catid": 50,
"catname": "Search Engines",
"catgroup": "Productivity"
},
{
"catid": 51,
"catname": "Internet Portals",
"catgroup": "Productivity"
},
{
"catid": 52,
"catname": "Web Advertisements",
"catgroup": "IT Resources"
},
{
"catid": 53,
"catname": "Cheating",
"catgroup": "Sensitive"
},
{
"catid": 54,
"catname": "Gross",
"catgroup": "Sensitive"
},
{
"catid": 55,
"catname": "Web-based Email",
"catgroup": "Privacy"
},
{
"catid": 56,
"catname": "Malware Sites",
"catgroup": "Security"
},
{
"catid": 57,
"catname": "Phishing and Other Frauds",
"catgroup": "Security"
},
{
"catid": 58,
"catname": "Proxy Avoidance and Anonymizers",
"catgroup": "Security"
},
{
"catid": 59,
"catname": "Spyware and Adware",
"catgroup": "Security"
},
{
"catid": 60,
"catname": "Music",
"catgroup": "Productivity"
},
{
"catid": 61,
"catname": "Government",
"catgroup": "Privacy"
},
{
"catid": 62,
"catname": "Nudity",
"catgroup": "Sensitive"
},
{
"catid": 63,
"catname": "News and Media",
"catgroup": "Productivity"
},
{
"catid": 64,
"catname": "Illegal",
"catgroup": "Sensitive"
},
{
"catid": 65,
"catname": "Content Delivery Networks",
"catgroup": "IT Resources"
},
{
"catid": 66,
"catname": "Internet Communications",
"catgroup": "IT Resources"
},
{
"catid": 67,
"catname": "Bot Nets",
"catgroup": "Security"
},
{
"catid": 68,
"catname": "Abortion",
"catgroup": "Sensitive"
},
{
"catid": 69,
"catname": "Health and Medicine",
"catgroup": "Privacy"
},
{
"catid": 70,
"catname": "Confirmed SPAM Sources",
"catgroup": "Unmaintained"
},
{
"catid": 71,
"catname": "SPAM URLs",
"catgroup": "Security"
},
{
"catid": 72,
"catname": "Unconfirmed SPAM Sources",
"catgroup": "Unmaintained"
},
{
"catid": 73,
"catname": "Open HTTP Proxies",
"catgroup": "Unmaintained"
},
{
"catid": 74,
"catname": "Dynamically Generated Content",
"catgroup": "Productivity"
},
{
"catid": 75,
"catname": "Parked Domains",
"catgroup": "Sensitive"
},
{
"catid": 76,
"catname": "Alcohol and Tobacco",
"catgroup": "Sensitive"
},
{
"catid": 77,
"catname": "Private IP Addresses",
"catgroup": "Unmaintained"
},
{
"catid": 78,
"catname": "Image and Video Search",
"catgroup": "Sensitive"
},
{
"catid": 79,
"catname": "Fashion and Beauty",
"catgroup": "Productivity"
},
{
"catid": 80,
"catname": "Recreation and Hobbies",
"catgroup": "Productivity"
},
{
"catid": 81,
"catname": "Motor Vehicles",
"catgroup": "Productivity"
},
{
"catid": 82,
"catname": "Web Hosting",
"catgroup": "IT Resources"
},
{
"catid": 83,
"catname": "Food and Dining",
"catgroup": "Unmaintained"
}
]
}
}
}
]
}

29315
src/main/resources/dns_server_v1.csv Normal file
View File

File diff suppressed because one or more lines are too long

3407
src/main/resources/doh_support_list.csv Normal file
View File

File diff suppressed because it is too large Load Diff

217
src/main/resources/mapper/DomainDao.xml Normal file
View File

@@ -0,0 +1,217 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.mesasoft.cn.sketch.dao.DomainDao">
<resultMap id="WhoisResultMap" type="com.mesasoft.cn.sketch.entity.domain.DomainInfo">
<result column="fqdn" jdbcType="VARCHAR" property="domain"/>
<collection property="result" ofType="com.mesasoft.cn.sketch.entity.domain.DomainWhois">
<id column="id" jdbcType="INTEGER" property="id"/>
<result column="fqdn" jdbcType="VARCHAR" property="fqdn"/>
<result column="source" jdbcType="VARCHAR" property="source"/>
<result column="query_success" jdbcType="BIT" property="query_success"/>
<result column="match_pattern" jdbcType="INTEGER" property="match_pattern"/>
<result column="whois_domain" jdbcType="VARCHAR" property="whois_domain"/>
<result column="whois_update_date" jdbcType="TIMESTAMP" property="whois_update_date"/>
<result column="whois_create_date" jdbcType="TIMESTAMP" property="whois_create_date"/>
<result column="whois_expire_date" jdbcType="TIMESTAMP" property="whois_expire_date"/>
<result column="whois_email" jdbcType="VARCHAR" property="whois_email"/>
<result column="whois_ns" jdbcType="VARCHAR" property="whois_ns"/>
<result column="whois_registrar_name" jdbcType="VARCHAR" property="whois_registrar_name"/>
<result column="whois_registrant_org" jdbcType="VARCHAR" property="whois_registrant_org"/>
<result column="whois_registrant_name" jdbcType="VARCHAR" property="whois_registrant_name"/>
<result column="whois_registrant_street" jdbcType="VARCHAR" property="whois_registrant_street"/>
<result column="whois_registrant_city" jdbcType="VARCHAR" property="whois_registrant_city"/>
<result column="whois_registrant_state" jdbcType="VARCHAR" property="whois_registrant_state"/>
<result column="whois_registrant_postcode" jdbcType="VARCHAR" property="whois_registrant_postcode"/>
<result column="whois_registrant_country" jdbcType="VARCHAR" property="whois_registrant_country"/>
<result column="whois_registrant_phone" jdbcType="VARCHAR" property="whois_registrant_phone"/>
<result column="status_code" jdbcType="INTEGER" property="status_code"/>
<result column="create_time" jdbcType="TIMESTAMP" property="create_time"/>
<result column="update_time" jdbcType="TIMESTAMP" property="update_time"/>
</collection>
</resultMap>
<resultMap id="CategoryResultMap" type="com.mesasoft.cn.sketch.entity.domain.DomainInfo">
<result column="fqdn" jdbcType="VARCHAR" property="domain"/>
<collection property="result" ofType="com.mesasoft.cn.sketch.entity.domain.DomainCategory">
<id column="id" jdbcType="INTEGER" property="id"/>
<result column="fqdn" jdbcType="VARCHAR" property="fqdn"/>
<result column="source" jdbcType="VARCHAR" property="source"/>
<result column="query_success" jdbcType="BIT" property="query_success"/>
<result column="match_pattern" jdbcType="INTEGER" property="match_pattern"/>
<result column="reputation_score" jdbcType="INTEGER" property="reputation_score"/>
<result column="reputation_level" jdbcType="VARCHAR" property="reputation_level"/>
<result column="category_id" jdbcType="INTEGER" property="category_id"/>
<result column="category_name" jdbcType="VARCHAR" property="category_name"/>
<result column="category_group" jdbcType="VARCHAR" property="category_group"/>
<result column="category_conf" jdbcType="INTEGER" property="category_conf"/>
<result column="is_a1_cat" jdbcType="BIT" property="is_a1_cat"/>
<result column="status_code" jdbcType="INTEGER" property="status_code"/>
<result column="submit_user" jdbcType="VARCHAR" property="submit_user"/>
<result column="create_time" jdbcType="TIMESTAMP" property="create_time"/>
<result column="update_time" jdbcType="TIMESTAMP" property="update_time"/>
</collection>
</resultMap>
<resultMap id="ICPResultMap" type="com.mesasoft.cn.sketch.entity.domain.DomainInfo">
<result column="fqdn" jdbcType="VARCHAR" property="domain"/>
<collection property="result" ofType="com.mesasoft.cn.sketch.entity.domain.DomainICP">
<id column="id" jdbcType="INTEGER" property="id"/>
<result column="fqdn" jdbcType="VARCHAR" property="fqdn"/>
<result column="source" jdbcType="VARCHAR" property="source"/>
<result column="query_success" jdbcType="BIT" property="query_success"/>
<result column="match_pattern" jdbcType="INTEGER" property="match_pattern"/>
<result column="icp_owner" jdbcType="VARCHAR" property="icp_owner"/>
<result column="icp_company_name" jdbcType="VARCHAR" property="icp_company_name"/>
<result column="icp_company_type" jdbcType="VARCHAR" property="icp_company_type"/>
<result column="icp_main_page" jdbcType="VARCHAR" property="icp_main_page"/>
<result column="icp_site_license" jdbcType="VARCHAR" property="icp_site_license"/>
<result column="icp_site_name" jdbcType="VARCHAR" property="icp_site_name"/>
<result column="icp_verify_time" jdbcType="VARCHAR" property="icp_verify_time"/>
<result column="status_code" jdbcType="INTEGER" property="status_code"/>
<result column="create_time" jdbcType="TIMESTAMP" property="create_time"/>
<result column="update_time" jdbcType="TIMESTAMP" property="update_time"/>
</collection>
</resultMap>
<!-- 查询字段 -->
<sql id="Category_Column_List">
id
, fqdn, `source`, query_success, match_pattern, reputation_score, reputation_level,
category_id, category_name, category_group, category_conf, is_a1_cat, status_code,
submit_user, create_time, update_time
</sql>
<sql id="Whois_Column_List">
id
, fqdn, `source`, query_success, match_pattern, whois_domain, whois_update_date,
whois_create_date, whois_expire_date, whois_email, whois_ns, whois_registrar_name,
whois_registrant_org, whois_registrant_name, whois_registrant_street, whois_registrant_city,
whois_registrant_state, whois_registrant_postcode, whois_registrant_country, whois_registrant_phone,
status_code, create_time, update_time
</sql>
<sql id="ICP_Column_List">
id
, fqdn, `source`, query_success, match_pattern, icp_owner, icp_company_name, icp_company_type,icp_main_page,
icp_site_license, icp_site_name, icp_verify_time, status_code, create_time, update_time
</sql>
<!-- whois info -->
<select id="getWhoisInfo" resultMap="WhoisResultMap">
select
<include refid="Whois_Column_List"/>
from domain_whois
<where>
fqdn in
<foreach collection="domain" item="listItem" open="(" separator="," close=")">
#{listItem}
</foreach>
</where>
limit 10
</select>
<!-- category info -->
<select id="getCategoryInfo" resultMap="CategoryResultMap">
select
<include refid="Category_Column_List"/>
from domain_category_reputation
<where>
fqdn in
<foreach collection="domain" item="listItem" open="(" separator="," close=")">
#{listItem}
</foreach>
</where>
limit 10
</select>
<!-- ICP info -->
<select id="getICPInfo" resultMap="ICPResultMap">
select
<include refid="ICP_Column_List"/>
from domain_icp
<where>
fqdn in
<foreach collection="domain" item="listItem" open="(" separator="," close=")">
#{listItem}
</foreach>
</where>
limit 10
</select>
<!-- category insert -->
<insert id="insertCategory" keyColumn="id" keyProperty="id"
parameterType="com.mesasoft.cn.sketch.entity.domain.DomainCategory" useGeneratedKeys="true">
insert into domain_category_reputation
(fqdn,
`source`,
query_success,
match_pattern,
reputation_score,
reputation_level,
category_id,
category_name,
category_group,
category_conf,
is_a1_cat,
status_code,
submit_user,
create_time,
update_time)
values (#{fqdn,jdbcType=VARCHAR},
#{source,jdbcType=VARCHAR},
#{query_success,jdbcType=BIT},
#{match_pattern,jdbcType=INTEGER},
#{reputation_score,jdbcType=INTEGER},
#{reputation_level,jdbcType=VARCHAR},
#{category_id,jdbcType=INTEGER},
#{category_name,jdbcType=VARCHAR},
#{category_group,jdbcType=VARCHAR},
#{category_conf,jdbcType=INTEGER},
#{is_a1_cat,jdbcType=BIT},
#{status_code,jdbcType=INTEGER},
#{submit_user,jdbcType=VARCHAR},
#{create_time,jdbcType=TIMESTAMP},
#{update_time,jdbcType=TIMESTAMP})
</insert>
<insert id="insertWhois" keyColumn="id" keyProperty="id"
parameterType="com.mesasoft.cn.sketch.entity.domain.DomainWhois" useGeneratedKeys="true">
insert into domain_whois
(fqdn, `source`, query_success,
match_pattern, whois_domain, whois_update_date,
whois_create_date, whois_expire_date,
whois_email, whois_ns, whois_registrar_name,
whois_registrant_org, whois_registrant_name,
whois_registrant_street, whois_registrant_city,
whois_registrant_state, whois_registrant_postcode,
whois_registrant_country, whois_registrant_phone,
status_code, create_time, update_time)
values (#{fqdn,jdbcType=VARCHAR}, #{source,jdbcType=VARCHAR}, #{query_success,jdbcType=BIT},
#{match_pattern,jdbcType=INTEGER}, #{whois_domain,jdbcType=VARCHAR},
#{whois_update_date,jdbcType=TIMESTAMP},
#{whois_create_date,jdbcType=TIMESTAMP}, #{whois_expire_date,jdbcType=TIMESTAMP},
#{whois_email,jdbcType=VARCHAR}, #{whois_ns,jdbcType=VARCHAR}, #{whois_registrar_name,jdbcType=VARCHAR},
#{whois_registrant_org,jdbcType=VARCHAR}, #{whois_registrant_name,jdbcType=VARCHAR},
#{whois_registrant_street,jdbcType=VARCHAR}, #{whois_registrant_city,jdbcType=VARCHAR},
#{whois_registrant_state,jdbcType=VARCHAR}, #{whois_registrant_postcode,jdbcType=VARCHAR},
#{whois_registrant_country,jdbcType=VARCHAR}, #{whois_registrant_phone,jdbcType=VARCHAR},
#{status_code,jdbcType=INTEGER}, #{create_time,jdbcType=TIMESTAMP}, #{update_time,jdbcType=TIMESTAMP})
</insert>
<insert id="insertICP" keyColumn="id" keyProperty="id"
parameterType="com.mesasoft.cn.sketch.entity.domain.DomainICP" useGeneratedKeys="true">
insert into domain_icp (fqdn, `source`, query_success,
match_pattern, icp_owner, icp_company_name,
icp_company_type, icp_main_page,icp_site_license, icp_site_name,
icp_verify_time, status_code, create_time,
update_time)
values (#{fqdn,jdbcType=VARCHAR}, #{source,jdbcType=VARCHAR}, #{query_success,jdbcType=BIT},
#{match_pattern,jdbcType=INTEGER}, #{icp_owner,jdbcType=VARCHAR}, #{icp_company_name,jdbcType=VARCHAR},
#{icp_company_type,jdbcType=VARCHAR}, #{icp_main_page,jdbcType=VARCHAR}, #{icp_site_license,jdbcType=VARCHAR},
#{icp_site_name,jdbcType=VARCHAR},
#{icp_verify_time,jdbcType=VARCHAR}, #{status_code,jdbcType=INTEGER}, #{create_time,jdbcType=TIMESTAMP},
#{update_time,jdbcType=TIMESTAMP})
</insert>
</mapper>

9142
src/main/resources/public_suffix_list_only.dat Normal file
View File

File diff suppressed because it is too large Load Diff

15
src/main/resources/static/index.html
View File

@@ -31,7 +31,7 @@
</head>
<body id="particles-js">
<nav class="navbar navbar-expand-md bg-dark navbar-dark navbar-toggler fixed-top" >
<img src="/assets/sketch/images/att-globe.svg" class="navbar-brand" style="width: 2rem;height: 2.5rem;"/>
<img src="/assets/sketch/images/att-globe.svg" class="navbar-brand" style="width: 2rem;height: 3rem;"/>
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#collapsibleNavbar">
<span class="navbar-toggler-icon"></span>
</button>
@@ -63,7 +63,7 @@
</div>-->
</nav>
<br/><br/><br/>
<div class="container" id="index">
<div class="container" id="index" style="max-width: 98%" >
<div class="row justify-content-center">
<div class="col-sm-12 col-10 tab-content">
<!--文件列表-->
@@ -71,13 +71,18 @@
<div class="row content-box rounded">
<!-- Header -->
<header id="header" class="d-flex align_item_center">
<div onclick="" class="logo_p">
<!-- <img src="/assets/img/file-favicon.png" class="rounded avatar d-block">-->
<img src="/assets/sketch/images/att-globe.svg" class="rounded avatar d-block">
<!--<div class="logo_p">
&lt;!&ndash; <img src="/assets/img/file-favicon.png" class="rounded avatar d-block">&ndash;&gt;
&lt;!&ndash; <img src="/assets/sketch/images/att-globe.svg" class="rounded avatar d-block">&ndash;&gt;
<h4 class="rounded avatar d-block" >Home</h4>
</div>-->
<div class="heading_title_p" style="white-space:nowrap;text-overflow:ellipsis;margin-top: 15px">
<h3 >Home</h3>
</div>
<div class="heading_title_p" style="white-space:nowrap;text-overflow:ellipsis;margin-top: 15px">
<h3 id="dir_id" value=""></h3>
</div>
</header>
</div>
<div class="row content-box rounded" style="margin-top: 10px">

36
src/test/java/com/mesasoft/cn/WebSketchApplicationTest.java Normal file
View File

@@ -0,0 +1,36 @@
package com.mesasoft.cn;
import cn.hutool.core.date.DateUtil;
import com.mesasoft.cn.sketch.config.ChinazConfig;
import com.zhazhapan.config.JsonParser;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@RunWith(SpringRunner.class)
@SpringBootTest
public class WebSketchApplicationTest {
public static void setSettings() {
try {
WebSketchApplication.settings = new JsonParser(WebSketchApplicationTest.class.getResource("/config.json"));
} catch (IOException e) {
e.printStackTrace();
}
}
@Test
public void contextLoads() {
List<String> domain = new ArrayList<>();
domain.add("a");
domain.remove("a");
System.err.println(DateUtil.date());
}
}

45
src/test/java/com/mesasoft/cn/api/ApiServiceTest.java Normal file
View File

@@ -0,0 +1,45 @@
package com.mesasoft.cn.api;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.api.BrightCloud;
import com.mesasoft.cn.sketch.api.ChinaZ;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author pantao
* @since 2018/2/9
*/
@RunWith(SpringRunner.class)
@SpringBootTest
public class ApiServiceTest {
@Test
public void brightCloudTest() {
JSONObject queryResults = new BrightCloud().getQueryResults(Arrays.asList("baidu.com"));
new BrightCloud().responseSparse(queryResults);
}
public void chinazTest() {
// ChinaZ t = new ChinaZ(null);
//单查询测试
// System.out.println(t.singleRequest("aaa.baidu.com"));
//批量查询测试
// List<String> domainList = new ArrayList<>();
// domainList.add("www.baidu.com");
//// domainList.add("aaa.qq.com");
//// domainList.add("doc.mesalab.com");
//
//// System.out.println(t.batchRequestController(domainList));
// System.out.println(t.getQueryResults(domainList));
}
}

6
src/test/java/com/mesasoft/cn/service/CategoryServiceTest.java
View File

@@ -1,11 +1,14 @@
package com.mesasoft.cn.service;
import cn.hutool.core.io.resource.ResourceUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.net.URL;
/**
* @author pantao
* @since 2018/2/9
@@ -20,5 +23,8 @@ public class CategoryServiceTest {
@Test
public void testGetIdByName() {
System.out.println(categoryService.getIdByName("fff"));
URL resource = ResourceUtil.getResource("config.json");
String path = resource.getPath();
System.err.println(path);
}
}