diff --git a/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java b/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java index a08c689..218b8f9 100644 --- a/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java +++ b/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java @@ -14,8 +14,6 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import java.io.BufferedReader; import java.io.InputStream; diff --git a/src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java b/src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java index 32947e8..545d743 100644 --- a/src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java +++ b/src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java @@ -1,11 +1,8 @@ package com.mesasoft.cn.sketch.config; -import cn.hutool.core.io.FileUtil; -import cn.hutool.core.io.resource.ResourceUtil; import com.mesasoft.cn.sketch.util.ConfigUtils; import lombok.Data; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; diff --git a/src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java b/src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java index 875b46d..182f97f 100644 --- a/src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java +++ b/src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java @@ -170,9 +170,9 @@ public class DomainController { * 查询大量数据 */ @GetMapping(value = "/getdomaininfobyfile") - public ReturnT getDomainInfoByFile(HttpServletRequest request, String srcFile, String queryType, SketchAuth sketchAuth) { + public ReturnT getDomainInfoByFile(HttpServletRequest request, String srcFile, String queryType, SketchAuth sketchAuth,Boolean export) { try { - Map domainInfoByFile = domainService.getDomainInfoByFile(SktUtil.getRequestUser(request), srcFile, queryType); + Map domainInfoByFile = domainService.getDomainInfoByFile(SktUtil.getRequestUser(request), srcFile, queryType,export); return ReturnT.succeed(sketchAuth.getRequestid(), domainInfoByFile); } catch (Exception e) { logger.error(e); diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java index b16bad0..667a923 100644 --- a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java +++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java @@ -1,10 +1,7 @@ package com.mesasoft.cn.sketch.entity.domain; -import cn.hutool.core.date.DateUtil; -import com.fasterxml.jackson.annotation.JsonFormat; import lombok.Data; import lombok.ToString; -import org.springframework.format.annotation.DateTimeFormat; import java.io.Serializable; import java.util.Date; diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainICP.java b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainICP.java index 5b56c82..f5fe5a0 100644 --- a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainICP.java +++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainICP.java @@ -1,6 +1,5 @@ package com.mesasoft.cn.sketch.entity.domain; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.mesasoft.cn.sketch.util.ValidationUtils; import lombok.AllArgsConstructor; import lombok.Data; @@ -13,7 +12,6 @@ import lombok.ToString; @ToString public class DomainICP extends DomainBase { - private Boolean is_accessible; private String icp_owner; private String icp_company_name; private String icp_company_type; @@ -23,7 +21,7 @@ public class DomainICP extends DomainBase { private String icp_verify_time; - public DomainICP(String fqdn, String source, Integer match_pattern, Boolean query_success,Boolean is_accessible, String icp_owner, String icp_company_name, + public DomainICP(String fqdn, String source, Integer match_pattern, Boolean query_success, String icp_owner, String icp_company_name, String icp_company_type,String icp_main_page, String icp_site_license, String icp_site_name, String icp_verify_time) { this.fqdn = fqdn; this.source = source; @@ -35,7 +33,6 @@ public class DomainICP extends DomainBase { this.match_pattern = match_pattern; } this.query_success = query_success; - this.is_accessible = is_accessible; this.icp_owner = icp_owner; this.icp_company_name = icp_company_name; diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java index c31414a..7540d0b 100644 --- a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java +++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java @@ -1,6 +1,5 @@ package com.mesasoft.cn.sketch.entity.domain; -import com.alibaba.fastjson.JSONObject; import com.mesasoft.cn.sketch.util.ConfigUtils; import com.mesasoft.cn.sketch.util.ValidationUtils; import lombok.AllArgsConstructor; @@ -9,11 +8,7 @@ import lombok.NoArgsConstructor; import lombok.ToString; import java.sql.Date; -import java.sql.ResultSet; -import java.sql.SQLException; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.List; /** * Created with IntelliJ IDEA. diff --git a/src/main/java/com/mesasoft/cn/sketch/service/DomainService.java b/src/main/java/com/mesasoft/cn/sketch/service/DomainService.java index bd4f420..bd9195b 100644 --- a/src/main/java/com/mesasoft/cn/sketch/service/DomainService.java +++ b/src/main/java/com/mesasoft/cn/sketch/service/DomainService.java @@ -23,7 +23,7 @@ public interface DomainService { List getICPInfoByFile(MultipartHttpServletRequest request) throws Exception; - Map getDomainInfoByFile(String requestUser, String srcFile, String queryType) throws Exception; + Map getDomainInfoByFile(String requestUser, String srcFile, String queryType,Boolean export) throws Exception; diff --git a/src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java b/src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java index 8444571..2388e96 100644 --- a/src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java +++ b/src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java @@ -3,6 +3,7 @@ package com.mesasoft.cn.sketch.service.impl; import cn.hutool.core.collection.ListUtil; import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.NumberUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.log.Log; import com.alibaba.fastjson.JSON; @@ -62,13 +63,14 @@ public class DomainServiceImpl implements DomainService { @Value("${query.readin.batch}") private int queryReadinBatch; - private final boolean isLocal = true; + @Value("${query.isLocal}") + private boolean isLocal; @Override public List getCategoryInfo(List domains, String username) throws Exception { domains = ValidationUtils.getCheckedFqdns(domains); - logger.info("domain category query total : {}", domains.size()); + logger.info("domain category query distinct total : {}", domains.size()); int queryNum = domains.size(); List results = new ArrayList(); //1. 查询本地数据库 @@ -298,22 +300,24 @@ public class DomainServiceImpl implements DomainService { } @Override - public Map getDomainInfoByFile(String requestUser, String srcFile, String queryType) throws Exception { + public Map getDomainInfoByFile(String requestUser, String srcFile, String queryType, Boolean export) throws Exception { //输出文件 String fileName = String.join("-", FileUtil.getPrefix(srcFile), queryType, DateUtil.format(new Date(), "yyyyMMdd-HHmmss") + ".json"); File resultFile = FileUtil.touch(queryOutputDir + File.separator + fileName); - cn.hutool.core.io.file.FileWriter fileWriter = new cn.hutool.core.io.file.FileWriter(resultFile); Map resultMsg = new HashMap<>(); resultMsg.put("result", resultFile.toString()); + resultMsg.put("export", export); // 文件读取 List readUtf8Lines = FileUtil.readUtf8Lines(srcFile); logger.info("[File query]-" + srcFile + " ,queryType " + queryType + " query read in batch " + queryReadinBatch + ",domains size " + readUtf8Lines.size()); List> partition = ListUtil.partition(readUtf8Lines, queryReadinBatch); - int total = 0; + int resultSize = 0; + int querySize = 0; + List queryResults = new ArrayList<>(); + cn.hutool.core.io.file.FileWriter fileWriter = new cn.hutool.core.io.file.FileWriter(resultFile); for (List domains : partition) { - List queryResults = new ArrayList<>(); switch (queryType) { case "category": queryResults = getCategoryInfo(domains, requestUser); @@ -325,17 +329,20 @@ public class DomainServiceImpl implements DomainService { queryResults = getICPInfo(domains); break; } - - if (ObjectUtil.isNotEmpty(queryResults)) { - total += queryResults.size(); + querySize +=domains.size(); + resultSize += queryResults.size(); + if (ObjectUtil.isNotEmpty(queryResults) && export) { List prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x)).collect(Collectors.toList()); //List prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x, true)).collect(Collectors.toList()); fileWriter.appendLines(prettyResults); + logger.info("[File query]-" + srcFile +" result: " + queryResults.size() + " Results saved in " + resultFile.getAbsolutePath()); } - logger.info("[File query]- " + queryType + " query result size " + queryResults.size()); + Double queryTotal = Double.valueOf(readUtf8Lines.size()); + logger.info("[File query]- " + queryType + " query result size: " + resultSize +", query size: " + querySize +", process: "+ NumberUtil.decimalFormat("#.##%", querySize/queryTotal)); + queryResults.clear(); + } - resultMsg.put("total", total); - logger.info("[File query]-" + srcFile + " Results saved in " + resultFile.getAbsolutePath()); + resultMsg.put("total_export", querySize); return resultMsg; } diff --git a/src/main/java/com/mesasoft/cn/sketch/util/SktUtil.java b/src/main/java/com/mesasoft/cn/sketch/util/SktUtil.java index 1547842..95e46d2 100644 --- a/src/main/java/com/mesasoft/cn/sketch/util/SktUtil.java +++ b/src/main/java/com/mesasoft/cn/sketch/util/SktUtil.java @@ -13,11 +13,13 @@ public class SktUtil { * @param request * @return */ + private static String defUsername = "default"; + public static String getRequestUser(HttpServletRequest request) { Object attribute = request.getSession().getAttribute(ValueConsts.USER_STRING); User user = null; if (attribute == null) { - return null; + return defUsername; } else { user = (User) attribute; } diff --git a/src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java b/src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java index 6893c4a..b001136 100644 --- a/src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java +++ b/src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java @@ -3,16 +3,10 @@ package com.mesasoft.cn.sketch.util; import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.resource.ResourceUtil; import cn.hutool.log.Log; -import com.mesasoft.cn.sketch.config.AppConfig; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Value; import sun.net.util.IPAddressUtil; -import javax.annotation.Resource; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStreamReader; +import java.io.*; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -26,15 +20,17 @@ public class ValidationUtils { private static String tldFilePath; -// @Value("${sketch.tld.file}") -// public void setTldFilePath(String ttlFilePath) { -// this.tldFilePath = ttlFilePath; -// if (StringUtils.isBlank(ttlFilePath) || !FileUtil.exist(ttlFilePath)) { -// this.tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath(); -// } -// } + static { - tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath(); + try { + tldFilePath = new File("").getCanonicalPath()+"/config/public_suffix_list_only.dat"; + } catch (IOException e) { + logger.error(e); + } + + if(!FileUtil.exist(tldFilePath)){ + tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath(); + } } /** * 获取二级域名 diff --git a/src/main/java/com/mesasoft/cn/web/controller/UserController.java b/src/main/java/com/mesasoft/cn/web/controller/UserController.java index ab3f380..6f57aa6 100644 --- a/src/main/java/com/mesasoft/cn/web/controller/UserController.java +++ b/src/main/java/com/mesasoft/cn/web/controller/UserController.java @@ -12,6 +12,7 @@ import com.mesasoft.cn.enums.InterceptorLevel; import com.mesasoft.cn.modules.constant.ConfigConsts; import com.mesasoft.cn.modules.constant.DefaultValues; import com.mesasoft.cn.service.IUserService; +import com.mesasoft.cn.sketch.util.SktUtil; import com.mesasoft.cn.util.ControllerUtils; import com.zhazhapan.modules.constant.ValueConsts; import com.zhazhapan.util.Checker; @@ -278,4 +279,16 @@ public class UserController { } return Result.success(resultObject); } + @ApiOperation(value = "登出") + @ApiImplicitParams({@ApiImplicitParam(name = "username", value = "用户名"), @ApiImplicitParam(name = "token", value = "用于自动登录")}) + @AuthInterceptor(InterceptorLevel.NONE) + @RequestMapping(value = "/logout", method = RequestMethod.PUT) + public String logOut(HttpServletRequest request, String token) { + User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING); + request.getSession().removeAttribute(ValueConsts.USER_STRING); + TokenConfig.removeTokenByValue(user.getId()); + + jsonObject.put("status", "success"); + return jsonObject.toString(); + } } diff --git a/src/main/resources/application-sketch.properties b/src/main/resources/application-sketch.properties index 7a470a1..df837bf 100644 --- a/src/main/resources/application-sketch.properties +++ b/src/main/resources/application-sketch.properties @@ -1,5 +1,5 @@ #datasource -spring.datasource.url=jdbc:mysql://192.168.44.12:3306/efo?useUnicode=true&characterEncoding=utf-8&useSSL=true +spring.datasource.url=jdbc:mysql://192.168.45.102:3306/web_sketch_v2?useUnicode=true&characterEncoding=utf-8&useSSL=true spring.datasource.username=root spring.datasource.password=galaxy2019 @@ -7,7 +7,7 @@ spring.datasource.password=galaxy2019 query.output.dir = c:\\test\\output_file # offline读取批处理量 -query.readin.batch = 1 +query.readin.batch = 10000 # 打印进度日志的查询条数 query.log.file.line.interval = 10000 @@ -17,6 +17,7 @@ database = web_sketch_v2 db.query.batch.size = 10000 ###################### api ######################### +query.isLocal=true ##### bright cloud ####### bright-cloud.oemid = GeedgeNet bright-cloud.deviceid = TSG-Dev @@ -45,5 +46,5 @@ chinaz.usereport-filepath = chinaz_query_count.csv #顶级域名 sketch.tld.file = public_suffix_list_only.dat -sketch.home.path=C:\\test +sketch.home.path=/home/test/sketch diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 1027b16..b494907 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -9,10 +9,10 @@ spring.datasource.type=com.alibaba.druid.pool.DruidDataSource spring.datasource.driverClassName=com.mysql.jdbc.Driver spring.datasource.druid.filters=stat,wall,slf4j,config spring.datasource.druid.maxActive=100 -spring.datasource.druid.initialSize=1 -spring.datasource.druid.maxWait=60000 -spring.datasource.druid.minIdle=1 -spring.datasource.druid.timeBetweenEvictionRunsMillis=60000 +spring.datasource.druid.initialSize=5 +spring.datasource.druid.maxWait=300000 +spring.datasource.druid.minIdle=5 +spring.datasource.druid.timeBetweenEvictionRunsMillis=300000 spring.datasource.druid.minEvictableIdleTimeMillis=300000 spring.datasource.druid.validationQuery=select 'x' spring.datasource.druid.testWhileIdle=true @@ -23,18 +23,21 @@ spring.datasource.druid.maxOpenPreparedStatements=50 spring.datasource.druid.maxPoolPreparedStatementPerConnectionSize=20 spring.datasource.druid.web-stat-filter.enabled=true spring.datasource.druid.stat-view-servlet.enabled=true +spring.datasource.druid.stat-view-servlet.url-pattern=/druid/* spring.datasource.druid.stat-view-servlet.login-username=test spring.datasource.druid.stat-view-servlet.login-password=test spring.datasource.druid.stat-view-servlet.allow=127.0.0.1 spring.datasource.druid.stat-view-servlet.deny= +spring.datasource.druid.keep-alive=true +spring.datasource.druid.keep-alive-between-time-millis=3600000 spring.jpa.show-sql=true spring.servlet.multipart.max-file-size=1099511627776 spring.servlet.multipart.max-request-size=1099511627776 #热部署生效 -spring.devtools.restart.enabled=true +spring.devtools.restart.enabled=false #页面热部署 -spring.thymeleaf.cache=false +spring.thymeleaf.cache=true spring.devtools.restart.additional-paths=src/main #classpath目录下的WEB-INF文件夹内容修改不重启 spring.devtools.restart.exclude=WEB-INF/** diff --git a/src/main/resources/assets/img/background_1.png b/src/main/resources/assets/img/background_1.png new file mode 100644 index 0000000..5444ab9 Binary files /dev/null and b/src/main/resources/assets/img/background_1.png differ diff --git a/src/main/resources/assets/js/admin.js b/src/main/resources/assets/js/admin.js index 695d795..e8286b1 100644 --- a/src/main/resources/assets/js/admin.js +++ b/src/main/resources/assets/js/admin.js @@ -761,4 +761,5 @@ $(document).ready(function () { $(".file-delete").click(function () { doDelete("#file-manager-table", app.files, ".file-index", "/file/batch/"); }); + }); \ No newline at end of file diff --git a/src/main/resources/assets/js/default.js b/src/main/resources/assets/js/default.js index 547cab0..94ecc05 100644 --- a/src/main/resources/assets/js/default.js +++ b/src/main/resources/assets/js/default.js @@ -70,7 +70,7 @@ $(document).ready(function () { } globalConfig.background.imageList = list; } - changeBackgroundImage(); + // changeBackgroundImage(); } setCSS(); }); diff --git a/src/main/resources/assets/js/index.js b/src/main/resources/assets/js/index.js index 772c6ab..78b841b 100644 --- a/src/main/resources/assets/js/index.js +++ b/src/main/resources/assets/js/index.js @@ -622,3 +622,18 @@ function fileDownLoad(path,name) { // 发送ajax请求 xhr.send() } +//退出登录 +function logout() { + layer.load(1); + $.ajax({ + url: "/user/logout", type: "PUT", data: { + token: getCookie("token") + }, success: function (data) { + layer.closeAll(); + var json = JSON.parse(data); + if (json.status === "success") { + window.location.href = "/signin"; + } + } + }); +} \ No newline at end of file diff --git a/src/main/resources/config.json b/src/main/resources/config.json index 542b973..3d48696 100644 --- a/src/main/resources/config.json +++ b/src/main/resources/config.json @@ -50,20 +50,18 @@ } ], "background": { - "useImage": true, + "useImage": false, "imageIndex": 0, "imageList": [ - "https://fakedog.xyz/homedb/srcm/upload/20180123/bg_1.jpg", - "https://fakedog.xyz/homedb/srcm/upload/20180123/bg_2.jpg", - "https://fakedog.xyz/homedb/srcm/upload/20180123/bg_3.jpg" + "assets/img/background_1.png" ], "random": true, "listGenerator": { - "enable": true, - "prefix": "https://fakedog.xyz/image/background/anime/bg_", - "suffix": ".jpg", + "enable": false, + "prefix": "assets/img/background_1", + "suffix": ".png", "start": 1, - "end": 158 + "end": 1 } } }, diff --git a/src/main/resources/mapper/DomainDao.xml b/src/main/resources/mapper/DomainDao.xml index 3eb6a45..e326092 100644 --- a/src/main/resources/mapper/DomainDao.xml +++ b/src/main/resources/mapper/DomainDao.xml @@ -109,7 +109,6 @@ #{listItem} - limit 10 @@ -123,7 +122,6 @@ #{listItem} - limit 10 diff --git a/src/main/resources/static/index.html b/src/main/resources/static/index.html index ae80d73..a643d02 100644 --- a/src/main/resources/static/index.html +++ b/src/main/resources/static/index.html @@ -61,6 +61,10 @@ --> +
+ + 退出 +