域名相关表去掉is_accessible字段

src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java

@@ -14,8 +14,6 @@ import org.apache.http.client.methods.HttpPost;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.message.BasicNameValuePair;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
 
 import java.io.BufferedReader;
 import java.io.InputStream;

src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java

@@ -1,11 +1,8 @@
 package com.mesasoft.cn.sketch.config;
 
 
-import cn.hutool.core.io.FileUtil;
-import cn.hutool.core.io.resource.ResourceUtil;
 import com.mesasoft.cn.sketch.util.ConfigUtils;
 import lombok.Data;
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 

src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java

@@ -170,9 +170,9 @@ public class DomainController {
      * 查询大量数据
      */
     @GetMapping(value = "/getdomaininfobyfile")
-    public ReturnT getDomainInfoByFile(HttpServletRequest request, String srcFile, String queryType, SketchAuth sketchAuth) {
+    public ReturnT getDomainInfoByFile(HttpServletRequest request, String srcFile, String queryType, SketchAuth sketchAuth,Boolean export) {
         try {
-            Map domainInfoByFile = domainService.getDomainInfoByFile(SktUtil.getRequestUser(request), srcFile, queryType);
+            Map domainInfoByFile = domainService.getDomainInfoByFile(SktUtil.getRequestUser(request), srcFile, queryType,export);
             return ReturnT.succeed(sketchAuth.getRequestid(), domainInfoByFile);
         } catch (Exception e) {
             logger.error(e);

src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java

@@ -1,10 +1,7 @@
 package com.mesasoft.cn.sketch.entity.domain;
 
-import cn.hutool.core.date.DateUtil;
-import com.fasterxml.jackson.annotation.JsonFormat;
 import lombok.Data;
 import lombok.ToString;
-import org.springframework.format.annotation.DateTimeFormat;
 
 import java.io.Serializable;
 import java.util.Date;

src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainICP.java

@@ -1,6 +1,5 @@
 package com.mesasoft.cn.sketch.entity.domain;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.mesasoft.cn.sketch.util.ValidationUtils;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -13,7 +12,6 @@ import lombok.ToString;
 @ToString
 public class DomainICP extends DomainBase {
 
-    private Boolean is_accessible;
     private String icp_owner;
     private String icp_company_name;
     private String icp_company_type;
@@ -23,7 +21,7 @@ public class DomainICP extends DomainBase {
     private String icp_verify_time;
 
 
-    public DomainICP(String fqdn, String source, Integer match_pattern, Boolean query_success,Boolean is_accessible, String icp_owner, String icp_company_name,
+    public DomainICP(String fqdn, String source, Integer match_pattern, Boolean query_success, String icp_owner, String icp_company_name,
                      String icp_company_type,String icp_main_page, String icp_site_license, String icp_site_name, String icp_verify_time) {
         this.fqdn = fqdn;
         this.source = source;
@@ -35,7 +33,6 @@ public class DomainICP extends DomainBase {
             this.match_pattern = match_pattern;
         }
         this.query_success = query_success;
-        this.is_accessible = is_accessible;
 
         this.icp_owner = icp_owner;
         this.icp_company_name = icp_company_name;

src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java

@@ -1,6 +1,5 @@
 package com.mesasoft.cn.sketch.entity.domain;
 
-import com.alibaba.fastjson.JSONObject;
 import com.mesasoft.cn.sketch.util.ConfigUtils;
 import com.mesasoft.cn.sketch.util.ValidationUtils;
 import lombok.AllArgsConstructor;
@@ -9,11 +8,7 @@ import lombok.NoArgsConstructor;
 import lombok.ToString;
 
 import java.sql.Date;
-import java.sql.ResultSet;
-import java.sql.SQLException;
 import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.List;
 
 /**
  * Created with IntelliJ IDEA.

src/main/java/com/mesasoft/cn/sketch/service/DomainService.java

@@ -23,7 +23,7 @@ public interface DomainService {
     List<DomainInfo> getICPInfoByFile(MultipartHttpServletRequest request) throws Exception;
 
 
-    Map getDomainInfoByFile(String requestUser, String srcFile, String queryType) throws Exception;
+    Map getDomainInfoByFile(String requestUser, String srcFile, String queryType,Boolean export) throws Exception;
 
 
 

src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java

@@ -3,6 +3,7 @@ package com.mesasoft.cn.sketch.service.impl;
 import cn.hutool.core.collection.ListUtil;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.util.NumberUtil;
 import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.log.Log;
 import com.alibaba.fastjson.JSON;
@@ -62,13 +63,14 @@ public class DomainServiceImpl implements DomainService {
     @Value("${query.readin.batch}")
     private int queryReadinBatch;
 
-    private final boolean isLocal = true;
+    @Value("${query.isLocal}")
+    private boolean isLocal;
 
     @Override
     public List<DomainInfo> getCategoryInfo(List<String> domains, String username) throws Exception {
 
         domains = ValidationUtils.getCheckedFqdns(domains);
-        logger.info("domain category query total : {}", domains.size());
+        logger.info("domain category query distinct total : {}", domains.size());
         int queryNum = domains.size();
         List<DomainInfo> results = new ArrayList<DomainInfo>();
         //1. 查询本地数据库
@@ -298,22 +300,24 @@ public class DomainServiceImpl implements DomainService {
     }
 
     @Override
-    public Map getDomainInfoByFile(String requestUser, String srcFile, String queryType) throws Exception {
+    public Map getDomainInfoByFile(String requestUser, String srcFile, String queryType, Boolean export) throws Exception {
         //输出文件
         String fileName = String.join("-", FileUtil.getPrefix(srcFile), queryType, DateUtil.format(new Date(), "yyyyMMdd-HHmmss") + ".json");
         File resultFile = FileUtil.touch(queryOutputDir + File.separator + fileName);
 
-        cn.hutool.core.io.file.FileWriter fileWriter = new cn.hutool.core.io.file.FileWriter(resultFile);
         Map<String, Object> resultMsg = new HashMap<>();
         resultMsg.put("result", resultFile.toString());
+        resultMsg.put("export", export);
 
         // 文件读取
         List<String> readUtf8Lines = FileUtil.readUtf8Lines(srcFile);
         logger.info("[File query]-" + srcFile + " ,queryType " + queryType + " query read in batch " + queryReadinBatch + ",domains size " + readUtf8Lines.size());
         List<List<String>> partition = ListUtil.partition(readUtf8Lines, queryReadinBatch);
-        int total = 0;
+        int resultSize = 0;
+        int querySize = 0;
+        List<DomainInfo> queryResults = new ArrayList<>();
+        cn.hutool.core.io.file.FileWriter fileWriter = new cn.hutool.core.io.file.FileWriter(resultFile);
         for (List<String> domains : partition) {
-            List<DomainInfo> queryResults = new ArrayList<>();
             switch (queryType) {
                 case "category":
                     queryResults = getCategoryInfo(domains, requestUser);
@@ -325,17 +329,20 @@ public class DomainServiceImpl implements DomainService {
                     queryResults = getICPInfo(domains);
                     break;
             }
-
+            querySize +=domains.size();
-            if (ObjectUtil.isNotEmpty(queryResults)) {
+            resultSize += queryResults.size();
-                total += queryResults.size();
+            if (ObjectUtil.isNotEmpty(queryResults) && export) {
                 List<String> prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x)).collect(Collectors.toList());
                 //List<String> prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x, true)).collect(Collectors.toList());
                 fileWriter.appendLines(prettyResults);
+                logger.info("[File query]-" + srcFile +" result: " + queryResults.size() + " Results saved in " + resultFile.getAbsolutePath());
             }
-            logger.info("[File query]- " + queryType + " query result size " + queryResults.size());
+            Double queryTotal = Double.valueOf(readUtf8Lines.size());
+            logger.info("[File query]- " + queryType + " query result size: " + resultSize  +", query size: " + querySize +", process: "+ NumberUtil.decimalFormat("#.##%", querySize/queryTotal));
+            queryResults.clear();
+
         }
-        resultMsg.put("total", total);
+        resultMsg.put("total_export", querySize);
-        logger.info("[File query]-" + srcFile + " Results saved in " + resultFile.getAbsolutePath());
 
         return resultMsg;
     }

src/main/java/com/mesasoft/cn/sketch/util/SktUtil.java

@@ -13,11 +13,13 @@ public class SktUtil {
      * @param request
      * @return
      */
+    private static String defUsername = "default";
+
     public static String getRequestUser(HttpServletRequest request) {
         Object attribute = request.getSession().getAttribute(ValueConsts.USER_STRING);
         User user = null;
         if (attribute == null) {
-            return null;
+            return defUsername;
         } else {
             user = (User) attribute;
         }

src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java

@@ -3,16 +3,10 @@ package com.mesasoft.cn.sketch.util;
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.resource.ResourceUtil;
 import cn.hutool.log.Log;
-import com.mesasoft.cn.sketch.config.AppConfig;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Value;
 import sun.net.util.IPAddressUtil;
 
-import javax.annotation.Resource;
+import java.io.*;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -26,15 +20,17 @@ public class ValidationUtils {
 
     private static String tldFilePath;
 
-//    @Value("${sketch.tld.file}")
+
-//    public void setTldFilePath(String ttlFilePath) {
-//        this.tldFilePath = ttlFilePath;
-//        if (StringUtils.isBlank(ttlFilePath) || !FileUtil.exist(ttlFilePath)) {
-//            this.tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
-//        }
-//    }
     static {
-        tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
+        try {
+            tldFilePath =  new File("").getCanonicalPath()+"/config/public_suffix_list_only.dat";
+        } catch (IOException e) {
+            logger.error(e);
+        }
+
+        if(!FileUtil.exist(tldFilePath)){
+            tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
+        }
     }
     /**
      * 获取二级域名

src/main/java/com/mesasoft/cn/web/controller/UserController.java

@@ -12,6 +12,7 @@ import com.mesasoft.cn.enums.InterceptorLevel;
 import com.mesasoft.cn.modules.constant.ConfigConsts;
 import com.mesasoft.cn.modules.constant.DefaultValues;
 import com.mesasoft.cn.service.IUserService;
+import com.mesasoft.cn.sketch.util.SktUtil;
 import com.mesasoft.cn.util.ControllerUtils;
 import com.zhazhapan.modules.constant.ValueConsts;
 import com.zhazhapan.util.Checker;
@@ -278,4 +279,16 @@ public class UserController {
         }
         return Result.success(resultObject);
     }
+    @ApiOperation(value = "登出")
+    @ApiImplicitParams({@ApiImplicitParam(name = "username", value = "用户名"), @ApiImplicitParam(name = "token", value = "用于自动登录")})
+    @AuthInterceptor(InterceptorLevel.NONE)
+    @RequestMapping(value = "/logout", method = RequestMethod.PUT)
+    public String logOut(HttpServletRequest request, String token) {
+        User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
+        request.getSession().removeAttribute(ValueConsts.USER_STRING);
+        TokenConfig.removeTokenByValue(user.getId());
+
+        jsonObject.put("status", "success");
+        return jsonObject.toString();
+    }
 }

src/main/resources/application-sketch.properties

@@ -1,5 +1,5 @@
 #datasource
-spring.datasource.url=jdbc:mysql://192.168.44.12:3306/efo?useUnicode=true&characterEncoding=utf-8&useSSL=true
+spring.datasource.url=jdbc:mysql://192.168.45.102:3306/web_sketch_v2?useUnicode=true&characterEncoding=utf-8&useSSL=true
 spring.datasource.username=root
 spring.datasource.password=galaxy2019
 
@@ -7,7 +7,7 @@ spring.datasource.password=galaxy2019
 query.output.dir = c:\\test\\output_file
 
 # offline读取批处理量
-query.readin.batch = 1
+query.readin.batch = 10000
 
 # 打印进度日志的查询条数
 query.log.file.line.interval = 10000
@@ -17,6 +17,7 @@ database = web_sketch_v2
 db.query.batch.size = 10000
 
 ###################### api #########################
+query.isLocal=true
 ##### bright cloud #######
 bright-cloud.oemid = GeedgeNet
 bright-cloud.deviceid = TSG-Dev
@@ -45,5 +46,5 @@ chinaz.usereport-filepath = chinaz_query_count.csv
 #顶级域名
 sketch.tld.file = public_suffix_list_only.dat
 
-sketch.home.path=C:\\test
+sketch.home.path=/home/test/sketch
 

src/main/resources/application.properties

@@ -9,10 +9,10 @@ spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
 spring.datasource.driverClassName=com.mysql.jdbc.Driver
 spring.datasource.druid.filters=stat,wall,slf4j,config
 spring.datasource.druid.maxActive=100
-spring.datasource.druid.initialSize=1
+spring.datasource.druid.initialSize=5
-spring.datasource.druid.maxWait=60000
+spring.datasource.druid.maxWait=300000
-spring.datasource.druid.minIdle=1
+spring.datasource.druid.minIdle=5
-spring.datasource.druid.timeBetweenEvictionRunsMillis=60000
+spring.datasource.druid.timeBetweenEvictionRunsMillis=300000
 spring.datasource.druid.minEvictableIdleTimeMillis=300000
 spring.datasource.druid.validationQuery=select 'x'
 spring.datasource.druid.testWhileIdle=true
@@ -23,18 +23,21 @@ spring.datasource.druid.maxOpenPreparedStatements=50
 spring.datasource.druid.maxPoolPreparedStatementPerConnectionSize=20
 spring.datasource.druid.web-stat-filter.enabled=true
 spring.datasource.druid.stat-view-servlet.enabled=true
+spring.datasource.druid.stat-view-servlet.url-pattern=/druid/*
 spring.datasource.druid.stat-view-servlet.login-username=test
 spring.datasource.druid.stat-view-servlet.login-password=test
 spring.datasource.druid.stat-view-servlet.allow=127.0.0.1
 spring.datasource.druid.stat-view-servlet.deny=
+spring.datasource.druid.keep-alive=true
+spring.datasource.druid.keep-alive-between-time-millis=3600000
 
 spring.jpa.show-sql=true
 spring.servlet.multipart.max-file-size=1099511627776
 spring.servlet.multipart.max-request-size=1099511627776
 #热部署生效
-spring.devtools.restart.enabled=true
+spring.devtools.restart.enabled=false
 #页面热部署
-spring.thymeleaf.cache=false
+spring.thymeleaf.cache=true
 spring.devtools.restart.additional-paths=src/main
 #classpath目录下的WEB-INF文件夹内容修改不重启
 spring.devtools.restart.exclude=WEB-INF/**

src/main/resources/assets/js/admin.js

@@ -761,4 +761,5 @@ $(document).ready(function () {
     $(".file-delete").click(function () {
         doDelete("#file-manager-table", app.files, ".file-index", "/file/batch/");
     });
+
 });

src/main/resources/assets/js/default.js

@@ -70,7 +70,7 @@ $(document).ready(function () {
                 }
                 globalConfig.background.imageList = list;
             }
-            changeBackgroundImage();
+            // changeBackgroundImage();
         }
         setCSS();
     });

src/main/resources/assets/js/index.js

@@ -622,3 +622,18 @@ function fileDownLoad(path,name) {
     // 发送ajax请求
     xhr.send()
 }
+//退出登录
+function logout() {
+    layer.load(1);
+    $.ajax({
+        url: "/user/logout", type: "PUT", data: {
+            token: getCookie("token")
+        }, success: function (data) {
+            layer.closeAll();
+            var json = JSON.parse(data);
+            if (json.status === "success") {
+                window.location.href = "/signin";
+            }
+        }
+    });
+}

src/main/resources/config.json

@@ -50,20 +50,18 @@
             }
         ],
         "background": {
-            "useImage": true,
+            "useImage": false,
             "imageIndex": 0,
             "imageList": [
-                "https://fakedog.xyz/homedb/srcm/upload/20180123/bg_1.jpg",
+                "assets/img/background_1.png"
-                "https://fakedog.xyz/homedb/srcm/upload/20180123/bg_2.jpg",
-                "https://fakedog.xyz/homedb/srcm/upload/20180123/bg_3.jpg"
             ],
             "random": true,
             "listGenerator": {
-                "enable": true,
+                "enable": false,
-                "prefix": "https://fakedog.xyz/image/background/anime/bg_",
+                "prefix": "assets/img/background_1",
-                "suffix": ".jpg",
+                "suffix": ".png",
                 "start": 1,
-                "end": 158
+                "end": 1
             }
         }
     },

src/main/resources/mapper/DomainDao.xml

@@ -109,7 +109,6 @@
                 #{listItem}
             </foreach>
         </where>
-        limit 10
     </select>
 
     <!-- category info   -->
@@ -123,7 +122,6 @@
                 #{listItem}
             </foreach>
         </where>
-        limit 10
     </select>
     <!-- ICP info   -->
     <select id="getICPInfo" resultMap="ICPResultMap">
@@ -136,7 +134,6 @@
                 #{listItem}
             </foreach>
         </where>
-        limit 10
     </select>
 
     <!-- category insert   -->

src/main/resources/static/index.html

@@ -61,6 +61,10 @@
         <input class="form-control border-dark bg-dark text-white form-control-sm w-100" type="search"
                placeholder="搜索" id="search"/>
     </div>-->
+    <div>
+<!--        <p id="logout" class="text-light" >退出</p>-->
+        <a class="nav-link text-muted" href="javascript:;" onclick="logout();">退出</a>
+    </div>
 </nav>
 <br/><br/><br/>
 <div class="container" id="index" style="max-width: 98%" >