域名相关表去掉is_accessible字段

2023-02-14 09:55:14 +08:00
parent 2a54823160
commit dbc926cbd2
20 changed files with 90 additions and 69 deletions
--- a/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java
+++ b/src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java
@@ -14,8 +14,6 @@ import org.apache.http.client.methods.HttpPost;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.message.BasicNameValuePair;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;

 import java.io.BufferedReader;
 import java.io.InputStream;
--- a/src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java
+++ b/src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java
@@ -1,11 +1,8 @@
 package com.mesasoft.cn.sketch.config;


-import cn.hutool.core.io.FileUtil;
-import cn.hutool.core.io.resource.ResourceUtil;
 import com.mesasoft.cn.sketch.util.ConfigUtils;
 import lombok.Data;
-import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;

--- a/src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java
+++ b/src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java
@@ -170,9 +170,9 @@ public class DomainController {
     * 查询大量数据
     */
    @GetMapping(value = "/getdomaininfobyfile")
-    public ReturnT getDomainInfoByFile(HttpServletRequest request, String srcFile, String queryType, SketchAuth sketchAuth) {
+    public ReturnT getDomainInfoByFile(HttpServletRequest request, String srcFile, String queryType, SketchAuth sketchAuth,Boolean export) {
        try {
-            Map domainInfoByFile = domainService.getDomainInfoByFile(SktUtil.getRequestUser(request), srcFile, queryType);
+            Map domainInfoByFile = domainService.getDomainInfoByFile(SktUtil.getRequestUser(request), srcFile, queryType,export);
            return ReturnT.succeed(sketchAuth.getRequestid(), domainInfoByFile);
        } catch (Exception e) {
            logger.error(e);
--- a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java
+++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java
@@ -1,10 +1,7 @@
 package com.mesasoft.cn.sketch.entity.domain;

-import cn.hutool.core.date.DateUtil;
-import com.fasterxml.jackson.annotation.JsonFormat;
 import lombok.Data;
 import lombok.ToString;
-import org.springframework.format.annotation.DateTimeFormat;

 import java.io.Serializable;
 import java.util.Date;
--- a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainICP.java
+++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainICP.java
@@ -1,6 +1,5 @@
 package com.mesasoft.cn.sketch.entity.domain;

-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.mesasoft.cn.sketch.util.ValidationUtils;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -13,7 +12,6 @@ import lombok.ToString;
@ToString
 public class DomainICP extends DomainBase {

-    private Boolean is_accessible;
    private String icp_owner;
    private String icp_company_name;
    private String icp_company_type;
@@ -23,7 +21,7 @@ public class DomainICP extends DomainBase {
    private String icp_verify_time;


-    public DomainICP(String fqdn, String source, Integer match_pattern, Boolean query_success,Boolean is_accessible, String icp_owner, String icp_company_name,
+    public DomainICP(String fqdn, String source, Integer match_pattern, Boolean query_success, String icp_owner, String icp_company_name,
                     String icp_company_type,String icp_main_page, String icp_site_license, String icp_site_name, String icp_verify_time) {
        this.fqdn = fqdn;
        this.source = source;
@@ -35,7 +33,6 @@ public class DomainICP extends DomainBase {
            this.match_pattern = match_pattern;
        }
        this.query_success = query_success;
-        this.is_accessible = is_accessible;

        this.icp_owner = icp_owner;
        this.icp_company_name = icp_company_name;
--- a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java
+++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java
@@ -1,6 +1,5 @@
 package com.mesasoft.cn.sketch.entity.domain;

-import com.alibaba.fastjson.JSONObject;
 import com.mesasoft.cn.sketch.util.ConfigUtils;
 import com.mesasoft.cn.sketch.util.ValidationUtils;
 import lombok.AllArgsConstructor;
@@ -9,11 +8,7 @@ import lombok.NoArgsConstructor;
 import lombok.ToString;

 import java.sql.Date;
-import java.sql.ResultSet;
-import java.sql.SQLException;
 import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.List;

 /**
 * Created with IntelliJ IDEA.
--- a/src/main/java/com/mesasoft/cn/sketch/service/DomainService.java
+++ b/src/main/java/com/mesasoft/cn/sketch/service/DomainService.java
@@ -23,7 +23,7 @@ public interface DomainService {
    List<DomainInfo> getICPInfoByFile(MultipartHttpServletRequest request) throws Exception;


-    Map getDomainInfoByFile(String requestUser, String srcFile, String queryType) throws Exception;
+    Map getDomainInfoByFile(String requestUser, String srcFile, String queryType,Boolean export) throws Exception;



--- a/src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java
+++ b/src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java
@@ -3,6 +3,7 @@ package com.mesasoft.cn.sketch.service.impl;
 import cn.hutool.core.collection.ListUtil;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.util.NumberUtil;
 import cn.hutool.core.util.ObjectUtil;
 import cn.hutool.log.Log;
 import com.alibaba.fastjson.JSON;
@@ -62,13 +63,14 @@ public class DomainServiceImpl implements DomainService {
    @Value("${query.readin.batch}")
    private int queryReadinBatch;

-    private final boolean isLocal = true;
+    @Value("${query.isLocal}")
+    private boolean isLocal;

    @Override
    public List<DomainInfo> getCategoryInfo(List<String> domains, String username) throws Exception {

        domains = ValidationUtils.getCheckedFqdns(domains);
-        logger.info("domain category query total : {}", domains.size());
+        logger.info("domain category query distinct total : {}", domains.size());
        int queryNum = domains.size();
        List<DomainInfo> results = new ArrayList<DomainInfo>();
        //1. 查询本地数据库
@@ -298,22 +300,24 @@ public class DomainServiceImpl implements DomainService {
    }

    @Override
-    public Map getDomainInfoByFile(String requestUser, String srcFile, String queryType) throws Exception {
+    public Map getDomainInfoByFile(String requestUser, String srcFile, String queryType, Boolean export) throws Exception {
        //输出文件
        String fileName = String.join("-", FileUtil.getPrefix(srcFile), queryType, DateUtil.format(new Date(), "yyyyMMdd-HHmmss") + ".json");
        File resultFile = FileUtil.touch(queryOutputDir + File.separator + fileName);

-        cn.hutool.core.io.file.FileWriter fileWriter = new cn.hutool.core.io.file.FileWriter(resultFile);
        Map<String, Object> resultMsg = new HashMap<>();
        resultMsg.put("result", resultFile.toString());
+        resultMsg.put("export", export);

        // 文件读取
        List<String> readUtf8Lines = FileUtil.readUtf8Lines(srcFile);
        logger.info("[File query]-" + srcFile + " ,queryType " + queryType + " query read in batch " + queryReadinBatch + ",domains size " + readUtf8Lines.size());
        List<List<String>> partition = ListUtil.partition(readUtf8Lines, queryReadinBatch);
-        int total = 0;
+        int resultSize = 0;
+        int querySize = 0;
+        List<DomainInfo> queryResults = new ArrayList<>();
+        cn.hutool.core.io.file.FileWriter fileWriter = new cn.hutool.core.io.file.FileWriter(resultFile);
        for (List<String> domains : partition) {
-            List<DomainInfo> queryResults = new ArrayList<>();
            switch (queryType) {
                case "category":
                    queryResults = getCategoryInfo(domains, requestUser);
@@ -325,17 +329,20 @@ public class DomainServiceImpl implements DomainService {
                    queryResults = getICPInfo(domains);
                    break;
            }
-
-            if (ObjectUtil.isNotEmpty(queryResults)) {
-                total += queryResults.size();
+            querySize +=domains.size();
+            resultSize += queryResults.size();
+            if (ObjectUtil.isNotEmpty(queryResults) && export) {
                List<String> prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x)).collect(Collectors.toList());
                //List<String> prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x, true)).collect(Collectors.toList());
                fileWriter.appendLines(prettyResults);
+                logger.info("[File query]-" + srcFile +" result: " + queryResults.size() + " Results saved in " + resultFile.getAbsolutePath());
            }
-            logger.info("[File query]- " + queryType + " query result size " + queryResults.size());
+            Double queryTotal = Double.valueOf(readUtf8Lines.size());
+            logger.info("[File query]- " + queryType + " query result size: " + resultSize  +", query size: " + querySize +", process: "+ NumberUtil.decimalFormat("#.##%", querySize/queryTotal));
+            queryResults.clear();
+
        }
-        resultMsg.put("total", total);
-        logger.info("[File query]-" + srcFile + " Results saved in " + resultFile.getAbsolutePath());
+        resultMsg.put("total_export", querySize);

        return resultMsg;
    }
--- a/src/main/java/com/mesasoft/cn/sketch/util/SktUtil.java
+++ b/src/main/java/com/mesasoft/cn/sketch/util/SktUtil.java
@@ -13,11 +13,13 @@ public class SktUtil {
     * @param request
     * @return
     */
+    private static String defUsername = "default";
+
    public static String getRequestUser(HttpServletRequest request) {
        Object attribute = request.getSession().getAttribute(ValueConsts.USER_STRING);
        User user = null;
        if (attribute == null) {
-            return null;
+            return defUsername;
        } else {
            user = (User) attribute;
        }
--- a/src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java
+++ b/src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java
@@ -3,16 +3,10 @@ package com.mesasoft.cn.sketch.util;
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.resource.ResourceUtil;
 import cn.hutool.log.Log;
-import com.mesasoft.cn.sketch.config.AppConfig;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.beans.factory.annotation.Value;
 import sun.net.util.IPAddressUtil;

-import javax.annotation.Resource;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
+import java.io.*;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -26,15 +20,17 @@ public class ValidationUtils {

    private static String tldFilePath;

-//    @Value("${sketch.tld.file}")
-//    public void setTldFilePath(String ttlFilePath) {
-//        this.tldFilePath = ttlFilePath;
-//        if (StringUtils.isBlank(ttlFilePath) || !FileUtil.exist(ttlFilePath)) {
-//            this.tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
-//        }
-//    }
+
    static {
-        tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
+        try {
+            tldFilePath =  new File("").getCanonicalPath()+"/config/public_suffix_list_only.dat";
+        } catch (IOException e) {
+            logger.error(e);
+        }
+
+        if(!FileUtil.exist(tldFilePath)){
+            tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
+        }
    }
    /**
     * 获取二级域名
--- a/src/main/java/com/mesasoft/cn/web/controller/UserController.java
+++ b/src/main/java/com/mesasoft/cn/web/controller/UserController.java
@@ -12,6 +12,7 @@ import com.mesasoft.cn.enums.InterceptorLevel;
 import com.mesasoft.cn.modules.constant.ConfigConsts;
 import com.mesasoft.cn.modules.constant.DefaultValues;
 import com.mesasoft.cn.service.IUserService;
+import com.mesasoft.cn.sketch.util.SktUtil;
 import com.mesasoft.cn.util.ControllerUtils;
 import com.zhazhapan.modules.constant.ValueConsts;
 import com.zhazhapan.util.Checker;
@@ -278,4 +279,16 @@ public class UserController {
        }
        return Result.success(resultObject);
    }
+    @ApiOperation(value = "登出")
+    @ApiImplicitParams({@ApiImplicitParam(name = "username", value = "用户名"), @ApiImplicitParam(name = "token", value = "用于自动登录")})
+    @AuthInterceptor(InterceptorLevel.NONE)
+    @RequestMapping(value = "/logout", method = RequestMethod.PUT)
+    public String logOut(HttpServletRequest request, String token) {
+        User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
+        request.getSession().removeAttribute(ValueConsts.USER_STRING);
+        TokenConfig.removeTokenByValue(user.getId());
+
+        jsonObject.put("status", "success");
+        return jsonObject.toString();
+    }
 }
--- a/src/main/resources/application-sketch.properties
+++ b/src/main/resources/application-sketch.properties
@@ -1,5 +1,5 @@
 #datasource
-spring.datasource.url=jdbc:mysql://192.168.44.12:3306/efo?useUnicode=true&characterEncoding=utf-8&useSSL=true
+spring.datasource.url=jdbc:mysql://192.168.45.102:3306/web_sketch_v2?useUnicode=true&characterEncoding=utf-8&useSSL=true
 spring.datasource.username=root
 spring.datasource.password=galaxy2019

@@ -7,7 +7,7 @@ spring.datasource.password=galaxy2019
 query.output.dir = c:\\test\\output_file

 # offline读取批处理量
-query.readin.batch = 1
+query.readin.batch = 10000

 # 打印进度日志的查询条数
 query.log.file.line.interval = 10000
@@ -17,6 +17,7 @@ database = web_sketch_v2
 db.query.batch.size = 10000

 ###################### api #########################
+query.isLocal=true
 ##### bright cloud #######
 bright-cloud.oemid = GeedgeNet
 bright-cloud.deviceid = TSG-Dev
@@ -45,5 +46,5 @@ chinaz.usereport-filepath = chinaz_query_count.csv
 #顶级域名
 sketch.tld.file = public_suffix_list_only.dat

-sketch.home.path=C:\\test
+sketch.home.path=/home/test/sketch

--- a/src/main/resources/application.properties
+++ b/src/main/resources/application.properties
@@ -9,10 +9,10 @@ spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
 spring.datasource.driverClassName=com.mysql.jdbc.Driver
 spring.datasource.druid.filters=stat,wall,slf4j,config
 spring.datasource.druid.maxActive=100
-spring.datasource.druid.initialSize=1
-spring.datasource.druid.maxWait=60000
-spring.datasource.druid.minIdle=1
-spring.datasource.druid.timeBetweenEvictionRunsMillis=60000
+spring.datasource.druid.initialSize=5
+spring.datasource.druid.maxWait=300000
+spring.datasource.druid.minIdle=5
+spring.datasource.druid.timeBetweenEvictionRunsMillis=300000
 spring.datasource.druid.minEvictableIdleTimeMillis=300000
 spring.datasource.druid.validationQuery=select 'x'
 spring.datasource.druid.testWhileIdle=true
@@ -23,18 +23,21 @@ spring.datasource.druid.maxOpenPreparedStatements=50
 spring.datasource.druid.maxPoolPreparedStatementPerConnectionSize=20
 spring.datasource.druid.web-stat-filter.enabled=true
 spring.datasource.druid.stat-view-servlet.enabled=true
+spring.datasource.druid.stat-view-servlet.url-pattern=/druid/*
 spring.datasource.druid.stat-view-servlet.login-username=test
 spring.datasource.druid.stat-view-servlet.login-password=test
 spring.datasource.druid.stat-view-servlet.allow=127.0.0.1
 spring.datasource.druid.stat-view-servlet.deny=
+spring.datasource.druid.keep-alive=true
+spring.datasource.druid.keep-alive-between-time-millis=3600000

 spring.jpa.show-sql=true
 spring.servlet.multipart.max-file-size=1099511627776
 spring.servlet.multipart.max-request-size=1099511627776
 #热部署生效
-spring.devtools.restart.enabled=true
+spring.devtools.restart.enabled=false
 #页面热部署
-spring.thymeleaf.cache=false
+spring.thymeleaf.cache=true
 spring.devtools.restart.additional-paths=src/main
 #classpath目录下的WEB-INF文件夹内容修改不重启
 spring.devtools.restart.exclude=WEB-INF/**
--- a/src/main/resources/assets/img/background_1.png
+++ b/src/main/resources/assets/img/background_1.png
--- a/src/main/resources/assets/js/admin.js
+++ b/src/main/resources/assets/js/admin.js
@@ -761,4 +761,5 @@ $(document).ready(function () {
    $(".file-delete").click(function () {
        doDelete("#file-manager-table", app.files, ".file-index", "/file/batch/");
    });
+
 });
--- a/src/main/resources/assets/js/default.js
+++ b/src/main/resources/assets/js/default.js
@@ -70,7 +70,7 @@ $(document).ready(function () {
                }
                globalConfig.background.imageList = list;
            }
-            changeBackgroundImage();
+            // changeBackgroundImage();
        }
        setCSS();
    });
--- a/src/main/resources/assets/js/index.js
+++ b/src/main/resources/assets/js/index.js
@@ -622,3 +622,18 @@ function fileDownLoad(path,name) {
    // 发送ajax请求
    xhr.send()
 }
+//退出登录
+function logout() {
+    layer.load(1);
+    $.ajax({
+        url: "/user/logout", type: "PUT", data: {
+            token: getCookie("token")
+        }, success: function (data) {
+            layer.closeAll();
+            var json = JSON.parse(data);
+            if (json.status === "success") {
+                window.location.href = "/signin";
+            }
+        }
+    });
+}
--- a/src/main/resources/config.json
+++ b/src/main/resources/config.json
@@ -50,20 +50,18 @@
            }
        ],
        "background": {
-            "useImage": true,
+            "useImage": false,
            "imageIndex": 0,
            "imageList": [
-                "https://fakedog.xyz/homedb/srcm/upload/20180123/bg_1.jpg",
-                "https://fakedog.xyz/homedb/srcm/upload/20180123/bg_2.jpg",
-                "https://fakedog.xyz/homedb/srcm/upload/20180123/bg_3.jpg"
+                "assets/img/background_1.png"
            ],
            "random": true,
            "listGenerator": {
-                "enable": true,
-                "prefix": "https://fakedog.xyz/image/background/anime/bg_",
-                "suffix": ".jpg",
+                "enable": false,
+                "prefix": "assets/img/background_1",
+                "suffix": ".png",
                "start": 1,
-                "end": 158
+                "end": 1
            }
        }
    },
--- a/src/main/resources/mapper/DomainDao.xml
+++ b/src/main/resources/mapper/DomainDao.xml
@@ -109,7 +109,6 @@
                #{listItem}
            </foreach>
        </where>
-        limit 10
    </select>

    <!-- category info   -->
@@ -123,7 +122,6 @@
                #{listItem}
            </foreach>
        </where>
-        limit 10
    </select>
    <!-- ICP info   -->
    <select id="getICPInfo" resultMap="ICPResultMap">
@@ -136,7 +134,6 @@
                #{listItem}
            </foreach>
        </where>
-        limit 10
    </select>

    <!-- category insert   -->
--- a/src/main/resources/static/index.html
+++ b/src/main/resources/static/index.html
@@ -61,6 +61,10 @@
        <input class="form-control border-dark bg-dark text-white form-control-sm w-100" type="search"
               placeholder="搜索" id="search"/>
    </div>-->
+    <div>
+<!--        <p id="logout" class="text-light" >退出</p>-->
+        <a class="nav-link text-muted" href="javascript:;" onclick="logout();">退出</a>
+    </div>
 </nav>
 <br/><br/><br/>
 <div class="container" id="index" style="max-width: 98%" >