gfwleak/web-sketch-webskt-query-agent
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加ICP调用第三方API chinaz 查询并存储结果到数据库，批量查询，文件查询，文件查询导出 CN-664

Browse Source
This commit is contained in:
zhanghongqing
2023-01-29 16:36:18 +08:00
parent bdf48f2b4f
commit 2a54823160
42 changed files with 43588 additions and 1350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java
View File

@@ -1,10 +1,14 @@
package com.mesasoft.cn.sketch.util;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.resource.ResourceUtil;
import cn.hutool.log.Log;
import com.mesasoft.cn.sketch.config.AppConfig;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import sun.net.util.IPAddressUtil;
import javax.annotation.Resource;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -16,20 +20,26 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ValidationUtils {
private static final Logger LOG = Logger.getLogger(ValidationUtils.class);
private static final Log logger = Log.get();
/**
* 获取二级域名
**/
private static String tldFilePath;
@Value("${sketch.tld.file}")
public void setTldFilePath(String ttlFilePath){
this.tldFilePath= ttlFilePath;
// @Value("${sketch.tld.file}")
// public void setTldFilePath(String ttlFilePath) {
// this.tldFilePath = ttlFilePath;
// if (StringUtils.isBlank(ttlFilePath) || !FileUtil.exist(ttlFilePath)) {
// this.tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
// }
// }
static {
tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
}
public static String getSecDomain(String fqdnOrUrl){
/**
* 获取二级域名
**/
public static String getSecDomain(String fqdnOrUrl) {
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(tldFilePath);
try {
@@ -56,13 +66,12 @@ public class ValidationUtils {
}
}
// 右匹配为顶级域名
if (secDomain == null){
if (secDomain == null) {
secDomain = fqdnOrUrl;
}
return secDomain;
} catch (Exception e) {
LOG.error("urlDomain:" + fqdnOrUrl);
e.printStackTrace();
logger.error("urlDomain:" + fqdnOrUrl);
return "---no---return---";
}
}
@@ -75,7 +84,7 @@ public class ValidationUtils {
if (StringUtils.isNotBlank(secDomain) && !("---no---return---".equals(secDomain))) {
secDomainList.add(secDomain);
} else {
System.out.println(oriDomain);
logger.info(oriDomain);
}
}
return secDomainList;
@@ -95,8 +104,7 @@ public class ValidationUtils {
String encoding = "UTF-8";
File file = new File(filePath);
if (file.isFile() && file.exists()) {
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null) {
@@ -113,11 +121,10 @@ public class ValidationUtils {
}
read.close();
} else {
LOG.error("TopDomainUtils>=>readTopDomainFile filePath is wrong--->{" + filePath + "}<---");
logger.error("TopDomainUtils>=>readTopDomainFile filePath is wrong--->{" + filePath + "}<---");
}
} catch (Exception e) {
LOG.error("TopDomainUtils>=>readTopDomainFile get filePathData error--->{" + e + "}<---");
e.printStackTrace();
logger.error("TopDomainUtils>=>readTopDomainFile get filePathData error--->{" + e + "}<---");
}
return maps;
}
@@ -132,55 +139,54 @@ public class ValidationUtils {
return maps;
}
public static List<String> getChecked(List<String> objectList, String type){
if (type.equals("ip")){
public static List<String> getChecked(List<String> objectList, String type) {
if (type.equals("ip")) {
return getCheckedIps(objectList);
}
if (type.equals("domain")){
if (type.equals("domain")) {
return getCheckedFqdns(objectList);
}
LOG.error("Wrong type to be checked: " + type);
logger.error("Wrong type to be checked: " + type);
return objectList;
}
public static List<String> getCheckedFqdns(List<String> fqdns){
public static List<String> getCheckedFqdns(List<String> fqdns) {
List<String> res = new ArrayList<>();
for (String fqdn:fqdns){
for (String fqdn : fqdns) {
//去端口号
fqdn = fqdn.split(":")[0];
// 去重 & 校验
if (isValidDomain(fqdn) && !res.contains(fqdn)){
if (isValidDomain(fqdn) && !res.contains(fqdn)) {
res.add(fqdn.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + fqdn);
logger.debug("Bad or duplicated fqdn:" + fqdn);
}
}
return res;
}
public static List<String> getCheckedIps(List<String> ipList){
public static List<String> getCheckedIps(List<String> ipList) {
List<String> res = new ArrayList<>();
for (String ip:ipList){
for (String ip : ipList) {
//去端口号
ip = ip.split(":")[0];
// 去重 & 校验
if (isValidIp(ip) && !res.contains(ip)){
if (isValidIp(ip) && !res.contains(ip)) {
res.add(ip.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + ip);
logger.debug("Bad or duplicated fqdn:" + ip);
}
}
return res;
}
public static boolean isValidIp(String ip){
public static boolean isValidIp(String ip) {
boolean iPv4LiteralAddress = IPAddressUtil.isIPv4LiteralAddress(ip);
boolean iPv6LiteralAddress = IPAddressUtil.isIPv6LiteralAddress(ip);
return iPv4LiteralAddress || iPv6LiteralAddress;
}
private static boolean isValidDomain(String str)
{
private static boolean isValidDomain(String str) {
String regex = "^((?!-)[A-Za-z0-9-_]"
+ "{1,63}(?<!-)\\.)"
+ "+[A-Za-z]{2,6}";
@@ -193,15 +199,13 @@ public class ValidationUtils {
return m.matches();
}
public static Integer getMatchPattern(String fqdn){
public static Integer getMatchPattern(String fqdn) {
int match_pattern = 2;
if (fqdn.equals(getSecDomain(fqdn))){
if (fqdn.equals(getSecDomain(fqdn))) {
match_pattern = 1; // 二级域名-右匹配
}
return match_pattern;
}
}