gfwleak/web-sketch-webskt-query-agent
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

merge into: gfwleak:main
Branches Tags
gfwleak:main gfwleak:dev
...
pull from: gfwleak:dev
Branches Tags
gfwleak:dev gfwleak:main

8 Commits
main ... dev

Author SHA1 Message Date
zhanghongqing
013c07192d 更新categoryinfo.json配置，取消对catid的限制 2023-11-03 16:12:55 +08:00
zhanghongqing
288ac70596 域名查询增加查询模式参数query.mode=2 
#update 2 第三方api优先，并更新数据库
#local 1 只查db
#general 0 db优先，不存在则查第三方api
 2023-08-04 18:50:41 +08:00
zhanghongqing
b9ae09c7d6 修复域名查询bright cloud时cats信息为空报错 2023-05-08 14:52:09 +08:00
zhanghongqing
d6d507b3b6 界面调整，隐藏冗余功能 2023-02-20 15:30:13 +08:00
zhanghongqing
dbc926cbd2 域名相关表去掉is_accessible字段 2023-02-14 09:55:14 +08:00
zhanghongqing
2a54823160 增加ICP调用第三方API chinaz 查询并存储结果到数据库，批量查询，文件查询，文件查询导出 CN-664 2023-01-29 16:36:18 +08:00
zhanghongqing
bdf48f2b4f web sketch-代码优化域名查询 CN-663 2023-01-10 16:02:56 +08:00
zhanghongqing
c125bb89cf web sketch-查询代理功能优化 CN-663 2023-01-10 15:51:02 +08:00
95 changed files with 46253 additions and 1968 deletions
Expand all files Collapse all files

16
README.md
View File

@@ -1,9 +1,5 @@
### 关于项目
- 项目名称 `efo` 是 `Easy File Online` 的缩写，字面意思就是让您轻松实现线上文件管理
- 本系统具有文件分享的功能，权限控制和自定义配置都很强大（可能还不完善）
- 系统后端框架有Spring Boot，Spring， SpringMVC，MyBatis; 前段框架有Bootstrap，Jquery, Layer, Vue。项目完全纯注解，零XML配置。
> 第一次运行系统，请先运行 [SQL代码](/mysql/efo.sql) , 并登陆系统修改用户 `system` （默认密码 `123456`）的密码
@@ -16,26 +12,24 @@
### 系统部分截图（背景图片可通过配置设置）
- 登录页面（包含登录、注册、密码重置），路径 `/signin`
- 登录页面（包含登录、注册、密码重置），路径 `/signin`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/signin.png)
- 资源首页，路径 `/index`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/index.png)
- 上传页面，路径 `/upload`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/upload.png)
- 管理员管理页面，路径 `/admin`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/admin.png)
- 远程文件管理（管理服务器端所有文件，只有系统用户才能进入此页面），路径 `/filemanager`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/filemanager.png)
> 此功能基于 [angular-filamanager](https://github.com/joni2back/angular-filemanager) 实现
**项目有不足的地方欢迎提出来哦，大家一起交流学习，觉得不错的话，Star来一个呗**

28
pom.xml
View File

@@ -4,11 +4,11 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.mesasoft.cn</groupId>
<artifactId>webskt-query-agent</artifactId>
<version>1.0</version>
<artifactId>webskt</artifactId>
<version>1.2</version>
<packaging>jar</packaging>
<name>webskt-query-agent</name>
<name>webskt</name>
<description>简单的线上文件管理系统</description>
<parent>
@@ -76,6 +76,7 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>mysql</groupId>
@@ -117,14 +118,27 @@
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.0.9</version>
<artifactId>druid-spring-boot-starter</artifactId>
<version>1.2.6</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<!-- websketch-->
<dependency>
<groupId>me.geso</groupId>
<artifactId>jdbcutils</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.7.17</version>
</dependency>
</dependencies>
<build>
@@ -134,7 +148,7 @@
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<fork>true</fork>
<mainClass>com.mesasoft.cn.SketchApplication</mainClass>
<mainClass>com.mesasoft.cn.WebSketchApplication</mainClass>
</configuration>
<executions>
<execution>
@@ -151,7 +165,7 @@
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>com.mesasoft.cn.SketchApplication</mainClass>
<mainClass>com.mesasoft.cn.WebSketchApplication</mainClass>
</manifest>
</archive>
<descriptorRefs>

12
src/main/java/com/mesasoft/cn/SketchApplication.java → src/main/java/com/mesasoft/cn/WebSketchApplication.java
View File

@@ -1,10 +1,10 @@
package com.mesasoft.cn;
import com.spring4all.swagger.EnableSwagger2Doc;
import com.zhazhapan.config.JsonParser;
import com.mesasoft.cn.config.TokenConfig;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.spring4all.swagger.EnableSwagger2Doc;
import com.zhazhapan.config.JsonParser;
import com.zhazhapan.util.FileExecutor;
import com.zhazhapan.util.MailSender;
import com.zhazhapan.util.ReflectUtils;
@@ -22,9 +22,9 @@ import java.util.List;
*/
@SpringBootApplication
@EnableSwagger2Doc
@MapperScan("com.mesasoft.cn.dao")
@MapperScan({"com.mesasoft.cn.dao","com.mesasoft.cn.sketch.dao"})
@EnableTransactionManagement
public class SketchApplication {
public class WebSketchApplication {
public static JsonParser settings = new JsonParser();
@@ -33,10 +33,10 @@ public class SketchApplication {
public static Hashtable<String, Integer> tokens;
public static void main(String[] args) throws IOException, ClassNotFoundException {
settings.setJsonObject(FileExecutor.read(SketchApplication.class.getResourceAsStream(DefaultValues.SETTING_PATH)));
settings.setJsonObject(FileExecutor.read(WebSketchApplication.class.getResourceAsStream(DefaultValues.SETTING_PATH)));
MailSender.config(settings.getObjectUseEval(ConfigConsts.EMAIL_CONFIG_OF_SETTINGS));
controllers = ReflectUtils.getClasses(DefaultValues.CONTROLLER_PACKAGE);
tokens = TokenConfig.loadToken();
SpringApplication.run(SketchApplication.class, args);
SpringApplication.run(WebSketchApplication.class, args);
}
}

10
src/main/java/com/mesasoft/cn/config/SettingConfig.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.config;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.util.CommonUtils;
import com.zhazhapan.modules.constant.ValueConsts;
@@ -12,6 +12,8 @@ import org.slf4j.LoggerFactory;
import java.util.Date;
import static com.mesasoft.cn.WebSketchApplication.settings;
/**
* @author pantao
* @since 2018/1/26
@@ -42,14 +44,14 @@ public class SettingConfig {
int[] auth = new int[5];
for (int i = 0; i < ConfigConsts.AUTH_OF_SETTINGS.length; i++) {
String key = jsonPath + ValueConsts.DOT_SIGN + ConfigConsts.AUTH_OF_SETTINGS[i];
auth[i] = SketchApplication.settings.getBooleanUseEval(key) ? 1 : 0;
auth[i] = settings.getBooleanUseEval(key) ? 1 : 0;
}
return auth;
}
public static String getUploadStoragePath() {
String parent = getStoragePath(ConfigConsts.UPLOAD_PATH_OF_SETTING);
String formatWay = SketchApplication.settings.getStringUseEval(ConfigConsts.UPLOAD_FORM_OF_SETTING);
String formatWay = WebSketchApplication.settings.getStringUseEval(ConfigConsts.UPLOAD_FORM_OF_SETTING);
String childPath = ValueConsts.SEPARATOR + Formatter.datetimeToCustomString(new Date(), formatWay);
String path = parent + childPath;
if (!FileExecutor.createFolder(path)) {
@@ -75,7 +77,7 @@ public class SettingConfig {
} else {
path += LINUX;
}
return CommonUtils.checkPath(SketchApplication.settings.getStringUseEval(path));
return CommonUtils.checkPath(WebSketchApplication.settings.getStringUseEval(path));
}
/**

14
src/main/java/com/mesasoft/cn/config/TokenConfig.java
View File

@@ -3,7 +3,7 @@ package com.mesasoft.cn.config;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
@@ -25,20 +25,20 @@ public class TokenConfig {
public static String generateToken(String token, int userId) {
if (Checker.isNotEmpty(token)) {
SketchApplication.tokens.remove(token);
WebSketchApplication.tokens.remove(token);
}
return generateToken(userId);
}
public static String generateToken(int userId) {
String token = RandomUtils.getRandomStringOnlyLetter(ValueConsts.THIRTY_TWO_INT);
SketchApplication.tokens.put(token, userId);
WebSketchApplication.tokens.put(token, userId);
saveToken();
return token;
}
public static void saveToken() {
String tokens = Formatter.mapToJson(SketchApplication.tokens);
String tokens = Formatter.mapToJson(WebSketchApplication.tokens);
try {
FileExecutor.saveFile(SettingConfig.getStoragePath(ConfigConsts.TOKEN_OF_SETTINGS), tokens);
} catch (Exception e) {
@@ -65,14 +65,14 @@ public class TokenConfig {
public static void removeTokenByValue(int userId) {
if (userId > 0) {
String removeKey = "";
for (String key : SketchApplication.tokens.keySet()) {
if (SketchApplication.tokens.get(key) == userId) {
for (String key : WebSketchApplication.tokens.keySet()) {
if (WebSketchApplication.tokens.get(key) == userId) {
removeKey = key;
break;
}
}
if (Checker.isNotEmpty(removeKey)) {
SketchApplication.tokens.remove(removeKey);
WebSketchApplication.tokens.remove(removeKey);
TokenConfig.saveToken();
}
}

6
src/main/java/com/mesasoft/cn/dao/sqlprovider/AuthSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.Checker;
import org.apache.ibatis.annotations.Param;
@@ -40,9 +40,9 @@ public class AuthSqlProvider {
} else if (Checker.isNotEmpty(fileName)) {
WHERE("f.local_url like '%" + fileName + "%'");
}
ORDER_BY("a." + SketchApplication.settings.getStringUseEval(ConfigConsts.AUTH_ORDER_BY_OF_SETTINGS));
ORDER_BY("a." + WebSketchApplication.settings.getStringUseEval(ConfigConsts.AUTH_ORDER_BY_OF_SETTINGS));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.AUTH_PAGE_SIZE_OF_SETTINGS);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.AUTH_PAGE_SIZE_OF_SETTINGS);
return sql + " limit " + (offset * size) + "," + size;
}
}

6
src/main/java/com/mesasoft/cn/dao/sqlprovider/DownloadedSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.Checker;
import org.apache.ibatis.annotations.Param;
@@ -40,9 +40,9 @@ public class DownloadedSqlProvider {
if (categoryId > 0) {
WHERE("c.id=#{categoryId}");
}
ORDER_BY("d." + SketchApplication.settings.getStringUseEval(ConfigConsts.DOWNLOAD_ORDER_BY_OF_SETTINGS));
ORDER_BY("d." + WebSketchApplication.settings.getStringUseEval(ConfigConsts.DOWNLOAD_ORDER_BY_OF_SETTINGS));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.DOWNLOAD_PAGE_SIZE_OF_SETTINGS);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.DOWNLOAD_PAGE_SIZE_OF_SETTINGS);
return sql + " limit " + (offset * size) + "," + size;
}
}

10
src/main/java/com/mesasoft/cn/dao/sqlprovider/FileSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
@@ -46,15 +46,15 @@ public class FileSqlProvider {
if (categoryId > 0) {
WHERE("c.id=#{categoryId}");
}
ORDER_BY("f." + SketchApplication.settings.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING));
ORDER_BY("f." + WebSketchApplication.settings.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
return sql + " limit " + (offset * size) + "," + size;
}
private String getSqlEnds(int offset, String orderBy, String search) {
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
return getSearch(search) + " order by " + (Checker.isEmpty(orderBy) ? SketchApplication.settings
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
return getSearch(search) + " order by " + (Checker.isEmpty(orderBy) ? WebSketchApplication.settings
.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING) : orderBy) + " limit " + offset * size
+ "," + size;
}

6
src/main/java/com/mesasoft/cn/dao/sqlprovider/UploadedSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.Checker;
import org.apache.ibatis.annotations.Param;
@@ -39,9 +39,9 @@ public class UploadedSqlProvider {
if (categoryId > 0) {
WHERE("c.id=#{categoryId}");
}
ORDER_BY("f." + SketchApplication.settings.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING));
ORDER_BY("f." + WebSketchApplication.settings.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
return sql + " limit " + (offset * size) + "," + size;
}
}

6
src/main/java/com/mesasoft/cn/dao/sqlprovider/UserSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.zhazhapan.util.Checker;
@@ -33,9 +33,9 @@ public class UserSqlProvider {
WHERE("username like '%" + condition + "%' or email like '%" + condition + "%' or real_name like '" +
condition + "'");
}
ORDER_BY(SketchApplication.settings.getStringUseEval(ConfigConsts.USER_ORDER_BY_OF_SETTINGS));
ORDER_BY(WebSketchApplication.settings.getStringUseEval(ConfigConsts.USER_ORDER_BY_OF_SETTINGS));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.USER_PAGE_SIZE_OF_SETTINGS);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.USER_PAGE_SIZE_OF_SETTINGS);
return sql + " limit " + (offset * size) + "," + size;
}
}

5
src/main/java/com/mesasoft/cn/exception/GlobalExceptionHandler.java
View File

@@ -1,7 +1,6 @@
package com.mesasoft.cn.exception;
import com.alibaba.fastjson.support.spring.FastJsonJsonView;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
import org.springframework.http.HttpStatus;
import org.springframework.web.servlet.HandlerExceptionResolver;
@@ -23,9 +22,9 @@ public class GlobalExceptionHandler implements HandlerExceptionResolver {
Exception ex) {
ModelAndView mv = new ModelAndView();
FastJsonJsonView view = new FastJsonJsonView();
Map<String, Object> attributes = new HashMap<>(ValueConsts.TWO_INT);
Map<String, Object> attributes = new HashMap<>();
attributes.put("code", "502");
attributes.put("message", ex.getMessage());
attributes.put("message", "request error "+ (Checker.isEmpty(ex.getMessage())||ex.getMessage()=="null"?ex.getCause():ex.getMessage()));
String queryString = request.getQueryString();
attributes.put("url", request.getRequestURI() + (Checker.isEmpty(queryString) ? "" : "?" + queryString));
view.setAttributesMap(attributes);

19
src/main/java/com/mesasoft/cn/interceptor/WebInterceptor.java
View File

@@ -1,11 +1,11 @@
package com.mesasoft.cn.interceptor;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.impl.UserServiceImpl;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.impl.UserServiceImpl;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
import com.zhazhapan.util.HttpUtils;
@@ -28,15 +28,16 @@ public class WebInterceptor implements HandlerInterceptor {
UserServiceImpl userService;
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws
Exception {
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
String url = request.getServletPath();
InterceptorLevel level = InterceptorLevel.NONE;
if (handler instanceof HandlerMethod) {
AuthInterceptor interceptor = ((HandlerMethod) handler).getMethodAnnotation(AuthInterceptor.class);
//注解到类上面的注解，无法直接获取，只能通过扫描
if (Checker.isNull(interceptor)) {
for (Class<?> type : SketchApplication.controllers) {
for (Class<?> type : WebSketchApplication.controllers) {
RequestMapping mapping = type.getAnnotation(RequestMapping.class);
if (Checker.isNotNull(mapping)) {
for (String path : mapping.value()) {
@@ -58,16 +59,14 @@ public class WebInterceptor implements HandlerInterceptor {
//读取token，自动登录
Cookie cookie = HttpUtils.getCookie(ValueConsts.TOKEN_STRING, request.getCookies());
if (Checker.isNotNull(cookie)) {
user = userService.login(ValueConsts.EMPTY_STRING, ValueConsts.EMPTY_STRING, cookie.getValue(),
response);
user = userService.login(ValueConsts.EMPTY_STRING, ValueConsts.EMPTY_STRING, cookie.getValue(), response);
if (Checker.isNotNull(user)) {
request.getSession().setAttribute(ValueConsts.USER_STRING, user);
}
}
}
if (level != InterceptorLevel.NONE) {
boolean isRedirect = Checker.isNull(user) || (level == InterceptorLevel.ADMIN && user.getPermission() <
2) || (level == InterceptorLevel.SYSTEM && user.getPermission() < 3);
boolean isRedirect = Checker.isNull(user) || (level == InterceptorLevel.ADMIN && user.getPermission() < 2) || (level == InterceptorLevel.SYSTEM && user.getPermission() < 3);
if (isRedirect) {
response.setStatus(401);
response.sendRedirect(DefaultValues.SIGNIN_PAGE);

3
src/main/java/com/mesasoft/cn/modules/constant/DefaultValues.java
View File

@@ -1,5 +1,6 @@
package com.mesasoft.cn.modules.constant;
import com.zhazhapan.modules.constant.ValueConsts;
/**
@@ -21,7 +22,7 @@ public class DefaultValues {
/**
* Controller包路径
*/
public static final String CONTROLLER_PACKAGE = "com.zhazhapan.efo.web.controller";
public static final String CONTROLLER_PACKAGE = "com.mesasoft.cn.web.controller";
/**
* 配置文件路径

4
src/main/java/com/mesasoft/cn/service/impl/AuthServiceImpl.java
View File

@@ -1,12 +1,12 @@
package com.mesasoft.cn.service.impl;
import com.mesasoft.cn.dao.AuthDAO;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.dao.AuthDAO;
import com.mesasoft.cn.entity.Auth;
import com.mesasoft.cn.model.AuthRecord;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.service.IAuthService;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.util.ServiceUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;

2
src/main/java/com/mesasoft/cn/service/impl/CommonServiceImpl.java
View File

@@ -1,7 +1,7 @@
package com.mesasoft.cn.service.impl;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.ICommonService;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;

6
src/main/java/com/mesasoft/cn/service/impl/ConfigServiceImpl.java
View File

@@ -1,7 +1,7 @@
package com.mesasoft.cn.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.service.IConfigService;
import org.springframework.stereotype.Service;
@@ -15,7 +15,7 @@ public class ConfigServiceImpl implements IConfigService {
@Override
public String getGlobalConfig() {
JSONObject jsonObject = (JSONObject) SketchApplication.settings.getObjectUseEval(ConfigConsts
JSONObject jsonObject = (JSONObject) WebSketchApplication.settings.getObjectUseEval(ConfigConsts
.GLOBAL_OF_SETTINGS).clone();
jsonObject.remove(ConfigConsts.UPLOAD_PATH_OF_GLOBAL);
jsonObject.remove(ConfigConsts.TOKEN_PATH_OF_GLOBAL);
@@ -25,7 +25,7 @@ public class ConfigServiceImpl implements IConfigService {
@Override
public String getUserConfig() {
JSONObject jsonObject = (JSONObject) SketchApplication.settings.getObjectUseEval(ConfigConsts.USER_OF_SETTINGS)
JSONObject jsonObject = (JSONObject) WebSketchApplication.settings.getObjectUseEval(ConfigConsts.USER_OF_SETTINGS)
.clone();
jsonObject.remove(ConfigConsts.EMAIL_CONFIG_OF_USER);
return jsonObject.toString();

24
src/main/java/com/mesasoft/cn/service/impl/FileServiceImpl.java
View File

@@ -1,10 +1,9 @@
package com.mesasoft.cn.service.impl;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.dao.DownloadedDAO;
import com.mesasoft.cn.dao.FileDAO;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.entity.Category;
import com.mesasoft.cn.entity.File;
import com.mesasoft.cn.entity.User;
@@ -17,6 +16,7 @@ import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.IAuthService;
import com.mesasoft.cn.service.ICategoryService;
import com.mesasoft.cn.service.IFileService;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.util.ServiceUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.*;
@@ -134,7 +134,7 @@ public class FileServiceImpl implements IFileService {
AuthRecord authRecord = authService.getByFileIdAndUserId(id, user.getId());
String suffix = FileExecutor.getFileSuffix(name);
boolean canUpdate = (Checker.isNull(authRecord) ? user.getIsUpdatable() == 1 :
authRecord.getIsUpdatable() == 1) && Checker.isNotEmpty(name) && Pattern.compile(SketchApplication.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher(suffix).matches();
authRecord.getIsUpdatable() == 1) && Checker.isNotEmpty(name) && Pattern.compile(WebSketchApplication.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher(suffix).matches();
if (canUpdate) {
String localUrl = file.getLocalUrl();
java.io.File newFile = new java.io.File(localUrl);
@@ -208,7 +208,7 @@ public class FileServiceImpl implements IFileService {
@Override
public String getResource(String visitUrl, HttpServletRequest request) {
logger.info("visit url: " + visitUrl);
boolean downloadable = SketchApplication.settings.getBooleanUseEval(ConfigConsts
boolean downloadable = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts
.ANONYMOUS_DOWNLOADABLE_OF_SETTING);
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
File file = fileDAO.getFileByVisitUrl(visitUrl);
@@ -249,15 +249,15 @@ public class FileServiceImpl implements IFileService {
String suffix = FileExecutor.getFileSuffix(name);
String localUrl = SettingConfig.getUploadStoragePath() + ValueConsts.SEPARATOR + name;
Category category = categoryService.getById(categoryId);
long maxSize = Formatter.sizeToLong(SketchApplication.settings.getStringUseEval(ConfigConsts
long maxSize = Formatter.sizeToLong(WebSketchApplication.settings.getStringUseEval(ConfigConsts
.FILE_MAX_SIZE_OF_SETTING));
long size = multipartFile.getSize();
boolean fileExists = localUrlExists(localUrl);
//检测标签是否合法
if (SketchApplication.settings.getBooleanUseEval(ConfigConsts.TAG_REQUIRE_OF_SETTING)) {
if (WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.TAG_REQUIRE_OF_SETTING)) {
String[] tags = Checker.checkNull(tag).split(ValueConsts.SPACE);
if (tags.length <= SketchApplication.settings.getIntegerUseEval(ConfigConsts.TAG_SIZE_OF_SETTING)) {
int maxLength = SketchApplication.settings.getIntegerUseEval(ConfigConsts.TAG_LENGTH_OF_SETTING);
if (tags.length <= WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.TAG_SIZE_OF_SETTING)) {
int maxLength = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.TAG_LENGTH_OF_SETTING);
for (String t : tags) {
if (t.length() > maxLength) {
return false;
@@ -268,15 +268,15 @@ public class FileServiceImpl implements IFileService {
}
}
//是否可以上传
boolean canUpload = !multipartFile.isEmpty() && size <= maxSize && Pattern.compile(SketchApplication
boolean canUpload = !multipartFile.isEmpty() && size <= maxSize && Pattern.compile(WebSketchApplication
.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher(suffix).matches()
&& (Checker.isNotExists(localUrl) || !fileExists || SketchApplication.settings.getBooleanUseEval
&& (Checker.isNotExists(localUrl) || !fileExists || WebSketchApplication.settings.getBooleanUseEval
(ConfigConsts.FILE_COVER_OF_SETTING));
logger.info("is empty [" + multipartFile.isEmpty() + "], file size [" + size + "], max file size [" +
maxSize + "]");
if (canUpload) {
String visitUrl = getRegularVisitUrl(Checker.isNotEmpty(prefix) && user.getPermission() > 1 ? prefix
: SketchApplication.settings.getStringUseEval(ConfigConsts.CUSTOM_LINK_RULE_OF_SETTING), user,
: WebSketchApplication.settings.getStringUseEval(ConfigConsts.CUSTOM_LINK_RULE_OF_SETTING), user,
name, suffix, category);
if (fileExists) {
removeByLocalUrl(localUrl);

23
src/main/java/com/mesasoft/cn/service/impl/UserServiceImpl.java
View File

@@ -1,13 +1,13 @@
package com.mesasoft.cn.service.impl;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.dao.UserDAO;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.config.TokenConfig;
import com.mesasoft.cn.dao.UserDAO;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.service.IUserService;
import com.mesasoft.cn.util.BeanUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
import com.zhazhapan.util.DateUtils;
@@ -67,14 +67,13 @@ public class UserServiceImpl implements IUserService {
@Override
public User login(String loginName, String password, String token, HttpServletResponse response) {
boolean allowLogin = SketchApplication.settings.getBooleanUseEval(ConfigConsts.ALLOW_LOGIN_OF_SETTINGS);
boolean allowLogin = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.ALLOW_LOGIN_OF_SETTINGS);
User user = null;
if (allowLogin) {
if (Checker.isNotEmpty(token) && SketchApplication.tokens.containsKey(token)) {
user = userDAO.getUserById(SketchApplication.tokens.get(token));
if (Checker.isNotEmpty(token) && WebSketchApplication.tokens.containsKey(token)) {
user = userDAO.getUserById(WebSketchApplication.tokens.get(token));
if (Checker.isNotNull(response)) {
Cookie cookie = new Cookie(ValueConsts.TOKEN_STRING, TokenConfig.generateToken(token, user.getId
()));
Cookie cookie = new Cookie(ValueConsts.TOKEN_STRING, TokenConfig.generateToken(token, user.getId ()));
cookie.setMaxAge(30 * 24 * 60 * 60);
response.addCookie(cookie);
}
@@ -92,9 +91,9 @@ public class UserServiceImpl implements IUserService {
@Override
public boolean register(String username, String email, String password) {
boolean allowRegister = SketchApplication.settings.getBooleanUseEval(ConfigConsts.ALLOW_REGISTER_OF_SETTINGS);
boolean allowRegister = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.ALLOW_REGISTER_OF_SETTINGS);
if (allowRegister) {
boolean isValid = Checker.isEmail(email) && checkPassword(password) && Pattern.compile(SketchApplication.settings
boolean isValid = Checker.isEmail(email) && checkPassword(password) && Pattern.compile(WebSketchApplication.settings
.getStringUseEval(ConfigConsts.USERNAME_PATTERN_OF_SETTINGS)).matcher(username).matches();
if (isValid) {
User user = new User(username, ValueConsts.EMPTY_STRING, email, password);
@@ -113,8 +112,8 @@ public class UserServiceImpl implements IUserService {
@Override
public boolean checkPassword(String password) {
int min = SketchApplication.settings.getIntegerUseEval(ConfigConsts.PASSWORD_MIN_LENGTH_OF_SETTINGS);
int max = SketchApplication.settings.getIntegerUseEval(ConfigConsts.PASSWORD_MAX_LENGTH_OF_SETTINGS);
int min = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.PASSWORD_MIN_LENGTH_OF_SETTINGS);
int max = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.PASSWORD_MAX_LENGTH_OF_SETTINGS);
return Checker.isLimited(password, min, max);
}

159
src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java
View File

@@ -1,13 +1,14 @@
package com.mesasoft.cn.sketch.api;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import com.mesasoft.cn.sketch.entity.DomainCategory;
import com.mesasoft.cn.util.FileUtils;
import com.mesasoft.cn.util.ValidationUtils;
import org.apache.log4j.Logger;
import com.mesasoft.cn.sketch.config.BrightCloudConfig;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import com.mesasoft.cn.sketch.util.FileUtils;
import com.mesasoft.cn.sketch.util.SpringContextUtil;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import java.io.*;
import java.net.HttpURLConnection;
@@ -20,45 +21,44 @@ import java.util.*;
* @version 1.0
* @date 2021/2/22 2:37 下午
*/
public class BrightCloud {
private static final Logger LOG = Logger.getLogger(BrightCloud.class);
private static final Log logger = Log.get();
private final Integer maxObjectNum = ApplicationConfig.API_BC_MAXIMUM_QUERYNUM;
private final HashMap<Integer, List<String>> catId2Info = new HashMap<>();
private static HashMap<Integer, List<String>> catId2Info = new HashMap<>();
private HttpURLConnection con;
public List<DomainCategory> getQueryFiles(List<String> domains){
private static BrightCloudConfig bcConfig = (BrightCloudConfig) SpringContextUtil.getBean("brightCloudConfig");
public List<DomainCategory> getBrightCloudDomainCategory(List<String> domains) {
JSONObject queryResults = getQueryResults(domains);
return responseSparse(queryResults);
}
// 获取json格式查询结果
public JSONObject getQueryResults(List<String> domains) {
if (domains.size()> ApplicationConfig.API_BC_MAXIMUM_QUERYNUM){
LOG.warn("Too many domains in a http post request, the number of fqdn/url should be no more than "
+ ApplicationConfig.API_BC_MAXIMUM_QUERYNUM + "!");
if (domains.size() > bcConfig.getMaximumQueryNum()) {
logger.warn("Too many domains in a http post request, the number of fqdn/url should be no more than "
+ bcConfig.getMaximumQueryNum() + "!");
}
JSONObject jsonRes = new JSONObject();
try {
URL url = new URL(ApplicationConfig.API_BC_URL);
URL url = new URL(bcConfig.getUrl());
con = (HttpURLConnection) url.openConnection();
con.setRequestMethod(ApplicationConfig.API_BC_METHOD);
con.setRequestMethod(bcConfig.getMethod());
con.setDoOutput(true);
con.setDoInput(true);
con.setRequestProperty("Content-Type", "application/json");
JSONObject param = new JSONObject();
param.put("oemid", ApplicationConfig.API_BC_OEMID);
param.put("deviceid", ApplicationConfig.API_BC_DEVICEID);
param.put("uid", ApplicationConfig.API_BC_UID);
param.put("queries", Collections.singletonList(ApplicationConfig.API_BC_QUERYTYPE));
param.put("a1cat", ApplicationConfig.API_BC_ISA1CAT);
param.put("reputation", ApplicationConfig.API_BC_ISREPU);
param.put("xml", ApplicationConfig.API_BC_ISXML); // json or xml格式
param.put("oemid", bcConfig.getOemid());
param.put("deviceid", bcConfig.getDeviceid());
param.put("uid", bcConfig.getUid());
param.put("queries", Collections.singletonList(bcConfig.getQueryType()));
param.put("a1cat", bcConfig.getIsa1cat());
param.put("reputation", bcConfig.getIsReputation());
// json or xml格式
param.put("xml", bcConfig.getIsxml());
param.put("urls", domains);
//建立实际的连接
@@ -67,7 +67,7 @@ public class BrightCloud {
writer.write(param.toString());
writer.flush();
} catch (IOException e) {
e.printStackTrace();
logger.error(e);
}
try {
@@ -85,19 +85,19 @@ public class BrightCloud {
jsonRes = JSONObject.parseObject(sbf.toString());
con.disconnect();
} catch (IOException e) {
e.printStackTrace();
logger.error(e);
}
return jsonRes;
}
// json响应内容解析
public List<DomainCategory> responseSparse(JSONObject records){
public List<DomainCategory> responseSparse(JSONObject records) {
List<DomainCategory> domainFiles = new ArrayList<>();
Boolean querySucess = records.get("status").equals(200);
if (!querySucess) {
System.out.print(records);
LOG.error("Wrong query. Query type: " + records.get("type"));
logger.error(JSON.toJSONString(records));
logger.error("Wrong query. Query type: " + records.get("type"));
} else {
JSONArray array = records.getJSONArray("results");
for (int i = 0; i < array.size(); i++) {
@@ -105,11 +105,16 @@ public class BrightCloud {
// json处理
JSONObject queries = jo.getJSONObject("queries");
JSONObject getInfo = queries.getJSONObject(ApplicationConfig.API_BC_QUERYTYPE);
JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0);
Integer catId = cat.getInteger("catid");
JSONObject getInfo = queries.getJSONObject(bcConfig.getQueryType());
String fqdn = jo.getString("url");
int catId = 0;
int catConf = 0;
if (getInfo.getJSONArray("cats") != null && getInfo.getJSONArray("cats").size() > 0) {
JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0);
catId = cat.getInteger("catid");
catConf = cat.getInteger("conf");
}
domainFiles.add(new DomainCategory(
fqdn,
"brightcloud",
@@ -120,74 +125,86 @@ public class BrightCloud {
catId,
getCatInfo(catId).get(0),
getCatInfo(catId).get(1),
cat.getInteger("conf"),
catConf,
getInfo.getBoolean("a1cat")));
}
}
return domainFiles;
}
private String getRepLevel(Integer repScore){
// json响应内容解析
public List<DomainCategory> responseSparseTest() {
List<DomainCategory> domainFiles = new ArrayList<>();
for (int i = 0; i < 2; i++) {
// json处理
String fqdn = "www.websketch.com" + i;
domainFiles.add(new DomainCategory(
fqdn,
"brightcloud",
true,
ValidationUtils.getMatchPattern(fqdn),
1,
"",
1,
"1",
"1",
1,
true));
}
return domainFiles;
}
private String getRepLevel(Integer repScore) {
String level = null; //用str存放数据
if (repScore > 80){ level="Trustworthy";}
else if (repScore > 60){ level="Low Risk";}
else if (repScore > 40){ level="Moderate Risk";}
else if (repScore > 20){ level="Suspicious";}
else if (repScore > 0){ level="High Risk";}
if (repScore > 80) {
level = "Trustworthy";
} else if (repScore > 60) {
level = "Low Risk";
} else if (repScore > 40) {
level = "Moderate Risk";
} else if (repScore > 20) {
level = "Suspicious";
} else if (repScore > 0) {
level = "High Risk";
}
return level;
}
// 获取类别id对应信息
public void geneCatInfo(){
if (catId2Info.size()==0){
public static void geneCatInfo() {
if (catId2Info.size() == 0) {
JSONObject jsonObject;
// String filePath = Objects.requireNonNull(BrightCloud.class.getClassLoader()
// .getResource(ApplicationConfig.API_BC_CATEINFO_FILE)).getFile();
String filePath =ApplicationConfig.API_BC_CATEINFO_FILE;
String s = FileUtils.readJsonFile(filePath);
String s = FileUtils.readJsonFile(bcConfig.getCateinfoFilepath());
jsonObject = JSON.parseObject(s);
if (!(jsonObject==null)){
if (!(jsonObject == null)) {
JSONObject tmp = (JSONObject) jsonObject.getJSONArray("results").get(0);
JSONArray catInfoArray = tmp.getJSONObject("queries").getJSONObject("getcatlist").getJSONArray("cats");
for (int i = 0; i < catInfoArray.size(); i++){
for (int i = 0; i < catInfoArray.size(); i++) {
JSONObject keyObject = catInfoArray.getJSONObject(i);
List<String> value = new ArrayList<>(Arrays.asList(
keyObject.getString("catname"),
keyObject.getString("catgroup")));
catId2Info.put(i+1, value);
catId2Info.put(i + 1, value);
}
}
}
}
public List<String> getCatInfo(Integer catId){
List<String> info = Arrays.asList("", "");
if (0 < catId && catId <= 83) {
if (catId2Info.size()==0){
geneCatInfo();
}
info = catId2Info.get(catId);
if (info == null){
LOG.error("Failed at geneCatInfo function");
System.out.print("Failed at geneCatInfo function");
}
public List<String> getCatInfo(Integer catId) {
if (catId2Info.size() == 0) {
geneCatInfo();
}
List<String> info = catId2Info.get(catId);
if (info == null || info.size()==0) {
info = Arrays.asList("", "");
logger.error("Failed at geneCatInfo function");
}
return info;
}
public Integer getMaxObjectNum() {
return maxObjectNum;
}
public static void main(String[] args) {
JSONObject queryResults = new BrightCloud().getQueryResults(Arrays.asList("baidu.com"));
new BrightCloud().responseSparse(queryResults);
}
}

201
src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java
View File

@@ -1,16 +1,12 @@
package com.mesasoft.cn.sketch.api;
/*
* @Description:
* @Author: chenxu
* @Date: 2021-12-27 13:59:29
* @LastEditTime: 2021-12-29 17:05:45
* @LastEditors: chenxu
*/
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import com.mesasoft.cn.sketch.entity.DomainWhois;
import com.mesasoft.cn.sketch.config.ChinazConfig;
import com.mesasoft.cn.sketch.entity.domain.DomainICP;
import com.mesasoft.cn.sketch.entity.domain.DomainWhois;
import com.mesasoft.cn.sketch.util.SpringContextUtil;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
@@ -18,7 +14,6 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.log4j.Logger;
import java.io.BufferedReader;
import java.io.InputStream;
@@ -29,26 +24,57 @@ import java.sql.Date;
import java.util.*;
import java.util.concurrent.TimeUnit;
public class ChinaZ {
private static final Logger LOG = Logger.getLogger(ChinaZ.class);
private final String apiKey = ApplicationConfig.API_CHINAZ_KEY;
/*
* @Description:
* @Author: chenxu
* @Date: 2021-12-27 13:59:29
* @LastEditTime: 2021-12-29 17:05:45
* @LastEditors: chenxu
*/
public List<DomainWhois> getQueryFiles(List<String> objectList) {
List<JSONObject> queryResults = getQueryResults(objectList);
public class ChinaZ {
private static final Log logger = Log.get();
private static final int REQUEST_MAX_BATCH = 50;
private static final String BATCH_SEPARATOR = "|";
private static final String CHINAZ = "chinaz";
private static final int SUFFIX_MATCHING = 1;
private static final int EXACTLY_MATCHING = 2;
private static ChinazConfig chinazConfig = (ChinazConfig) SpringContextUtil.getBean("chinazConfig");
public List<DomainWhois> getDomainsWhois(List<String> domainList) {
List<JSONObject> queryResults = getQueryResults(domainList);
return responseSparse(queryResults);
}
public List<DomainICP> getDomainsICP(List<String> domainList) {
//查询ICP
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("key", chinazConfig.getKey());
List<DomainICP> queryResults = new ArrayList<>();
logger.info("query chainz icp domain size {}", domainList.size());
for (String domain : domainList) {
params.put("domain", domain);
JSONObject r = doPost(chinazConfig.getICPSingle(), params);
DomainICP domainICP = icpInfoResolve(r, domain);
queryResults.add(domainICP);
}
return queryResults;
}
/**
* @Description: 站长之家单查询
* @Param : 域名
* @Return: 查询结果
*/
public JSONObject getQueryResult(String domain){
String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE;
Map<String, String> params = new LinkedHashMap<String,String>();
params.put("key", apiKey);
public JSONObject getQueryResult(String domain) {
String urlString = chinazConfig.getWhoisSingle();
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("key", chinazConfig.getKey());
params.put("domain", domain);
return whoisInfoResolve(doPost(urlString, params),domain);
return whoisInfoResolve(doPost(urlString, params), domain);
}
/**
@@ -56,43 +82,42 @@ public class ChinaZ {
* @Param : 域名
* @Return: 查询结果
*/
public List<JSONObject> getQueryResults(List<String> domainList){
String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE;
public List<JSONObject> getQueryResults(List<String> domainList) {
List<JSONObject> whoisInfoList = new ArrayList<>();
for (String s : domainList) {
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("key", apiKey);
params.put("key", chinazConfig.getKey());
params.put("domain", s);
JSONObject r = doPost(urlString, params);
JSONObject r = doPost(chinazConfig.getWhoisSingle(), params);
whoisInfoList.add(whoisInfoResolve(r, s));
}
return whoisInfoList;
return whoisInfoList;
}
public List<DomainWhois> responseSparse(List<JSONObject> records){
public List<DomainWhois> responseSparse(List<JSONObject> records) {
List<DomainWhois> whoisFiles = new ArrayList<>();
for(JSONObject record: records) {
for (JSONObject record : records) {
Boolean querySucess = record.getBoolean("isSuccess");
if (!querySucess) {
LOG.error("Failed query. Query response: " + record);
logger.error("Failed query. Query response: " + record);
break;
}
String fqdn = record.getString("domain_name");
String domainName = record.getString("domain_host");
Integer matchPattern = fqdn.equals(domainName)? 1 : 2 ;
Integer matchPattern = fqdn.equals(domainName) ? 1 : 2;
String source = "chinaz";
// json处理
Date creatDate = null;
Date expiraDate = null;
java.util.Date tmpDate = record.getDate("domain_whois_create_time");
if(tmpDate!=null){
if (tmpDate != null) {
creatDate = new Date(tmpDate.getTime());
}
tmpDate = record.getDate("domain_whois_expiration_time");
if(tmpDate!=null){
if (tmpDate != null) {
expiraDate = new Date(tmpDate.getTime());
}
whoisFiles.add(new DomainWhois(
@@ -119,17 +144,18 @@ public class ChinaZ {
}
return whoisFiles;
}
/**
* @Description: 解析并重构JSON串
* @Param : 查询得到的“单个”JSON串
* @Return: 返回重构的JSON串
*/
public JSONObject whoisInfoResolve(JSONObject jsonRes,String queryDomain){
public JSONObject whoisInfoResolve(JSONObject jsonRes, String queryDomain) {
JSONObject whoisInfo = new JSONObject(true);
JSONObject res = jsonRes.getJSONObject("Result");
if(jsonRes.get("StateCode").equals(1)){
if (jsonRes.get("StateCode").equals(1)) {
whoisInfo.put("isSuccess", jsonRes.get("StateCode"));
whoisInfo.put("domain_name",queryDomain);
whoisInfo.put("domain_name", queryDomain);
whoisInfo.put("domain_host", res.get("Host"));
whoisInfo.put("domain_whois_create_time", res.get("CreationDate"));
whoisInfo.put("domain_whois_expiration_time", res.get("ExpirationDate"));
@@ -139,10 +165,31 @@ public class ChinaZ {
whoisInfo.put("domain_whois_phone", res.get("Phone"));
whoisInfo.put("domain_whois_name_servers", res.get("DnsServer"));
whoisInfo.put("domain_whois_status", res.get("DomainStatus"));
}else{
} else {
whoisInfo.put("isSuccess", jsonRes.get("StateCode"));
}
return whoisInfo;
return whoisInfo;
}
public DomainICP icpInfoResolve(JSONObject jsonRes, String queryDomain) {
JSONObject res = jsonRes.getJSONObject("Result");
DomainICP icp = new DomainICP();
if (jsonRes.get("StateCode").equals(1)) {
icp.setFqdn(queryDomain);
icp.setSource(CHINAZ);
icp.setMatch_pattern(queryDomain.equals(res.getString("MainPage")) ? EXACTLY_MATCHING : SUFFIX_MATCHING);
icp.setQuery_success(jsonRes.getBoolean("StateCode"));
icp.setIcp_owner(res.getString("Owner"));
icp.setIcp_company_name(res.getString("CompanyName"));
icp.setIcp_company_type(res.getString("CompanyType"));
icp.setIcp_main_page(res.getString("MainPage"));
icp.setIcp_site_license(res.getString("SiteLicense"));
icp.setIcp_site_name(res.getString("SiteName"));
icp.setIcp_verify_time(res.getString("VerifyTime"));
} else {
icp.setQuery_success(jsonRes.getBoolean("StateCode"));
}
return icp;
}
/**
@@ -150,19 +197,19 @@ public class ChinaZ {
* @Param : 待查询域名
* @Return: 拼接好的URL
*/
public List<String> queryStringBuilder(List<String> domainList){
public List<String> queryStringBuilder(List<String> domainList) {
//将域名每50个划分一组
int CHINAZ_REQUEST_LIMIT = 50 ;
int CHINAZ_REQUEST_LIMIT = 50;
int domainListSize = domainList.size();
int toIndex = CHINAZ_REQUEST_LIMIT;
Map domainListMap = new HashMap();
int keyToken = 0;
for(int i = 0;i<domainList.size();i+=CHINAZ_REQUEST_LIMIT){
if(i+CHINAZ_REQUEST_LIMIT>domainListSize){ //作用为toIndex最后没有50条数据则剩余几条newList中就装几条
toIndex=domainListSize-i;
for (int i = 0; i < domainList.size(); i += CHINAZ_REQUEST_LIMIT) {
if (i + CHINAZ_REQUEST_LIMIT > domainListSize) { //作用为toIndex最后没有50条数据则剩余几条newList中就装几条
toIndex = domainListSize - i;
}
List<String> newList = domainList.subList(i,i+toIndex);
domainListMap.put("keyName"+keyToken, newList);
List<String> newList = domainList.subList(i, i + toIndex);
domainListMap.put("keyName" + keyToken, newList);
keyToken++;
}
//将批量查询的域名，构造成CHINAZ的格式
@@ -174,8 +221,8 @@ public class ChinaZ {
Object val = entry.getValue();
String urlString = "";
urlString = String.valueOf(val);
urlString = urlString.replace(", ","|");
urlString =urlString.replace("[","").replace("]","");
urlString = urlString.replace(", ", "|");
urlString = urlString.replace("[", "").replace("]", "");
domainListString.add(urlString);
}
return domainListString;
@@ -186,18 +233,18 @@ public class ChinaZ {
* @Param : 域名集合（不能超过50个）
* @Return: 查询结果
*/
public String batchRequest_step1(String domainsString){
public String batchRequest_step1(String domainsString) {
String TaskID = "";
String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH;
String urlString = chinazConfig.getWhoisBatch();
Map<String, String> params = new LinkedHashMap<String,String>();
if (!Objects.equals(domainsString, "overflow")){
params.put("domains",domainsString);
params.put("key", apiKey);
JSONObject r = doPost(urlString, params);
Map<String, String> params = new LinkedHashMap<String, String>();
if (!Objects.equals(domainsString, "overflow")) {
params.put("domains", domainsString);
params.put("key", chinazConfig.getKey());
JSONObject r = doPost(urlString, params);
TaskID = r.get("TaskID").toString();
return TaskID;
}else{
} else {
return TaskID;
}
@@ -208,11 +255,11 @@ public class ChinaZ {
* @Param : 任务ID
* @Return: 查询结果
*/
public JSONObject batchRequest_step2(String TaskID){
String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH;
public JSONObject batchRequest_step2(String TaskID) {
String urlString = chinazConfig.getWhoisBatch();
Map<String, String> params = new LinkedHashMap<String,String>();
params.put("taskid",TaskID);
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("taskid", TaskID);
JSONObject requestTotal = null;
requestTotal = doPost(urlString, params);
return requestTotal;
@@ -223,17 +270,15 @@ public class ChinaZ {
* @Param : 域名列表
* @Return: whois记录列表
*/
public List<String> batchRequestController(List<String> domainList){
public List<String> batchRequestController(List<String> domainList) {
List<String> result = new ArrayList<>();
if (domainList.size()> 5000){
System.out.println("Too many urls in a http post request!");
List<String> result = new ArrayList<>();
if (domainList.size() > 5000) {
logger.info("Too many urls in a http post request!");
}
List<String> domainListString = new ArrayList<>();
List<String> TaskID = new ArrayList<>();
List<String> TaskID = new ArrayList<>();
// Queue<String> queue = new LinkedList<String>();
domainListString = queryStringBuilder(domainList);
//循环发送请求，收集每个请求的TaskID
for (String domainParam : domainListString) {
@@ -276,7 +321,7 @@ public class ChinaZ {
* @Param : 请求对URL，POST请求体需要添加的 k-v 数据
* @Return: API JSON数据
*/
public JSONObject doPost(String url, Map params){
public JSONObject doPost(String url, Map params) {
JSONObject jsonRes = null;
try {
// 定义HttpClient
@@ -291,17 +336,14 @@ public class ChinaZ {
String name = (String) o;
String value = String.valueOf(params.get(name));
nvps.add(new BasicNameValuePair(name, value));
//System.out.println(name +"-"+value);
}
request.setEntity(new UrlEncodedFormEntity(nvps));
//发送请求
HttpResponse httpResponse = client.execute(request);
HttpResponse httpResponse = client.execute(request);
// 获取响应输入流
InputStream inStream = httpResponse.getEntity().getContent();
//对放回数据进行处理
BufferedReader reader = new BufferedReader(new InputStreamReader(inStream , StandardCharsets.UTF_8));
StringBuilder strber = new StringBuilder();
BufferedReader reader = new BufferedReader(new InputStreamReader(inStream, StandardCharsets.UTF_8));
StringBuilder sbf = new StringBuilder();
String strRead = null;
while ((strRead = reader.readLine()) != null) {
@@ -311,26 +353,11 @@ public class ChinaZ {
// 关闭输入流
inStream.close();
jsonRes = JSONObject.parseObject(sbf.toString());
}catch (Exception e) {
System.out.println("请求接口异常");
} catch (Exception e) {
logger.error("chinaZ request error", e);
}
return jsonRes;
}
public static void main(String[] args){
ChinaZ t = new ChinaZ();
//单查询测试
// System.out.println(t.singleRequest("aaa.baidu.com"));
//批量查询测试
List<String> domainList = new ArrayList<>();
domainList.add("www.baidu.com");
// domainList.add("aaa.qq.com");
// domainList.add("doc.mesalab.com");
// System.out.println(t.batchRequestController(domainList));
System.out.println(t.getQueryResults(domainList));
}
}

20
src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java Normal file
View File

@@ -0,0 +1,20 @@
package com.mesasoft.cn.sketch.config;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Data
@Component
public class AppConfig {
// Mariadb
public static final String DATABASE = ConfigUtils.getStringProperty("database");
@Value("${db.query.batch.size}")
private Integer dbQueryBatchSize;
}

54
src/main/java/com/mesasoft/cn/sketch/config/ApplicationConfig.java
View File

@@ -1,54 +0,0 @@
package com.mesasoft.cn.sketch.config;
import com.mesasoft.cn.util.ConfigUtils;
public class ApplicationConfig {
public static final String QUERY_OUTPUT_DIR = ConfigUtils.getStringProperty("query.output.dir");
public static final String QUERY_TYPES_DOMAIN = ConfigUtils.getStringProperty("query.types.domain");
public static final String QUERY_TYPES_IP = ConfigUtils.getStringProperty("query.types.ip");
public static final Integer UPDATE_EXPIRED_DAY = ConfigUtils.getIntProperty("update.expired.day"); // 更新任务中过期时间长度（天数）
public static final Integer QUERY_READIN_BATCH = ConfigUtils.getIntProperty("query.readin.batch");
public static final Integer QUERY_LOG_FILE_LINE_INTERVAL = ConfigUtils.getIntProperty("query.log.file.line.interval"); // 文件查询时，打印log的读取行数间隔
// api参数
// brightcloud
public static final String API_BC_OEMID = ConfigUtils.getStringProperty("bc.oemid");
public static final String API_BC_DEVICEID = ConfigUtils.getStringProperty("bc.deviceid");
public static final String API_BC_UID = ConfigUtils.getStringProperty("bc.uid");
public static final String API_BC_URL = ConfigUtils.getStringProperty("bc.url");
public static final String API_BC_METHOD = ConfigUtils.getStringProperty("bc.method");
public static final String API_BC_ISA1CAT = ConfigUtils.getStringProperty("bc.isa1cat");
public static final String API_BC_ISREPU = ConfigUtils.getStringProperty("bc.isReputation");
public static final String API_BC_ISXML = ConfigUtils.getStringProperty("bc.isxml");
public static final Integer API_BC_MAXIMUM_QUERYNUM = ConfigUtils.getIntProperty("bc.maximum.query.num"); // brightcloud单次查询条数上线
public static final String API_BC_QUERYTYPE = ConfigUtils.getStringProperty("bc.queryType");
public static final String API_BC_USE_REPORT_FILE = ConfigUtils.getStringProperty("bc.usereport.filepath"); // brightcloud使用报告导出文件目录
public static final String API_BC_CATEINFO_FILE = ConfigUtils.getStringProperty("bc.cateinfo.filepath"); // brightcloud使用报告导出文件目录
// chinaz
public static final String API_CHINAZ_URL_SINGLE = ConfigUtils.getStringProperty("chinaz.url.single");
public static final String API_CHINAZ_URL_BATCH = ConfigUtils.getStringProperty("chinaz.url.batch");
public static final String API_CHINAZ_KEY = ConfigUtils.getStringProperty("chinaz.key");
public static final Integer API_CHINAZ_MAXIMUM_QUERYNUM = ConfigUtils.getIntProperty("chinaz.maximum.query.num");
public static final String API_CHINAZ_USE_REPORT_FILE = ConfigUtils.getStringProperty("chinaz.usereport.filepath");
// Mariadb
public static final String DATABASE = ConfigUtils.getStringProperty("database");
public static final String DOMAIN_CATE_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.category");
public static final String DOMAIN_WHOIS_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.whois");
public static final String DNS_SERVER_TABLENAME = ConfigUtils.getStringProperty("tablename.dns.server");
public static final Integer DB_QUERY_BATCH_SIZE = ConfigUtils.getIntProperty("db.query.batch.size");
// 其他
public static final String TLD_FILE = ConfigUtils.getStringProperty("tld.file"); // 顶级域名公开列表文件
}

26
src/main/java/com/mesasoft/cn/sketch/config/BrightCloudConfig.java Normal file
View File

@@ -0,0 +1,26 @@
package com.mesasoft.cn.sketch.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Data
@Component
@ConfigurationProperties(prefix = "bright-cloud")
public class BrightCloudConfig {
private String oemid;
private String deviceid;
private String uid;
private String url;
private String method;
private String queryType;
private String isa1cat;
private String isReputation;
private String isxml;
private int maximumQueryNum;
private String cateinfoFilepath;
private String usereportFilePath;
}

36
src/main/java/com/mesasoft/cn/sketch/config/ChinazConfig.java Normal file
View File

@@ -0,0 +1,36 @@
package com.mesasoft.cn.sketch.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Data
@Component
@ConfigurationProperties(prefix = "chinaz")
public class ChinazConfig {
private String urlSingle;
private String urlBatch;
private String key;
private int maximumQueryNum;
private String usereportFilePath;
public String getWhoisSingle() {
return getUrlSingle() + "/whois";
}
public String getWhoisBatch() {
return getUrlBatch() + "/whois";
}
public String getICPSingle() {
return getUrlSingle() + "/icp";
}
public String getICPBatch() {
return getUrlBatch() + "/icp";
}
}

258
src/main/java/com/mesasoft/cn/sketch/config/SketchDatabaseConfig.java
View File

@@ -1,125 +1,133 @@
//package com.zhazhapan.efo.sketch.config;
//
//import com.alibaba.druid.pool.DruidDataSource;
//import lombok.Data;
//import org.apache.ibatis.session.SqlSessionFactory;
//import org.mybatis.spring.SqlSessionFactoryBean;
//import org.mybatis.spring.annotation.MapperScan;
//import org.springframework.beans.factory.annotation.Qualifier;
//import org.springframework.beans.factory.annotation.Value;
//import org.springframework.boot.context.properties.ConfigurationProperties;
//import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.Configuration;
//import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
//import org.springframework.jdbc.datasource.DataSourceTransactionManager;
//
//import javax.sql.DataSource;
//import java.sql.SQLException;
//
///**
// * @ProjectName
// * @Description: 后台数据源配置类
// */
//@Data
//@Configuration
//@ConfigurationProperties(prefix = "sketch.datasource.druid")
//@MapperScan(basePackages = SketchDatabaseConfig.PACKAGE, sqlSessionFactoryRef = "sketchSqlSessionFactory")
//public class SketchDatabaseConfig {
// /**
// * dao层的包路径
// */
// static final String PACKAGE = "com.mao.mysqlhive.demomh.mapper.sketch";
//
// /**
// * mapper文件的相对路径
// */
// private static final String MAPPER_LOCATION = "classpath:mappers/sketch/*Mapper.xml";
//
// @Value("${sketch.datasource.druid.filters}")
// private String filters;
// @Value("${sketch.datasource.druid.driverClassName}")
// private String url;
// @Value("${sketch.datasource.druid.url}")
// private String username;
// @Value("${sketch.datasource.druid.username}")
// private String password;
// @Value("${sketch.datasource.druid.password}")
// private String driverClassName;
// @Value("${sketch.datasource.druid.initialSize}")
// private int initialSize;
// @Value("${sketch.datasource.druid.minIdle}")
// private int minIdle;
// @Value("${sketch.datasource.druid.maxActive}")
// private int maxActive;
// @Value("${sketch.datasource.druid.maxWait}")
// private long maxWait;
// @Value("${sketch.datasource.druid.timeBetweenEvictionRunsMillis}")
// private long timeBetweenEvictionRunsMillis;
// @Value("${sketch.datasource.druid.minEvictableIdleTimeMillis}")
// private long minEvictableIdleTimeMillis;
// @Value("${sketch.datasource.druid.validationQuery}")
// private String validationQuery;
// @Value("${sketch.datasource.druid.testWhileIdle}")
// private boolean testWhileIdle;
// @Value("${sketch.datasource.druid.testOnBorrow}")
// private boolean testOnBorrow;
// @Value("${sketch.datasource.druid.testOnReturn}")
// private boolean testOnReturn;
// @Value("${sketch.datasource.druid.poolPreparedStatements}")
// private boolean poolPreparedStatements;
// @Value("${sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize}")
// private int maxPoolPreparedStatementPerConnectionSize;
//
//
// @Bean(name = "sketchDataSource")
// public DataSource sketchDataSource() throws SQLException {
// DruidDataSource druid = new DruidDataSource();
// // 监控统计拦截的filters
// druid.setFilters(filters);
//
// // 配置基本属性
// druid.setDriverClassName(driverClassName);
// druid.setUsername(username);
// druid.setPassword(password);
// druid.setUrl(url);
//
// //初始化时建立物理连接的个数
// druid.setInitialSize(initialSize);
// //最大连接池数量
// druid.setMaxActive(maxActive);
// //最小连接池数量
// druid.setMinIdle(minIdle);
// //获取连接时最大等待时间，单位毫秒。
// druid.setMaxWait(maxWait);
// //间隔多久进行一次检测，检测需要关闭的空闲连接
// druid.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
// //一个连接在池中最小生存的时间
// druid.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
// //用来检测连接是否有效的sql
// druid.setValidationQuery(validationQuery);
// //建议配置为true，不影响性能，并且保证安全性。
// druid.setTestWhileIdle(testWhileIdle);
// //申请连接时执行validationQuery检测连接是否有效
// druid.setTestOnBorrow(testOnBorrow);
// druid.setTestOnReturn(testOnReturn);
// //是否缓存preparedStatement，也就是PSCache，oracle设为true，mysql设为false。分库分表较多推荐设置为false
// druid.setPoolPreparedStatements(poolPreparedStatements);
// // 打开PSCache时，指定每个连接上PSCache的大小
// druid.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize);
// return druid;
// }
//
// @Bean(name = "sketchTransactionManager")
// public DataSourceTransactionManager sketchTransactionManager() throws SQLException {
// return new DataSourceTransactionManager(sketchDataSource());
// }
//
// @Bean(name = "sketchSqlSessionFactory")
// public SqlSessionFactory sketchSqlSessionFactory(@Qualifier("sketchDataSource") DataSource sketchDataSource) throws Exception {
// final SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
// sessionFactory.setDataSource(sketchDataSource);
// sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(SketchDatabaseConfig.MAPPER_LOCATION));
//
// return sessionFactory.getObject();
// }
//}
/*
package cn.ac.iie.efo.sketch.config;
import com.alibaba.druid.pool.DruidDataSource;
import lombok.Data;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import javax.sql.DataSource;
import java.sql.SQLException;
*/
/**
* @ProjectName
* @Description: 后台数据源配置类
*//*
@Data
@Configuration
@ConfigurationProperties(prefix = "sketch.datasource.druid")
@MapperScan(basePackages = SketchDatabaseConfig.PACKAGE, sqlSessionFactoryRef = "sketchSqlSessionFactory")
public class SketchDatabaseConfig {
*/
/**
* dao层的包路径
*//*
static final String PACKAGE = "com.mao.mysqlhive.demomh.mapper.sketch";
*/
/**
* mapper文件的相对路径
*//*
private static final String MAPPER_LOCATION = "classpath:mappers/sketch/*Mapper.xml";
@Value("${sketch.datasource.druid.filters}")
private String filters;
@Value("${sketch.datasource.druid.driverClassName}")
private String url;
@Value("${sketch.datasource.druid.url}")
private String username;
@Value("${sketch.datasource.druid.username}")
private String password;
@Value("${sketch.datasource.druid.password}")
private String driverClassName;
@Value("${sketch.datasource.druid.initialSize}")
private int initialSize;
@Value("${sketch.datasource.druid.minIdle}")
private int minIdle;
@Value("${sketch.datasource.druid.maxActive}")
private int maxActive;
@Value("${sketch.datasource.druid.maxWait}")
private long maxWait;
@Value("${sketch.datasource.druid.timeBetweenEvictionRunsMillis}")
private long timeBetweenEvictionRunsMillis;
@Value("${sketch.datasource.druid.minEvictableIdleTimeMillis}")
private long minEvictableIdleTimeMillis;
@Value("${sketch.datasource.druid.validationQuery}")
private String validationQuery;
@Value("${sketch.datasource.druid.testWhileIdle}")
private boolean testWhileIdle;
@Value("${sketch.datasource.druid.testOnBorrow}")
private boolean testOnBorrow;
@Value("${sketch.datasource.druid.testOnReturn}")
private boolean testOnReturn;
@Value("${sketch.datasource.druid.poolPreparedStatements}")
private boolean poolPreparedStatements;
@Value("${sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize}")
private int maxPoolPreparedStatementPerConnectionSize;
@Bean(name = "sketchDataSource")
public DataSource sketchDataSource() throws SQLException {
DruidDataSource druid = new DruidDataSource();
// 监控统计拦截的filters
druid.setFilters(filters);
// 配置基本属性
druid.setDriverClassName(driverClassName);
druid.setUsername(username);
druid.setPassword(password);
druid.setUrl(url);
//初始化时建立物理连接的个数
druid.setInitialSize(initialSize);
//最大连接池数量
druid.setMaxActive(maxActive);
//最小连接池数量
druid.setMinIdle(minIdle);
//获取连接时最大等待时间，单位毫秒。
druid.setMaxWait(maxWait);
//间隔多久进行一次检测，检测需要关闭的空闲连接
druid.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
//一个连接在池中最小生存的时间
druid.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
//用来检测连接是否有效的sql
druid.setValidationQuery(validationQuery);
//建议配置为true，不影响性能，并且保证安全性。
druid.setTestWhileIdle(testWhileIdle);
//申请连接时执行validationQuery检测连接是否有效
druid.setTestOnBorrow(testOnBorrow);
druid.setTestOnReturn(testOnReturn);
//是否缓存preparedStatement，也就是PSCache，oracle设为true，mysql设为false。分库分表较多推荐设置为false
druid.setPoolPreparedStatements(poolPreparedStatements);
// 打开PSCache时，指定每个连接上PSCache的大小
druid.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize);
return druid;
}
@Bean(name = "sketchTransactionManager")
public DataSourceTransactionManager sketchTransactionManager() throws SQLException {
return new DataSourceTransactionManager(sketchDataSource());
}
@Bean(name = "sketchSqlSessionFactory")
public SqlSessionFactory sketchSqlSessionFactory(@Qualifier("sketchDataSource") DataSource sketchDataSource) throws Exception {
final SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
sessionFactory.setDataSource(sketchDataSource);
sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(SketchDatabaseConfig.MAPPER_LOCATION));
return sessionFactory.getObject();
}
}
*/

183
src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java Normal file
View File

@@ -0,0 +1,183 @@
package com.mesasoft.cn.sketch.controller;
import cn.hutool.core.date.DateUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONArray;
import com.mesasoft.cn.sketch.entity.ReturnT;
import com.mesasoft.cn.sketch.entity.SketchAuth;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import com.mesasoft.cn.sketch.service.DomainService;
import com.mesasoft.cn.sketch.util.SktUtil;
import com.mesasoft.cn.util.ControllerUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
import java.util.Map;
@RestController
@RequestMapping("/v1/domain/")
public class DomainController {
private Log logger = Log.get();
@Resource
private DomainService domainService;
@GetMapping(value = "info")
@ResponseBody
public ReturnT info(SketchAuth sketchAuth) {
return ReturnT.succeed(sketchAuth.getRequestid(), "Hello WebSketch !");
}
@GetMapping(value = "getcategoryinfo")
@ResponseBody
public ReturnT getCategoryInfo1(HttpServletRequest request, SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getCategoryInfo(domain, SktUtil.getRequestUser(request));
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@PostMapping(value = "getcategoryinfo")
@ResponseBody
public ReturnT getCategoryInfo2(HttpServletRequest request, SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getCategoryInfo(domain, SktUtil.getRequestUser(request));
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@GetMapping(value = "getwhoisinfo")
@ResponseBody
public ReturnT getWhoisInfo1(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getWhoisInfo(domain);
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@PostMapping(value = "getwhoisinfo")
@ResponseBody
public ReturnT getWhoisInfo2(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getWhoisInfo(domain);
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@GetMapping(value = "geticpinfo")
@ResponseBody
public ReturnT getICPInfo1(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getICPInfo(domain);
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
@PostMapping(value = "geticpinfo")
@ResponseBody
public ReturnT getICPInfo2(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
List<DomainInfo> domainInfo = null;
try {
domainInfo = domainService.getICPInfo(domain);
} catch (Exception e) {
logger.error(e);
}
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfo);
}
/**
* 文件查询
*
* @param sketchAuth
* @param request
* @return
*/
@PostMapping(value = "/getcategoryinfobyfile")
public void getCategoryInfo(HttpServletResponse response, SketchAuth sketchAuth, MultipartHttpServletRequest request) {
try {
List<DomainInfo> categoryInfoByFile = domainService.getCategoryInfoByFile(request, SktUtil.getRequestUser(request));
ControllerUtils.loadFile(response, "domaincategory-" + DateUtil.date() + ".json", JSONArray.toJSONString(categoryInfoByFile, true));
} catch (Exception e) {
logger.error(e);
}
}
/**
* 文件查询whois
*
* @param sketchAuth
* @param request
* @return curl -o ./domaintest.json --request POST 'http://IP:8196/v1/domain/getcategoryinfobyfile' --form 'file=@"/home/test/domain.txt"' --form 'omeid=1'
*/
@PostMapping(value = "/getwhoisinfobyfile")
public void getWhoisInfo(HttpServletResponse response, SketchAuth sketchAuth, MultipartHttpServletRequest request) throws Exception {
try {
List<DomainInfo> whoisInfoByFile = domainService.getWhoisInfoByFile(request);
ControllerUtils.loadFile(response, "domainwhois-" + DateUtil.date() + ".json", JSONArray.toJSONString(whoisInfoByFile, true));
} catch (Exception e) {
logger.error(e);
}
}
/**
* 文件查询whois
*
* @param sketchAuth
* @param request
* @return curl -o ./domaintest.json --request POST 'http://IP:8196/v1/domain/getcategoryinfobyfile' --form 'file=@"/home/test/domain.txt"' --form 'omeid=1'
*/
@PostMapping(value = "/geticpinfobyfile")
public void getICPInfo(HttpServletResponse response, SketchAuth sketchAuth, MultipartHttpServletRequest request) throws Exception {
try {
List<DomainInfo> domainInfo = domainService.getICPInfoByFile(request);
ControllerUtils.loadFile(response, "domainicp-" + DateUtil.date() + ".json", JSONArray.toJSONString(domainInfo, true));
} catch (Exception e) {
logger.error(e);
}
}
/**
* 本地文件上传
* 查询大量数据
*/
@GetMapping(value = "/getdomaininfobyfile")
public ReturnT getDomainInfoByFile(HttpServletRequest request, String srcFile, String queryType, SketchAuth sketchAuth,Boolean export) {
try {
Map domainInfoByFile = domainService.getDomainInfoByFile(SktUtil.getRequestUser(request), srcFile, queryType,export);
return ReturnT.succeed(sketchAuth.getRequestid(), domainInfoByFile);
} catch (Exception e) {
logger.error(e);
return ReturnT.failed(sketchAuth.getRequestid(), e.getMessage());
}
}
}

49
src/main/java/com/mesasoft/cn/sketch/controller/GlobalException.java Normal file
View File

@@ -0,0 +1,49 @@
package com.mesasoft.cn.sketch.controller;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.mesasoft.cn.entity.Result;
import com.mesasoft.cn.entity.ResultEntity;
import com.mesasoft.cn.enums.StatusEnum;
import com.mesasoft.cn.exception.BusinessException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.context.request.async.AsyncRequestTimeoutException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@RestControllerAdvice
public class GlobalException {
private static final Log log = LogFactory.get();
@ExceptionHandler(AsyncRequestTimeoutException.class) //捕获特定异常
public void handleAsyncRequestTimeoutException(AsyncRequestTimeoutException e, HttpServletRequest request) {
log.info("Handle Async Request Timeout Exception");
}
@ExceptionHandler(Exception.class)
public ResultEntity handleException(Exception e, HttpServletRequest request, HttpServletResponse response) {
response.setStatus(StatusEnum.FAIL.getStatus());
String message = e.getMessage() + (e.getCause() != null ? e.getCause().getMessage() : "");
log.error("message:{}, stackTrace:{}", message, getStackTrace(e));
return Result.fail(e.getMessage());
}
@ExceptionHandler({BusinessException.class})
public ResultEntity handleBusinessException(BusinessException e, HttpServletRequest request, HttpServletResponse response) {
response.setStatus(e.getStatus());
String message = (e.getMessage() != null ? e.getMessage() : e.getMessage()) + " " + (e.getCause() != null ? e.getCause().getMessage() : "");
log.error("message:{}.stackTrace:{}", message, getStackTrace(e));
return Result.fail(e.getStatus(), e.getCode(), message);
}
private String getStackTrace(Exception e) {
return ObjectUtil.isNotNull(e.getStackTrace()) ? e.getStackTrace()[0].toString() : "";
}
}

54
src/main/java/com/mesasoft/cn/sketch/controller/SketchDomainController.java
View File

@@ -1,54 +0,0 @@
package com.mesasoft.cn.sketch.controller;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.service.IFileManagerService;
import io.swagger.annotations.Api;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
/**
* @description:
* @author: zhq
* @create: 2022-03-10
**/
@RestController
@RequestMapping("/sketch/domain")
@Api(value = "/sketch/domain", description = "文件相关操作")
@Slf4j
public class SketchDomainController {
private final IFileManagerService fileManagerService;
private final HttpServletRequest request;
private JSONObject jsonObject;
@Value("${sketch.path.admin}")
private String pathAdmin;
@Value("${sketch.path.user}")
private String pathUser;
@Autowired
public SketchDomainController(HttpServletRequest request, IFileManagerService fileManagerService, JSONObject jsonObject) {
this.fileManagerService = fileManagerService;
this.jsonObject = jsonObject;
this.request = request;
}
@AuthInterceptor(InterceptorLevel.USER)
@RequestMapping(value = "/list", method = RequestMethod.GET)
public String list(String searchPath) {
log.info("search path :", searchPath);
return jsonObject.toJSONString();
}
}

58
src/main/java/com/mesasoft/cn/sketch/controller/SketchFileController.java
View File

@@ -2,9 +2,10 @@ package com.mesasoft.cn.sketch.controller;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.service.IFileManagerService;
import com.mesasoft.cn.sketch.util.FileUtils;
import com.mesasoft.cn.sketch.util.SktUtil;
import com.mesasoft.cn.util.ControllerUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.ArrayUtils;
@@ -21,6 +22,7 @@ import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.util.Map;
@@ -39,10 +41,8 @@ public class SketchFileController {
private final HttpServletRequest request;
private JSONObject jsonObject;
@Value("${sketch.path.admin}")
private String pathAdmin;
@Value("${sketch.path.user}")
private String pathUser;
@Value("${sketch.home.path}")
private String sketchHomePath;
@Autowired
public SketchFileController(HttpServletRequest request, IFileManagerService fileManagerService, JSONObject jsonObject) {
@@ -55,39 +55,49 @@ public class SketchFileController {
@RequestMapping(value = "/list", method = RequestMethod.GET)
public String list(String searchPath) {
log.info("search path :", searchPath);
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
log.info("search path : {}", searchPath);
JSONObject json = new JSONObject();
String rootPath = "";
if ("system".equals(user.getUsername())) {
rootPath = pathAdmin;
} else {
rootPath = pathUser;
}
searchPath = searchPath.replace("\\\\", "");
searchPath = searchPath.replace("//", "");
if (StringUtils.isNotBlank(searchPath) && !rootPath.contains(searchPath)) {
json.fluentPut("path", searchPath);
} else {
json.fluentPut("path", rootPath);
}
json.fluentPut("path", getAbsolutePath(searchPath));
log.info("current path : {}", json.getString("path"));
//返回结果
jsonObject.put("path", json.getString("path"));
jsonObject.put("path", searchPath);
jsonObject.put("result", fileManagerService.list(json));
return jsonObject.toJSONString();
}
@RequestMapping(value = "/upload", method = RequestMethod.POST)
public String upload(String destination, MultipartHttpServletRequest request) {
String absolutePath = getAbsolutePath(destination);
log.info("file upload {}", absolutePath);
Map<String, MultipartFile> fileMap = request.getFileMap();
MultipartFile[] files = ArrayUtils.mapToArray(fileMap, MultipartFile.class);
jsonObject.put("result", fileManagerService.upload(destination, files));
jsonObject.put("result", fileManagerService.upload(absolutePath, files));
return jsonObject.toJSONString();
}
@RequestMapping(value = "/download", method = RequestMethod.GET)
public void download(HttpServletRequest request, HttpServletResponse response, String path) throws IOException, ClassNotFoundException {
ControllerUtils.loadResource2(response, path, ValueConsts.TRUE);
log.info("search path : {}", path);
String absolutePath = getAbsolutePath(path);
log.info("down load absolute file path : {} ", absolutePath);
ControllerUtils.loadResource2(response, absolutePath, ValueConsts.TRUE);
}
private String getAbsolutePath(String searchPath) {
String requestUser = SktUtil.getRequestUser(request);
String absolutePath = StringUtils.join(sketchHomePath, File.separator, requestUser, File.separator);
if ("system".equals(requestUser) || "default".equals(requestUser)) {
absolutePath = StringUtils.join(sketchHomePath, File.separator);
}
searchPath = FileUtils.formatPath(searchPath);
if (StringUtils.isNotBlank(searchPath) && !absolutePath.contains(searchPath)) {
absolutePath = StringUtils.join(absolutePath, searchPath, File.separator);
}
log.info(" absolute path : {}", absolutePath);
return absolutePath;
}
}

27
src/main/java/com/mesasoft/cn/sketch/dao/DomainDao.java Normal file
View File

@@ -0,0 +1,27 @@
package com.mesasoft.cn.sketch.dao;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import com.mesasoft.cn.sketch.entity.domain.DomainICP;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import com.mesasoft.cn.sketch.entity.domain.DomainWhois;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface DomainDao {
List<DomainInfo> getCategoryInfo(@Param("domain") List<String> domain);
List<DomainInfo> getWhoisInfo(@Param("domain") List<String> domain);
List<DomainInfo> getICPInfo(@Param("domain") List<String> domain);
void insertCategory(DomainCategory categories);
void insertWhois(DomainWhois data);
void insertICP(DomainICP data);
}

28
src/main/java/com/mesasoft/cn/sketch/config/MariaDbBase.java → src/main/java/com/mesasoft/cn/sketch/dao/MariaDbBase.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.sketch.config;
package com.mesasoft.cn.sketch.dao;
import com.mesasoft.cn.util.TimeUtils;
import com.mesasoft.cn.sketch.util.TimeUtils;
import org.apache.log4j.Logger;
import java.sql.*;
@@ -19,12 +19,13 @@ public class MariaDbBase {
private static final Logger LOG = Logger.getLogger(MariaDbBase.class);
private static final Properties props = new Properties();
private final Statement statement;
private Statement statement;
public MariaDbBase(Connection conn, Statement stat) {
statement = stat;
}
public MariaDbBase() {
}
/**
* 执行写入sql
*/
@@ -53,23 +54,6 @@ public class MariaDbBase {
}
/**
* 获得指定表格、按指定时间字段的过期记录
* @param tableName 库表名称
* @param timeColumnName 时间列名
* @return 查询结果
*/
public ResultSet getExpiredRecord(String tableName, String timeColumnName){
Date lastUpdateTime = new Timestamp(getExpiredTime(ApplicationConfig.UPDATE_EXPIRED_DAY).getTime());
String resSql = "SELECT *"
+ " FROM " + ApplicationConfig.DATABASE + "." + tableName
+ " WHERE " + timeColumnName + " < '" + lastUpdateTime + '\'';
LOG.debug("Update task: expired query sql" + resSql);
return querySqlExecute(resSql);
}
/**
* TODO: getUnlabeledRecord() 考虑多个来源的情况
@@ -81,4 +65,6 @@ public class MariaDbBase {
public static Date getExpiredTime(int expiredRangeDays){
return new Timestamp(TimeUtils.getStartOfDay(-expiredRangeDays).getTime());
}
}

4
src/main/java/com/mesasoft/cn/sketch/dao/SketchDAO.java
View File

@@ -1,8 +1,8 @@
package com.mesasoft.cn.sketch.dao;
import com.mesasoft.cn.sketch.dao.sqlprovider.SketchSqlProvider;
import com.mesasoft.cn.sketch.entity.DomainCategory;
import com.mesasoft.cn.dao.sqlprovider.UserSqlProvider;
import com.mesasoft.cn.sketch.dao.sqlprovider.SketchSqlProvider;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import org.apache.ibatis.annotations.*;
import org.springframework.stereotype.Repository;

6
src/main/java/com/mesasoft/cn/sketch/dao/sqlprovider/SketchSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.sketch.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.Checker;
import org.apache.ibatis.annotations.Param;
@@ -19,9 +19,9 @@ public class SketchSqlProvider {
if (Checker.isNotEmpty(condition)) {
WHERE("fqdn like '%" + condition + "'");
}
ORDER_BY(SketchApplication.settings.getStringUseEval(ConfigConsts.USER_ORDER_BY_OF_SETTINGS));
ORDER_BY(WebSketchApplication.settings.getStringUseEval(ConfigConsts.USER_ORDER_BY_OF_SETTINGS));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.USER_PAGE_SIZE_OF_SETTINGS);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.USER_PAGE_SIZE_OF_SETTINGS);
return sql + " limit " + (offset * size) + "," + size;
}
}

423
src/main/java/com/mesasoft/cn/sketch/entity/DomainWhois.java
View File

@@ -1,423 +0,0 @@
package com.mesasoft.cn.sketch.entity;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import com.mesasoft.cn.util.ConfigUtils;
import com.mesasoft.cn.util.MariaDbBase;
import com.mesasoft.cn.util.ValidationUtils;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
public class DomainWhois {
private static final String dataBase = ApplicationConfig.DATABASE;
private static final String tableName = ApplicationConfig.DOMAIN_WHOIS_TABLENAME;
private String fqdn;
private String source;
private Boolean query_success;
private Integer match_pattern;
private String whois_domain;
private Timestamp whois_update_date;
private Timestamp whois_create_date;
private Timestamp whois_expire_date;
private String whois_email;
private String whois_ns;
private String whois_registrar_name;
private String whois_registrant_org;
private String whois_registrant_name;
private String whois_registrant_street;
private String whois_registrant_city;
private String whois_registrant_state;
private String whois_registrant_postcode;
private String whois_registrant_country;
private String whois_registrant_phone;
// category schema
public DomainWhois(String fqdn,
String source,
Integer match_pattern,
Boolean query_success,
String whoisDomain,
Date whoisUpdateDate,
Date whoisCreateDate,
Date whoisExpireDate,
String whoisEmail,
String whoisNs,
String whoisRegistrarName,
String whoisRegistrantOrg,
String whoisRegistrantName,
String whoisRegistrantStreet,
String whoisRegistrantCity,
String whoisRegistrantState,
String whoisRegistrantPostcode,
String whoisRegistrantCountry,
String whoisRegistrantPhone
) {
this.fqdn = fqdn;
this.source = source;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.query_success = query_success;
this.whois_domain = ConfigUtils.getEffectiveString(whoisDomain);
this.whois_update_date = whoisUpdateDate == null ? null : new Timestamp(whoisUpdateDate.getTime());
this.whois_create_date = whoisCreateDate == null ? null : new Timestamp(whoisCreateDate.getTime());
this.whois_expire_date = whoisExpireDate == null ? null : new Timestamp(whoisExpireDate.getTime());
this.whois_email = ConfigUtils.getEffectiveString(whoisEmail);
this.whois_ns = ConfigUtils.getEffectiveString(whoisNs);
this.whois_registrar_name = ConfigUtils.getEffectiveString(whoisRegistrarName);
this.whois_registrant_org = ConfigUtils.getEffectiveString(whoisRegistrantOrg);
this.whois_registrant_name = ConfigUtils.getEffectiveString(whoisRegistrantName);
this.whois_registrant_street = ConfigUtils.getEffectiveString(whoisRegistrantStreet);
this.whois_registrant_city = ConfigUtils.getEffectiveString(whoisRegistrantCity);
this.whois_registrant_state = ConfigUtils.getEffectiveString(whoisRegistrantState);
this.whois_registrant_postcode = ConfigUtils.getEffectiveString(whoisRegistrantPostcode);
this.whois_registrant_country = ConfigUtils.getEffectiveString(whoisRegistrantCountry);
this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone);
}
public static void insertRecords(List<DomainWhois> whoisFiles, MariaDbBase mariaDbBase) {
for (DomainWhois whoisFile : whoisFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static String insertSql(List<DomainWhois> whoisFiles) {
DomainWhois whoisFile = whoisFiles.get(0);
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
return resSql;
}
public static void updateRecords(List<DomainWhois> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainWhois categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainWhois> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") ";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}
public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", '" + whois_domain + '\'' +
", '" + whois_update_date + '\'' +
", '" + whois_create_date + '\'' +
", '" + whois_expire_date + '\'' +
", '" + whois_email + '\'' +
", '" + whois_ns + '\'' +
", '" + whois_registrar_name + '\'' +
", '" + whois_registrant_org + '\'' +
", '" + whois_registrant_name + '\'' +
", '" + whois_registrant_street + '\'' +
", '" + whois_registrant_city + '\'' +
", '" + whois_registrant_state + '\'' +
", '" + whois_registrant_postcode + '\'' +
", '" + whois_registrant_country + '\'' +
", '" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public static List<DomainWhois> rs2schema(ResultSet rs) throws SQLException {
List<DomainWhois> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainWhois(
rs.getString("fqdn"),
rs.getString("source"),
rs.getInt("match_pattern"),
rs.getBoolean("query_success"),
rs.getString("whois_domain"),
(Date) rs.getDate("whois_update_date"),
(Date) rs.getDate("whois_create_date"),
(Date) rs.getDate("whois_expire_date"),
rs.getString("whois_email"),
rs.getString("whois_ns"),
rs.getString("whois_registrar_name"),
rs.getString("whois_registrant_org"),
rs.getString("whois_registrant_name"),
rs.getString("whois_registrant_street"),
rs.getString("whois_registrant_city"),
rs.getString("whois_registrant_state"),
rs.getString("whois_registrant_postcode"),
rs.getString("whois_registrant_country"),
rs.getString("whois_registrant_phone")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainWhois schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("whois_domain", schema.getWhois_domain());
jsonObject.put("whois_update_date", schema.getWhois_update_date());
jsonObject.put("whois_create_date", schema.getWhois_create_date());
jsonObject.put("whois_expire_date", schema.getWhois_expire_date());
jsonObject.put("whois_email", schema.getWhois_email());
jsonObject.put("whois_ns", schema.getWhois_ns());
jsonObject.put("whois_registrar_name", schema.getWhois_registrar_name());
jsonObject.put("whois_registrant_org", schema.getWhois_registrant_org());
jsonObject.put("whois_registrant_name", schema.getWhois_registrant_name());
jsonObject.put("whois_registrant_street", schema.getWhois_registrant_street());
jsonObject.put("whois_registrant_city", schema.getWhois_registrant_city());
jsonObject.put("whois_registrant_state", schema.getWhois_registrant_state());
jsonObject.put("whois_registrant_postcode", schema.getWhois_registrant_postcode());
jsonObject.put("whois_registrant_country", schema.getWhois_registrant_country());
jsonObject.put("whois_registrant_phone", schema.getWhois_registrant_phone());
return jsonObject;
}
public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", whois_domain" +
", whois_update_date" +
", whois_create_date" +
", whois_expire_date" +
", whois_email" +
", whois_ns" +
", whois_registrar_name" +
", whois_registrant_org" +
", whois_registrant_name" +
", whois_registrant_street" +
", whois_registrant_city" +
", whois_registrant_state" +
", whois_registrant_postcode" +
", whois_registrant_country" +
", whois_registrant_phone";
return resString;
}
public String getKeyValues() {
String resString = "query_success=" + query_success +
", source='" + source + '\'' +
", match_pattern=" + match_pattern +
", whois_domain='" + whois_domain + '\'' +
", whois_update_date='" + whois_update_date + '\'' +
", whois_create_date='" + whois_create_date + '\'' +
", whois_expire_date='" + whois_expire_date + '\'' +
", whois_email='" + whois_email + '\'' +
", whois_ns='" + whois_ns + '\'' +
", whois_registrar_name='" + whois_registrar_name + '\'' +
", whois_registrant_org='" + whois_registrant_org + '\'' +
", whois_registrant_name='" + whois_registrant_name + '\'' +
", whois_registrant_street='" + whois_registrant_street + '\'' +
", whois_registrant_city='" + whois_registrant_city + '\'' +
", whois_registrant_state='" + whois_registrant_state + '\'' +
", whois_registrant_postcode='" + whois_registrant_postcode + '\'' +
", whois_registrant_country='" + whois_registrant_country + '\'' +
", whois_registrant_phone='" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
public Boolean getQuery_success() {
return query_success;
}
public void setQuery_success(Boolean query_success) {
this.query_success = query_success;
}
public Integer getMatch_pattern() {
return match_pattern;
}
public void setMatch_pattern(Integer match_pattern) {
this.match_pattern = match_pattern;
}
public String getWhois_domain() {
return whois_domain;
}
public void setWhois_domain(String whois_domain) {
this.whois_domain = whois_domain;
}
public Timestamp getWhois_update_date() {
return whois_update_date;
}
public void setWhois_update_date(Timestamp whois_update_date) {
this.whois_update_date = whois_update_date;
}
public Timestamp getWhois_create_date() {
return whois_create_date;
}
public void setWhois_create_date(Timestamp whois_create_date) {
this.whois_create_date = whois_create_date;
}
public Timestamp getWhois_expire_date() {
return whois_expire_date;
}
public void setWhois_expire_date(Timestamp whois_expire_date) {
this.whois_expire_date = whois_expire_date;
}
public String getWhois_email() {
return whois_email;
}
public void setWhois_email(String whois_email) {
this.whois_email = whois_email;
}
public String getWhois_ns() {
return whois_ns;
}
public void setWhois_ns(String whois_ns) {
this.whois_ns = whois_ns;
}
public String getWhois_registrar_name() {
return whois_registrar_name;
}
public void setWhois_registrar_name(String whois_registrar_name) {
this.whois_registrar_name = whois_registrar_name;
}
public String getWhois_registrant_org() {
return whois_registrant_org;
}
public void setWhois_registrant_org(String whois_registrant_org) {
this.whois_registrant_org = whois_registrant_org;
}
public String getWhois_registrant_name() {
return whois_registrant_name;
}
public void setWhois_registrant_name(String whois_registrant_name) {
this.whois_registrant_name = whois_registrant_name;
}
public String getWhois_registrant_street() {
return whois_registrant_street;
}
public void setWhois_registrant_street(String whois_registrant_street) {
this.whois_registrant_street = whois_registrant_street;
}
public String getWhois_registrant_city() {
return whois_registrant_city;
}
public void setWhois_registrant_city(String whois_registrant_city) {
this.whois_registrant_city = whois_registrant_city;
}
public String getWhois_registrant_state() {
return whois_registrant_state;
}
public void setWhois_registrant_state(String whois_registrant_state) {
this.whois_registrant_state = whois_registrant_state;
}
public String getWhois_registrant_postcode() {
return whois_registrant_postcode;
}
public void setWhois_registrant_postcode(String whois_registrant_postcode) {
this.whois_registrant_postcode = whois_registrant_postcode;
}
public String getWhois_registrant_country() {
return whois_registrant_country;
}
public void setWhois_registrant_country(String whois_registrant_country) {
this.whois_registrant_country = whois_registrant_country;
}
public String getWhois_registrant_phone() {
return whois_registrant_phone;
}
public void setWhois_registrant_phone(String whois_registrant_phone) {
this.whois_registrant_phone = whois_registrant_phone;
}
}

52
src/main/java/com/mesasoft/cn/sketch/entity/ReturnT.java Normal file
View File

@@ -0,0 +1,52 @@
package com.mesasoft.cn.sketch.entity;
import com.mesasoft.cn.sketch.enums.ResultStatus;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.io.Serializable;
@Data
@ToString
@NoArgsConstructor
@AllArgsConstructor
public class ReturnT<T> implements Serializable {
public static final int SUCCESS_STATUS = ResultStatus.SUCCESS.getStatus();
public static final int FAIL_STATUS = ResultStatus.FAIL.getStatus();
private int status;
private String requestid;
private String message;
private T data;
public static <T> ReturnT succeed(String requestid) {
return new ReturnT(SUCCESS_STATUS, requestid, ResultStatus.SUCCESS.getMessage(), null);
}
public static <T> ReturnT succeed(String requestid, T data) {
return new ReturnT(SUCCESS_STATUS, requestid, ResultStatus.SUCCESS.getMessage(), data);
}
public static <T> ReturnT failed(String requestid) {
return new ReturnT(FAIL_STATUS, requestid, ResultStatus.FAIL.getMessage(), null);
}
public static <T> ReturnT failed(String requestid, String message) {
return new ReturnT(FAIL_STATUS, requestid, message, null);
}
public static <T> ReturnT failed(int status, String requestid, String message) {
return new ReturnT(status, requestid, message, null);
}
}

12
src/main/java/com/mesasoft/cn/sketch/entity/SketchAuth.java Normal file
View File

@@ -0,0 +1,12 @@
package com.mesasoft.cn.sketch.entity;
import lombok.Data;
@Data
public class SketchAuth {
private String omeid;
private String deviceid;
private String uid;
private String requestid;
}

28
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java Normal file
View File

@@ -0,0 +1,28 @@
package com.mesasoft.cn.sketch.entity.domain;
import lombok.Data;
import lombok.ToString;
import java.io.Serializable;
import java.util.Date;
@Data
@ToString
public class DomainBase implements Serializable {
Integer id;
String fqdn;
String source;
Boolean query_success;
Integer match_pattern;
Integer status_code = 0;
// @JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd")
Date create_time;
// @JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd")
Date update_time;
}

141
src/main/java/com/mesasoft/cn/sketch/entity/DomainCategory.java → src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainCategory.java
View File

@@ -1,16 +1,17 @@
package com.mesasoft.cn.sketch.entity;
package com.mesasoft.cn.sketch.entity.domain;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.util.ConfigUtils;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import com.mesasoft.cn.util.MariaDbBase;
import com.mesasoft.cn.util.ValidationUtils;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Created with IntelliJ IDEA.
@@ -19,14 +20,14 @@ import java.util.stream.Collectors;
* Time: 9:27 AM
* Description: No Description
*/
public class DomainCategory {
private static final String dataBase = ApplicationConfig.DATABASE;
private static final String tableName = ApplicationConfig.DOMAIN_CATE_TABLENAME;
private String fqdn;
private String source;
private Boolean query_success;
private Integer match_pattern;
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class DomainCategory extends DomainBase{
private Integer reputation_score;
private String reputation_level;
private Integer category_id;
@@ -34,7 +35,7 @@ public class DomainCategory {
private String category_group;
private Integer category_conf;
private Boolean is_a1_cat;
private Integer status_code = 0;
private String submit_user ="''";
@@ -50,7 +51,6 @@ public class DomainCategory {
String categoryGroup,
Integer categoryConf,
Boolean isA1Cat, Integer statusCode, String submitUser) {
this.fqdn = fqdn;
this.source = source; // 默认应为为brightcloud
this.query_success = query_success;
@@ -105,7 +105,7 @@ public class DomainCategory {
this.is_a1_cat = isA1Cat;
}
public static void insertRecords(List<DomainCategory> categoryFiles, MariaDbBase mariaDbBase) {
/* public static void insertRecords(List<DomainCategory> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainCategory categoryFile : categoryFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
@@ -138,7 +138,7 @@ public class DomainCategory {
" WHERE fqdn in (" + queryFqdns + ") and source = '" + source + "'";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}
}*/
public static List<DomainCategory> rs2schema(ResultSet rs) throws SQLException {
List<DomainCategory> schemaFiles = new ArrayList<>();
@@ -183,7 +183,7 @@ public class DomainCategory {
return jsonObject;
}
public String getValues() {
/*public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
@@ -232,109 +232,6 @@ public class DomainCategory {
", is_a1_cat=" + is_a1_cat;
return resString.replace("'null'", "null");
}
}*/
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public Boolean getQuery_success() {
return query_success;
}
public void setQuery_success(Boolean query_success) {
this.query_success = query_success;
}
public Integer getMatch_pattern() {
return match_pattern;
}
public void setMatch_pattern(Integer match_pattern) {
this.match_pattern = match_pattern;
}
public Integer getReputation_score() {
return reputation_score;
}
public void setReputation_score(Integer reputation_score) {
this.reputation_score = reputation_score;
}
public String getReputation_level() {
return reputation_level;
}
public void setReputation_level(String reputation_level) {
this.reputation_level = reputation_level;
}
public Integer getCategory_id() {
return category_id;
}
public void setCategory_id(Integer category_id) {
this.category_id = category_id;
}
public String getCategory_name() {
return category_name;
}
public void setCategory_name(String category_name) {
this.category_name = category_name;
}
public String getCategory_group() {
return category_group;
}
public void setCategory_group(String category_group) {
this.category_group = category_group;
}
public Integer getCategory_conf() {
return category_conf;
}
public void setCategory_conf(Integer category_conf) {
this.category_conf = category_conf;
}
public Boolean getIs_a1_cat() {
return is_a1_cat;
}
public void setIs_a1_cat(Boolean is_a1_cat) {
this.is_a1_cat = is_a1_cat;
}
public Integer getStatus_code() {
return status_code;
}
public void setStatus_code(Integer status_code) {
this.status_code = status_code;
}
public String getSubmit_user() {
return submit_user;
}
public void setSubmit_user(String submit_user) {
this.submit_user = submit_user;
}
}

46
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainICP.java Normal file
View File

@@ -0,0 +1,46 @@
package com.mesasoft.cn.sketch.entity.domain;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class DomainICP extends DomainBase {
private String icp_owner;
private String icp_company_name;
private String icp_company_type;
private String icp_main_page;
private String icp_site_license;
private String icp_site_name;
private String icp_verify_time;
public DomainICP(String fqdn, String source, Integer match_pattern, Boolean query_success, String icp_owner, String icp_company_name,
String icp_company_type,String icp_main_page, String icp_site_license, String icp_site_name, String icp_verify_time) {
this.fqdn = fqdn;
this.source = source;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.query_success = query_success;
this.icp_owner = icp_owner;
this.icp_company_name = icp_company_name;
this.icp_company_type = icp_company_type;
this.icp_main_page = icp_main_page;
this.icp_site_license = icp_site_license;
this.icp_site_name = icp_site_name;
this.icp_verify_time = icp_verify_time;
}
}

18
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainInfo.java Normal file
View File

@@ -0,0 +1,18 @@
package com.mesasoft.cn.sketch.entity.domain;
import lombok.*;
import java.util.List;
@Data
@Builder
@AllArgsConstructor
@ToString
@NoArgsConstructor
public class DomainInfo<T> {
private String domain;
private List<T> result;
}

96
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java Normal file
View File

@@ -0,0 +1,96 @@
package com.mesasoft.cn.sketch.entity.domain;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.sql.Date;
import java.sql.Timestamp;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
@ToString
public class DomainWhois extends DomainBase{
private String whois_domain;
private Timestamp whois_update_date;
private Timestamp whois_create_date;
private Timestamp whois_expire_date;
private String whois_email;
private String whois_ns;
private String whois_registrar_name;
private String whois_registrant_org;
private String whois_registrant_name;
private String whois_registrant_street;
private String whois_registrant_city;
private String whois_registrant_state;
private String whois_registrant_postcode;
private String whois_registrant_country;
private String whois_registrant_phone;
// category schema
public DomainWhois(String fqdn,
String source,
Integer match_pattern,
Boolean query_success,
String whoisDomain,
Date whoisUpdateDate,
Date whoisCreateDate,
Date whoisExpireDate,
String whoisEmail,
String whoisNs,
String whoisRegistrarName,
String whoisRegistrantOrg,
String whoisRegistrantName,
String whoisRegistrantStreet,
String whoisRegistrantCity,
String whoisRegistrantState,
String whoisRegistrantPostcode,
String whoisRegistrantCountry,
String whoisRegistrantPhone
) {
this.fqdn = fqdn;
this.source = source;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.query_success = query_success;
this.whois_domain = ConfigUtils.getEffectiveString(whoisDomain);
this.whois_update_date = whoisUpdateDate == null ? null : new Timestamp(whoisUpdateDate.getTime());
this.whois_create_date = whoisCreateDate == null ? null : new Timestamp(whoisCreateDate.getTime());
this.whois_expire_date = whoisExpireDate == null ? null : new Timestamp(whoisExpireDate.getTime());
this.whois_email = ConfigUtils.getEffectiveString(whoisEmail);
this.whois_ns = ConfigUtils.getEffectiveString(whoisNs);
this.whois_registrar_name = ConfigUtils.getEffectiveString(whoisRegistrarName);
this.whois_registrant_org = ConfigUtils.getEffectiveString(whoisRegistrantOrg);
this.whois_registrant_name = ConfigUtils.getEffectiveString(whoisRegistrantName);
this.whois_registrant_street = ConfigUtils.getEffectiveString(whoisRegistrantStreet);
this.whois_registrant_city = ConfigUtils.getEffectiveString(whoisRegistrantCity);
this.whois_registrant_state = ConfigUtils.getEffectiveString(whoisRegistrantState);
this.whois_registrant_postcode = ConfigUtils.getEffectiveString(whoisRegistrantPostcode);
this.whois_registrant_country = ConfigUtils.getEffectiveString(whoisRegistrantCountry);
this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone);
}
}

77
src/main/java/com/mesasoft/cn/sketch/enums/ResultStatus.java Normal file
View File

@@ -0,0 +1,77 @@
package com.mesasoft.cn.sketch.enums;
/**
* @Description: 返回码定义
* 规定:
* #1表示成功
* #1001～1999 区间表示参数错误
* #2001～2999 区间表示用户错误
* #3001～3999 区间表示接口异常
* @Date Create in 2019/7/22 19:28
*/
public enum ResultStatus {
/* 成功 */
SUCCESS(200, "success"),
/* 默认失败 */
COMMON_FAIL(400, "fail"),
FAIL(500, "request fail"),
/* 参数错误：1000～1999 */
PARAM_NOT_VALID(1001, "参数无效"),
PARAM_IS_BLANK(1002, "参数为空"),
PARAM_TYPE_ERROR(1003, "参数类型错误"),
PARAM_NOT_COMPLETE(1004, "参数缺失"),
/* 用户错误 */
USER_NOT_LOGIN(2001, "用户未登录"),
USER_ACCOUNT_EXPIRED(2002, "账号已过期"),
USER_CREDENTIALS_ERROR(2003, "密码错误"),
USER_CREDENTIALS_EXPIRED(2004, "密码过期"),
USER_ACCOUNT_DISABLE(2005, "账号不可用"),
USER_ACCOUNT_LOCKED(2006, "账号被锁定"),
USER_ACCOUNT_NOT_EXIST(2007, "账号不存在"),
USER_ACCOUNT_ALREADY_EXIST(2008, "账号已存在"),
USER_ACCOUNT_USE_BY_OTHERS(2009, "账号下线"),
/* 业务错误 */
NO_PERMISSION(3001, "没有权限");
private Integer status;
private String message;
ResultStatus(Integer status, String message) {
this.status = status;
this.message = message;
}
public Integer getStatus() {
return status;
}
public void setStatus(Integer status) {
this.status = status;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
/**
* 根据status获取message
*
* @param status
* @return
*/
public static String getMessageBystatus(Integer status) {
for (ResultStatus ele : values()) {
if (ele.getStatus().equals(status)) {
return ele.getMessage();
}
}
return null;
}
}

461
src/main/java/com/mesasoft/cn/sketch/pojo/IocDarkWeb.java Normal file
View File

@@ -0,0 +1,461 @@
package com.mesasoft.cn.sketch.pojo;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.dao.MariaDbBase;
import java.util.Arrays;
import java.util.List;
public class IocDarkWeb {
private Long id;
private String iocValue;
private String iocType;
private String nodeType;
private String ipv6;
private String isp;
private String asn;
private String operatingSystem;
private String country;
private String city;
private String latitude;
private String Longtitude;
private String torFingerprint;
private Integer torDirPort;
private Integer torOrPort;
private String torFlags;
private String torVersion;
private String torNickname;
private String torExitPolicy;
private String i2PHash;
private String i2PMethod;
private String i2PVersion;
private Boolean i2PIsFloodfill;
private String i2PBandwidth;
private Boolean i2PIsHidden;
private Boolean i2PIsReachable;
private String i2PPublishTime;
private Integer mtproxyPort;
private String mtproxySecret;
private Integer obfs4Port;
private String obfs4Fingerprint;
private String obfs4Cert;
private Integer obfs4IatMode;
private Integer snowflakePort;
private Long firstAppearTime;
private Long lastAppearTime;
private Long createTime;
private Long updateTime;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getIocValue() {
return iocValue;
}
public void setIocValue(String iocValue) {
this.iocValue = iocValue;
}
public String getIocType() {
return iocType;
}
public void setIocType(String iocType) {
this.iocType = iocType;
}
public String getIpv6() {
return ipv6;
}
public void setIpv6(String ipv6) {
this.ipv6 = ipv6;
}
public String getIsp() {
return isp;
}
public void setIsp(String isp) {
this.isp = isp;
}
public String getAsn() {
return asn;
}
public void setAsn(String asn) {
this.asn = asn;
}
public String getOperatingSystem() {
return operatingSystem;
}
public void setOperatingSystem(String operatingSystem) {
this.operatingSystem = operatingSystem;
}
public String getCountry() {
return country;
}
public void setCountry(String country) {
this.country = country;
}
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
public String getLatitude() {
return latitude;
}
public void setLatitude(String latitude) {
this.latitude = latitude;
}
public String getLongtitude() {
return Longtitude;
}
public void setLongtitude(String Longtitude) {
this.Longtitude = Longtitude;
}
public String getTorFingerprint() {
return torFingerprint;
}
public void setTorFingerprint(String torFingerprint) {
this.torFingerprint = torFingerprint;
}
public Integer getTorDirPort() {
return torDirPort;
}
public void setTorDirPort(Integer torDirPort) {
this.torDirPort = torDirPort;
}
public Integer getTorOrPort() {
return torOrPort;
}
public void setTorOrPort(Integer torOrPort) {
this.torOrPort = torOrPort;
}
public String getTorFlags() {
return torFlags;
}
public void setTorFlags(String torFlags) {
this.torFlags = torFlags;
}
public String getTorVersion() {
return torVersion;
}
public void setTorVersion(String torVersion) {
this.torVersion = torVersion;
}
public String getTorNickname() {
return torNickname;
}
public void setTorNickname(String torNickname) {
this.torNickname = torNickname;
}
public String getTorExitPolicy() {
return torExitPolicy;
}
public void setTorExitPolicy(String torExitPolicy) {
this.torExitPolicy = torExitPolicy;
}
public String getI2PHash() {
return i2PHash;
}
public void setI2PHash(String i2PHash) {
this.i2PHash = i2PHash;
}
public String getI2PMethod() {
return i2PMethod;
}
public void setI2PMethod(String i2PMethod) {
this.i2PMethod = i2PMethod;
}
public String getI2PVersion() {
return i2PVersion;
}
public void setI2PVersion(String i2PVersion) {
this.i2PVersion = i2PVersion;
}
public Boolean getI2PIsFloodfill() {
return i2PIsFloodfill;
}
public void setI2PIsFloodfill(Boolean i2PIsFloodfill) {
this.i2PIsFloodfill = i2PIsFloodfill;
}
public String getI2PBandwidth() {
return i2PBandwidth;
}
public void setI2PBandwidth(String i2PBandwidth) {
this.i2PBandwidth = i2PBandwidth;
}
public Boolean getI2PIsHidden() {
return i2PIsHidden;
}
public void setI2PIsHidden(Boolean i2PIsHidden) {
this.i2PIsHidden = i2PIsHidden;
}
public Boolean getI2PIsReachable() {
return i2PIsReachable;
}
public void setI2PIsReachable(Boolean i2PIsReachable) {
this.i2PIsReachable = i2PIsReachable;
}
public String getI2PPublishTime() {
return i2PPublishTime;
}
public void setI2PPublishTime(String i2PPublishTime) {
this.i2PPublishTime = i2PPublishTime;
}
public Integer getMtproxyPort() {
return mtproxyPort;
}
public void setMtproxyPort(Integer mtproxyPort) {
this.mtproxyPort = mtproxyPort;
}
public String getMtproxySecret() {
return mtproxySecret;
}
public void setMtproxySecret(String mtproxySecret) {
this.mtproxySecret = mtproxySecret;
}
public Integer getObfs4Port() {
return obfs4Port;
}
public void setObfs4Port(Integer obfs4Port) {
this.obfs4Port = obfs4Port;
}
public String getObfs4Fingerprint() {
return obfs4Fingerprint;
}
public void setObfs4Fingerprint(String obfs4Fingerprint) {
this.obfs4Fingerprint = obfs4Fingerprint;
}
public String getObfs4Cert() {
return obfs4Cert;
}
public void setObfs4Cert(String obfs4Cert) {
this.obfs4Cert = obfs4Cert;
}
public Integer getObfs4IatMode() {
return obfs4IatMode;
}
public void setObfs4IatMode(Integer obfs4IatMode) {
this.obfs4IatMode = obfs4IatMode;
}
public Integer getSnowflakePort() {
return snowflakePort;
}
public void setSnowflakePort(Integer snowflakePort) {
this.snowflakePort = snowflakePort;
}
public Long getFirstAppearTime() {
return firstAppearTime;
}
public void setFirstAppearTime(Long firstAppearTime) {
this.firstAppearTime = firstAppearTime;
}
public Long getLastAppearTime() {
return lastAppearTime;
}
public void setLastAppearTime(Long lastAppearTime) {
this.lastAppearTime = lastAppearTime;
}
public Long getCreateTime() {
return createTime;
}
public void setCreateTime(Long createTime) {
this.createTime = createTime;
}
public Long getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Long updateTime) {
this.updateTime = updateTime;
}
public String getNodeType() {
return nodeType;
}
public void setNodeType(String nodeType) {
this.nodeType = nodeType;
}
public String getValues() {
return
" '" + iocValue + '\'' +
", '" + iocType + '\'' +
", '" + nodeType + '\'' +
", '" + ipv6 + '\'' +
", '" + isp + '\'' +
", '" + asn + '\'' +
", '" + operatingSystem + '\'' +
", '" + country + '\'' +
", '" + city + '\'' +
", " + latitude +
", " + Longtitude +
", '" + torFingerprint + '\'' +
", " + torDirPort +
", " + torOrPort +
", '" + torFlags + '\'' +
", '" + torVersion + '\'' +
", '" + torNickname + '\'' +
", '" + torExitPolicy + '\'' +
", '" + i2PHash + '\'' +
", '" + i2PMethod + '\'' +
", '" + i2PVersion + '\'' +
", " + i2PIsFloodfill +
", '" + i2PBandwidth + '\'' +
", " + i2PIsHidden +
", " + i2PIsReachable +
", '" + i2PPublishTime + '\'' +
", " + mtproxyPort +
", '" + mtproxySecret + '\'' +
", " + obfs4Port +
", '" + obfs4Fingerprint + '\'' +
", '" + obfs4Cert + '\'' +
", " + obfs4IatMode +
", " + snowflakePort +
", " + firstAppearTime +
", " + lastAppearTime +
", " + createTime +
", " + updateTime;
}
private static final String dataBase = AppConfig.DATABASE;
private static final String tableName = "ioc_darkweb";
private static final String fields = "ioc_value, ioc_type,node_type, ipv6, isp, asn, operating_system, country, city, latitude, Longtitude, tor_fingerprint, " +
"tor_dir_port, tor_or_port, tor_flags, tor_version, tor_nickname, tor_exit_policy, i2p_hash, i2p_method, i2p_version, i2p_is_floodfill, i2p_bandwidth, " +
"i2p_is_hidden, i2p_is_reachable, i2p_publish_time, mtproxy_port, mtproxy_secret, obfs4_port, obfs4_fingerprint, obfs4_cert, obfs4_iat_mode, snowflake_port, " +
"first_appear_time, last_appear_time, create_time, update_time";
public static void insertRecords(List<IocDarkWeb> iocDarkWebs, MariaDbBase mariaDbBase) {
for (IocDarkWeb iocDarkWeb : iocDarkWebs) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + fields + ") values" +
'(' + iocDarkWeb.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static String getInsertSql() {
String[] split = fields.split(",");
Arrays.fill(split, "?");
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + fields + ") values" +
'(' + String.join(",", split) + ')';
return resSql;
}
}

321
src/main/java/com/mesasoft/cn/sketch/service/DarkWebService.java Normal file
View File

@@ -0,0 +1,321 @@
package com.mesasoft.cn.sketch.service;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.file.FileReader;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONNull;
import cn.hutool.json.JSONUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.util.TypeUtils;
import com.google.common.collect.Lists;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.pojo.IocDarkWeb;
import com.mesasoft.cn.sketch.util.MariaDBUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.File;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
public class DarkWebService {
private static Log logger = Log.get();
@Autowired
static AppConfig appConfig;
public static void main(String[] args) {
//1.读取文件，转为list<Map> ，参数 文件目录，多个目录
//tor
//i2p
//mtproxy
//obfs4
//snowflake
//generalbridge
try {
String parentPath = "C:\\test\\h3\\";
String[] apis = {"tor", "i2p", "mtproxy", "obfs4", "snowflake"/*, "generalbridge"*/};
String[] t = {"07", "09", "10", "11", "12", "13", "14", "15", "17", "20"};
String apiType = apis[0];
for (String s : t) {
List<Map> tors = readJson(parentPath + apiType +"\\"+s+"\\");
System.err.println(apiType+"读取日志" + tors.size());
List<IocDarkWeb> list = new ArrayList<>();
//2.转成对象
for (Map data : tors) {
Map result = (Map) data.get("result");
List<Map> nodeList = (List<Map>) result.get("node_list");
for (Map node : nodeList) {
String ipv6s = "";
if (node.get("ipv6") instanceof List) {
List ipv6 = TypeUtils.castToJavaBean(node.get("ipv6"), List.class);
ipv6s = String.join(",", ipv6);
} else {
ipv6s = TypeUtils.castToString(Optional.ofNullable(node.get("ipv6")).orElse(""));
}
//location
Map detail = (Map) node.get("detail");
Map location = (Map) detail.get("location");
String isp = TypeUtils.castToString(location.get("isp"));
String asn = TypeUtils.castToString(location.get("as_number"));
String country = TypeUtils.castToString(location.get("country"));
String city = TypeUtils.castToString(location.get("city"));
String latitude = TypeUtils.castToString(location.get("latitude"));
String longtitude = TypeUtils.castToString(location.get("longtitude"));
Long firstAppearTime = TypeUtils.castToLong(node.get("first_appear_time")) / 1000;
Long lastAppearTime = TypeUtils.castToLong(node.get("last_appear_time")) / 1000;
//创建对象
IocDarkWeb iocDarkWeb = new IocDarkWeb();
iocDarkWeb.setNodeType(apiType);
iocDarkWeb.setIpv6(ipv6s);
//location
iocDarkWeb.setIsp(isp);
iocDarkWeb.setAsn(asn);
iocDarkWeb.setCountry(country);
iocDarkWeb.setCity(city);
iocDarkWeb.setLatitude(latitude);
iocDarkWeb.setLongtitude(longtitude);
//时间
iocDarkWeb.setFirstAppearTime(firstAppearTime);
iocDarkWeb.setLastAppearTime(lastAppearTime);
iocDarkWeb.setCreateTime(DateUtil.currentSeconds());
iocDarkWeb.setUpdateTime(DateUtil.currentSeconds());
iocDarkWeb.setIocType("ip");
switch (apiType) {
case "tor":
//tor
String ip = TypeUtils.castToString(node.get("ip"));
iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
torDetail(node, detail, iocDarkWeb);
break;
case "i2p":
ip = TypeUtils.castToString(node.get("ipv4"));
iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
i2pDetail(detail, iocDarkWeb);
break;
case "mtproxy":
//特殊处理，mtproxy节点中server字段支持域名和ip两种形式
String server = TypeUtils.castToString(node.get("server"));
iocDarkWeb.setIocType(ValidationUtils.isValidIp(server) ? "ip" : "domain");
iocDarkWeb.setIocValue(server);
mtproxyDetail(detail, iocDarkWeb);
break;
case "obfs4":
ip = TypeUtils.castToString(node.get("ipv4"));
iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
obfs4Detail(detail, iocDarkWeb);
break;
case "snowflake":
ip = TypeUtils.castToString(node.get("ipv4"));
iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
snowflakeDetail(detail, iocDarkWeb);
break;
}
list.add(iocDarkWeb);
}
}
//3.写入数据库
List<List<IocDarkWeb>> apiPartitions = Lists.partition(list, appConfig.getDbQueryBatchSize());
for (List<IocDarkWeb> partition : apiPartitions) {
System.err.println("开始写入" + partition.size());
insert(partition);
System.err.println("成功写入" + partition.size());
}
}
System.err.println(apiType+"写入完成");
} catch (Exception e) {
e.printStackTrace();
}
}
private static void snowflakeDetail(Map detail, IocDarkWeb iocDarkWeb) {
//snowflake
Integer snowflakePort = TypeUtils.castToInt(detail.get("port"));
//snowflake
iocDarkWeb.setSnowflakePort(snowflakePort);
}
private static void obfs4Detail(Map detail, IocDarkWeb iocDarkWeb) {
//obfs4
Integer obfs4Port = TypeUtils.castToInt(detail.get("port"));
String obfs4Fingerprint = TypeUtils.castToString(detail.get("fingerprint"));
String obfs4Cert = TypeUtils.castToString(detail.get("cert"));
Integer obfs4IatMode = TypeUtils.castToInt(detail.get("iatMode"));
//obfs4
iocDarkWeb.setObfs4Port(obfs4Port);
iocDarkWeb.setObfs4Fingerprint(obfs4Fingerprint);
iocDarkWeb.setObfs4Cert(obfs4Cert);
iocDarkWeb.setObfs4IatMode(obfs4IatMode);
}
private static void mtproxyDetail(Map detail, IocDarkWeb iocDarkWeb) {
//mtproxy
Integer mtproxyPort = TypeUtils.castToInt(detail.get("port"));
String mtproxySecret = TypeUtils.castToString(detail.get("secret"));
//mtproxy
iocDarkWeb.setMtproxyPort(mtproxyPort);
iocDarkWeb.setMtproxySecret(mtproxySecret);
}
private static void i2pDetail(Map detail, IocDarkWeb iocDarkWeb) {
//i2p
String i2pHash = TypeUtils.castToString(detail.get("hash"));
String i2pMethod = TypeUtils.castToString(detail.get("method"));
String i2pVersion = TypeUtils.castToString(detail.get("version"));
Boolean i2pFloodFill = TypeUtils.castToBoolean(detail.get("isFloodfill"));
String i2pWidth = TypeUtils.castToString(detail.get("width"));
Boolean i2pHidden = TypeUtils.castToBoolean(detail.get("isHidden"));
Boolean i2pReachable = TypeUtils.castToBoolean(detail.get("isReachable"));
String i2pPublish = TypeUtils.castToString(detail.get("publish"));
//i2p
iocDarkWeb.setI2PHash(i2pHash);
iocDarkWeb.setI2PMethod(i2pMethod);
iocDarkWeb.setI2PVersion(i2pVersion);
iocDarkWeb.setI2PIsFloodfill(i2pFloodFill);
iocDarkWeb.setI2PBandwidth(i2pWidth);
iocDarkWeb.setI2PIsHidden(i2pHidden);
iocDarkWeb.setI2PIsReachable(i2pReachable);
iocDarkWeb.setI2PPublishTime(i2pPublish);
}
private static void torDetail(Map node, Map detail, IocDarkWeb iocDarkWeb) {
String fingerprint = TypeUtils.castToString(node.get("fingerprint"));
Integer dirPort = node.get("dir_port") == JSONNull.NULL ? null : TypeUtils.castToInt(node.get("dir_port"));
Integer orPort = node.get("or_port") == JSONNull.NULL ? null : TypeUtils.castToInt(node.get("or_port"));
List flagsList = TypeUtils.castToJavaBean(detail.get("flags"), List.class);
String flags = String.join(",", flagsList);
String nickname = TypeUtils.castToString(detail.get("nickname"));
String exitPolicy = TypeUtils.castToString(detail.get("exit_policy"));
String torVirsion = TypeUtils.castToString(detail.get("tor_virsion"));
String operatingSystem = TypeUtils.castToString(detail.get("host_system"));
//tor
iocDarkWeb.setTorFingerprint(fingerprint);
iocDarkWeb.setTorDirPort(dirPort);
iocDarkWeb.setTorOrPort(orPort);
iocDarkWeb.setTorFlags(flags);
iocDarkWeb.setTorNickname(nickname);
iocDarkWeb.setTorExitPolicy(exitPolicy);
iocDarkWeb.setOperatingSystem(operatingSystem);
iocDarkWeb.setTorVersion(torVirsion);
}
public static List<Map> readJson(String path) {
List<File> files = FileUtil.loopFiles(Paths.get(path), 1, FileFilterUtils.suffixFileFilter("json"));
logger.info("读取文件数量 {}", files.size());
List<Map> jsonList = files.stream().map(f -> {
FileReader fr = new FileReader(f);
Map objMap = JSONUtil.toBean(fr.readString(), Map.class);
return objMap;
}).collect(Collectors.toList());
return jsonList;
}
public static void insert(List<IocDarkWeb> iocDarkWebs) throws Exception {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
Connection connection = MariaDBUtils.getConnection();
connection.setAutoCommit(false);
PreparedStatement pst = connection.prepareStatement(IocDarkWeb.getInsertSql());
for (IocDarkWeb i : iocDarkWebs) {
pst.setString(1, i.getIocValue());
pst.setString(2, i.getIocType());
pst.setString(3, i.getNodeType());
pst.setString(4, i.getIpv6());
pst.setString(5, i.getIsp());
pst.setString(6, i.getAsn());
pst.setString(7, i.getOperatingSystem());
pst.setString(8, i.getCountry());
pst.setString(9, i.getCity());
pst.setString(10, i.getLatitude());
pst.setString(11, i.getLongtitude());
pst.setString(12, i.getTorFingerprint());
pst.setObject(13, i.getTorDirPort());
pst.setObject(14, i.getTorOrPort());
pst.setString(15, i.getTorFlags());
pst.setString(16, i.getTorVersion());
pst.setString(17, i.getTorNickname());
pst.setString(18, i.getTorExitPolicy());
pst.setString(19, i.getI2PHash());
pst.setString(20, i.getI2PMethod());
pst.setString(21, i.getI2PVersion());
pst.setObject(22, i.getI2PIsFloodfill());
pst.setString(23, i.getI2PBandwidth());
pst.setObject(24, i.getI2PIsHidden());
pst.setObject(25, i.getI2PIsReachable());
pst.setString(26, i.getI2PPublishTime());
pst.setObject(27, i.getMtproxyPort());
pst.setString(28, i.getMtproxySecret());
pst.setObject(29, i.getObfs4Port());
pst.setString(30, i.getObfs4Fingerprint());
pst.setString(31, i.getObfs4Cert());
pst.setObject(32, i.getObfs4IatMode());
pst.setObject(33, i.getSnowflakePort());
pst.setLong(34, i.getFirstAppearTime());
pst.setLong(35, i.getLastAppearTime());
pst.setLong(36, i.getCreateTime());
pst.setLong(37, i.getUpdateTime());
pst.addBatch();
}
pst.executeBatch();
connection.commit();
stopWatch.stop();
logger.info("成功提交数据 {} ,花费时间 {} ms", iocDarkWebs.size(), stopWatch.getTime());
MariaDBUtils.close(pst, connection);
}
}

30
src/main/java/com/mesasoft/cn/sketch/service/DomainService.java Normal file
View File

@@ -0,0 +1,30 @@
package com.mesasoft.cn.sketch.service;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import java.util.List;
import java.util.Map;
@Service
public interface DomainService {
List<DomainInfo> getCategoryInfo(List<String> domains, String username) throws Exception;
List<DomainInfo> getCategoryInfoByFile(MultipartHttpServletRequest request, String username) throws Exception;
List<DomainInfo> getWhoisInfo(List<String> domains) throws Exception;
List<DomainInfo> getWhoisInfoByFile(MultipartHttpServletRequest request) throws Exception;
List<DomainInfo> getICPInfo(List<String> domains) throws Exception;
List<DomainInfo> getICPInfoByFile(MultipartHttpServletRequest request) throws Exception;
Map getDomainInfoByFile(String requestUser, String srcFile, String queryType,Boolean export) throws Exception;
}

117
src/main/java/com/mesasoft/cn/sketch/service/FileQuery.java Normal file
View File

@@ -0,0 +1,117 @@
/*
package cn.ac.iie.efo.sketch.service;
import cn.ac.iie.efo.sketch.config.ApplicationConfig;
import cn.ac.iie.efo.sketch.util.FileUtils;
import cn.ac.iie.efo.sketch.util.JsonTool;
import cn.ac.iie.efo.sketch.util.MariaDBUtils;
import cn.ac.iie.efo.sketch.util.ValidationUtils;
import cn.hutool.core.io.file.FileNameUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONArray;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.DecimalFormat;
import java.util.List;
*/
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2022/1/3
* Time: 5:33 PM
* Description: No Description
*//*
public class FileQuery {
private static final Log logger = Log.get();
long queryNum = 0;
long dbResultNum = 0;
long apiResultNum = 0;
long failedQueryNum = 0;
public String queryExecutor(String objectType, String queryType, String queryFile, String username, Boolean isLocal) throws SQLException, IOException {
Connection mariaConn = MariaDBUtils.getConnection();
Statement mariaStat = mariaConn.createStatement();
// 文件读取
InputStreamReader inputStreamReader = new InputStreamReader( new FileInputStream(queryFile), "GBK");
BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
File importFile = new File(queryFile);
String importFileName = FileNameUtil.getName(importFile);
long fileLineNum = Files.lines(importFile.toPath()).count();
logger.info("[File query]-" + importFileName + ": File Found.");
logger.info("[File query]-" + queryType + ".");
String outputFileName = importFileName + "." + queryType + ".json";
File outputFile = new File(ApplicationConfig.QUERY_OUTPUT_DIR + "/" + outputFileName);
if (!outputFile.exists()) {
FileUtils.createFile(new File(ApplicationConfig.QUERY_OUTPUT_DIR), outputFileName);
}
OutputStream outStream = new FileOutputStream(outputFile);
OutputStreamWriter outWriter = new OutputStreamWriter(outStream, StandardCharsets.UTF_8);
outWriter.write("[\n");
logger.info("[File query]-Start query! Progress: 0.00%");
// 执行查询
List<String> objectList;
JSONArray resultJson = new JSONArray();
ListQuery listQueryUtils = new ListQuery();
int batch_index = 1;
// 批量读取
while ((objectList = FileUtils.getBatchLineReadIn(bufferedReader, ApplicationConfig.QUERY_READIN_BATCH)).size() > 0) {
// 校验
objectList = ValidationUtils.getChecked(objectList, objectType);
JSONArray queryResults = listQueryUtils.getQueryResults(objectType, queryType, objectList,username, isLocal);
resultJson.addAll(queryResults);
queryNum += listQueryUtils.queryNum;
dbResultNum += listQueryUtils.dbResultNum;
apiResultNum += listQueryUtils.apiResultNum;
failedQueryNum += listQueryUtils.failedQueryNum;
// 写入output
// todo 写入格式优化 check
for (Object json : queryResults) {
outWriter.write(JsonTool.prettyJson(json.toString()));
}
// 进度日志
if (batch_index * ApplicationConfig.QUERY_READIN_BATCH % ApplicationConfig.QUERY_LOG_FILE_LINE_INTERVAL == 0) {
String percent = new DecimalFormat("##.00%").format((float) queryNum / fileLineNum);
logger.info("[File query]-" + queryType + "-" + importFileName + " Progress:" + percent);
}
batch_index++;
}
logger.info("[File query]-" + queryType + "-" + importFileName + " "
+ "Query result: submit " + queryNum + " valid objects, "
+ dbResultNum + " (" + new DecimalFormat("##.0%").format((float) dbResultNum / queryNum) + ")" + " results from database，"
+ apiResultNum + " (" + new DecimalFormat("##.0%").format((float) apiResultNum / queryNum) + ")" + " results from api. "
+ failedQueryNum + " (" + new DecimalFormat("##.0%").format((float) failedQueryNum / queryNum) + ")" + " failed queries,");
logger.info("[File query]-" + importFileName + " Results saved in " + outputFile.toString());
outWriter.write("\n]");
FileUtils.writerClose(outWriter, outStream);
FileUtils.readerClose(bufferedReader, inputStreamReader);
MariaDBUtils.close(mariaStat, mariaConn);
// todo check 写入null时的过滤情况
return resultJson.toJSONString();
}
}
*/

241
src/main/java/com/mesasoft/cn/sketch/service/ListQuery.java Normal file
View File

File diff suppressed because one or more lines are too long

14
src/main/java/com/mesasoft/cn/sketch/service/SketchService.java
View File

@@ -1,5 +1,5 @@
/*
package com.zhazhapan.efo.sketch.service;
package cn.ac.iie.efo.sketch.service;
import cn.ac.iie.api.ChinaZ;
import cn.ac.iie.dao.MariaDbBase;
@@ -8,12 +8,12 @@ import cn.ac.iie.util.ValidationUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.zhazhapan.efo.sketch.api.BrightCloud;
import com.zhazhapan.efo.sketch.api.ChinaZ;
import com.zhazhapan.efo.sketch.config.ApplicationConfig;
import com.zhazhapan.efo.sketch.entity.DomainCategory;
import com.zhazhapan.efo.sketch.entity.DomainWhois;
import com.zhazhapan.efo.util.FileUtils;
import cn.ac.iie.efo.sketch.api.BrightCloud;
import cn.ac.iie.efo.sketch.api.ChinaZ;
import cn.ac.iie.efo.sketch.config.ApplicationConfig;
import cn.ac.iie.efo.sketch.entity.domain.DomainCategory;
import cn.ac.iie.efo.sketch.entity.domain.DomainWhois;
import cn.ac.iie.efo.util.FileUtils;
import org.apache.log4j.Logger;
import java.io.FileOutputStream;

426
src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java Normal file
View File

@@ -0,0 +1,426 @@
package com.mesasoft.cn.sketch.service.impl;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.Lists;
import com.mesasoft.cn.sketch.api.BrightCloud;
import com.mesasoft.cn.sketch.api.ChinaZ;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.config.BrightCloudConfig;
import com.mesasoft.cn.sketch.config.ChinazConfig;
import com.mesasoft.cn.sketch.dao.DomainDao;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import com.mesasoft.cn.sketch.entity.domain.DomainICP;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import com.mesasoft.cn.sketch.entity.domain.DomainWhois;
import com.mesasoft.cn.sketch.service.DomainService;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import com.zhazhapan.util.ArrayUtils;
import org.apache.commons.io.IOUtils;
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class DomainServiceImpl implements DomainService {
private static final Log logger = Log.get();
@Resource
private BrightCloudConfig bcConfig;
@Resource
private ChinazConfig chinazConfig;
@Resource
private AppConfig appConfig;
@Autowired
private DomainDao domainDao;
@Resource
SqlSessionFactory sqlSessionFactory;
@Value("${query.output.dir}")
private String queryOutputDir;
@Value("${query.readin.batch}")
private int queryReadinBatch;
@Value("${query.mode}")
private int queryMode;
// private static ThreadLocal<Integer> bcSuccessCounter = ThreadLocal.withInitial(() -> 0);
// private static ThreadLocal<Integer> bcFailureCounter = ThreadLocal.withInitial(() -> 0);
@Override
public List<DomainInfo> getCategoryInfo(List<String> domains, String username) throws Exception {
domains = ValidationUtils.getCheckedFqdns(domains);
logger.info("domain category query distinct total : {}", domains.size());
int queryNum = domains.size();
List<DomainInfo> results = new ArrayList<DomainInfo>();
if (queryMode == 1) {
//1. 查询本地数据库
getDomainsByDB(domains, results);
} else if (queryMode == 2) {
//2. 查第三方api ，并更新数据库
getDomainByBrightCloud(domains, username, queryNum, results, 0);
getDomainsByDB(domains, results);
} else {
//3. 查数据库，再查第三方api
int domainsByDBSize = getDomainsByDB(domains, results);
getDomainByBrightCloud(domains, username, queryNum, results, domainsByDBSize);
}
return results;
}
/**
* @param domains
* @param username
* @param queryNum
* @param results
* @param dbResultNum
* @throws IOException
*/
private void getDomainByBrightCloud(List<String> domains, String username, int queryNum, List<DomainInfo> results, int dbResultNum) throws IOException {
//2. 调用api查询
int apiResultNum = 0;
int failedQueryNum = 0;
List<DomainCategory> bcResults = new ArrayList<>();
//批量查询API
BrightCloud brightCloud = new BrightCloud();
List<List<String>> apiPartitions = Lists.partition(domains, bcConfig.getMaximumQueryNum());
List<String> bcDomainRecords = new ArrayList<>();
for (List<String> partition : apiPartitions) {
List<DomainInfo> bcDomainInfos = new ArrayList<>();
List<DomainCategory> recordsFromBcApi = brightCloud.getBrightCloudDomainCategory(partition);
for (DomainCategory record : recordsFromBcApi) {
//查询成功的结果
if (record.getQuery_success().equals(true)) {
record.setSubmit_user(username);
record.setCreate_time(new Date());
record.setUpdate_time(new Date());
bcResults.add(record);
bcDomainInfos.add(DomainInfo.builder()
.domain(record.getFqdn())
.result(Lists.newArrayList(record))
.build());
apiResultNum += 1;
// bcSuccessCounter.set(bcSuccessCounter.get() + 1);
bcDomainRecords.add(record.getFqdn());
} else {
failedQueryNum += 1;
// bcFailureCounter.set(bcFailureCounter.get() + 1);
}
}
if (bcResults.size() > 0) {
insertBatchDomains(bcResults);
results.addAll(bcDomainInfos);
bcResults.clear();
bcDomainInfos.clear();
}
processLog("category", queryNum, apiResultNum, failedQueryNum, dbResultNum);
}
//移除查询到的域名
domains.removeAll(bcDomainRecords);
// // 记录api调用次数
FileWriter fileWriter = new FileWriter(bcConfig.getUsereportFilePath(), true);
fileWriter.write(DateUtil.date() + "," + " domain category query, brightCloud, success records " + apiResultNum + ", fail records " + failedQueryNum + "\n");
fileWriter.close();
logger.info("domain category query brightCloud success records : " + apiResultNum + ",fail records : " + failedQueryNum);
}
private int getDomainsByDB(List<String> domains, List<DomainInfo> results) {
List<List<String>> partitionDomains = Lists.partition(domains, appConfig.getDbQueryBatchSize());
List<String> dBDomains = new ArrayList<>();
for (List<String> partitionDomain : partitionDomains) {
List<DomainInfo> categoryInfos = domainDao.getCategoryInfo(partitionDomain);
results.addAll(categoryInfos);
//移除DB查询结果
categoryInfos.forEach(domainInfo -> dBDomains.add(domainInfo.getDomain()));
}
domains.removeAll(dBDomains);
logger.info("domain category query DB record {}", results.size());
return dBDomains.size();
}
@Override
public List<DomainInfo> getCategoryInfoByFile(MultipartHttpServletRequest request, String username) throws Exception {
return getCategoryInfo(readDomainsByFile(request), username);
}
@Override
public List<DomainInfo> getWhoisInfoByFile(MultipartHttpServletRequest request) throws Exception {
return getWhoisInfo(readDomainsByFile(request));
}
@Override
public List<DomainInfo> getWhoisInfo(List<String> domains) throws Exception {
domains = ValidationUtils.getCheckedFqdns(domains);
logger.info("domain whois query total size {}", domains.size());
int queryNum = domains.size();
ArrayList<DomainInfo> results = new ArrayList<DomainInfo>();
//1.查询本地数据库
List<String> dBDomains = new ArrayList<>();
List<List<String>> partitions = Lists.partition(domains, appConfig.getDbQueryBatchSize());
for (List<String> domainWhois : partitions) {
//批量查询
List<DomainInfo> dbRecords = domainDao.getWhoisInfo(domainWhois);
results.addAll(dbRecords);
dbRecords.forEach(domainInfo -> dBDomains.add(domainInfo.getDomain()));
}
domains.removeAll(dBDomains);
logger.info("domain whois query DB record {}", results.size());
if (queryMode == 1 && domains.size() > 0) {
//2. 调用api
int apiResultNum = 0;
int failedQueryNum = 0;
int dbResultNum = dBDomains.size();
List<DomainWhois> chinazResults = new ArrayList<>();
ChinaZ chinaz = new ChinaZ();
List<List<String>> apiPartitions = Lists.partition(domains, chinazConfig.getMaximumQueryNum());
for (List<String> partition : apiPartitions) {
List<DomainInfo> chinazDomainInfos = new ArrayList<>();
List<DomainWhois> recordsFromApi = chinaz.getDomainsWhois(partition);
for (DomainWhois record : recordsFromApi) {
if (record.getQuery_success().equals(true)) {
// 查询成功的结果
record.setCreate_time(new Date());
record.setUpdate_time(new Date());
chinazResults.add(record);
chinazDomainInfos.add(DomainInfo.builder()
.domain(record.getFqdn())
.result(Lists.newArrayList(record))
.build());
apiResultNum += 1;
} else { // 查询失败的结果
failedQueryNum += 1;
}
}
if (chinazResults.size() > 0) {
insertBatchDomains(chinazResults);
results.addAll(chinazDomainInfos);
chinazResults.clear();
chinazDomainInfos.clear();
}
processLog("whois", queryNum, apiResultNum, failedQueryNum, dbResultNum);
}
// 记录api调用次数
try {
FileWriter fileWriter = new FileWriter(chinazConfig.getUsereportFilePath(), true);
fileWriter.write(DateUtil.date() + "," + " domain whois query, chinaz, success records " + apiResultNum + ", fail records " + failedQueryNum + "\n");
fileWriter.close();
logger.info("domain whois query chinaz success records " + apiResultNum + ",fail records " + failedQueryNum);
} catch (IOException e) {
logger.error(e);
}
}
return results;
}
@Override
public List<DomainInfo> getICPInfo(List<String> domains) throws Exception {
domains = ValidationUtils.getCheckedFqdns(domains);
logger.info("domain ICP query total size {}", domains.size());
int queryNum = domains.size();
ArrayList<DomainInfo> results = new ArrayList<DomainInfo>();
//1.查询本地数据库
List<String> dBDomains = new ArrayList<>();
List<List<String>> partitions = Lists.partition(domains, appConfig.getDbQueryBatchSize());
for (List<String> domainICP : partitions) {
//批量查询
List<DomainInfo> dbRecords = domainDao.getICPInfo(domainICP);
results.addAll(dbRecords);
dbRecords.forEach(domainInfo -> dBDomains.add(domainInfo.getDomain()));
}
domains.removeAll(dBDomains);
logger.info("domain ICP query DB record {}", results.size());
if (queryMode == 1 && domains.size() > 0) {
//2. 调用api
int apiResultNum = 0;
int failedQueryNum = 0;
int dbResultNum = dBDomains.size();
List<DomainICP> chinazResults = new ArrayList<>();
ChinaZ chinaz = new ChinaZ();
List<List<String>> apiPartitions = Lists.partition(domains, chinazConfig.getMaximumQueryNum());
for (List<String> partition : apiPartitions) {
List<DomainInfo> chinazDomainInfos = new ArrayList<>();
List<DomainICP> recordsFromApi = chinaz.getDomainsICP(partition);
for (DomainICP record : recordsFromApi) {
if (record.getQuery_success().equals(true)) {
// 查询成功的结果
record.setCreate_time(new Date());
record.setUpdate_time(new Date());
chinazResults.add(record);
chinazDomainInfos.add(DomainInfo.builder()
.domain(record.getFqdn())
.result(Lists.newArrayList(record))
.build());
apiResultNum += 1;
} else { // 查询失败的结果
failedQueryNum += 1;
}
}
if (chinazResults.size() > 0) {
insertBatchDomains(chinazResults);
results.addAll(chinazDomainInfos);
chinazResults.clear();
chinazDomainInfos.clear();
}
processLog("ICP", queryNum, apiResultNum, failedQueryNum, dbResultNum);
}
// 记录api调用次数
try {
FileWriter fileWriter = new FileWriter(chinazConfig.getUsereportFilePath(), true);
fileWriter.write(DateUtil.date() + "," + " domain ICP query, chinaz, success records " + apiResultNum + ", fail records " + failedQueryNum + "\n");
fileWriter.close();
logger.info("domain ICP query chinaz success records " + apiResultNum + ",fail records " + failedQueryNum);
} catch (IOException e) {
logger.error(e);
}
}
return results;
}
@Override
public List<DomainInfo> getICPInfoByFile(MultipartHttpServletRequest request) throws Exception {
return getICPInfo(readDomainsByFile(request));
}
private List<String> readDomainsByFile(MultipartHttpServletRequest request) {
List<String> domains = new ArrayList<>();
MultipartFile[] domainFiles = ArrayUtils.mapToArray(request.getFileMap(), MultipartFile.class);
for (MultipartFile domainFile : domainFiles) {
logger.info("domain query file ", domainFile.getOriginalFilename());
if (domainFile != null && !domainFile.isEmpty()) {
try {
List<String> lines = IOUtils.readLines(domainFile.getInputStream(), StandardCharsets.UTF_8);
domains.addAll(lines);
} catch (IOException e) {
logger.error("domain query file error {}", e);
}
}
}
return domains;
}
@Override
public Map getDomainInfoByFile(String requestUser, String srcFile, String queryType, Boolean export) throws Exception {
//查询的源文件
String fileName = String.join("-", FileUtil.getPrefix(srcFile), queryType, DateUtil.format(new Date(), "yyyyMMdd-HHmmss") + ".json");
Map<String, Object> resultMsg = new HashMap<>();
resultMsg.put("export", export);
// 文件读取
List<String> readUtf8Lines = FileUtil.readUtf8Lines(srcFile);
logger.info("[File query]-" + srcFile + " ,queryType " + queryType + " query read in batch " + queryReadinBatch + ",domains size " + readUtf8Lines.size());
List<List<String>> partition = ListUtil.partition(readUtf8Lines, queryReadinBatch);
int resultSize = 0;
int querySize = 0;
List<DomainInfo> queryResults = new ArrayList<>();
for (List<String> domains : partition) {
switch (queryType) {
case "category":
queryResults = getCategoryInfo(domains, requestUser);
break;
case "whois":
queryResults = getWhoisInfo(domains);
break;
case "icp":
queryResults = getICPInfo(domains);
break;
}
querySize += domains.size();
resultSize += queryResults.size();
if (ObjectUtil.isNotEmpty(queryResults) && export) {
exportFile(srcFile, fileName, resultMsg, querySize, queryResults);
}
Double queryTotal = Double.valueOf(readUtf8Lines.size());
logger.info("[File query]- " + queryType + " query result records: " + resultSize + ", query records: " + querySize + ", process: " + NumberUtil.decimalFormat("#.##%", querySize / queryTotal));
queryResults.clear();
}
return resultMsg;
}
private void exportFile(String srcFile, String fileName, Map<String, Object> resultMsg, int querySize, List<DomainInfo> queryResults) {
File resultFile = FileUtil.touch(queryOutputDir + File.separator + fileName);
cn.hutool.core.io.file.FileWriter fileWriter = new cn.hutool.core.io.file.FileWriter(resultFile);
List<String> prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x)).collect(Collectors.toList());
//List<String> prettyResults = queryResults.stream().map(x -> JSON.toJSONString(x, true)).collect(Collectors.toList());
fileWriter.appendLines(prettyResults);
resultMsg.put("filePath", resultFile.toString());
resultMsg.put("fileSize", resultFile.length());
resultMsg.put("records", querySize);
logger.info("[File query]-" + srcFile + " result: " + queryResults.size() + " Results saved in " + resultFile.getAbsolutePath());
}
/**
* 域名入库
*
* @param dataList
* @return
*/
public <T> int insertBatchDomains(List<T> dataList) {
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
try {
DomainDao domainDaoBatch = sqlSession.getMapper(DomainDao.class);
dataList.stream().forEach(data -> {
if (data.getClass() == DomainCategory.class) {
domainDaoBatch.insertCategory((DomainCategory) data);
} else if (data.getClass() == DomainWhois.class) {
domainDaoBatch.insertWhois((DomainWhois) data);
} else if (data.getClass() == DomainICP.class) {
domainDaoBatch.insertICP((DomainICP) data);
}
});
sqlSession.commit();
logger.info("insert domain data success ,data size {}", dataList.size());
} catch (Exception e) {
sqlSession.rollback();
logger.error("domain insert error, {}", e);
return 0;
} finally {
sqlSession.clearCache();
sqlSession.close();
}
return dataList.size();
}
private void processLog(String logType, int queryNum, int apiResultNum, int failedQueryNum, int dbResultNum) {
logger.info("[domain " + logType + "]-"
+ "Query result: submit " + queryNum + " valid objects, "
+ dbResultNum + " (" + new DecimalFormat("##.0%").format((float) dbResultNum / queryNum) + ")" + " results from database. "
+ apiResultNum + " (" + new DecimalFormat("##.0%").format((float) apiResultNum / queryNum) + ")" + " results from api. "
+ failedQueryNum + " (" + new DecimalFormat("##.0%").format((float) failedQueryNum / queryNum) + ")" + " failed queries,");
}
}

6
src/main/java/com/mesasoft/cn/util/ConfigUtils.java → src/main/java/com/mesasoft/cn/sketch/util/ConfigUtils.java
View File

@@ -1,4 +1,4 @@
package com.mesasoft.cn.util;
package com.mesasoft.cn.sketch.util;
import org.apache.log4j.Logger;
@@ -35,12 +35,12 @@ public class ConfigUtils {
static {
try {
propCommon.load(ConfigUtils.class.getClassLoader().getResourceAsStream("sketch.properties"));
propCommon.load(ConfigUtils.class.getClassLoader().getResourceAsStream("application.properties"));
// propCommon.load(MariaDbConnection.class.getClassLoader().getResourceAsStream("mariadb.properties"));
} catch (Exception e) {
propCommon = null;
LOG.error("配置加载失败");
}
}
}

72
src/main/java/com/mesasoft/cn/util/FileUtils.java → src/main/java/com/mesasoft/cn/sketch/util/FileUtils.java
View File

@@ -1,4 +1,4 @@
package com.mesasoft.cn.util;
package com.mesasoft.cn.sketch.util;
import org.apache.log4j.Logger;
@@ -14,36 +14,28 @@ import java.util.List;
public class FileUtils {
private static final Logger LOG = Logger.getLogger(FileUtils.class);
public static List<String> readTxtFileIntoStringArrList(String filePath)
{
public static List<String> readTxtFileIntoStringArrList(String filePath) {
List<String> list = new ArrayList<>();
try
{
try {
String encoding = "GBK";
File file = new File(filePath);
if (file.isFile() && file.exists())
{ // 判断文件是否存在
if (file.isFile() && file.exists()) { // 判断文件是否存在
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null)
{
while ((lineTxt = bufferedReader.readLine()) != null) {
if (!lineTxt.equals("")) {
list.add(lineTxt.trim());
}
}
bufferedReader.close();
read.close();
}
else
{
} else {
System.out.println("Can not find file: " + filePath);
}
}
catch (Exception e)
{
} catch (Exception e) {
System.out.println("Error occurred in Function 'readTxtFileIntoStringArrList'");
e.printStackTrace();
}
@@ -51,32 +43,31 @@ public class FileUtils {
return list;
}
public static List<String> getBatchLineReadIn(BufferedReader bufferedReader, int batchSize){
public static List<String> getBatchLineReadIn(BufferedReader bufferedReader, int batchSize) {
List<String> list = new ArrayList<>();
String lineTxt;
try{
while ((lineTxt = bufferedReader.readLine()) != null && list.size()<batchSize)
{
try {
while ((lineTxt = bufferedReader.readLine()) != null && list.size() < batchSize) {
if (!lineTxt.equals("")) {
list.add(lineTxt.trim());
}
}
} catch (IOException e){
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
public static void createFile(File filePath, String fileName){
public static void createFile(File filePath, String fileName) {
try {
File file = new File(filePath.toString() + "/" + fileName);
if (!filePath.exists()){
if (!filePath.exists()) {
filePath.mkdirs();
}
boolean isCreate = file.createNewFile();
if (isCreate){
if (isCreate) {
LOG.info("File " + fileName + " is created.");
}
} catch (IOException e) {
@@ -85,10 +76,10 @@ public class FileUtils {
}
public static void createFile(File file){
public static void createFile(File file) {
try {
boolean isCreate = file.createNewFile();
if (isCreate){
if (isCreate) {
LOG.info("File " + file + " is created.");
}
} catch (IOException e) {
@@ -121,20 +112,20 @@ public class FileUtils {
}
}
public static String getFileName(File file){
public static String getFileName(File file) {
String[] tmp = file.toString().split("/");
String fileName = tmp[tmp.length-1];
String fileName = tmp[tmp.length - 1];
return fileName;
}
public static void writerClose(OutputStreamWriter outWriter, OutputStream outStream) throws IOException {
public static void writerClose(OutputStreamWriter outWriter, OutputStream outStream) throws IOException {
assert outWriter != null;
outWriter.close();
outStream.close();
}
public static void readerClose(BufferedReader bufferedReader, InputStreamReader inputStreamReader) throws IOException {
public static void readerClose(BufferedReader bufferedReader, InputStreamReader inputStreamReader) throws IOException {
assert inputStreamReader != null;
bufferedReader.close();
inputStreamReader.close();
@@ -142,14 +133,13 @@ public class FileUtils {
//执行cmd命令，获取返回结果
public static String execCMD(String command) {
StringBuilder sb =new StringBuilder();
StringBuilder sb = new StringBuilder();
try {
Process process=Runtime.getRuntime().exec(command);
BufferedReader bufferedReader=new BufferedReader(new InputStreamReader(process.getInputStream()));
Process process = Runtime.getRuntime().exec(command);
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while((line=bufferedReader.readLine())!=null)
{
sb.append(line+"\n");
while ((line = bufferedReader.readLine()) != null) {
sb.append(line + "\n");
}
} catch (Exception e) {
return e.toString();
@@ -157,9 +147,9 @@ public class FileUtils {
return sb.toString();
}
public static Long getFileLineNum(File file){
public static Long getFileLineNum(File file) {
Long num = 0L;
if (!file.exists()){
if (!file.exists()) {
LOG.error("File not exist: " + file.toString());
} else {
String res = FileUtils.execCMD("wc -l " + file.toString());
@@ -194,4 +184,12 @@ public class FileUtils {
}
return lengthDomain;
}
public static String formatPath(String path) {
path = path.replace("\\\\", "");
path = path.replace("//", "");
return path;
}
}

70
src/main/java/com/mesasoft/cn/sketch/util/JsonTool.java Normal file
View File

@@ -0,0 +1,70 @@
package com.mesasoft.cn.sketch.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.parser.Feature;
import com.alibaba.fastjson.serializer.SerializerFeature;
import org.apache.commons.lang3.StringUtils;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2022/1/3
* Time: 7:12 PM
* Description: No Description
*/
public class JsonTool {
private boolean isTab = true;
public String stringToJSON(String strJson) {
int tabNum = 0;
StringBuffer jsonFormat = new StringBuffer();
int length = strJson.length();
for (int i = 0; i < length; i++) {
char c = strJson.charAt(i);
if (c == '{') {
tabNum++;
jsonFormat.append(c + "\n");
jsonFormat.append(getSpaceOrTab(tabNum));
} else if (c == '}') {
tabNum--;
jsonFormat.append("\n");
jsonFormat.append(getSpaceOrTab(tabNum));
jsonFormat.append(c);
} else if (c == ',') {
jsonFormat.append(c + "\n");
jsonFormat.append(getSpaceOrTab(tabNum));
} else {
jsonFormat.append(c);
}
}
return jsonFormat.toString();
}
public String getSpaceOrTab(int tabNum) {
StringBuffer sbTab = new StringBuffer();
for (int i = 0; i < tabNum; i++) {
if (isTab) {
sbTab.append('\t');
} else {
sbTab.append(" ");
}
}
return sbTab.toString();
}
public static String prettyJson(String jsonString){
if(StringUtils.isBlank(jsonString)){
return jsonString;
}
String pretty;
try {
JSONObject object = JSONObject.parseObject(jsonString, Feature.OrderedField);
pretty = JSON.toJSONString(object, SerializerFeature.PrettyFormat, SerializerFeature.WriteMapNullValue,
SerializerFeature.WriteDateUseDateFormat);
}catch (Exception e){
return jsonString;
}
return pretty;
}
}

133
src/main/java/com/mesasoft/cn/sketch/util/MariaDBUtils.java Normal file
View File

@@ -0,0 +1,133 @@
package com.mesasoft.cn.sketch.util;
import cn.hutool.core.date.DateUtil;
import com.alibaba.druid.pool.DruidDataSourceFactory;
import me.geso.jdbcutils.JDBCUtils;
import org.apache.commons.lang3.time.StopWatch;
import javax.sql.DataSource;
import java.math.BigInteger;
import java.sql.*;
import java.util.Properties;
/**
* Druid连接池的工具类
*/
public class MariaDBUtils {
private static DataSource ds;
static {
try {
Properties pro = new Properties();
pro.load(JDBCUtils.class.getClassLoader().getResourceAsStream("druid.properties"));
ds = DruidDataSourceFactory.createDataSource(pro);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 获取连接
*/
public static Connection getConnection() throws SQLException {
return ds.getConnection();
}
/**
* 释放资源
*/
public static void close(Statement stmt, Connection conn) {
close(null, stmt, conn);
}
public static void close(ResultSet rs, Statement stmt, Connection conn) {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (stmt != null) {
try {
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
/**
* 获取连接池方法
*/
public static DataSource getDataSource() {
return ds;
}
public static void main(String[] args) {
try {
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");// 驱动包
String url = "jdbc:clickhouse://192.168.44.85:8123/tsg_galaxy_v3";// url路径
String user = "tsg_insert";// 账号
String password = "galaxy2019";// 密码
Connection connection = DriverManager.getConnection(url, user, password);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
System.err.println("开始写入ck数据 ：{}");
connection.setAutoCommit(false);
PreparedStatement preparedStatement = null;
long time = DateUtil.currentSeconds();
time = time - 1;
// String sql = "INSERT INTO tsg_galaxy_v3.connection_record_log_local" +
// "(cap_ip,recv_ip,src_ip,dst_ip,src_port,dst_port,addr_type,protocol,fxo_id,link_status,dir_status,total_cs_pkts,total_sc_pkts,total_cs_bytes,total_sc_bytes,log_gen_time,aa,wv,yy,user_mask,conn_start_time,app_class,app_id,http_host,http_url,http_cookie,`http_user-agent`,http_method,http_accept,http_accept_encoding,http_referer,http_rescode,tls_sni,tls_cert,phone_num,imei,imsi) VALUES " +
// "('127.0.0.1', '192.168.44.12', 'aaa', 'String', 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 1657526027, 123456789, 123456789, 123456789, 123456789, "+ time +", 123456789, 123456789, 'String', 'String', 'String', 'String', 'String', 'String', 'String', 'String', 123456789, 'String', 'String', 123456789, 123456789, 123456789);";
// String sql ="INSERT INTO tsg_galaxy_v3.session_record\n" +
// "(common_recv_time, common_log_id, common_stream_trace_id, common_direction, common_stream_dir, common_start_time, common_end_time, common_con_duration_ms, common_establish_latency_ms, common_processing_time, common_ingestion_time, common_entrance_id, common_device_id, common_egress_link_id, common_ingress_link_id, common_isp, common_data_center, common_sled_ip, common_device_group, common_app_behavior, common_action, common_sub_action, common_policy_id, common_user_tags, common_user_region, common_client_ip, common_internal_ip, common_client_port, common_client_location, common_client_asn, common_subscriber_id, common_imei, common_imsi, common_phone_number, common_server_ip, common_external_ip, common_server_port, common_server_location, common_server_asn, common_app_id, common_userdefine_app_name, common_app_identify_info, common_app_label, common_app_surrogate_id, common_l7_protocol, common_protocol_label, common_service_category, common_service, common_l4_protocol, common_sessions, common_c2s_pkt_num, common_s2c_pkt_num, common_c2s_pkt_diff, common_s2c_pkt_diff, common_c2s_byte_num, common_s2c_byte_num, common_c2s_byte_diff, common_s2c_byte_diff, common_c2s_ipfrag_num, common_s2c_ipfrag_num, common_c2s_tcp_lostlen, common_s2c_tcp_lostlen, common_c2s_tcp_unorder_num, common_s2c_tcp_unorder_num, common_c2s_pkt_retrans, common_s2c_pkt_retrans, common_c2s_byte_retrans, common_s2c_byte_retrans, common_first_ttl, common_tcp_client_isn, common_tcp_server_isn, common_mirrored_pkts, common_mirrored_bytes, common_address_type, common_schema_type, common_device_tag, common_encapsulation, common_tunnels, common_address_list, common_has_dup_traffic, common_stream_error, common_link_info_c2s, common_link_info_s2c, common_packet_capture_file, http_url, http_host, http_domain, http_request_line, http_response_line, http_request_header, http_response_header, http_request_content, http_response_content, http_request_body, http_response_body, http_request_body_key, http_response_body_key, http_proxy_flag, http_sequence, http_snapshot, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_content_length, http_content_type, http_set_cookie, http_version, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_to_cmd, mail_from_cmd, mail_from, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_content, mail_content_charset, mail_attachment_name, mail_attachment_name_charset, mail_attachment_content, mail_eml_file, mail_snapshot, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_pinningst, ssl_intercept_state, ssl_passthrough_reason, ssl_server_side_latency, ssl_client_side_latency, ssl_server_side_version, ssl_client_side_version, ssl_cert_verify, ssl_error, ssl_con_latency_ms, ssl_ja3_fingerprint, ssl_ja3_hash, ssl_cert_issuer, ssl_cert_subject, quic_version, quic_sni, quic_user_agent, ftp_account, ftp_url, ftp_content, ftp_link_type, bgp_type, bgp_as_num, bgp_route, voip_calling_account, voip_called_account, voip_calling_number, voip_called_number, streaming_media_url, streaming_media_protocol, app_extra_info, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, rdp_encryption_method, rdp_encryption_level, rdp_certificate_permanent, rdp_certificate_count, rdp_certificate_type, rdp_requested_color_depth, rdp_desktop_height, rdp_desktop_width, rdp_client_product_id, rdp_client_name, rdp_client_version, rdp_keyboard_layout, rdp_client_channels, rdp_security_protocol, rdp_cookie)\n" +
// "VALUES("+time+", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', 0, 0, '', '', '', '', '', 0, '', 0, '', '', '', '', 0, '', '', '', '', '', '', '', '', 0, '', '', '', '', '', '', '', '', '', null, 0, '', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', '', 0, '', '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', 0, 0, '', 0, '', 0, '', '', '', '', 0, 0, '', 0, 0, '', '', 0, '', 0, '', '', '', '', '', '', '', '', '', '', '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, '', '', '', 0, '', '', 0, '', 0, 0, '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, '', '', '', '', '', '', '', '', '', '', '');\n";
preparedStatement = connection.prepareStatement("INSERT INTO tsg_galaxy_v3.connection_relation_log" +
"(start_time, src_ip, dst_ip,bytes)" +
"VALUES(?, ?, ?, ?)");
for (int i = 0; i < 10; i++) {
preparedStatement.setLong(1, time);
preparedStatement.setString(2, "127.0.0.1");
preparedStatement.setString(3, "127.0.0.1");
BigInteger bytes = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(Long.MAX_VALUE));
System.err.println(bytes);
System.err.println(bytes.bitLength());
preparedStatement.setObject(4, bytes);
preparedStatement.addBatch();
}
preparedStatement.executeBatch();
connection.commit();
stopWatch.stop();
System.err.println("总共花费时间 {}" + stopWatch.getTime());
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException throwables) {
throwables.printStackTrace();
}
}
}

28
src/main/java/com/mesasoft/cn/sketch/util/SktUtil.java Normal file
View File

@@ -0,0 +1,28 @@
package com.mesasoft.cn.sketch.util;
import com.mesasoft.cn.entity.User;
import com.zhazhapan.modules.constant.ValueConsts;
import javax.servlet.http.HttpServletRequest;
public class SktUtil {
/**
* 获取当前请求用户
*
* @param request
* @return
*/
private static String defUsername = "default";
public static String getRequestUser(HttpServletRequest request) {
Object attribute = request.getSession().getAttribute(ValueConsts.USER_STRING);
User user = null;
if (attribute == null) {
return defUsername;
} else {
user = (User) attribute;
}
return user.getUsername();
}
}

95
src/main/java/com/mesasoft/cn/sketch/util/SpringContextUtil.java Normal file
View File

@@ -0,0 +1,95 @@
package com.mesasoft.cn.sketch.util;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
import java.util.Map;
/**
* Spring上下文工具类
*
* @author dazzlzy
* @date 2018/4/25
*/
@Component
public class SpringContextUtil implements ApplicationContextAware {
private static ApplicationContext context = null;
private SpringContextUtil() {
super();
}
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
context = applicationContext;
}
/**
* 根据名称获取bean
*
* @param beanName bean的名字
* @return bean对象
*/
public static Object getBean(String beanName) {
return context.getBean(beanName);
}
/**
* 根据bean名称获取指定类型bean
*
* @param beanName bean名称
* @param clazz 返回的bean类型,若类型不匹配,将抛出异常
*/
public static <T> T getBean(String beanName, Class<T> clazz) {
return context.getBean(beanName, clazz);
}
/**
* 根据类型获取bean
*
* @param clazz bean类型
* @return 指定类型的bean
*/
public static <T> T getBean(Class<T> clazz) {
T t = null;
Map<String, T> map = context.getBeansOfType(clazz);
for (Map.Entry<String, T> entry : map.entrySet()) {
t = entry.getValue();
}
return t;
}
/**
* 是否包含bean
*
* @param beanName bean名字
* @return boolean，是否包含指定名字的bean
*/
public static boolean containsBean(String beanName) {
return context.containsBean(beanName);
}
/**
* 是否是单例
*
* @param beanName bean名字
* @return boolean， 是否是单例
*/
public static boolean isSingleton(String beanName) {
return context.isSingleton(beanName);
}
/**
* bean的类型
*
* @param beanName bean名字
* @return bean的类型
*/
public static Class getType(String beanName) {
return context.getType(beanName);
}
}

2
src/main/java/com/mesasoft/cn/util/TimeUtils.java → src/main/java/com/mesasoft/cn/sketch/util/TimeUtils.java
View File

@@ -1,4 +1,4 @@
package com.mesasoft.cn.util;
package com.mesasoft.cn.sketch.util;
import java.util.Calendar;
import java.util.Date;

93
src/main/java/com/mesasoft/cn/util/ValidationUtils.java → src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java
View File

@@ -1,32 +1,43 @@
package com.mesasoft.cn.util;
package com.mesasoft.cn.sketch.util;
import com.mesasoft.cn.sketch.api.BrightCloud;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.resource.ResourceUtil;
import cn.hutool.log.Log;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import sun.net.util.IPAddressUtil;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ValidationUtils {
private static final Logger LOG = Logger.getLogger(ValidationUtils.class);
private static final Log logger = Log.get();
private static String tldFilePath;
static {
try {
tldFilePath = new File("").getCanonicalPath()+"/config/public_suffix_list_only.dat";
} catch (IOException e) {
logger.error(e);
}
if(!FileUtil.exist(tldFilePath)){
tldFilePath = ResourceUtil.getResource("public_suffix_list_only.dat").getPath();
}
}
/**
* 获取二级域名
**/
public static String getSecDomain(String fqdnOrUrl){
String filePath = Objects.requireNonNull(BrightCloud.class.getClassLoader()
.getResource(ApplicationConfig.TLD_FILE)).getFile();
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(filePath);
**/
public static String getSecDomain(String fqdnOrUrl) {
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(tldFilePath);
try {
String[] split = fqdnOrUrl.split("\\.");
String secDomain = null;
@@ -51,26 +62,25 @@ public class ValidationUtils {
}
}
// 右匹配为顶级域名
if (secDomain == null){
if (secDomain == null) {
secDomain = fqdnOrUrl;
}
return secDomain;
} catch (Exception e) {
LOG.error("urlDomain:" + fqdnOrUrl);
e.printStackTrace();
logger.error("urlDomain:" + fqdnOrUrl);
return "---no---return---";
}
}
public static List<String> getSecDomain(List<String> fqdnOrUrls) {
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(ApplicationConfig.TLD_FILE);
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(tldFilePath);
List<String> secDomainList = new ArrayList<>();
for (String oriDomain : fqdnOrUrls) {
String secDomain = getSecDomain(oriDomain);
if (StringUtils.isNotBlank(secDomain) && !("---no---return---".equals(secDomain))) {
secDomainList.add(secDomain);
} else {
System.out.println(oriDomain);
logger.info(oriDomain);
}
}
return secDomainList;
@@ -90,8 +100,7 @@ public class ValidationUtils {
String encoding = "UTF-8";
File file = new File(filePath);
if (file.isFile() && file.exists()) {
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null) {
@@ -108,11 +117,10 @@ public class ValidationUtils {
}
read.close();
} else {
LOG.error("TopDomainUtils>=>readTopDomainFile filePath is wrong--->{" + filePath + "}<---");
logger.error("TopDomainUtils>=>readTopDomainFile filePath is wrong--->{" + filePath + "}<---");
}
} catch (Exception e) {
LOG.error("TopDomainUtils>=>readTopDomainFile get filePathData error--->{" + e + "}<---");
e.printStackTrace();
logger.error("TopDomainUtils>=>readTopDomainFile get filePathData error--->{" + e + "}<---");
}
return maps;
}
@@ -127,55 +135,54 @@ public class ValidationUtils {
return maps;
}
public static List<String> getChecked(List<String> objectList, String type){
if (type.equals("ip")){
public static List<String> getChecked(List<String> objectList, String type) {
if (type.equals("ip")) {
return getCheckedIps(objectList);
}
if (type.equals("domain")){
if (type.equals("domain")) {
return getCheckedFqdns(objectList);
}
LOG.error("Wrong type to be checked: " + type);
logger.error("Wrong type to be checked: " + type);
return objectList;
}
public static List<String> getCheckedFqdns(List<String> fqdns){
public static List<String> getCheckedFqdns(List<String> fqdns) {
List<String> res = new ArrayList<>();
for (String fqdn:fqdns){
for (String fqdn : fqdns) {
//去端口号
fqdn = fqdn.split(":")[0];
// 去重 & 校验
if (isValidDomain(fqdn) && !res.contains(fqdn)){
if (isValidDomain(fqdn) && !res.contains(fqdn)) {
res.add(fqdn.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + fqdn);
logger.debug("Bad or duplicated fqdn:" + fqdn);
}
}
return res;
}
public static List<String> getCheckedIps(List<String> ipList){
public static List<String> getCheckedIps(List<String> ipList) {
List<String> res = new ArrayList<>();
for (String ip:ipList){
for (String ip : ipList) {
//去端口号
ip = ip.split(":")[0];
// 去重 & 校验
if (isValidIp(ip) && !res.contains(ip)){
if (isValidIp(ip) && !res.contains(ip)) {
res.add(ip.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + ip);
logger.debug("Bad or duplicated fqdn:" + ip);
}
}
return res;
}
public static boolean isValidIp(String ip){
public static boolean isValidIp(String ip) {
boolean iPv4LiteralAddress = IPAddressUtil.isIPv4LiteralAddress(ip);
boolean iPv6LiteralAddress = IPAddressUtil.isIPv6LiteralAddress(ip);
return iPv4LiteralAddress || iPv6LiteralAddress;
}
private static boolean isValidDomain(String str)
{
private static boolean isValidDomain(String str) {
String regex = "^((?!-)[A-Za-z0-9-_]"
+ "{1,63}(?<!-)\\.)"
+ "+[A-Za-z]{2,6}";
@@ -188,15 +195,13 @@ public class ValidationUtils {
return m.matches();
}
public static Integer getMatchPattern(String fqdn){
public static Integer getMatchPattern(String fqdn) {
int match_pattern = 2;
if (fqdn.equals(getSecDomain(fqdn))){
if (fqdn.equals(getSecDomain(fqdn))) {
match_pattern = 1; // 二级域名-右匹配
}
return match_pattern;
}
}

36
src/main/java/com/mesasoft/cn/util/ControllerUtils.java
View File

@@ -1,8 +1,8 @@
package com.mesasoft.cn.util;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.text.UnicodeUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.zhazhapan.util.Checker;
@@ -10,9 +10,11 @@ import org.apache.commons.lang3.StringUtils;
import javax.activation.MimetypesFileTypeMap;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
/**
* @author pantao
@@ -50,7 +52,7 @@ public class ControllerUtils {
if (Checker.isNotEmpty(path)) {
File file = new File(path);
if (download) {
response.setContentType(getContentType(file)+";charset=UTF-8");
response.setContentType(getContentType(file) + ";charset=UTF-8");
setResponseFileName2(response, file.getName());
}
FileInputStream in = new FileInputStream(file);
@@ -68,12 +70,13 @@ public class ControllerUtils {
response.sendRedirect(DefaultValues.NOT_FOUND_PAGE);
}
}
public static void loadResource(HttpServletResponse response, String path, boolean download) throws IOException {
if (Checker.isNotEmpty(path)) {
File file = new File(path);
if (download) {
response.setContentType(getContentType(file));
setResponseFileName( response, file.getName());
setResponseFileName(response, file.getName());
response.setCharacterEncoding("UTF-8");
}
FileInputStream in = new FileInputStream(file);
@@ -91,10 +94,11 @@ public class ControllerUtils {
response.sendRedirect(DefaultValues.NOT_FOUND_PAGE);
}
}
public static String getContentType(File file) {
String defContentType = "application/octet-stream";
String fileName = file.getName();
String fileTyle=fileName.substring(fileName.lastIndexOf(".")+1,fileName.length());
String fileTyle = fileName.substring(fileName.lastIndexOf(".") + 1, fileName.length());
if (StringUtils.isNotBlank(fileTyle)) {
String type2 = Contants.CONTENT_TYPES.get(fileTyle);
if (StringUtils.isNotBlank(type2)) {
@@ -118,29 +122,26 @@ public class ControllerUtils {
public static void setResponseFileName2(HttpServletResponse response, String fileName) {
response.setHeader("Content-Disposition", "attachment;filename=" + UnicodeUtil.toUnicode(fileName));
}
public static void setResponseFileName( HttpServletResponse response, String fileName) throws
public static void setResponseFileName(HttpServletResponse response, String fileName) throws
UnsupportedEncodingException {
response.setHeader("Content-Disposition", "attachment;filename=" + new String(fileName.getBytes("UTF-8"),
"ISO-8859-1"));
}
public static void loadFile(HttpServletRequest request, HttpServletResponse response,String filePath,boolean download) throws IOException {
request.setCharacterEncoding("utf-8");
// 文件存储路径
// 从请求中获取文件名
File file=new File(filePath);
String fileName=file.getName();
public static void loadFile(HttpServletResponse response, String fileName, String data) throws IOException {
// 创建输出流对象
ServletOutputStream outputStream = response.getOutputStream();
//以字节数组的形式读取文件
byte[] bytes = FileUtil.readBytes(filePath);
//以字节数组的形式读取字符串
byte[] bytes = StrUtil.utf8Bytes(data);
// 设置返回内容格式
response.setContentType("application/octet-stream;charset=UTF-8");
String suffixName = StrUtil.subAfter(fileName, ".", true);
response.setContentType("application/" + suffixName + ";charset=UTF-8");
// 把文件名按UTF-8取出并按ISO8859-1编码，保证弹出窗口中的文件名中文不乱码
// 中文不要太多，最多支持17个中文，因为header有150个字节限制。
response.setHeader("filename", UnicodeUtil.toUnicode(fileName));
// 这一步一定要在读取文件之后进行，否则文件名会乱码，找不到文件
fileName = new String(fileName.getBytes("UTF-8"),"ISO-8859-1");
fileName = new String(fileName.getBytes("UTF-8"));
// 设置下载弹窗的文件名和格式（文件名要包括名字和文件格式）
response.setHeader("Content-Disposition", "attachment;filename=" + fileName);
// 返回数据到输出流对象中
@@ -149,4 +150,5 @@ public class ControllerUtils {
IoUtil.close(outputStream);
}
}

80
src/main/java/com/mesasoft/cn/util/MariaDBUtils.java
View File

@@ -1,80 +0,0 @@
package com.mesasoft.cn.util;
import com.alibaba.druid.pool.DruidDataSourceFactory;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
/**
* Druid连接池的工具类
*/
public class MariaDBUtils {
private static DataSource ds ;
static{
try {
Properties pro = new Properties();
// pro.load(JDBCUtils.class.getClassLoader().getResourceAsStream("druid.properties"));
ds = DruidDataSourceFactory.createDataSource(pro);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 获取连接
*/
public static Connection getConnection() throws SQLException {
return ds.getConnection();
}
/**
* 释放资源
*/
public static void close(Statement stmt,Connection conn){
close(null,stmt,conn);
}
public static void close(ResultSet rs , Statement stmt, Connection conn){
if(rs != null){
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if(stmt != null){
try {
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if(conn != null){
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
/**
* 获取连接池方法
*/
public static DataSource getDataSource(){
return ds;
}
}

84
src/main/java/com/mesasoft/cn/util/MariaDbBase.java
View File

@@ -1,84 +0,0 @@
package com.mesasoft.cn.util;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import org.apache.log4j.Logger;
import java.sql.*;
import java.util.Date;
import java.util.Properties;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/28
* Time: 2:56 PM
* Description: No Description
*/
public class MariaDbBase {
private static final Logger LOG = Logger.getLogger(MariaDbBase.class);
private static final Properties props = new Properties();
private final Statement statement;
public MariaDbBase(Connection conn, Statement stat) {
statement = stat;
}
/**
* 执行写入sql
*/
public void writeSqlExecute(String sql){
try {
statement.executeUpdate(sql);
} catch (SQLIntegrityConstraintViolationException e){
LOG.error("Duplicated entry for key 'PRIMARY'");
} catch (SQLException exception) {
LOG.error("Sql : " + sql);
exception.printStackTrace();
}
}
/**
* 执行查询sql
*/
public ResultSet querySqlExecute(String sql){
ResultSet set = null;
try {
set = statement.executeQuery(sql);
} catch (SQLException exception) {
exception.printStackTrace();
}
return set;
}
/**
* 获得指定表格、按指定时间字段的过期记录
* @param tableName 库表名称
* @param timeColumnName 时间列名
* @return 查询结果
*/
public ResultSet getExpiredRecord(String tableName, String timeColumnName){
Date lastUpdateTime = new Timestamp(getExpiredTime(ApplicationConfig.UPDATE_EXPIRED_DAY).getTime());
String resSql = "SELECT *"
+ " FROM " + ApplicationConfig.DATABASE + "." + tableName
+ " WHERE " + timeColumnName + " < '" + lastUpdateTime + '\'';
LOG.debug("Update task: expired query sql" + resSql);
return querySqlExecute(resSql);
}
/**
* TODO: getUnlabeledRecord() 考虑多个来源的情况
*/
/**
* 获得过期时间, 当前时间的expiredRangeDays天之前的日期为过期日期
*/
public static Date getExpiredTime(int expiredRangeDays){
return new Timestamp(TimeUtils.getStartOfDay(-expiredRangeDays).getTime());
}
}

2
src/main/java/com/mesasoft/cn/web/controller/CommonController.java
View File

@@ -1,10 +1,10 @@
package com.mesasoft.cn.web.controller;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.ICommonService;
import com.mesasoft.cn.util.ControllerUtils;
import com.zhazhapan.modules.constant.ValueConsts;

12
src/main/java/com/mesasoft/cn/web/controller/ConfigController.java
View File

@@ -1,10 +1,10 @@
package com.mesasoft.cn.web.controller;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.IConfigService;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.FileExecutor;
@@ -49,11 +49,11 @@ public class ConfigController {
public String updateConfig(String config) {
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
if (user.getPermission() > ValueConsts.TWO_INT) {
SketchApplication.settings.setJsonObject(config);
WebSketchApplication.settings.setJsonObject(config);
//打包成jar之后无法修改config.json文件
try {
FileExecutor.saveFile(NetUtils.urlToString(SketchApplication.class.getResource(DefaultValues
.SETTING_PATH)), SketchApplication.settings.toString());
FileExecutor.saveFile(NetUtils.urlToString(WebSketchApplication.class.getResource(DefaultValues
.SETTING_PATH)), WebSketchApplication.settings.toString());
} catch (IOException e) {
logger.error(e.getMessage());
return "{\"message\":\"internal error, cannot save\"}";
@@ -70,7 +70,7 @@ public class ConfigController {
public String getAll() {
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
if (user.getPermission() > ValueConsts.TWO_INT) {
return SketchApplication.settings.toString();
return WebSketchApplication.settings.toString();
} else {
return "{\"message\":\"permission denied\"}";
}

8
src/main/java/com/mesasoft/cn/web/controller/FileController.java
View File

@@ -2,13 +2,13 @@ package com.mesasoft.cn.web.controller;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.service.IFileService;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.util.ControllerUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
@@ -90,7 +90,7 @@ public class FileController {
@RequestMapping(value = "/all", method = RequestMethod.GET)
public String getAll(int offset, int categoryId, String orderBy, String search) {
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
boolean canGet = SketchApplication.settings.getBooleanUseEval(ConfigConsts.ANONYMOUS_VISIBLE_OF_SETTING) ||
boolean canGet = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.ANONYMOUS_VISIBLE_OF_SETTING) ||
(Checker.isNotNull(user) && user.getIsVisible() == 1);
if (canGet) {
int userId = Checker.isNull(user) ? 0 : user.getId();

3
src/main/java/com/mesasoft/cn/web/controller/UploadedController.java
View File

@@ -3,7 +3,6 @@ package com.mesasoft.cn.web.controller;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.service.IUploadedService;
import com.zhazhapan.util.Formatter;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
@@ -34,6 +33,6 @@ public class UploadedController {
@AuthInterceptor(InterceptorLevel.ADMIN)
@RequestMapping(value = "all", method = RequestMethod.GET)
public String getAll(String user, String file, String category, int offset) {
return Formatter.listToJson(uploadedService.list(user, file, category, offset));
return com.zhazhapan.util.Formatter.listToJson(uploadedService.list(user, file, category, offset));
}
}

23
src/main/java/com/mesasoft/cn/web/controller/UserController.java
View File

@@ -2,16 +2,17 @@ package com.mesasoft.cn.web.controller;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.config.TokenConfig;
import com.mesasoft.cn.entity.Result;
import com.mesasoft.cn.entity.ResultEntity;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.IUserService;
import com.mesasoft.cn.sketch.util.SktUtil;
import com.mesasoft.cn.util.ControllerUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
@@ -102,7 +103,7 @@ public class UserController {
public String updateBasicInfo(String avatar, String realName, String email, String code) {
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
jsonObject.put("message", "保存成功");
boolean emilVerify = SketchApplication.settings.getBooleanUseEval(ConfigConsts.EMAIL_VERIFY_OF_SETTINGS);
boolean emilVerify = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.EMAIL_VERIFY_OF_SETTINGS);
if (Checker.isNotEmpty(email) && !email.equals(user.getEmail())) {
if (!emilVerify || isCodeValidate(code)) {
if (userService.emailExists(email)) {
@@ -192,7 +193,7 @@ public class UserController {
@AuthInterceptor(InterceptorLevel.NONE)
@RequestMapping(value = "/register", method = RequestMethod.POST)
public String register(String username, String email, String password, String code) {
boolean emilVerify = SketchApplication.settings.getBooleanUseEval(ConfigConsts.EMAIL_VERIFY_OF_SETTINGS);
boolean emilVerify = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.EMAIL_VERIFY_OF_SETTINGS);
jsonObject.put("status", "error");
if (!emilVerify || isCodeValidate(code)) {
if (userService.usernameExists(username)) {
@@ -278,4 +279,16 @@ public class UserController {
}
return Result.success(resultObject);
}
@ApiOperation(value = "登出")
@ApiImplicitParams({@ApiImplicitParam(name = "username", value = "用户名"), @ApiImplicitParam(name = "token", value = "用于自动登录")})
@AuthInterceptor(InterceptorLevel.NONE)
@RequestMapping(value = "/logout", method = RequestMethod.PUT)
public String logOut(HttpServletRequest request, String token) {
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
request.getSession().removeAttribute(ValueConsts.USER_STRING);
TokenConfig.removeTokenByValue(user.getId());
jsonObject.put("status", "success");
return jsonObject.toString();
}
}

8
src/main/resources/META-INF/additional-spring-configuration-metadata.json Normal file
View File

@@ -0,0 +1,8 @@
{
"properties": [
{
"name": "bright-cloud.maximum.query.num",
"type": "java.lang.String",
"description": "Description for bright-cloud.maximum.query.num."
}
] }

93
src/main/resources/application-sketch.properties
View File

@@ -1,85 +1,54 @@
driverClassName=org.mariadb.jdbc.Driver
url=jdbc:mariadb://192.168.44.12:3306/web_sketch_v2
username=root
password=galaxy2019
initialSize=5
maxActive=10
maxWait=1000
#datasource
#spring.datasource.url=jdbc:mysql://api.geedge.net:3306/web_sketch_v2?useUnicode=true&characterEncoding=utf-8&useSSL=true&autoReconnect=true&failOverReadOnly=false
spring.datasource.url=jdbc:mysql://192.168.44.12:3306/web_sketch_v2?useUnicode=true&characterEncoding=utf-8&useSSL=true&autoReconnect=true&failOverReadOnly=false
spring.datasource.username=root
spring.datasource.password=galaxy2019
validationQuery=SELECT 1
####################################################
################### application ####################
####################################################
query.types.domain = domain_category, domain_whois
query.types.ip = dns_server
query.output.dir = /home/bigdata/domain/output_file
query.output.dir = c:\\test\\output_file
# 数据库更新定时任务时间间隔1d 1000 * 3600 * 24 = 86400000
update.expired.day = 7
# offline读取批处理量
query.readin.batch = 10000
# 打印进度日志的查询条数
query.log.file.line.interval = 10000
####################################################
################### mariadb ########################
####################################################
database = web_sketch_v2
tablename.domain.category = domain_category_reputation
tablename.domain.whois = domain_whois
tablename.dns.server =
db.query.batch.size = 10000
####################################################
###################### api #########################
####################################################
#update 2 第三方api优先，并更新数据库
#local 1 只查db
#general 0 db优先，不存在则查第三方api
query.mode=2
##### bright cloud #######
bc.oemid = GeedgeNet
bc.deviceid = TSG-Dev
bc.uid = GN0001
bc.url = https://api.bcti.brightcloud.com/1.0/url/getinfo
bc.method = POST
bright-cloud.oemid = GeedgeNet
bright-cloud.deviceid = TSG-Dev
bright-cloud.uid = GN0001
bright-cloud.url = https://api.bcti.brightcloud.com/1.0/url/getinfo
bright-cloud.method = POST
bc.queryType = getinfo
bc.isa1cat = 1
bc.isReputation = 1
# 返回json格式
bc.isxml = 0
# bc api单次查询url长度限制 API最高限制
bc.maximum.query.num = 100
bc.cateinfo.filepath = categoryinfo.json
bc.usereport.filepath = bright_cloud_query_count.csv
bright-cloud.queryType = getinfo
bright-cloud.isa1cat = 1
bright-cloud.isReputation = 1
#bright-cloud返回json格式
bright-cloud.isxml = 0
# api单次查询url长度限制 API最高限制
bright-cloud.maximum-query-num = 100
bright-cloud.cateinfo-filepath = C:\\code\\cn\\webskt-query-agent\\src\\main\\resources\\categoryinfo.json
bright-cloud.usereport-filepath = C:\\code\\cn\\webskt-query-agent\\src\\main\\resources\\bright_cloud_query_count.csv
######### chinaz #########
chinaz.url.single = https://apidatav2.chinaz.com/single/whois
chinaz.url.batch = https://apidatav2.chinaz.com/batch/whois
chinaz.url-single = https://apidatav2.chinaz.com/single
chinaz.url-batch = https://apidatav2.chinaz.com/batch
chinaz.key = ffc9be4141bd49a093ed0185a54dc6a2
chinaz.maximum.query.num = 50
chinaz.usereport.filepath = chinaz_query_count.csv
chinaz.maximum-query-num = 50
chinaz.usereport-filepath = chinaz_query_count.csv
####################################################
###################### 其他 #########################
####################################################
tld.file = public_suffix_list_only.dat
#顶级域名
sketch.tld.file = public_suffix_list_only.dat
sketch.home.path=C:\\test
sketch.datasource.druid.filters=stat
sketch.datasource.druid.driverClassName=com.mysql.jdbc.Driver
sketch.datasource.druid.url=jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&autoReconnect=true&useSSL=false
sketch.datasource.druid.username=root
sketch.datasource.druid.password=12311
sketch.datasource.druid.initialSize=1
sketch.datasource.druid.minIdle=1
sketch.datasource.druid.maxActive=10
sketch.datasource.druid.maxWait=60000
sketch.datasource.druid.timeBetweenEvictionRunsMillis=60000
sketch.datasource.druid.minEvictableIdleTimeMillis=300000
sketch.datasource.druid.validationQuery=SELECT 'x'
sketch.datasource.druid.testWhileIdle=true
sketch.datasource.druid.testOnBorrow=false
sketch.datasource.druid.testOnReturn=false
sketch.datasource.druid.poolPreparedStatements=false
sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize=10

58
src/main/resources/application.properties
View File

@@ -1,28 +1,60 @@
server.port=8196
mybatis.type-aliases-package=com.mesasoft.cn.entity
spring.profiles.include=sketch
mybatis.type-aliases-package=com.mesasoft.cn.entity,com.mesasoft.cn.sketch.entity
mybatis.mapper-locations=classpath:mapper/*.xml
spring.datasource.name=druidDataSource
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driverClassName=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://192.168.44.12:3306/efo?useUnicode=true&characterEncoding=utf-8&useSSL=true
spring.datasource.username=root
spring.datasource.password=galaxy2019
spring.datasource.druid.filters=stat,wall,slf4j,config
spring.datasource.druid.maxActive=100
spring.datasource.druid.initialSize=5
spring.datasource.druid.maxWait=300000
spring.datasource.druid.minIdle=5
spring.datasource.druid.timeBetweenEvictionRunsMillis=300000
spring.datasource.druid.minEvictableIdleTimeMillis=300000
spring.datasource.druid.validationQuery=select 'x'
spring.datasource.druid.testWhileIdle=true
spring.datasource.druid.testOnBorrow=false
spring.datasource.druid.testOnReturn=false
spring.datasource.druid.poolPreparedStatements=true
spring.datasource.druid.maxOpenPreparedStatements=50
spring.datasource.druid.maxPoolPreparedStatementPerConnectionSize=20
spring.datasource.druid.web-stat-filter.enabled=true
spring.datasource.druid.stat-view-servlet.enabled=true
spring.datasource.druid.stat-view-servlet.url-pattern=/druid/*
spring.datasource.druid.stat-view-servlet.login-username=test
spring.datasource.druid.stat-view-servlet.login-password=test
spring.datasource.druid.stat-view-servlet.allow=127.0.0.1
spring.datasource.druid.stat-view-servlet.deny=
spring.datasource.druid.keep-alive=true
spring.datasource.druid.keep-alive-between-time-millis=3600000
spring.jpa.show-sql=true
spring.servlet.multipart.max-file-size=1099511627776
spring.servlet.multipart.max-request-size=1099511627776
#热部署生效
spring.devtools.restart.enabled=false
#页面热部署
spring.thymeleaf.cache=true
spring.devtools.restart.additional-paths=src/main
#classpath目录下的WEB-INF文件夹内容修改不重启
spring.devtools.restart.exclude=WEB-INF/**
#spring.profiles.active=prod
#spring.mvc.favicon.enabled=false
swagger.enabled=true
swagger.title=\u7EBF\u4E0A\u6587\u4EF6\u7BA1\u7406\u7CFB\u7EDF
swagger.description=\u4E0A\u4F20\u3001\u4E0B\u8F7D\u3001\u5206\u4EAB\u3001\u8FDC\u7A0B\u6587\u4EF6\u7BA1\u7406
swagger.title=File Manager
swagger.description=upload,download,export,search
swagger.version=1.2
swagger.license=MIT
swagger.licenseUrl=https://opensource.org/licenses/MIT
swagger.termsOfServiceUrl=https://github.com/code4everything/efo
swagger.contact.name=zhazhapan
swagger.contact.url=https://github.zhazhapan.com
swagger.contact.email=zhazhapan.com
swagger.licenseUrl=test
swagger.termsOfServiceUrl=test
swagger.contact.name=e
swagger.contact.url=test
swagger.contact.email=test.com
swagger.base-package=com.mesasoft.cn
swagger.base-path=/**
swagger.exclude-path=/error, /ops/**
@@ -36,7 +68,3 @@ swagger.global-response-message.get[1].modelRef=ERROR
swagger.global-response-message.post[0].code=500
swagger.global-response-message.post[0].message=500post
swagger.global-response-message.post[0].modelRef=ERROR
sketch.path.admin=C:\\test
sketch.path.user=C:\\test\\1\\

BIN
src/main/resources/assets/img/background_1.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 43 KiB

1
src/main/resources/assets/js/admin.js
View File

@@ -761,4 +761,5 @@ $(document).ready(function () {
$(".file-delete").click(function () {
doDelete("#file-manager-table", app.files, ".file-index", "/file/batch/");
});
});

2
src/main/resources/assets/js/default.js
View File

@@ -70,7 +70,7 @@ $(document).ready(function () {
}
globalConfig.background.imageList = list;
}
changeBackgroundImage();
// changeBackgroundImage();
}
setCSS();
});

21
src/main/resources/assets/js/index.js
View File

@@ -565,12 +565,12 @@ function backPage() {
if(currentPath.endsWith("\\")){
currentPath = currentPath.substring(0, currentPath.lastIndexOf('\\'));
}
backPath = currentPath == '' ? '' : currentPath.substring(0, currentPath.lastIndexOf('\\'))+"\\";
backPath = currentPath == '' ? '' : currentPath.substring(0, currentPath.lastIndexOf('\\'));
} else {
if(currentPath.endsWith("/")){
currentPath = currentPath.substring(0, currentPath.lastIndexOf('\\'));
currentPath = currentPath.substring(0, currentPath.lastIndexOf('/'));
}
backPath = currentPath == '' ? '' : currentPath.substring(0, currentPath.lastIndexOf('/'))+"/";
backPath = currentPath == '' ? '' : currentPath.substring(0, currentPath.lastIndexOf('/'));
}
getFiles(backPath);
if ( $("#history_path").attr("value")==backPath){
@@ -622,3 +622,18 @@ function fileDownLoad(path,name) {
// 发送ajax请求
xhr.send()
}
//退出登录
function logout() {
layer.load(1);
$.ajax({
url: "/user/logout", type: "PUT", data: {
token: getCookie("token")
}, success: function (data) {
layer.closeAll();
var json = JSON.parse(data);
if (json.status === "success") {
window.location.href = "/signin";
}
}
});
}

451
src/main/resources/categoryinfo.json Normal file
View File

@@ -0,0 +1,451 @@
{
"status": 200,
"requestid": "12345",
"type": "url",
"results": [
{
"url": "getcatlist",
"queries": {
"getcatlist": {
"cats": [
{
"catid": 1,
"catname": "Real Estate",
"catgroup": "Productivity"
},
{
"catid": 2,
"catname": "Computer and Internet Security",
"catgroup": "Productivity"
},
{
"catid": 3,
"catname": "Financial Services",
"catgroup": "Privacy"
},
{
"catid": 4,
"catname": "Business and Economy",
"catgroup": "Productivity"
},
{
"catid": 5,
"catname": "Computer and Internet Info",
"catgroup": "Productivity"
},
{
"catid": 6,
"catname": "Auctions",
"catgroup": "Productivity"
},
{
"catid": 7,
"catname": "Shopping",
"catgroup": "Productivity"
},
{
"catid": 8,
"catname": "Cult and Occult",
"catgroup": "Sensitive"
},
{
"catid": 9,
"catname": "Travel",
"catgroup": "Productivity"
},
{
"catid": 10,
"catname": "Abused Drugs",
"catgroup": "Sensitive"
},
{
"catid": 11,
"catname": "Adult and Pornography",
"catgroup": "Sensitive"
},
{
"catid": 12,
"catname": "Home and Garden",
"catgroup": "Productivity"
},
{
"catid": 13,
"catname": "Military",
"catgroup": "Productivity"
},
{
"catid": 14,
"catname": "Social Networking",
"catgroup": "Productivity"
},
{
"catid": 15,
"catname": "Dead Sites",
"catgroup": "Misc"
},
{
"catid": 16,
"catname": "Individual Stock Advice and Tools",
"catgroup": "Productivity"
},
{
"catid": 17,
"catname": "Training and Tools",
"catgroup": "Productivity"
},
{
"catid": 18,
"catname": "Dating",
"catgroup": "Sensitive"
},
{
"catid": 19,
"catname": "Sex Education",
"catgroup": "Sensitive"
},
{
"catid": 20,
"catname": "Religion",
"catgroup": "Sensitive"
},
{
"catid": 21,
"catname": "Entertainment and Arts",
"catgroup": "Productivity"
},
{
"catid": 22,
"catname": "Personal sites and Blogs",
"catgroup": "Productivity"
},
{
"catid": 23,
"catname": "Legal",
"catgroup": "Privacy"
},
{
"catid": 24,
"catname": "Local Information",
"catgroup": "Productivity"
},
{
"catid": 25,
"catname": "Streaming Media",
"catgroup": "IT Resources"
},
{
"catid": 26,
"catname": "Job Search",
"catgroup": "Productivity"
},
{
"catid": 27,
"catname": "Gambling",
"catgroup": "Sensitive"
},
{
"catid": 28,
"catname": "Translation",
"catgroup": "Sensitive"
},
{
"catid": 29,
"catname": "Reference and Research",
"catgroup": "Productivity"
},
{
"catid": 30,
"catname": "Shareware and Freeware",
"catgroup": "IT Resources"
},
{
"catid": 31,
"catname": "Peer to Peer",
"catgroup": "IT Resources"
},
{
"catid": 32,
"catname": "Marijuana",
"catgroup": "Sensitive"
},
{
"catid": 33,
"catname": "Hacking",
"catgroup": "Sensitive"
},
{
"catid": 34,
"catname": "Games",
"catgroup": "Productivity"
},
{
"catid": 35,
"catname": "Philosophy and Political Advocacy",
"catgroup": "Productivity"
},
{
"catid": 36,
"catname": "Weapons",
"catgroup": "Sensitive"
},
{
"catid": 37,
"catname": "Pay to Surf",
"catgroup": "Productivity"
},
{
"catid": 38,
"catname": "Hunting and Fishing",
"catgroup": "Productivity"
},
{
"catid": 39,
"catname": "Society",
"catgroup": "Productivity"
},
{
"catid": 40,
"catname": "Educational Institutions",
"catgroup": "Productivity"
},
{
"catid": 41,
"catname": "Online Greeting Cards",
"catgroup": "IT Resources"
},
{
"catid": 42,
"catname": "Sports",
"catgroup": "Productivity"
},
{
"catid": 43,
"catname": "Swimsuits and Intimate Apparel",
"catgroup": "Sensitive"
},
{
"catid": 44,
"catname": "Questionable",
"catgroup": "Sensitive"
},
{
"catid": 45,
"catname": "Kids",
"catgroup": "Productivity"
},
{
"catid": 46,
"catname": "Hate and Racism",
"catgroup": "Sensitive"
},
{
"catid": 47,
"catname": "Personal Storage",
"catgroup": "IT Resources"
},
{
"catid": 48,
"catname": "Violence",
"catgroup": "Sensitive"
},
{
"catid": 49,
"catname": "Keyloggers and Monitoring",
"catgroup": "Security"
},
{
"catid": 50,
"catname": "Search Engines",
"catgroup": "Productivity"
},
{
"catid": 51,
"catname": "Internet Portals",
"catgroup": "Productivity"
},
{
"catid": 52,
"catname": "Web Advertisements",
"catgroup": "IT Resources"
},
{
"catid": 53,
"catname": "Cheating",
"catgroup": "Sensitive"
},
{
"catid": 54,
"catname": "Gross",
"catgroup": "Sensitive"
},
{
"catid": 55,
"catname": "Web-based Email",
"catgroup": "Privacy"
},
{
"catid": 56,
"catname": "Malware Sites",
"catgroup": "Security"
},
{
"catid": 57,
"catname": "Phishing and Other Frauds",
"catgroup": "Security"
},
{
"catid": 58,
"catname": "Proxy Avoidance and Anonymizers",
"catgroup": "Security"
},
{
"catid": 59,
"catname": "Spyware and Adware",
"catgroup": "Security"
},
{
"catid": 60,
"catname": "Music",
"catgroup": "Productivity"
},
{
"catid": 61,
"catname": "Government",
"catgroup": "Privacy"
},
{
"catid": 62,
"catname": "Nudity",
"catgroup": "Sensitive"
},
{
"catid": 63,
"catname": "News and Media",
"catgroup": "Productivity"
},
{
"catid": 64,
"catname": "Illegal",
"catgroup": "Sensitive"
},
{
"catid": 65,
"catname": "Content Delivery Networks",
"catgroup": "IT Resources"
},
{
"catid": 66,
"catname": "Internet Communications",
"catgroup": "IT Resources"
},
{
"catid": 67,
"catname": "Bot Nets",
"catgroup": "Security"
},
{
"catid": 68,
"catname": "Abortion",
"catgroup": "Sensitive"
},
{
"catid": 69,
"catname": "Health and Medicine",
"catgroup": "Privacy"
},
{
"catid": 70,
"catname": "Confirmed SPAM Sources",
"catgroup": "Unmaintained"
},
{
"catid": 71,
"catname": "SPAM URLs",
"catgroup": "Security"
},
{
"catid": 72,
"catname": "Unconfirmed SPAM Sources",
"catgroup": "Unmaintained"
},
{
"catid": 73,
"catname": "Open HTTP Proxies",
"catgroup": "Unmaintained"
},
{
"catid": 74,
"catname": "Dynamically Generated Content",
"catgroup": "Productivity"
},
{
"catid": 75,
"catname": "Parked Domains",
"catgroup": "Sensitive"
},
{
"catid": 76,
"catname": "Alcohol and Tobacco",
"catgroup": "Sensitive"
},
{
"catid": 77,
"catname": "Private IP Addresses",
"catgroup": "Unmaintained"
},
{
"catid": 78,
"catname": "Image and Video Search",
"catgroup": "Sensitive"
},
{
"catid": 79,
"catname": "Fashion and Beauty",
"catgroup": "Productivity"
},
{
"catid": 80,
"catname": "Recreation and Hobbies",
"catgroup": "Productivity"
},
{
"catid": 81,
"catname": "Motor Vehicles",
"catgroup": "Productivity"
},
{
"catid": 82,
"catname": "Web Hosting",
"catgroup": "IT Resources"
},
{
"catid": 83,
"catname": "Food and Dining",
"catgroup": "Unmaintained"
},
{
"catid": 85,
"catname": "Self Harm",
"catgroup": "Sensitive"
},
{
"catid": 86,
"catname": "Low-THC Cannabis Products",
"catgroup": "Security"
},
{
"catid": 87,
"catname": "Self Harm",
"catgroup": "Sensitive"
},
{
"catid": 88,
"catname": "Generative AI",
"catgroup": "Productivity"
}
]
}
}
}
]
}

18
src/main/resources/config.json
View File

@@ -50,20 +50,18 @@
}
],
"background": {
"useImage": true,
"useImage": false,
"imageIndex": 0,
"imageList": [
"https://fakedog.xyz/homedb/srcm/upload/20180123/bg_1.jpg",
"https://fakedog.xyz/homedb/srcm/upload/20180123/bg_2.jpg",
"https://fakedog.xyz/homedb/srcm/upload/20180123/bg_3.jpg"
"assets/img/background_1.png"
],
"random": true,
"listGenerator": {
"enable": true,
"prefix": "https://fakedog.xyz/image/background/anime/bg_",
"suffix": ".jpg",
"enable": false,
"prefix": "assets/img/background_1",
"suffix": ".png",
"start": 1,
"end": 158
"end": 1
}
}
},
@@ -77,8 +75,8 @@
"emailConfig": {
"host": "",
"port": 587,
"personal": "efo",
"from": "efo-admin@qq.com",
"personal": "webskt",
"from": "webskt-admin@qq.com",
"key": "123456",
"ssl": false
},

29315
src/main/resources/dns_server_v1.csv Normal file
View File

File diff suppressed because one or more lines are too long

3407
src/main/resources/doh_support_list.csv Normal file
View File

File diff suppressed because it is too large Load Diff

230
src/main/resources/mapper/DomainDao.xml Normal file
View File

@@ -0,0 +1,230 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.mesasoft.cn.sketch.dao.DomainDao">
<resultMap id="WhoisResultMap" type="com.mesasoft.cn.sketch.entity.domain.DomainInfo">
<result column="fqdn" jdbcType="VARCHAR" property="domain"/>
<collection property="result" ofType="com.mesasoft.cn.sketch.entity.domain.DomainWhois">
<id column="id" jdbcType="INTEGER" property="id"/>
<result column="fqdn" jdbcType="VARCHAR" property="fqdn"/>
<result column="source" jdbcType="VARCHAR" property="source"/>
<result column="query_success" jdbcType="BIT" property="query_success"/>
<result column="match_pattern" jdbcType="INTEGER" property="match_pattern"/>
<result column="whois_domain" jdbcType="VARCHAR" property="whois_domain"/>
<result column="whois_update_date" jdbcType="TIMESTAMP" property="whois_update_date"/>
<result column="whois_create_date" jdbcType="TIMESTAMP" property="whois_create_date"/>
<result column="whois_expire_date" jdbcType="TIMESTAMP" property="whois_expire_date"/>
<result column="whois_email" jdbcType="VARCHAR" property="whois_email"/>
<result column="whois_ns" jdbcType="VARCHAR" property="whois_ns"/>
<result column="whois_registrar_name" jdbcType="VARCHAR" property="whois_registrar_name"/>
<result column="whois_registrant_org" jdbcType="VARCHAR" property="whois_registrant_org"/>
<result column="whois_registrant_name" jdbcType="VARCHAR" property="whois_registrant_name"/>
<result column="whois_registrant_street" jdbcType="VARCHAR" property="whois_registrant_street"/>
<result column="whois_registrant_city" jdbcType="VARCHAR" property="whois_registrant_city"/>
<result column="whois_registrant_state" jdbcType="VARCHAR" property="whois_registrant_state"/>
<result column="whois_registrant_postcode" jdbcType="VARCHAR" property="whois_registrant_postcode"/>
<result column="whois_registrant_country" jdbcType="VARCHAR" property="whois_registrant_country"/>
<result column="whois_registrant_phone" jdbcType="VARCHAR" property="whois_registrant_phone"/>
<result column="status_code" jdbcType="INTEGER" property="status_code"/>
<result column="create_time" jdbcType="TIMESTAMP" property="create_time"/>
<result column="update_time" jdbcType="TIMESTAMP" property="update_time"/>
</collection>
</resultMap>
<resultMap id="CategoryResultMap" type="com.mesasoft.cn.sketch.entity.domain.DomainInfo">
<result column="fqdn" jdbcType="VARCHAR" property="domain"/>
<collection property="result" ofType="com.mesasoft.cn.sketch.entity.domain.DomainCategory">
<id column="id" jdbcType="INTEGER" property="id"/>
<result column="fqdn" jdbcType="VARCHAR" property="fqdn"/>
<result column="source" jdbcType="VARCHAR" property="source"/>
<result column="query_success" jdbcType="BIT" property="query_success"/>
<result column="match_pattern" jdbcType="INTEGER" property="match_pattern"/>
<result column="reputation_score" jdbcType="INTEGER" property="reputation_score"/>
<result column="reputation_level" jdbcType="VARCHAR" property="reputation_level"/>
<result column="category_id" jdbcType="INTEGER" property="category_id"/>
<result column="category_name" jdbcType="VARCHAR" property="category_name"/>
<result column="category_group" jdbcType="VARCHAR" property="category_group"/>
<result column="category_conf" jdbcType="INTEGER" property="category_conf"/>
<result column="is_a1_cat" jdbcType="BIT" property="is_a1_cat"/>
<result column="status_code" jdbcType="INTEGER" property="status_code"/>
<result column="submit_user" jdbcType="VARCHAR" property="submit_user"/>
<result column="create_time" jdbcType="TIMESTAMP" property="create_time"/>
<result column="update_time" jdbcType="TIMESTAMP" property="update_time"/>
</collection>
</resultMap>
<resultMap id="ICPResultMap" type="com.mesasoft.cn.sketch.entity.domain.DomainInfo">
<result column="fqdn" jdbcType="VARCHAR" property="domain"/>
<collection property="result" ofType="com.mesasoft.cn.sketch.entity.domain.DomainICP">
<id column="id" jdbcType="INTEGER" property="id"/>
<result column="fqdn" jdbcType="VARCHAR" property="fqdn"/>
<result column="source" jdbcType="VARCHAR" property="source"/>
<result column="query_success" jdbcType="BIT" property="query_success"/>
<result column="match_pattern" jdbcType="INTEGER" property="match_pattern"/>
<result column="icp_owner" jdbcType="VARCHAR" property="icp_owner"/>
<result column="icp_company_name" jdbcType="VARCHAR" property="icp_company_name"/>
<result column="icp_company_type" jdbcType="VARCHAR" property="icp_company_type"/>
<result column="icp_main_page" jdbcType="VARCHAR" property="icp_main_page"/>
<result column="icp_site_license" jdbcType="VARCHAR" property="icp_site_license"/>
<result column="icp_site_name" jdbcType="VARCHAR" property="icp_site_name"/>
<result column="icp_verify_time" jdbcType="VARCHAR" property="icp_verify_time"/>
<result column="status_code" jdbcType="INTEGER" property="status_code"/>
<result column="create_time" jdbcType="TIMESTAMP" property="create_time"/>
<result column="update_time" jdbcType="TIMESTAMP" property="update_time"/>
</collection>
</resultMap>
<!-- 查询字段 -->
<sql id="Category_Column_List">
id
, fqdn, `source`, query_success, match_pattern, reputation_score, reputation_level,
category_id, category_name, category_group, category_conf, is_a1_cat, status_code,
submit_user, create_time, update_time
</sql>
<sql id="Whois_Column_List">
id
, fqdn, `source`, query_success, match_pattern, whois_domain, whois_update_date,
whois_create_date, whois_expire_date, whois_email, whois_ns, whois_registrar_name,
whois_registrant_org, whois_registrant_name, whois_registrant_street, whois_registrant_city,
whois_registrant_state, whois_registrant_postcode, whois_registrant_country, whois_registrant_phone,
status_code, create_time, update_time
</sql>
<sql id="ICP_Column_List">
id
, fqdn, `source`, query_success, match_pattern, icp_owner, icp_company_name, icp_company_type,icp_main_page,
icp_site_license, icp_site_name, icp_verify_time, status_code, create_time, update_time
</sql>
<!-- whois info -->
<select id="getWhoisInfo" resultMap="WhoisResultMap">
select
<include refid="Whois_Column_List"/>
from domain_whois
<where>
fqdn in
<foreach collection="domain" item="listItem" open="(" separator="," close=")">
#{listItem}
</foreach>
</where>
</select>
<!-- category info -->
<select id="getCategoryInfo" resultMap="CategoryResultMap">
select
<include refid="Category_Column_List"/>
from domain_category_reputation
<where>
fqdn in
<foreach collection="domain" item="listItem" open="(" separator="," close=")">
#{listItem}
</foreach>
</where>
</select>
<!-- ICP info -->
<select id="getICPInfo" resultMap="ICPResultMap">
select
<include refid="ICP_Column_List"/>
from domain_icp
<where>
fqdn in
<foreach collection="domain" item="listItem" open="(" separator="," close=")">
#{listItem}
</foreach>
</where>
</select>
<!-- category insert -->
<insert id="insertCategory" keyColumn="id" keyProperty="id"
parameterType="com.mesasoft.cn.sketch.entity.domain.DomainCategory" useGeneratedKeys="true">
insert into domain_category_reputation
(fqdn,
`source`,
query_success,
match_pattern,
reputation_score,
reputation_level,
category_id,
category_name,
category_group,
category_conf,
is_a1_cat,
status_code,
submit_user,
create_time,
update_time)
values (#{fqdn,jdbcType=VARCHAR},
#{source,jdbcType=VARCHAR},
#{query_success,jdbcType=BIT},
#{match_pattern,jdbcType=INTEGER},
#{reputation_score,jdbcType=INTEGER},
#{reputation_level,jdbcType=VARCHAR},
#{category_id,jdbcType=INTEGER},
#{category_name,jdbcType=VARCHAR},
#{category_group,jdbcType=VARCHAR},
#{category_conf,jdbcType=INTEGER},
#{is_a1_cat,jdbcType=BIT},
#{status_code,jdbcType=INTEGER},
#{submit_user,jdbcType=VARCHAR},
#{create_time,jdbcType=TIMESTAMP},
#{update_time,jdbcType=TIMESTAMP})
ON DUPLICATE KEY UPDATE
`source` = VALUES(`source`),
query_success = VALUES(query_success),
match_pattern = VALUES(match_pattern),
reputation_score = VALUES(reputation_score),
reputation_level = VALUES(reputation_level),
category_id = VALUES(category_id),
category_name = VALUES(category_name),
category_group = VALUES(category_group),
category_conf = VALUES(category_conf),
is_a1_cat = VALUES(is_a1_cat),
status_code = VALUES(status_code),
submit_user = VALUES(submit_user),
create_time = VALUES(create_time),
update_time = VALUES(update_time);
</insert>
<insert id="insertWhois" keyColumn="id" keyProperty="id"
parameterType="com.mesasoft.cn.sketch.entity.domain.DomainWhois" useGeneratedKeys="true">
insert into domain_whois
(fqdn, `source`, query_success,
match_pattern, whois_domain, whois_update_date,
whois_create_date, whois_expire_date,
whois_email, whois_ns, whois_registrar_name,
whois_registrant_org, whois_registrant_name,
whois_registrant_street, whois_registrant_city,
whois_registrant_state, whois_registrant_postcode,
whois_registrant_country, whois_registrant_phone,
status_code, create_time, update_time)
values (#{fqdn,jdbcType=VARCHAR}, #{source,jdbcType=VARCHAR}, #{query_success,jdbcType=BIT},
#{match_pattern,jdbcType=INTEGER}, #{whois_domain,jdbcType=VARCHAR},
#{whois_update_date,jdbcType=TIMESTAMP},
#{whois_create_date,jdbcType=TIMESTAMP}, #{whois_expire_date,jdbcType=TIMESTAMP},
#{whois_email,jdbcType=VARCHAR}, #{whois_ns,jdbcType=VARCHAR}, #{whois_registrar_name,jdbcType=VARCHAR},
#{whois_registrant_org,jdbcType=VARCHAR}, #{whois_registrant_name,jdbcType=VARCHAR},
#{whois_registrant_street,jdbcType=VARCHAR}, #{whois_registrant_city,jdbcType=VARCHAR},
#{whois_registrant_state,jdbcType=VARCHAR}, #{whois_registrant_postcode,jdbcType=VARCHAR},
#{whois_registrant_country,jdbcType=VARCHAR}, #{whois_registrant_phone,jdbcType=VARCHAR},
#{status_code,jdbcType=INTEGER}, #{create_time,jdbcType=TIMESTAMP}, #{update_time,jdbcType=TIMESTAMP})
</insert>
<insert id="insertICP" keyColumn="id" keyProperty="id"
parameterType="com.mesasoft.cn.sketch.entity.domain.DomainICP" useGeneratedKeys="true">
insert into domain_icp (fqdn, `source`, query_success,
match_pattern, icp_owner, icp_company_name,
icp_company_type, icp_main_page,icp_site_license, icp_site_name,
icp_verify_time, status_code, create_time,
update_time)
values (#{fqdn,jdbcType=VARCHAR}, #{source,jdbcType=VARCHAR}, #{query_success,jdbcType=BIT},
#{match_pattern,jdbcType=INTEGER}, #{icp_owner,jdbcType=VARCHAR}, #{icp_company_name,jdbcType=VARCHAR},
#{icp_company_type,jdbcType=VARCHAR}, #{icp_main_page,jdbcType=VARCHAR}, #{icp_site_license,jdbcType=VARCHAR},
#{icp_site_name,jdbcType=VARCHAR},
#{icp_verify_time,jdbcType=VARCHAR}, #{status_code,jdbcType=INTEGER}, #{create_time,jdbcType=TIMESTAMP},
#{update_time,jdbcType=TIMESTAMP})
</insert>
</mapper>

9142
src/main/resources/public_suffix_list_only.dat Normal file
View File

File diff suppressed because it is too large Load Diff

85
src/main/resources/sketch.properties
View File

@@ -1,85 +0,0 @@
driverClassName=org.mariadb.jdbc.Driver
url=jdbc:mariadb://192.168.44.12:3306/web_sketch_v2
username=root
password=galaxy2019
initialSize=5
maxActive=10
maxWait=1000
validationQuery=SELECT 1
####################################################
################### application ####################
####################################################
query.types.domain = domain_category, domain_whois
query.types.ip = dns_server
query.output.dir = /home/bigdata/domain/output_file
# 数据库更新定时任务时间间隔1d 1000 * 3600 * 24 = 86400000
update.expired.day = 7
# offline读取批处理量
query.readin.batch = 10000
# 打印进度日志的查询条数
query.log.file.line.interval = 10000
####################################################
################### mariadb ########################
####################################################
database = web_sketch_v2
tablename.domain.category = domain_category_reputation
tablename.domain.whois = domain_whois
tablename.dns.server =
db.query.batch.size = 10000
####################################################
###################### api #########################
####################################################
##### bright cloud #######
bc.oemid = GeedgeNet
bc.deviceid = TSG-Dev
bc.uid = GN0001
bc.url = https://api.bcti.brightcloud.com/1.0/url/getinfo
bc.method = POST
bc.queryType = getinfo
bc.isa1cat = 1
bc.isReputation = 1
# 返回json格式
bc.isxml = 0
# bc api单次查询url长度限制 API最高限制
bc.maximum.query.num = 100
bc.cateinfo.filepath = categoryinfo.json
bc.usereport.filepath = bright_cloud_query_count.csv
######### chinaz #########
chinaz.url.single = https://apidatav2.chinaz.com/single/whois
chinaz.url.batch = https://apidatav2.chinaz.com/batch/whois
chinaz.key = ffc9be4141bd49a093ed0185a54dc6a2
chinaz.maximum.query.num = 50
chinaz.usereport.filepath = chinaz_query_count.csv
####################################################
###################### 其他 #########################
####################################################
tld.file = public_suffix_list_only.dat
sketch.datasource.druid.filters=stat
sketch.datasource.druid.driverClassName=com.mysql.jdbc.Driver
sketch.datasource.druid.url=jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&autoReconnect=true&useSSL=false
sketch.datasource.druid.username=root
sketch.datasource.druid.password=12311
sketch.datasource.druid.initialSize=1
sketch.datasource.druid.minIdle=1
sketch.datasource.druid.maxActive=10
sketch.datasource.druid.maxWait=60000
sketch.datasource.druid.timeBetweenEvictionRunsMillis=60000
sketch.datasource.druid.minEvictableIdleTimeMillis=300000
sketch.datasource.druid.validationQuery=SELECT 'x'
sketch.datasource.druid.testWhileIdle=true
sketch.datasource.druid.testOnBorrow=false
sketch.datasource.druid.testOnReturn=false
sketch.datasource.druid.poolPreparedStatements=false
sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize=10

28
src/main/resources/static/index.html
View File

@@ -31,7 +31,7 @@
</head>
<body id="particles-js">
<nav class="navbar navbar-expand-md bg-dark navbar-dark navbar-toggler fixed-top" >
<img src="/assets/sketch/images/att-globe.svg" class="navbar-brand" style="width: 2rem;height: 2.5rem;"/>
<img src="/assets/sketch/images/att-globe.svg" class="navbar-brand" style="width: 2rem;height: 3rem;"/>
<button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#collapsibleNavbar">
<span class="navbar-toggler-icon"></span>
</button>
@@ -40,19 +40,19 @@
<li class="nav-item">
<a class="nav-link active" data-toggle="pill" href="#files-tab">文件列表</a>
</li>
<li class="nav-item">
<li class="nav-item" hidden>
<a class="nav-link" data-toggle="pill" href="#resource-tab">资源列表</a>
</li>
<li class="nav-item">
<li class="nav-item" hidden>
<a class="nav-link" data-toggle="pill" href="#uploaded-tab">我的资源</a>
</li>
<li class="nav-item">
<li class="nav-item" hidden>
<a class="nav-link" data-toggle="pill" href="#downloaded-tab">我的下载</a>
</li>
<li class="nav-item">
<li class="nav-item" hidden>
<a class="nav-link" data-toggle="pill" href="#bio-tab">我的资料</a>
</li>
<li class="nav-item">
<li class="nav-item" hidden>
<a class="nav-link" href="upload" target="_blank">文件上传</a>
</li>
</ul>
@@ -61,9 +61,12 @@
<input class="form-control border-dark bg-dark text-white form-control-sm w-100" type="search"
placeholder="搜索" id="search"/>
</div>-->
<div>
<a class="nav-link text-muted" href="javascript:;" onclick="logout();">退出</a>
</div>
</nav>
<br/><br/><br/>
<div class="container" id="index">
<div class="container" id="index" style="max-width: 98%" >
<div class="row justify-content-center">
<div class="col-sm-12 col-10 tab-content">
<!--文件列表-->
@@ -71,13 +74,18 @@
<div class="row content-box rounded">
<!-- Header -->
<header id="header" class="d-flex align_item_center">
<div onclick="" class="logo_p">
<!-- <img src="/assets/img/file-favicon.png" class="rounded avatar d-block">-->
<img src="/assets/sketch/images/att-globe.svg" class="rounded avatar d-block">
<!--<div class="logo_p">
&lt;!&ndash; <img src="/assets/img/file-favicon.png" class="rounded avatar d-block">&ndash;&gt;
&lt;!&ndash; <img src="/assets/sketch/images/att-globe.svg" class="rounded avatar d-block">&ndash;&gt;
<h4 class="rounded avatar d-block" >Home</h4>
</div>-->
<div class="heading_title_p" style="white-space:nowrap;text-overflow:ellipsis;margin-top: 15px">
<h3 >Home</h3>
</div>
<div class="heading_title_p" style="white-space:nowrap;text-overflow:ellipsis;margin-top: 15px">
<h3 id="dir_id" value=""></h3>
</div>
</header>
</div>
<div class="row content-box rounded" style="margin-top: 10px">

4
src/main/resources/static/signin.html
View File

@@ -3,7 +3,7 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0,maximum-scale=1.0,user-scalable=no">
<link rel="shortcut icon" href="/assets/img/file-favicon.png"/>
<link rel="shortcut icon" href="/assets/sketch/images/favicon.ico"/>
<link rel="stylesheet" href="/assets/css/bootstrap.min.css">
<link rel="stylesheet" href="/assets/css/glyphicons.css">
<script src="/assets/js/popper.min.js"></script>
@@ -37,7 +37,7 @@
<div class="col-sm-6 col-6 col-md-6 text-left">
<button class="btn btn-link text-success" onclick="switchToRegister();">还没有账号？</button>
</div>
<div class="col-sm-6 col-6 col-md-6 text-right">
<div class="col-sm-6 col-6 col-md-6 text-right" hidden>
<button class="btn btn-link text-danger" onclick="switchToReset();">忘记密码？</button>
</div>
</div>

358
src/main/resources/style.scss
View File

@@ -1,358 +0,0 @@
@import "assets/sketch/css/font";
// -------- Variables ----------
$fontLight: 300;
$fontBold: 600;
$desktop: 1024px;
$tablet: 768px;
$mobile: 576px;
$primary-color: #03A2DC;
$secondary-color: #fff6bb;
$light-color: #ffffff;
$dark-color: #555555;
$alt-bg-color: #f9f9f9;
$slider-bg-color: #ccc;
$tableHoverColor: #f9f9f9;
// --------Mixins ------------
@mixin mQ($size) {
@media (max-width: $size) {
@content;
}
}
// -------- Normalize ----------
* {
margin: 0;
padding: 0;
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
ul {
margin: 0;
}
button,
input[type='button'] {
cursor: pointer;
}
button:focus,
input:focus,
textarea:focus {
outline: none;
}
input, textarea {
border: none;
}
button {
border: none;
background: none;
}
img {
max-width: 100%;
height: auto;
}
p {
margin: 0;
}
.align_item_center {
align-items: center;
}
a,
a:hover,
a:active,
a:visited {
text-decoration: none;
}
// ------- Typo ------
body {
font-family: "Poppins", arial, sans-serif;
font-weight: 300;
line-height: 1.625;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
color: $dark-color;
}
h1,
h2,
h3,
h4,
h5,
h6 {
color: $dark-color;
font-weight: 600;
margin: 0;
line-height: 1.2;
}
h1 {
font-size: 36px;
}
h2 {
font-size: 30px;
}
h3 {
font-size: 24px;
}
h4 {
font-size: 18px;
}
h5 {
font-size: 15px;
}
h6 {
font-size: 13px;
}
// ------- Styling ------
.body_bg {
width: 100%;
height: 100%;
position: absolute;
z-index: -1;
background-color: $alt-bg-color;
clip-path: polygon(100% 100%, 0% 100%, 100% 0);
}
.conn {
padding: 20px 0;
}
// ----- header ----
#header {
margin: 0px 0;
margin-left: 20px;
padding-bottom: 20px;
border-bottom: 5px solid $alt-bg-color;
@include mQ($tablet) {
justify-content: center;
}
@include mQ($mobile) {
margin-left: 10px;
flex-direction: column;
}
// -- logo P --
.logo_p {
//background-color: $primary-color;
display: flex;
justify-content: center;
align-items: center;
img{
width: 65px;
}
}
// --- Heading Title --
.heading_title_p {
margin-left: 20px;
@include mQ($mobile) {
margin-left: 0;
margin-top: 20px;
}
h2 {
font-size: 22px;
@include mQ($mobile) {
text-align: center;
}
}
}
}
// --- Upload
.inputUploadP {
margin: 5px 0px 0px 20px;
@include mQ($mobile) {
margin: 20px 10px;
}
.uploadForm {
display: flex;
align-items: center;
.uploadFile {
width: 0.1px;
height: 0.1px;
opacity: 0;
overflow: hidden;
position: absolute;
z-index: -1;
}
svg{
fill: $light-color;
}
.uploadFile + label {
font-size: 1em;
font-weight: 700;
color: $light-color;
background-color: $primary-color;
display: inline-block;
transition: .2s ease;
padding: 8px 20px;
border-radius: 3px;
cursor: pointer;
}
.uploadFile:focus + label,
.uploadFile + label:hover {
background-color: $dark-color;
}
.uploadFile:focus + label {
outline: 1px dotted $dark-color;
outline: -webkit-focus-ring-color auto 5px;
}
.uploadFile + label * {
pointer-events: none;
}
.js .uploadFile {
width: 0.1px;
height: 0.1px;
opacity: 0;
overflow: hidden;
position: absolute;
z-index: -1;
}
.no-js .uploadFile + label {
display: none;
}
.uploadBtn_P {
margin-top: -7px;
margin-left: 10px;
.uploadBtn {
transition: .2s ease;
&:hover {
background-color: $secondary-color;
color: var(--dark-color);
border: 1px solid $dark-color;
}
i {
padding-right: 10px;
}
}
}
}
}
// ---- BackBtn P ---
.backBtn_p {
margin: 10px 0px 0px 20px;
@include mQ($mobile) {
margin: 20px 10px;
}
a {
font-size: 1.3em;
display: flex;
align-items: center;
transition: .2s ease;
width: 150px;
color: $primary-color;
&:hover {
color: $dark-color;
i {
color: $dark-color;
}
}
i {
padding-right: 10px;
font-size: 1.5em;
}
}
}
// -- Table - P
.table_p {
//margin: 20px;
//margin-top: 0;
//width: calc(100% - 40px);
.table-hover tbody tr:hover td, .table-hover tbody tr:hover th {
background-color: $tableHoverColor;
}
// th {
// @include mQ($mobile) {
// font-size: .9em;
// }
// }
td {
font-size: .95em;
vertical-align: middle;
&:nth-child(1) {
text-align: center;
}
}
.file_ic, .folder_ic {
color: $primary-color;
font-size: 1.5em;
transition: .2s ease;
&:hover {
color: #0056b3;
}
}
@include mQ($mobile) {
margin: 20px 10px;
width: calc(100% - 20px);
}
}
#tableData > tbody > tr > td {
padding: 0;
height: 20px;
}
.dataTables_filter input {
border: 1px solid $dark-color;
padding: 3px 20px;
}
// -- Footer --
footer {
text-align: center;
font-size: .8em;
font-weight: bold;
margin: 30px 0;
margin-bottom: 0;
}

27
src/test/java/com/mesasoft/cn/SketchApplicationTest.java
View File

@@ -1,27 +0,0 @@
package com.mesasoft.cn;
import com.zhazhapan.config.JsonParser;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
@RunWith(SpringRunner.class)
@SpringBootTest
public class SketchApplicationTest {
public static void setSettings() {
try {
SketchApplication.settings = new JsonParser(SketchApplicationTest.class.getResource("/config.json"));
} catch (IOException e) {
e.printStackTrace();
}
}
@Test
public void contextLoads() {
}
}

40
src/test/java/com/mesasoft/cn/WebSketchApplicationTest.java Normal file
View File

@@ -0,0 +1,40 @@
package com.mesasoft.cn;
import cn.hutool.core.date.DateUtil;
import com.mesasoft.cn.sketch.config.BrightCloudConfig;
import com.mesasoft.cn.sketch.config.ChinazConfig;
import com.mesasoft.cn.sketch.util.FileUtils;
import com.zhazhapan.config.JsonParser;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@RunWith(SpringRunner.class)
@SpringBootTest
public class WebSketchApplicationTest {
public static void setSettings() {
try {
WebSketchApplication.settings = new JsonParser(WebSketchApplicationTest.class.getResource("/config.json"));
} catch (IOException e) {
e.printStackTrace();
}
}
@Resource
BrightCloudConfig brightCloudConfig;
@Test
public void contextLoads() {
String s = FileUtils.readJsonFile(brightCloudConfig.getCateinfoFilepath());
System.err.println(s);
}
}

45
src/test/java/com/mesasoft/cn/api/ApiServiceTest.java Normal file
View File

@@ -0,0 +1,45 @@
package com.mesasoft.cn.api;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.api.BrightCloud;
import com.mesasoft.cn.sketch.api.ChinaZ;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @author pantao
* @since 2018/2/9
*/
@RunWith(SpringRunner.class)
@SpringBootTest
public class ApiServiceTest {
@Test
public void brightCloudTest() {
JSONObject queryResults = new BrightCloud().getQueryResults(Arrays.asList("baidu.com"));
new BrightCloud().responseSparse(queryResults);
}
public void chinazTest() {
// ChinaZ t = new ChinaZ(null);
//单查询测试
// System.out.println(t.singleRequest("aaa.baidu.com"));
//批量查询测试
// List<String> domainList = new ArrayList<>();
// domainList.add("www.baidu.com");
//// domainList.add("aaa.qq.com");
//// domainList.add("doc.mesalab.com");
//
//// System.out.println(t.batchRequestController(domainList));
// System.out.println(t.getQueryResults(domainList));
}
}

8
src/test/java/com/mesasoft/cn/common/CommonTest.java
View File

@@ -1,7 +1,7 @@
package com.mesasoft.cn.common;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.FileExecutor;
import com.zhazhapan.util.Formatter;
@@ -19,8 +19,8 @@ public class CommonTest {
@Test
public void testSendEmail() throws Exception {
SketchApplicationTest.setSettings();
MailSender.config(SketchApplication.settings.getObjectUseEval(ConfigConsts.EMAIL_CONFIG_OF_SETTINGS));
WebSketchApplicationTest.setSettings();
MailSender.config(WebSketchApplication.settings.getObjectUseEval(ConfigConsts.EMAIL_CONFIG_OF_SETTINGS));
MailSender.sendMail("tao@zhazhapan.com", "test", "test");
}

53
src/test/java/com/mesasoft/cn/common/SketchTest.java
View File

File diff suppressed because one or more lines are too long

10
src/test/java/com/mesasoft/cn/config/SettingConfigTest.java
View File

@@ -1,7 +1,7 @@
package com.mesasoft.cn.config;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import org.junit.Test;
@@ -15,13 +15,13 @@ public class SettingConfigTest {
@Test
public void testFileSuffixPattern() {
SketchApplicationTest.setSettings();
assert Pattern.compile(SketchApplication.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher("jpg").matches();
WebSketchApplicationTest.setSettings();
assert Pattern.compile(WebSketchApplication.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher("jpg").matches();
}
@Test
public void testGetStoragePath() {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
System.out.println(SettingConfig.getStoragePath(ConfigConsts.TOKEN_OF_SETTINGS));
}
}

4
src/test/java/com/mesasoft/cn/dao/AuthDAOTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.zhazhapan.util.Formatter;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -17,7 +17,7 @@ import org.springframework.test.context.junit4.SpringRunner;
public class AuthDAOTest {
static {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
}
@Autowired

4
src/test/java/com/mesasoft/cn/dao/DownloadedDAOTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.zhazhapan.modules.constant.ValueConsts;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -17,7 +17,7 @@ import org.springframework.test.context.junit4.SpringRunner;
public class DownloadedDAOTest {
static {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
}
@Autowired

4
src/test/java/com/mesasoft/cn/dao/FileDAOTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.zhazhapan.util.Formatter;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -26,7 +26,7 @@ public class FileDAOTest {
@Test
public void testGetUserDownloaded() {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
System.out.println(Formatter.listToJson(fileDAO.listUserDownloaded(2, 0, "")));
}
}

4
src/test/java/com/mesasoft/cn/dao/UserDAOTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.mesasoft.cn.entity.User;
import com.zhazhapan.util.Checker;
import com.zhazhapan.util.Formatter;
@@ -20,7 +20,7 @@ import org.springframework.test.context.junit4.SpringRunner;
public class UserDAOTest {
static {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
}
@Autowired

4
src/test/java/com/mesasoft/cn/dao/sqlprovider/FileSqlProviderTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@@ -25,7 +25,7 @@ public class FileSqlProviderTest {
@Test
public void getAll() {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
System.out.println(fileSqlProvider.getAll(0, 0, "", ""));
}

6
src/test/java/com/mesasoft/cn/service/CategoryServiceTest.java
View File

@@ -1,11 +1,14 @@
package com.mesasoft.cn.service;
import cn.hutool.core.io.resource.ResourceUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.net.URL;
/**
* @author pantao
* @since 2018/2/9
@@ -20,5 +23,8 @@ public class CategoryServiceTest {
@Test
public void testGetIdByName() {
System.out.println(categoryService.getIdByName("fff"));
URL resource = ResourceUtil.getResource("config.json");
String path = resource.getPath();
System.err.println(path);
}
}

178
webskt-query-agent.iml Normal file
View File

@@ -0,0 +1,178 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="FacetManager">
<facet type="Spring" name="Spring">
<configuration />
</facet>
<facet type="web" name="Web">
<configuration>
<webroots />
<sourceRoots>
<root url="file://$MODULE_DIR$/src/main/java" />
<root url="file://$MODULE_DIR$/src/main/resources" />
</sourceRoots>
</configuration>
</facet>
</component>
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-all:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-lang:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-config-core:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-config-ogdl:1.4.0" level="project" />
<orderEntry type="library" name="Maven: commons-beanutils:commons-beanutils:1.9.3" level="project" />
<orderEntry type="library" name="Maven: commons-collections:commons-collections:3.2.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-crypto-core:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-crypto-hash:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-crypto-cipher:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-event:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-core:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-cache:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-web:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-aspectj:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.aspectj:aspectjrt:1.8.13" level="project" />
<orderEntry type="library" name="Maven: org.aspectj:aspectjweaver:1.8.13" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-ehcache:1.4.0" level="project" />
<orderEntry type="library" name="Maven: net.sf.ehcache:ehcache-core:2.6.11" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-guice:1.4.0" level="project" />
<orderEntry type="library" name="Maven: com.google.inject:guice:3.0" level="project" />
<orderEntry type="library" name="Maven: javax.inject:javax.inject:1" level="project" />
<orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />
<orderEntry type="library" name="Maven: com.google.inject.extensions:guice-multibindings:3.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-hazelcast:1.4.0" level="project" />
<orderEntry type="library" name="Maven: com.hazelcast:hazelcast:3.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-quartz:1.4.0" level="project" />
<orderEntry type="library" name="Maven: org.opensymphony.quartz:quartz:1.6.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.shiro:shiro-spring:1.4.0" level="project" />
<orderEntry type="library" name="Maven: commons-fileupload:commons-fileupload:1.3.3" level="project" />
<orderEntry type="library" name="Maven: com.zhazhapan:util:1.1.0" level="project" />
<orderEntry type="library" name="Maven: junit:junit:4.12" level="project" />
<orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.8.2" level="project" />
<orderEntry type="library" name="Maven: com.alibaba:fastjson:1.2.47" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.7" level="project" />
<orderEntry type="library" name="Maven: javax.mail:mail:1.5.0-b01" level="project" />
<orderEntry type="library" name="Maven: javax.activation:activation:1.1" level="project" />
<orderEntry type="library" name="Maven: commons-io:commons-io:2.6" level="project" />
<orderEntry type="library" name="Maven: cglib:cglib:3.2.6" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm:6.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.ant:ant:1.9.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.ant:ant-launcher:1.9.6" level="project" />
<orderEntry type="library" name="Maven: javax.servlet:javax.servlet-api:3.1.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:3.17" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi:3.17" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-collections4:4.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-schemas:3.17" level="project" />
<orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:2.6.0" level="project" />
<orderEntry type="library" name="Maven: stax:stax-api:1.0.1" level="project" />
<orderEntry type="library" name="Maven: com.github.virtuald:curvesapi:1.04" level="project" />
<orderEntry type="library" name="Maven: org.jsoup:jsoup:1.11.2" level="project" />
<orderEntry type="library" name="Maven: javax.xml:jaxp-api:1.4.2" level="project" />
<orderEntry type="library" name="Maven: net.sourceforge.htmlcleaner:htmlcleaner:2.21" level="project" />
<orderEntry type="library" name="Maven: org.jdom:jdom2:2.0.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-jexl3:3.1" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.2" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:25.1-jre" level="project" />
<orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:3.0.2" level="project" />
<orderEntry type="library" name="Maven: org.checkerframework:checker-qual:2.0.0" level="project" />
<orderEntry type="library" name="Maven: com.google.errorprone:error_prone_annotations:2.1.3" level="project" />
<orderEntry type="library" name="Maven: com.google.j2objc:j2objc-annotations:1.1" level="project" />
<orderEntry type="library" name="Maven: org.codehaus.mojo:animal-sniffer-annotations:1.14" level="project" />
<orderEntry type="library" name="Maven: org.mybatis.spring.boot:mybatis-spring-boot-starter:1.3.1" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter:2.0.0.M7" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-logging:2.0.0.M7" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:1.2.3" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.2.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-to-slf4j:2.10.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.10.0" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:jul-to-slf4j:1.7.25" level="project" />
<orderEntry type="library" name="Maven: javax.annotation:javax.annotation-api:1.3.1" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.yaml:snakeyaml:1.19" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jdbc:2.0.0.M7" level="project" />
<orderEntry type="library" name="Maven: com.zaxxer:HikariCP:2.7.4" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-jdbc:5.0.2.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-tx:5.0.2.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.mybatis.spring.boot:mybatis-spring-boot-autoconfigure:1.3.1" level="project" />
<orderEntry type="library" name="Maven: org.mybatis:mybatis:3.4.5" level="project" />
<orderEntry type="library" name="Maven: org.mybatis:mybatis-spring:1.3.1" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-web:2.0.0.M7" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-json:2.0.0.M7" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.9.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.9.0" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.9.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.9.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.9.2" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.module:jackson-module-parameter-names:2.9.2" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-tomcat:2.0.0.M7" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-core:8.5.23" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat:tomcat-annotations-api:8.5.23" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-el:8.5.23" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-websocket:8.5.23" level="project" />
<orderEntry type="library" name="Maven: org.hibernate.validator:hibernate-validator:6.0.5.Final" level="project" />
<orderEntry type="library" name="Maven: javax.validation:validation-api:2.0.0.Final" level="project" />
<orderEntry type="library" name="Maven: org.jboss.logging:jboss-logging:3.3.1.Final" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml:classmate:1.3.4" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-web:5.0.2.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-beans:5.0.2.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-webmvc:5.0.2.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-aop:5.0.2.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-context:5.0.2.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-expression:5.0.2.RELEASE" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: org.springframework.boot:spring-boot-devtools:2.0.0.M7" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot:2.0.0.M7" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-autoconfigure:2.0.0.M7" level="project" />
<orderEntry type="library" scope="RUNTIME" name="Maven: mysql:mysql-connector-java:5.1.44" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-starter-test:2.0.0.M7" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-test:2.0.0.M7" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.springframework.boot:spring-boot-test-autoconfigure:2.0.0.M7" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: com.jayway.jsonpath:json-path:2.4.0" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: net.minidev:json-smart:2.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: net.minidev:accessors-smart:1.2" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.assertj:assertj-core:3.8.0" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.mockito:mockito-core:2.12.0" level="project" />
<orderEntry type="library" name="Maven: net.bytebuddy:byte-buddy:1.7.9" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: net.bytebuddy:byte-buddy-agent:1.7.9" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.objenesis:objenesis:2.6" level="project" />
<orderEntry type="library" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-library:1.3" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.skyscreamer:jsonassert:1.5.0" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-core:5.0.2.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-jcl:5.0.2.RELEASE" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.springframework:spring-test:5.0.2.RELEASE" level="project" />
<orderEntry type="library" scope="TEST" name="Maven: org.xmlunit:xmlunit-core:2.5.1" level="project" />
<orderEntry type="library" name="Maven: com.spring4all:swagger-spring-boot-starter:1.7.1.RELEASE" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-swagger-ui:2.9.0" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-spring-web:2.9.0" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-swagger2:2.9.0" level="project" />
<orderEntry type="library" name="Maven: io.swagger:swagger-annotations:1.5.18" level="project" />
<orderEntry type="library" name="Maven: io.swagger:swagger-models:1.5.18" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-spi:2.9.0" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-core:2.9.0" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-schema:2.9.0" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-swagger-common:2.9.0" level="project" />
<orderEntry type="library" name="Maven: org.springframework.plugin:spring-plugin-core:1.2.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.plugin:spring-plugin-metadata:1.2.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.mapstruct:mapstruct:1.2.0.Final" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-bean-validators:2.9.0" level="project" />
<orderEntry type="library" name="Maven: org.projectlombok:lombok:1.18.4" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.6" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.8" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.11" level="project" />
<orderEntry type="library" name="Maven: com.alibaba:druid-spring-boot-starter:1.2.6" level="project" />
<orderEntry type="library" name="Maven: com.alibaba:druid:1.2.6" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.25" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-configuration-processor:2.0.0.M7" level="project" />
<orderEntry type="library" name="Maven: com.vaadin.external.google:android-json:0.0.20131108.vaadin1" level="project" />
<orderEntry type="library" name="Maven: me.geso:jdbcutils:1.0.0" level="project" />
<orderEntry type="library" name="Maven: cn.hutool:hutool-all:5.7.17" level="project" />
</component>
</module>