gfwleak/web-sketch-webskt-query-agent
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

web sketch-查询代理功能优化 CN-663

Browse Source
This commit is contained in:
zhanghongqing
2023-01-10 15:51:02 +08:00
parent b3fa11d4b1
commit c125bb89cf
54 changed files with 515 additions and 2557 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
README.md
View File

@@ -1,9 +1,5 @@
### 关于项目
- 项目名称 `efo` 是 `Easy File Online` 的缩写，字面意思就是让您轻松实现线上文件管理
- 本系统具有文件分享的功能，权限控制和自定义配置都很强大（可能还不完善）
- 系统后端框架有Spring Boot，Spring， SpringMVC，MyBatis; 前段框架有Bootstrap，Jquery, Layer, Vue。项目完全纯注解，零XML配置。
> 第一次运行系统，请先运行 [SQL代码](/mysql/efo.sql) , 并登陆系统修改用户 `system` （默认密码 `123456`）的密码
@@ -16,26 +12,24 @@
### 系统部分截图（背景图片可通过配置设置）
- 登录页面（包含登录、注册、密码重置），路径 `/signin`
- 登录页面（包含登录、注册、密码重置），路径 `/signin`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/signin.png)
- 资源首页，路径 `/index`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/index.png)
- 上传页面，路径 `/upload`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/upload.png)
- 管理员管理页面，路径 `/admin`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/admin.png)
- 远程文件管理（管理服务器端所有文件，只有系统用户才能进入此页面），路径 `/filemanager`
![登录页面](http://towerpan.qiniu.segocat.com/git/efo/filemanager.png)
> 此功能基于 [angular-filamanager](https://github.com/joni2back/angular-filemanager) 实现
**项目有不足的地方欢迎提出来哦，大家一起交流学习，觉得不错的话，Star来一个呗**

28
pom.xml
View File

@@ -4,11 +4,11 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.mesasoft.cn</groupId>
<artifactId>webskt-query-agent</artifactId>
<version>1.0</version>
<artifactId>webskt</artifactId>
<version>1.2</version>
<packaging>jar</packaging>
<name>webskt-query-agent</name>
<name>webskt</name>
<description>简单的线上文件管理系统</description>
<parent>
@@ -76,6 +76,7 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<scope>runtime</scope>
<optional>true</optional>
</dependency>
<dependency>
<groupId>mysql</groupId>
@@ -117,14 +118,27 @@
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.0.9</version>
<artifactId>druid-spring-boot-starter</artifactId>
<version>1.2.6</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<!-- websketch-->
<dependency>
<groupId>me.geso</groupId>
<artifactId>jdbcutils</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.7.17</version>
</dependency>
</dependencies>
<build>
@@ -134,7 +148,7 @@
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<fork>true</fork>
<mainClass>com.mesasoft.cn.SketchApplication</mainClass>
<mainClass>com.mesasoft.cn.WebSketchApplication</mainClass>
</configuration>
<executions>
<execution>
@@ -151,7 +165,7 @@
<archive>
<manifest>
<addClasspath>true</addClasspath>
<mainClass>com.mesasoft.cn.SketchApplication</mainClass>
<mainClass>com.mesasoft.cn.WebSketchApplication</mainClass>
</manifest>
</archive>
<descriptorRefs>

42
src/main/java/com/mesasoft/cn/SketchApplication.java
View File

@@ -1,42 +0,0 @@
package com.mesasoft.cn;
import com.spring4all.swagger.EnableSwagger2Doc;
import com.zhazhapan.config.JsonParser;
import com.mesasoft.cn.config.TokenConfig;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.zhazhapan.util.FileExecutor;
import com.zhazhapan.util.MailSender;
import com.zhazhapan.util.ReflectUtils;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import java.io.IOException;
import java.util.Hashtable;
import java.util.List;
/**
* @author pantao
*/
@SpringBootApplication
@EnableSwagger2Doc
@MapperScan("com.mesasoft.cn.dao")
@EnableTransactionManagement
public class SketchApplication {
public static JsonParser settings = new JsonParser();
public static List<Class<?>> controllers;
public static Hashtable<String, Integer> tokens;
public static void main(String[] args) throws IOException, ClassNotFoundException {
settings.setJsonObject(FileExecutor.read(SketchApplication.class.getResourceAsStream(DefaultValues.SETTING_PATH)));
MailSender.config(settings.getObjectUseEval(ConfigConsts.EMAIL_CONFIG_OF_SETTINGS));
controllers = ReflectUtils.getClasses(DefaultValues.CONTROLLER_PACKAGE);
tokens = TokenConfig.loadToken();
SpringApplication.run(SketchApplication.class, args);
}
}

10
src/main/java/com/mesasoft/cn/config/SettingConfig.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.config;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.util.CommonUtils;
import com.zhazhapan.modules.constant.ValueConsts;
@@ -12,6 +12,8 @@ import org.slf4j.LoggerFactory;
import java.util.Date;
import static com.mesasoft.cn.WebSketchApplication.settings;
/**
* @author pantao
* @since 2018/1/26
@@ -42,14 +44,14 @@ public class SettingConfig {
int[] auth = new int[5];
for (int i = 0; i < ConfigConsts.AUTH_OF_SETTINGS.length; i++) {
String key = jsonPath + ValueConsts.DOT_SIGN + ConfigConsts.AUTH_OF_SETTINGS[i];
auth[i] = SketchApplication.settings.getBooleanUseEval(key) ? 1 : 0;
auth[i] = settings.getBooleanUseEval(key) ? 1 : 0;
}
return auth;
}
public static String getUploadStoragePath() {
String parent = getStoragePath(ConfigConsts.UPLOAD_PATH_OF_SETTING);
String formatWay = SketchApplication.settings.getStringUseEval(ConfigConsts.UPLOAD_FORM_OF_SETTING);
String formatWay = WebSketchApplication.settings.getStringUseEval(ConfigConsts.UPLOAD_FORM_OF_SETTING);
String childPath = ValueConsts.SEPARATOR + Formatter.datetimeToCustomString(new Date(), formatWay);
String path = parent + childPath;
if (!FileExecutor.createFolder(path)) {
@@ -75,7 +77,7 @@ public class SettingConfig {
} else {
path += LINUX;
}
return CommonUtils.checkPath(SketchApplication.settings.getStringUseEval(path));
return CommonUtils.checkPath(WebSketchApplication.settings.getStringUseEval(path));
}
/**

14
src/main/java/com/mesasoft/cn/config/TokenConfig.java
View File

@@ -3,7 +3,7 @@ package com.mesasoft.cn.config;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
@@ -25,20 +25,20 @@ public class TokenConfig {
public static String generateToken(String token, int userId) {
if (Checker.isNotEmpty(token)) {
SketchApplication.tokens.remove(token);
WebSketchApplication.tokens.remove(token);
}
return generateToken(userId);
}
public static String generateToken(int userId) {
String token = RandomUtils.getRandomStringOnlyLetter(ValueConsts.THIRTY_TWO_INT);
SketchApplication.tokens.put(token, userId);
WebSketchApplication.tokens.put(token, userId);
saveToken();
return token;
}
public static void saveToken() {
String tokens = Formatter.mapToJson(SketchApplication.tokens);
String tokens = Formatter.mapToJson(WebSketchApplication.tokens);
try {
FileExecutor.saveFile(SettingConfig.getStoragePath(ConfigConsts.TOKEN_OF_SETTINGS), tokens);
} catch (Exception e) {
@@ -65,14 +65,14 @@ public class TokenConfig {
public static void removeTokenByValue(int userId) {
if (userId > 0) {
String removeKey = "";
for (String key : SketchApplication.tokens.keySet()) {
if (SketchApplication.tokens.get(key) == userId) {
for (String key : WebSketchApplication.tokens.keySet()) {
if (WebSketchApplication.tokens.get(key) == userId) {
removeKey = key;
break;
}
}
if (Checker.isNotEmpty(removeKey)) {
SketchApplication.tokens.remove(removeKey);
WebSketchApplication.tokens.remove(removeKey);
TokenConfig.saveToken();
}
}

6
src/main/java/com/mesasoft/cn/dao/sqlprovider/AuthSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.Checker;
import org.apache.ibatis.annotations.Param;
@@ -40,9 +40,9 @@ public class AuthSqlProvider {
} else if (Checker.isNotEmpty(fileName)) {
WHERE("f.local_url like '%" + fileName + "%'");
}
ORDER_BY("a." + SketchApplication.settings.getStringUseEval(ConfigConsts.AUTH_ORDER_BY_OF_SETTINGS));
ORDER_BY("a." + WebSketchApplication.settings.getStringUseEval(ConfigConsts.AUTH_ORDER_BY_OF_SETTINGS));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.AUTH_PAGE_SIZE_OF_SETTINGS);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.AUTH_PAGE_SIZE_OF_SETTINGS);
return sql + " limit " + (offset * size) + "," + size;
}
}

6
src/main/java/com/mesasoft/cn/dao/sqlprovider/DownloadedSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.Checker;
import org.apache.ibatis.annotations.Param;
@@ -40,9 +40,9 @@ public class DownloadedSqlProvider {
if (categoryId > 0) {
WHERE("c.id=#{categoryId}");
}
ORDER_BY("d." + SketchApplication.settings.getStringUseEval(ConfigConsts.DOWNLOAD_ORDER_BY_OF_SETTINGS));
ORDER_BY("d." + WebSketchApplication.settings.getStringUseEval(ConfigConsts.DOWNLOAD_ORDER_BY_OF_SETTINGS));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.DOWNLOAD_PAGE_SIZE_OF_SETTINGS);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.DOWNLOAD_PAGE_SIZE_OF_SETTINGS);
return sql + " limit " + (offset * size) + "," + size;
}
}

10
src/main/java/com/mesasoft/cn/dao/sqlprovider/FileSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
@@ -46,15 +46,15 @@ public class FileSqlProvider {
if (categoryId > 0) {
WHERE("c.id=#{categoryId}");
}
ORDER_BY("f." + SketchApplication.settings.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING));
ORDER_BY("f." + WebSketchApplication.settings.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
return sql + " limit " + (offset * size) + "," + size;
}
private String getSqlEnds(int offset, String orderBy, String search) {
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
return getSearch(search) + " order by " + (Checker.isEmpty(orderBy) ? SketchApplication.settings
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
return getSearch(search) + " order by " + (Checker.isEmpty(orderBy) ? WebSketchApplication.settings
.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING) : orderBy) + " limit " + offset * size
+ "," + size;
}

6
src/main/java/com/mesasoft/cn/dao/sqlprovider/UploadedSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.Checker;
import org.apache.ibatis.annotations.Param;
@@ -39,9 +39,9 @@ public class UploadedSqlProvider {
if (categoryId > 0) {
WHERE("c.id=#{categoryId}");
}
ORDER_BY("f." + SketchApplication.settings.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING));
ORDER_BY("f." + WebSketchApplication.settings.getStringUseEval(ConfigConsts.FILE_ORDER_BY_OF_SETTING));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.FILE_PAGE_SIZE_OF_SETTING);
return sql + " limit " + (offset * size) + "," + size;
}
}

6
src/main/java/com/mesasoft/cn/dao/sqlprovider/UserSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.zhazhapan.util.Checker;
@@ -33,9 +33,9 @@ public class UserSqlProvider {
WHERE("username like '%" + condition + "%' or email like '%" + condition + "%' or real_name like '" +
condition + "'");
}
ORDER_BY(SketchApplication.settings.getStringUseEval(ConfigConsts.USER_ORDER_BY_OF_SETTINGS));
ORDER_BY(WebSketchApplication.settings.getStringUseEval(ConfigConsts.USER_ORDER_BY_OF_SETTINGS));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.USER_PAGE_SIZE_OF_SETTINGS);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.USER_PAGE_SIZE_OF_SETTINGS);
return sql + " limit " + (offset * size) + "," + size;
}
}

5
src/main/java/com/mesasoft/cn/exception/GlobalExceptionHandler.java
View File

@@ -1,7 +1,6 @@
package com.mesasoft.cn.exception;
import com.alibaba.fastjson.support.spring.FastJsonJsonView;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
import org.springframework.http.HttpStatus;
import org.springframework.web.servlet.HandlerExceptionResolver;
@@ -23,9 +22,9 @@ public class GlobalExceptionHandler implements HandlerExceptionResolver {
Exception ex) {
ModelAndView mv = new ModelAndView();
FastJsonJsonView view = new FastJsonJsonView();
Map<String, Object> attributes = new HashMap<>(ValueConsts.TWO_INT);
Map<String, Object> attributes = new HashMap<>();
attributes.put("code", "502");
attributes.put("message", ex.getMessage());
attributes.put("message", "request error "+ (Checker.isEmpty(ex.getMessage())||ex.getMessage()=="null"?ex.getCause():ex.getMessage()));
String queryString = request.getQueryString();
attributes.put("url", request.getRequestURI() + (Checker.isEmpty(queryString) ? "" : "?" + queryString));
view.setAttributesMap(attributes);

8
src/main/java/com/mesasoft/cn/interceptor/WebInterceptor.java
View File

@@ -1,11 +1,11 @@
package com.mesasoft.cn.interceptor;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.impl.UserServiceImpl;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.impl.UserServiceImpl;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
import com.zhazhapan.util.HttpUtils;
@@ -36,7 +36,7 @@ public class WebInterceptor implements HandlerInterceptor {
AuthInterceptor interceptor = ((HandlerMethod) handler).getMethodAnnotation(AuthInterceptor.class);
//注解到类上面的注解，无法直接获取，只能通过扫描
if (Checker.isNull(interceptor)) {
for (Class<?> type : SketchApplication.controllers) {
for (Class<?> type : WebSketchApplication.controllers) {
RequestMapping mapping = type.getAnnotation(RequestMapping.class);
if (Checker.isNotNull(mapping)) {
for (String path : mapping.value()) {

3
src/main/java/com/mesasoft/cn/modules/constant/DefaultValues.java
View File

@@ -1,5 +1,6 @@
package com.mesasoft.cn.modules.constant;
import com.zhazhapan.modules.constant.ValueConsts;
/**
@@ -21,7 +22,7 @@ public class DefaultValues {
/**
* Controller包路径
*/
public static final String CONTROLLER_PACKAGE = "com.zhazhapan.efo.web.controller";
public static final String CONTROLLER_PACKAGE = "com.mesasoft.cn.web.controller";
/**
* 配置文件路径

4
src/main/java/com/mesasoft/cn/service/impl/AuthServiceImpl.java
View File

@@ -1,12 +1,12 @@
package com.mesasoft.cn.service.impl;
import com.mesasoft.cn.dao.AuthDAO;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.dao.AuthDAO;
import com.mesasoft.cn.entity.Auth;
import com.mesasoft.cn.model.AuthRecord;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.service.IAuthService;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.util.ServiceUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;

2
src/main/java/com/mesasoft/cn/service/impl/CommonServiceImpl.java
View File

@@ -1,7 +1,7 @@
package com.mesasoft.cn.service.impl;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.ICommonService;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;

6
src/main/java/com/mesasoft/cn/service/impl/ConfigServiceImpl.java
View File

@@ -1,7 +1,7 @@
package com.mesasoft.cn.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.service.IConfigService;
import org.springframework.stereotype.Service;
@@ -15,7 +15,7 @@ public class ConfigServiceImpl implements IConfigService {
@Override
public String getGlobalConfig() {
JSONObject jsonObject = (JSONObject) SketchApplication.settings.getObjectUseEval(ConfigConsts
JSONObject jsonObject = (JSONObject) WebSketchApplication.settings.getObjectUseEval(ConfigConsts
.GLOBAL_OF_SETTINGS).clone();
jsonObject.remove(ConfigConsts.UPLOAD_PATH_OF_GLOBAL);
jsonObject.remove(ConfigConsts.TOKEN_PATH_OF_GLOBAL);
@@ -25,7 +25,7 @@ public class ConfigServiceImpl implements IConfigService {
@Override
public String getUserConfig() {
JSONObject jsonObject = (JSONObject) SketchApplication.settings.getObjectUseEval(ConfigConsts.USER_OF_SETTINGS)
JSONObject jsonObject = (JSONObject) WebSketchApplication.settings.getObjectUseEval(ConfigConsts.USER_OF_SETTINGS)
.clone();
jsonObject.remove(ConfigConsts.EMAIL_CONFIG_OF_USER);
return jsonObject.toString();

24
src/main/java/com/mesasoft/cn/service/impl/FileServiceImpl.java
View File

@@ -1,10 +1,9 @@
package com.mesasoft.cn.service.impl;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.dao.DownloadedDAO;
import com.mesasoft.cn.dao.FileDAO;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.entity.Category;
import com.mesasoft.cn.entity.File;
import com.mesasoft.cn.entity.User;
@@ -17,6 +16,7 @@ import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.IAuthService;
import com.mesasoft.cn.service.ICategoryService;
import com.mesasoft.cn.service.IFileService;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.util.ServiceUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.*;
@@ -134,7 +134,7 @@ public class FileServiceImpl implements IFileService {
AuthRecord authRecord = authService.getByFileIdAndUserId(id, user.getId());
String suffix = FileExecutor.getFileSuffix(name);
boolean canUpdate = (Checker.isNull(authRecord) ? user.getIsUpdatable() == 1 :
authRecord.getIsUpdatable() == 1) && Checker.isNotEmpty(name) && Pattern.compile(SketchApplication.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher(suffix).matches();
authRecord.getIsUpdatable() == 1) && Checker.isNotEmpty(name) && Pattern.compile(WebSketchApplication.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher(suffix).matches();
if (canUpdate) {
String localUrl = file.getLocalUrl();
java.io.File newFile = new java.io.File(localUrl);
@@ -208,7 +208,7 @@ public class FileServiceImpl implements IFileService {
@Override
public String getResource(String visitUrl, HttpServletRequest request) {
logger.info("visit url: " + visitUrl);
boolean downloadable = SketchApplication.settings.getBooleanUseEval(ConfigConsts
boolean downloadable = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts
.ANONYMOUS_DOWNLOADABLE_OF_SETTING);
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
File file = fileDAO.getFileByVisitUrl(visitUrl);
@@ -249,15 +249,15 @@ public class FileServiceImpl implements IFileService {
String suffix = FileExecutor.getFileSuffix(name);
String localUrl = SettingConfig.getUploadStoragePath() + ValueConsts.SEPARATOR + name;
Category category = categoryService.getById(categoryId);
long maxSize = Formatter.sizeToLong(SketchApplication.settings.getStringUseEval(ConfigConsts
long maxSize = Formatter.sizeToLong(WebSketchApplication.settings.getStringUseEval(ConfigConsts
.FILE_MAX_SIZE_OF_SETTING));
long size = multipartFile.getSize();
boolean fileExists = localUrlExists(localUrl);
//检测标签是否合法
if (SketchApplication.settings.getBooleanUseEval(ConfigConsts.TAG_REQUIRE_OF_SETTING)) {
if (WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.TAG_REQUIRE_OF_SETTING)) {
String[] tags = Checker.checkNull(tag).split(ValueConsts.SPACE);
if (tags.length <= SketchApplication.settings.getIntegerUseEval(ConfigConsts.TAG_SIZE_OF_SETTING)) {
int maxLength = SketchApplication.settings.getIntegerUseEval(ConfigConsts.TAG_LENGTH_OF_SETTING);
if (tags.length <= WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.TAG_SIZE_OF_SETTING)) {
int maxLength = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.TAG_LENGTH_OF_SETTING);
for (String t : tags) {
if (t.length() > maxLength) {
return false;
@@ -268,15 +268,15 @@ public class FileServiceImpl implements IFileService {
}
}
//是否可以上传
boolean canUpload = !multipartFile.isEmpty() && size <= maxSize && Pattern.compile(SketchApplication
boolean canUpload = !multipartFile.isEmpty() && size <= maxSize && Pattern.compile(WebSketchApplication
.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher(suffix).matches()
&& (Checker.isNotExists(localUrl) || !fileExists || SketchApplication.settings.getBooleanUseEval
&& (Checker.isNotExists(localUrl) || !fileExists || WebSketchApplication.settings.getBooleanUseEval
(ConfigConsts.FILE_COVER_OF_SETTING));
logger.info("is empty [" + multipartFile.isEmpty() + "], file size [" + size + "], max file size [" +
maxSize + "]");
if (canUpload) {
String visitUrl = getRegularVisitUrl(Checker.isNotEmpty(prefix) && user.getPermission() > 1 ? prefix
: SketchApplication.settings.getStringUseEval(ConfigConsts.CUSTOM_LINK_RULE_OF_SETTING), user,
: WebSketchApplication.settings.getStringUseEval(ConfigConsts.CUSTOM_LINK_RULE_OF_SETTING), user,
name, suffix, category);
if (fileExists) {
removeByLocalUrl(localUrl);

20
src/main/java/com/mesasoft/cn/service/impl/UserServiceImpl.java
View File

@@ -1,13 +1,13 @@
package com.mesasoft.cn.service.impl;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.dao.UserDAO;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.config.TokenConfig;
import com.mesasoft.cn.dao.UserDAO;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.service.IUserService;
import com.mesasoft.cn.util.BeanUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
import com.zhazhapan.util.DateUtils;
@@ -67,11 +67,11 @@ public class UserServiceImpl implements IUserService {
@Override
public User login(String loginName, String password, String token, HttpServletResponse response) {
boolean allowLogin = SketchApplication.settings.getBooleanUseEval(ConfigConsts.ALLOW_LOGIN_OF_SETTINGS);
boolean allowLogin = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.ALLOW_LOGIN_OF_SETTINGS);
User user = null;
if (allowLogin) {
if (Checker.isNotEmpty(token) && SketchApplication.tokens.containsKey(token)) {
user = userDAO.getUserById(SketchApplication.tokens.get(token));
if (Checker.isNotEmpty(token) && WebSketchApplication.tokens.containsKey(token)) {
user = userDAO.getUserById(WebSketchApplication.tokens.get(token));
if (Checker.isNotNull(response)) {
Cookie cookie = new Cookie(ValueConsts.TOKEN_STRING, TokenConfig.generateToken(token, user.getId
()));
@@ -92,9 +92,9 @@ public class UserServiceImpl implements IUserService {
@Override
public boolean register(String username, String email, String password) {
boolean allowRegister = SketchApplication.settings.getBooleanUseEval(ConfigConsts.ALLOW_REGISTER_OF_SETTINGS);
boolean allowRegister = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.ALLOW_REGISTER_OF_SETTINGS);
if (allowRegister) {
boolean isValid = Checker.isEmail(email) && checkPassword(password) && Pattern.compile(SketchApplication.settings
boolean isValid = Checker.isEmail(email) && checkPassword(password) && Pattern.compile(WebSketchApplication.settings
.getStringUseEval(ConfigConsts.USERNAME_PATTERN_OF_SETTINGS)).matcher(username).matches();
if (isValid) {
User user = new User(username, ValueConsts.EMPTY_STRING, email, password);
@@ -113,8 +113,8 @@ public class UserServiceImpl implements IUserService {
@Override
public boolean checkPassword(String password) {
int min = SketchApplication.settings.getIntegerUseEval(ConfigConsts.PASSWORD_MIN_LENGTH_OF_SETTINGS);
int max = SketchApplication.settings.getIntegerUseEval(ConfigConsts.PASSWORD_MAX_LENGTH_OF_SETTINGS);
int min = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.PASSWORD_MIN_LENGTH_OF_SETTINGS);
int max = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.PASSWORD_MAX_LENGTH_OF_SETTINGS);
return Checker.isLimited(password, min, max);
}

108
src/main/java/com/mesasoft/cn/sketch/api/BrightCloud.java
View File

@@ -1,13 +1,14 @@
package com.mesasoft.cn.sketch.api;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import com.mesasoft.cn.sketch.entity.DomainCategory;
import com.mesasoft.cn.util.FileUtils;
import com.mesasoft.cn.util.ValidationUtils;
import org.apache.log4j.Logger;
import com.mesasoft.cn.sketch.config.BrightCloudConfig;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import com.mesasoft.cn.sketch.util.FileUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.*;
import java.net.HttpURLConnection;
@@ -22,43 +23,44 @@ import java.util.*;
*/
public class BrightCloud {
private static final Logger LOG = Logger.getLogger(BrightCloud.class);
private static final Log logger = Log.get();
private final Integer maxObjectNum = ApplicationConfig.API_BC_MAXIMUM_QUERYNUM;
private final HashMap<Integer, List<String>> catId2Info = new HashMap<>();
private HttpURLConnection con;
public List<DomainCategory> getQueryFiles(List<String> domains){
@Autowired
private BrightCloudConfig bcConfig;
public List<DomainCategory> getBrightCloudDomains(List<String> domains) {
JSONObject queryResults = getQueryResults(domains);
return responseSparse(queryResults);
}
// 获取json格式查询结果
public JSONObject getQueryResults(List<String> domains) {
if (domains.size()> ApplicationConfig.API_BC_MAXIMUM_QUERYNUM){
LOG.warn("Too many domains in a http post request, the number of fqdn/url should be no more than "
+ ApplicationConfig.API_BC_MAXIMUM_QUERYNUM + "!");
if (domains.size() > bcConfig.getMaximumQueryNum()) {
logger.warn("Too many domains in a http post request, the number of fqdn/url should be no more than "
+ bcConfig.getMaximumQueryNum() + "!");
}
JSONObject jsonRes = new JSONObject();
try {
URL url = new URL(ApplicationConfig.API_BC_URL);
URL url = new URL(bcConfig.getUrl());
con = (HttpURLConnection) url.openConnection();
con.setRequestMethod(ApplicationConfig.API_BC_METHOD);
con.setRequestMethod(bcConfig.getMethod());
con.setDoOutput(true);
con.setDoInput(true);
con.setRequestProperty("Content-Type", "application/json");
JSONObject param = new JSONObject();
param.put("oemid", ApplicationConfig.API_BC_OEMID);
param.put("deviceid", ApplicationConfig.API_BC_DEVICEID);
param.put("uid", ApplicationConfig.API_BC_UID);
param.put("queries", Collections.singletonList(ApplicationConfig.API_BC_QUERYTYPE));
param.put("a1cat", ApplicationConfig.API_BC_ISA1CAT);
param.put("reputation", ApplicationConfig.API_BC_ISREPU);
param.put("xml", ApplicationConfig.API_BC_ISXML); // json or xml格式
param.put("oemid", bcConfig.getOemid());
param.put("deviceid", bcConfig.getDeviceid());
param.put("uid", bcConfig.getUid());
param.put("queries", Collections.singletonList(bcConfig.getQueryType()));
param.put("a1cat", bcConfig.getIsa1cat());
param.put("reputation", bcConfig.getIsReputation());
// json or xml格式
param.put("xml", bcConfig.getIsxml());
param.put("urls", domains);
//建立实际的连接
@@ -67,7 +69,7 @@ public class BrightCloud {
writer.write(param.toString());
writer.flush();
} catch (IOException e) {
e.printStackTrace();
logger.error(e);
}
try {
@@ -85,19 +87,19 @@ public class BrightCloud {
jsonRes = JSONObject.parseObject(sbf.toString());
con.disconnect();
} catch (IOException e) {
e.printStackTrace();
logger.error(e);
}
return jsonRes;
}
// json响应内容解析
public List<DomainCategory> responseSparse(JSONObject records){
public List<DomainCategory> responseSparse(JSONObject records) {
List<DomainCategory> domainFiles = new ArrayList<>();
Boolean querySucess = records.get("status").equals(200);
if (!querySucess) {
System.out.print(records);
LOG.error("Wrong query. Query type: " + records.get("type"));
logger.error(JSON.toJSONString(records));
logger.error("Wrong query. Query type: " + records.get("type"));
} else {
JSONArray array = records.getJSONArray("results");
for (int i = 0; i < array.size(); i++) {
@@ -105,7 +107,7 @@ public class BrightCloud {
// json处理
JSONObject queries = jo.getJSONObject("queries");
JSONObject getInfo = queries.getJSONObject(ApplicationConfig.API_BC_QUERYTYPE);
JSONObject getInfo = queries.getJSONObject(bcConfig.getQueryType());
JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0);
Integer catId = cat.getInteger("catid");
@@ -127,67 +129,61 @@ public class BrightCloud {
return domainFiles;
}
private String getRepLevel(Integer repScore){
private String getRepLevel(Integer repScore) {
String level = null; //用str存放数据
if (repScore > 80){ level="Trustworthy";}
else if (repScore > 60){ level="Low Risk";}
else if (repScore > 40){ level="Moderate Risk";}
else if (repScore > 20){ level="Suspicious";}
else if (repScore > 0){ level="High Risk";}
if (repScore > 80) {
level = "Trustworthy";
} else if (repScore > 60) {
level = "Low Risk";
} else if (repScore > 40) {
level = "Moderate Risk";
} else if (repScore > 20) {
level = "Suspicious";
} else if (repScore > 0) {
level = "High Risk";
}
return level;
}
// 获取类别id对应信息
public void geneCatInfo(){
if (catId2Info.size()==0){
public void geneCatInfo() {
if (catId2Info.size() == 0) {
JSONObject jsonObject;
// String filePath = Objects.requireNonNull(BrightCloud.class.getClassLoader()
// .getResource(ApplicationConfig.API_BC_CATEINFO_FILE)).getFile();
String filePath =ApplicationConfig.API_BC_CATEINFO_FILE;
String s = FileUtils.readJsonFile(filePath);
String s = FileUtils.readJsonFile(bcConfig.getCateinfoFilepath());
jsonObject = JSON.parseObject(s);
if (!(jsonObject==null)){
if (!(jsonObject == null)) {
JSONObject tmp = (JSONObject) jsonObject.getJSONArray("results").get(0);
JSONArray catInfoArray = tmp.getJSONObject("queries").getJSONObject("getcatlist").getJSONArray("cats");
for (int i = 0; i < catInfoArray.size(); i++){
for (int i = 0; i < catInfoArray.size(); i++) {
JSONObject keyObject = catInfoArray.getJSONObject(i);
List<String> value = new ArrayList<>(Arrays.asList(
keyObject.getString("catname"),
keyObject.getString("catgroup")));
catId2Info.put(i+1, value);
catId2Info.put(i + 1, value);
}
}
}
}
public List<String> getCatInfo(Integer catId){
public List<String> getCatInfo(Integer catId) {
List<String> info = Arrays.asList("", "");
if (0 < catId && catId <= 83) {
if (catId2Info.size()==0){
if (catId2Info.size() == 0) {
geneCatInfo();
}
info = catId2Info.get(catId);
if (info == null){
LOG.error("Failed at geneCatInfo function");
System.out.print("Failed at geneCatInfo function");
if (info == null) {
logger.error("Failed at geneCatInfo function");
}
}
return info;
}
public Integer getMaxObjectNum() {
return maxObjectNum;
}
public static void main(String[] args) {
JSONObject queryResults = new BrightCloud().getQueryResults(Arrays.asList("baidu.com"));
new BrightCloud().responseSparse(queryResults);
}
}

152
src/main/java/com/mesasoft/cn/sketch/api/ChinaZ.java
View File

@@ -1,16 +1,10 @@
package com.mesasoft.cn.sketch.api;
/*
* @Description:
* @Author: chenxu
* @Date: 2021-12-27 13:59:29
* @LastEditTime: 2021-12-29 17:05:45
* @LastEditors: chenxu
*/
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import com.mesasoft.cn.sketch.entity.DomainWhois;
import com.mesasoft.cn.sketch.config.ChinazConfig;
import com.mesasoft.cn.sketch.entity.domain.DomainWhois;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
@@ -18,7 +12,7 @@ import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.BufferedReader;
import java.io.InputStream;
@@ -29,11 +23,20 @@ import java.sql.Date;
import java.util.*;
import java.util.concurrent.TimeUnit;
/*
* @Description:
* @Author: chenxu
* @Date: 2021-12-27 13:59:29
* @LastEditTime: 2021-12-29 17:05:45
* @LastEditors: chenxu
*/
public class ChinaZ {
private static final Logger LOG = Logger.getLogger(ChinaZ.class);
private final String apiKey = ApplicationConfig.API_CHINAZ_KEY;
private static final Log logger = Log.get();
public List<DomainWhois> getQueryFiles(List<String> objectList) {
@Autowired
private ChinazConfig chinazConfig;
public List<DomainWhois> getChinaZDomains(List<String> objectList) {
List<JSONObject> queryResults = getQueryResults(objectList);
return responseSparse(queryResults);
}
@@ -43,12 +46,12 @@ public class ChinaZ {
* @Param : 域名
* @Return: 查询结果
*/
public JSONObject getQueryResult(String domain){
String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE;
Map<String, String> params = new LinkedHashMap<String,String>();
params.put("key", apiKey);
public JSONObject getQueryResult(String domain) {
String urlString = chinazConfig.getUrlSingle();
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("key", chinazConfig.getKey());
params.put("domain", domain);
return whoisInfoResolve(doPost(urlString, params),domain);
return whoisInfoResolve(doPost(urlString, params), domain);
}
/**
@@ -56,43 +59,42 @@ public class ChinaZ {
* @Param : 域名
* @Return: 查询结果
*/
public List<JSONObject> getQueryResults(List<String> domainList){
String urlString = ApplicationConfig.API_CHINAZ_URL_SINGLE;
public List<JSONObject> getQueryResults(List<String> domainList) {
List<JSONObject> whoisInfoList = new ArrayList<>();
for (String s : domainList) {
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("key", apiKey);
params.put("key", chinazConfig.getKey());
params.put("domain", s);
JSONObject r = doPost(urlString, params);
JSONObject r = doPost(chinazConfig.getUrlSingle(), params);
whoisInfoList.add(whoisInfoResolve(r, s));
}
return whoisInfoList;
return whoisInfoList;
}
public List<DomainWhois> responseSparse(List<JSONObject> records){
public List<DomainWhois> responseSparse(List<JSONObject> records) {
List<DomainWhois> whoisFiles = new ArrayList<>();
for(JSONObject record: records) {
for (JSONObject record : records) {
Boolean querySucess = record.getBoolean("isSuccess");
if (!querySucess) {
LOG.error("Failed query. Query response: " + record);
logger.error("Failed query. Query response: " + record);
break;
}
String fqdn = record.getString("domain_name");
String domainName = record.getString("domain_host");
Integer matchPattern = fqdn.equals(domainName)? 1 : 2 ;
Integer matchPattern = fqdn.equals(domainName) ? 1 : 2;
String source = "chinaz";
// json处理
Date creatDate = null;
Date expiraDate = null;
java.util.Date tmpDate = record.getDate("domain_whois_create_time");
if(tmpDate!=null){
if (tmpDate != null) {
creatDate = new Date(tmpDate.getTime());
}
tmpDate = record.getDate("domain_whois_expiration_time");
if(tmpDate!=null){
if (tmpDate != null) {
expiraDate = new Date(tmpDate.getTime());
}
whoisFiles.add(new DomainWhois(
@@ -119,17 +121,18 @@ public class ChinaZ {
}
return whoisFiles;
}
/**
* @Description: 解析并重构JSON串
* @Param : 查询得到的“单个”JSON串
* @Return: 返回重构的JSON串
*/
public JSONObject whoisInfoResolve(JSONObject jsonRes,String queryDomain){
public JSONObject whoisInfoResolve(JSONObject jsonRes, String queryDomain) {
JSONObject whoisInfo = new JSONObject(true);
JSONObject res = jsonRes.getJSONObject("Result");
if(jsonRes.get("StateCode").equals(1)){
if (jsonRes.get("StateCode").equals(1)) {
whoisInfo.put("isSuccess", jsonRes.get("StateCode"));
whoisInfo.put("domain_name",queryDomain);
whoisInfo.put("domain_name", queryDomain);
whoisInfo.put("domain_host", res.get("Host"));
whoisInfo.put("domain_whois_create_time", res.get("CreationDate"));
whoisInfo.put("domain_whois_expiration_time", res.get("ExpirationDate"));
@@ -139,10 +142,10 @@ public class ChinaZ {
whoisInfo.put("domain_whois_phone", res.get("Phone"));
whoisInfo.put("domain_whois_name_servers", res.get("DnsServer"));
whoisInfo.put("domain_whois_status", res.get("DomainStatus"));
}else{
} else {
whoisInfo.put("isSuccess", jsonRes.get("StateCode"));
}
return whoisInfo;
return whoisInfo;
}
/**
@@ -150,19 +153,19 @@ public class ChinaZ {
* @Param : 待查询域名
* @Return: 拼接好的URL
*/
public List<String> queryStringBuilder(List<String> domainList){
public List<String> queryStringBuilder(List<String> domainList) {
//将域名每50个划分一组
int CHINAZ_REQUEST_LIMIT = 50 ;
int CHINAZ_REQUEST_LIMIT = 50;
int domainListSize = domainList.size();
int toIndex = CHINAZ_REQUEST_LIMIT;
Map domainListMap = new HashMap();
int keyToken = 0;
for(int i = 0;i<domainList.size();i+=CHINAZ_REQUEST_LIMIT){
if(i+CHINAZ_REQUEST_LIMIT>domainListSize){ //作用为toIndex最后没有50条数据则剩余几条newList中就装几条
toIndex=domainListSize-i;
for (int i = 0; i < domainList.size(); i += CHINAZ_REQUEST_LIMIT) {
if (i + CHINAZ_REQUEST_LIMIT > domainListSize) { //作用为toIndex最后没有50条数据则剩余几条newList中就装几条
toIndex = domainListSize - i;
}
List<String> newList = domainList.subList(i,i+toIndex);
domainListMap.put("keyName"+keyToken, newList);
List<String> newList = domainList.subList(i, i + toIndex);
domainListMap.put("keyName" + keyToken, newList);
keyToken++;
}
//将批量查询的域名，构造成CHINAZ的格式
@@ -174,8 +177,8 @@ public class ChinaZ {
Object val = entry.getValue();
String urlString = "";
urlString = String.valueOf(val);
urlString = urlString.replace(", ","|");
urlString =urlString.replace("[","").replace("]","");
urlString = urlString.replace(", ", "|");
urlString = urlString.replace("[", "").replace("]", "");
domainListString.add(urlString);
}
return domainListString;
@@ -186,18 +189,18 @@ public class ChinaZ {
* @Param : 域名集合（不能超过50个）
* @Return: 查询结果
*/
public String batchRequest_step1(String domainsString){
public String batchRequest_step1(String domainsString) {
String TaskID = "";
String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH;
String urlString = chinazConfig.getUrlBatch();
Map<String, String> params = new LinkedHashMap<String,String>();
if (!Objects.equals(domainsString, "overflow")){
params.put("domains",domainsString);
params.put("key", apiKey);
JSONObject r = doPost(urlString, params);
Map<String, String> params = new LinkedHashMap<String, String>();
if (!Objects.equals(domainsString, "overflow")) {
params.put("domains", domainsString);
params.put("key", chinazConfig.getKey());
JSONObject r = doPost(urlString, params);
TaskID = r.get("TaskID").toString();
return TaskID;
}else{
} else {
return TaskID;
}
@@ -208,11 +211,11 @@ public class ChinaZ {
* @Param : 任务ID
* @Return: 查询结果
*/
public JSONObject batchRequest_step2(String TaskID){
String urlString = ApplicationConfig.API_CHINAZ_URL_BATCH;
public JSONObject batchRequest_step2(String TaskID) {
String urlString = chinazConfig.getUrlBatch();
Map<String, String> params = new LinkedHashMap<String,String>();
params.put("taskid",TaskID);
Map<String, String> params = new LinkedHashMap<String, String>();
params.put("taskid", TaskID);
JSONObject requestTotal = null;
requestTotal = doPost(urlString, params);
return requestTotal;
@@ -223,17 +226,15 @@ public class ChinaZ {
* @Param : 域名列表
* @Return: whois记录列表
*/
public List<String> batchRequestController(List<String> domainList){
public List<String> batchRequestController(List<String> domainList) {
List<String> result = new ArrayList<>();
if (domainList.size()> 5000){
System.out.println("Too many urls in a http post request!");
List<String> result = new ArrayList<>();
if (domainList.size() > 5000) {
logger.info("Too many urls in a http post request!");
}
List<String> domainListString = new ArrayList<>();
List<String> TaskID = new ArrayList<>();
List<String> TaskID = new ArrayList<>();
// Queue<String> queue = new LinkedList<String>();
domainListString = queryStringBuilder(domainList);
//循环发送请求，收集每个请求的TaskID
for (String domainParam : domainListString) {
@@ -276,7 +277,7 @@ public class ChinaZ {
* @Param : 请求对URL，POST请求体需要添加的 k-v 数据
* @Return: API JSON数据
*/
public JSONObject doPost(String url, Map params){
public JSONObject doPost(String url, Map params) {
JSONObject jsonRes = null;
try {
// 定义HttpClient
@@ -291,16 +292,14 @@ public class ChinaZ {
String name = (String) o;
String value = String.valueOf(params.get(name));
nvps.add(new BasicNameValuePair(name, value));
//System.out.println(name +"-"+value);
}
request.setEntity(new UrlEncodedFormEntity(nvps));
//发送请求
HttpResponse httpResponse = client.execute(request);
HttpResponse httpResponse = client.execute(request);
// 获取响应输入流
InputStream inStream = httpResponse.getEntity().getContent();
//对放回数据进行处理
BufferedReader reader = new BufferedReader(new InputStreamReader(inStream , StandardCharsets.UTF_8));
BufferedReader reader = new BufferedReader(new InputStreamReader(inStream, StandardCharsets.UTF_8));
StringBuilder strber = new StringBuilder();
StringBuilder sbf = new StringBuilder();
String strRead = null;
@@ -311,26 +310,11 @@ public class ChinaZ {
// 关闭输入流
inStream.close();
jsonRes = JSONObject.parseObject(sbf.toString());
}catch (Exception e) {
System.out.println("请求接口异常");
} catch (Exception e) {
logger.error("chinaZ request error", e);
}
return jsonRes;
}
public static void main(String[] args){
ChinaZ t = new ChinaZ();
//单查询测试
// System.out.println(t.singleRequest("aaa.baidu.com"));
//批量查询测试
List<String> domainList = new ArrayList<>();
domainList.add("www.baidu.com");
// domainList.add("aaa.qq.com");
// domainList.add("doc.mesalab.com");
// System.out.println(t.batchRequestController(domainList));
System.out.println(t.getQueryResults(domainList));
}
}

54
src/main/java/com/mesasoft/cn/sketch/config/ApplicationConfig.java
View File

@@ -1,54 +0,0 @@
package com.mesasoft.cn.sketch.config;
import com.mesasoft.cn.util.ConfigUtils;
public class ApplicationConfig {
public static final String QUERY_OUTPUT_DIR = ConfigUtils.getStringProperty("query.output.dir");
public static final String QUERY_TYPES_DOMAIN = ConfigUtils.getStringProperty("query.types.domain");
public static final String QUERY_TYPES_IP = ConfigUtils.getStringProperty("query.types.ip");
public static final Integer UPDATE_EXPIRED_DAY = ConfigUtils.getIntProperty("update.expired.day"); // 更新任务中过期时间长度（天数）
public static final Integer QUERY_READIN_BATCH = ConfigUtils.getIntProperty("query.readin.batch");
public static final Integer QUERY_LOG_FILE_LINE_INTERVAL = ConfigUtils.getIntProperty("query.log.file.line.interval"); // 文件查询时，打印log的读取行数间隔
// api参数
// brightcloud
public static final String API_BC_OEMID = ConfigUtils.getStringProperty("bc.oemid");
public static final String API_BC_DEVICEID = ConfigUtils.getStringProperty("bc.deviceid");
public static final String API_BC_UID = ConfigUtils.getStringProperty("bc.uid");
public static final String API_BC_URL = ConfigUtils.getStringProperty("bc.url");
public static final String API_BC_METHOD = ConfigUtils.getStringProperty("bc.method");
public static final String API_BC_ISA1CAT = ConfigUtils.getStringProperty("bc.isa1cat");
public static final String API_BC_ISREPU = ConfigUtils.getStringProperty("bc.isReputation");
public static final String API_BC_ISXML = ConfigUtils.getStringProperty("bc.isxml");
public static final Integer API_BC_MAXIMUM_QUERYNUM = ConfigUtils.getIntProperty("bc.maximum.query.num"); // brightcloud单次查询条数上线
public static final String API_BC_QUERYTYPE = ConfigUtils.getStringProperty("bc.queryType");
public static final String API_BC_USE_REPORT_FILE = ConfigUtils.getStringProperty("bc.usereport.filepath"); // brightcloud使用报告导出文件目录
public static final String API_BC_CATEINFO_FILE = ConfigUtils.getStringProperty("bc.cateinfo.filepath"); // brightcloud使用报告导出文件目录
// chinaz
public static final String API_CHINAZ_URL_SINGLE = ConfigUtils.getStringProperty("chinaz.url.single");
public static final String API_CHINAZ_URL_BATCH = ConfigUtils.getStringProperty("chinaz.url.batch");
public static final String API_CHINAZ_KEY = ConfigUtils.getStringProperty("chinaz.key");
public static final Integer API_CHINAZ_MAXIMUM_QUERYNUM = ConfigUtils.getIntProperty("chinaz.maximum.query.num");
public static final String API_CHINAZ_USE_REPORT_FILE = ConfigUtils.getStringProperty("chinaz.usereport.filepath");
// Mariadb
public static final String DATABASE = ConfigUtils.getStringProperty("database");
public static final String DOMAIN_CATE_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.category");
public static final String DOMAIN_WHOIS_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.whois");
public static final String DNS_SERVER_TABLENAME = ConfigUtils.getStringProperty("tablename.dns.server");
public static final Integer DB_QUERY_BATCH_SIZE = ConfigUtils.getIntProperty("db.query.batch.size");
// 其他
public static final String TLD_FILE = ConfigUtils.getStringProperty("tld.file"); // 顶级域名公开列表文件
}

84
src/main/java/com/mesasoft/cn/sketch/config/MariaDbBase.java
View File

@@ -1,84 +0,0 @@
package com.mesasoft.cn.sketch.config;
import com.mesasoft.cn.util.TimeUtils;
import org.apache.log4j.Logger;
import java.sql.*;
import java.util.Date;
import java.util.Properties;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/28
* Time: 2:56 PM
* Description: No Description
*/
public class MariaDbBase {
private static final Logger LOG = Logger.getLogger(MariaDbBase.class);
private static final Properties props = new Properties();
private final Statement statement;
public MariaDbBase(Connection conn, Statement stat) {
statement = stat;
}
/**
* 执行写入sql
*/
public void writeSqlExecute(String sql){
try {
statement.executeUpdate(sql);
} catch (SQLIntegrityConstraintViolationException e){
LOG.error("Duplicated entry for key 'PRIMARY'");
} catch (SQLException exception) {
LOG.error("Sql : " + sql);
exception.printStackTrace();
}
}
/**
* 执行查询sql
*/
public ResultSet querySqlExecute(String sql){
ResultSet set = null;
try {
set = statement.executeQuery(sql);
} catch (SQLException exception) {
exception.printStackTrace();
}
return set;
}
/**
* 获得指定表格、按指定时间字段的过期记录
* @param tableName 库表名称
* @param timeColumnName 时间列名
* @return 查询结果
*/
public ResultSet getExpiredRecord(String tableName, String timeColumnName){
Date lastUpdateTime = new Timestamp(getExpiredTime(ApplicationConfig.UPDATE_EXPIRED_DAY).getTime());
String resSql = "SELECT *"
+ " FROM " + ApplicationConfig.DATABASE + "." + tableName
+ " WHERE " + timeColumnName + " < '" + lastUpdateTime + '\'';
LOG.debug("Update task: expired query sql" + resSql);
return querySqlExecute(resSql);
}
/**
* TODO: getUnlabeledRecord() 考虑多个来源的情况
*/
/**
* 获得过期时间, 当前时间的expiredRangeDays天之前的日期为过期日期
*/
public static Date getExpiredTime(int expiredRangeDays){
return new Timestamp(TimeUtils.getStartOfDay(-expiredRangeDays).getTime());
}
}

258
src/main/java/com/mesasoft/cn/sketch/config/SketchDatabaseConfig.java
View File

@@ -1,125 +1,133 @@
//package com.zhazhapan.efo.sketch.config;
//
//import com.alibaba.druid.pool.DruidDataSource;
//import lombok.Data;
//import org.apache.ibatis.session.SqlSessionFactory;
//import org.mybatis.spring.SqlSessionFactoryBean;
//import org.mybatis.spring.annotation.MapperScan;
//import org.springframework.beans.factory.annotation.Qualifier;
//import org.springframework.beans.factory.annotation.Value;
//import org.springframework.boot.context.properties.ConfigurationProperties;
//import org.springframework.context.annotation.Bean;
//import org.springframework.context.annotation.Configuration;
//import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
//import org.springframework.jdbc.datasource.DataSourceTransactionManager;
//
//import javax.sql.DataSource;
//import java.sql.SQLException;
//
///**
// * @ProjectName
// * @Description: 后台数据源配置类
// */
//@Data
//@Configuration
//@ConfigurationProperties(prefix = "sketch.datasource.druid")
//@MapperScan(basePackages = SketchDatabaseConfig.PACKAGE, sqlSessionFactoryRef = "sketchSqlSessionFactory")
//public class SketchDatabaseConfig {
// /**
// * dao层的包路径
// */
// static final String PACKAGE = "com.mao.mysqlhive.demomh.mapper.sketch";
//
// /**
// * mapper文件的相对路径
// */
// private static final String MAPPER_LOCATION = "classpath:mappers/sketch/*Mapper.xml";
//
// @Value("${sketch.datasource.druid.filters}")
// private String filters;
// @Value("${sketch.datasource.druid.driverClassName}")
// private String url;
// @Value("${sketch.datasource.druid.url}")
// private String username;
// @Value("${sketch.datasource.druid.username}")
// private String password;
// @Value("${sketch.datasource.druid.password}")
// private String driverClassName;
// @Value("${sketch.datasource.druid.initialSize}")
// private int initialSize;
// @Value("${sketch.datasource.druid.minIdle}")
// private int minIdle;
// @Value("${sketch.datasource.druid.maxActive}")
// private int maxActive;
// @Value("${sketch.datasource.druid.maxWait}")
// private long maxWait;
// @Value("${sketch.datasource.druid.timeBetweenEvictionRunsMillis}")
// private long timeBetweenEvictionRunsMillis;
// @Value("${sketch.datasource.druid.minEvictableIdleTimeMillis}")
// private long minEvictableIdleTimeMillis;
// @Value("${sketch.datasource.druid.validationQuery}")
// private String validationQuery;
// @Value("${sketch.datasource.druid.testWhileIdle}")
// private boolean testWhileIdle;
// @Value("${sketch.datasource.druid.testOnBorrow}")
// private boolean testOnBorrow;
// @Value("${sketch.datasource.druid.testOnReturn}")
// private boolean testOnReturn;
// @Value("${sketch.datasource.druid.poolPreparedStatements}")
// private boolean poolPreparedStatements;
// @Value("${sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize}")
// private int maxPoolPreparedStatementPerConnectionSize;
//
//
// @Bean(name = "sketchDataSource")
// public DataSource sketchDataSource() throws SQLException {
// DruidDataSource druid = new DruidDataSource();
// // 监控统计拦截的filters
// druid.setFilters(filters);
//
// // 配置基本属性
// druid.setDriverClassName(driverClassName);
// druid.setUsername(username);
// druid.setPassword(password);
// druid.setUrl(url);
//
// //初始化时建立物理连接的个数
// druid.setInitialSize(initialSize);
// //最大连接池数量
// druid.setMaxActive(maxActive);
// //最小连接池数量
// druid.setMinIdle(minIdle);
// //获取连接时最大等待时间，单位毫秒。
// druid.setMaxWait(maxWait);
// //间隔多久进行一次检测，检测需要关闭的空闲连接
// druid.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
// //一个连接在池中最小生存的时间
// druid.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
// //用来检测连接是否有效的sql
// druid.setValidationQuery(validationQuery);
// //建议配置为true，不影响性能，并且保证安全性。
// druid.setTestWhileIdle(testWhileIdle);
// //申请连接时执行validationQuery检测连接是否有效
// druid.setTestOnBorrow(testOnBorrow);
// druid.setTestOnReturn(testOnReturn);
// //是否缓存preparedStatement，也就是PSCache，oracle设为true，mysql设为false。分库分表较多推荐设置为false
// druid.setPoolPreparedStatements(poolPreparedStatements);
// // 打开PSCache时，指定每个连接上PSCache的大小
// druid.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize);
// return druid;
// }
//
// @Bean(name = "sketchTransactionManager")
// public DataSourceTransactionManager sketchTransactionManager() throws SQLException {
// return new DataSourceTransactionManager(sketchDataSource());
// }
//
// @Bean(name = "sketchSqlSessionFactory")
// public SqlSessionFactory sketchSqlSessionFactory(@Qualifier("sketchDataSource") DataSource sketchDataSource) throws Exception {
// final SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
// sessionFactory.setDataSource(sketchDataSource);
// sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(SketchDatabaseConfig.MAPPER_LOCATION));
//
// return sessionFactory.getObject();
// }
//}
/*
package cn.ac.iie.efo.sketch.config;
import com.alibaba.druid.pool.DruidDataSource;
import lombok.Data;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import javax.sql.DataSource;
import java.sql.SQLException;
*/
/**
* @ProjectName
* @Description: 后台数据源配置类
*//*
@Data
@Configuration
@ConfigurationProperties(prefix = "sketch.datasource.druid")
@MapperScan(basePackages = SketchDatabaseConfig.PACKAGE, sqlSessionFactoryRef = "sketchSqlSessionFactory")
public class SketchDatabaseConfig {
*/
/**
* dao层的包路径
*//*
static final String PACKAGE = "com.mao.mysqlhive.demomh.mapper.sketch";
*/
/**
* mapper文件的相对路径
*//*
private static final String MAPPER_LOCATION = "classpath:mappers/sketch/*Mapper.xml";
@Value("${sketch.datasource.druid.filters}")
private String filters;
@Value("${sketch.datasource.druid.driverClassName}")
private String url;
@Value("${sketch.datasource.druid.url}")
private String username;
@Value("${sketch.datasource.druid.username}")
private String password;
@Value("${sketch.datasource.druid.password}")
private String driverClassName;
@Value("${sketch.datasource.druid.initialSize}")
private int initialSize;
@Value("${sketch.datasource.druid.minIdle}")
private int minIdle;
@Value("${sketch.datasource.druid.maxActive}")
private int maxActive;
@Value("${sketch.datasource.druid.maxWait}")
private long maxWait;
@Value("${sketch.datasource.druid.timeBetweenEvictionRunsMillis}")
private long timeBetweenEvictionRunsMillis;
@Value("${sketch.datasource.druid.minEvictableIdleTimeMillis}")
private long minEvictableIdleTimeMillis;
@Value("${sketch.datasource.druid.validationQuery}")
private String validationQuery;
@Value("${sketch.datasource.druid.testWhileIdle}")
private boolean testWhileIdle;
@Value("${sketch.datasource.druid.testOnBorrow}")
private boolean testOnBorrow;
@Value("${sketch.datasource.druid.testOnReturn}")
private boolean testOnReturn;
@Value("${sketch.datasource.druid.poolPreparedStatements}")
private boolean poolPreparedStatements;
@Value("${sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize}")
private int maxPoolPreparedStatementPerConnectionSize;
@Bean(name = "sketchDataSource")
public DataSource sketchDataSource() throws SQLException {
DruidDataSource druid = new DruidDataSource();
// 监控统计拦截的filters
druid.setFilters(filters);
// 配置基本属性
druid.setDriverClassName(driverClassName);
druid.setUsername(username);
druid.setPassword(password);
druid.setUrl(url);
//初始化时建立物理连接的个数
druid.setInitialSize(initialSize);
//最大连接池数量
druid.setMaxActive(maxActive);
//最小连接池数量
druid.setMinIdle(minIdle);
//获取连接时最大等待时间，单位毫秒。
druid.setMaxWait(maxWait);
//间隔多久进行一次检测，检测需要关闭的空闲连接
druid.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis);
//一个连接在池中最小生存的时间
druid.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis);
//用来检测连接是否有效的sql
druid.setValidationQuery(validationQuery);
//建议配置为true，不影响性能，并且保证安全性。
druid.setTestWhileIdle(testWhileIdle);
//申请连接时执行validationQuery检测连接是否有效
druid.setTestOnBorrow(testOnBorrow);
druid.setTestOnReturn(testOnReturn);
//是否缓存preparedStatement，也就是PSCache，oracle设为true，mysql设为false。分库分表较多推荐设置为false
druid.setPoolPreparedStatements(poolPreparedStatements);
// 打开PSCache时，指定每个连接上PSCache的大小
druid.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize);
return druid;
}
@Bean(name = "sketchTransactionManager")
public DataSourceTransactionManager sketchTransactionManager() throws SQLException {
return new DataSourceTransactionManager(sketchDataSource());
}
@Bean(name = "sketchSqlSessionFactory")
public SqlSessionFactory sketchSqlSessionFactory(@Qualifier("sketchDataSource") DataSource sketchDataSource) throws Exception {
final SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean();
sessionFactory.setDataSource(sketchDataSource);
sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(SketchDatabaseConfig.MAPPER_LOCATION));
return sessionFactory.getObject();
}
}
*/

4
src/main/java/com/mesasoft/cn/sketch/dao/SketchDAO.java
View File

@@ -1,8 +1,8 @@
package com.mesasoft.cn.sketch.dao;
import com.mesasoft.cn.sketch.dao.sqlprovider.SketchSqlProvider;
import com.mesasoft.cn.sketch.entity.DomainCategory;
import com.mesasoft.cn.dao.sqlprovider.UserSqlProvider;
import com.mesasoft.cn.sketch.dao.sqlprovider.SketchSqlProvider;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import org.apache.ibatis.annotations.*;
import org.springframework.stereotype.Repository;

6
src/main/java/com/mesasoft/cn/sketch/dao/sqlprovider/SketchSqlProvider.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.sketch.dao.sqlprovider;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.Checker;
import org.apache.ibatis.annotations.Param;
@@ -19,9 +19,9 @@ public class SketchSqlProvider {
if (Checker.isNotEmpty(condition)) {
WHERE("fqdn like '%" + condition + "'");
}
ORDER_BY(SketchApplication.settings.getStringUseEval(ConfigConsts.USER_ORDER_BY_OF_SETTINGS));
ORDER_BY(WebSketchApplication.settings.getStringUseEval(ConfigConsts.USER_ORDER_BY_OF_SETTINGS));
}}.toString();
int size = SketchApplication.settings.getIntegerUseEval(ConfigConsts.USER_PAGE_SIZE_OF_SETTINGS);
int size = WebSketchApplication.settings.getIntegerUseEval(ConfigConsts.USER_PAGE_SIZE_OF_SETTINGS);
return sql + " limit " + (offset * size) + "," + size;
}
}

340
src/main/java/com/mesasoft/cn/sketch/entity/DomainCategory.java
View File

@@ -1,340 +0,0 @@
package com.mesasoft.cn.sketch.entity;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.util.ConfigUtils;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import com.mesasoft.cn.util.MariaDbBase;
import com.mesasoft.cn.util.ValidationUtils;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
public class DomainCategory {
private static final String dataBase = ApplicationConfig.DATABASE;
private static final String tableName = ApplicationConfig.DOMAIN_CATE_TABLENAME;
private String fqdn;
private String source;
private Boolean query_success;
private Integer match_pattern;
private Integer reputation_score;
private String reputation_level;
private Integer category_id;
private String category_name;
private String category_group;
private Integer category_conf;
private Boolean is_a1_cat;
private Integer status_code = 0;
private String submit_user ="''";
// category schema
public DomainCategory(String fqdn,
String source,
Boolean query_success,
Integer match_pattern,
Integer reputation_score,
String reputationLevel,
Integer categoryId,
String categoryName,
String categoryGroup,
Integer categoryConf,
Boolean isA1Cat, Integer statusCode, String submitUser) {
this.fqdn = fqdn;
this.source = source; // 默认应为为brightcloud
this.query_success = query_success;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.reputation_score = reputation_score;
this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel);
this.category_id = categoryId;
this.category_name = ConfigUtils.getEffectiveString(categoryName);
this.category_group = ConfigUtils.getEffectiveString(categoryGroup);
this.category_conf = categoryConf;
this.is_a1_cat = isA1Cat;
this.status_code = statusCode;
this.submit_user = submitUser;
}
public DomainCategory(String fqdn,
String source,
Boolean query_success,
Integer match_pattern,
Integer reputation_score,
String reputationLevel,
Integer categoryId,
String categoryName,
String categoryGroup,
Integer categoryConf,
Boolean isA1Cat) {
this.fqdn = fqdn;
this.source = source; // 默认应为为brightcloud
this.query_success = query_success;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.reputation_score = reputation_score;
this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel);
this.category_id = categoryId;
this.category_name = ConfigUtils.getEffectiveString(categoryName);
this.category_group = ConfigUtils.getEffectiveString(categoryGroup);
this.category_conf = categoryConf;
this.is_a1_cat = isA1Cat;
}
public static void insertRecords(List<DomainCategory> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainCategory categoryFile : categoryFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + categoryFile.getKeys() + ") values" +
'(' + categoryFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public void updateRecords(List<DomainCategory> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainCategory categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainCategory> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") and source = '" + source + "'";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}
public static List<DomainCategory> rs2schema(ResultSet rs) throws SQLException {
List<DomainCategory> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainCategory(
rs.getString("fqdn"),
rs.getString("source"),
rs.getBoolean("query_success"),
rs.getInt("match_pattern"),
rs.getInt("reputation_score"),
rs.getString("reputation_level"),
rs.getInt("category_id"),
rs.getString("category_name"),
rs.getString("category_group"),
rs.getInt("category_conf"),
rs.getBoolean("is_a1_cat"),
rs.getInt("status_code"),
rs.getString("submit_user")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainCategory schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("reputation_score", schema.getReputation_score());
jsonObject.put("reputation_level", schema.getReputation_level());
jsonObject.put("category_id", schema.getCategory_id());
jsonObject.put("category_group", schema.getCategory_group());
jsonObject.put("category_name", schema.getCategory_name());
jsonObject.put("category_conf", schema.getCategory_conf());
jsonObject.put("is_a1_cat", schema.getIs_a1_cat());
jsonObject.put("status_code", schema.getStatus_code());
jsonObject.put("submit_user", schema.getSubmit_user());
return jsonObject;
}
public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", " + reputation_score +
", '" + reputation_level + '\'' +
", " + category_id +
", '" + category_name + '\'' +
", '" + category_group + '\'' +
", " + category_conf +
", " + status_code +
", " + submit_user +
", " + is_a1_cat;
return resString.replace("'null'", "null");
}
public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", reputation_score" +
", reputation_level" +
", category_id" +
", category_name" +
", category_group" +
", category_conf" +
", status_code" +
", submit_user" +
", is_a1_cat";
return resString;
}
public String getKeyValues() {
String resString = "source='" + source + '\'' +
", query_success=" + query_success +
", match_pattern=" + match_pattern +
", reputation_score=" + reputation_score +
", reputation_level='" + reputation_level + '\'' +
", category_id=" + category_id +
", category_name='" + category_name + '\'' +
", category_group='" + category_group + '\'' +
", category_conf=" + category_conf +
", is_a1_cat=" + is_a1_cat;
return resString.replace("'null'", "null");
}
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public Boolean getQuery_success() {
return query_success;
}
public void setQuery_success(Boolean query_success) {
this.query_success = query_success;
}
public Integer getMatch_pattern() {
return match_pattern;
}
public void setMatch_pattern(Integer match_pattern) {
this.match_pattern = match_pattern;
}
public Integer getReputation_score() {
return reputation_score;
}
public void setReputation_score(Integer reputation_score) {
this.reputation_score = reputation_score;
}
public String getReputation_level() {
return reputation_level;
}
public void setReputation_level(String reputation_level) {
this.reputation_level = reputation_level;
}
public Integer getCategory_id() {
return category_id;
}
public void setCategory_id(Integer category_id) {
this.category_id = category_id;
}
public String getCategory_name() {
return category_name;
}
public void setCategory_name(String category_name) {
this.category_name = category_name;
}
public String getCategory_group() {
return category_group;
}
public void setCategory_group(String category_group) {
this.category_group = category_group;
}
public Integer getCategory_conf() {
return category_conf;
}
public void setCategory_conf(Integer category_conf) {
this.category_conf = category_conf;
}
public Boolean getIs_a1_cat() {
return is_a1_cat;
}
public void setIs_a1_cat(Boolean is_a1_cat) {
this.is_a1_cat = is_a1_cat;
}
public Integer getStatus_code() {
return status_code;
}
public void setStatus_code(Integer status_code) {
this.status_code = status_code;
}
public String getSubmit_user() {
return submit_user;
}
public void setSubmit_user(String submit_user) {
this.submit_user = submit_user;
}
}

423
src/main/java/com/mesasoft/cn/sketch/entity/DomainWhois.java
View File

@@ -1,423 +0,0 @@
package com.mesasoft.cn.sketch.entity;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import com.mesasoft.cn.util.ConfigUtils;
import com.mesasoft.cn.util.MariaDbBase;
import com.mesasoft.cn.util.ValidationUtils;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
public class DomainWhois {
private static final String dataBase = ApplicationConfig.DATABASE;
private static final String tableName = ApplicationConfig.DOMAIN_WHOIS_TABLENAME;
private String fqdn;
private String source;
private Boolean query_success;
private Integer match_pattern;
private String whois_domain;
private Timestamp whois_update_date;
private Timestamp whois_create_date;
private Timestamp whois_expire_date;
private String whois_email;
private String whois_ns;
private String whois_registrar_name;
private String whois_registrant_org;
private String whois_registrant_name;
private String whois_registrant_street;
private String whois_registrant_city;
private String whois_registrant_state;
private String whois_registrant_postcode;
private String whois_registrant_country;
private String whois_registrant_phone;
// category schema
public DomainWhois(String fqdn,
String source,
Integer match_pattern,
Boolean query_success,
String whoisDomain,
Date whoisUpdateDate,
Date whoisCreateDate,
Date whoisExpireDate,
String whoisEmail,
String whoisNs,
String whoisRegistrarName,
String whoisRegistrantOrg,
String whoisRegistrantName,
String whoisRegistrantStreet,
String whoisRegistrantCity,
String whoisRegistrantState,
String whoisRegistrantPostcode,
String whoisRegistrantCountry,
String whoisRegistrantPhone
) {
this.fqdn = fqdn;
this.source = source;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.query_success = query_success;
this.whois_domain = ConfigUtils.getEffectiveString(whoisDomain);
this.whois_update_date = whoisUpdateDate == null ? null : new Timestamp(whoisUpdateDate.getTime());
this.whois_create_date = whoisCreateDate == null ? null : new Timestamp(whoisCreateDate.getTime());
this.whois_expire_date = whoisExpireDate == null ? null : new Timestamp(whoisExpireDate.getTime());
this.whois_email = ConfigUtils.getEffectiveString(whoisEmail);
this.whois_ns = ConfigUtils.getEffectiveString(whoisNs);
this.whois_registrar_name = ConfigUtils.getEffectiveString(whoisRegistrarName);
this.whois_registrant_org = ConfigUtils.getEffectiveString(whoisRegistrantOrg);
this.whois_registrant_name = ConfigUtils.getEffectiveString(whoisRegistrantName);
this.whois_registrant_street = ConfigUtils.getEffectiveString(whoisRegistrantStreet);
this.whois_registrant_city = ConfigUtils.getEffectiveString(whoisRegistrantCity);
this.whois_registrant_state = ConfigUtils.getEffectiveString(whoisRegistrantState);
this.whois_registrant_postcode = ConfigUtils.getEffectiveString(whoisRegistrantPostcode);
this.whois_registrant_country = ConfigUtils.getEffectiveString(whoisRegistrantCountry);
this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone);
}
public static void insertRecords(List<DomainWhois> whoisFiles, MariaDbBase mariaDbBase) {
for (DomainWhois whoisFile : whoisFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static String insertSql(List<DomainWhois> whoisFiles) {
DomainWhois whoisFile = whoisFiles.get(0);
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
return resSql;
}
public static void updateRecords(List<DomainWhois> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainWhois categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainWhois> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") ";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}
public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", '" + whois_domain + '\'' +
", '" + whois_update_date + '\'' +
", '" + whois_create_date + '\'' +
", '" + whois_expire_date + '\'' +
", '" + whois_email + '\'' +
", '" + whois_ns + '\'' +
", '" + whois_registrar_name + '\'' +
", '" + whois_registrant_org + '\'' +
", '" + whois_registrant_name + '\'' +
", '" + whois_registrant_street + '\'' +
", '" + whois_registrant_city + '\'' +
", '" + whois_registrant_state + '\'' +
", '" + whois_registrant_postcode + '\'' +
", '" + whois_registrant_country + '\'' +
", '" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public static List<DomainWhois> rs2schema(ResultSet rs) throws SQLException {
List<DomainWhois> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainWhois(
rs.getString("fqdn"),
rs.getString("source"),
rs.getInt("match_pattern"),
rs.getBoolean("query_success"),
rs.getString("whois_domain"),
(Date) rs.getDate("whois_update_date"),
(Date) rs.getDate("whois_create_date"),
(Date) rs.getDate("whois_expire_date"),
rs.getString("whois_email"),
rs.getString("whois_ns"),
rs.getString("whois_registrar_name"),
rs.getString("whois_registrant_org"),
rs.getString("whois_registrant_name"),
rs.getString("whois_registrant_street"),
rs.getString("whois_registrant_city"),
rs.getString("whois_registrant_state"),
rs.getString("whois_registrant_postcode"),
rs.getString("whois_registrant_country"),
rs.getString("whois_registrant_phone")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainWhois schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("whois_domain", schema.getWhois_domain());
jsonObject.put("whois_update_date", schema.getWhois_update_date());
jsonObject.put("whois_create_date", schema.getWhois_create_date());
jsonObject.put("whois_expire_date", schema.getWhois_expire_date());
jsonObject.put("whois_email", schema.getWhois_email());
jsonObject.put("whois_ns", schema.getWhois_ns());
jsonObject.put("whois_registrar_name", schema.getWhois_registrar_name());
jsonObject.put("whois_registrant_org", schema.getWhois_registrant_org());
jsonObject.put("whois_registrant_name", schema.getWhois_registrant_name());
jsonObject.put("whois_registrant_street", schema.getWhois_registrant_street());
jsonObject.put("whois_registrant_city", schema.getWhois_registrant_city());
jsonObject.put("whois_registrant_state", schema.getWhois_registrant_state());
jsonObject.put("whois_registrant_postcode", schema.getWhois_registrant_postcode());
jsonObject.put("whois_registrant_country", schema.getWhois_registrant_country());
jsonObject.put("whois_registrant_phone", schema.getWhois_registrant_phone());
return jsonObject;
}
public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", whois_domain" +
", whois_update_date" +
", whois_create_date" +
", whois_expire_date" +
", whois_email" +
", whois_ns" +
", whois_registrar_name" +
", whois_registrant_org" +
", whois_registrant_name" +
", whois_registrant_street" +
", whois_registrant_city" +
", whois_registrant_state" +
", whois_registrant_postcode" +
", whois_registrant_country" +
", whois_registrant_phone";
return resString;
}
public String getKeyValues() {
String resString = "query_success=" + query_success +
", source='" + source + '\'' +
", match_pattern=" + match_pattern +
", whois_domain='" + whois_domain + '\'' +
", whois_update_date='" + whois_update_date + '\'' +
", whois_create_date='" + whois_create_date + '\'' +
", whois_expire_date='" + whois_expire_date + '\'' +
", whois_email='" + whois_email + '\'' +
", whois_ns='" + whois_ns + '\'' +
", whois_registrar_name='" + whois_registrar_name + '\'' +
", whois_registrant_org='" + whois_registrant_org + '\'' +
", whois_registrant_name='" + whois_registrant_name + '\'' +
", whois_registrant_street='" + whois_registrant_street + '\'' +
", whois_registrant_city='" + whois_registrant_city + '\'' +
", whois_registrant_state='" + whois_registrant_state + '\'' +
", whois_registrant_postcode='" + whois_registrant_postcode + '\'' +
", whois_registrant_country='" + whois_registrant_country + '\'' +
", whois_registrant_phone='" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
public Boolean getQuery_success() {
return query_success;
}
public void setQuery_success(Boolean query_success) {
this.query_success = query_success;
}
public Integer getMatch_pattern() {
return match_pattern;
}
public void setMatch_pattern(Integer match_pattern) {
this.match_pattern = match_pattern;
}
public String getWhois_domain() {
return whois_domain;
}
public void setWhois_domain(String whois_domain) {
this.whois_domain = whois_domain;
}
public Timestamp getWhois_update_date() {
return whois_update_date;
}
public void setWhois_update_date(Timestamp whois_update_date) {
this.whois_update_date = whois_update_date;
}
public Timestamp getWhois_create_date() {
return whois_create_date;
}
public void setWhois_create_date(Timestamp whois_create_date) {
this.whois_create_date = whois_create_date;
}
public Timestamp getWhois_expire_date() {
return whois_expire_date;
}
public void setWhois_expire_date(Timestamp whois_expire_date) {
this.whois_expire_date = whois_expire_date;
}
public String getWhois_email() {
return whois_email;
}
public void setWhois_email(String whois_email) {
this.whois_email = whois_email;
}
public String getWhois_ns() {
return whois_ns;
}
public void setWhois_ns(String whois_ns) {
this.whois_ns = whois_ns;
}
public String getWhois_registrar_name() {
return whois_registrar_name;
}
public void setWhois_registrar_name(String whois_registrar_name) {
this.whois_registrar_name = whois_registrar_name;
}
public String getWhois_registrant_org() {
return whois_registrant_org;
}
public void setWhois_registrant_org(String whois_registrant_org) {
this.whois_registrant_org = whois_registrant_org;
}
public String getWhois_registrant_name() {
return whois_registrant_name;
}
public void setWhois_registrant_name(String whois_registrant_name) {
this.whois_registrant_name = whois_registrant_name;
}
public String getWhois_registrant_street() {
return whois_registrant_street;
}
public void setWhois_registrant_street(String whois_registrant_street) {
this.whois_registrant_street = whois_registrant_street;
}
public String getWhois_registrant_city() {
return whois_registrant_city;
}
public void setWhois_registrant_city(String whois_registrant_city) {
this.whois_registrant_city = whois_registrant_city;
}
public String getWhois_registrant_state() {
return whois_registrant_state;
}
public void setWhois_registrant_state(String whois_registrant_state) {
this.whois_registrant_state = whois_registrant_state;
}
public String getWhois_registrant_postcode() {
return whois_registrant_postcode;
}
public void setWhois_registrant_postcode(String whois_registrant_postcode) {
this.whois_registrant_postcode = whois_registrant_postcode;
}
public String getWhois_registrant_country() {
return whois_registrant_country;
}
public void setWhois_registrant_country(String whois_registrant_country) {
this.whois_registrant_country = whois_registrant_country;
}
public String getWhois_registrant_phone() {
return whois_registrant_phone;
}
public void setWhois_registrant_phone(String whois_registrant_phone) {
this.whois_registrant_phone = whois_registrant_phone;
}
}

14
src/main/java/com/mesasoft/cn/sketch/service/SketchService.java
View File

@@ -1,5 +1,5 @@
/*
package com.zhazhapan.efo.sketch.service;
package cn.ac.iie.efo.sketch.service;
import cn.ac.iie.api.ChinaZ;
import cn.ac.iie.dao.MariaDbBase;
@@ -8,12 +8,12 @@ import cn.ac.iie.util.ValidationUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.zhazhapan.efo.sketch.api.BrightCloud;
import com.zhazhapan.efo.sketch.api.ChinaZ;
import com.zhazhapan.efo.sketch.config.ApplicationConfig;
import com.zhazhapan.efo.sketch.entity.DomainCategory;
import com.zhazhapan.efo.sketch.entity.DomainWhois;
import com.zhazhapan.efo.util.FileUtils;
import cn.ac.iie.efo.sketch.api.BrightCloud;
import cn.ac.iie.efo.sketch.api.ChinaZ;
import cn.ac.iie.efo.sketch.config.ApplicationConfig;
import cn.ac.iie.efo.sketch.entity.domain.DomainCategory;
import cn.ac.iie.efo.sketch.entity.domain.DomainWhois;
import cn.ac.iie.efo.util.FileUtils;
import org.apache.log4j.Logger;
import java.io.FileOutputStream;

46
src/main/java/com/mesasoft/cn/util/ConfigUtils.java
View File

@@ -1,46 +0,0 @@
package com.mesasoft.cn.util;
import org.apache.log4j.Logger;
import java.util.Properties;
public class ConfigUtils {
private static final Logger LOG = Logger.getLogger(ConfigUtils.class);
private static Properties propCommon = new Properties();
public static String getStringProperty(String key) {
return propCommon.getProperty(key);
}
public static Integer getIntProperty(String key) {
return Integer.parseInt(propCommon.getProperty(key));
}
public static Long getLongProperty(String key) {
return Long.parseLong(propCommon.getProperty(key));
}
public static Boolean getBooleanProperty(String key) {
return "true".equals(propCommon.getProperty(key).toLowerCase().trim());
}
public static String getEffectiveString(String s) {
if (!(s == null)) {
return s.length() == 0 ? null : s;
} else {
return null;
}
}
static {
try {
propCommon.load(ConfigUtils.class.getClassLoader().getResourceAsStream("sketch.properties"));
} catch (Exception e) {
propCommon = null;
LOG.error("配置加载失败");
}
}
}

5
src/main/java/com/mesasoft/cn/util/ControllerUtils.java
View File

@@ -12,7 +12,10 @@ import javax.activation.MimetypesFileTypeMap;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
/**
* @author pantao

197
src/main/java/com/mesasoft/cn/util/FileUtils.java
View File

@@ -1,197 +0,0 @@
package com.mesasoft.cn.util;
import org.apache.log4j.Logger;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 6:11 下午
*/
public class FileUtils {
private static final Logger LOG = Logger.getLogger(FileUtils.class);
public static List<String> readTxtFileIntoStringArrList(String filePath)
{
List<String> list = new ArrayList<>();
try
{
String encoding = "GBK";
File file = new File(filePath);
if (file.isFile() && file.exists())
{ // 判断文件是否存在
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null)
{
if (!lineTxt.equals("")) {
list.add(lineTxt.trim());
}
}
bufferedReader.close();
read.close();
}
else
{
System.out.println("Can not find file: " + filePath);
}
}
catch (Exception e)
{
System.out.println("Error occurred in Function 'readTxtFileIntoStringArrList'");
e.printStackTrace();
}
return list;
}
public static List<String> getBatchLineReadIn(BufferedReader bufferedReader, int batchSize){
List<String> list = new ArrayList<>();
String lineTxt;
try{
while ((lineTxt = bufferedReader.readLine()) != null && list.size()<batchSize)
{
if (!lineTxt.equals("")) {
list.add(lineTxt.trim());
}
}
} catch (IOException e){
e.printStackTrace();
}
return list;
}
public static void createFile(File filePath, String fileName){
try {
File file = new File(filePath.toString() + "/" + fileName);
if (!filePath.exists()){
filePath.mkdirs();
}
boolean isCreate = file.createNewFile();
if (isCreate){
LOG.info("File " + fileName + " is created.");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void createFile(File file){
try {
boolean isCreate = file.createNewFile();
if (isCreate){
LOG.info("File " + file + " is created.");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static String readJsonFile(String fileName) {
String jsonStr = "";
try {
File jsonFile = new File(fileName);
FileReader fileReader = new FileReader(jsonFile);
Reader reader = new InputStreamReader(new FileInputStream(jsonFile), "utf-8");
int ch = 0;
StringBuffer sb = new StringBuffer();
while ((ch = reader.read()) != -1) {
sb.append((char) ch);
}
fileReader.close();
reader.close();
jsonStr = sb.toString();
return jsonStr;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
public static String getFileName(File file){
String[] tmp = file.toString().split("/");
String fileName = tmp[tmp.length-1];
return fileName;
}
public static void writerClose(OutputStreamWriter outWriter, OutputStream outStream) throws IOException {
assert outWriter != null;
outWriter.close();
outStream.close();
}
public static void readerClose(BufferedReader bufferedReader, InputStreamReader inputStreamReader) throws IOException {
assert inputStreamReader != null;
bufferedReader.close();
inputStreamReader.close();
}
//执行cmd命令，获取返回结果
public static String execCMD(String command) {
StringBuilder sb =new StringBuilder();
try {
Process process=Runtime.getRuntime().exec(command);
BufferedReader bufferedReader=new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while((line=bufferedReader.readLine())!=null)
{
sb.append(line+"\n");
}
} catch (Exception e) {
return e.toString();
}
return sb.toString();
}
public static Long getFileLineNum(File file){
Long num = 0L;
if (!file.exists()){
LOG.error("File not exist: " + file.toString());
} else {
String res = FileUtils.execCMD("wc -l " + file.toString());
num = Long.parseLong(res.trim().split(" ")[0]);
}
return num;
}
public static int getMaxLength(String filePath) {
int lengthDomain = 0;
try {
String encoding = "UTF-8";
File file = new File(filePath);
if (file.isFile() && file.exists()) {
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null) {
String[] split = lineTxt.split("\\.");
if (split.length > lengthDomain) {
lengthDomain = split.length;
}
}
read.close();
} else {
LOG.error("FilePath is wrong--->{" + filePath + "}<---");
}
} catch (Exception e) {
LOG.error("Get filePathData error--->{" + e + "}<---");
e.printStackTrace();
}
return lengthDomain;
}
}

80
src/main/java/com/mesasoft/cn/util/MariaDBUtils.java
View File

@@ -1,80 +0,0 @@
package com.mesasoft.cn.util;
import com.alibaba.druid.pool.DruidDataSourceFactory;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
/**
* Druid连接池的工具类
*/
public class MariaDBUtils {
private static DataSource ds ;
static{
try {
Properties pro = new Properties();
// pro.load(JDBCUtils.class.getClassLoader().getResourceAsStream("druid.properties"));
ds = DruidDataSourceFactory.createDataSource(pro);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 获取连接
*/
public static Connection getConnection() throws SQLException {
return ds.getConnection();
}
/**
* 释放资源
*/
public static void close(Statement stmt,Connection conn){
close(null,stmt,conn);
}
public static void close(ResultSet rs , Statement stmt, Connection conn){
if(rs != null){
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if(stmt != null){
try {
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if(conn != null){
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
/**
* 获取连接池方法
*/
public static DataSource getDataSource(){
return ds;
}
}

84
src/main/java/com/mesasoft/cn/util/MariaDbBase.java
View File

@@ -1,84 +0,0 @@
package com.mesasoft.cn.util;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import org.apache.log4j.Logger;
import java.sql.*;
import java.util.Date;
import java.util.Properties;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/28
* Time: 2:56 PM
* Description: No Description
*/
public class MariaDbBase {
private static final Logger LOG = Logger.getLogger(MariaDbBase.class);
private static final Properties props = new Properties();
private final Statement statement;
public MariaDbBase(Connection conn, Statement stat) {
statement = stat;
}
/**
* 执行写入sql
*/
public void writeSqlExecute(String sql){
try {
statement.executeUpdate(sql);
} catch (SQLIntegrityConstraintViolationException e){
LOG.error("Duplicated entry for key 'PRIMARY'");
} catch (SQLException exception) {
LOG.error("Sql : " + sql);
exception.printStackTrace();
}
}
/**
* 执行查询sql
*/
public ResultSet querySqlExecute(String sql){
ResultSet set = null;
try {
set = statement.executeQuery(sql);
} catch (SQLException exception) {
exception.printStackTrace();
}
return set;
}
/**
* 获得指定表格、按指定时间字段的过期记录
* @param tableName 库表名称
* @param timeColumnName 时间列名
* @return 查询结果
*/
public ResultSet getExpiredRecord(String tableName, String timeColumnName){
Date lastUpdateTime = new Timestamp(getExpiredTime(ApplicationConfig.UPDATE_EXPIRED_DAY).getTime());
String resSql = "SELECT *"
+ " FROM " + ApplicationConfig.DATABASE + "." + tableName
+ " WHERE " + timeColumnName + " < '" + lastUpdateTime + '\'';
LOG.debug("Update task: expired query sql" + resSql);
return querySqlExecute(resSql);
}
/**
* TODO: getUnlabeledRecord() 考虑多个来源的情况
*/
/**
* 获得过期时间, 当前时间的expiredRangeDays天之前的日期为过期日期
*/
public static Date getExpiredTime(int expiredRangeDays){
return new Timestamp(TimeUtils.getStartOfDay(-expiredRangeDays).getTime());
}
}

59
src/main/java/com/mesasoft/cn/util/TimeUtils.java
View File

@@ -1,59 +0,0 @@
package com.mesasoft.cn.util;
import java.util.Calendar;
import java.util.Date;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 11:26 上午
*/
public class TimeUtils {
public static final Long HOUR_TO_MILLISECONDS = 3600000L;
public static final Long DAY_TO_MILLSEDONDS = 86400000L;
public static final Integer SECOND_TO_MILLSEDONDS = 1000;
/**
* 获得当前时间小时的起始（0分钟）时间
*/
public static Date getStartOfHour() {
return getStartOfHour(0);
}
public static Date getStartOfHour(Integer offset) {
Calendar ca = Calendar.getInstance();
ca.add(Calendar.HOUR, offset);
ca.set(Calendar.MINUTE, 0);
ca.set(Calendar.SECOND, 0);
ca.set(Calendar.MILLISECOND, 0);
return ca.getTime();
}
/**
* 获得当前日期的起始（0时）时间
*/
public static Date getStartOfDay() {
return getStartOfDay(0);
}
public static Date getStartOfDay(Integer bias) {
Calendar ca = Calendar.getInstance();
ca.add(Calendar.DATE, bias);
ca.set(Calendar.HOUR, -12);
ca.set(Calendar.MINUTE, 0);
ca.set(Calendar.SECOND, 0);
ca.set(Calendar.MILLISECOND, 0);
return ca.getTime();
}
public static Date getStartOfMonth() {
Calendar ca = Calendar.getInstance();
ca.set(Calendar.DATE, 1);
ca.set(Calendar.HOUR, -12);
ca.set(Calendar.MINUTE, 0);
ca.set(Calendar.SECOND, 0);
ca.set(Calendar.MILLISECOND, 0);
return ca.getTime();
}
}

202
src/main/java/com/mesasoft/cn/util/ValidationUtils.java
View File

@@ -1,202 +0,0 @@
package com.mesasoft.cn.util;
import com.mesasoft.cn.sketch.api.BrightCloud;
import com.mesasoft.cn.sketch.config.ApplicationConfig;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import sun.net.util.IPAddressUtil;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ValidationUtils {
private static final Logger LOG = Logger.getLogger(ValidationUtils.class);
/**
* 获取二级域名
**/
public static String getSecDomain(String fqdnOrUrl){
String filePath = Objects.requireNonNull(BrightCloud.class.getClassLoader()
.getResource(ApplicationConfig.TLD_FILE)).getFile();
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(filePath);
try {
String[] split = fqdnOrUrl.split("\\.");
String secDomain = null;
for (int i = split.length - 1; i >= 0; i--) {
int maps_index = split.length - (i + 1);
HashMap<String, String> innerMap = maps.get("map_id_" + maps_index);
HashMap<String, String> fullTop = maps.get("full");
if (!(innerMap.containsKey(split[i]))) {
String strSec = "";
for (int j = i; j < split.length; j++) {
strSec += (split[j] + ".");
}
secDomain = strSec.substring(0, strSec.length() - 1);
if (fullTop.containsKey(getTopFromSecDomain(secDomain))) {
break;
} else {
while (!fullTop.containsKey(getTopFromSecDomain(secDomain)) && getTopFromSecDomain(secDomain).contains(".")) {
secDomain = getTopFromSecDomain(secDomain);
}
break;
}
}
}
// 右匹配为顶级域名
if (secDomain == null){
secDomain = fqdnOrUrl;
}
return secDomain;
} catch (Exception e) {
LOG.error("urlDomain:" + fqdnOrUrl);
e.printStackTrace();
return "---no---return---";
}
}
public static List<String> getSecDomain(List<String> fqdnOrUrls) {
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(ApplicationConfig.TLD_FILE);
List<String> secDomainList = new ArrayList<>();
for (String oriDomain : fqdnOrUrls) {
String secDomain = getSecDomain(oriDomain);
if (StringUtils.isNotBlank(secDomain) && !("---no---return---".equals(secDomain))) {
secDomainList.add(secDomain);
} else {
System.out.println(oriDomain);
}
}
return secDomainList;
}
public static String getTopFromSecDomain(String secDomain) {
String quFirstDian = secDomain;
if (secDomain.contains(".")) {
quFirstDian = secDomain.substring(secDomain.indexOf(".")).substring(1);
}
return quFirstDian;
}
public static HashMap<String, HashMap<String, String>> readTopDomainFile(String filePath) {
HashMap<String, HashMap<String, String>> maps = makeHashMap(filePath);
try {
String encoding = "UTF-8";
File file = new File(filePath);
if (file.isFile() && file.exists()) {
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null) {
HashMap<String, String> fullTop = maps.get("full");
fullTop.put(lineTxt, lineTxt);
maps.put("full", fullTop);
String[] split = lineTxt.split("\\.");
for (int i = split.length - 1; i >= 0; i--) {
int maps_index = split.length - (i + 1);
HashMap<String, String> innerMap = maps.get("map_id_" + maps_index);
innerMap.put(split[i], split[i]);
maps.put("map_id_" + maps_index, innerMap);
}
}
read.close();
} else {
LOG.error("TopDomainUtils>=>readTopDomainFile filePath is wrong--->{" + filePath + "}<---");
}
} catch (Exception e) {
LOG.error("TopDomainUtils>=>readTopDomainFile get filePathData error--->{" + e + "}<---");
e.printStackTrace();
}
return maps;
}
public static HashMap<String, HashMap<String, String>> makeHashMap(String filePath) {
int maxLength = FileUtils.getMaxLength(filePath);
HashMap<String, HashMap<String, String>> maps = new HashMap<String, HashMap<String, String>>();
for (int i = 0; i < maxLength; i++) {
maps.put("map_id_" + i, new HashMap<String, String>());
}
maps.put("full", new HashMap<String, String>());
return maps;
}
public static List<String> getChecked(List<String> objectList, String type){
if (type.equals("ip")){
return getCheckedIps(objectList);
}
if (type.equals("domain")){
return getCheckedFqdns(objectList);
}
LOG.error("Wrong type to be checked: " + type);
return objectList;
}
public static List<String> getCheckedFqdns(List<String> fqdns){
List<String> res = new ArrayList<>();
for (String fqdn:fqdns){
//去端口号
fqdn = fqdn.split(":")[0];
// 去重 & 校验
if (isValidDomain(fqdn) && !res.contains(fqdn)){
res.add(fqdn.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + fqdn);
}
}
return res;
}
public static List<String> getCheckedIps(List<String> ipList){
List<String> res = new ArrayList<>();
for (String ip:ipList){
//去端口号
ip = ip.split(":")[0];
// 去重 & 校验
if (isValidIp(ip) && !res.contains(ip)){
res.add(ip.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + ip);
}
}
return res;
}
public static boolean isValidIp(String ip){
boolean iPv4LiteralAddress = IPAddressUtil.isIPv4LiteralAddress(ip);
boolean iPv6LiteralAddress = IPAddressUtil.isIPv6LiteralAddress(ip);
return iPv4LiteralAddress || iPv6LiteralAddress;
}
private static boolean isValidDomain(String str)
{
String regex = "^((?!-)[A-Za-z0-9-_]"
+ "{1,63}(?<!-)\\.)"
+ "+[A-Za-z]{2,6}";
Pattern p = Pattern.compile(regex);
if (str == null) {
return false;
}
Matcher m = p.matcher(str);
return m.matches();
}
public static Integer getMatchPattern(String fqdn){
int match_pattern = 2;
if (fqdn.equals(getSecDomain(fqdn))){
match_pattern = 1; // 二级域名-右匹配
}
return match_pattern;
}
}

2
src/main/java/com/mesasoft/cn/web/controller/CommonController.java
View File

@@ -1,10 +1,10 @@
package com.mesasoft.cn.web.controller;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.config.SettingConfig;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.ICommonService;
import com.mesasoft.cn.util.ControllerUtils;
import com.zhazhapan.modules.constant.ValueConsts;

12
src/main/java/com/mesasoft/cn/web/controller/ConfigController.java
View File

@@ -1,10 +1,10 @@
package com.mesasoft.cn.web.controller;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.IConfigService;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.FileExecutor;
@@ -49,11 +49,11 @@ public class ConfigController {
public String updateConfig(String config) {
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
if (user.getPermission() > ValueConsts.TWO_INT) {
SketchApplication.settings.setJsonObject(config);
WebSketchApplication.settings.setJsonObject(config);
//打包成jar之后无法修改config.json文件
try {
FileExecutor.saveFile(NetUtils.urlToString(SketchApplication.class.getResource(DefaultValues
.SETTING_PATH)), SketchApplication.settings.toString());
FileExecutor.saveFile(NetUtils.urlToString(WebSketchApplication.class.getResource(DefaultValues
.SETTING_PATH)), WebSketchApplication.settings.toString());
} catch (IOException e) {
logger.error(e.getMessage());
return "{\"message\":\"internal error, cannot save\"}";
@@ -70,7 +70,7 @@ public class ConfigController {
public String getAll() {
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
if (user.getPermission() > ValueConsts.TWO_INT) {
return SketchApplication.settings.toString();
return WebSketchApplication.settings.toString();
} else {
return "{\"message\":\"permission denied\"}";
}

8
src/main/java/com/mesasoft/cn/web/controller/FileController.java
View File

@@ -2,13 +2,13 @@ package com.mesasoft.cn.web.controller;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.service.IFileService;
import com.mesasoft.cn.util.BeanUtils;
import com.mesasoft.cn.util.ControllerUtils;
import com.zhazhapan.modules.constant.ValueConsts;
import com.zhazhapan.util.Checker;
@@ -90,7 +90,7 @@ public class FileController {
@RequestMapping(value = "/all", method = RequestMethod.GET)
public String getAll(int offset, int categoryId, String orderBy, String search) {
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
boolean canGet = SketchApplication.settings.getBooleanUseEval(ConfigConsts.ANONYMOUS_VISIBLE_OF_SETTING) ||
boolean canGet = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.ANONYMOUS_VISIBLE_OF_SETTING) ||
(Checker.isNotNull(user) && user.getIsVisible() == 1);
if (canGet) {
int userId = Checker.isNull(user) ? 0 : user.getId();

3
src/main/java/com/mesasoft/cn/web/controller/UploadedController.java
View File

@@ -3,7 +3,6 @@ package com.mesasoft.cn.web.controller;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.service.IUploadedService;
import com.zhazhapan.util.Formatter;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
@@ -34,6 +33,6 @@ public class UploadedController {
@AuthInterceptor(InterceptorLevel.ADMIN)
@RequestMapping(value = "all", method = RequestMethod.GET)
public String getAll(String user, String file, String category, int offset) {
return Formatter.listToJson(uploadedService.list(user, file, category, offset));
return com.zhazhapan.util.Formatter.listToJson(uploadedService.list(user, file, category, offset));
}
}

10
src/main/java/com/mesasoft/cn/web/controller/UserController.java
View File

@@ -2,15 +2,15 @@ package com.mesasoft.cn.web.controller;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.annotation.AuthInterceptor;
import com.mesasoft.cn.config.TokenConfig;
import com.mesasoft.cn.entity.Result;
import com.mesasoft.cn.entity.ResultEntity;
import com.mesasoft.cn.entity.User;
import com.mesasoft.cn.enums.InterceptorLevel;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.mesasoft.cn.service.IUserService;
import com.mesasoft.cn.util.ControllerUtils;
import com.zhazhapan.modules.constant.ValueConsts;
@@ -102,7 +102,7 @@ public class UserController {
public String updateBasicInfo(String avatar, String realName, String email, String code) {
User user = (User) request.getSession().getAttribute(ValueConsts.USER_STRING);
jsonObject.put("message", "保存成功");
boolean emilVerify = SketchApplication.settings.getBooleanUseEval(ConfigConsts.EMAIL_VERIFY_OF_SETTINGS);
boolean emilVerify = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.EMAIL_VERIFY_OF_SETTINGS);
if (Checker.isNotEmpty(email) && !email.equals(user.getEmail())) {
if (!emilVerify || isCodeValidate(code)) {
if (userService.emailExists(email)) {
@@ -192,7 +192,7 @@ public class UserController {
@AuthInterceptor(InterceptorLevel.NONE)
@RequestMapping(value = "/register", method = RequestMethod.POST)
public String register(String username, String email, String password, String code) {
boolean emilVerify = SketchApplication.settings.getBooleanUseEval(ConfigConsts.EMAIL_VERIFY_OF_SETTINGS);
boolean emilVerify = WebSketchApplication.settings.getBooleanUseEval(ConfigConsts.EMAIL_VERIFY_OF_SETTINGS);
jsonObject.put("status", "error");
if (!emilVerify || isCodeValidate(code)) {
if (userService.usernameExists(username)) {

91
src/main/resources/application-sketch.properties
View File

@@ -1,85 +1,52 @@
driverClassName=org.mariadb.jdbc.Driver
url=jdbc:mariadb://192.168.44.12:3306/web_sketch_v2
username=root
password=galaxy2019
initialSize=5
maxActive=10
maxWait=1000
#datasource
spring.datasource.url=jdbc:mysql://192.168.44.12:3306/efo?useUnicode=true&characterEncoding=utf-8&useSSL=true
spring.datasource.username=root
spring.datasource.password=galaxy2019
validationQuery=SELECT 1
####################################################
################### application ####################
####################################################
query.types.domain = domain_category, domain_whois
query.types.ip = dns_server
query.output.dir = /home/bigdata/domain/output_file
# 数据库更新定时任务时间间隔1d 1000 * 3600 * 24 = 86400000
update.expired.day = 7
# offline读取批处理量
# offline\u8BFB\u53D6\u6279\u5904\u7406\u91CF
query.readin.batch = 10000
# 打印进度日志的查询条数
# \u6253\u5370\u8FDB\u5EA6\u65E5\u5FD7\u7684\u67E5\u8BE2\u6761\u6570
query.log.file.line.interval = 10000
####################################################
################### mariadb ########################
####################################################
database = web_sketch_v2
tablename.domain.category = domain_category_reputation
tablename.domain.whois = domain_whois
tablename.dns.server =
db.query.batch.size = 10000
####################################################
###################### api #########################
####################################################
##### bright cloud #######
bc.oemid = GeedgeNet
bc.deviceid = TSG-Dev
bc.uid = GN0001
bc.url = https://api.bcti.brightcloud.com/1.0/url/getinfo
bc.method = POST
bright-cloud.oemid = GeedgeNet
bright-cloud.deviceid = TSG-Dev
bright-cloud.uid = GN0001
bright-cloud.url = https://api.bcti.brightcloud.com/1.0/url/getinfo
bright-cloud.method = POST
bc.queryType = getinfo
bc.isa1cat = 1
bc.isReputation = 1
# 返回json格式
bc.isxml = 0
# bc api单次查询url长度限制 API最高限制
bc.maximum.query.num = 100
bc.cateinfo.filepath = categoryinfo.json
bc.usereport.filepath = bright_cloud_query_count.csv
bright-cloud.queryType = getinfo
bright-cloud.isa1cat = 1
bright-cloud.isReputation = 1
#bright-cloud\u8FD4\u56DEjson\u683C\u5F0F
bright-cloud.isxml = 0
# api\u5355\u6B21\u67E5\u8BE2url\u957F\u5EA6\u9650\u5236 API\u6700\u9AD8\u9650\u5236
bright-cloud.maximum-query-num = 100
bright-cloud.cateinfo-filepath = categoryinfo.json
bright-cloud.usereport-filepath = bright_cloud_query_count.csv
######### chinaz #########
chinaz.url.single = https://apidatav2.chinaz.com/single/whois
chinaz.url.batch = https://apidatav2.chinaz.com/batch/whois
chinaz.url-single = https://apidatav2.chinaz.com/single/whois
chinaz.url-batch = https://apidatav2.chinaz.com/batch/whois
chinaz.key = ffc9be4141bd49a093ed0185a54dc6a2
chinaz.maximum.query.num = 50
chinaz.usereport.filepath = chinaz_query_count.csv
chinaz.maximum-query-num = 50
chinaz.usereport-filepath = chinaz_query_count.csv
####################################################
###################### 其他 #########################
####################################################
tld.file = public_suffix_list_only.dat
###################### \u5176\u4ED6 #########################
#\u9876\u7EA7\u57DF\u540D
sketch.tld.file = public_suffix_list_only.dat
sketch.path.admin=C:\\test
sketch.path.user=C:\\test\\1\\
sketch.datasource.druid.filters=stat
sketch.datasource.druid.driverClassName=com.mysql.jdbc.Driver
sketch.datasource.druid.url=jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&autoReconnect=true&useSSL=false
sketch.datasource.druid.username=root
sketch.datasource.druid.password=12311
sketch.datasource.druid.initialSize=1
sketch.datasource.druid.minIdle=1
sketch.datasource.druid.maxActive=10
sketch.datasource.druid.maxWait=60000
sketch.datasource.druid.timeBetweenEvictionRunsMillis=60000
sketch.datasource.druid.minEvictableIdleTimeMillis=300000
sketch.datasource.druid.validationQuery=SELECT 'x'
sketch.datasource.druid.testWhileIdle=true
sketch.datasource.druid.testOnBorrow=false
sketch.datasource.druid.testOnReturn=false
sketch.datasource.druid.poolPreparedStatements=false
sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize=10

49
src/main/resources/application.properties
View File

@@ -1,14 +1,43 @@
server.port=8196
mybatis.type-aliases-package=com.mesasoft.cn.entity
spring.profiles.include=sketch
mybatis.type-aliases-package=com.mesasoft.cn.entity,com.mesasoft.cn.sketch.entity
mybatis.mapper-locations=classpath:mapper/*.xml
spring.datasource.name=druidDataSource
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driverClassName=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://192.168.44.12:3306/efo?useUnicode=true&characterEncoding=utf-8&useSSL=true
spring.datasource.username=root
spring.datasource.password=galaxy2019
spring.datasource.druid.filters=stat,wall,slf4j,config
spring.datasource.druid.maxActive=100
spring.datasource.druid.initialSize=1
spring.datasource.druid.maxWait=60000
spring.datasource.druid.minIdle=1
spring.datasource.druid.timeBetweenEvictionRunsMillis=60000
spring.datasource.druid.minEvictableIdleTimeMillis=300000
spring.datasource.druid.validationQuery=select 'x'
spring.datasource.druid.testWhileIdle=true
spring.datasource.druid.testOnBorrow=false
spring.datasource.druid.testOnReturn=false
spring.datasource.druid.poolPreparedStatements=true
spring.datasource.druid.maxOpenPreparedStatements=50
spring.datasource.druid.maxPoolPreparedStatementPerConnectionSize=20
spring.datasource.druid.web-stat-filter.enabled=true
spring.datasource.druid.stat-view-servlet.enabled=true
spring.datasource.druid.stat-view-servlet.login-username=test
spring.datasource.druid.stat-view-servlet.login-password=test
spring.datasource.druid.stat-view-servlet.allow=127.0.0.1
spring.datasource.druid.stat-view-servlet.deny=
spring.jpa.show-sql=true
spring.servlet.multipart.max-file-size=1099511627776
spring.servlet.multipart.max-request-size=1099511627776
spring.devtools.restart.enabled=false
#\u70ED\u90E8\u7F72\u751F\u6548
spring.devtools.restart.enabled=true
#\u9875\u9762\u70ED\u90E8\u7F72
spring.thymeleaf.cache=false
spring.devtools.restart.additional-paths=src/main
#classpath\u76EE\u5F55\u4E0B\u7684WEB-INF\u6587\u4EF6\u5939\u5185\u5BB9\u4FEE\u6539\u4E0D\u91CD\u542F
spring.devtools.restart.exclude=WEB-INF/**
#spring.profiles.active=prod
#spring.mvc.favicon.enabled=false
@@ -20,9 +49,9 @@ swagger.version=1.2
swagger.license=MIT
swagger.licenseUrl=https://opensource.org/licenses/MIT
swagger.termsOfServiceUrl=https://github.com/code4everything/efo
swagger.contact.name=zhazhapan
swagger.contact.url=https://github.zhazhapan.com
swagger.contact.email=zhazhapan.com
swagger.contact.name=e
swagger.contact.url=https://github.e.com
swagger.contact.email=.com
swagger.base-package=com.mesasoft.cn
swagger.base-path=/**
swagger.exclude-path=/error, /ops/**
@@ -36,7 +65,3 @@ swagger.global-response-message.get[1].modelRef=ERROR
swagger.global-response-message.post[0].code=500
swagger.global-response-message.post[0].message=500post
swagger.global-response-message.post[0].modelRef=ERROR
sketch.path.admin=C:\\test
sketch.path.user=C:\\test\\1\\

4
src/main/resources/config.json
View File

@@ -77,8 +77,8 @@
"emailConfig": {
"host": "",
"port": 587,
"personal": "efo",
"from": "efo-admin@qq.com",
"personal": "webskt",
"from": "webskt-admin@qq.com",
"key": "123456",
"ssl": false
},

85
src/main/resources/sketch.properties
View File

@@ -1,85 +0,0 @@
driverClassName=org.mariadb.jdbc.Driver
url=jdbc:mariadb://192.168.44.12:3306/web_sketch_v2
username=root
password=galaxy2019
initialSize=5
maxActive=10
maxWait=1000
validationQuery=SELECT 1
####################################################
################### application ####################
####################################################
query.types.domain = domain_category, domain_whois
query.types.ip = dns_server
query.output.dir = /home/bigdata/domain/output_file
# 数据库更新定时任务时间间隔1d 1000 * 3600 * 24 = 86400000
update.expired.day = 7
# offline读取批处理量
query.readin.batch = 10000
# 打印进度日志的查询条数
query.log.file.line.interval = 10000
####################################################
################### mariadb ########################
####################################################
database = web_sketch_v2
tablename.domain.category = domain_category_reputation
tablename.domain.whois = domain_whois
tablename.dns.server =
db.query.batch.size = 10000
####################################################
###################### api #########################
####################################################
##### bright cloud #######
bc.oemid = GeedgeNet
bc.deviceid = TSG-Dev
bc.uid = GN0001
bc.url = https://api.bcti.brightcloud.com/1.0/url/getinfo
bc.method = POST
bc.queryType = getinfo
bc.isa1cat = 1
bc.isReputation = 1
# 返回json格式
bc.isxml = 0
# bc api单次查询url长度限制 API最高限制
bc.maximum.query.num = 100
bc.cateinfo.filepath = categoryinfo.json
bc.usereport.filepath = bright_cloud_query_count.csv
######### chinaz #########
chinaz.url.single = https://apidatav2.chinaz.com/single/whois
chinaz.url.batch = https://apidatav2.chinaz.com/batch/whois
chinaz.key = ffc9be4141bd49a093ed0185a54dc6a2
chinaz.maximum.query.num = 50
chinaz.usereport.filepath = chinaz_query_count.csv
####################################################
###################### 其他 #########################
####################################################
tld.file = public_suffix_list_only.dat
sketch.datasource.druid.filters=stat
sketch.datasource.druid.driverClassName=com.mysql.jdbc.Driver
sketch.datasource.druid.url=jdbc:mysql://127.0.0.1:3306/test?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&autoReconnect=true&useSSL=false
sketch.datasource.druid.username=root
sketch.datasource.druid.password=12311
sketch.datasource.druid.initialSize=1
sketch.datasource.druid.minIdle=1
sketch.datasource.druid.maxActive=10
sketch.datasource.druid.maxWait=60000
sketch.datasource.druid.timeBetweenEvictionRunsMillis=60000
sketch.datasource.druid.minEvictableIdleTimeMillis=300000
sketch.datasource.druid.validationQuery=SELECT 'x'
sketch.datasource.druid.testWhileIdle=true
sketch.datasource.druid.testOnBorrow=false
sketch.datasource.druid.testOnReturn=false
sketch.datasource.druid.poolPreparedStatements=false
sketch.datasource.druid.maxPoolPreparedStatementPerConnectionSize=10

358
src/main/resources/style.scss
View File

@@ -1,358 +0,0 @@
@import "assets/sketch/css/font";
// -------- Variables ----------
$fontLight: 300;
$fontBold: 600;
$desktop: 1024px;
$tablet: 768px;
$mobile: 576px;
$primary-color: #03A2DC;
$secondary-color: #fff6bb;
$light-color: #ffffff;
$dark-color: #555555;
$alt-bg-color: #f9f9f9;
$slider-bg-color: #ccc;
$tableHoverColor: #f9f9f9;
// --------Mixins ------------
@mixin mQ($size) {
@media (max-width: $size) {
@content;
}
}
// -------- Normalize ----------
* {
margin: 0;
padding: 0;
-webkit-box-sizing: border-box;
-moz-box-sizing: border-box;
box-sizing: border-box;
}
ul {
margin: 0;
}
button,
input[type='button'] {
cursor: pointer;
}
button:focus,
input:focus,
textarea:focus {
outline: none;
}
input, textarea {
border: none;
}
button {
border: none;
background: none;
}
img {
max-width: 100%;
height: auto;
}
p {
margin: 0;
}
.align_item_center {
align-items: center;
}
a,
a:hover,
a:active,
a:visited {
text-decoration: none;
}
// ------- Typo ------
body {
font-family: "Poppins", arial, sans-serif;
font-weight: 300;
line-height: 1.625;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
color: $dark-color;
}
h1,
h2,
h3,
h4,
h5,
h6 {
color: $dark-color;
font-weight: 600;
margin: 0;
line-height: 1.2;
}
h1 {
font-size: 36px;
}
h2 {
font-size: 30px;
}
h3 {
font-size: 24px;
}
h4 {
font-size: 18px;
}
h5 {
font-size: 15px;
}
h6 {
font-size: 13px;
}
// ------- Styling ------
.body_bg {
width: 100%;
height: 100%;
position: absolute;
z-index: -1;
background-color: $alt-bg-color;
clip-path: polygon(100% 100%, 0% 100%, 100% 0);
}
.conn {
padding: 20px 0;
}
// ----- header ----
#header {
margin: 0px 0;
margin-left: 20px;
padding-bottom: 20px;
border-bottom: 5px solid $alt-bg-color;
@include mQ($tablet) {
justify-content: center;
}
@include mQ($mobile) {
margin-left: 10px;
flex-direction: column;
}
// -- logo P --
.logo_p {
//background-color: $primary-color;
display: flex;
justify-content: center;
align-items: center;
img{
width: 65px;
}
}
// --- Heading Title --
.heading_title_p {
margin-left: 20px;
@include mQ($mobile) {
margin-left: 0;
margin-top: 20px;
}
h2 {
font-size: 22px;
@include mQ($mobile) {
text-align: center;
}
}
}
}
// --- Upload
.inputUploadP {
margin: 5px 0px 0px 20px;
@include mQ($mobile) {
margin: 20px 10px;
}
.uploadForm {
display: flex;
align-items: center;
.uploadFile {
width: 0.1px;
height: 0.1px;
opacity: 0;
overflow: hidden;
position: absolute;
z-index: -1;
}
svg{
fill: $light-color;
}
.uploadFile + label {
font-size: 1em;
font-weight: 700;
color: $light-color;
background-color: $primary-color;
display: inline-block;
transition: .2s ease;
padding: 8px 20px;
border-radius: 3px;
cursor: pointer;
}
.uploadFile:focus + label,
.uploadFile + label:hover {
background-color: $dark-color;
}
.uploadFile:focus + label {
outline: 1px dotted $dark-color;
outline: -webkit-focus-ring-color auto 5px;
}
.uploadFile + label * {
pointer-events: none;
}
.js .uploadFile {
width: 0.1px;
height: 0.1px;
opacity: 0;
overflow: hidden;
position: absolute;
z-index: -1;
}
.no-js .uploadFile + label {
display: none;
}
.uploadBtn_P {
margin-top: -7px;
margin-left: 10px;
.uploadBtn {
transition: .2s ease;
&:hover {
background-color: $secondary-color;
color: var(--dark-color);
border: 1px solid $dark-color;
}
i {
padding-right: 10px;
}
}
}
}
}
// ---- BackBtn P ---
.backBtn_p {
margin: 10px 0px 0px 20px;
@include mQ($mobile) {
margin: 20px 10px;
}
a {
font-size: 1.3em;
display: flex;
align-items: center;
transition: .2s ease;
width: 150px;
color: $primary-color;
&:hover {
color: $dark-color;
i {
color: $dark-color;
}
}
i {
padding-right: 10px;
font-size: 1.5em;
}
}
}
// -- Table - P
.table_p {
//margin: 20px;
//margin-top: 0;
//width: calc(100% - 40px);
.table-hover tbody tr:hover td, .table-hover tbody tr:hover th {
background-color: $tableHoverColor;
}
// th {
// @include mQ($mobile) {
// font-size: .9em;
// }
// }
td {
font-size: .95em;
vertical-align: middle;
&:nth-child(1) {
text-align: center;
}
}
.file_ic, .folder_ic {
color: $primary-color;
font-size: 1.5em;
transition: .2s ease;
&:hover {
color: #0056b3;
}
}
@include mQ($mobile) {
margin: 20px 10px;
width: calc(100% - 20px);
}
}
#tableData > tbody > tr > td {
padding: 0;
height: 20px;
}
.dataTables_filter input {
border: 1px solid $dark-color;
padding: 3px 20px;
}
// -- Footer --
footer {
text-align: center;
font-size: .8em;
font-weight: bold;
margin: 30px 0;
margin-bottom: 0;
}

27
src/test/java/com/mesasoft/cn/SketchApplicationTest.java
View File

@@ -1,27 +0,0 @@
package com.mesasoft.cn;
import com.zhazhapan.config.JsonParser;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.IOException;
@RunWith(SpringRunner.class)
@SpringBootTest
public class SketchApplicationTest {
public static void setSettings() {
try {
SketchApplication.settings = new JsonParser(SketchApplicationTest.class.getResource("/config.json"));
} catch (IOException e) {
e.printStackTrace();
}
}
@Test
public void contextLoads() {
}
}

8
src/test/java/com/mesasoft/cn/common/CommonTest.java
View File

@@ -1,7 +1,7 @@
package com.mesasoft.cn.common;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.zhazhapan.util.FileExecutor;
import com.zhazhapan.util.Formatter;
@@ -19,8 +19,8 @@ public class CommonTest {
@Test
public void testSendEmail() throws Exception {
SketchApplicationTest.setSettings();
MailSender.config(SketchApplication.settings.getObjectUseEval(ConfigConsts.EMAIL_CONFIG_OF_SETTINGS));
WebSketchApplicationTest.setSettings();
MailSender.config(WebSketchApplication.settings.getObjectUseEval(ConfigConsts.EMAIL_CONFIG_OF_SETTINGS));
MailSender.sendMail("tao@zhazhapan.com", "test", "test");
}

53
src/test/java/com/mesasoft/cn/common/SketchTest.java
View File

File diff suppressed because one or more lines are too long

10
src/test/java/com/mesasoft/cn/config/SettingConfigTest.java
View File

@@ -1,7 +1,7 @@
package com.mesasoft.cn.config;
import com.mesasoft.cn.SketchApplication;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplication;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import org.junit.Test;
@@ -15,13 +15,13 @@ public class SettingConfigTest {
@Test
public void testFileSuffixPattern() {
SketchApplicationTest.setSettings();
assert Pattern.compile(SketchApplication.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher("jpg").matches();
WebSketchApplicationTest.setSettings();
assert Pattern.compile(WebSketchApplication.settings.getStringUseEval(ConfigConsts.FILE_SUFFIX_MATCH_OF_SETTING)).matcher("jpg").matches();
}
@Test
public void testGetStoragePath() {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
System.out.println(SettingConfig.getStoragePath(ConfigConsts.TOKEN_OF_SETTINGS));
}
}

4
src/test/java/com/mesasoft/cn/dao/AuthDAOTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.zhazhapan.util.Formatter;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -17,7 +17,7 @@ import org.springframework.test.context.junit4.SpringRunner;
public class AuthDAOTest {
static {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
}
@Autowired

4
src/test/java/com/mesasoft/cn/dao/DownloadedDAOTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.zhazhapan.modules.constant.ValueConsts;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -17,7 +17,7 @@ import org.springframework.test.context.junit4.SpringRunner;
public class DownloadedDAOTest {
static {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
}
@Autowired

4
src/test/java/com/mesasoft/cn/dao/FileDAOTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.zhazhapan.util.Formatter;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -26,7 +26,7 @@ public class FileDAOTest {
@Test
public void testGetUserDownloaded() {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
System.out.println(Formatter.listToJson(fileDAO.listUserDownloaded(2, 0, "")));
}
}

4
src/test/java/com/mesasoft/cn/dao/UserDAOTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import com.mesasoft.cn.entity.User;
import com.zhazhapan.util.Checker;
import com.zhazhapan.util.Formatter;
@@ -20,7 +20,7 @@ import org.springframework.test.context.junit4.SpringRunner;
public class UserDAOTest {
static {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
}
@Autowired

4
src/test/java/com/mesasoft/cn/dao/sqlprovider/FileSqlProviderTest.java
View File

@@ -1,6 +1,6 @@
package com.mesasoft.cn.dao.sqlprovider;
import com.mesasoft.cn.SketchApplicationTest;
import com.mesasoft.cn.WebSketchApplicationTest;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
@@ -25,7 +25,7 @@ public class FileSqlProviderTest {
@Test
public void getAll() {
SketchApplicationTest.setSettings();
WebSketchApplicationTest.setSettings();
System.out.println(fileSqlProvider.getAll(0, 0, "", ""));
}