gfwleak/web-sketch-webskt-query-agent
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

web sketch-代码优化域名查询 CN-663

Browse Source
This commit is contained in:
zhanghongqing
2023-01-10 16:02:56 +08:00
parent c125bb89cf
commit bdf48f2b4f
29 changed files with 3815 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/main/java/com/mesasoft/cn/WebSketchApplication.java Normal file
View File

@@ -0,0 +1,42 @@
package com.mesasoft.cn;
import com.mesasoft.cn.config.TokenConfig;
import com.mesasoft.cn.modules.constant.ConfigConsts;
import com.mesasoft.cn.modules.constant.DefaultValues;
import com.spring4all.swagger.EnableSwagger2Doc;
import com.zhazhapan.config.JsonParser;
import com.zhazhapan.util.FileExecutor;
import com.zhazhapan.util.MailSender;
import com.zhazhapan.util.ReflectUtils;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import java.io.IOException;
import java.util.Hashtable;
import java.util.List;
/**
* @author pantao
*/
@SpringBootApplication
@EnableSwagger2Doc
@MapperScan({"com.mesasoft.cn.dao","com.mesasoft.cn.sketch.dao"})
@EnableTransactionManagement
public class WebSketchApplication {
public static JsonParser settings = new JsonParser();
public static List<Class<?>> controllers;
public static Hashtable<String, Integer> tokens;
public static void main(String[] args) throws IOException, ClassNotFoundException {
settings.setJsonObject(FileExecutor.read(WebSketchApplication.class.getResourceAsStream(DefaultValues.SETTING_PATH)));
MailSender.config(settings.getObjectUseEval(ConfigConsts.EMAIL_CONFIG_OF_SETTINGS));
controllers = ReflectUtils.getClasses(DefaultValues.CONTROLLER_PACKAGE);
tokens = TokenConfig.loadToken();
SpringApplication.run(WebSketchApplication.class, args);
}
}

22
src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java Normal file
View File

@@ -0,0 +1,22 @@
package com.mesasoft.cn.sketch.config;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Data
@Component
public class AppConfig {
// Mariadb
public static final String DATABASE = ConfigUtils.getStringProperty("database");
public static final String DOMAIN_CATE_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.category");
public static final String DOMAIN_WHOIS_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.whois");
@Value("${db.query.batch.size}")
private Integer dbQueryBatchSize;
}

26
src/main/java/com/mesasoft/cn/sketch/config/BrightCloudConfig.java Normal file
View File

@@ -0,0 +1,26 @@
package com.mesasoft.cn.sketch.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Data
@Component
@ConfigurationProperties(prefix = "bright-cloud")
public class BrightCloudConfig {
private String oemid;
private String deviceid;
private String uid;
private String url;
private String method;
private String queryType;
private String isa1cat;
private String isReputation;
private String isxml;
private int maximumQueryNum;
private String cateinfoFilepath;
private String usereportFilePath;
}

17
src/main/java/com/mesasoft/cn/sketch/config/ChinazConfig.java Normal file
View File

@@ -0,0 +1,17 @@
package com.mesasoft.cn.sketch.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Data
@Component
@ConfigurationProperties(prefix = "chinaz")
public class ChinazConfig {
private String urlSingle;
private String urlBatch;
private String key;
private int maximumQueryNum;
private String usereportFilePath;
}

44
src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java Normal file
View File

@@ -0,0 +1,44 @@
package com.mesasoft.cn.sketch.controller;
import com.alibaba.fastjson.JSONArray;
import com.mesasoft.cn.sketch.entity.ReturnT;
import com.mesasoft.cn.sketch.entity.SketchAuth;
import com.mesasoft.cn.sketch.service.DomainService;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.util.List;
@RestController
@RequestMapping("/v1/domain/")
public class DomainController {
@Resource
private DomainService domainService;
@GetMapping(value = "info")
@ResponseBody
public ReturnT info(SketchAuth sketchAuth) {
return ReturnT.succeed(sketchAuth.getRequestid(), "Hello WebSketch !");
}
@GetMapping(value = "getcategoryinfo")
@ResponseBody
public ReturnT getCategoryInfo(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
JSONArray categoryInfo = domainService.getCategoryInfo(domain, "", true);
return ReturnT.succeed(sketchAuth.getRequestid(), categoryInfo);
}
@GetMapping(value = "getwhoisinfo")
@ResponseBody
public ReturnT getWhoisInfo(SketchAuth sketchAuth, @RequestParam List<String> domain) throws Exception {
JSONArray whoisInfo = domainService.getWhoisInfo(domain, true);
return ReturnT.succeed(sketchAuth.getRequestid(), whoisInfo);
}
}

340
src/main/java/com/mesasoft/cn/sketch/dao/DomainCategorySchema.java Normal file
View File

@@ -0,0 +1,340 @@
package com.mesasoft.cn.sketch.dao;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
@Deprecated
public class DomainCategorySchema {
private static final String dataBase = AppConfig.DATABASE;
private static final String tableName = AppConfig.DOMAIN_CATE_TABLENAME;
private String fqdn;
private String source;
private Boolean query_success;
private Integer match_pattern;
private Integer reputation_score;
private String reputation_level;
private Integer category_id;
private String category_name;
private String category_group;
private Integer category_conf;
private Boolean is_a1_cat;
private Integer status_code = 0;
private String submit_user ="";
// category schema
public DomainCategorySchema(String fqdn,
String source,
Boolean query_success,
Integer match_pattern,
Integer reputation_score,
String reputationLevel,
Integer categoryId,
String categoryName,
String categoryGroup,
Integer categoryConf,
Boolean isA1Cat,Integer statusCode, String submitUser) {
this.fqdn = fqdn;
this.source = source; // 默认应为为brightcloud
this.query_success = query_success;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.reputation_score = reputation_score;
this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel);
this.category_id = categoryId;
this.category_name = ConfigUtils.getEffectiveString(categoryName);
this.category_group = ConfigUtils.getEffectiveString(categoryGroup);
this.category_conf = categoryConf;
this.is_a1_cat = isA1Cat;
this.status_code = statusCode;
this.submit_user = submitUser;
}
public DomainCategorySchema(String fqdn,
String source,
Boolean query_success,
Integer match_pattern,
Integer reputation_score,
String reputationLevel,
Integer categoryId,
String categoryName,
String categoryGroup,
Integer categoryConf,
Boolean isA1Cat) {
this.fqdn = fqdn;
this.source = source; // 默认应为为brightcloud
this.query_success = query_success;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.reputation_score = reputation_score;
this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel);
this.category_id = categoryId;
this.category_name = ConfigUtils.getEffectiveString(categoryName);
this.category_group = ConfigUtils.getEffectiveString(categoryGroup);
this.category_conf = categoryConf;
this.is_a1_cat = isA1Cat;
}
public static void insertRecords(List<DomainCategorySchema> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainCategorySchema categoryFile : categoryFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + categoryFile.getKeys() + ") values" +
'(' + categoryFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public void updateRecords(List<DomainCategorySchema> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainCategorySchema categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainCategorySchema> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") and source = '" + source + "'";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}
public static List<DomainCategorySchema> rs2schema(ResultSet rs) throws SQLException {
List<DomainCategorySchema> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainCategorySchema(
rs.getString("fqdn"),
rs.getString("source"),
rs.getBoolean("query_success"),
rs.getInt("match_pattern"),
rs.getInt("reputation_score"),
rs.getString("reputation_level"),
rs.getInt("category_id"),
rs.getString("category_name"),
rs.getString("category_group"),
rs.getInt("category_conf"),
rs.getBoolean("is_a1_cat"),
rs.getInt("status_code"),
rs.getString("submit_user")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainCategorySchema schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("reputation_score", schema.getReputation_score());
jsonObject.put("reputation_level", schema.getReputation_level());
jsonObject.put("category_id", schema.getCategory_id());
jsonObject.put("category_group", schema.getCategory_group());
jsonObject.put("category_name", schema.getCategory_name());
jsonObject.put("category_conf", schema.getCategory_conf());
jsonObject.put("is_a1_cat", schema.getIs_a1_cat());
jsonObject.put("status_code", schema.getStatus_code());
jsonObject.put("submit_user", schema.getSubmit_user());
return jsonObject;
}
public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", " + reputation_score +
", '" + reputation_level + '\'' +
", " + category_id +
", '" + category_name + '\'' +
", '" + category_group + '\'' +
", " + category_conf +
", " + status_code +
", " + submit_user +
", " + is_a1_cat;
return resString.replace("'null'", "null");
}
public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", reputation_score" +
", reputation_level" +
", category_id" +
", category_name" +
", category_group" +
", category_conf" +
", status_code" +
", submit_user" +
", is_a1_cat";
return resString;
}
public String getKeyValues() {
String resString = "source='" + source + '\'' +
", query_success=" + query_success +
", match_pattern=" + match_pattern +
", reputation_score=" + reputation_score +
", reputation_level='" + reputation_level + '\'' +
", category_id=" + category_id +
", category_name='" + category_name + '\'' +
", category_group='" + category_group + '\'' +
", category_conf=" + category_conf +
", is_a1_cat=" + is_a1_cat;
return resString.replace("'null'", "null");
}
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public Boolean getQuery_success() {
return query_success;
}
public void setQuery_success(Boolean query_success) {
this.query_success = query_success;
}
public Integer getMatch_pattern() {
return match_pattern;
}
public void setMatch_pattern(Integer match_pattern) {
this.match_pattern = match_pattern;
}
public Integer getReputation_score() {
return reputation_score;
}
public void setReputation_score(Integer reputation_score) {
this.reputation_score = reputation_score;
}
public String getReputation_level() {
return reputation_level;
}
public void setReputation_level(String reputation_level) {
this.reputation_level = reputation_level;
}
public Integer getCategory_id() {
return category_id;
}
public void setCategory_id(Integer category_id) {
this.category_id = category_id;
}
public String getCategory_name() {
return category_name;
}
public void setCategory_name(String category_name) {
this.category_name = category_name;
}
public String getCategory_group() {
return category_group;
}
public void setCategory_group(String category_group) {
this.category_group = category_group;
}
public Integer getCategory_conf() {
return category_conf;
}
public void setCategory_conf(Integer category_conf) {
this.category_conf = category_conf;
}
public Boolean getIs_a1_cat() {
return is_a1_cat;
}
public void setIs_a1_cat(Boolean is_a1_cat) {
this.is_a1_cat = is_a1_cat;
}
public Integer getStatus_code() {
return status_code;
}
public void setStatus_code(Integer status_code) {
this.status_code = status_code;
}
public String getSubmit_user() {
return submit_user;
}
public void setSubmit_user(String submit_user) {
this.submit_user = submit_user;
}
}

22
src/main/java/com/mesasoft/cn/sketch/dao/DomainDao.java Normal file
View File

@@ -0,0 +1,22 @@
package com.mesasoft.cn.sketch.dao;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import com.mesasoft.cn.sketch.entity.domain.DomainWhois;
import org.apache.ibatis.annotations.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface DomainDao {
List<DomainInfo> getCategoryInfo(@Param("domain") List<String> domain);
List<DomainInfo> getWhoisInfo(@Param("domain") List<String> domain);
void insertCategory(DomainCategory categories);
void insertWhois(DomainWhois data);
}

423
src/main/java/com/mesasoft/cn/sketch/dao/DomainWhoisSchema.java Normal file
View File

@@ -0,0 +1,423 @@
package com.mesasoft.cn.sketch.dao;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
@Deprecated
public class DomainWhoisSchema {
private static final String dataBase = AppConfig.DATABASE;
private static final String tableName = AppConfig.DOMAIN_WHOIS_TABLENAME;
private String fqdn;
private String source;
private Boolean query_success;
private Integer match_pattern;
private String whois_domain;
private Timestamp whois_update_date;
private Timestamp whois_create_date;
private Timestamp whois_expire_date;
private String whois_email;
private String whois_ns;
private String whois_registrar_name;
private String whois_registrant_org;
private String whois_registrant_name;
private String whois_registrant_street;
private String whois_registrant_city;
private String whois_registrant_state;
private String whois_registrant_postcode;
private String whois_registrant_country;
private String whois_registrant_phone;
// category schema
public DomainWhoisSchema(String fqdn,
String source,
Integer match_pattern,
Boolean query_success,
String whoisDomain,
Date whoisUpdateDate,
Date whoisCreateDate,
Date whoisExpireDate,
String whoisEmail,
String whoisNs,
String whoisRegistrarName,
String whoisRegistrantOrg,
String whoisRegistrantName,
String whoisRegistrantStreet,
String whoisRegistrantCity,
String whoisRegistrantState,
String whoisRegistrantPostcode,
String whoisRegistrantCountry,
String whoisRegistrantPhone
) {
this.fqdn = fqdn;
this.source = source;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.query_success = query_success;
this.whois_domain = ConfigUtils.getEffectiveString(whoisDomain);
this.whois_update_date = whoisUpdateDate == null ? null : new Timestamp(whoisUpdateDate.getTime());
this.whois_create_date = whoisCreateDate == null ? null : new Timestamp(whoisCreateDate.getTime());
this.whois_expire_date = whoisExpireDate == null ? null : new Timestamp(whoisExpireDate.getTime());
this.whois_email = ConfigUtils.getEffectiveString(whoisEmail);
this.whois_ns = ConfigUtils.getEffectiveString(whoisNs);
this.whois_registrar_name = ConfigUtils.getEffectiveString(whoisRegistrarName);
this.whois_registrant_org = ConfigUtils.getEffectiveString(whoisRegistrantOrg);
this.whois_registrant_name = ConfigUtils.getEffectiveString(whoisRegistrantName);
this.whois_registrant_street = ConfigUtils.getEffectiveString(whoisRegistrantStreet);
this.whois_registrant_city = ConfigUtils.getEffectiveString(whoisRegistrantCity);
this.whois_registrant_state = ConfigUtils.getEffectiveString(whoisRegistrantState);
this.whois_registrant_postcode = ConfigUtils.getEffectiveString(whoisRegistrantPostcode);
this.whois_registrant_country = ConfigUtils.getEffectiveString(whoisRegistrantCountry);
this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone);
}
public static void insertRecords(List<DomainWhoisSchema> whoisFiles, MariaDbBase mariaDbBase) {
for (DomainWhoisSchema whoisFile : whoisFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static String insertSql(List<DomainWhoisSchema> whoisFiles) {
DomainWhoisSchema whoisFile = whoisFiles.get(0);
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
return resSql;
}
public static void updateRecords(List<DomainWhoisSchema> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainWhoisSchema categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainWhoisSchema> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") ";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}
public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", '" + whois_domain + '\'' +
", '" + whois_update_date + '\'' +
", '" + whois_create_date + '\'' +
", '" + whois_expire_date + '\'' +
", '" + whois_email + '\'' +
", '" + whois_ns + '\'' +
", '" + whois_registrar_name + '\'' +
", '" + whois_registrant_org + '\'' +
", '" + whois_registrant_name + '\'' +
", '" + whois_registrant_street + '\'' +
", '" + whois_registrant_city + '\'' +
", '" + whois_registrant_state + '\'' +
", '" + whois_registrant_postcode + '\'' +
", '" + whois_registrant_country + '\'' +
", '" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public static List<DomainWhoisSchema> rs2schema(ResultSet rs) throws SQLException {
List<DomainWhoisSchema> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainWhoisSchema(
rs.getString("fqdn"),
rs.getString("source"),
rs.getInt("match_pattern"),
rs.getBoolean("query_success"),
rs.getString("whois_domain"),
(Date) rs.getDate("whois_update_date"),
(Date) rs.getDate("whois_create_date"),
(Date) rs.getDate("whois_expire_date"),
rs.getString("whois_email"),
rs.getString("whois_ns"),
rs.getString("whois_registrar_name"),
rs.getString("whois_registrant_org"),
rs.getString("whois_registrant_name"),
rs.getString("whois_registrant_street"),
rs.getString("whois_registrant_city"),
rs.getString("whois_registrant_state"),
rs.getString("whois_registrant_postcode"),
rs.getString("whois_registrant_country"),
rs.getString("whois_registrant_phone")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainWhoisSchema schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("whois_domain", schema.getWhois_domain());
jsonObject.put("whois_update_date", schema.getWhois_update_date());
jsonObject.put("whois_create_date", schema.getWhois_create_date());
jsonObject.put("whois_expire_date", schema.getWhois_expire_date());
jsonObject.put("whois_email", schema.getWhois_email());
jsonObject.put("whois_ns", schema.getWhois_ns());
jsonObject.put("whois_registrar_name", schema.getWhois_registrar_name());
jsonObject.put("whois_registrant_org", schema.getWhois_registrant_org());
jsonObject.put("whois_registrant_name", schema.getWhois_registrant_name());
jsonObject.put("whois_registrant_street", schema.getWhois_registrant_street());
jsonObject.put("whois_registrant_city", schema.getWhois_registrant_city());
jsonObject.put("whois_registrant_state", schema.getWhois_registrant_state());
jsonObject.put("whois_registrant_postcode", schema.getWhois_registrant_postcode());
jsonObject.put("whois_registrant_country", schema.getWhois_registrant_country());
jsonObject.put("whois_registrant_phone", schema.getWhois_registrant_phone());
return jsonObject;
}
public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", whois_domain" +
", whois_update_date" +
", whois_create_date" +
", whois_expire_date" +
", whois_email" +
", whois_ns" +
", whois_registrar_name" +
", whois_registrant_org" +
", whois_registrant_name" +
", whois_registrant_street" +
", whois_registrant_city" +
", whois_registrant_state" +
", whois_registrant_postcode" +
", whois_registrant_country" +
", whois_registrant_phone";
return resString;
}
public String getKeyValues() {
String resString = "query_success=" + query_success +
", source='" + source + '\'' +
", match_pattern=" + match_pattern +
", whois_domain='" + whois_domain + '\'' +
", whois_update_date='" + whois_update_date + '\'' +
", whois_create_date='" + whois_create_date + '\'' +
", whois_expire_date='" + whois_expire_date + '\'' +
", whois_email='" + whois_email + '\'' +
", whois_ns='" + whois_ns + '\'' +
", whois_registrar_name='" + whois_registrar_name + '\'' +
", whois_registrant_org='" + whois_registrant_org + '\'' +
", whois_registrant_name='" + whois_registrant_name + '\'' +
", whois_registrant_street='" + whois_registrant_street + '\'' +
", whois_registrant_city='" + whois_registrant_city + '\'' +
", whois_registrant_state='" + whois_registrant_state + '\'' +
", whois_registrant_postcode='" + whois_registrant_postcode + '\'' +
", whois_registrant_country='" + whois_registrant_country + '\'' +
", whois_registrant_phone='" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
public Boolean getQuery_success() {
return query_success;
}
public void setQuery_success(Boolean query_success) {
this.query_success = query_success;
}
public Integer getMatch_pattern() {
return match_pattern;
}
public void setMatch_pattern(Integer match_pattern) {
this.match_pattern = match_pattern;
}
public String getWhois_domain() {
return whois_domain;
}
public void setWhois_domain(String whois_domain) {
this.whois_domain = whois_domain;
}
public Timestamp getWhois_update_date() {
return whois_update_date;
}
public void setWhois_update_date(Timestamp whois_update_date) {
this.whois_update_date = whois_update_date;
}
public Timestamp getWhois_create_date() {
return whois_create_date;
}
public void setWhois_create_date(Timestamp whois_create_date) {
this.whois_create_date = whois_create_date;
}
public Timestamp getWhois_expire_date() {
return whois_expire_date;
}
public void setWhois_expire_date(Timestamp whois_expire_date) {
this.whois_expire_date = whois_expire_date;
}
public String getWhois_email() {
return whois_email;
}
public void setWhois_email(String whois_email) {
this.whois_email = whois_email;
}
public String getWhois_ns() {
return whois_ns;
}
public void setWhois_ns(String whois_ns) {
this.whois_ns = whois_ns;
}
public String getWhois_registrar_name() {
return whois_registrar_name;
}
public void setWhois_registrar_name(String whois_registrar_name) {
this.whois_registrar_name = whois_registrar_name;
}
public String getWhois_registrant_org() {
return whois_registrant_org;
}
public void setWhois_registrant_org(String whois_registrant_org) {
this.whois_registrant_org = whois_registrant_org;
}
public String getWhois_registrant_name() {
return whois_registrant_name;
}
public void setWhois_registrant_name(String whois_registrant_name) {
this.whois_registrant_name = whois_registrant_name;
}
public String getWhois_registrant_street() {
return whois_registrant_street;
}
public void setWhois_registrant_street(String whois_registrant_street) {
this.whois_registrant_street = whois_registrant_street;
}
public String getWhois_registrant_city() {
return whois_registrant_city;
}
public void setWhois_registrant_city(String whois_registrant_city) {
this.whois_registrant_city = whois_registrant_city;
}
public String getWhois_registrant_state() {
return whois_registrant_state;
}
public void setWhois_registrant_state(String whois_registrant_state) {
this.whois_registrant_state = whois_registrant_state;
}
public String getWhois_registrant_postcode() {
return whois_registrant_postcode;
}
public void setWhois_registrant_postcode(String whois_registrant_postcode) {
this.whois_registrant_postcode = whois_registrant_postcode;
}
public String getWhois_registrant_country() {
return whois_registrant_country;
}
public void setWhois_registrant_country(String whois_registrant_country) {
this.whois_registrant_country = whois_registrant_country;
}
public String getWhois_registrant_phone() {
return whois_registrant_phone;
}
public void setWhois_registrant_phone(String whois_registrant_phone) {
this.whois_registrant_phone = whois_registrant_phone;
}
}

70
src/main/java/com/mesasoft/cn/sketch/dao/MariaDbBase.java Normal file
View File

@@ -0,0 +1,70 @@
package com.mesasoft.cn.sketch.dao;
import com.mesasoft.cn.sketch.util.TimeUtils;
import org.apache.log4j.Logger;
import java.sql.*;
import java.util.Date;
import java.util.Properties;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/28
* Time: 2:56 PM
* Description: No Description
*/
public class MariaDbBase {
private static final Logger LOG = Logger.getLogger(MariaDbBase.class);
private static final Properties props = new Properties();
private Statement statement;
public MariaDbBase(Connection conn, Statement stat) {
statement = stat;
}
public MariaDbBase() {
}
/**
* 执行写入sql
*/
public void writeSqlExecute(String sql){
try {
statement.executeUpdate(sql);
} catch (SQLIntegrityConstraintViolationException e){
LOG.error("Duplicated entry for key 'PRIMARY'");
} catch (SQLException exception) {
LOG.error("Sql : " + sql);
exception.printStackTrace();
}
}
/**
* 执行查询sql
*/
public ResultSet querySqlExecute(String sql){
ResultSet set = null;
try {
set = statement.executeQuery(sql);
} catch (SQLException exception) {
exception.printStackTrace();
}
return set;
}
/**
* TODO: getUnlabeledRecord() 考虑多个来源的情况
*/
/**
* 获得过期时间, 当前时间的expiredRangeDays天之前的日期为过期日期
*/
public static Date getExpiredTime(int expiredRangeDays){
return new Timestamp(TimeUtils.getStartOfDay(-expiredRangeDays).getTime());
}
}

31
src/main/java/com/mesasoft/cn/sketch/dao/src/main/java/generator/DomainCategoryReputationDao.java Normal file
View File

@@ -0,0 +1,31 @@
/*
package generator;
import generator.DomainCategoryReputation;
import generator.DomainCategoryReputationExample;
import java.util.List;
import org.apache.ibatis.annotations.Param;
public interface DomainCategoryReputationDao {
long countByExample(DomainCategoryReputationExample example);
int deleteByExample(DomainCategoryReputationExample example);
int deleteByPrimaryKey(Integer id);
int insert(DomainCategoryReputation record);
int insertSelective(DomainCategoryReputation record);
List<DomainCategoryReputation> selectByExample(DomainCategoryReputationExample example);
DomainCategoryReputation selectByPrimaryKey(Integer id);
int updateByExampleSelective(@Param("record") DomainCategoryReputation record, @Param("example") DomainCategoryReputationExample example);
int updateByExample(@Param("record") DomainCategoryReputation record, @Param("example") DomainCategoryReputationExample example);
int updateByPrimaryKeySelective(DomainCategoryReputation record);
int updateByPrimaryKey(DomainCategoryReputation record);
}*/

40
src/main/java/com/mesasoft/cn/sketch/entity/ReturnT.java Normal file
View File

@@ -0,0 +1,40 @@
package com.mesasoft.cn.sketch.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import java.io.Serializable;
@Data
@ToString
@NoArgsConstructor
@AllArgsConstructor
public class ReturnT<T> implements Serializable {
public static final int SUCCESS_STATUS = 200;
public static final int FAIL_STATUS = 500;
private int status;
private String requestid;
private String message;
private T data;
public static <T> ReturnT succeed(String requestid, T data) {
return new ReturnT(SUCCESS_STATUS, requestid, null, data);
}
public static <T> ReturnT failed(String requestid, String message) {
return new ReturnT(FAIL_STATUS, requestid, message, null);
}
public static <T> ReturnT failed(int status, String requestid, String message) {
return new ReturnT(status, requestid, message, null);
}
}

12
src/main/java/com/mesasoft/cn/sketch/entity/SketchAuth.java Normal file
View File

@@ -0,0 +1,12 @@
package com.mesasoft.cn.sketch.entity;
import lombok.Data;
@Data
public class SketchAuth {
private String omeid;
private String deviceid;
private String uid;
private String requestid;
}

24
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java Normal file
View File

@@ -0,0 +1,24 @@
package com.mesasoft.cn.sketch.entity.domain;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import java.util.Date;
@Data
public class DomainBase {
Integer id;
String fqdn;
String source;
Boolean query_success;
Integer match_pattern;
Integer status_code = 0;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd")
Date create_time;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd")
Date update_time;
}

235
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainCategory.java Normal file
View File

@@ -0,0 +1,235 @@
package com.mesasoft.cn.sketch.entity.domain;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class DomainCategory extends DomainBase{
private Integer reputation_score;
private String reputation_level;
private Integer category_id;
private String category_name;
private String category_group;
private Integer category_conf;
private Boolean is_a1_cat;
private String submit_user ="''";
// category schema
public DomainCategory(String fqdn,
String source,
Boolean query_success,
Integer match_pattern,
Integer reputation_score,
String reputationLevel,
Integer categoryId,
String categoryName,
String categoryGroup,
Integer categoryConf,
Boolean isA1Cat, Integer statusCode, String submitUser) {
this.fqdn = fqdn;
this.source = source; // 默认应为为brightcloud
this.query_success = query_success;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.reputation_score = reputation_score;
this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel);
this.category_id = categoryId;
this.category_name = ConfigUtils.getEffectiveString(categoryName);
this.category_group = ConfigUtils.getEffectiveString(categoryGroup);
this.category_conf = categoryConf;
this.is_a1_cat = isA1Cat;
this.status_code = statusCode;
this.submit_user = submitUser;
}
public DomainCategory(String fqdn,
String source,
Boolean query_success,
Integer match_pattern,
Integer reputation_score,
String reputationLevel,
Integer categoryId,
String categoryName,
String categoryGroup,
Integer categoryConf,
Boolean isA1Cat) {
this.fqdn = fqdn;
this.source = source; // 默认应为为brightcloud
this.query_success = query_success;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.reputation_score = reputation_score;
this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel);
this.category_id = categoryId;
this.category_name = ConfigUtils.getEffectiveString(categoryName);
this.category_group = ConfigUtils.getEffectiveString(categoryGroup);
this.category_conf = categoryConf;
this.is_a1_cat = isA1Cat;
}
/* public static void insertRecords(List<DomainCategory> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainCategory categoryFile : categoryFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + categoryFile.getKeys() + ") values" +
'(' + categoryFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public void updateRecords(List<DomainCategory> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainCategory categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainCategory> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") and source = '" + source + "'";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}*/
public static List<DomainCategory> rs2schema(ResultSet rs) throws SQLException {
List<DomainCategory> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainCategory(
rs.getString("fqdn"),
rs.getString("source"),
rs.getBoolean("query_success"),
rs.getInt("match_pattern"),
rs.getInt("reputation_score"),
rs.getString("reputation_level"),
rs.getInt("category_id"),
rs.getString("category_name"),
rs.getString("category_group"),
rs.getInt("category_conf"),
rs.getBoolean("is_a1_cat"),
rs.getInt("status_code"),
rs.getString("submit_user")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainCategory schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("reputation_score", schema.getReputation_score());
jsonObject.put("reputation_level", schema.getReputation_level());
jsonObject.put("category_id", schema.getCategory_id());
jsonObject.put("category_group", schema.getCategory_group());
jsonObject.put("category_name", schema.getCategory_name());
jsonObject.put("category_conf", schema.getCategory_conf());
jsonObject.put("is_a1_cat", schema.getIs_a1_cat());
jsonObject.put("status_code", schema.getStatus_code());
jsonObject.put("submit_user", schema.getSubmit_user());
return jsonObject;
}
/*public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", " + reputation_score +
", '" + reputation_level + '\'' +
", " + category_id +
", '" + category_name + '\'' +
", '" + category_group + '\'' +
", " + category_conf +
", " + status_code +
", " + submit_user +
", " + is_a1_cat;
return resString.replace("'null'", "null");
}
public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", reputation_score" +
", reputation_level" +
", category_id" +
", category_name" +
", category_group" +
", category_conf" +
", status_code" +
", submit_user" +
", is_a1_cat";
return resString;
}
public String getKeyValues() {
String resString = "source='" + source + '\'' +
", query_success=" + query_success +
", match_pattern=" + match_pattern +
", reputation_score=" + reputation_score +
", reputation_level='" + reputation_level + '\'' +
", category_id=" + category_id +
", category_name='" + category_name + '\'' +
", category_group='" + category_group + '\'' +
", category_conf=" + category_conf +
", is_a1_cat=" + is_a1_cat;
return resString.replace("'null'", "null");
}*/
}

20
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainInfo.java Normal file
View File

@@ -0,0 +1,20 @@
package com.mesasoft.cn.sketch.entity.domain;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class DomainInfo<T> {
private String domain;
private List<T> result;
}

271
src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java Normal file
View File

@@ -0,0 +1,271 @@
package com.mesasoft.cn.sketch.entity.domain;
import com.alibaba.fastjson.JSONObject;
import com.mesasoft.cn.sketch.util.ConfigUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/29
* Time: 9:27 AM
* Description: No Description
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class DomainWhois extends DomainBase{
private String whois_domain;
private Timestamp whois_update_date;
private Timestamp whois_create_date;
private Timestamp whois_expire_date;
private String whois_email;
private String whois_ns;
private String whois_registrar_name;
private String whois_registrant_org;
private String whois_registrant_name;
private String whois_registrant_street;
private String whois_registrant_city;
private String whois_registrant_state;
private String whois_registrant_postcode;
private String whois_registrant_country;
private String whois_registrant_phone;
// category schema
public DomainWhois(String fqdn,
String source,
Integer match_pattern,
Boolean query_success,
String whoisDomain,
Date whoisUpdateDate,
Date whoisCreateDate,
Date whoisExpireDate,
String whoisEmail,
String whoisNs,
String whoisRegistrarName,
String whoisRegistrantOrg,
String whoisRegistrantName,
String whoisRegistrantStreet,
String whoisRegistrantCity,
String whoisRegistrantState,
String whoisRegistrantPostcode,
String whoisRegistrantCountry,
String whoisRegistrantPhone
) {
this.fqdn = fqdn;
this.source = source;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
}
this.query_success = query_success;
this.whois_domain = ConfigUtils.getEffectiveString(whoisDomain);
this.whois_update_date = whoisUpdateDate == null ? null : new Timestamp(whoisUpdateDate.getTime());
this.whois_create_date = whoisCreateDate == null ? null : new Timestamp(whoisCreateDate.getTime());
this.whois_expire_date = whoisExpireDate == null ? null : new Timestamp(whoisExpireDate.getTime());
this.whois_email = ConfigUtils.getEffectiveString(whoisEmail);
this.whois_ns = ConfigUtils.getEffectiveString(whoisNs);
this.whois_registrar_name = ConfigUtils.getEffectiveString(whoisRegistrarName);
this.whois_registrant_org = ConfigUtils.getEffectiveString(whoisRegistrantOrg);
this.whois_registrant_name = ConfigUtils.getEffectiveString(whoisRegistrantName);
this.whois_registrant_street = ConfigUtils.getEffectiveString(whoisRegistrantStreet);
this.whois_registrant_city = ConfigUtils.getEffectiveString(whoisRegistrantCity);
this.whois_registrant_state = ConfigUtils.getEffectiveString(whoisRegistrantState);
this.whois_registrant_postcode = ConfigUtils.getEffectiveString(whoisRegistrantPostcode);
this.whois_registrant_country = ConfigUtils.getEffectiveString(whoisRegistrantCountry);
this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone);
}
/* public static void insertRecords(List<DomainWhois> whoisFiles, MariaDbBase mariaDbBase) {
for (DomainWhois whoisFile : whoisFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static String insertSql(List<DomainWhois> whoisFiles) {
DomainWhois whoisFile = whoisFiles.get(0);
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + whoisFile.getKeys() + ") values" +
'(' + whoisFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
return resSql;
}
public static void updateRecords(List<DomainWhois> categoryFiles, MariaDbBase mariaDbBase) {
for (DomainWhois categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
"SET " + categoryFile.getKeyValues() +
", update_time = current_time() " +
" WHERE fqdn = '" + categoryFile.getFqdn() + '\'';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static List<DomainWhois> getDbRecord(List<String> fqdns, MariaDbBase mariaDbBase, String source) throws SQLException {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
" WHERE fqdn in (" + queryFqdns + ") ";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}*/
public String getValues() {
String resString = "'" + fqdn + '\'' +
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
", '" + whois_domain + '\'' +
", '" + whois_update_date + '\'' +
", '" + whois_create_date + '\'' +
", '" + whois_expire_date + '\'' +
", '" + whois_email + '\'' +
", '" + whois_ns + '\'' +
", '" + whois_registrar_name + '\'' +
", '" + whois_registrant_org + '\'' +
", '" + whois_registrant_name + '\'' +
", '" + whois_registrant_street + '\'' +
", '" + whois_registrant_city + '\'' +
", '" + whois_registrant_state + '\'' +
", '" + whois_registrant_postcode + '\'' +
", '" + whois_registrant_country + '\'' +
", '" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}
public static List<DomainWhois> rs2schema(ResultSet rs) throws SQLException {
List<DomainWhois> schemaFiles = new ArrayList<>();
while (rs.next()) {
schemaFiles.add(
new DomainWhois(
rs.getString("fqdn"),
rs.getString("source"),
rs.getInt("match_pattern"),
rs.getBoolean("query_success"),
rs.getString("whois_domain"),
(Date) rs.getDate("whois_update_date"),
(Date) rs.getDate("whois_create_date"),
(Date) rs.getDate("whois_expire_date"),
rs.getString("whois_email"),
rs.getString("whois_ns"),
rs.getString("whois_registrar_name"),
rs.getString("whois_registrant_org"),
rs.getString("whois_registrant_name"),
rs.getString("whois_registrant_street"),
rs.getString("whois_registrant_city"),
rs.getString("whois_registrant_state"),
rs.getString("whois_registrant_postcode"),
rs.getString("whois_registrant_country"),
rs.getString("whois_registrant_phone")
));
}
return schemaFiles;
}
public static JSONObject schema2json(DomainWhois schema) throws SQLException {
JSONObject jsonObject = new JSONObject(true);
jsonObject.put("fqdn", schema.getFqdn());
jsonObject.put("source", schema.getSource());
jsonObject.put("match_pattern", schema.getMatch_pattern());
jsonObject.put("query_success", schema.getQuery_success());
jsonObject.put("whois_domain", schema.getWhois_domain());
jsonObject.put("whois_update_date", schema.getWhois_update_date());
jsonObject.put("whois_create_date", schema.getWhois_create_date());
jsonObject.put("whois_expire_date", schema.getWhois_expire_date());
jsonObject.put("whois_email", schema.getWhois_email());
jsonObject.put("whois_ns", schema.getWhois_ns());
jsonObject.put("whois_registrar_name", schema.getWhois_registrar_name());
jsonObject.put("whois_registrant_org", schema.getWhois_registrant_org());
jsonObject.put("whois_registrant_name", schema.getWhois_registrant_name());
jsonObject.put("whois_registrant_street", schema.getWhois_registrant_street());
jsonObject.put("whois_registrant_city", schema.getWhois_registrant_city());
jsonObject.put("whois_registrant_state", schema.getWhois_registrant_state());
jsonObject.put("whois_registrant_postcode", schema.getWhois_registrant_postcode());
jsonObject.put("whois_registrant_country", schema.getWhois_registrant_country());
jsonObject.put("whois_registrant_phone", schema.getWhois_registrant_phone());
return jsonObject;
}
/* public String getKeys() {
String resString;
resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
", whois_domain" +
", whois_update_date" +
", whois_create_date" +
", whois_expire_date" +
", whois_email" +
", whois_ns" +
", whois_registrar_name" +
", whois_registrant_org" +
", whois_registrant_name" +
", whois_registrant_street" +
", whois_registrant_city" +
", whois_registrant_state" +
", whois_registrant_postcode" +
", whois_registrant_country" +
", whois_registrant_phone";
return resString;
}
public String getKeyValues() {
String resString = "query_success=" + query_success +
", source='" + source + '\'' +
", match_pattern=" + match_pattern +
", whois_domain='" + whois_domain + '\'' +
", whois_update_date='" + whois_update_date + '\'' +
", whois_create_date='" + whois_create_date + '\'' +
", whois_expire_date='" + whois_expire_date + '\'' +
", whois_email='" + whois_email + '\'' +
", whois_ns='" + whois_ns + '\'' +
", whois_registrar_name='" + whois_registrar_name + '\'' +
", whois_registrant_org='" + whois_registrant_org + '\'' +
", whois_registrant_name='" + whois_registrant_name + '\'' +
", whois_registrant_street='" + whois_registrant_street + '\'' +
", whois_registrant_city='" + whois_registrant_city + '\'' +
", whois_registrant_state='" + whois_registrant_state + '\'' +
", whois_registrant_postcode='" + whois_registrant_postcode + '\'' +
", whois_registrant_country='" + whois_registrant_country + '\'' +
", whois_registrant_phone='" + whois_registrant_phone + '\'';
return resString.replace("'null'", "null");
}*/
}

461
src/main/java/com/mesasoft/cn/sketch/pojo/IocDarkWeb.java Normal file
View File

@@ -0,0 +1,461 @@
package com.mesasoft.cn.sketch.pojo;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.dao.MariaDbBase;
import java.util.Arrays;
import java.util.List;
public class IocDarkWeb {
private Long id;
private String iocValue;
private String iocType;
private String nodeType;
private String ipv6;
private String isp;
private String asn;
private String operatingSystem;
private String country;
private String city;
private String latitude;
private String Longtitude;
private String torFingerprint;
private Integer torDirPort;
private Integer torOrPort;
private String torFlags;
private String torVersion;
private String torNickname;
private String torExitPolicy;
private String i2PHash;
private String i2PMethod;
private String i2PVersion;
private Boolean i2PIsFloodfill;
private String i2PBandwidth;
private Boolean i2PIsHidden;
private Boolean i2PIsReachable;
private String i2PPublishTime;
private Integer mtproxyPort;
private String mtproxySecret;
private Integer obfs4Port;
private String obfs4Fingerprint;
private String obfs4Cert;
private Integer obfs4IatMode;
private Integer snowflakePort;
private Long firstAppearTime;
private Long lastAppearTime;
private Long createTime;
private Long updateTime;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getIocValue() {
return iocValue;
}
public void setIocValue(String iocValue) {
this.iocValue = iocValue;
}
public String getIocType() {
return iocType;
}
public void setIocType(String iocType) {
this.iocType = iocType;
}
public String getIpv6() {
return ipv6;
}
public void setIpv6(String ipv6) {
this.ipv6 = ipv6;
}
public String getIsp() {
return isp;
}
public void setIsp(String isp) {
this.isp = isp;
}
public String getAsn() {
return asn;
}
public void setAsn(String asn) {
this.asn = asn;
}
public String getOperatingSystem() {
return operatingSystem;
}
public void setOperatingSystem(String operatingSystem) {
this.operatingSystem = operatingSystem;
}
public String getCountry() {
return country;
}
public void setCountry(String country) {
this.country = country;
}
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
public String getLatitude() {
return latitude;
}
public void setLatitude(String latitude) {
this.latitude = latitude;
}
public String getLongtitude() {
return Longtitude;
}
public void setLongtitude(String Longtitude) {
this.Longtitude = Longtitude;
}
public String getTorFingerprint() {
return torFingerprint;
}
public void setTorFingerprint(String torFingerprint) {
this.torFingerprint = torFingerprint;
}
public Integer getTorDirPort() {
return torDirPort;
}
public void setTorDirPort(Integer torDirPort) {
this.torDirPort = torDirPort;
}
public Integer getTorOrPort() {
return torOrPort;
}
public void setTorOrPort(Integer torOrPort) {
this.torOrPort = torOrPort;
}
public String getTorFlags() {
return torFlags;
}
public void setTorFlags(String torFlags) {
this.torFlags = torFlags;
}
public String getTorVersion() {
return torVersion;
}
public void setTorVersion(String torVersion) {
this.torVersion = torVersion;
}
public String getTorNickname() {
return torNickname;
}
public void setTorNickname(String torNickname) {
this.torNickname = torNickname;
}
public String getTorExitPolicy() {
return torExitPolicy;
}
public void setTorExitPolicy(String torExitPolicy) {
this.torExitPolicy = torExitPolicy;
}
public String getI2PHash() {
return i2PHash;
}
public void setI2PHash(String i2PHash) {
this.i2PHash = i2PHash;
}
public String getI2PMethod() {
return i2PMethod;
}
public void setI2PMethod(String i2PMethod) {
this.i2PMethod = i2PMethod;
}
public String getI2PVersion() {
return i2PVersion;
}
public void setI2PVersion(String i2PVersion) {
this.i2PVersion = i2PVersion;
}
public Boolean getI2PIsFloodfill() {
return i2PIsFloodfill;
}
public void setI2PIsFloodfill(Boolean i2PIsFloodfill) {
this.i2PIsFloodfill = i2PIsFloodfill;
}
public String getI2PBandwidth() {
return i2PBandwidth;
}
public void setI2PBandwidth(String i2PBandwidth) {
this.i2PBandwidth = i2PBandwidth;
}
public Boolean getI2PIsHidden() {
return i2PIsHidden;
}
public void setI2PIsHidden(Boolean i2PIsHidden) {
this.i2PIsHidden = i2PIsHidden;
}
public Boolean getI2PIsReachable() {
return i2PIsReachable;
}
public void setI2PIsReachable(Boolean i2PIsReachable) {
this.i2PIsReachable = i2PIsReachable;
}
public String getI2PPublishTime() {
return i2PPublishTime;
}
public void setI2PPublishTime(String i2PPublishTime) {
this.i2PPublishTime = i2PPublishTime;
}
public Integer getMtproxyPort() {
return mtproxyPort;
}
public void setMtproxyPort(Integer mtproxyPort) {
this.mtproxyPort = mtproxyPort;
}
public String getMtproxySecret() {
return mtproxySecret;
}
public void setMtproxySecret(String mtproxySecret) {
this.mtproxySecret = mtproxySecret;
}
public Integer getObfs4Port() {
return obfs4Port;
}
public void setObfs4Port(Integer obfs4Port) {
this.obfs4Port = obfs4Port;
}
public String getObfs4Fingerprint() {
return obfs4Fingerprint;
}
public void setObfs4Fingerprint(String obfs4Fingerprint) {
this.obfs4Fingerprint = obfs4Fingerprint;
}
public String getObfs4Cert() {
return obfs4Cert;
}
public void setObfs4Cert(String obfs4Cert) {
this.obfs4Cert = obfs4Cert;
}
public Integer getObfs4IatMode() {
return obfs4IatMode;
}
public void setObfs4IatMode(Integer obfs4IatMode) {
this.obfs4IatMode = obfs4IatMode;
}
public Integer getSnowflakePort() {
return snowflakePort;
}
public void setSnowflakePort(Integer snowflakePort) {
this.snowflakePort = snowflakePort;
}
public Long getFirstAppearTime() {
return firstAppearTime;
}
public void setFirstAppearTime(Long firstAppearTime) {
this.firstAppearTime = firstAppearTime;
}
public Long getLastAppearTime() {
return lastAppearTime;
}
public void setLastAppearTime(Long lastAppearTime) {
this.lastAppearTime = lastAppearTime;
}
public Long getCreateTime() {
return createTime;
}
public void setCreateTime(Long createTime) {
this.createTime = createTime;
}
public Long getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Long updateTime) {
this.updateTime = updateTime;
}
public String getNodeType() {
return nodeType;
}
public void setNodeType(String nodeType) {
this.nodeType = nodeType;
}
public String getValues() {
return
" '" + iocValue + '\'' +
", '" + iocType + '\'' +
", '" + nodeType + '\'' +
", '" + ipv6 + '\'' +
", '" + isp + '\'' +
", '" + asn + '\'' +
", '" + operatingSystem + '\'' +
", '" + country + '\'' +
", '" + city + '\'' +
", " + latitude +
", " + Longtitude +
", '" + torFingerprint + '\'' +
", " + torDirPort +
", " + torOrPort +
", '" + torFlags + '\'' +
", '" + torVersion + '\'' +
", '" + torNickname + '\'' +
", '" + torExitPolicy + '\'' +
", '" + i2PHash + '\'' +
", '" + i2PMethod + '\'' +
", '" + i2PVersion + '\'' +
", " + i2PIsFloodfill +
", '" + i2PBandwidth + '\'' +
", " + i2PIsHidden +
", " + i2PIsReachable +
", '" + i2PPublishTime + '\'' +
", " + mtproxyPort +
", '" + mtproxySecret + '\'' +
", " + obfs4Port +
", '" + obfs4Fingerprint + '\'' +
", '" + obfs4Cert + '\'' +
", " + obfs4IatMode +
", " + snowflakePort +
", " + firstAppearTime +
", " + lastAppearTime +
", " + createTime +
", " + updateTime;
}
private static final String dataBase = AppConfig.DATABASE;
private static final String tableName = "ioc_darkweb";
private static final String fields = "ioc_value, ioc_type,node_type, ipv6, isp, asn, operating_system, country, city, latitude, Longtitude, tor_fingerprint, " +
"tor_dir_port, tor_or_port, tor_flags, tor_version, tor_nickname, tor_exit_policy, i2p_hash, i2p_method, i2p_version, i2p_is_floodfill, i2p_bandwidth, " +
"i2p_is_hidden, i2p_is_reachable, i2p_publish_time, mtproxy_port, mtproxy_secret, obfs4_port, obfs4_fingerprint, obfs4_cert, obfs4_iat_mode, snowflake_port, " +
"first_appear_time, last_appear_time, create_time, update_time";
public static void insertRecords(List<IocDarkWeb> iocDarkWebs, MariaDbBase mariaDbBase) {
for (IocDarkWeb iocDarkWeb : iocDarkWebs) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + fields + ") values" +
'(' + iocDarkWeb.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDbBase.writeSqlExecute(resSql);
}
}
public static String getInsertSql() {
String[] split = fields.split(",");
Arrays.fill(split, "?");
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + fields + ") values" +
'(' + String.join(",", split) + ')';
return resSql;
}
}

321
src/main/java/com/mesasoft/cn/sketch/service/DarkWebService.java Normal file
View File

@@ -0,0 +1,321 @@
package com.mesasoft.cn.sketch.service;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.file.FileReader;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONNull;
import cn.hutool.json.JSONUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.util.TypeUtils;
import com.google.common.collect.Lists;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.pojo.IocDarkWeb;
import com.mesasoft.cn.sketch.util.MariaDBUtils;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import org.apache.commons.io.filefilter.FileFilterUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.springframework.beans.factory.annotation.Autowired;
import java.io.File;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
public class DarkWebService {
private static Log logger = Log.get();
@Autowired
static AppConfig appConfig;
public static void main(String[] args) {
//1.读取文件，转为list<Map> ，参数 文件目录，多个目录
//tor
//i2p
//mtproxy
//obfs4
//snowflake
//generalbridge
try {
String parentPath = "C:\\test\\h3\\";
String[] apis = {"tor", "i2p", "mtproxy", "obfs4", "snowflake"/*, "generalbridge"*/};
String[] t = {"07", "09", "10", "11", "12", "13", "14", "15", "17", "20"};
String apiType = apis[0];
for (String s : t) {
List<Map> tors = readJson(parentPath + apiType +"\\"+s+"\\");
System.err.println(apiType+"读取日志" + tors.size());
List<IocDarkWeb> list = new ArrayList<>();
//2.转成对象
for (Map data : tors) {
Map result = (Map) data.get("result");
List<Map> nodeList = (List<Map>) result.get("node_list");
for (Map node : nodeList) {
String ipv6s = "";
if (node.get("ipv6") instanceof List) {
List ipv6 = TypeUtils.castToJavaBean(node.get("ipv6"), List.class);
ipv6s = String.join(",", ipv6);
} else {
ipv6s = TypeUtils.castToString(Optional.ofNullable(node.get("ipv6")).orElse(""));
}
//location
Map detail = (Map) node.get("detail");
Map location = (Map) detail.get("location");
String isp = TypeUtils.castToString(location.get("isp"));
String asn = TypeUtils.castToString(location.get("as_number"));
String country = TypeUtils.castToString(location.get("country"));
String city = TypeUtils.castToString(location.get("city"));
String latitude = TypeUtils.castToString(location.get("latitude"));
String longtitude = TypeUtils.castToString(location.get("longtitude"));
Long firstAppearTime = TypeUtils.castToLong(node.get("first_appear_time")) / 1000;
Long lastAppearTime = TypeUtils.castToLong(node.get("last_appear_time")) / 1000;
//创建对象
IocDarkWeb iocDarkWeb = new IocDarkWeb();
iocDarkWeb.setNodeType(apiType);
iocDarkWeb.setIpv6(ipv6s);
//location
iocDarkWeb.setIsp(isp);
iocDarkWeb.setAsn(asn);
iocDarkWeb.setCountry(country);
iocDarkWeb.setCity(city);
iocDarkWeb.setLatitude(latitude);
iocDarkWeb.setLongtitude(longtitude);
//时间
iocDarkWeb.setFirstAppearTime(firstAppearTime);
iocDarkWeb.setLastAppearTime(lastAppearTime);
iocDarkWeb.setCreateTime(DateUtil.currentSeconds());
iocDarkWeb.setUpdateTime(DateUtil.currentSeconds());
iocDarkWeb.setIocType("ip");
switch (apiType) {
case "tor":
//tor
String ip = TypeUtils.castToString(node.get("ip"));
iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
torDetail(node, detail, iocDarkWeb);
break;
case "i2p":
ip = TypeUtils.castToString(node.get("ipv4"));
iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
i2pDetail(detail, iocDarkWeb);
break;
case "mtproxy":
//特殊处理，mtproxy节点中server字段支持域名和ip两种形式
String server = TypeUtils.castToString(node.get("server"));
iocDarkWeb.setIocType(ValidationUtils.isValidIp(server) ? "ip" : "domain");
iocDarkWeb.setIocValue(server);
mtproxyDetail(detail, iocDarkWeb);
break;
case "obfs4":
ip = TypeUtils.castToString(node.get("ipv4"));
iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
obfs4Detail(detail, iocDarkWeb);
break;
case "snowflake":
ip = TypeUtils.castToString(node.get("ipv4"));
iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
snowflakeDetail(detail, iocDarkWeb);
break;
}
list.add(iocDarkWeb);
}
}
//3.写入数据库
List<List<IocDarkWeb>> apiPartitions = Lists.partition(list, appConfig.getDbQueryBatchSize());
for (List<IocDarkWeb> partition : apiPartitions) {
System.err.println("开始写入" + partition.size());
insert(partition);
System.err.println("成功写入" + partition.size());
}
}
System.err.println(apiType+"写入完成");
} catch (Exception e) {
e.printStackTrace();
}
}
private static void snowflakeDetail(Map detail, IocDarkWeb iocDarkWeb) {
//snowflake
Integer snowflakePort = TypeUtils.castToInt(detail.get("port"));
//snowflake
iocDarkWeb.setSnowflakePort(snowflakePort);
}
private static void obfs4Detail(Map detail, IocDarkWeb iocDarkWeb) {
//obfs4
Integer obfs4Port = TypeUtils.castToInt(detail.get("port"));
String obfs4Fingerprint = TypeUtils.castToString(detail.get("fingerprint"));
String obfs4Cert = TypeUtils.castToString(detail.get("cert"));
Integer obfs4IatMode = TypeUtils.castToInt(detail.get("iatMode"));
//obfs4
iocDarkWeb.setObfs4Port(obfs4Port);
iocDarkWeb.setObfs4Fingerprint(obfs4Fingerprint);
iocDarkWeb.setObfs4Cert(obfs4Cert);
iocDarkWeb.setObfs4IatMode(obfs4IatMode);
}
private static void mtproxyDetail(Map detail, IocDarkWeb iocDarkWeb) {
//mtproxy
Integer mtproxyPort = TypeUtils.castToInt(detail.get("port"));
String mtproxySecret = TypeUtils.castToString(detail.get("secret"));
//mtproxy
iocDarkWeb.setMtproxyPort(mtproxyPort);
iocDarkWeb.setMtproxySecret(mtproxySecret);
}
private static void i2pDetail(Map detail, IocDarkWeb iocDarkWeb) {
//i2p
String i2pHash = TypeUtils.castToString(detail.get("hash"));
String i2pMethod = TypeUtils.castToString(detail.get("method"));
String i2pVersion = TypeUtils.castToString(detail.get("version"));
Boolean i2pFloodFill = TypeUtils.castToBoolean(detail.get("isFloodfill"));
String i2pWidth = TypeUtils.castToString(detail.get("width"));
Boolean i2pHidden = TypeUtils.castToBoolean(detail.get("isHidden"));
Boolean i2pReachable = TypeUtils.castToBoolean(detail.get("isReachable"));
String i2pPublish = TypeUtils.castToString(detail.get("publish"));
//i2p
iocDarkWeb.setI2PHash(i2pHash);
iocDarkWeb.setI2PMethod(i2pMethod);
iocDarkWeb.setI2PVersion(i2pVersion);
iocDarkWeb.setI2PIsFloodfill(i2pFloodFill);
iocDarkWeb.setI2PBandwidth(i2pWidth);
iocDarkWeb.setI2PIsHidden(i2pHidden);
iocDarkWeb.setI2PIsReachable(i2pReachable);
iocDarkWeb.setI2PPublishTime(i2pPublish);
}
private static void torDetail(Map node, Map detail, IocDarkWeb iocDarkWeb) {
String fingerprint = TypeUtils.castToString(node.get("fingerprint"));
Integer dirPort = node.get("dir_port") == JSONNull.NULL ? null : TypeUtils.castToInt(node.get("dir_port"));
Integer orPort = node.get("or_port") == JSONNull.NULL ? null : TypeUtils.castToInt(node.get("or_port"));
List flagsList = TypeUtils.castToJavaBean(detail.get("flags"), List.class);
String flags = String.join(",", flagsList);
String nickname = TypeUtils.castToString(detail.get("nickname"));
String exitPolicy = TypeUtils.castToString(detail.get("exit_policy"));
String torVirsion = TypeUtils.castToString(detail.get("tor_virsion"));
String operatingSystem = TypeUtils.castToString(detail.get("host_system"));
//tor
iocDarkWeb.setTorFingerprint(fingerprint);
iocDarkWeb.setTorDirPort(dirPort);
iocDarkWeb.setTorOrPort(orPort);
iocDarkWeb.setTorFlags(flags);
iocDarkWeb.setTorNickname(nickname);
iocDarkWeb.setTorExitPolicy(exitPolicy);
iocDarkWeb.setOperatingSystem(operatingSystem);
iocDarkWeb.setTorVersion(torVirsion);
}
public static List<Map> readJson(String path) {
List<File> files = FileUtil.loopFiles(Paths.get(path), 1, FileFilterUtils.suffixFileFilter("json"));
logger.info("读取文件数量 {}", files.size());
List<Map> jsonList = files.stream().map(f -> {
FileReader fr = new FileReader(f);
Map objMap = JSONUtil.toBean(fr.readString(), Map.class);
return objMap;
}).collect(Collectors.toList());
return jsonList;
}
public static void insert(List<IocDarkWeb> iocDarkWebs) throws Exception {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
Connection connection = MariaDBUtils.getConnection();
connection.setAutoCommit(false);
PreparedStatement pst = connection.prepareStatement(IocDarkWeb.getInsertSql());
for (IocDarkWeb i : iocDarkWebs) {
pst.setString(1, i.getIocValue());
pst.setString(2, i.getIocType());
pst.setString(3, i.getNodeType());
pst.setString(4, i.getIpv6());
pst.setString(5, i.getIsp());
pst.setString(6, i.getAsn());
pst.setString(7, i.getOperatingSystem());
pst.setString(8, i.getCountry());
pst.setString(9, i.getCity());
pst.setString(10, i.getLatitude());
pst.setString(11, i.getLongtitude());
pst.setString(12, i.getTorFingerprint());
pst.setObject(13, i.getTorDirPort());
pst.setObject(14, i.getTorOrPort());
pst.setString(15, i.getTorFlags());
pst.setString(16, i.getTorVersion());
pst.setString(17, i.getTorNickname());
pst.setString(18, i.getTorExitPolicy());
pst.setString(19, i.getI2PHash());
pst.setString(20, i.getI2PMethod());
pst.setString(21, i.getI2PVersion());
pst.setObject(22, i.getI2PIsFloodfill());
pst.setString(23, i.getI2PBandwidth());
pst.setObject(24, i.getI2PIsHidden());
pst.setObject(25, i.getI2PIsReachable());
pst.setString(26, i.getI2PPublishTime());
pst.setObject(27, i.getMtproxyPort());
pst.setString(28, i.getMtproxySecret());
pst.setObject(29, i.getObfs4Port());
pst.setString(30, i.getObfs4Fingerprint());
pst.setString(31, i.getObfs4Cert());
pst.setObject(32, i.getObfs4IatMode());
pst.setObject(33, i.getSnowflakePort());
pst.setLong(34, i.getFirstAppearTime());
pst.setLong(35, i.getLastAppearTime());
pst.setLong(36, i.getCreateTime());
pst.setLong(37, i.getUpdateTime());
pst.addBatch();
}
pst.executeBatch();
connection.commit();
stopWatch.stop();
logger.info("成功提交数据 {} ,花费时间 {} ms", iocDarkWebs.size(), stopWatch.getTime());
MariaDBUtils.close(pst, connection);
}
}

93
src/main/java/com/mesasoft/cn/sketch/service/DbService.java Normal file
View File

@@ -0,0 +1,93 @@
package com.mesasoft.cn.sketch.service;
import cn.hutool.log.Log;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.dao.DomainCategorySchema;
import com.mesasoft.cn.sketch.dao.MariaDbBase;
import java.util.List;
public class DbService {
private Log logger = Log.get();
private static final String dataBase = AppConfig.DATABASE;
private static final String tableName = AppConfig.DOMAIN_CATE_TABLENAME;
private MariaDbBase mariaDB;
public void execute(List<DomainCategorySchema> recordsFromBcApi) {
for (DomainCategorySchema categoryFile : recordsFromBcApi) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + categoryFile.getKeys() + ") values" +
'(' + categoryFile.getValues() + ')';
resSql = resSql.replace("'null'", "null");
mariaDB.writeSqlExecute(resSql);
}
}
/**
* @throws Exception
*/
/* public void exec(List<Map> data, String sql) throws Exception {
if (Objects.isNull(data)) {
logger.info(" data is empty ");
return;
}
Connection conn = null;
PreparedStatement pst = null;
try {
//创建数据库连接库对象
conn = MariaDBUtils.getConnection();
long start = System.currentTimeMillis();
conn.setAutoCommit(false);
pst = conn.prepareStatement(sql);
int count = 0;
int index = 1;
for (int i = 1; i <= data.size(); i++) {
for (Object val : data.get(i - 1).values()) {
if (val instanceof Long) {
pst.setLong((index++), Long.valueOf(String.valueOf(val)));
} else if (val instanceof Integer) {
pst.setInt((index++), Integer.valueOf(String.valueOf(val)));
} else if (val instanceof Boolean) {
pst.setBoolean((index++), Boolean.valueOf(String.valueOf(val)));
} else {
pst.setString((index++), String.valueOf(val));
}
}
index = 1;
pst.addBatch();
//1w提交一次
if (i % ApplicationConfig.DB_QUERY_BATCH_SIZE == 0) {
int[] ints = pst.executeBatch();
count = count + ints.length;
conn.commit();
pst.clearBatch();
}
}
int[] ints = pst.executeBatch();
count = count + ints.length;
conn.commit();
logger.info("sql {} , count {} , take {}", sql, tableName, count, (System.currentTimeMillis() - start) + "ms");
} catch (Exception e) {
if (conn != null) {
try {
conn.rollback();
} catch (SQLException e1) {
logger.error(e1);
}
}
logger.error(e);
throw e;
} finally {
IoUtil.close(pst);
IoUtil.close(conn);
}
}*/
}

14
src/main/java/com/mesasoft/cn/sketch/service/DomainService.java Normal file
View File

@@ -0,0 +1,14 @@
package com.mesasoft.cn.sketch.service;
import com.alibaba.fastjson.JSONArray;
import java.util.List;
public interface DomainService {
JSONArray getCategoryInfo(List<String> domains, String username, boolean isLocal) throws Exception ;
JSONArray getWhoisInfo(List<String> domains, boolean isLocal) throws Exception ;
}

117
src/main/java/com/mesasoft/cn/sketch/service/FileQuery.java Normal file
View File

@@ -0,0 +1,117 @@
/*
package cn.ac.iie.efo.sketch.service;
import cn.ac.iie.efo.sketch.config.ApplicationConfig;
import cn.ac.iie.efo.sketch.util.FileUtils;
import cn.ac.iie.efo.sketch.util.JsonTool;
import cn.ac.iie.efo.sketch.util.MariaDBUtils;
import cn.ac.iie.efo.sketch.util.ValidationUtils;
import cn.hutool.core.io.file.FileNameUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONArray;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.DecimalFormat;
import java.util.List;
*/
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2022/1/3
* Time: 5:33 PM
* Description: No Description
*//*
public class FileQuery {
private static final Log logger = Log.get();
long queryNum = 0;
long dbResultNum = 0;
long apiResultNum = 0;
long failedQueryNum = 0;
public String queryExecutor(String objectType, String queryType, String queryFile, String username, Boolean isLocal) throws SQLException, IOException {
Connection mariaConn = MariaDBUtils.getConnection();
Statement mariaStat = mariaConn.createStatement();
// 文件读取
InputStreamReader inputStreamReader = new InputStreamReader( new FileInputStream(queryFile), "GBK");
BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
File importFile = new File(queryFile);
String importFileName = FileNameUtil.getName(importFile);
long fileLineNum = Files.lines(importFile.toPath()).count();
logger.info("[File query]-" + importFileName + ": File Found.");
logger.info("[File query]-" + queryType + ".");
String outputFileName = importFileName + "." + queryType + ".json";
File outputFile = new File(ApplicationConfig.QUERY_OUTPUT_DIR + "/" + outputFileName);
if (!outputFile.exists()) {
FileUtils.createFile(new File(ApplicationConfig.QUERY_OUTPUT_DIR), outputFileName);
}
OutputStream outStream = new FileOutputStream(outputFile);
OutputStreamWriter outWriter = new OutputStreamWriter(outStream, StandardCharsets.UTF_8);
outWriter.write("[\n");
logger.info("[File query]-Start query! Progress: 0.00%");
// 执行查询
List<String> objectList;
JSONArray resultJson = new JSONArray();
ListQuery listQueryUtils = new ListQuery();
int batch_index = 1;
// 批量读取
while ((objectList = FileUtils.getBatchLineReadIn(bufferedReader, ApplicationConfig.QUERY_READIN_BATCH)).size() > 0) {
// 校验
objectList = ValidationUtils.getChecked(objectList, objectType);
JSONArray queryResults = listQueryUtils.getQueryResults(objectType, queryType, objectList,username, isLocal);
resultJson.addAll(queryResults);
queryNum += listQueryUtils.queryNum;
dbResultNum += listQueryUtils.dbResultNum;
apiResultNum += listQueryUtils.apiResultNum;
failedQueryNum += listQueryUtils.failedQueryNum;
// 写入output
// todo 写入格式优化 check
for (Object json : queryResults) {
outWriter.write(JsonTool.prettyJson(json.toString()));
}
// 进度日志
if (batch_index * ApplicationConfig.QUERY_READIN_BATCH % ApplicationConfig.QUERY_LOG_FILE_LINE_INTERVAL == 0) {
String percent = new DecimalFormat("##.00%").format((float) queryNum / fileLineNum);
logger.info("[File query]-" + queryType + "-" + importFileName + " Progress:" + percent);
}
batch_index++;
}
logger.info("[File query]-" + queryType + "-" + importFileName + " "
+ "Query result: submit " + queryNum + " valid objects, "
+ dbResultNum + " (" + new DecimalFormat("##.0%").format((float) dbResultNum / queryNum) + ")" + " results from database，"
+ apiResultNum + " (" + new DecimalFormat("##.0%").format((float) apiResultNum / queryNum) + ")" + " results from api. "
+ failedQueryNum + " (" + new DecimalFormat("##.0%").format((float) failedQueryNum / queryNum) + ")" + " failed queries,");
logger.info("[File query]-" + importFileName + " Results saved in " + outputFile.toString());
outWriter.write("\n]");
FileUtils.writerClose(outWriter, outStream);
FileUtils.readerClose(bufferedReader, inputStreamReader);
MariaDBUtils.close(mariaStat, mariaConn);
// todo check 写入null时的过滤情况
return resultJson.toJSONString();
}
}
*/

241
src/main/java/com/mesasoft/cn/sketch/service/ListQuery.java Normal file
View File

File diff suppressed because one or more lines are too long

217
src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java Normal file
View File

@@ -0,0 +1,217 @@
package com.mesasoft.cn.sketch.service.impl;
import cn.hutool.core.date.DateUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONArray;
import com.google.common.collect.Lists;
import com.mesasoft.cn.sketch.api.BrightCloud;
import com.mesasoft.cn.sketch.api.ChinaZ;
import com.mesasoft.cn.sketch.config.AppConfig;
import com.mesasoft.cn.sketch.config.BrightCloudConfig;
import com.mesasoft.cn.sketch.config.ChinazConfig;
import com.mesasoft.cn.sketch.dao.DomainDao;
import com.mesasoft.cn.sketch.entity.domain.DomainCategory;
import com.mesasoft.cn.sketch.entity.domain.DomainInfo;
import com.mesasoft.cn.sketch.entity.domain.DomainWhois;
import com.mesasoft.cn.sketch.service.DomainService;
import com.mesasoft.cn.sketch.util.ValidationUtils;
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.List;
@Service
public class DomainServiceImpl implements DomainService {
private static final Log logger = Log.get();
@Autowired
private BrightCloudConfig bcConfig;
@Autowired
private ChinazConfig chinazConfig;
@Autowired
private AppConfig appConfig;
@Autowired
private DomainDao domainDao;
@Resource
SqlSessionFactory sqlSessionFactory;
@Override
public JSONArray getCategoryInfo(List<String> domains, String username, boolean isLocal) throws Exception {
domains = ValidationUtils.getCheckedFqdns(domains);
logger.info("domain category query total size {}", domains.size());
JSONArray results = new JSONArray();
//1. 查询本地数据库
List<List<String>> partitionDomains = Lists.partition(domains, appConfig.getDbQueryBatchSize());
List<String> dBDomains = new ArrayList<>();
for (List<String> partitionDomain : partitionDomains) {
List<DomainInfo> categoryInfos = domainDao.getCategoryInfo(partitionDomain);
results.addAll(categoryInfos);
//移除DB查询结果
categoryInfos.forEach(domainInfo -> dBDomains.add(domainInfo.getDomain()));
}
domains.removeAll(dBDomains);
logger.info("domain category query DB record {}", results.size());
//2. 调用api查询
int apiResultNum = 0;
int failedQueryNum = 0;
if (!isLocal && domains.size() > 0) {
List<DomainCategory> bcResults = new ArrayList<>();
//批量查询API
BrightCloud brightCloud = new BrightCloud();
List<List<String>> apiPartitions = Lists.partition(domains, bcConfig.getMaximumQueryNum());
for (List<String> partition : apiPartitions) {
List<DomainInfo> bcDomainInfos = new ArrayList<>();
List<DomainCategory> recordsFromBcApi = brightCloud.getBrightCloudDomains(partition);
for (DomainCategory record : recordsFromBcApi) {
//查询成功的结果
if (record.getQuery_success().equals(true)) {
record.setSubmit_user(username);
bcResults.add(record);
bcDomainInfos.add(DomainInfo.builder()
.domain(record.getFqdn())
.result(Lists.newArrayList(record))
.build());
apiResultNum += 1;
} else {
failedQueryNum += 1;
}
}
if (bcResults.size() > 0) {
insertBatchDomainCategory(bcResults);
results.addAll(bcDomainInfos);
bcResults.clear();
bcDomainInfos.clear();
}
}
// 记录api调用次数
FileWriter fileWriter = new FileWriter(bcConfig.getUsereportFilePath(), true);
fileWriter.write(DateUtil.date() + "," + " domain category query brightCloud success record " + apiResultNum + ",fail query record " + failedQueryNum + "\n");
fileWriter.close();
logger.info("domain category query brightCloud success record " + apiResultNum + ",fail query record " + failedQueryNum);
}
return results;
}
@Override
public JSONArray getWhoisInfo(List<String> domains, boolean isLocal) throws Exception {
domains = ValidationUtils.getCheckedFqdns(domains);
logger.info("domain whois query total size {}", domains.size());
JSONArray results = new JSONArray();
//1.查询本地数据库
List<String> dBDomains = new ArrayList<>();
List<List<String>> partitions = Lists.partition(domains, appConfig.getDbQueryBatchSize());
for (List<String> domainWhois : partitions) {
//批量查询
List<DomainInfo> dbRecords = domainDao.getWhoisInfo(domainWhois);
results.addAll(dbRecords);
dbRecords.forEach(domainInfo -> dBDomains.add(domainInfo.getDomain()));
}
domains.removeAll(dBDomains);
logger.info("domain whois query DB record {}", results.size());
//2. 调用api
int apiResultNum = 0;
int failedQueryNum = 0;
if (!isLocal && domains.size() > 0) {
List<DomainWhois> chinazResults = new ArrayList<>();
ChinaZ chinaz = new ChinaZ();
List<List<String>> apiPartitions = Lists.partition(domains, chinazConfig.getMaximumQueryNum());
for (List<String> partition : apiPartitions) {
List<DomainInfo> chinazDomainInfos = new ArrayList<>();
List<DomainWhois> recordsFromApi = chinaz.getChinaZDomains(partition);
for (DomainWhois record : recordsFromApi) {
if (record.getQuery_success().equals(true)) {
// 查询成功的结果
chinazResults.add(record);
chinazDomainInfos.add(DomainInfo.builder()
.domain(record.getFqdn())
.result(Lists.newArrayList(record))
.build());
apiResultNum += 1;
} else { // 查询失败的结果
failedQueryNum += 1;
}
}
if (chinazResults.size() > 0) {
insertBatchDomainWhois(chinazResults);
results.addAll(chinazDomainInfos);
chinazResults.clear();
chinazDomainInfos.clear();
}
}
// 记录api调用次数
FileWriter fileWriter = new FileWriter(chinazConfig.getUsereportFilePath(), true);
fileWriter.write(DateUtil.date() + "," + " domain whois query chinaz success record " + apiResultNum + ",fail query record " + failedQueryNum + "\n");
fileWriter.close();
logger.info("domain whois query chinaz success record " + apiResultNum + ",fail query record " + failedQueryNum);
}
return results;
}
/**
* 域名分类 批量入库
*
* @param dataList
* @return
*/
public int insertBatchDomainCategory(List<DomainCategory> dataList) {
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
try {
DomainDao domainDaoBatch = sqlSession.getMapper(DomainDao.class);
dataList.stream().forEach(data -> domainDaoBatch.insertCategory(data));
sqlSession.commit();
sqlSession.clearCache();
logger.info("insert domain category data success ,data size {}", dataList.size());
} catch (Exception e) {
logger.error("domainCategory insert error, {}", e);
sqlSession.rollback();
return 0;
} finally {
sqlSession.close();
}
return dataList.size();
}
/**
* 域名whois入库
*
* @param dataList
* @return
*/
public int insertBatchDomainWhois(List<DomainWhois> dataList) {
SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
try {
DomainDao domainDaoBatch = sqlSession.getMapper(DomainDao.class);
dataList.stream().forEach(data -> domainDaoBatch.insertWhois(data));
sqlSession.commit();
sqlSession.clearCache();
logger.info("insert domain whois data success ,data size {}", dataList.size());
} catch (Exception e) {
logger.error("domainWhois insert error, {}", e);
sqlSession.rollback();
return 0;
} finally {
sqlSession.close();
}
return dataList.size();
}
}

46
src/main/java/com/mesasoft/cn/sketch/util/ConfigUtils.java Normal file
View File

@@ -0,0 +1,46 @@
package com.mesasoft.cn.sketch.util;
import org.apache.log4j.Logger;
import java.util.Properties;
public class ConfigUtils {
private static final Logger LOG = Logger.getLogger(ConfigUtils.class);
private static Properties propCommon = new Properties();
public static String getStringProperty(String key) {
return propCommon.getProperty(key);
}
public static Integer getIntProperty(String key) {
return Integer.parseInt(propCommon.getProperty(key));
}
public static Long getLongProperty(String key) {
return Long.parseLong(propCommon.getProperty(key));
}
public static Boolean getBooleanProperty(String key) {
return "true".equals(propCommon.getProperty(key).toLowerCase().trim());
}
public static String getEffectiveString(String s) {
if (!(s == null)) {
return s.length() == 0 ? null : s;
} else {
return null;
}
}
static {
try {
propCommon.load(ConfigUtils.class.getClassLoader().getResourceAsStream("application.properties"));
// propCommon.load(MariaDbConnection.class.getClassLoader().getResourceAsStream("mariadb.properties"));
} catch (Exception e) {
propCommon = null;
LOG.error("配置加载失败");
}
}
}

197
src/main/java/com/mesasoft/cn/sketch/util/FileUtils.java Normal file
View File

@@ -0,0 +1,197 @@
package com.mesasoft.cn.sketch.util;
import org.apache.log4j.Logger;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 6:11 下午
*/
public class FileUtils {
private static final Logger LOG = Logger.getLogger(FileUtils.class);
public static List<String> readTxtFileIntoStringArrList(String filePath)
{
List<String> list = new ArrayList<>();
try
{
String encoding = "GBK";
File file = new File(filePath);
if (file.isFile() && file.exists())
{ // 判断文件是否存在
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null)
{
if (!lineTxt.equals("")) {
list.add(lineTxt.trim());
}
}
bufferedReader.close();
read.close();
}
else
{
System.out.println("Can not find file: " + filePath);
}
}
catch (Exception e)
{
System.out.println("Error occurred in Function 'readTxtFileIntoStringArrList'");
e.printStackTrace();
}
return list;
}
public static List<String> getBatchLineReadIn(BufferedReader bufferedReader, int batchSize){
List<String> list = new ArrayList<>();
String lineTxt;
try{
while ((lineTxt = bufferedReader.readLine()) != null && list.size()<batchSize)
{
if (!lineTxt.equals("")) {
list.add(lineTxt.trim());
}
}
} catch (IOException e){
e.printStackTrace();
}
return list;
}
public static void createFile(File filePath, String fileName){
try {
File file = new File(filePath.toString() + "/" + fileName);
if (!filePath.exists()){
filePath.mkdirs();
}
boolean isCreate = file.createNewFile();
if (isCreate){
LOG.info("File " + fileName + " is created.");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void createFile(File file){
try {
boolean isCreate = file.createNewFile();
if (isCreate){
LOG.info("File " + file + " is created.");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static String readJsonFile(String fileName) {
String jsonStr = "";
try {
File jsonFile = new File(fileName);
FileReader fileReader = new FileReader(jsonFile);
Reader reader = new InputStreamReader(new FileInputStream(jsonFile), "utf-8");
int ch = 0;
StringBuffer sb = new StringBuffer();
while ((ch = reader.read()) != -1) {
sb.append((char) ch);
}
fileReader.close();
reader.close();
jsonStr = sb.toString();
return jsonStr;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
public static String getFileName(File file){
String[] tmp = file.toString().split("/");
String fileName = tmp[tmp.length-1];
return fileName;
}
public static void writerClose(OutputStreamWriter outWriter, OutputStream outStream) throws IOException {
assert outWriter != null;
outWriter.close();
outStream.close();
}
public static void readerClose(BufferedReader bufferedReader, InputStreamReader inputStreamReader) throws IOException {
assert inputStreamReader != null;
bufferedReader.close();
inputStreamReader.close();
}
//执行cmd命令，获取返回结果
public static String execCMD(String command) {
StringBuilder sb =new StringBuilder();
try {
Process process=Runtime.getRuntime().exec(command);
BufferedReader bufferedReader=new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while((line=bufferedReader.readLine())!=null)
{
sb.append(line+"\n");
}
} catch (Exception e) {
return e.toString();
}
return sb.toString();
}
public static Long getFileLineNum(File file){
Long num = 0L;
if (!file.exists()){
LOG.error("File not exist: " + file.toString());
} else {
String res = FileUtils.execCMD("wc -l " + file.toString());
num = Long.parseLong(res.trim().split(" ")[0]);
}
return num;
}
public static int getMaxLength(String filePath) {
int lengthDomain = 0;
try {
String encoding = "UTF-8";
File file = new File(filePath);
if (file.isFile() && file.exists()) {
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null) {
String[] split = lineTxt.split("\\.");
if (split.length > lengthDomain) {
lengthDomain = split.length;
}
}
read.close();
} else {
LOG.error("FilePath is wrong--->{" + filePath + "}<---");
}
} catch (Exception e) {
LOG.error("Get filePathData error--->{" + e + "}<---");
e.printStackTrace();
}
return lengthDomain;
}
}

70
src/main/java/com/mesasoft/cn/sketch/util/JsonTool.java Normal file
View File

@@ -0,0 +1,70 @@
package com.mesasoft.cn.sketch.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.parser.Feature;
import com.alibaba.fastjson.serializer.SerializerFeature;
import org.apache.commons.lang3.StringUtils;
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2022/1/3
* Time: 7:12 PM
* Description: No Description
*/
public class JsonTool {
private boolean isTab = true;
public String stringToJSON(String strJson) {
int tabNum = 0;
StringBuffer jsonFormat = new StringBuffer();
int length = strJson.length();
for (int i = 0; i < length; i++) {
char c = strJson.charAt(i);
if (c == '{') {
tabNum++;
jsonFormat.append(c + "\n");
jsonFormat.append(getSpaceOrTab(tabNum));
} else if (c == '}') {
tabNum--;
jsonFormat.append("\n");
jsonFormat.append(getSpaceOrTab(tabNum));
jsonFormat.append(c);
} else if (c == ',') {
jsonFormat.append(c + "\n");
jsonFormat.append(getSpaceOrTab(tabNum));
} else {
jsonFormat.append(c);
}
}
return jsonFormat.toString();
}
public String getSpaceOrTab(int tabNum) {
StringBuffer sbTab = new StringBuffer();
for (int i = 0; i < tabNum; i++) {
if (isTab) {
sbTab.append('\t');
} else {
sbTab.append(" ");
}
}
return sbTab.toString();
}
public static String prettyJson(String jsonString){
if(StringUtils.isBlank(jsonString)){
return jsonString;
}
String pretty;
try {
JSONObject object = JSONObject.parseObject(jsonString, Feature.OrderedField);
pretty = JSON.toJSONString(object, SerializerFeature.PrettyFormat, SerializerFeature.WriteMapNullValue,
SerializerFeature.WriteDateUseDateFormat);
}catch (Exception e){
return jsonString;
}
return pretty;
}
}

133
src/main/java/com/mesasoft/cn/sketch/util/MariaDBUtils.java Normal file
View File

@@ -0,0 +1,133 @@
package com.mesasoft.cn.sketch.util;
import cn.hutool.core.date.DateUtil;
import com.alibaba.druid.pool.DruidDataSourceFactory;
import me.geso.jdbcutils.JDBCUtils;
import org.apache.commons.lang3.time.StopWatch;
import javax.sql.DataSource;
import java.math.BigInteger;
import java.sql.*;
import java.util.Properties;
/**
* Druid连接池的工具类
*/
public class MariaDBUtils {
private static DataSource ds;
static {
try {
Properties pro = new Properties();
pro.load(JDBCUtils.class.getClassLoader().getResourceAsStream("druid.properties"));
ds = DruidDataSourceFactory.createDataSource(pro);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 获取连接
*/
public static Connection getConnection() throws SQLException {
return ds.getConnection();
}
/**
* 释放资源
*/
public static void close(Statement stmt, Connection conn) {
close(null, stmt, conn);
}
public static void close(ResultSet rs, Statement stmt, Connection conn) {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (stmt != null) {
try {
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
/**
* 获取连接池方法
*/
public static DataSource getDataSource() {
return ds;
}
public static void main(String[] args) {
try {
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");// 驱动包
String url = "jdbc:clickhouse://192.168.44.85:8123/tsg_galaxy_v3";// url路径
String user = "tsg_insert";// 账号
String password = "galaxy2019";// 密码
Connection connection = DriverManager.getConnection(url, user, password);
StopWatch stopWatch = new StopWatch();
stopWatch.start();
System.err.println("开始写入ck数据 ：{}");
connection.setAutoCommit(false);
PreparedStatement preparedStatement = null;
long time = DateUtil.currentSeconds();
time = time - 1;
// String sql = "INSERT INTO tsg_galaxy_v3.connection_record_log_local" +
// "(cap_ip,recv_ip,src_ip,dst_ip,src_port,dst_port,addr_type,protocol,fxo_id,link_status,dir_status,total_cs_pkts,total_sc_pkts,total_cs_bytes,total_sc_bytes,log_gen_time,aa,wv,yy,user_mask,conn_start_time,app_class,app_id,http_host,http_url,http_cookie,`http_user-agent`,http_method,http_accept,http_accept_encoding,http_referer,http_rescode,tls_sni,tls_cert,phone_num,imei,imsi) VALUES " +
// "('127.0.0.1', '192.168.44.12', 'aaa', 'String', 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 1657526027, 123456789, 123456789, 123456789, 123456789, "+ time +", 123456789, 123456789, 'String', 'String', 'String', 'String', 'String', 'String', 'String', 'String', 123456789, 'String', 'String', 123456789, 123456789, 123456789);";
// String sql ="INSERT INTO tsg_galaxy_v3.session_record\n" +
// "(common_recv_time, common_log_id, common_stream_trace_id, common_direction, common_stream_dir, common_start_time, common_end_time, common_con_duration_ms, common_establish_latency_ms, common_processing_time, common_ingestion_time, common_entrance_id, common_device_id, common_egress_link_id, common_ingress_link_id, common_isp, common_data_center, common_sled_ip, common_device_group, common_app_behavior, common_action, common_sub_action, common_policy_id, common_user_tags, common_user_region, common_client_ip, common_internal_ip, common_client_port, common_client_location, common_client_asn, common_subscriber_id, common_imei, common_imsi, common_phone_number, common_server_ip, common_external_ip, common_server_port, common_server_location, common_server_asn, common_app_id, common_userdefine_app_name, common_app_identify_info, common_app_label, common_app_surrogate_id, common_l7_protocol, common_protocol_label, common_service_category, common_service, common_l4_protocol, common_sessions, common_c2s_pkt_num, common_s2c_pkt_num, common_c2s_pkt_diff, common_s2c_pkt_diff, common_c2s_byte_num, common_s2c_byte_num, common_c2s_byte_diff, common_s2c_byte_diff, common_c2s_ipfrag_num, common_s2c_ipfrag_num, common_c2s_tcp_lostlen, common_s2c_tcp_lostlen, common_c2s_tcp_unorder_num, common_s2c_tcp_unorder_num, common_c2s_pkt_retrans, common_s2c_pkt_retrans, common_c2s_byte_retrans, common_s2c_byte_retrans, common_first_ttl, common_tcp_client_isn, common_tcp_server_isn, common_mirrored_pkts, common_mirrored_bytes, common_address_type, common_schema_type, common_device_tag, common_encapsulation, common_tunnels, common_address_list, common_has_dup_traffic, common_stream_error, common_link_info_c2s, common_link_info_s2c, common_packet_capture_file, http_url, http_host, http_domain, http_request_line, http_response_line, http_request_header, http_response_header, http_request_content, http_response_content, http_request_body, http_response_body, http_request_body_key, http_response_body_key, http_proxy_flag, http_sequence, http_snapshot, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_content_length, http_content_type, http_set_cookie, http_version, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_to_cmd, mail_from_cmd, mail_from, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_content, mail_content_charset, mail_attachment_name, mail_attachment_name_charset, mail_attachment_content, mail_eml_file, mail_snapshot, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_pinningst, ssl_intercept_state, ssl_passthrough_reason, ssl_server_side_latency, ssl_client_side_latency, ssl_server_side_version, ssl_client_side_version, ssl_cert_verify, ssl_error, ssl_con_latency_ms, ssl_ja3_fingerprint, ssl_ja3_hash, ssl_cert_issuer, ssl_cert_subject, quic_version, quic_sni, quic_user_agent, ftp_account, ftp_url, ftp_content, ftp_link_type, bgp_type, bgp_as_num, bgp_route, voip_calling_account, voip_called_account, voip_calling_number, voip_called_number, streaming_media_url, streaming_media_protocol, app_extra_info, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, rdp_encryption_method, rdp_encryption_level, rdp_certificate_permanent, rdp_certificate_count, rdp_certificate_type, rdp_requested_color_depth, rdp_desktop_height, rdp_desktop_width, rdp_client_product_id, rdp_client_name, rdp_client_version, rdp_keyboard_layout, rdp_client_channels, rdp_security_protocol, rdp_cookie)\n" +
// "VALUES("+time+", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', 0, 0, '', '', '', '', '', 0, '', 0, '', '', '', '', 0, '', '', '', '', '', '', '', '', 0, '', '', '', '', '', '', '', '', '', null, 0, '', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', '', 0, '', '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', 0, 0, '', 0, '', 0, '', '', '', '', 0, 0, '', 0, 0, '', '', 0, '', 0, '', '', '', '', '', '', '', '', '', '', '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, '', '', '', 0, '', '', 0, '', 0, 0, '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, '', '', '', '', '', '', '', '', '', '', '');\n";
preparedStatement = connection.prepareStatement("INSERT INTO tsg_galaxy_v3.connection_relation_log" +
"(start_time, src_ip, dst_ip,bytes)" +
"VALUES(?, ?, ?, ?)");
for (int i = 0; i < 10; i++) {
preparedStatement.setLong(1, time);
preparedStatement.setString(2, "127.0.0.1");
preparedStatement.setString(3, "127.0.0.1");
BigInteger bytes = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(Long.MAX_VALUE));
System.err.println(bytes);
System.err.println(bytes.bitLength());
preparedStatement.setObject(4, bytes);
preparedStatement.addBatch();
}
preparedStatement.executeBatch();
connection.commit();
stopWatch.stop();
System.err.println("总共花费时间 {}" + stopWatch.getTime());
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (SQLException throwables) {
throwables.printStackTrace();
}
}
}

59
src/main/java/com/mesasoft/cn/sketch/util/TimeUtils.java Normal file
View File

@@ -0,0 +1,59 @@
package com.mesasoft.cn.sketch.util;
import java.util.Calendar;
import java.util.Date;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 11:26 上午
*/
public class TimeUtils {
public static final Long HOUR_TO_MILLISECONDS = 3600000L;
public static final Long DAY_TO_MILLSEDONDS = 86400000L;
public static final Integer SECOND_TO_MILLSEDONDS = 1000;
/**
* 获得当前时间小时的起始（0分钟）时间
*/
public static Date getStartOfHour() {
return getStartOfHour(0);
}
public static Date getStartOfHour(Integer offset) {
Calendar ca = Calendar.getInstance();
ca.add(Calendar.HOUR, offset);
ca.set(Calendar.MINUTE, 0);
ca.set(Calendar.SECOND, 0);
ca.set(Calendar.MILLISECOND, 0);
return ca.getTime();
}
/**
* 获得当前日期的起始（0时）时间
*/
public static Date getStartOfDay() {
return getStartOfDay(0);
}
public static Date getStartOfDay(Integer bias) {
Calendar ca = Calendar.getInstance();
ca.add(Calendar.DATE, bias);
ca.set(Calendar.HOUR, -12);
ca.set(Calendar.MINUTE, 0);
ca.set(Calendar.SECOND, 0);
ca.set(Calendar.MILLISECOND, 0);
return ca.getTime();
}
public static Date getStartOfMonth() {
Calendar ca = Calendar.getInstance();
ca.set(Calendar.DATE, 1);
ca.set(Calendar.HOUR, -12);
ca.set(Calendar.MINUTE, 0);
ca.set(Calendar.SECOND, 0);
ca.set(Calendar.MILLISECOND, 0);
return ca.getTime();
}
}

207
src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java Normal file
View File

@@ -0,0 +1,207 @@
package com.mesasoft.cn.sketch.util;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import sun.net.util.IPAddressUtil;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ValidationUtils {
private static final Logger LOG = Logger.getLogger(ValidationUtils.class);
/**
* 获取二级域名
**/
private static String tldFilePath;
@Value("${sketch.tld.file}")
public void setTldFilePath(String ttlFilePath){
this.tldFilePath= ttlFilePath;
}
public static String getSecDomain(String fqdnOrUrl){
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(tldFilePath);
try {
String[] split = fqdnOrUrl.split("\\.");
String secDomain = null;
for (int i = split.length - 1; i >= 0; i--) {
int maps_index = split.length - (i + 1);
HashMap<String, String> innerMap = maps.get("map_id_" + maps_index);
HashMap<String, String> fullTop = maps.get("full");
if (!(innerMap.containsKey(split[i]))) {
String strSec = "";
for (int j = i; j < split.length; j++) {
strSec += (split[j] + ".");
}
secDomain = strSec.substring(0, strSec.length() - 1);
if (fullTop.containsKey(getTopFromSecDomain(secDomain))) {
break;
} else {
while (!fullTop.containsKey(getTopFromSecDomain(secDomain)) && getTopFromSecDomain(secDomain).contains(".")) {
secDomain = getTopFromSecDomain(secDomain);
}
break;
}
}
}
// 右匹配为顶级域名
if (secDomain == null){
secDomain = fqdnOrUrl;
}
return secDomain;
} catch (Exception e) {
LOG.error("urlDomain:" + fqdnOrUrl);
e.printStackTrace();
return "---no---return---";
}
}
public static List<String> getSecDomain(List<String> fqdnOrUrls) {
HashMap<String, HashMap<String, String>> maps = readTopDomainFile(tldFilePath);
List<String> secDomainList = new ArrayList<>();
for (String oriDomain : fqdnOrUrls) {
String secDomain = getSecDomain(oriDomain);
if (StringUtils.isNotBlank(secDomain) && !("---no---return---".equals(secDomain))) {
secDomainList.add(secDomain);
} else {
System.out.println(oriDomain);
}
}
return secDomainList;
}
public static String getTopFromSecDomain(String secDomain) {
String quFirstDian = secDomain;
if (secDomain.contains(".")) {
quFirstDian = secDomain.substring(secDomain.indexOf(".")).substring(1);
}
return quFirstDian;
}
public static HashMap<String, HashMap<String, String>> readTopDomainFile(String filePath) {
HashMap<String, HashMap<String, String>> maps = makeHashMap(filePath);
try {
String encoding = "UTF-8";
File file = new File(filePath);
if (file.isFile() && file.exists()) {
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null) {
HashMap<String, String> fullTop = maps.get("full");
fullTop.put(lineTxt, lineTxt);
maps.put("full", fullTop);
String[] split = lineTxt.split("\\.");
for (int i = split.length - 1; i >= 0; i--) {
int maps_index = split.length - (i + 1);
HashMap<String, String> innerMap = maps.get("map_id_" + maps_index);
innerMap.put(split[i], split[i]);
maps.put("map_id_" + maps_index, innerMap);
}
}
read.close();
} else {
LOG.error("TopDomainUtils>=>readTopDomainFile filePath is wrong--->{" + filePath + "}<---");
}
} catch (Exception e) {
LOG.error("TopDomainUtils>=>readTopDomainFile get filePathData error--->{" + e + "}<---");
e.printStackTrace();
}
return maps;
}
public static HashMap<String, HashMap<String, String>> makeHashMap(String filePath) {
int maxLength = FileUtils.getMaxLength(filePath);
HashMap<String, HashMap<String, String>> maps = new HashMap<String, HashMap<String, String>>();
for (int i = 0; i < maxLength; i++) {
maps.put("map_id_" + i, new HashMap<String, String>());
}
maps.put("full", new HashMap<String, String>());
return maps;
}
public static List<String> getChecked(List<String> objectList, String type){
if (type.equals("ip")){
return getCheckedIps(objectList);
}
if (type.equals("domain")){
return getCheckedFqdns(objectList);
}
LOG.error("Wrong type to be checked: " + type);
return objectList;
}
public static List<String> getCheckedFqdns(List<String> fqdns){
List<String> res = new ArrayList<>();
for (String fqdn:fqdns){
//去端口号
fqdn = fqdn.split(":")[0];
// 去重 & 校验
if (isValidDomain(fqdn) && !res.contains(fqdn)){
res.add(fqdn.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + fqdn);
}
}
return res;
}
public static List<String> getCheckedIps(List<String> ipList){
List<String> res = new ArrayList<>();
for (String ip:ipList){
//去端口号
ip = ip.split(":")[0];
// 去重 & 校验
if (isValidIp(ip) && !res.contains(ip)){
res.add(ip.toLowerCase());
} else {
LOG.debug("Bad or duplicated fqdn:" + ip);
}
}
return res;
}
public static boolean isValidIp(String ip){
boolean iPv4LiteralAddress = IPAddressUtil.isIPv4LiteralAddress(ip);
boolean iPv6LiteralAddress = IPAddressUtil.isIPv6LiteralAddress(ip);
return iPv4LiteralAddress || iPv6LiteralAddress;
}
private static boolean isValidDomain(String str)
{
String regex = "^((?!-)[A-Za-z0-9-_]"
+ "{1,63}(?<!-)\\.)"
+ "+[A-Za-z]{2,6}";
Pattern p = Pattern.compile(regex);
if (str == null) {
return false;
}
Matcher m = p.matcher(str);
return m.matches();
}
public static Integer getMatchPattern(String fqdn){
int match_pattern = 2;
if (fqdn.equals(getSecDomain(fqdn))){
match_pattern = 1; // 二级域名-右匹配
}
return match_pattern;
}
}