From bdf48f2b4f147445a7f7723f59d1acaf5819972a Mon Sep 17 00:00:00 2001 From: zhanghongqing Date: Tue, 10 Jan 2023 16:02:56 +0800 Subject: [PATCH] =?UTF-8?q?web=20sketch-=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E5=9F=9F=E5=90=8D=E6=9F=A5=E8=AF=A2=20CN-663?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/mesasoft/cn/WebSketchApplication.java | 42 ++ .../mesasoft/cn/sketch/config/AppConfig.java | 22 + .../cn/sketch/config/BrightCloudConfig.java | 26 + .../cn/sketch/config/ChinazConfig.java | 17 + .../sketch/controller/DomainController.java | 44 ++ .../cn/sketch/dao/DomainCategorySchema.java | 340 +++++++++++++ .../com/mesasoft/cn/sketch/dao/DomainDao.java | 22 + .../cn/sketch/dao/DomainWhoisSchema.java | 423 ++++++++++++++++ .../mesasoft/cn/sketch/dao/MariaDbBase.java | 70 +++ .../DomainCategoryReputationDao.java | 31 ++ .../mesasoft/cn/sketch/entity/ReturnT.java | 40 ++ .../mesasoft/cn/sketch/entity/SketchAuth.java | 12 + .../cn/sketch/entity/domain/DomainBase.java | 24 + .../sketch/entity/domain/DomainCategory.java | 235 +++++++++ .../cn/sketch/entity/domain/DomainInfo.java | 20 + .../cn/sketch/entity/domain/DomainWhois.java | 271 ++++++++++ .../mesasoft/cn/sketch/pojo/IocDarkWeb.java | 461 ++++++++++++++++++ .../cn/sketch/service/DarkWebService.java | 321 ++++++++++++ .../mesasoft/cn/sketch/service/DbService.java | 93 ++++ .../cn/sketch/service/DomainService.java | 14 + .../mesasoft/cn/sketch/service/FileQuery.java | 117 +++++ .../mesasoft/cn/sketch/service/ListQuery.java | 241 +++++++++ .../service/impl/DomainServiceImpl.java | 217 +++++++++ .../mesasoft/cn/sketch/util/ConfigUtils.java | 46 ++ .../mesasoft/cn/sketch/util/FileUtils.java | 197 ++++++++ .../com/mesasoft/cn/sketch/util/JsonTool.java | 70 +++ .../mesasoft/cn/sketch/util/MariaDBUtils.java | 133 +++++ .../mesasoft/cn/sketch/util/TimeUtils.java | 59 +++ .../cn/sketch/util/ValidationUtils.java | 207 ++++++++ 29 files changed, 3815 insertions(+) create mode 100644 src/main/java/com/mesasoft/cn/WebSketchApplication.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/config/BrightCloudConfig.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/config/ChinazConfig.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/dao/DomainCategorySchema.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/dao/DomainDao.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/dao/DomainWhoisSchema.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/dao/MariaDbBase.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/dao/src/main/java/generator/DomainCategoryReputationDao.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/entity/ReturnT.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/entity/SketchAuth.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainCategory.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainInfo.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/pojo/IocDarkWeb.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/service/DarkWebService.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/service/DbService.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/service/DomainService.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/service/FileQuery.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/service/ListQuery.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/util/ConfigUtils.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/util/FileUtils.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/util/JsonTool.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/util/MariaDBUtils.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/util/TimeUtils.java create mode 100644 src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java diff --git a/src/main/java/com/mesasoft/cn/WebSketchApplication.java b/src/main/java/com/mesasoft/cn/WebSketchApplication.java new file mode 100644 index 0000000..1ed52df --- /dev/null +++ b/src/main/java/com/mesasoft/cn/WebSketchApplication.java @@ -0,0 +1,42 @@ +package com.mesasoft.cn; + +import com.mesasoft.cn.config.TokenConfig; +import com.mesasoft.cn.modules.constant.ConfigConsts; +import com.mesasoft.cn.modules.constant.DefaultValues; +import com.spring4all.swagger.EnableSwagger2Doc; +import com.zhazhapan.config.JsonParser; +import com.zhazhapan.util.FileExecutor; +import com.zhazhapan.util.MailSender; +import com.zhazhapan.util.ReflectUtils; +import org.mybatis.spring.annotation.MapperScan; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import java.io.IOException; +import java.util.Hashtable; +import java.util.List; + +/** + * @author pantao + */ +@SpringBootApplication +@EnableSwagger2Doc +@MapperScan({"com.mesasoft.cn.dao","com.mesasoft.cn.sketch.dao"}) +@EnableTransactionManagement +public class WebSketchApplication { + + public static JsonParser settings = new JsonParser(); + + public static List> controllers; + + public static Hashtable tokens; + + public static void main(String[] args) throws IOException, ClassNotFoundException { + settings.setJsonObject(FileExecutor.read(WebSketchApplication.class.getResourceAsStream(DefaultValues.SETTING_PATH))); + MailSender.config(settings.getObjectUseEval(ConfigConsts.EMAIL_CONFIG_OF_SETTINGS)); + controllers = ReflectUtils.getClasses(DefaultValues.CONTROLLER_PACKAGE); + tokens = TokenConfig.loadToken(); + SpringApplication.run(WebSketchApplication.class, args); + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java b/src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java new file mode 100644 index 0000000..ef6aa20 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/config/AppConfig.java @@ -0,0 +1,22 @@ +package com.mesasoft.cn.sketch.config; + + +import com.mesasoft.cn.sketch.util.ConfigUtils; +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class AppConfig { + + // Mariadb + public static final String DATABASE = ConfigUtils.getStringProperty("database"); + public static final String DOMAIN_CATE_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.category"); + public static final String DOMAIN_WHOIS_TABLENAME = ConfigUtils.getStringProperty("tablename.domain.whois"); + + @Value("${db.query.batch.size}") + private Integer dbQueryBatchSize; + + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/config/BrightCloudConfig.java b/src/main/java/com/mesasoft/cn/sketch/config/BrightCloudConfig.java new file mode 100644 index 0000000..581b361 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/config/BrightCloudConfig.java @@ -0,0 +1,26 @@ +package com.mesasoft.cn.sketch.config; + + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Data +@Component +@ConfigurationProperties(prefix = "bright-cloud") +public class BrightCloudConfig { + + private String oemid; + private String deviceid; + private String uid; + private String url; + private String method; + private String queryType; + private String isa1cat; + private String isReputation; + private String isxml; + private int maximumQueryNum; + private String cateinfoFilepath; + private String usereportFilePath; + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/config/ChinazConfig.java b/src/main/java/com/mesasoft/cn/sketch/config/ChinazConfig.java new file mode 100644 index 0000000..5da0b3f --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/config/ChinazConfig.java @@ -0,0 +1,17 @@ +package com.mesasoft.cn.sketch.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Data +@Component +@ConfigurationProperties(prefix = "chinaz") +public class ChinazConfig { + + private String urlSingle; + private String urlBatch; + private String key; + private int maximumQueryNum; + private String usereportFilePath; +} diff --git a/src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java b/src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java new file mode 100644 index 0000000..4425b09 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/controller/DomainController.java @@ -0,0 +1,44 @@ +package com.mesasoft.cn.sketch.controller; + +import com.alibaba.fastjson.JSONArray; +import com.mesasoft.cn.sketch.entity.ReturnT; +import com.mesasoft.cn.sketch.entity.SketchAuth; +import com.mesasoft.cn.sketch.service.DomainService; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import java.util.List; + +@RestController +@RequestMapping("/v1/domain/") +public class DomainController { + + @Resource + private DomainService domainService; + + @GetMapping(value = "info") + @ResponseBody + public ReturnT info(SketchAuth sketchAuth) { + return ReturnT.succeed(sketchAuth.getRequestid(), "Hello WebSketch !"); + } + + @GetMapping(value = "getcategoryinfo") + @ResponseBody + public ReturnT getCategoryInfo(SketchAuth sketchAuth, @RequestParam List domain) throws Exception { + + JSONArray categoryInfo = domainService.getCategoryInfo(domain, "", true); + + return ReturnT.succeed(sketchAuth.getRequestid(), categoryInfo); + } + + @GetMapping(value = "getwhoisinfo") + @ResponseBody + public ReturnT getWhoisInfo(SketchAuth sketchAuth, @RequestParam List domain) throws Exception { + + JSONArray whoisInfo = domainService.getWhoisInfo(domain, true); + + return ReturnT.succeed(sketchAuth.getRequestid(), whoisInfo); + } + + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/dao/DomainCategorySchema.java b/src/main/java/com/mesasoft/cn/sketch/dao/DomainCategorySchema.java new file mode 100644 index 0000000..8100fc1 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/dao/DomainCategorySchema.java @@ -0,0 +1,340 @@ +package com.mesasoft.cn.sketch.dao; + +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.sketch.config.AppConfig; +import com.mesasoft.cn.sketch.util.ConfigUtils; +import com.mesasoft.cn.sketch.util.ValidationUtils; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/29 + * Time: 9:27 AM + * Description: No Description + */ +@Deprecated +public class DomainCategorySchema { + private static final String dataBase = AppConfig.DATABASE; + private static final String tableName = AppConfig.DOMAIN_CATE_TABLENAME; + + private String fqdn; + private String source; + private Boolean query_success; + private Integer match_pattern; + private Integer reputation_score; + private String reputation_level; + private Integer category_id; + private String category_name; + private String category_group; + private Integer category_conf; + private Boolean is_a1_cat; + private Integer status_code = 0; + private String submit_user =""; + + + // category schema + public DomainCategorySchema(String fqdn, + String source, + Boolean query_success, + Integer match_pattern, + Integer reputation_score, + String reputationLevel, + Integer categoryId, + String categoryName, + String categoryGroup, + Integer categoryConf, + Boolean isA1Cat,Integer statusCode, String submitUser) { + + this.fqdn = fqdn; + this.source = source; // 默认应为为brightcloud + this.query_success = query_success; + + // 没有设置match_pattern，则二级域名为右匹配，其余为全匹配 + if (match_pattern == null) { + this.match_pattern = ValidationUtils.getMatchPattern(fqdn); + } else { + this.match_pattern = match_pattern; + } + + this.reputation_score = reputation_score; + this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel); + this.category_id = categoryId; + this.category_name = ConfigUtils.getEffectiveString(categoryName); + this.category_group = ConfigUtils.getEffectiveString(categoryGroup); + this.category_conf = categoryConf; + this.is_a1_cat = isA1Cat; + this.status_code = statusCode; + this.submit_user = submitUser; + + } + public DomainCategorySchema(String fqdn, + String source, + Boolean query_success, + Integer match_pattern, + Integer reputation_score, + String reputationLevel, + Integer categoryId, + String categoryName, + String categoryGroup, + Integer categoryConf, + Boolean isA1Cat) { + + this.fqdn = fqdn; + this.source = source; // 默认应为为brightcloud + this.query_success = query_success; + + // 没有设置match_pattern，则二级域名为右匹配，其余为全匹配 + if (match_pattern == null) { + this.match_pattern = ValidationUtils.getMatchPattern(fqdn); + } else { + this.match_pattern = match_pattern; + } + + this.reputation_score = reputation_score; + this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel); + this.category_id = categoryId; + this.category_name = ConfigUtils.getEffectiveString(categoryName); + this.category_group = ConfigUtils.getEffectiveString(categoryGroup); + this.category_conf = categoryConf; + this.is_a1_cat = isA1Cat; + + } + public static void insertRecords(List categoryFiles, MariaDbBase mariaDbBase) { + for (DomainCategorySchema categoryFile : categoryFiles) { + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + categoryFile.getKeys() + ") values" + + '(' + categoryFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public void updateRecords(List categoryFiles, MariaDbBase mariaDbBase) { + for (DomainCategorySchema categoryFile : categoryFiles) { + + String resSql = "UPDATE " + dataBase + "." + + tableName + ' ' + + "SET " + categoryFile.getKeyValues() + + ", update_time = current_time() " + + " WHERE fqdn = '" + categoryFile.getFqdn() + '\''; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static List getDbRecord(List fqdns, MariaDbBase mariaDbBase, String source) throws SQLException { + String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(",")); + String sql = "SELECT * FROM " + dataBase + "." + + tableName + ' ' + + " WHERE fqdn in (" + queryFqdns + ") and source = '" + source + "'"; + + return rs2schema(mariaDbBase.querySqlExecute(sql)); + } + + public static List rs2schema(ResultSet rs) throws SQLException { + List schemaFiles = new ArrayList<>(); + while (rs.next()) { + schemaFiles.add( + new DomainCategorySchema( + rs.getString("fqdn"), + rs.getString("source"), + rs.getBoolean("query_success"), + rs.getInt("match_pattern"), + rs.getInt("reputation_score"), + rs.getString("reputation_level"), + rs.getInt("category_id"), + rs.getString("category_name"), + rs.getString("category_group"), + rs.getInt("category_conf"), + rs.getBoolean("is_a1_cat"), + rs.getInt("status_code"), + rs.getString("submit_user") + + )); + } + return schemaFiles; + } + + public static JSONObject schema2json(DomainCategorySchema schema) throws SQLException { + JSONObject jsonObject = new JSONObject(true); + jsonObject.put("fqdn", schema.getFqdn()); + jsonObject.put("source", schema.getSource()); + jsonObject.put("query_success", schema.getQuery_success()); + jsonObject.put("match_pattern", schema.getMatch_pattern()); + jsonObject.put("reputation_score", schema.getReputation_score()); + jsonObject.put("reputation_level", schema.getReputation_level()); + jsonObject.put("category_id", schema.getCategory_id()); + jsonObject.put("category_group", schema.getCategory_group()); + jsonObject.put("category_name", schema.getCategory_name()); + jsonObject.put("category_conf", schema.getCategory_conf()); + jsonObject.put("is_a1_cat", schema.getIs_a1_cat()); + jsonObject.put("status_code", schema.getStatus_code()); + jsonObject.put("submit_user", schema.getSubmit_user()); + + return jsonObject; + } + + public String getValues() { + String resString = "'" + fqdn + '\'' + + ", '" + source + '\'' + + ", " + query_success + + ", " + match_pattern + + ", " + reputation_score + + ", '" + reputation_level + '\'' + + ", " + category_id + + ", '" + category_name + '\'' + + ", '" + category_group + '\'' + + ", " + category_conf + + ", " + status_code + + ", " + submit_user + + ", " + is_a1_cat; + + return resString.replace("'null'", "null"); + } + + public String getKeys() { + String resString; + resString = "fqdn" + + ", source" + + ", query_success" + + ", match_pattern" + + ", reputation_score" + + ", reputation_level" + + ", category_id" + + ", category_name" + + ", category_group" + + ", category_conf" + + ", status_code" + + ", submit_user" + + ", is_a1_cat"; + return resString; + } + + public String getKeyValues() { + String resString = "source='" + source + '\'' + + ", query_success=" + query_success + + ", match_pattern=" + match_pattern + + ", reputation_score=" + reputation_score + + ", reputation_level='" + reputation_level + '\'' + + ", category_id=" + category_id + + ", category_name='" + category_name + '\'' + + ", category_group='" + category_group + '\'' + + ", category_conf=" + category_conf + + ", is_a1_cat=" + is_a1_cat; + + return resString.replace("'null'", "null"); + } + + public String getFqdn() { + return fqdn; + } + + public void setFqdn(String fqdn) { + this.fqdn = fqdn; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public Boolean getQuery_success() { + return query_success; + } + + public void setQuery_success(Boolean query_success) { + this.query_success = query_success; + } + + public Integer getMatch_pattern() { + return match_pattern; + } + + public void setMatch_pattern(Integer match_pattern) { + this.match_pattern = match_pattern; + } + + public Integer getReputation_score() { + return reputation_score; + } + + public void setReputation_score(Integer reputation_score) { + this.reputation_score = reputation_score; + } + + public String getReputation_level() { + return reputation_level; + } + + public void setReputation_level(String reputation_level) { + this.reputation_level = reputation_level; + } + + public Integer getCategory_id() { + return category_id; + } + + public void setCategory_id(Integer category_id) { + this.category_id = category_id; + } + + public String getCategory_name() { + return category_name; + } + + public void setCategory_name(String category_name) { + this.category_name = category_name; + } + + public String getCategory_group() { + return category_group; + } + + public void setCategory_group(String category_group) { + this.category_group = category_group; + } + + public Integer getCategory_conf() { + return category_conf; + } + + public void setCategory_conf(Integer category_conf) { + this.category_conf = category_conf; + } + + public Boolean getIs_a1_cat() { + return is_a1_cat; + } + + public void setIs_a1_cat(Boolean is_a1_cat) { + this.is_a1_cat = is_a1_cat; + } + + public Integer getStatus_code() { + return status_code; + } + + public void setStatus_code(Integer status_code) { + this.status_code = status_code; + } + + public String getSubmit_user() { + return submit_user; + } + + public void setSubmit_user(String submit_user) { + this.submit_user = submit_user; + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/dao/DomainDao.java b/src/main/java/com/mesasoft/cn/sketch/dao/DomainDao.java new file mode 100644 index 0000000..e05bc31 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/dao/DomainDao.java @@ -0,0 +1,22 @@ +package com.mesasoft.cn.sketch.dao; + +import com.mesasoft.cn.sketch.entity.domain.DomainCategory; +import com.mesasoft.cn.sketch.entity.domain.DomainInfo; +import com.mesasoft.cn.sketch.entity.domain.DomainWhois; +import org.apache.ibatis.annotations.Param; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public interface DomainDao { + + + List getCategoryInfo(@Param("domain") List domain); + + List getWhoisInfo(@Param("domain") List domain); + + void insertCategory(DomainCategory categories); + + void insertWhois(DomainWhois data); +} diff --git a/src/main/java/com/mesasoft/cn/sketch/dao/DomainWhoisSchema.java b/src/main/java/com/mesasoft/cn/sketch/dao/DomainWhoisSchema.java new file mode 100644 index 0000000..b5ddfae --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/dao/DomainWhoisSchema.java @@ -0,0 +1,423 @@ +package com.mesasoft.cn.sketch.dao; + +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.sketch.config.AppConfig; +import com.mesasoft.cn.sketch.util.ConfigUtils; +import com.mesasoft.cn.sketch.util.ValidationUtils; + +import java.sql.Date; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/29 + * Time: 9:27 AM + * Description: No Description + */ +@Deprecated +public class DomainWhoisSchema { + private static final String dataBase = AppConfig.DATABASE; + private static final String tableName = AppConfig.DOMAIN_WHOIS_TABLENAME; + + private String fqdn; + private String source; + private Boolean query_success; + private Integer match_pattern; + private String whois_domain; + private Timestamp whois_update_date; + private Timestamp whois_create_date; + private Timestamp whois_expire_date; + private String whois_email; + private String whois_ns; + private String whois_registrar_name; + private String whois_registrant_org; + private String whois_registrant_name; + private String whois_registrant_street; + private String whois_registrant_city; + private String whois_registrant_state; + private String whois_registrant_postcode; + private String whois_registrant_country; + private String whois_registrant_phone; + + + // category schema + public DomainWhoisSchema(String fqdn, + String source, + Integer match_pattern, + Boolean query_success, + String whoisDomain, + Date whoisUpdateDate, + Date whoisCreateDate, + Date whoisExpireDate, + String whoisEmail, + String whoisNs, + String whoisRegistrarName, + String whoisRegistrantOrg, + String whoisRegistrantName, + String whoisRegistrantStreet, + String whoisRegistrantCity, + String whoisRegistrantState, + String whoisRegistrantPostcode, + String whoisRegistrantCountry, + String whoisRegistrantPhone + ) { + + this.fqdn = fqdn; + this.source = source; + + // 没有设置match_pattern，则二级域名为右匹配，其余为全匹配 + if (match_pattern == null) { + this.match_pattern = ValidationUtils.getMatchPattern(fqdn); + } else { + this.match_pattern = match_pattern; + } + + this.query_success = query_success; + this.whois_domain = ConfigUtils.getEffectiveString(whoisDomain); + this.whois_update_date = whoisUpdateDate == null ? null : new Timestamp(whoisUpdateDate.getTime()); + this.whois_create_date = whoisCreateDate == null ? null : new Timestamp(whoisCreateDate.getTime()); + this.whois_expire_date = whoisExpireDate == null ? null : new Timestamp(whoisExpireDate.getTime()); + this.whois_email = ConfigUtils.getEffectiveString(whoisEmail); + this.whois_ns = ConfigUtils.getEffectiveString(whoisNs); + this.whois_registrar_name = ConfigUtils.getEffectiveString(whoisRegistrarName); + this.whois_registrant_org = ConfigUtils.getEffectiveString(whoisRegistrantOrg); + this.whois_registrant_name = ConfigUtils.getEffectiveString(whoisRegistrantName); + this.whois_registrant_street = ConfigUtils.getEffectiveString(whoisRegistrantStreet); + this.whois_registrant_city = ConfigUtils.getEffectiveString(whoisRegistrantCity); + this.whois_registrant_state = ConfigUtils.getEffectiveString(whoisRegistrantState); + this.whois_registrant_postcode = ConfigUtils.getEffectiveString(whoisRegistrantPostcode); + this.whois_registrant_country = ConfigUtils.getEffectiveString(whoisRegistrantCountry); + this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone); + } + + public static void insertRecords(List whoisFiles, MariaDbBase mariaDbBase) { + for (DomainWhoisSchema whoisFile : whoisFiles) { + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + whoisFile.getKeys() + ") values" + + '(' + whoisFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static String insertSql(List whoisFiles) { + DomainWhoisSchema whoisFile = whoisFiles.get(0); + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + whoisFile.getKeys() + ") values" + + '(' + whoisFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + return resSql; + } + + public static void updateRecords(List categoryFiles, MariaDbBase mariaDbBase) { + for (DomainWhoisSchema categoryFile : categoryFiles) { + + String resSql = "UPDATE " + dataBase + "." + + tableName + ' ' + + "SET " + categoryFile.getKeyValues() + + ", update_time = current_time() " + + " WHERE fqdn = '" + categoryFile.getFqdn() + '\''; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static List getDbRecord(List fqdns, MariaDbBase mariaDbBase, String source) throws SQLException { + String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(",")); + String sql = "SELECT * FROM " + dataBase + "." + + tableName + ' ' + + " WHERE fqdn in (" + queryFqdns + ") "; + + return rs2schema(mariaDbBase.querySqlExecute(sql)); + } + + public String getValues() { + String resString = "'" + fqdn + '\'' + + ", '" + source + '\'' + + ", " + query_success + + ", " + match_pattern + + ", '" + whois_domain + '\'' + + ", '" + whois_update_date + '\'' + + ", '" + whois_create_date + '\'' + + ", '" + whois_expire_date + '\'' + + ", '" + whois_email + '\'' + + ", '" + whois_ns + '\'' + + ", '" + whois_registrar_name + '\'' + + ", '" + whois_registrant_org + '\'' + + ", '" + whois_registrant_name + '\'' + + ", '" + whois_registrant_street + '\'' + + ", '" + whois_registrant_city + '\'' + + ", '" + whois_registrant_state + '\'' + + ", '" + whois_registrant_postcode + '\'' + + ", '" + whois_registrant_country + '\'' + + ", '" + whois_registrant_phone + '\''; + + return resString.replace("'null'", "null"); + } + + public static List rs2schema(ResultSet rs) throws SQLException { + List schemaFiles = new ArrayList<>(); + while (rs.next()) { + schemaFiles.add( + new DomainWhoisSchema( + rs.getString("fqdn"), + rs.getString("source"), + rs.getInt("match_pattern"), + rs.getBoolean("query_success"), + rs.getString("whois_domain"), + (Date) rs.getDate("whois_update_date"), + (Date) rs.getDate("whois_create_date"), + (Date) rs.getDate("whois_expire_date"), + rs.getString("whois_email"), + rs.getString("whois_ns"), + rs.getString("whois_registrar_name"), + rs.getString("whois_registrant_org"), + rs.getString("whois_registrant_name"), + rs.getString("whois_registrant_street"), + rs.getString("whois_registrant_city"), + rs.getString("whois_registrant_state"), + rs.getString("whois_registrant_postcode"), + rs.getString("whois_registrant_country"), + rs.getString("whois_registrant_phone") + )); + } + return schemaFiles; + } + + public static JSONObject schema2json(DomainWhoisSchema schema) throws SQLException { + JSONObject jsonObject = new JSONObject(true); + jsonObject.put("fqdn", schema.getFqdn()); + jsonObject.put("source", schema.getSource()); + jsonObject.put("match_pattern", schema.getMatch_pattern()); + jsonObject.put("query_success", schema.getQuery_success()); + jsonObject.put("whois_domain", schema.getWhois_domain()); + jsonObject.put("whois_update_date", schema.getWhois_update_date()); + jsonObject.put("whois_create_date", schema.getWhois_create_date()); + jsonObject.put("whois_expire_date", schema.getWhois_expire_date()); + jsonObject.put("whois_email", schema.getWhois_email()); + jsonObject.put("whois_ns", schema.getWhois_ns()); + jsonObject.put("whois_registrar_name", schema.getWhois_registrar_name()); + jsonObject.put("whois_registrant_org", schema.getWhois_registrant_org()); + jsonObject.put("whois_registrant_name", schema.getWhois_registrant_name()); + jsonObject.put("whois_registrant_street", schema.getWhois_registrant_street()); + jsonObject.put("whois_registrant_city", schema.getWhois_registrant_city()); + jsonObject.put("whois_registrant_state", schema.getWhois_registrant_state()); + jsonObject.put("whois_registrant_postcode", schema.getWhois_registrant_postcode()); + jsonObject.put("whois_registrant_country", schema.getWhois_registrant_country()); + jsonObject.put("whois_registrant_phone", schema.getWhois_registrant_phone()); + + return jsonObject; + } + + public String getKeys() { + String resString; + resString = "fqdn" + + ", source" + + ", query_success" + + ", match_pattern" + + ", whois_domain" + + ", whois_update_date" + + ", whois_create_date" + + ", whois_expire_date" + + ", whois_email" + + ", whois_ns" + + ", whois_registrar_name" + + ", whois_registrant_org" + + ", whois_registrant_name" + + ", whois_registrant_street" + + ", whois_registrant_city" + + ", whois_registrant_state" + + ", whois_registrant_postcode" + + ", whois_registrant_country" + + ", whois_registrant_phone"; + return resString; + } + + public String getKeyValues() { + String resString = "query_success=" + query_success + + ", source='" + source + '\'' + + ", match_pattern=" + match_pattern + + ", whois_domain='" + whois_domain + '\'' + + ", whois_update_date='" + whois_update_date + '\'' + + ", whois_create_date='" + whois_create_date + '\'' + + ", whois_expire_date='" + whois_expire_date + '\'' + + ", whois_email='" + whois_email + '\'' + + ", whois_ns='" + whois_ns + '\'' + + ", whois_registrar_name='" + whois_registrar_name + '\'' + + ", whois_registrant_org='" + whois_registrant_org + '\'' + + ", whois_registrant_name='" + whois_registrant_name + '\'' + + ", whois_registrant_street='" + whois_registrant_street + '\'' + + ", whois_registrant_city='" + whois_registrant_city + '\'' + + ", whois_registrant_state='" + whois_registrant_state + '\'' + + ", whois_registrant_postcode='" + whois_registrant_postcode + '\'' + + ", whois_registrant_country='" + whois_registrant_country + '\'' + + ", whois_registrant_phone='" + whois_registrant_phone + '\''; + + return resString.replace("'null'", "null"); + } + + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getFqdn() { + return fqdn; + } + + public void setFqdn(String fqdn) { + this.fqdn = fqdn; + } + + public Boolean getQuery_success() { + return query_success; + } + + public void setQuery_success(Boolean query_success) { + this.query_success = query_success; + } + + public Integer getMatch_pattern() { + return match_pattern; + } + + public void setMatch_pattern(Integer match_pattern) { + this.match_pattern = match_pattern; + } + + public String getWhois_domain() { + return whois_domain; + } + + public void setWhois_domain(String whois_domain) { + this.whois_domain = whois_domain; + } + + public Timestamp getWhois_update_date() { + return whois_update_date; + } + + public void setWhois_update_date(Timestamp whois_update_date) { + this.whois_update_date = whois_update_date; + } + + public Timestamp getWhois_create_date() { + return whois_create_date; + } + + public void setWhois_create_date(Timestamp whois_create_date) { + this.whois_create_date = whois_create_date; + } + + public Timestamp getWhois_expire_date() { + return whois_expire_date; + } + + public void setWhois_expire_date(Timestamp whois_expire_date) { + this.whois_expire_date = whois_expire_date; + } + + public String getWhois_email() { + return whois_email; + } + + public void setWhois_email(String whois_email) { + this.whois_email = whois_email; + } + + public String getWhois_ns() { + return whois_ns; + } + + public void setWhois_ns(String whois_ns) { + this.whois_ns = whois_ns; + } + + public String getWhois_registrar_name() { + return whois_registrar_name; + } + + public void setWhois_registrar_name(String whois_registrar_name) { + this.whois_registrar_name = whois_registrar_name; + } + + public String getWhois_registrant_org() { + return whois_registrant_org; + } + + public void setWhois_registrant_org(String whois_registrant_org) { + this.whois_registrant_org = whois_registrant_org; + } + + public String getWhois_registrant_name() { + return whois_registrant_name; + } + + public void setWhois_registrant_name(String whois_registrant_name) { + this.whois_registrant_name = whois_registrant_name; + } + + public String getWhois_registrant_street() { + return whois_registrant_street; + } + + public void setWhois_registrant_street(String whois_registrant_street) { + this.whois_registrant_street = whois_registrant_street; + } + + public String getWhois_registrant_city() { + return whois_registrant_city; + } + + public void setWhois_registrant_city(String whois_registrant_city) { + this.whois_registrant_city = whois_registrant_city; + } + + public String getWhois_registrant_state() { + return whois_registrant_state; + } + + public void setWhois_registrant_state(String whois_registrant_state) { + this.whois_registrant_state = whois_registrant_state; + } + + public String getWhois_registrant_postcode() { + return whois_registrant_postcode; + } + + public void setWhois_registrant_postcode(String whois_registrant_postcode) { + this.whois_registrant_postcode = whois_registrant_postcode; + } + + public String getWhois_registrant_country() { + return whois_registrant_country; + } + + public void setWhois_registrant_country(String whois_registrant_country) { + this.whois_registrant_country = whois_registrant_country; + } + + public String getWhois_registrant_phone() { + return whois_registrant_phone; + } + + public void setWhois_registrant_phone(String whois_registrant_phone) { + this.whois_registrant_phone = whois_registrant_phone; + } +} + diff --git a/src/main/java/com/mesasoft/cn/sketch/dao/MariaDbBase.java b/src/main/java/com/mesasoft/cn/sketch/dao/MariaDbBase.java new file mode 100644 index 0000000..0fafabb --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/dao/MariaDbBase.java @@ -0,0 +1,70 @@ +package com.mesasoft.cn.sketch.dao; + +import com.mesasoft.cn.sketch.util.TimeUtils; +import org.apache.log4j.Logger; + +import java.sql.*; +import java.util.Date; +import java.util.Properties; + +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/28 + * Time: 2:56 PM + * Description: No Description + */ +public class MariaDbBase { + + private static final Logger LOG = Logger.getLogger(MariaDbBase.class); + private static final Properties props = new Properties(); + + private Statement statement; + + public MariaDbBase(Connection conn, Statement stat) { + statement = stat; + } + public MariaDbBase() { + } + /** + * 执行写入sql + */ + public void writeSqlExecute(String sql){ + try { + statement.executeUpdate(sql); + } catch (SQLIntegrityConstraintViolationException e){ + LOG.error("Duplicated entry for key 'PRIMARY'"); + } catch (SQLException exception) { + LOG.error("Sql : " + sql); + exception.printStackTrace(); + } + } + + /** + * 执行查询sql + */ + public ResultSet querySqlExecute(String sql){ + ResultSet set = null; + try { + set = statement.executeQuery(sql); + } catch (SQLException exception) { + exception.printStackTrace(); + } + return set; + } + + + + /** + * TODO: getUnlabeledRecord() 考虑多个来源的情况 + */ + + /** + * 获得过期时间, 当前时间的expiredRangeDays天之前的日期为过期日期 + */ + public static Date getExpiredTime(int expiredRangeDays){ + return new Timestamp(TimeUtils.getStartOfDay(-expiredRangeDays).getTime()); + } + + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/dao/src/main/java/generator/DomainCategoryReputationDao.java b/src/main/java/com/mesasoft/cn/sketch/dao/src/main/java/generator/DomainCategoryReputationDao.java new file mode 100644 index 0000000..a19a3b5 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/dao/src/main/java/generator/DomainCategoryReputationDao.java @@ -0,0 +1,31 @@ +/* +package generator; + +import generator.DomainCategoryReputation; +import generator.DomainCategoryReputationExample; +import java.util.List; +import org.apache.ibatis.annotations.Param; + +public interface DomainCategoryReputationDao { + long countByExample(DomainCategoryReputationExample example); + + int deleteByExample(DomainCategoryReputationExample example); + + int deleteByPrimaryKey(Integer id); + + int insert(DomainCategoryReputation record); + + int insertSelective(DomainCategoryReputation record); + + List selectByExample(DomainCategoryReputationExample example); + + DomainCategoryReputation selectByPrimaryKey(Integer id); + + int updateByExampleSelective(@Param("record") DomainCategoryReputation record, @Param("example") DomainCategoryReputationExample example); + + int updateByExample(@Param("record") DomainCategoryReputation record, @Param("example") DomainCategoryReputationExample example); + + int updateByPrimaryKeySelective(DomainCategoryReputation record); + + int updateByPrimaryKey(DomainCategoryReputation record); +}*/ diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/ReturnT.java b/src/main/java/com/mesasoft/cn/sketch/entity/ReturnT.java new file mode 100644 index 0000000..09089dc --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/entity/ReturnT.java @@ -0,0 +1,40 @@ +package com.mesasoft.cn.sketch.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +import java.io.Serializable; + +@Data +@ToString +@NoArgsConstructor +@AllArgsConstructor +public class ReturnT implements Serializable { + + public static final int SUCCESS_STATUS = 200; + public static final int FAIL_STATUS = 500; + + + private int status; + private String requestid; + private String message; + private T data; + + public static ReturnT succeed(String requestid, T data) { + + return new ReturnT(SUCCESS_STATUS, requestid, null, data); + } + + public static ReturnT failed(String requestid, String message) { + + return new ReturnT(FAIL_STATUS, requestid, message, null); + } + + public static ReturnT failed(int status, String requestid, String message) { + + return new ReturnT(status, requestid, message, null); + } +} + diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/SketchAuth.java b/src/main/java/com/mesasoft/cn/sketch/entity/SketchAuth.java new file mode 100644 index 0000000..2ffa26b --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/entity/SketchAuth.java @@ -0,0 +1,12 @@ +package com.mesasoft.cn.sketch.entity; + +import lombok.Data; + +@Data +public class SketchAuth { + + private String omeid; + private String deviceid; + private String uid; + private String requestid; +} diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java new file mode 100644 index 0000000..90466cb --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainBase.java @@ -0,0 +1,24 @@ +package com.mesasoft.cn.sketch.entity.domain; + +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.Data; + +import java.util.Date; + +@Data +public class DomainBase { + + Integer id; + String fqdn; + String source; + Boolean query_success; + Integer match_pattern; + + Integer status_code = 0; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd") + Date create_time; + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:dd") + Date update_time; +} diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainCategory.java b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainCategory.java new file mode 100644 index 0000000..2d6b690 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainCategory.java @@ -0,0 +1,235 @@ +package com.mesasoft.cn.sketch.entity.domain; + +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.sketch.util.ConfigUtils; +import com.mesasoft.cn.sketch.util.ValidationUtils; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/29 + * Time: 9:27 AM + * Description: No Description + */ + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class DomainCategory extends DomainBase{ + + + private Integer reputation_score; + private String reputation_level; + private Integer category_id; + private String category_name; + private String category_group; + private Integer category_conf; + private Boolean is_a1_cat; + + private String submit_user ="''"; + + + // category schema + public DomainCategory(String fqdn, + String source, + Boolean query_success, + Integer match_pattern, + Integer reputation_score, + String reputationLevel, + Integer categoryId, + String categoryName, + String categoryGroup, + Integer categoryConf, + Boolean isA1Cat, Integer statusCode, String submitUser) { + this.fqdn = fqdn; + this.source = source; // 默认应为为brightcloud + this.query_success = query_success; + + // 没有设置match_pattern，则二级域名为右匹配，其余为全匹配 + if (match_pattern == null) { + this.match_pattern = ValidationUtils.getMatchPattern(fqdn); + } else { + this.match_pattern = match_pattern; + } + + this.reputation_score = reputation_score; + this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel); + this.category_id = categoryId; + this.category_name = ConfigUtils.getEffectiveString(categoryName); + this.category_group = ConfigUtils.getEffectiveString(categoryGroup); + this.category_conf = categoryConf; + this.is_a1_cat = isA1Cat; + this.status_code = statusCode; + this.submit_user = submitUser; + + } + public DomainCategory(String fqdn, + String source, + Boolean query_success, + Integer match_pattern, + Integer reputation_score, + String reputationLevel, + Integer categoryId, + String categoryName, + String categoryGroup, + Integer categoryConf, + Boolean isA1Cat) { + + this.fqdn = fqdn; + this.source = source; // 默认应为为brightcloud + this.query_success = query_success; + + // 没有设置match_pattern，则二级域名为右匹配，其余为全匹配 + if (match_pattern == null) { + this.match_pattern = ValidationUtils.getMatchPattern(fqdn); + } else { + this.match_pattern = match_pattern; + } + + this.reputation_score = reputation_score; + this.reputation_level = ConfigUtils.getEffectiveString(reputationLevel); + this.category_id = categoryId; + this.category_name = ConfigUtils.getEffectiveString(categoryName); + this.category_group = ConfigUtils.getEffectiveString(categoryGroup); + this.category_conf = categoryConf; + this.is_a1_cat = isA1Cat; + + } +/* public static void insertRecords(List categoryFiles, MariaDbBase mariaDbBase) { + for (DomainCategory categoryFile : categoryFiles) { + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + categoryFile.getKeys() + ") values" + + '(' + categoryFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public void updateRecords(List categoryFiles, MariaDbBase mariaDbBase) { + for (DomainCategory categoryFile : categoryFiles) { + + String resSql = "UPDATE " + dataBase + "." + + tableName + ' ' + + "SET " + categoryFile.getKeyValues() + + ", update_time = current_time() " + + " WHERE fqdn = '" + categoryFile.getFqdn() + '\''; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static List getDbRecord(List fqdns, MariaDbBase mariaDbBase, String source) throws SQLException { + String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(",")); + String sql = "SELECT * FROM " + dataBase + "." + + tableName + ' ' + + " WHERE fqdn in (" + queryFqdns + ") and source = '" + source + "'"; + + return rs2schema(mariaDbBase.querySqlExecute(sql)); + }*/ + + public static List rs2schema(ResultSet rs) throws SQLException { + List schemaFiles = new ArrayList<>(); + while (rs.next()) { + schemaFiles.add( + new DomainCategory( + rs.getString("fqdn"), + rs.getString("source"), + rs.getBoolean("query_success"), + rs.getInt("match_pattern"), + rs.getInt("reputation_score"), + rs.getString("reputation_level"), + rs.getInt("category_id"), + rs.getString("category_name"), + rs.getString("category_group"), + rs.getInt("category_conf"), + rs.getBoolean("is_a1_cat"), + rs.getInt("status_code"), + rs.getString("submit_user") + + )); + } + return schemaFiles; + } + + public static JSONObject schema2json(DomainCategory schema) throws SQLException { + JSONObject jsonObject = new JSONObject(true); + jsonObject.put("fqdn", schema.getFqdn()); + jsonObject.put("source", schema.getSource()); + jsonObject.put("query_success", schema.getQuery_success()); + jsonObject.put("match_pattern", schema.getMatch_pattern()); + jsonObject.put("reputation_score", schema.getReputation_score()); + jsonObject.put("reputation_level", schema.getReputation_level()); + jsonObject.put("category_id", schema.getCategory_id()); + jsonObject.put("category_group", schema.getCategory_group()); + jsonObject.put("category_name", schema.getCategory_name()); + jsonObject.put("category_conf", schema.getCategory_conf()); + jsonObject.put("is_a1_cat", schema.getIs_a1_cat()); + jsonObject.put("status_code", schema.getStatus_code()); + jsonObject.put("submit_user", schema.getSubmit_user()); + + return jsonObject; + } + + /*public String getValues() { + String resString = "'" + fqdn + '\'' + + ", '" + source + '\'' + + ", " + query_success + + ", " + match_pattern + + ", " + reputation_score + + ", '" + reputation_level + '\'' + + ", " + category_id + + ", '" + category_name + '\'' + + ", '" + category_group + '\'' + + ", " + category_conf + + ", " + status_code + + ", " + submit_user + + ", " + is_a1_cat; + + return resString.replace("'null'", "null"); + } + + public String getKeys() { + String resString; + resString = "fqdn" + + ", source" + + ", query_success" + + ", match_pattern" + + ", reputation_score" + + ", reputation_level" + + ", category_id" + + ", category_name" + + ", category_group" + + ", category_conf" + + ", status_code" + + ", submit_user" + + ", is_a1_cat"; + return resString; + } + + public String getKeyValues() { + String resString = "source='" + source + '\'' + + ", query_success=" + query_success + + ", match_pattern=" + match_pattern + + ", reputation_score=" + reputation_score + + ", reputation_level='" + reputation_level + '\'' + + ", category_id=" + category_id + + ", category_name='" + category_name + '\'' + + ", category_group='" + category_group + '\'' + + ", category_conf=" + category_conf + + ", is_a1_cat=" + is_a1_cat; + + return resString.replace("'null'", "null"); + }*/ + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainInfo.java b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainInfo.java new file mode 100644 index 0000000..dc5a95f --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainInfo.java @@ -0,0 +1,20 @@ +package com.mesasoft.cn.sketch.entity.domain; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class DomainInfo { + + private String domain; + private List result; + + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java new file mode 100644 index 0000000..a59b42b --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/entity/domain/DomainWhois.java @@ -0,0 +1,271 @@ +package com.mesasoft.cn.sketch.entity.domain; + +import com.alibaba.fastjson.JSONObject; +import com.mesasoft.cn.sketch.util.ConfigUtils; +import com.mesasoft.cn.sketch.util.ValidationUtils; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.sql.Date; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; + +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/29 + * Time: 9:27 AM + * Description: No Description + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class DomainWhois extends DomainBase{ + + + + private String whois_domain; + private Timestamp whois_update_date; + private Timestamp whois_create_date; + private Timestamp whois_expire_date; + private String whois_email; + private String whois_ns; + private String whois_registrar_name; + private String whois_registrant_org; + private String whois_registrant_name; + private String whois_registrant_street; + private String whois_registrant_city; + private String whois_registrant_state; + private String whois_registrant_postcode; + private String whois_registrant_country; + private String whois_registrant_phone; + + + // category schema + public DomainWhois(String fqdn, + String source, + Integer match_pattern, + Boolean query_success, + String whoisDomain, + Date whoisUpdateDate, + Date whoisCreateDate, + Date whoisExpireDate, + String whoisEmail, + String whoisNs, + String whoisRegistrarName, + String whoisRegistrantOrg, + String whoisRegistrantName, + String whoisRegistrantStreet, + String whoisRegistrantCity, + String whoisRegistrantState, + String whoisRegistrantPostcode, + String whoisRegistrantCountry, + String whoisRegistrantPhone + ) { + + this.fqdn = fqdn; + this.source = source; + + // 没有设置match_pattern，则二级域名为右匹配，其余为全匹配 + if (match_pattern == null) { + this.match_pattern = ValidationUtils.getMatchPattern(fqdn); + } else { + this.match_pattern = match_pattern; + } + + this.query_success = query_success; + this.whois_domain = ConfigUtils.getEffectiveString(whoisDomain); + this.whois_update_date = whoisUpdateDate == null ? null : new Timestamp(whoisUpdateDate.getTime()); + this.whois_create_date = whoisCreateDate == null ? null : new Timestamp(whoisCreateDate.getTime()); + this.whois_expire_date = whoisExpireDate == null ? null : new Timestamp(whoisExpireDate.getTime()); + this.whois_email = ConfigUtils.getEffectiveString(whoisEmail); + this.whois_ns = ConfigUtils.getEffectiveString(whoisNs); + this.whois_registrar_name = ConfigUtils.getEffectiveString(whoisRegistrarName); + this.whois_registrant_org = ConfigUtils.getEffectiveString(whoisRegistrantOrg); + this.whois_registrant_name = ConfigUtils.getEffectiveString(whoisRegistrantName); + this.whois_registrant_street = ConfigUtils.getEffectiveString(whoisRegistrantStreet); + this.whois_registrant_city = ConfigUtils.getEffectiveString(whoisRegistrantCity); + this.whois_registrant_state = ConfigUtils.getEffectiveString(whoisRegistrantState); + this.whois_registrant_postcode = ConfigUtils.getEffectiveString(whoisRegistrantPostcode); + this.whois_registrant_country = ConfigUtils.getEffectiveString(whoisRegistrantCountry); + this.whois_registrant_phone = ConfigUtils.getEffectiveString(whoisRegistrantPhone); + } + + /* public static void insertRecords(List whoisFiles, MariaDbBase mariaDbBase) { + for (DomainWhois whoisFile : whoisFiles) { + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + whoisFile.getKeys() + ") values" + + '(' + whoisFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static String insertSql(List whoisFiles) { + DomainWhois whoisFile = whoisFiles.get(0); + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + whoisFile.getKeys() + ") values" + + '(' + whoisFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + return resSql; + } + + public static void updateRecords(List categoryFiles, MariaDbBase mariaDbBase) { + for (DomainWhois categoryFile : categoryFiles) { + + String resSql = "UPDATE " + dataBase + "." + + tableName + ' ' + + "SET " + categoryFile.getKeyValues() + + ", update_time = current_time() " + + " WHERE fqdn = '" + categoryFile.getFqdn() + '\''; + resSql = resSql.replace("'null'", "null"); + + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static List getDbRecord(List fqdns, MariaDbBase mariaDbBase, String source) throws SQLException { + String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(",")); + String sql = "SELECT * FROM " + dataBase + "." + + tableName + ' ' + + " WHERE fqdn in (" + queryFqdns + ") "; + + return rs2schema(mariaDbBase.querySqlExecute(sql)); + }*/ + + public String getValues() { + String resString = "'" + fqdn + '\'' + + ", '" + source + '\'' + + ", " + query_success + + ", " + match_pattern + + ", '" + whois_domain + '\'' + + ", '" + whois_update_date + '\'' + + ", '" + whois_create_date + '\'' + + ", '" + whois_expire_date + '\'' + + ", '" + whois_email + '\'' + + ", '" + whois_ns + '\'' + + ", '" + whois_registrar_name + '\'' + + ", '" + whois_registrant_org + '\'' + + ", '" + whois_registrant_name + '\'' + + ", '" + whois_registrant_street + '\'' + + ", '" + whois_registrant_city + '\'' + + ", '" + whois_registrant_state + '\'' + + ", '" + whois_registrant_postcode + '\'' + + ", '" + whois_registrant_country + '\'' + + ", '" + whois_registrant_phone + '\''; + + return resString.replace("'null'", "null"); + } + + public static List rs2schema(ResultSet rs) throws SQLException { + List schemaFiles = new ArrayList<>(); + while (rs.next()) { + schemaFiles.add( + new DomainWhois( + rs.getString("fqdn"), + rs.getString("source"), + rs.getInt("match_pattern"), + rs.getBoolean("query_success"), + rs.getString("whois_domain"), + (Date) rs.getDate("whois_update_date"), + (Date) rs.getDate("whois_create_date"), + (Date) rs.getDate("whois_expire_date"), + rs.getString("whois_email"), + rs.getString("whois_ns"), + rs.getString("whois_registrar_name"), + rs.getString("whois_registrant_org"), + rs.getString("whois_registrant_name"), + rs.getString("whois_registrant_street"), + rs.getString("whois_registrant_city"), + rs.getString("whois_registrant_state"), + rs.getString("whois_registrant_postcode"), + rs.getString("whois_registrant_country"), + rs.getString("whois_registrant_phone") + )); + } + return schemaFiles; + } + + public static JSONObject schema2json(DomainWhois schema) throws SQLException { + JSONObject jsonObject = new JSONObject(true); + jsonObject.put("fqdn", schema.getFqdn()); + jsonObject.put("source", schema.getSource()); + jsonObject.put("match_pattern", schema.getMatch_pattern()); + jsonObject.put("query_success", schema.getQuery_success()); + jsonObject.put("whois_domain", schema.getWhois_domain()); + jsonObject.put("whois_update_date", schema.getWhois_update_date()); + jsonObject.put("whois_create_date", schema.getWhois_create_date()); + jsonObject.put("whois_expire_date", schema.getWhois_expire_date()); + jsonObject.put("whois_email", schema.getWhois_email()); + jsonObject.put("whois_ns", schema.getWhois_ns()); + jsonObject.put("whois_registrar_name", schema.getWhois_registrar_name()); + jsonObject.put("whois_registrant_org", schema.getWhois_registrant_org()); + jsonObject.put("whois_registrant_name", schema.getWhois_registrant_name()); + jsonObject.put("whois_registrant_street", schema.getWhois_registrant_street()); + jsonObject.put("whois_registrant_city", schema.getWhois_registrant_city()); + jsonObject.put("whois_registrant_state", schema.getWhois_registrant_state()); + jsonObject.put("whois_registrant_postcode", schema.getWhois_registrant_postcode()); + jsonObject.put("whois_registrant_country", schema.getWhois_registrant_country()); + jsonObject.put("whois_registrant_phone", schema.getWhois_registrant_phone()); + + return jsonObject; + } + + /* public String getKeys() { + String resString; + resString = "fqdn" + + ", source" + + ", query_success" + + ", match_pattern" + + ", whois_domain" + + ", whois_update_date" + + ", whois_create_date" + + ", whois_expire_date" + + ", whois_email" + + ", whois_ns" + + ", whois_registrar_name" + + ", whois_registrant_org" + + ", whois_registrant_name" + + ", whois_registrant_street" + + ", whois_registrant_city" + + ", whois_registrant_state" + + ", whois_registrant_postcode" + + ", whois_registrant_country" + + ", whois_registrant_phone"; + return resString; + } + + public String getKeyValues() { + String resString = "query_success=" + query_success + + ", source='" + source + '\'' + + ", match_pattern=" + match_pattern + + ", whois_domain='" + whois_domain + '\'' + + ", whois_update_date='" + whois_update_date + '\'' + + ", whois_create_date='" + whois_create_date + '\'' + + ", whois_expire_date='" + whois_expire_date + '\'' + + ", whois_email='" + whois_email + '\'' + + ", whois_ns='" + whois_ns + '\'' + + ", whois_registrar_name='" + whois_registrar_name + '\'' + + ", whois_registrant_org='" + whois_registrant_org + '\'' + + ", whois_registrant_name='" + whois_registrant_name + '\'' + + ", whois_registrant_street='" + whois_registrant_street + '\'' + + ", whois_registrant_city='" + whois_registrant_city + '\'' + + ", whois_registrant_state='" + whois_registrant_state + '\'' + + ", whois_registrant_postcode='" + whois_registrant_postcode + '\'' + + ", whois_registrant_country='" + whois_registrant_country + '\'' + + ", whois_registrant_phone='" + whois_registrant_phone + '\''; + + return resString.replace("'null'", "null"); + }*/ + + +} + diff --git a/src/main/java/com/mesasoft/cn/sketch/pojo/IocDarkWeb.java b/src/main/java/com/mesasoft/cn/sketch/pojo/IocDarkWeb.java new file mode 100644 index 0000000..75efb4b --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/pojo/IocDarkWeb.java @@ -0,0 +1,461 @@ +package com.mesasoft.cn.sketch.pojo; + + +import com.mesasoft.cn.sketch.config.AppConfig; +import com.mesasoft.cn.sketch.dao.MariaDbBase; + +import java.util.Arrays; +import java.util.List; + +public class IocDarkWeb { + + private Long id; + private String iocValue; + private String iocType; + private String nodeType; + private String ipv6; + private String isp; + private String asn; + private String operatingSystem; + private String country; + private String city; + private String latitude; + private String Longtitude; + private String torFingerprint; + private Integer torDirPort; + private Integer torOrPort; + private String torFlags; + private String torVersion; + private String torNickname; + private String torExitPolicy; + private String i2PHash; + private String i2PMethod; + private String i2PVersion; + private Boolean i2PIsFloodfill; + private String i2PBandwidth; + private Boolean i2PIsHidden; + private Boolean i2PIsReachable; + private String i2PPublishTime; + private Integer mtproxyPort; + private String mtproxySecret; + private Integer obfs4Port; + private String obfs4Fingerprint; + private String obfs4Cert; + private Integer obfs4IatMode; + private Integer snowflakePort; + private Long firstAppearTime; + private Long lastAppearTime; + private Long createTime; + private Long updateTime; + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + + public String getIocValue() { + return iocValue; + } + + public void setIocValue(String iocValue) { + this.iocValue = iocValue; + } + + + public String getIocType() { + return iocType; + } + + public void setIocType(String iocType) { + this.iocType = iocType; + } + + + public String getIpv6() { + return ipv6; + } + + public void setIpv6(String ipv6) { + this.ipv6 = ipv6; + } + + + public String getIsp() { + return isp; + } + + public void setIsp(String isp) { + this.isp = isp; + } + + + public String getAsn() { + return asn; + } + + public void setAsn(String asn) { + this.asn = asn; + } + + + public String getOperatingSystem() { + return operatingSystem; + } + + public void setOperatingSystem(String operatingSystem) { + this.operatingSystem = operatingSystem; + } + + + public String getCountry() { + return country; + } + + public void setCountry(String country) { + this.country = country; + } + + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + + public String getLatitude() { + return latitude; + } + + public void setLatitude(String latitude) { + this.latitude = latitude; + } + + + public String getLongtitude() { + return Longtitude; + } + + public void setLongtitude(String Longtitude) { + this.Longtitude = Longtitude; + } + + + public String getTorFingerprint() { + return torFingerprint; + } + + public void setTorFingerprint(String torFingerprint) { + this.torFingerprint = torFingerprint; + } + + + public Integer getTorDirPort() { + return torDirPort; + } + + public void setTorDirPort(Integer torDirPort) { + this.torDirPort = torDirPort; + } + + + public Integer getTorOrPort() { + return torOrPort; + } + + public void setTorOrPort(Integer torOrPort) { + this.torOrPort = torOrPort; + } + + + public String getTorFlags() { + return torFlags; + } + + public void setTorFlags(String torFlags) { + this.torFlags = torFlags; + } + + + public String getTorVersion() { + return torVersion; + } + + public void setTorVersion(String torVersion) { + this.torVersion = torVersion; + } + + + public String getTorNickname() { + return torNickname; + } + + public void setTorNickname(String torNickname) { + this.torNickname = torNickname; + } + + + public String getTorExitPolicy() { + return torExitPolicy; + } + + public void setTorExitPolicy(String torExitPolicy) { + this.torExitPolicy = torExitPolicy; + } + + + public String getI2PHash() { + return i2PHash; + } + + public void setI2PHash(String i2PHash) { + this.i2PHash = i2PHash; + } + + + public String getI2PMethod() { + return i2PMethod; + } + + public void setI2PMethod(String i2PMethod) { + this.i2PMethod = i2PMethod; + } + + + public String getI2PVersion() { + return i2PVersion; + } + + public void setI2PVersion(String i2PVersion) { + this.i2PVersion = i2PVersion; + } + + + public Boolean getI2PIsFloodfill() { + return i2PIsFloodfill; + } + + public void setI2PIsFloodfill(Boolean i2PIsFloodfill) { + this.i2PIsFloodfill = i2PIsFloodfill; + } + + + public String getI2PBandwidth() { + return i2PBandwidth; + } + + public void setI2PBandwidth(String i2PBandwidth) { + this.i2PBandwidth = i2PBandwidth; + } + + + public Boolean getI2PIsHidden() { + return i2PIsHidden; + } + + public void setI2PIsHidden(Boolean i2PIsHidden) { + this.i2PIsHidden = i2PIsHidden; + } + + + public Boolean getI2PIsReachable() { + return i2PIsReachable; + } + + public void setI2PIsReachable(Boolean i2PIsReachable) { + this.i2PIsReachable = i2PIsReachable; + } + + + public String getI2PPublishTime() { + return i2PPublishTime; + } + + public void setI2PPublishTime(String i2PPublishTime) { + this.i2PPublishTime = i2PPublishTime; + } + + + public Integer getMtproxyPort() { + return mtproxyPort; + } + + public void setMtproxyPort(Integer mtproxyPort) { + this.mtproxyPort = mtproxyPort; + } + + + public String getMtproxySecret() { + return mtproxySecret; + } + + public void setMtproxySecret(String mtproxySecret) { + this.mtproxySecret = mtproxySecret; + } + + + public Integer getObfs4Port() { + return obfs4Port; + } + + public void setObfs4Port(Integer obfs4Port) { + this.obfs4Port = obfs4Port; + } + + + public String getObfs4Fingerprint() { + return obfs4Fingerprint; + } + + public void setObfs4Fingerprint(String obfs4Fingerprint) { + this.obfs4Fingerprint = obfs4Fingerprint; + } + + + public String getObfs4Cert() { + return obfs4Cert; + } + + public void setObfs4Cert(String obfs4Cert) { + this.obfs4Cert = obfs4Cert; + } + + + public Integer getObfs4IatMode() { + return obfs4IatMode; + } + + public void setObfs4IatMode(Integer obfs4IatMode) { + this.obfs4IatMode = obfs4IatMode; + } + + + public Integer getSnowflakePort() { + return snowflakePort; + } + + public void setSnowflakePort(Integer snowflakePort) { + this.snowflakePort = snowflakePort; + } + + + public Long getFirstAppearTime() { + return firstAppearTime; + } + + public void setFirstAppearTime(Long firstAppearTime) { + this.firstAppearTime = firstAppearTime; + } + + + public Long getLastAppearTime() { + return lastAppearTime; + } + + public void setLastAppearTime(Long lastAppearTime) { + this.lastAppearTime = lastAppearTime; + } + + + public Long getCreateTime() { + return createTime; + } + + public void setCreateTime(Long createTime) { + this.createTime = createTime; + } + + + public Long getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Long updateTime) { + this.updateTime = updateTime; + } + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public String getValues() { + return + " '" + iocValue + '\'' + + ", '" + iocType + '\'' + + ", '" + nodeType + '\'' + + ", '" + ipv6 + '\'' + + ", '" + isp + '\'' + + ", '" + asn + '\'' + + ", '" + operatingSystem + '\'' + + ", '" + country + '\'' + + ", '" + city + '\'' + + ", " + latitude + + ", " + Longtitude + + ", '" + torFingerprint + '\'' + + ", " + torDirPort + + ", " + torOrPort + + ", '" + torFlags + '\'' + + ", '" + torVersion + '\'' + + ", '" + torNickname + '\'' + + ", '" + torExitPolicy + '\'' + + ", '" + i2PHash + '\'' + + ", '" + i2PMethod + '\'' + + ", '" + i2PVersion + '\'' + + ", " + i2PIsFloodfill + + ", '" + i2PBandwidth + '\'' + + ", " + i2PIsHidden + + ", " + i2PIsReachable + + ", '" + i2PPublishTime + '\'' + + ", " + mtproxyPort + + ", '" + mtproxySecret + '\'' + + ", " + obfs4Port + + ", '" + obfs4Fingerprint + '\'' + + ", '" + obfs4Cert + '\'' + + ", " + obfs4IatMode + + ", " + snowflakePort + + ", " + firstAppearTime + + ", " + lastAppearTime + + ", " + createTime + + ", " + updateTime; + } + + private static final String dataBase = AppConfig.DATABASE; + private static final String tableName = "ioc_darkweb"; + private static final String fields = "ioc_value, ioc_type,node_type, ipv6, isp, asn, operating_system, country, city, latitude, Longtitude, tor_fingerprint, " + + "tor_dir_port, tor_or_port, tor_flags, tor_version, tor_nickname, tor_exit_policy, i2p_hash, i2p_method, i2p_version, i2p_is_floodfill, i2p_bandwidth, " + + "i2p_is_hidden, i2p_is_reachable, i2p_publish_time, mtproxy_port, mtproxy_secret, obfs4_port, obfs4_fingerprint, obfs4_cert, obfs4_iat_mode, snowflake_port, " + + "first_appear_time, last_appear_time, create_time, update_time"; + + public static void insertRecords(List iocDarkWebs, MariaDbBase mariaDbBase) { + for (IocDarkWeb iocDarkWeb : iocDarkWebs) { + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + fields + ") values" + + '(' + iocDarkWeb.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + mariaDbBase.writeSqlExecute(resSql); + } + } + + public static String getInsertSql() { + String[] split = fields.split(","); + Arrays.fill(split, "?"); + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + fields + ") values" + + '(' + String.join(",", split) + ')'; + + return resSql; + } + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/service/DarkWebService.java b/src/main/java/com/mesasoft/cn/sketch/service/DarkWebService.java new file mode 100644 index 0000000..c131057 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/service/DarkWebService.java @@ -0,0 +1,321 @@ +package com.mesasoft.cn.sketch.service; + +import cn.hutool.core.date.DateUtil; +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.io.file.FileReader; +import cn.hutool.core.util.ObjectUtil; +import cn.hutool.json.JSONNull; +import cn.hutool.json.JSONUtil; +import cn.hutool.log.Log; +import com.alibaba.fastjson.util.TypeUtils; +import com.google.common.collect.Lists; +import com.mesasoft.cn.sketch.config.AppConfig; +import com.mesasoft.cn.sketch.pojo.IocDarkWeb; +import com.mesasoft.cn.sketch.util.MariaDBUtils; +import com.mesasoft.cn.sketch.util.ValidationUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.lang3.time.StopWatch; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.File; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + + + +public class DarkWebService { + + private static Log logger = Log.get(); + + + @Autowired + static AppConfig appConfig; + + + public static void main(String[] args) { + + //1.读取文件，转为list ，参数 文件目录，多个目录 + //tor + //i2p + //mtproxy + //obfs4 + //snowflake + //generalbridge + try { + String parentPath = "C:\\test\\h3\\"; + String[] apis = {"tor", "i2p", "mtproxy", "obfs4", "snowflake"/*, "generalbridge"*/}; + String[] t = {"07", "09", "10", "11", "12", "13", "14", "15", "17", "20"}; + + String apiType = apis[0]; + for (String s : t) { + + List tors = readJson(parentPath + apiType +"\\"+s+"\\"); + System.err.println(apiType+"读取日志" + tors.size()); + + List list = new ArrayList<>(); + //2.转成对象 + for (Map data : tors) { + Map result = (Map) data.get("result"); + List nodeList = (List) result.get("node_list"); + for (Map node : nodeList) { + + String ipv6s = ""; + if (node.get("ipv6") instanceof List) { + List ipv6 = TypeUtils.castToJavaBean(node.get("ipv6"), List.class); + ipv6s = String.join(",", ipv6); + } else { + ipv6s = TypeUtils.castToString(Optional.ofNullable(node.get("ipv6")).orElse("")); + } + + //location + Map detail = (Map) node.get("detail"); + Map location = (Map) detail.get("location"); + String isp = TypeUtils.castToString(location.get("isp")); + String asn = TypeUtils.castToString(location.get("as_number")); + String country = TypeUtils.castToString(location.get("country")); + + String city = TypeUtils.castToString(location.get("city")); + String latitude = TypeUtils.castToString(location.get("latitude")); + String longtitude = TypeUtils.castToString(location.get("longtitude")); + + Long firstAppearTime = TypeUtils.castToLong(node.get("first_appear_time")) / 1000; + Long lastAppearTime = TypeUtils.castToLong(node.get("last_appear_time")) / 1000; + + //创建对象 + IocDarkWeb iocDarkWeb = new IocDarkWeb(); + + iocDarkWeb.setNodeType(apiType); + + iocDarkWeb.setIpv6(ipv6s); + + //location + iocDarkWeb.setIsp(isp); + iocDarkWeb.setAsn(asn); + iocDarkWeb.setCountry(country); + iocDarkWeb.setCity(city); + iocDarkWeb.setLatitude(latitude); + iocDarkWeb.setLongtitude(longtitude); + + //时间 + iocDarkWeb.setFirstAppearTime(firstAppearTime); + iocDarkWeb.setLastAppearTime(lastAppearTime); + iocDarkWeb.setCreateTime(DateUtil.currentSeconds()); + iocDarkWeb.setUpdateTime(DateUtil.currentSeconds()); + + iocDarkWeb.setIocType("ip"); + + switch (apiType) { + case "tor": + //tor + String ip = TypeUtils.castToString(node.get("ip")); + iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip); + torDetail(node, detail, iocDarkWeb); + break; + case "i2p": + ip = TypeUtils.castToString(node.get("ipv4")); + iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip); + i2pDetail(detail, iocDarkWeb); + break; + case "mtproxy": + //特殊处理，mtproxy节点中server字段支持域名和ip两种形式 + String server = TypeUtils.castToString(node.get("server")); + iocDarkWeb.setIocType(ValidationUtils.isValidIp(server) ? "ip" : "domain"); + iocDarkWeb.setIocValue(server); + + mtproxyDetail(detail, iocDarkWeb); + break; + case "obfs4": + ip = TypeUtils.castToString(node.get("ipv4")); + iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip); + obfs4Detail(detail, iocDarkWeb); + break; + case "snowflake": + ip = TypeUtils.castToString(node.get("ipv4")); + iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip); + snowflakeDetail(detail, iocDarkWeb); + break; + } + + list.add(iocDarkWeb); + } + } + + + //3.写入数据库 + List> apiPartitions = Lists.partition(list, appConfig.getDbQueryBatchSize()); + for (List partition : apiPartitions) { + System.err.println("开始写入" + partition.size()); + insert(partition); + System.err.println("成功写入" + partition.size()); + } + } + System.err.println(apiType+"写入完成"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static void snowflakeDetail(Map detail, IocDarkWeb iocDarkWeb) { + //snowflake + Integer snowflakePort = TypeUtils.castToInt(detail.get("port")); + //snowflake + iocDarkWeb.setSnowflakePort(snowflakePort); + } + + private static void obfs4Detail(Map detail, IocDarkWeb iocDarkWeb) { + //obfs4 + Integer obfs4Port = TypeUtils.castToInt(detail.get("port")); + String obfs4Fingerprint = TypeUtils.castToString(detail.get("fingerprint")); + String obfs4Cert = TypeUtils.castToString(detail.get("cert")); + Integer obfs4IatMode = TypeUtils.castToInt(detail.get("iatMode")); + //obfs4 + iocDarkWeb.setObfs4Port(obfs4Port); + iocDarkWeb.setObfs4Fingerprint(obfs4Fingerprint); + iocDarkWeb.setObfs4Cert(obfs4Cert); + iocDarkWeb.setObfs4IatMode(obfs4IatMode); + } + + private static void mtproxyDetail(Map detail, IocDarkWeb iocDarkWeb) { + //mtproxy + Integer mtproxyPort = TypeUtils.castToInt(detail.get("port")); + String mtproxySecret = TypeUtils.castToString(detail.get("secret")); + //mtproxy + iocDarkWeb.setMtproxyPort(mtproxyPort); + iocDarkWeb.setMtproxySecret(mtproxySecret); + } + + private static void i2pDetail(Map detail, IocDarkWeb iocDarkWeb) { + //i2p + String i2pHash = TypeUtils.castToString(detail.get("hash")); + String i2pMethod = TypeUtils.castToString(detail.get("method")); + String i2pVersion = TypeUtils.castToString(detail.get("version")); + Boolean i2pFloodFill = TypeUtils.castToBoolean(detail.get("isFloodfill")); + String i2pWidth = TypeUtils.castToString(detail.get("width")); + Boolean i2pHidden = TypeUtils.castToBoolean(detail.get("isHidden")); + Boolean i2pReachable = TypeUtils.castToBoolean(detail.get("isReachable")); + String i2pPublish = TypeUtils.castToString(detail.get("publish")); + //i2p + iocDarkWeb.setI2PHash(i2pHash); + iocDarkWeb.setI2PMethod(i2pMethod); + iocDarkWeb.setI2PVersion(i2pVersion); + iocDarkWeb.setI2PIsFloodfill(i2pFloodFill); + iocDarkWeb.setI2PBandwidth(i2pWidth); + iocDarkWeb.setI2PIsHidden(i2pHidden); + iocDarkWeb.setI2PIsReachable(i2pReachable); + iocDarkWeb.setI2PPublishTime(i2pPublish); + } + + private static void torDetail(Map node, Map detail, IocDarkWeb iocDarkWeb) { + String fingerprint = TypeUtils.castToString(node.get("fingerprint")); + Integer dirPort = node.get("dir_port") == JSONNull.NULL ? null : TypeUtils.castToInt(node.get("dir_port")); + Integer orPort = node.get("or_port") == JSONNull.NULL ? null : TypeUtils.castToInt(node.get("or_port")); + List flagsList = TypeUtils.castToJavaBean(detail.get("flags"), List.class); + String flags = String.join(",", flagsList); + String nickname = TypeUtils.castToString(detail.get("nickname")); + String exitPolicy = TypeUtils.castToString(detail.get("exit_policy")); + + String torVirsion = TypeUtils.castToString(detail.get("tor_virsion")); + String operatingSystem = TypeUtils.castToString(detail.get("host_system")); + //tor + iocDarkWeb.setTorFingerprint(fingerprint); + iocDarkWeb.setTorDirPort(dirPort); + iocDarkWeb.setTorOrPort(orPort); + iocDarkWeb.setTorFlags(flags); + iocDarkWeb.setTorNickname(nickname); + iocDarkWeb.setTorExitPolicy(exitPolicy); + iocDarkWeb.setOperatingSystem(operatingSystem); + + iocDarkWeb.setTorVersion(torVirsion); + } + + + public static List readJson(String path) { + + List files = FileUtil.loopFiles(Paths.get(path), 1, FileFilterUtils.suffixFileFilter("json")); + + logger.info("读取文件数量 {}", files.size()); + List jsonList = files.stream().map(f -> { + FileReader fr = new FileReader(f); + Map objMap = JSONUtil.toBean(fr.readString(), Map.class); + + return objMap; + }).collect(Collectors.toList()); + + return jsonList; + } + + + public static void insert(List iocDarkWebs) throws Exception { + + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + + Connection connection = MariaDBUtils.getConnection(); + connection.setAutoCommit(false); + + PreparedStatement pst = connection.prepareStatement(IocDarkWeb.getInsertSql()); + + for (IocDarkWeb i : iocDarkWebs) { + + pst.setString(1, i.getIocValue()); + pst.setString(2, i.getIocType()); + pst.setString(3, i.getNodeType()); + pst.setString(4, i.getIpv6()); + pst.setString(5, i.getIsp()); + pst.setString(6, i.getAsn()); + pst.setString(7, i.getOperatingSystem()); + pst.setString(8, i.getCountry()); + pst.setString(9, i.getCity()); + pst.setString(10, i.getLatitude()); + pst.setString(11, i.getLongtitude()); + pst.setString(12, i.getTorFingerprint()); + pst.setObject(13, i.getTorDirPort()); + pst.setObject(14, i.getTorOrPort()); + pst.setString(15, i.getTorFlags()); + pst.setString(16, i.getTorVersion()); + pst.setString(17, i.getTorNickname()); + pst.setString(18, i.getTorExitPolicy()); + + pst.setString(19, i.getI2PHash()); + pst.setString(20, i.getI2PMethod()); + pst.setString(21, i.getI2PVersion()); + pst.setObject(22, i.getI2PIsFloodfill()); + pst.setString(23, i.getI2PBandwidth()); + pst.setObject(24, i.getI2PIsHidden()); + pst.setObject(25, i.getI2PIsReachable()); + pst.setString(26, i.getI2PPublishTime()); + + pst.setObject(27, i.getMtproxyPort()); + pst.setString(28, i.getMtproxySecret()); + + pst.setObject(29, i.getObfs4Port()); + pst.setString(30, i.getObfs4Fingerprint()); + pst.setString(31, i.getObfs4Cert()); + pst.setObject(32, i.getObfs4IatMode()); + + pst.setObject(33, i.getSnowflakePort()); + + pst.setLong(34, i.getFirstAppearTime()); + pst.setLong(35, i.getLastAppearTime()); + pst.setLong(36, i.getCreateTime()); + pst.setLong(37, i.getUpdateTime()); + + pst.addBatch(); + + } + pst.executeBatch(); + connection.commit(); + + stopWatch.stop(); + logger.info("成功提交数据 {} ,花费时间 {} ms", iocDarkWebs.size(), stopWatch.getTime()); + MariaDBUtils.close(pst, connection); + } + + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/service/DbService.java b/src/main/java/com/mesasoft/cn/sketch/service/DbService.java new file mode 100644 index 0000000..cbde7b4 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/service/DbService.java @@ -0,0 +1,93 @@ +package com.mesasoft.cn.sketch.service; + +import cn.hutool.log.Log; +import com.mesasoft.cn.sketch.config.AppConfig; +import com.mesasoft.cn.sketch.dao.DomainCategorySchema; +import com.mesasoft.cn.sketch.dao.MariaDbBase; + +import java.util.List; + +public class DbService { + + + private Log logger = Log.get(); + + private static final String dataBase = AppConfig.DATABASE; + private static final String tableName = AppConfig.DOMAIN_CATE_TABLENAME; + + private MariaDbBase mariaDB; + + public void execute(List recordsFromBcApi) { + + for (DomainCategorySchema categoryFile : recordsFromBcApi) { + // 生成sql + String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' + + " (" + categoryFile.getKeys() + ") values" + + '(' + categoryFile.getValues() + ')'; + resSql = resSql.replace("'null'", "null"); + + mariaDB.writeSqlExecute(resSql); + } + + } + + /** + * @throws Exception + */ +/* public void exec(List data, String sql) throws Exception { + if (Objects.isNull(data)) { + logger.info(" data is empty "); + return; + } + Connection conn = null; + PreparedStatement pst = null; + try { + //创建数据库连接库对象 + conn = MariaDBUtils.getConnection(); + long start = System.currentTimeMillis(); + conn.setAutoCommit(false); + pst = conn.prepareStatement(sql); + int count = 0; + int index = 1; + for (int i = 1; i <= data.size(); i++) { + for (Object val : data.get(i - 1).values()) { + if (val instanceof Long) { + pst.setLong((index++), Long.valueOf(String.valueOf(val))); + } else if (val instanceof Integer) { + pst.setInt((index++), Integer.valueOf(String.valueOf(val))); + } else if (val instanceof Boolean) { + pst.setBoolean((index++), Boolean.valueOf(String.valueOf(val))); + } else { + pst.setString((index++), String.valueOf(val)); + } + } + index = 1; + pst.addBatch(); + //1w提交一次 + if (i % ApplicationConfig.DB_QUERY_BATCH_SIZE == 0) { + int[] ints = pst.executeBatch(); + count = count + ints.length; + conn.commit(); + pst.clearBatch(); + } + } + int[] ints = pst.executeBatch(); + count = count + ints.length; + conn.commit(); + logger.info("sql {} , count {} , take {}", sql, tableName, count, (System.currentTimeMillis() - start) + "ms"); + } catch (Exception e) { + if (conn != null) { + try { + conn.rollback(); + } catch (SQLException e1) { + logger.error(e1); + } + } + logger.error(e); + throw e; + } finally { + IoUtil.close(pst); + IoUtil.close(conn); + } + }*/ +} diff --git a/src/main/java/com/mesasoft/cn/sketch/service/DomainService.java b/src/main/java/com/mesasoft/cn/sketch/service/DomainService.java new file mode 100644 index 0000000..8850e84 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/service/DomainService.java @@ -0,0 +1,14 @@ +package com.mesasoft.cn.sketch.service; + +import com.alibaba.fastjson.JSONArray; + +import java.util.List; + +public interface DomainService { + + JSONArray getCategoryInfo(List domains, String username, boolean isLocal) throws Exception ; + + JSONArray getWhoisInfo(List domains, boolean isLocal) throws Exception ; + + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/service/FileQuery.java b/src/main/java/com/mesasoft/cn/sketch/service/FileQuery.java new file mode 100644 index 0000000..921ac8e --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/service/FileQuery.java @@ -0,0 +1,117 @@ +/* +package cn.ac.iie.efo.sketch.service; + +import cn.ac.iie.efo.sketch.config.ApplicationConfig; +import cn.ac.iie.efo.sketch.util.FileUtils; +import cn.ac.iie.efo.sketch.util.JsonTool; +import cn.ac.iie.efo.sketch.util.MariaDBUtils; +import cn.ac.iie.efo.sketch.util.ValidationUtils; +import cn.hutool.core.io.file.FileNameUtil; +import cn.hutool.log.Log; +import com.alibaba.fastjson.JSONArray; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.DecimalFormat; +import java.util.List; + +*/ +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2022/1/3 + * Time: 5:33 PM + * Description: No Description + *//* + +public class FileQuery { + private static final Log logger = Log.get(); + + long queryNum = 0; + long dbResultNum = 0; + long apiResultNum = 0; + long failedQueryNum = 0; + + public String queryExecutor(String objectType, String queryType, String queryFile, String username, Boolean isLocal) throws SQLException, IOException { + + Connection mariaConn = MariaDBUtils.getConnection(); + Statement mariaStat = mariaConn.createStatement(); + + // 文件读取 + InputStreamReader inputStreamReader = new InputStreamReader( new FileInputStream(queryFile), "GBK"); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader); + File importFile = new File(queryFile); + String importFileName = FileNameUtil.getName(importFile); + long fileLineNum = Files.lines(importFile.toPath()).count(); + + logger.info("[File query]-" + importFileName + ": File Found."); + logger.info("[File query]-" + queryType + "."); + String outputFileName = importFileName + "." + queryType + ".json"; + + File outputFile = new File(ApplicationConfig.QUERY_OUTPUT_DIR + "/" + outputFileName); + if (!outputFile.exists()) { + FileUtils.createFile(new File(ApplicationConfig.QUERY_OUTPUT_DIR), outputFileName); + } + OutputStream outStream = new FileOutputStream(outputFile); + OutputStreamWriter outWriter = new OutputStreamWriter(outStream, StandardCharsets.UTF_8); + outWriter.write("[\n"); + + logger.info("[File query]-Start query! Progress: 0.00%"); + + // 执行查询 + List objectList; + JSONArray resultJson = new JSONArray(); + ListQuery listQueryUtils = new ListQuery(); + + int batch_index = 1; + // 批量读取 + while ((objectList = FileUtils.getBatchLineReadIn(bufferedReader, ApplicationConfig.QUERY_READIN_BATCH)).size() > 0) { + // 校验 + objectList = ValidationUtils.getChecked(objectList, objectType); + JSONArray queryResults = listQueryUtils.getQueryResults(objectType, queryType, objectList,username, isLocal); + + resultJson.addAll(queryResults); + + queryNum += listQueryUtils.queryNum; + dbResultNum += listQueryUtils.dbResultNum; + apiResultNum += listQueryUtils.apiResultNum; + failedQueryNum += listQueryUtils.failedQueryNum; + + // 写入output + // todo 写入格式优化 check + for (Object json : queryResults) { + outWriter.write(JsonTool.prettyJson(json.toString())); + } + + // 进度日志 + if (batch_index * ApplicationConfig.QUERY_READIN_BATCH % ApplicationConfig.QUERY_LOG_FILE_LINE_INTERVAL == 0) { + String percent = new DecimalFormat("##.00%").format((float) queryNum / fileLineNum); + logger.info("[File query]-" + queryType + "-" + importFileName + " Progress:" + percent); + } + + batch_index++; + } + + logger.info("[File query]-" + queryType + "-" + importFileName + " " + + "Query result: submit " + queryNum + " valid objects, " + + dbResultNum + " (" + new DecimalFormat("##.0%").format((float) dbResultNum / queryNum) + ")" + " results from database，" + + apiResultNum + " (" + new DecimalFormat("##.0%").format((float) apiResultNum / queryNum) + ")" + " results from api. " + + failedQueryNum + " (" + new DecimalFormat("##.0%").format((float) failedQueryNum / queryNum) + ")" + " failed queries,"); + logger.info("[File query]-" + importFileName + " Results saved in " + outputFile.toString()); + + outWriter.write("\n]"); + FileUtils.writerClose(outWriter, outStream); + FileUtils.readerClose(bufferedReader, inputStreamReader); + MariaDBUtils.close(mariaStat, mariaConn); + + // todo check 写入null时的过滤情况 + return resultJson.toJSONString(); + } + + +} +*/ diff --git a/src/main/java/com/mesasoft/cn/sketch/service/ListQuery.java b/src/main/java/com/mesasoft/cn/sketch/service/ListQuery.java new file mode 100644 index 0000000..c64a899 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/service/ListQuery.java @@ -0,0 +1,241 @@ +/* +package cn.ac.iie.efo.sketch.service; + +import cn.ac.iie.efo.sketch.api.BrightCloud; +import cn.ac.iie.efo.sketch.api.ChinaZ; +import cn.ac.iie.efo.sketch.config.ApplicationConfig; +import cn.ac.iie.efo.sketch.config.BrightCloudConfig; +import cn.ac.iie.efo.sketch.config.ChinazConfig; +import cn.ac.iie.efo.sketch.dao.DomainCategorySchema; +import cn.ac.iie.efo.sketch.dao.DomainWhoisSchema; +import cn.ac.iie.efo.sketch.dao.MariaDbBase; +import cn.ac.iie.efo.sketch.util.FileUtils; +import cn.ac.iie.efo.sketch.util.MariaDBUtils; +import cn.ac.iie.efo.sketch.util.ValidationUtils; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.Lists; +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.Date; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +*/ +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2021/12/31 + * Time: 11:28 AM + * Description: No Description + *//* + +public class ListQuery { + private static final Logger LOG = Logger.getLogger(ListQuery.class); + + @Autowired + private BrightCloudConfig bcConfig; + + @Autowired + private ChinazConfig chinazConfig; + + private static final int MAX_DB_BATCH_SIZE = ApplicationConfig.DB_QUERY_BATCH_SIZE; + + private List queryObjects; + private String curQueryType; + + private MariaDbBase mariaDB; + + long queryNum; + long dbResultNum; + long apiResultNum; + long failedQueryNum; + + public JSONArray getQueryResults(String objectType, String queryType, List queryObjects, String username, Boolean isLocal) throws SQLException, IOException { + queryNum = 0; + dbResultNum = 0; + apiResultNum = 0; + failedQueryNum = 0; + + this.queryObjects = queryObjects; + this.curQueryType = queryType; + JSONArray json = null; + + Connection mariaConn = MariaDBUtils.getConnection(); + Statement mariaStat = mariaConn.createStatement(); + mariaDB = new MariaDbBase(mariaConn, mariaStat); + + //去重 + queryObjects = queryObjects.stream().distinct().collect(Collectors.toList()); + + // 校验 + queryObjects = ValidationUtils.getChecked(queryObjects, objectType); + this.queryNum = queryObjects.size(); + + // 执行查询 + if (queryType.equals("domain_category")) { + json = getDomainCategory(queryObjects, username, isLocal); + } else if (queryType.equals("domain_whois")) { + json = getDomainWhois(queryObjects, isLocal); + } else { + // TODO： get dns server info + LOG.error("Wrong query type: " + queryType); + } + + LOG.info("Query Statistic - number of query objects: " + queryNum + + "\nresults from local db: " + dbResultNum + + " results from external api: " + apiResultNum); + MariaDBUtils.close(mariaStat, mariaConn); + return json; + } + + public JSONArray getDomainCategory(List domains, String username, boolean isLocal) throws SQLException, IOException { + + JSONArray results = new JSONArray(); + + // 查询本地数据库 + ArrayList objectsFromDB = new ArrayList<>(); + List> partitions = Lists.partition(queryObjects, MAX_DB_BATCH_SIZE); + for (List partition : partitions) { // 批量查询 + List dbRecords = DomainCategorySchema.getDbRecord(partition, mariaDB, "brightcloud"); + for (DomainCategorySchema record : dbRecords) { + objectsFromDB.add(record.getFqdn()); // 保存查询记录 + JSONObject jsonObject = DomainCategorySchema.schema2json(record); + results.add(jsonObject); // 保存查询结果 + } + } + dbResultNum = results.size(); + + // 调用api + List bcResults = new ArrayList<>(); + List objectsFromApi = new ArrayList<>(queryObjects); + objectsFromApi.removeAll(objectsFromDB); + if (!isLocal && objectsFromApi.size() > 0) { + BrightCloud brightCloud = new BrightCloud(); + + List> apiPartitions = Lists.partition(objectsFromApi, bcConfig.getMaximumQueryNum()); + for (List partition : apiPartitions) { // 批量查询 + List recordsFromBcApi = brightCloud.responseSparse(brightCloud.getQueryResults(partition)); + for (DomainCategorySchema record : recordsFromBcApi) { + if (record.getQuery_success().equals(true)) { //查询成功的结果 + record.setSubmit_user(username); + bcResults.add(record); + if (bcResults.size() > MAX_DB_BATCH_SIZE) { //超过一定量时写入数据库 + DomainCategorySchema.insertRecords(bcResults, mariaDB); + bcResults.clear(); + } + results.add(DomainCategorySchema.schema2json(record)); + } else { // 查询失败的结果 + failedQueryNum += 1; + } + } + } + } + apiResultNum = results.size() - dbResultNum - failedQueryNum; + DomainCategorySchema.insertRecords(bcResults, mariaDB); + + if (apiResultNum > 0) { // 记录api调用次数 + OutputStream bcQueryLogStream = new FileOutputStream(bcConfig.getUsereportFilePath(), true); + OutputStreamWriter bcQueryLogWriter = new OutputStreamWriter(bcQueryLogStream, StandardCharsets.UTF_8); + Date d = new Date(System.currentTimeMillis()); + bcQueryLogWriter.write(d + "," + "List Query," + "-" + "," + curQueryType + "," + apiResultNum + "\n"); + FileUtils.writerClose(bcQueryLogWriter, bcQueryLogStream); + } + + return results; + } + + public JSONArray getDomainWhois(List domains, boolean isLocal) throws SQLException, IOException { + + JSONArray results = new JSONArray(); + + // 查询本地数据库 + ArrayList objectsFromDB = new ArrayList<>(); + List> partitions = Lists.partition(queryObjects, MAX_DB_BATCH_SIZE); + for (List partition : partitions) { // 批量查询 + List dbRecords = + DomainWhoisSchema.getDbRecord(partition, mariaDB, "chinaz"); + for (DomainWhoisSchema record : dbRecords) { + objectsFromDB.add(record.getFqdn()); // 保存查询记录 + JSONObject jsonObject = DomainWhoisSchema.schema2json(record); + results.add(jsonObject); // 保存查询结果 + } + } + dbResultNum = results.size(); + + // 调用api + List chinazResults = new ArrayList<>(); + if (!isLocal) { + ChinaZ chinaz = new ChinaZ(); + + List objectsFromApi = new ArrayList<>(queryObjects); + objectsFromApi.removeAll(objectsFromDB); // 需要调用api查询的部分对象 + + List> apiPartitions = Lists.partition(objectsFromApi, chinazConfig.getMaximumQueryNum()); + for (List partition : apiPartitions) { // 批量查询 + List recordsFromApi = chinaz.responseSparse(chinaz.getQueryResults(partition)); + for (DomainWhoisSchema record : recordsFromApi) { + if (record.getQuery_success().equals(true)) { // 查询成功的结果 + chinazResults.add(record); + if (chinazResults.size() > MAX_DB_BATCH_SIZE) { // 超过一定量时写入数据库 + DomainWhoisSchema.insertRecords(chinazResults, mariaDB); + chinazResults.clear(); + } + results.add(DomainWhoisSchema.schema2json(record)); + } else { // 查询失败的结果 + failedQueryNum += 1; + } + } + } + } + + DomainWhoisSchema.insertRecords(chinazResults, mariaDB); + apiResultNum = results.size() - dbResultNum - failedQueryNum; + // todo 不直接写入，维护变量，导出一次 + if (apiResultNum > 0) { // 记录api调用次数 + OutputStream bcQueryLogStream = new FileOutputStream(chinazConfig.getUsereportFilePath(), true); + OutputStreamWriter bcQueryLogWriter = new OutputStreamWriter(bcQueryLogStream, StandardCharsets.UTF_8); + Date d = new Date(System.currentTimeMillis()); + bcQueryLogWriter.write(d + "," + "List Query," + "-" + "," + curQueryType + "," + apiResultNum + "\n"); + FileUtils.writerClose(bcQueryLogWriter, bcQueryLogStream); + } + + + return results; + } + + + public static void main(String[] args) throws Exception { +// String objectType = args[0]; +// String queryType = args[1]; +// List queryObjects = Arrays.asList(args[2].split(",")); +// String username = args[4]; +// boolean isLocal = true; +// if (args.length >= 3) { +// isLocal = Boolean.parseBoolean(args[3]); +// } + +// System.out.println(new ListQuery().getQueryResults(objectType, queryType, queryObjects, username, isLocal).toString()); + + JSONArray queryResults = new ListQuery().getQueryResults("domain", + "domain_category", + Arrays.asList("baidu.com", "cctv.com"), "", true); + System.err.println(queryResults.toJSONString()); + String json = "{\"namespace\":\"log.session\",\"type\":\"record\",\"name\":\"session\",\"fields\":[{\"name\":\"common_log_id\",\"type\":\"long\"},{\"name\":\"common_service\",\"type\":\"long\"},{\"name\":\"common_recv_time\",\"type\":\"long\"},{\"name\":\"common_direction\",\"type\":\"long\"},{\"name\":\"common_l4_protocol\",\"type\":\"string\"},{\"name\":\"common_address_type\",\"type\":\"long\"},{\"name\":\"common_schema_type\",\"type\":\"string\"},{\"name\":\"common_policy_id\",\"type\":\"long\"},{\"name\":\"common_user_tags\",\"type\":\"string\"},{\"name\":\"common_action\",\"type\":\"long\"},{\"name\":\"common_sub_action\",\"type\":\"string\"},{\"name\":\"common_user_region\",\"type\":\"string\"},{\"name\":\"common_client_ip\",\"type\":\"string\"},{\"name\":\"common_client_port\",\"type\":\"long\"},{\"name\":\"common_internal_ip\",\"type\":\"string\"},{\"name\":\"common_entrance_id\",\"type\":\"long\"},{\"name\":\"common_device_id\",\"type\":\"string\"},{\"name\":\"common_egress_link_id\",\"type\":\"long\"},{\"name\":\"common_ingress_link_id\",\"type\":\"long\"},{\"name\":\"common_isp\",\"type\":\"string\"},{\"name\":\"common_device_tag\",\"type\":\"string\"},{\"name\":\"common_data_center\",\"type\":\"string\"},{\"name\":\"common_encapsulation\",\"type\":\"long\"},{\"name\":\"common_tunnels\",\"type\":\"string\"},{\"name\":\"common_sled_ip\",\"type\":\"string\"},{\"name\":\"common_device_group\",\"type\":\"string\"},{\"name\":\"common_app_behavior\",\"type\":\"string\"},{\"name\":\"common_client_location\",\"type\":\"string\"},{\"name\":\"common_client_asn\",\"type\":\"string\"},{\"name\":\"common_subscriber_id\",\"type\":\"string\"},{\"name\":\"common_imei\",\"type\":\"string\"},{\"name\":\"common_imsi\",\"type\":\"string\"},{\"name\":\"common_phone_number\",\"type\":\"string\"},{\"name\":\"common_server_ip\",\"type\":\"string\"},{\"name\":\"common_server_port\",\"type\":\"long\"},{\"name\":\"common_external_ip\",\"type\":\"string\"},{\"name\":\"common_server_location\",\"type\":\"string\"},{\"name\":\"common_server_asn\",\"type\":\"string\"},{\"name\":\"common_protocol_label\",\"type\":\"string\"},{\"name\":\"common_service_category\",\"type\":{\"type\":\"array\",\"items\":\"long\"}},{\"name\":\"common_app_label\",\"type\":\"string\"},{\"name\":\"common_app_id\",\"type\":\"string\"},{\"name\":\"common_userdefine_app_name\",\"type\":\"string\"},{\"name\":\"common_app_surrogate_id\",\"type\":\"string\"},{\"name\":\"common_l7_protocol\",\"type\":\"string\"},{\"name\":\"common_sessions\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_num\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_num\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_diff\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_diff\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_diff\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_diff\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_num\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_num\",\"type\":\"long\"},{\"name\":\"common_start_time\",\"type\":\"long\"},{\"name\":\"common_end_time\",\"type\":\"long\"},{\"name\":\"common_establish_latency_ms\",\"type\":\"long\"},{\"name\":\"common_con_duration_ms\",\"type\":\"long\"},{\"name\":\"common_stream_dir\",\"type\":\"long\"},{\"name\":\"common_address_list\",\"type\":\"string\"},{\"name\":\"common_has_dup_traffic\",\"type\":\"long\"},{\"name\":\"common_stream_error\",\"type\":\"string\"},{\"name\":\"common_stream_trace_id\",\"type\":\"long\"},{\"name\":\"common_link_info_c2s\",\"type\":\"string\"},{\"name\":\"common_link_info_s2c\",\"type\":\"string\"},{\"name\":\"common_packet_capture_file\",\"type\":\"string\"},{\"name\":\"common_c2s_ipfrag_num\",\"type\":\"long\"},{\"name\":\"common_s2c_ipfrag_num\",\"type\":\"long\"},{\"name\":\"common_c2s_tcp_lostlen\",\"type\":\"long\"},{\"name\":\"common_s2c_tcp_lostlen\",\"type\":\"long\"},{\"name\":\"common_c2s_tcp_unorder_num\",\"type\":\"long\"},{\"name\":\"common_s2c_tcp_unorder_num\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_retrans\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_retrans\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_retrans\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_retrans\",\"type\":\"long\"},{\"name\":\"common_tcp_client_isn\",\"type\":\"long\"},{\"name\":\"common_tcp_server_isn\",\"type\":\"long\"},{\"name\":\"common_mirrored_pkts\",\"type\":\"long\"},{\"name\":\"common_mirrored_bytes\",\"type\":\"long\"},{\"name\":\"common_first_ttl\",\"type\":\"long\"},{\"name\":\"common_processing_time\",\"type\":\"long\"},{\"name\":\"http_url\",\"type\":\"string\"},{\"name\":\"http_host\",\"type\":\"string\"},{\"name\":\"http_domain\",\"type\":\"string\"},{\"name\":\"http_request_line\",\"type\":\"string\"},{\"name\":\"http_response_line\",\"type\":\"string\"},{\"name\":\"http_request_header\",\"type\":\"string\"},{\"name\":\"http_response_header\",\"type\":\"string\"},{\"name\":\"http_request_content\",\"type\":\"string\"},{\"name\":\"http_response_content\",\"type\":\"string\"},{\"name\":\"http_request_body\",\"type\":\"string\"},{\"name\":\"http_response_body\",\"type\":\"string\"},{\"name\":\"http_request_body_key\",\"type\":\"string\"},{\"name\":\"http_response_body_key\",\"type\":\"string\"},{\"name\":\"http_proxy_flag\",\"type\":\"long\"},{\"name\":\"http_sequence\",\"type\":\"long\"},{\"name\":\"http_snapshot\",\"type\":\"string\"},{\"name\":\"http_cookie\",\"type\":\"string\"},{\"name\":\"http_referer\",\"type\":\"string\"},{\"name\":\"http_user_agent\",\"type\":\"string\"},{\"name\":\"http_request_content_length\",\"type\":\"string\"},{\"name\":\"http_request_content_type\",\"type\":\"string\"},{\"name\":\"http_response_content_length\",\"type\":\"string\"},{\"name\":\"http_response_content_type\",\"type\":\"string\"},{\"name\":\"http_content_length\",\"type\":\"string\"},{\"name\":\"http_content_type\",\"type\":\"string\"},{\"name\":\"http_set_cookie\",\"type\":\"string\"},{\"name\":\"http_version\",\"type\":\"string\"},{\"name\":\"http_response_latency_ms\",\"type\":\"long\"},{\"name\":\"http_session_duration_ms\",\"type\":\"long\"},{\"name\":\"http_action_file_size\",\"type\":\"long\"},{\"name\":\"mail_protocol_type\",\"type\":\"string\"},{\"name\":\"mail_account\",\"type\":\"string\"},{\"name\":\"mail_to_cmd\",\"type\":\"string\"},{\"name\":\"mail_from_cmd\",\"type\":\"string\"},{\"name\":\"mail_from\",\"type\":\"string\"},{\"name\":\"mail_to\",\"type\":\"string\"},{\"name\":\"mail_cc\",\"type\":\"string\"},{\"name\":\"mail_bcc\",\"type\":\"string\"},{\"name\":\"mail_subject\",\"type\":\"string\"},{\"name\":\"mail_subject_charset\",\"type\":\"string\"},{\"name\":\"mail_content\",\"type\":\"string\"},{\"name\":\"mail_content_charset\",\"type\":\"string\"},{\"name\":\"mail_attachment_name\",\"type\":\"string\"},{\"name\":\"mail_attachment_name_charset\",\"type\":\"string\"},{\"name\":\"mail_attachment_content\",\"type\":\"string\"},{\"name\":\"mail_eml_file\",\"type\":\"string\"},{\"name\":\"mail_snapshot\",\"type\":\"string\"},{\"name\":\"dns_message_id\",\"type\":\"long\"},{\"name\":\"dns_qr\",\"type\":\"long\"},{\"name\":\"dns_opcode\",\"type\":\"long\"},{\"name\":\"dns_aa\",\"type\":\"long\"},{\"name\":\"dns_tc\",\"type\":\"long\"},{\"name\":\"dns_rd\",\"type\":\"long\"},{\"name\":\"dns_ra\",\"type\":\"long\"},{\"name\":\"dns_rcode\",\"type\":\"long\"},{\"name\":\"dns_qdcount\",\"type\":\"long\"},{\"name\":\"dns_ancount\",\"type\":\"long\"},{\"name\":\"dns_nscount\",\"type\":\"long\"},{\"name\":\"dns_arcount\",\"type\":\"long\"},{\"name\":\"dns_qname\",\"type\":\"string\"},{\"name\":\"dns_qtype\",\"type\":\"long\"},{\"name\":\"dns_qclass\",\"type\":\"long\"},{\"name\":\"dns_cname\",\"type\":\"string\"},{\"name\":\"dns_sub\",\"type\":\"long\"},{\"name\":\"dns_rr\",\"type\":\"string\"},{\"name\":\"ssl_version\",\"type\":\"string\"},{\"name\":\"ssl_sni\",\"type\":\"string\"},{\"name\":\"ssl_san\",\"type\":\"string\"},{\"name\":\"ssl_cn\",\"type\":\"string\"},{\"name\":\"ssl_pinningst\",\"type\":\"long\"},{\"name\":\"ssl_intercept_state\",\"type\":\"long\"},{\"name\":\"ssl_passthrough_reason\",\"type\":\"string\"},{\"name\":\"ssl_server_side_latency\",\"type\":\"long\"},{\"name\":\"ssl_client_side_latency\",\"type\":\"long\"},{\"name\":\"ssl_server_side_version\",\"type\":\"string\"},{\"name\":\"ssl_client_side_version\",\"type\":\"string\"},{\"name\":\"ssl_cert_verify\",\"type\":\"long\"},{\"name\":\"ssl_error\",\"type\":\"string\"},{\"name\":\"ssl_con_latency_ms\",\"type\":\"long\"},{\"name\":\"ssl_ja3_fingerprint\",\"type\":\"string\"},{\"name\":\"ssl_ja3_hash\",\"type\":\"string\"},{\"name\":\"ssl_cert_issuer\",\"type\":\"string\"},{\"name\":\"ssl_cert_subject\",\"type\":\"string\"},{\"name\":\"quic_version\",\"type\":\"string\"},{\"name\":\"quic_sni\",\"type\":\"string\"},{\"name\":\"quic_user_agent\",\"type\":\"string\"},{\"name\":\"ftp_account\",\"type\":\"string\"},{\"name\":\"ftp_url\",\"type\":\"string\"},{\"name\":\"ftp_content\",\"type\":\"string\"},{\"name\":\"ftp_link_type\",\"type\":\"string\"},{\"name\":\"bgp_type\",\"type\":\"long\"},{\"name\":\"bgp_as_num\",\"type\":\"string\"},{\"name\":\"bgp_route\",\"type\":\"string\"},{\"name\":\"voip_calling_account\",\"type\":\"string\"},{\"name\":\"voip_called_account\",\"type\":\"string\"},{\"name\":\"voip_calling_number\",\"type\":\"string\"},{\"name\":\"voip_called_number\",\"type\":\"string\"},{\"name\":\"sip_call_id\",\"type\":\"string\"},{\"name\":\"sip_originator_description\",\"type\":\"string\"},{\"name\":\"sip_responder_description\",\"type\":\"string\"},{\"name\":\"sip_user_agent\",\"type\":\"string\"},{\"name\":\"sip_server\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_connect_ip\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_media_port\",\"type\":\"long\"},{\"name\":\"sip_originator_sdp_media_type\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_content\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_connect_ip\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_media_port\",\"type\":\"long\"},{\"name\":\"sip_responder_sdp_media_type\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_content\",\"type\":\"string\"},{\"name\":\"sip_duration_s\",\"type\":\"long\"},{\"name\":\"sip_bye\",\"type\":\"string\"},{\"name\":\"rtp_payload_type_c2s\",\"type\":\"long\"},{\"name\":\"rtp_payload_type_s2c\",\"type\":\"long\"},{\"name\":\"rtp_pcap_path\",\"type\":\"string\"},{\"name\":\"rtp_originator_dir\",\"type\":\"long\"},{\"name\":\"ssh_version\",\"type\":\"string\"},{\"name\":\"ssh_auth_success\",\"type\":\"string\"},{\"name\":\"ssh_client_version\",\"type\":\"string\"},{\"name\":\"ssh_server_version\",\"type\":\"string\"},{\"name\":\"ssh_cipher_alg\",\"type\":\"string\"},{\"name\":\"ssh_mac_alg\",\"type\":\"string\"},{\"name\":\"ssh_compression_alg\",\"type\":\"string\"},{\"name\":\"ssh_kex_alg\",\"type\":\"string\"},{\"name\":\"ssh_host_key_alg\",\"type\":\"string\"},{\"name\":\"ssh_host_key\",\"type\":\"string\"},{\"name\":\"ssh_hassh\",\"type\":\"string\"},{\"name\":\"stratum_cryptocurrency\",\"type\":\"string\"},{\"name\":\"stratum_mining_pools\",\"type\":\"string\"},{\"name\":\"stratum_mining_program\",\"type\":\"string\"},{\"name\":\"streaming_media_url\",\"type\":\"string\"},{\"name\":\"streaming_media_protocol\",\"type\":\"string\"},{\"name\":\"app_extra_info\",\"type\":\"string\"}]}"; + Object parse = JSONObject.parse(json); + + System.err.println(parse); + } +} +*/ diff --git a/src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java b/src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java new file mode 100644 index 0000000..506efb7 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/service/impl/DomainServiceImpl.java @@ -0,0 +1,217 @@ +package com.mesasoft.cn.sketch.service.impl; + +import cn.hutool.core.date.DateUtil; +import cn.hutool.log.Log; +import com.alibaba.fastjson.JSONArray; +import com.google.common.collect.Lists; +import com.mesasoft.cn.sketch.api.BrightCloud; +import com.mesasoft.cn.sketch.api.ChinaZ; +import com.mesasoft.cn.sketch.config.AppConfig; +import com.mesasoft.cn.sketch.config.BrightCloudConfig; +import com.mesasoft.cn.sketch.config.ChinazConfig; +import com.mesasoft.cn.sketch.dao.DomainDao; +import com.mesasoft.cn.sketch.entity.domain.DomainCategory; +import com.mesasoft.cn.sketch.entity.domain.DomainInfo; +import com.mesasoft.cn.sketch.entity.domain.DomainWhois; +import com.mesasoft.cn.sketch.service.DomainService; +import com.mesasoft.cn.sketch.util.ValidationUtils; +import org.apache.ibatis.session.ExecutorType; +import org.apache.ibatis.session.SqlSession; +import org.apache.ibatis.session.SqlSessionFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.List; + +@Service +public class DomainServiceImpl implements DomainService { + + private static final Log logger = Log.get(); + + @Autowired + private BrightCloudConfig bcConfig; + @Autowired + private ChinazConfig chinazConfig; + @Autowired + private AppConfig appConfig; + + @Autowired + private DomainDao domainDao; + @Resource + SqlSessionFactory sqlSessionFactory; + + @Override + public JSONArray getCategoryInfo(List domains, String username, boolean isLocal) throws Exception { + + domains = ValidationUtils.getCheckedFqdns(domains); + logger.info("domain category query total size {}", domains.size()); + JSONArray results = new JSONArray(); + //1. 查询本地数据库 + List> partitionDomains = Lists.partition(domains, appConfig.getDbQueryBatchSize()); + List dBDomains = new ArrayList<>(); + for (List partitionDomain : partitionDomains) { + List categoryInfos = domainDao.getCategoryInfo(partitionDomain); + results.addAll(categoryInfos); + //移除DB查询结果 + categoryInfos.forEach(domainInfo -> dBDomains.add(domainInfo.getDomain())); + } + domains.removeAll(dBDomains); + logger.info("domain category query DB record {}", results.size()); + + //2. 调用api查询 + int apiResultNum = 0; + int failedQueryNum = 0; + + if (!isLocal && domains.size() > 0) { + List bcResults = new ArrayList<>(); + //批量查询API + BrightCloud brightCloud = new BrightCloud(); + List> apiPartitions = Lists.partition(domains, bcConfig.getMaximumQueryNum()); + + for (List partition : apiPartitions) { + List bcDomainInfos = new ArrayList<>(); + List recordsFromBcApi = brightCloud.getBrightCloudDomains(partition); + for (DomainCategory record : recordsFromBcApi) { + //查询成功的结果 + if (record.getQuery_success().equals(true)) { + record.setSubmit_user(username); + bcResults.add(record); + bcDomainInfos.add(DomainInfo.builder() + .domain(record.getFqdn()) + .result(Lists.newArrayList(record)) + .build()); + apiResultNum += 1; + } else { + failedQueryNum += 1; + } + } + if (bcResults.size() > 0) { + insertBatchDomainCategory(bcResults); + results.addAll(bcDomainInfos); + bcResults.clear(); + bcDomainInfos.clear(); + } + } + // 记录api调用次数 + FileWriter fileWriter = new FileWriter(bcConfig.getUsereportFilePath(), true); + fileWriter.write(DateUtil.date() + "," + " domain category query brightCloud success record " + apiResultNum + ",fail query record " + failedQueryNum + "\n"); + fileWriter.close(); + logger.info("domain category query brightCloud success record " + apiResultNum + ",fail query record " + failedQueryNum); + } + return results; + } + + @Override + public JSONArray getWhoisInfo(List domains, boolean isLocal) throws Exception { + domains = ValidationUtils.getCheckedFqdns(domains); + logger.info("domain whois query total size {}", domains.size()); + + JSONArray results = new JSONArray(); + //1.查询本地数据库 + List dBDomains = new ArrayList<>(); + List> partitions = Lists.partition(domains, appConfig.getDbQueryBatchSize()); + for (List domainWhois : partitions) { + //批量查询 + List dbRecords = domainDao.getWhoisInfo(domainWhois); + results.addAll(dbRecords); + dbRecords.forEach(domainInfo -> dBDomains.add(domainInfo.getDomain())); + } + domains.removeAll(dBDomains); + logger.info("domain whois query DB record {}", results.size()); + + //2. 调用api + int apiResultNum = 0; + int failedQueryNum = 0; + + if (!isLocal && domains.size() > 0) { + List chinazResults = new ArrayList<>(); + ChinaZ chinaz = new ChinaZ(); + List> apiPartitions = Lists.partition(domains, chinazConfig.getMaximumQueryNum()); + + for (List partition : apiPartitions) { + List chinazDomainInfos = new ArrayList<>(); + List recordsFromApi = chinaz.getChinaZDomains(partition); + for (DomainWhois record : recordsFromApi) { + if (record.getQuery_success().equals(true)) { + // 查询成功的结果 + chinazResults.add(record); + chinazDomainInfos.add(DomainInfo.builder() + .domain(record.getFqdn()) + .result(Lists.newArrayList(record)) + .build()); + apiResultNum += 1; + } else { // 查询失败的结果 + failedQueryNum += 1; + } + } + if (chinazResults.size() > 0) { + insertBatchDomainWhois(chinazResults); + results.addAll(chinazDomainInfos); + chinazResults.clear(); + chinazDomainInfos.clear(); + } + } + // 记录api调用次数 + FileWriter fileWriter = new FileWriter(chinazConfig.getUsereportFilePath(), true); + fileWriter.write(DateUtil.date() + "," + " domain whois query chinaz success record " + apiResultNum + ",fail query record " + failedQueryNum + "\n"); + fileWriter.close(); + logger.info("domain whois query chinaz success record " + apiResultNum + ",fail query record " + failedQueryNum); + } + return results; + } + + /** + * 域名分类 批量入库 + * + * @param dataList + * @return + */ + public int insertBatchDomainCategory(List dataList) { + + SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false); + try { + DomainDao domainDaoBatch = sqlSession.getMapper(DomainDao.class); + dataList.stream().forEach(data -> domainDaoBatch.insertCategory(data)); + sqlSession.commit(); + sqlSession.clearCache(); + logger.info("insert domain category data success ,data size {}", dataList.size()); + + } catch (Exception e) { + logger.error("domainCategory insert error, {}", e); + sqlSession.rollback(); + return 0; + } finally { + sqlSession.close(); + } + return dataList.size(); + } + + /** + * 域名whois入库 + * + * @param dataList + * @return + */ + public int insertBatchDomainWhois(List dataList) { + + SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false); + try { + DomainDao domainDaoBatch = sqlSession.getMapper(DomainDao.class); + dataList.stream().forEach(data -> domainDaoBatch.insertWhois(data)); + sqlSession.commit(); + sqlSession.clearCache(); + logger.info("insert domain whois data success ,data size {}", dataList.size()); + } catch (Exception e) { + logger.error("domainWhois insert error, {}", e); + sqlSession.rollback(); + return 0; + } finally { + sqlSession.close(); + } + return dataList.size(); + } + +} diff --git a/src/main/java/com/mesasoft/cn/sketch/util/ConfigUtils.java b/src/main/java/com/mesasoft/cn/sketch/util/ConfigUtils.java new file mode 100644 index 0000000..5df40ea --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/util/ConfigUtils.java @@ -0,0 +1,46 @@ +package com.mesasoft.cn.sketch.util; + + +import org.apache.log4j.Logger; + +import java.util.Properties; + +public class ConfigUtils { + private static final Logger LOG = Logger.getLogger(ConfigUtils.class); + private static Properties propCommon = new Properties(); + + public static String getStringProperty(String key) { + return propCommon.getProperty(key); + } + + public static Integer getIntProperty(String key) { + return Integer.parseInt(propCommon.getProperty(key)); + } + + public static Long getLongProperty(String key) { + return Long.parseLong(propCommon.getProperty(key)); + } + + public static Boolean getBooleanProperty(String key) { + return "true".equals(propCommon.getProperty(key).toLowerCase().trim()); + } + + public static String getEffectiveString(String s) { + if (!(s == null)) { + return s.length() == 0 ? null : s; + } else { + return null; + } + } + + static { + try { + propCommon.load(ConfigUtils.class.getClassLoader().getResourceAsStream("application.properties")); +// propCommon.load(MariaDbConnection.class.getClassLoader().getResourceAsStream("mariadb.properties")); + + } catch (Exception e) { + propCommon = null; + LOG.error("配置加载失败"); + } + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/util/FileUtils.java b/src/main/java/com/mesasoft/cn/sketch/util/FileUtils.java new file mode 100644 index 0000000..6e0738a --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/util/FileUtils.java @@ -0,0 +1,197 @@ +package com.mesasoft.cn.sketch.util; + +import org.apache.log4j.Logger; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +/** + * @author yjy + * @version 1.0 + * @date 2021/2/25 6:11 下午 + */ +public class FileUtils { + private static final Logger LOG = Logger.getLogger(FileUtils.class); + + public static List readTxtFileIntoStringArrList(String filePath) + { + List list = new ArrayList<>(); + try + { + String encoding = "GBK"; + File file = new File(filePath); + if (file.isFile() && file.exists()) + { // 判断文件是否存在 + InputStreamReader read = new InputStreamReader( + new FileInputStream(file), encoding); + BufferedReader bufferedReader = new BufferedReader(read); + String lineTxt = null; + + while ((lineTxt = bufferedReader.readLine()) != null) + { + if (!lineTxt.equals("")) { + list.add(lineTxt.trim()); + } + } + bufferedReader.close(); + read.close(); + } + else + { + System.out.println("Can not find file: " + filePath); + } + } + catch (Exception e) + { + System.out.println("Error occurred in Function 'readTxtFileIntoStringArrList'"); + e.printStackTrace(); + } + + return list; + } + + public static List getBatchLineReadIn(BufferedReader bufferedReader, int batchSize){ + List list = new ArrayList<>(); + String lineTxt; + try{ + while ((lineTxt = bufferedReader.readLine()) != null && list.size() lengthDomain) { + lengthDomain = split.length; + } + } + read.close(); + } else { + LOG.error("FilePath is wrong--->{" + filePath + "}<---"); + } + } catch (Exception e) { + LOG.error("Get filePathData error--->{" + e + "}<---"); + e.printStackTrace(); + } + return lengthDomain; + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/util/JsonTool.java b/src/main/java/com/mesasoft/cn/sketch/util/JsonTool.java new file mode 100644 index 0000000..544eec9 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/util/JsonTool.java @@ -0,0 +1,70 @@ +package com.mesasoft.cn.sketch.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.parser.Feature; +import com.alibaba.fastjson.serializer.SerializerFeature; +import org.apache.commons.lang3.StringUtils; + +/** + * Created with IntelliJ IDEA. + * User: joy + * Date: 2022/1/3 + * Time: 7:12 PM + * Description: No Description + */ +public class JsonTool { + private boolean isTab = true; + public String stringToJSON(String strJson) { + int tabNum = 0; + StringBuffer jsonFormat = new StringBuffer(); + int length = strJson.length(); + for (int i = 0; i < length; i++) { + char c = strJson.charAt(i); + if (c == '{') { + tabNum++; + jsonFormat.append(c + "\n"); + jsonFormat.append(getSpaceOrTab(tabNum)); + } else if (c == '}') { + tabNum--; + jsonFormat.append("\n"); + jsonFormat.append(getSpaceOrTab(tabNum)); + jsonFormat.append(c); + } else if (c == ',') { + jsonFormat.append(c + "\n"); + jsonFormat.append(getSpaceOrTab(tabNum)); + } else { + jsonFormat.append(c); + } + } + return jsonFormat.toString(); + } + public String getSpaceOrTab(int tabNum) { + StringBuffer sbTab = new StringBuffer(); + for (int i = 0; i < tabNum; i++) { + if (isTab) { + sbTab.append('\t'); + } else { + sbTab.append(" "); + } + } + return sbTab.toString(); + } + + + public static String prettyJson(String jsonString){ + if(StringUtils.isBlank(jsonString)){ + return jsonString; + } + String pretty; + try { + JSONObject object = JSONObject.parseObject(jsonString, Feature.OrderedField); + pretty = JSON.toJSONString(object, SerializerFeature.PrettyFormat, SerializerFeature.WriteMapNullValue, + SerializerFeature.WriteDateUseDateFormat); + + }catch (Exception e){ + return jsonString; + } + return pretty; + } +} diff --git a/src/main/java/com/mesasoft/cn/sketch/util/MariaDBUtils.java b/src/main/java/com/mesasoft/cn/sketch/util/MariaDBUtils.java new file mode 100644 index 0000000..7032147 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/util/MariaDBUtils.java @@ -0,0 +1,133 @@ +package com.mesasoft.cn.sketch.util; + +import cn.hutool.core.date.DateUtil; +import com.alibaba.druid.pool.DruidDataSourceFactory; +import me.geso.jdbcutils.JDBCUtils; +import org.apache.commons.lang3.time.StopWatch; + +import javax.sql.DataSource; +import java.math.BigInteger; +import java.sql.*; +import java.util.Properties; + +/** + * Druid连接池的工具类 + */ +public class MariaDBUtils { + private static DataSource ds; + + static { + try { + Properties pro = new Properties(); + pro.load(JDBCUtils.class.getClassLoader().getResourceAsStream("druid.properties")); + ds = DruidDataSourceFactory.createDataSource(pro); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 获取连接 + */ + public static Connection getConnection() throws SQLException { + return ds.getConnection(); + } + + /** + * 释放资源 + */ + public static void close(Statement stmt, Connection conn) { + + close(null, stmt, conn); + } + + + public static void close(ResultSet rs, Statement stmt, Connection conn) { + + + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + + if (stmt != null) { + try { + stmt.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + /** + * 获取连接池方法 + */ + public static DataSource getDataSource() { + return ds; + } + + public static void main(String[] args) { + + try { + Class.forName("ru.yandex.clickhouse.ClickHouseDriver");// 驱动包 + String url = "jdbc:clickhouse://192.168.44.85:8123/tsg_galaxy_v3";// url路径 + String user = "tsg_insert";// 账号 + String password = "galaxy2019";// 密码 + Connection connection = DriverManager.getConnection(url, user, password); + + + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + System.err.println("开始写入ck数据 ：{}"); + + connection.setAutoCommit(false); + + PreparedStatement preparedStatement = null; + long time = DateUtil.currentSeconds(); + time = time - 1; +// String sql = "INSERT INTO tsg_galaxy_v3.connection_record_log_local" + +// "(cap_ip,recv_ip,src_ip,dst_ip,src_port,dst_port,addr_type,protocol,fxo_id,link_status,dir_status,total_cs_pkts,total_sc_pkts,total_cs_bytes,total_sc_bytes,log_gen_time,aa,wv,yy,user_mask,conn_start_time,app_class,app_id,http_host,http_url,http_cookie,`http_user-agent`,http_method,http_accept,http_accept_encoding,http_referer,http_rescode,tls_sni,tls_cert,phone_num,imei,imsi) VALUES " + +// "('127.0.0.1', '192.168.44.12', 'aaa', 'String', 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 1657526027, 123456789, 123456789, 123456789, 123456789, "+ time +", 123456789, 123456789, 'String', 'String', 'String', 'String', 'String', 'String', 'String', 'String', 123456789, 'String', 'String', 123456789, 123456789, 123456789);"; + +// String sql ="INSERT INTO tsg_galaxy_v3.session_record\n" + +// "(common_recv_time, common_log_id, common_stream_trace_id, common_direction, common_stream_dir, common_start_time, common_end_time, common_con_duration_ms, common_establish_latency_ms, common_processing_time, common_ingestion_time, common_entrance_id, common_device_id, common_egress_link_id, common_ingress_link_id, common_isp, common_data_center, common_sled_ip, common_device_group, common_app_behavior, common_action, common_sub_action, common_policy_id, common_user_tags, common_user_region, common_client_ip, common_internal_ip, common_client_port, common_client_location, common_client_asn, common_subscriber_id, common_imei, common_imsi, common_phone_number, common_server_ip, common_external_ip, common_server_port, common_server_location, common_server_asn, common_app_id, common_userdefine_app_name, common_app_identify_info, common_app_label, common_app_surrogate_id, common_l7_protocol, common_protocol_label, common_service_category, common_service, common_l4_protocol, common_sessions, common_c2s_pkt_num, common_s2c_pkt_num, common_c2s_pkt_diff, common_s2c_pkt_diff, common_c2s_byte_num, common_s2c_byte_num, common_c2s_byte_diff, common_s2c_byte_diff, common_c2s_ipfrag_num, common_s2c_ipfrag_num, common_c2s_tcp_lostlen, common_s2c_tcp_lostlen, common_c2s_tcp_unorder_num, common_s2c_tcp_unorder_num, common_c2s_pkt_retrans, common_s2c_pkt_retrans, common_c2s_byte_retrans, common_s2c_byte_retrans, common_first_ttl, common_tcp_client_isn, common_tcp_server_isn, common_mirrored_pkts, common_mirrored_bytes, common_address_type, common_schema_type, common_device_tag, common_encapsulation, common_tunnels, common_address_list, common_has_dup_traffic, common_stream_error, common_link_info_c2s, common_link_info_s2c, common_packet_capture_file, http_url, http_host, http_domain, http_request_line, http_response_line, http_request_header, http_response_header, http_request_content, http_response_content, http_request_body, http_response_body, http_request_body_key, http_response_body_key, http_proxy_flag, http_sequence, http_snapshot, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_content_length, http_content_type, http_set_cookie, http_version, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_to_cmd, mail_from_cmd, mail_from, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_content, mail_content_charset, mail_attachment_name, mail_attachment_name_charset, mail_attachment_content, mail_eml_file, mail_snapshot, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_pinningst, ssl_intercept_state, ssl_passthrough_reason, ssl_server_side_latency, ssl_client_side_latency, ssl_server_side_version, ssl_client_side_version, ssl_cert_verify, ssl_error, ssl_con_latency_ms, ssl_ja3_fingerprint, ssl_ja3_hash, ssl_cert_issuer, ssl_cert_subject, quic_version, quic_sni, quic_user_agent, ftp_account, ftp_url, ftp_content, ftp_link_type, bgp_type, bgp_as_num, bgp_route, voip_calling_account, voip_called_account, voip_calling_number, voip_called_number, streaming_media_url, streaming_media_protocol, app_extra_info, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, rdp_encryption_method, rdp_encryption_level, rdp_certificate_permanent, rdp_certificate_count, rdp_certificate_type, rdp_requested_color_depth, rdp_desktop_height, rdp_desktop_width, rdp_client_product_id, rdp_client_name, rdp_client_version, rdp_keyboard_layout, rdp_client_channels, rdp_security_protocol, rdp_cookie)\n" + +// "VALUES("+time+", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', 0, 0, '', '', '', '', '', 0, '', 0, '', '', '', '', 0, '', '', '', '', '', '', '', '', 0, '', '', '', '', '', '', '', '', '', null, 0, '', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', '', 0, '', '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', 0, 0, '', 0, '', 0, '', '', '', '', 0, 0, '', 0, 0, '', '', 0, '', 0, '', '', '', '', '', '', '', '', '', '', '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, '', '', '', 0, '', '', 0, '', 0, 0, '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, '', '', '', '', '', '', '', '', '', '', '');\n"; + preparedStatement = connection.prepareStatement("INSERT INTO tsg_galaxy_v3.connection_relation_log" + + "(start_time, src_ip, dst_ip,bytes)" + + "VALUES(?, ?, ?, ?)"); + for (int i = 0; i < 10; i++) { + preparedStatement.setLong(1, time); + preparedStatement.setString(2, "127.0.0.1"); + preparedStatement.setString(3, "127.0.0.1"); + BigInteger bytes = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(Long.MAX_VALUE)); + System.err.println(bytes); + System.err.println(bytes.bitLength()); + preparedStatement.setObject(4, bytes); + preparedStatement.addBatch(); + } + preparedStatement.executeBatch(); + connection.commit(); + stopWatch.stop(); + System.err.println("总共花费时间 {}" + stopWatch.getTime()); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + + + } + +} \ No newline at end of file diff --git a/src/main/java/com/mesasoft/cn/sketch/util/TimeUtils.java b/src/main/java/com/mesasoft/cn/sketch/util/TimeUtils.java new file mode 100644 index 0000000..800126c --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/util/TimeUtils.java @@ -0,0 +1,59 @@ +package com.mesasoft.cn.sketch.util; + +import java.util.Calendar; +import java.util.Date; + +/** + * @author yjy + * @version 1.0 + * @date 2021/2/25 11:26 上午 + */ +public class TimeUtils { + public static final Long HOUR_TO_MILLISECONDS = 3600000L; + public static final Long DAY_TO_MILLSEDONDS = 86400000L; + public static final Integer SECOND_TO_MILLSEDONDS = 1000; + + + /** + * 获得当前时间小时的起始（0分钟）时间 + */ + public static Date getStartOfHour() { + return getStartOfHour(0); + } + public static Date getStartOfHour(Integer offset) { + Calendar ca = Calendar.getInstance(); + ca.add(Calendar.HOUR, offset); + ca.set(Calendar.MINUTE, 0); + ca.set(Calendar.SECOND, 0); + ca.set(Calendar.MILLISECOND, 0); + return ca.getTime(); + } + + /** + * 获得当前日期的起始（0时）时间 + */ + public static Date getStartOfDay() { + return getStartOfDay(0); + } + public static Date getStartOfDay(Integer bias) { + Calendar ca = Calendar.getInstance(); + ca.add(Calendar.DATE, bias); + ca.set(Calendar.HOUR, -12); + ca.set(Calendar.MINUTE, 0); + ca.set(Calendar.SECOND, 0); + ca.set(Calendar.MILLISECOND, 0); + return ca.getTime(); + } + + public static Date getStartOfMonth() { + Calendar ca = Calendar.getInstance(); + ca.set(Calendar.DATE, 1); + ca.set(Calendar.HOUR, -12); + ca.set(Calendar.MINUTE, 0); + ca.set(Calendar.SECOND, 0); + ca.set(Calendar.MILLISECOND, 0); + return ca.getTime(); + } + +} + diff --git a/src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java b/src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java new file mode 100644 index 0000000..76eaa46 --- /dev/null +++ b/src/main/java/com/mesasoft/cn/sketch/util/ValidationUtils.java @@ -0,0 +1,207 @@ +package com.mesasoft.cn.sketch.util; + +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Value; +import sun.net.util.IPAddressUtil; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ValidationUtils { + private static final Logger LOG = Logger.getLogger(ValidationUtils.class); + + /** + * 获取二级域名 + **/ + + private static String tldFilePath; + + @Value("${sketch.tld.file}") + public void setTldFilePath(String ttlFilePath){ + this.tldFilePath= ttlFilePath; + } + + public static String getSecDomain(String fqdnOrUrl){ + + HashMap> maps = readTopDomainFile(tldFilePath); + try { + String[] split = fqdnOrUrl.split("\\."); + String secDomain = null; + for (int i = split.length - 1; i >= 0; i--) { + int maps_index = split.length - (i + 1); + HashMap innerMap = maps.get("map_id_" + maps_index); + HashMap fullTop = maps.get("full"); + if (!(innerMap.containsKey(split[i]))) { + String strSec = ""; + for (int j = i; j < split.length; j++) { + strSec += (split[j] + "."); + } + secDomain = strSec.substring(0, strSec.length() - 1); + if (fullTop.containsKey(getTopFromSecDomain(secDomain))) { + break; + } else { + while (!fullTop.containsKey(getTopFromSecDomain(secDomain)) && getTopFromSecDomain(secDomain).contains(".")) { + secDomain = getTopFromSecDomain(secDomain); + } + break; + } + } + } + // 右匹配为顶级域名 + if (secDomain == null){ + secDomain = fqdnOrUrl; + } + return secDomain; + } catch (Exception e) { + LOG.error("urlDomain:" + fqdnOrUrl); + e.printStackTrace(); + return "---no---return---"; + } + } + + public static List getSecDomain(List fqdnOrUrls) { + HashMap> maps = readTopDomainFile(tldFilePath); + List secDomainList = new ArrayList<>(); + for (String oriDomain : fqdnOrUrls) { + String secDomain = getSecDomain(oriDomain); + if (StringUtils.isNotBlank(secDomain) && !("---no---return---".equals(secDomain))) { + secDomainList.add(secDomain); + } else { + System.out.println(oriDomain); + } + } + return secDomainList; + } + + public static String getTopFromSecDomain(String secDomain) { + String quFirstDian = secDomain; + if (secDomain.contains(".")) { + quFirstDian = secDomain.substring(secDomain.indexOf(".")).substring(1); + } + return quFirstDian; + } + + public static HashMap> readTopDomainFile(String filePath) { + HashMap> maps = makeHashMap(filePath); + try { + String encoding = "UTF-8"; + File file = new File(filePath); + if (file.isFile() && file.exists()) { + InputStreamReader read = new InputStreamReader( + new FileInputStream(file), encoding); + BufferedReader bufferedReader = new BufferedReader(read); + String lineTxt = null; + while ((lineTxt = bufferedReader.readLine()) != null) { + HashMap fullTop = maps.get("full"); + fullTop.put(lineTxt, lineTxt); + maps.put("full", fullTop); + String[] split = lineTxt.split("\\."); + for (int i = split.length - 1; i >= 0; i--) { + int maps_index = split.length - (i + 1); + HashMap innerMap = maps.get("map_id_" + maps_index); + innerMap.put(split[i], split[i]); + maps.put("map_id_" + maps_index, innerMap); + } + } + read.close(); + } else { + LOG.error("TopDomainUtils>=>readTopDomainFile filePath is wrong--->{" + filePath + "}<---"); + } + } catch (Exception e) { + LOG.error("TopDomainUtils>=>readTopDomainFile get filePathData error--->{" + e + "}<---"); + e.printStackTrace(); + } + return maps; + } + + public static HashMap> makeHashMap(String filePath) { + int maxLength = FileUtils.getMaxLength(filePath); + HashMap> maps = new HashMap>(); + for (int i = 0; i < maxLength; i++) { + maps.put("map_id_" + i, new HashMap()); + } + maps.put("full", new HashMap()); + return maps; + } + + public static List getChecked(List objectList, String type){ + if (type.equals("ip")){ + return getCheckedIps(objectList); + } + if (type.equals("domain")){ + return getCheckedFqdns(objectList); + } + LOG.error("Wrong type to be checked: " + type); + return objectList; + } + + public static List getCheckedFqdns(List fqdns){ + List res = new ArrayList<>(); + for (String fqdn:fqdns){ + //去端口号 + fqdn = fqdn.split(":")[0]; + // 去重 & 校验 + if (isValidDomain(fqdn) && !res.contains(fqdn)){ + res.add(fqdn.toLowerCase()); + } else { + LOG.debug("Bad or duplicated fqdn:" + fqdn); + } + } + return res; + } + + public static List getCheckedIps(List ipList){ + List res = new ArrayList<>(); + for (String ip:ipList){ + //去端口号 + ip = ip.split(":")[0]; + // 去重 & 校验 + if (isValidIp(ip) && !res.contains(ip)){ + res.add(ip.toLowerCase()); + } else { + LOG.debug("Bad or duplicated fqdn:" + ip); + } + } + return res; + } + + public static boolean isValidIp(String ip){ + boolean iPv4LiteralAddress = IPAddressUtil.isIPv4LiteralAddress(ip); + boolean iPv6LiteralAddress = IPAddressUtil.isIPv6LiteralAddress(ip); + return iPv4LiteralAddress || iPv6LiteralAddress; + } + + private static boolean isValidDomain(String str) + { + String regex = "^((?!-)[A-Za-z0-9-_]" + + "{1,63}(?