gfwleak/yinjiangyi-webskt-query-agent
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V1版本定时查询更新

Browse Source
This commit is contained in:
yinjiangyi
2021-06-17 10:06:34 +08:00
parent 33f504abee
commit 11566a272a
20 changed files with 2397 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
pom.xml Normal file
View File

@@ -0,0 +1,99 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>iie.ac.cn</groupId>
<artifactId>QueryAgent</artifactId>
<version>1.0-SNAPSHOT</version>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>cn.ac.iie.MainScheduleTasks</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>7</source>
<target>7</target>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>org.mariadb.jdbc</groupId>
<artifactId>mariadb-java-client</artifactId>
<version>2.7.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.6</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
<version>2.5.6</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.31</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.0.9</version>
</dependency>
<dependency>
<groupId>me.geso</groupId>
<artifactId>jdbcutils</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>21.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
</dependencies>
</project>

55
src/main/java/cn/ac/iie/MainScheduleTasks.java Normal file
View File

@@ -0,0 +1,55 @@
package cn.ac.iie;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.service.OfflineTask;
import cn.ac.iie.service.UpdateTask;
import cn.ac.iie.utils.TimeUtils;
import org.apache.log4j.Logger;
import java.util.Calendar;
import java.util.Date;
import java.util.Timer;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 11:27 上午
*/
public class MainScheduleTasks {
private static final Logger LOG = Logger.getLogger(MainScheduleTasks.class);
private Date offlineStartTime;
/**
* offline 每小时扫描一次离线目录
* update 每天00:00更新一次库表内容
*/
private void Timers() {
if (ApplicationConfig.OFFLINE_SWITCH){
LOG.info("Start offline schedule task");
Timer offlineTimer = new Timer();
Date offlineStartTime = Calendar.getInstance().getTime();
offlineTimer.schedule(new OfflineTask(), offlineStartTime,
ApplicationConfig.OFFLINE_SCHEDULE_SECOND * TimeUtils.SECOND_TO_MILLSEDONDS);
}
if (ApplicationConfig.UPDATE_SWITCH){
LOG.info("Start update schedule task");
Timer updateTimer = new Timer();
Date updateStartTime = TimeUtils.getStartOfDay(1);
updateTimer.schedule(new UpdateTask(), updateStartTime,
ApplicationConfig.UPDATE_SCHEDULE_DAY * TimeUtils.DAY_TO_MILLSEDONDS);
}
}
public static void main(String[] args) {
MainScheduleTasks tasks = new MainScheduleTasks();
tasks.Timers();
}
}

33
src/main/java/cn/ac/iie/config/ApplicationConfig.java Normal file
View File

@@ -0,0 +1,33 @@
package cn.ac.iie.config;
import cn.ac.iie.utils.ConfigUtils;
public class ApplicationConfig {
public static final Boolean OFFLINE_SWITCH = ConfigUtils.getBooleanProperty("offline.switch");
public static final String OFFLINE_IMPORT_FILENAME_SUFFIX = ConfigUtils.getStringProperty("offline.import.filename.suffix");
public static final String OFFLINE_IMPORT_FILEDONE_SUFFIX = ConfigUtils.getStringProperty("offline.import.filedone.suffix");
public static final String OFFLINE_OUTPUT_FILENAME_SUFFIX = ConfigUtils.getStringProperty("offline.output.filename.suffix");
public static final Integer OFFLINE_SCHEDULE_SECOND = ConfigUtils.getIntProperty("offline.schedule.second");
public static final String OFFLINE_IMPORT_PATH = ConfigUtils.getStringProperty("offline.import.path");
public static final String OFFLINE_OUTPUT_PATH = ConfigUtils.getStringProperty("offline.output.path");
public static final Boolean UPDATE_SWITCH = ConfigUtils.getBooleanProperty("update.switch");
public static final Integer UPDATE_SCHEDULE_DAY = ConfigUtils.getIntProperty("update.schedule.day");
public static final Integer QUERY_STANDARD_FQDN_LEVEL = ConfigUtils.getIntProperty("query.standard.fqdn.level");
public static final Boolean QUERY_URL_INFO_SWITCH = ConfigUtils.getBooleanProperty("query.url_info.switch");
public static final Boolean QUERY_URL_REP_SWITCH = ConfigUtils.getBooleanProperty("query.url_rep.switch");
public static final Boolean QUERY_URL_WHOIS_SWITCH = ConfigUtils.getBooleanProperty("query.url_whois.switch");
public static final Integer LOG_OFFLINE_NUMBER = ConfigUtils.getIntProperty("log.offline.number");
public static final String LOG_BC_QUERY_REPORT_FILE = ConfigUtils.getStringProperty("log.bc.query.report.file");
public static final Integer MAXIMUM_URL_ONCE = ConfigUtils.getIntProperty("maximum.url.once");
}

16
src/main/java/cn/ac/iie/config/MariaDBConfig.java Normal file
View File

@@ -0,0 +1,16 @@
package cn.ac.iie.config;
import cn.ac.iie.utils.ConfigUtils;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 11:09 上午
*/
public class MariaDBConfig {
public static final String MARIADB_DATABASE = ConfigUtils.getStringProperty("database");
public static final String MARIADB_TABLE = ConfigUtils.getStringProperty("table");
}

177
src/main/java/cn/ac/iie/dao/BaseMariaDB.java Normal file
View File

@@ -0,0 +1,177 @@
package cn.ac.iie.dao;
import cn.ac.iie.config.MariaDBConfig;
import cn.ac.iie.utils.TimeUtils;
import org.apache.log4j.Logger;
import java.sql.*;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Properties;
/**
* @author yjy
* @version 1.0
* @date 2021/2/19 6:25 下午
* //TODO 非持久连接，一次实例化满足一次数据库操作
*/
public class BaseMariaDB {
private static final Logger LOG = Logger.getLogger(BaseMariaDB.class);
private static final Properties props = new Properties();
private Statement statement;
public BaseMariaDB(Connection conn, Statement stat) {
statement = stat;
}
public void insertRecords(List<FqdnFile> fqdnFiles){
for (FqdnFile fqdnFile : fqdnFiles) {
fqdnFileWrite(fqdnFile, "insert");
}
}
public void updateRecords(List<FqdnFile> fqdnFiles){
for (FqdnFile fqdnFile : fqdnFiles) {
fqdnFileWrite(fqdnFile, "update");
}
long last = System.currentTimeMillis();
}
/**
* 单条FqdnFile写入操作：更新、插入
*/
public void fqdnFileWrite(FqdnFile fqdnFile, String method){
String sql = null;
List<String> methods = Arrays.asList("insert", "update");
assert methods.contains(method);
if (method.equals("insert")){
sql = getInsertSql(fqdnFile);
} else {
sql = getUpdateSql(fqdnFile);
}
writeSqlExecute(sql);
}
public ResultSet getExpiredRecord(){
return querySqlExecute(getExpiredRecordSql());
}
public ResultSet getUnlabelRecord(){
return querySqlExecute(getUnlabelRecordSql());
}
public ResultSet getDatabaseRecord(String fqdn){
return querySqlExecute(getQueryRecordSql(fqdn));
}
public void writeSqlExecute(String sql){
try {
statement.executeUpdate(sql);
} catch (SQLException exception) {
LOG.debug("Sql : " + sql);
exception.printStackTrace();
}
}
public ResultSet querySqlExecute(String sql){
ResultSet set = null;
try {
set = statement.executeQuery(sql);
} catch (SQLException exception) {
exception.printStackTrace();
}
return set;
}
public Boolean isInDatabase(String standardFqdn){
boolean isInDb = false;
try {
String querySql = getQueryExistSql(standardFqdn);
ResultSet set = statement.executeQuery(querySql);
set.next();
isInDb = set.getInt("COUNT(*)") > 0;
} catch (SQLException e) {
e.printStackTrace();
}
return isInDb;
}
public String getInsertSql(FqdnFile fqdnFile){
long start = System.currentTimeMillis();
StringBuilder sql = new StringBuilder("INSERT INTO ");
sql.append(MariaDBConfig.MARIADB_DATABASE).append(".")
.append(MariaDBConfig.MARIADB_TABLE).append(' ');
sql.append(" (fqdn,query_success,reputation_score,reputation_level,")
.append("category_id,category_name,category_group,category_conf,is_a1_cat,popularity,observ_age,country,threat_history,")
.append("whois_domain,whois_update_date,whois_create_date,whois_expire_date,whois_email,whois_ns,")
.append("whois_registrar_name,whois_registrant_org,whois_regisrante_name,whois_regisrante_street,")
.append("whois_regisrante_city,whois_regisrante_state,whois_regisrante_postcode,whois_regisrante_country,")
.append("whois_regisrante_phone) values");
sql.append('(').append(fqdnFile.getValues()).append(')');
String resSql = sql.toString();
resSql = resSql.replace("'null'", "null");
return resSql;
}
public String getUpdateSql(FqdnFile fqdnFile){
StringBuilder sql = new StringBuilder("UPDATE ");
sql.append(MariaDBConfig.MARIADB_DATABASE).append(".")
.append(MariaDBConfig.MARIADB_TABLE).append(' ')
.append("SET ").append(fqdnFile.getKeyValues())
.append(", update_time = current_time() ")
.append(" WHERE fqdn = '").append(fqdnFile.getFqdn()).append('\'');
String resSql = sql.toString();
resSql = resSql.replace("'null'", "null");
return resSql;
}
public String getQueryExistSql(String fqdn){
StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM ");
sql.append(MariaDBConfig.MARIADB_DATABASE).append(".")
.append(MariaDBConfig.MARIADB_TABLE).append(' ')
.append(" WHERE fqdn = '").append(fqdn).append('\'');
return sql.toString();
}
public String getQueryRecordSql(String fqdn){
StringBuilder sql = new StringBuilder("SELECT * FROM ");
sql.append(MariaDBConfig.MARIADB_DATABASE).append(".")
.append(MariaDBConfig.MARIADB_TABLE).append(' ')
.append(" WHERE fqdn = '").append(fqdn).append('\'');
String resSql = sql.toString();
resSql = resSql.replace("'null'", "null");
return resSql;
}
public static String getExpiredRecordSql(){
Date lastUpdateTime = new Timestamp(TimeUtils.getExpiredTime().getTime());
String resSql = "SELECT x.fqdn FROM " + MariaDBConfig.MARIADB_DATABASE + "." +
MariaDBConfig.MARIADB_TABLE +
" x WHERE x.update_time < '" + lastUpdateTime + '\'';
LOG.debug("Update task: expired query sql" + resSql);
return resSql;
}
public static String getUnlabelRecordSql(){
return "SELECT x.fqdn FROM " + MariaDBConfig.MARIADB_DATABASE + "." +
MariaDBConfig.MARIADB_TABLE +
" x WHERE x.category_id = 0 or x.query_success != 1";
}
}

449
src/main/java/cn/ac/iie/dao/FqdnFile.java Normal file
View File

@@ -0,0 +1,449 @@
package cn.ac.iie.dao;
import java.util.Date;
/**
* @author yjy
* @version 1.0
* @date 2021/2/22 7:40 下午
* field name 和 mariadb 一致
*
*/
public class FqdnFile
{
private String fqdn;
private Boolean query_success;
private Integer reputation_score;
private String reputation_level;
private Integer category_id;
private String category_name;
private String category_group;
private Integer category_conf;
private Boolean is_a1_cat;
private Integer popularity;
private Integer observ_age;
private String country;
private Integer threat_history;
private String whois_domain;
private java.sql.Timestamp whois_update_date;
private java.sql.Timestamp whois_create_date;
private java.sql.Timestamp whois_expire_date;
private String whois_email;
private String whois_ns;
private String whois_registrar_name;
private String whois_registrant_org;
private String whois_regisrante_name;
private String whois_regisrante_street;
private String whois_regisrante_city;
private String whois_regisrante_state;
private String whois_regisrante_postcode;
private String whois_regisrante_country;
private String whois_regisrante_phone;
// update
public FqdnFile(String fqdn, Boolean query_success,
Integer reputation_score, String reputationLevel,
Integer categoryId, String categoryName, String categoryGroup, Integer categoryConf, Boolean isA1Cat,
Integer popularity, Integer observAge, String country, Integer threatHistory,
String whoisDomain, Date whoisUpdateDate, Date whoisCreateDate, Date whoisExpireDate,
String whoisEmail, String whoisNs, String whoisRegistrarName, String whoisRegistrantOrg,
String whoisRegisranteName, String whoisRegisranteStreet, String whoisRegisranteCity,
String whoisRegisranteState, String whoisRegisrantePostcode, String whoisRegisranteCountry,
String whoisRegisrantePhone) {
this.fqdn = fqdn;
this.query_success = query_success;
this.reputation_score = reputation_score;
this.reputation_level = getEffectiveString(reputationLevel);
this.category_id = categoryId;
this.category_name = getEffectiveString(categoryName);
this.category_group = getEffectiveString(categoryGroup);
this.category_conf = categoryConf;
this.is_a1_cat = isA1Cat;
this.popularity = popularity;
this.observ_age = observAge;
this.country = getEffectiveString(country);
this.threat_history = threatHistory;
this.whois_domain = getEffectiveString(whoisDomain);
this.whois_update_date = whoisUpdateDate == null? null : new java.sql.Timestamp(whoisUpdateDate.getTime());
this.whois_create_date = whoisCreateDate == null? null : new java.sql.Timestamp(whoisCreateDate.getTime());
this.whois_expire_date = whoisExpireDate == null? null : new java.sql.Timestamp(whoisExpireDate.getTime());
this.whois_email = getEffectiveString(whoisEmail);
this.whois_ns = getEffectiveString(whoisNs);
this.whois_registrar_name = getEffectiveString(whoisRegistrarName);
this.whois_registrant_org = getEffectiveString(whoisRegistrantOrg);
this.whois_regisrante_name = getEffectiveString(whoisRegisranteName);
this.whois_regisrante_street = getEffectiveString(whoisRegisranteStreet);
this.whois_regisrante_city = getEffectiveString(whoisRegisranteCity);
this.whois_regisrante_state = getEffectiveString(whoisRegisranteState);
this.whois_regisrante_postcode = getEffectiveString(whoisRegisrantePostcode);
this.whois_regisrante_country = getEffectiveString(whoisRegisranteCountry);
this.whois_regisrante_phone = getEffectiveString(whoisRegisrantePhone);
}
private String getEffectiveString(String s){
if (!(s ==null)){
return s.length() == 0 ? null : s;
} else{
return null;
}
}
public String getFqdn() {
return fqdn;
}
public void setFqdn(String fqdn) {
this.fqdn = fqdn;
}
public Boolean getQuery_success() {
return query_success;
}
public void setQuery_success(Boolean query_success) {
this.query_success = query_success;
}
public Integer getReputation_score() {
return reputation_score;
}
public void setReputation_score(Integer reputation_score) {
this.reputation_score = reputation_score;
}
public String getReputation_level() {
return reputation_level;
}
public void setReputation_level(String reputation_level) {
this.reputation_level = reputation_level;
}
public Integer getCategory_id() {
return category_id;
}
public void setCategory_id(Integer category_id) {
this.category_id = category_id;
}
public String getCategory_name() {
return category_name;
}
public void setCategory_name(String category_name) {
this.category_name = category_name;
}
public String getCategory_group() {
return category_group;
}
public void setCategory_group(String category_group) {
this.category_group = category_group;
}
public Integer getCategory_conf() {
return category_conf;
}
public void setCategory_conf(Integer category_conf) {
this.category_conf = category_conf;
}
public Boolean getA1Cat() {
return is_a1_cat;
}
public void setA1Cat(Boolean a1Cat) {
is_a1_cat = a1Cat;
}
public Integer getPopularity() {
return popularity;
}
public void setPopularity(Integer popularity) {
this.popularity = popularity;
}
public String getCountry() {
return country;
}
public void setCountry(String country) {
this.country = country;
}
public Integer getObserv_age() {
return observ_age;
}
public void setObserv_age(Integer observ_age) {
this.observ_age = observ_age;
}
public Integer getThreat_history() {
return threat_history;
}
public void setThreat_history(Integer threat_history) {
this.threat_history = threat_history;
}
public String getWhois_domain() {
return whois_domain;
}
public void setWhois_domain(String whois_domain) {
this.whois_domain = whois_domain;
}
public Date getWhois_update_date() {
return whois_update_date;
}
public void setWhois_update_date(java.sql.Timestamp whois_update_date) {
this.whois_update_date = whois_update_date;
}
public Date getWhoisCreateDate() {
return whois_create_date;
}
public void setWhoisCreateDate(java.sql.Timestamp whoisCreateDate) {
this.whois_create_date = whoisCreateDate;
}
public Date getWhois_expire_date() {
return whois_expire_date;
}
public void setWhois_expire_date(java.sql.Timestamp whois_expire_date) {
this.whois_expire_date = whois_expire_date;
}
public String getWhois_email() {
return whois_email;
}
public void setWhois_email(String whois_email) {
this.whois_email = whois_email;
}
public String getWhois_ns() {
return whois_ns;
}
public void setWhois_ns(String whois_ns) {
this.whois_ns = whois_ns;
}
public String getWhois_registrar_name() {
return whois_registrar_name;
}
public void setWhois_registrar_name(String whois_registrar_name) {
this.whois_registrar_name = whois_registrar_name;
}
public String getWhois_registrant_org() {
return whois_registrant_org;
}
public void setWhois_registrant_org(String whois_registrant_org) {
this.whois_registrant_org = whois_registrant_org;
}
public String getWhois_regisrante_name() {
return whois_regisrante_name;
}
public void setWhois_regisrante_name(String whois_regisrante_name) {
this.whois_regisrante_name = whois_regisrante_name;
}
public String getWhois_regisrante_street() {
return whois_regisrante_street;
}
public void setWhois_regisrante_street(String whois_regisrante_street) {
this.whois_regisrante_street = whois_regisrante_street;
}
public String getWhois_regisrante_city() {
return whois_regisrante_city;
}
public void setWhois_regisrante_city(String whois_regisrante_city) {
this.whois_regisrante_city = whois_regisrante_city;
}
public String getWhois_regisrante_state() {
return whois_regisrante_state;
}
public void setWhois_regisrante_state(String whois_regisrante_state) {
this.whois_regisrante_state = whois_regisrante_state;
}
public String getWhois_regisrante_postcode() {
return whois_regisrante_postcode;
}
public void setWhois_regisrante_postcode(String whois_regisrante_postcode) {
this.whois_regisrante_postcode = whois_regisrante_postcode;
}
public String getWhois_regisrante_country() {
return whois_regisrante_country;
}
public void setWhois_regisrante_country(String whois_regisrante_country) {
this.whois_regisrante_country = whois_regisrante_country;
}
public String getWhois_regisrante_phone() {
return whois_regisrante_phone;
}
public void setWhois_regisrante_phone(String whois_regisrante_phone) {
this.whois_regisrante_phone = whois_regisrante_phone;
}
@Override
public String toString() {
return "FqdnFile{" +
"fqdn='" + fqdn + '\'' +
", query_success=" + query_success +
", reputation_score=" + reputation_score +
", reputation_level='" + reputation_level + '\'' +
", category_id=" + category_id +
", category_name='" + category_name + '\'' +
", category_group='" + category_group + '\'' +
", category_conf=" + category_conf +
", is_a1_cat=" + is_a1_cat +
", popularity=" + popularity +
", observ_age=" + observ_age +
", country='" + country + '\'' +
", threat_history=" + threat_history +
", whois_domain='" + whois_domain + '\'' +
", whois_update_date=" + whois_update_date +
", whois_create_date=" + whois_create_date +
", whois_expire_date=" + whois_expire_date +
", whois_email='" + whois_email + '\'' +
", whois_ns='" + whois_ns + '\'' +
", whois_registrar_name='" + whois_registrar_name + '\'' +
", whois_registrant_org='" + whois_registrant_org + '\'' +
", whois_regisrante_name='" + whois_regisrante_name + '\'' +
", whois_regisrante_street='" + whois_regisrante_street + '\'' +
", whois_regisrante_city='" + whois_regisrante_city + '\'' +
", whois_regisrante_state='" + whois_regisrante_state + '\'' +
", whois_regisrante_postcode='" + whois_regisrante_postcode + '\'' +
", whois_regisrante_country='" + whois_regisrante_country + '\'' +
", whois_regisrante_phone='" + whois_regisrante_phone + '\'' +
'}';
}
public String getValues(){
String resString =
"'" + fqdn + '\'' +
"," + query_success +
", " + reputation_score +
", '" + reputation_level + '\'' +
", " + category_id +
", '" + category_name + '\'' +
", '" + category_group + '\'' +
", " + category_conf +
", " + is_a1_cat +
", " + popularity +
", " + observ_age +
", '" + country + '\'' +
", " + threat_history +
", '" + whois_domain + '\'' +
", '" + whois_update_date + '\'' +
", '" + whois_create_date + '\'' +
", '" + whois_expire_date + '\'' +
", '" + whois_email + '\'' +
", '" + whois_ns + '\'' +
", '" + whois_registrar_name + '\'' +
", '" + whois_registrant_org + '\'' +
", '" + whois_regisrante_name + '\'' +
", '" + whois_regisrante_street + '\'' +
", '" + whois_regisrante_city + '\'' +
", '" + whois_regisrante_state + '\'' +
", '" + whois_regisrante_postcode + '\'' +
", '" + whois_regisrante_country + '\'' +
", '" + whois_regisrante_phone + '\'' ;
return resString.replace("'null'", "null");
}
public static String getKeys() {
return "fqdn" +
", query_success" +
", reputation_score" +
", reputation_level" +
", category_id" +
", category_name" +
", category_group" +
", category_conf" +
", is_a1_cat" +
", popularity" +
", observ_age" +
", country" +
", threat_history" +
", whois_domain" +
", whois_update_date" +
", whois_create_date" +
", whois_expire_date" +
", whois_email" +
", whois_ns" +
", whois_registrar_name" +
", whois_registrant_org" +
", whois_regisrante_name" +
", whois_regisrante_street" +
", whois_regisrante_city" +
", whois_regisrante_state" +
", whois_regisrante_postcode" +
", whois_regisrante_country" +
", whois_regisrante_phone";
}
public String getKeyValues(){
String resString =
"query_success=" + query_success +
", reputation_score=" + reputation_score +
", reputation_level='" + reputation_level + '\'' +
", category_id=" + category_id +
", category_name='" + category_name + '\'' +
", category_group='" + category_group + '\'' +
", category_conf=" + category_conf +
", is_a1_cat=" + is_a1_cat +
", popularity=" + popularity +
", observ_age=" + observ_age +
", country='" + country + '\'' +
", threat_history=" + threat_history +
", whois_domain='" + whois_domain + '\'' +
", whois_update_date='" + whois_update_date + '\'' +
", whois_create_date='" + whois_create_date + '\'' +
", whois_expire_date='" + whois_expire_date + '\'' +
", whois_email='" + whois_email + '\'' +
", whois_ns='" + whois_ns + '\'' +
", whois_registrar_name='" + whois_registrar_name + '\'' +
", whois_registrant_org='" + whois_registrant_org + '\'' +
", whois_regisrante_name='" + whois_regisrante_name + '\'' +
", whois_regisrante_street='" + whois_regisrante_street + '\'' +
", whois_regisrante_city='" + whois_regisrante_city + '\'' +
", whois_regisrante_state='" + whois_regisrante_state + '\'' +
", whois_regisrante_postcode='" + whois_regisrante_postcode + '\'' +
", whois_regisrante_country='" + whois_regisrante_country + '\'' +
", whois_regisrante_phone='" + whois_regisrante_phone + '\'';
return resString.replace("'null'", "null");
}
}

237
src/main/java/cn/ac/iie/service/OfflineTask.java Normal file
View File

@@ -0,0 +1,237 @@
package cn.ac.iie.service;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.BaseMariaDB;
import cn.ac.iie.dao.FqdnFile;
import cn.ac.iie.utils.BrightCloudUtils;
import cn.ac.iie.utils.FileUtils;
import cn.ac.iie.utils.MariaDBConnect;
import cn.ac.iie.utils.LogUtils;
import com.alibaba.fastjson.JSONObject;
import org.apache.log4j.Logger;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.DecimalFormat;
import java.util.*;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 11:29 上午
*/
public class OfflineTask extends TimerTask {
private static final Logger LOG = Logger.getLogger(OfflineTask.class);
private static final LogUtils logutils = new LogUtils();
@Override
public void run() {
try {
runTask();
} catch (SQLException | IOException exception) {
exception.printStackTrace();
}
}
private void runTask() throws SQLException, IOException {
List<String> fileNames = catpureNewFiles();
List<String> fqdns;
Connection mariaConn = MariaDBConnect.getConnection();
Statement mariaStat = mariaConn.createStatement();
BaseMariaDB mariaDB = new BaseMariaDB(mariaConn, mariaStat);
BrightCloudUtils brightCloudUtils = new BrightCloudUtils();
List<String> queryTypes = BrightCloudUtils.getQueryTypes();
for (String fileName: fileNames){
File importFile = new File(fileName);
String importFileName = FileUtils.getFileName(importFile);
LOG.info("[Offline import file query]-" + importFileName + ": File Found.");
String outputFileName =
importFileName.substring(0, importFileName.length()-ApplicationConfig.OFFLINE_IMPORT_FILENAME_SUFFIX.length())
+ ApplicationConfig.OFFLINE_OUTPUT_FILENAME_SUFFIX;
// TODO 遍历处理消耗太大
fqdns = FileUtils.readTxtFileIntoStringArrList(importFile.toString());
fqdns = SingleTermTask.getCheckedFqdn(fqdns);
long standardFqdnNum = fqdns.size();
long dbQueryNum = 0;
long bcQueryNum = 0;
long failQueryNum = 0;
long effecResNum = 0;
long noLabelNum = 0;
if (standardFqdnNum>0){
// 创建结果保存文件
File outputFile = new File(ApplicationConfig.OFFLINE_OUTPUT_PATH + "/" + outputFileName);
if (!outputFile.exists()){
FileUtils.createFile(new File(ApplicationConfig.OFFLINE_OUTPUT_PATH), outputFileName);
}
OutputStream outStream = new FileOutputStream(outputFile);
OutputStreamWriter outWriter = new OutputStreamWriter(outStream, StandardCharsets.UTF_8);
outWriter.write(FqdnFile.getKeys() + "\n");
List<String> fqdnToQuery = new ArrayList<>();
// 遍历列表域名
LOG.info("[Offline import file query]-" + importFileName + " Progress: 0%");
for (int index=0; index<standardFqdnNum; index=index+1){
String fqdn = fqdns.get(index);
FqdnFile fqdnFile;
if (fqdn.equals("")){
continue;
}
ResultSet rs = mariaDB.getDatabaseRecord(fqdn);
if (rs.next()){
// 查db
fqdnFile = SingleTermTask.ResSet2FqdnFile(rs);
dbQueryNum += 1;
outWriter.write(fqdnFile.getValues() + "\n");
if (fqdnFile.getQuery_success().equals(false)){
failQueryNum += 1;
} else if (fqdnFile.getCategory_id().equals(0)){
noLabelNum += 1;
} else {
effecResNum += 1;
}
} else {
// 查bc
fqdnToQuery.add(fqdn);
bcQueryNum = bcQueryNum + 1;
// 缓存满
if (fqdnToQuery.size() == ApplicationConfig.MAXIMUM_URL_ONCE){
// 批量查
JSONObject resObj = brightCloudUtils.getQueryResults(fqdnToQuery);
List<FqdnFile> fqdnFiles = brightCloudUtils.responseSparse(resObj);
assert fqdnFiles.size()>0;
// 存数据库
mariaDB.insertRecords(fqdnFiles);
// 写入output
for (FqdnFile tmpFile:fqdnFiles){
outWriter.write(tmpFile.getValues() + "\n");
if (tmpFile.getQuery_success().equals(false)){
failQueryNum += 1;
} else if (tmpFile.getCategory_id().equals(0)){
noLabelNum += 1;
} else {
effecResNum += 1;
}
}
// // 打印处理进度日志
// String percent = new DecimalFormat("##.0%").format((float)index/standardFqdnNum);
// LOG.info("[Offline import file query]-" + importFileName + " Progress:" + percent);
// 缓存复位
fqdnToQuery = new ArrayList<>();
}
}
// 打印处理进度至日志
if (index>0 && index % ApplicationConfig.LOG_OFFLINE_NUMBER==0){
String percent = new DecimalFormat("##.0%").format((float)index/standardFqdnNum);
LOG.info("[Offline import file query]-" + importFileName + " Progress:" + percent);
}
}
// 剩余待查bc
if (fqdnToQuery.size()>0){
JSONObject resObj = brightCloudUtils.getQueryResults(fqdnToQuery);
List<FqdnFile> fqdnFiles = brightCloudUtils.responseSparse(resObj);
assert fqdnFiles.size()>0;
// 存数据库
mariaDB.insertRecords(fqdnFiles);
// 写入output
for (FqdnFile tmpFile:fqdnFiles){
outWriter.write(tmpFile.getValues() + "\n");
if (tmpFile.getQuery_success().equals(false)){
failQueryNum += 1;
} else if (tmpFile.getCategory_id().equals(0)){
noLabelNum += 1;
} else {
effecResNum += 1;
}
}
}
LOG.info("[Offline import file query]-" + importFileName + " Progress: 100%");
// 打印处理结果至日志
LOG.info("[Offline import file query]-" + importFileName + " "
+ "Query result: submit " + standardFqdnNum+" valid fqdns, "
+ dbQueryNum + " (" + new DecimalFormat("##.0%").format((float)dbQueryNum/standardFqdnNum) + ")" + " results from database，"
+ bcQueryNum + " (" + new DecimalFormat("##.0%").format((float)bcQueryNum/standardFqdnNum) + ")" + " results from bright cloud. "
+ effecResNum + " (" + new DecimalFormat("##.0%").format((float)effecResNum/standardFqdnNum) + ")" + " effective results，"
+ failQueryNum + " (" + new DecimalFormat("##.0%").format((float)failQueryNum/standardFqdnNum) + ")" + " failed queries，"
+ noLabelNum + " (" + new DecimalFormat("##.0%").format((float)noLabelNum/standardFqdnNum) + ")" + " unlabeled results");
LOG.info("[Offline import file query]-" + importFileName + " Results saved in " + outputFile.toString());
// 打印查询服务调用记录日志
if (bcQueryNum > 0){
OutputStream bcQueryLogStream = new FileOutputStream(ApplicationConfig.LOG_BC_QUERY_REPORT_FILE, true);
OutputStreamWriter bcQueryLogWriter = new OutputStreamWriter(bcQueryLogStream, StandardCharsets.UTF_8);
for (String type : queryTypes) {
java.sql.Date d = new java.sql.Date(System.currentTimeMillis());
bcQueryLogWriter.write(d + "," + "OfflineTask," + importFileName + "," + type + "," + bcQueryNum + "\n");
}
FileUtils.writerClose(bcQueryLogWriter, bcQueryLogStream);
}
FileUtils.writerClose(outWriter, outStream);
} else {
continue;
}
// 查询结束修改后缀
importFile.renameTo(new File(
importFile.toString().substring(0, importFile.toString().length()-ApplicationConfig.OFFLINE_IMPORT_FILENAME_SUFFIX.length())
+ ApplicationConfig.OFFLINE_IMPORT_FILEDONE_SUFFIX));
}
MariaDBConnect.close(mariaStat, mariaConn);
}
private List<String> catpureNewFiles() {
List<String> newFiles = new ArrayList<>();
// Get all files
File dir = new File(ApplicationConfig.OFFLINE_IMPORT_PATH);
if (!dir.exists()) {
boolean isCreateDir = dir.mkdir();
if (isCreateDir) {
LOG.info("Create new dictionary: " + ApplicationConfig.OFFLINE_IMPORT_PATH);
} else {
LOG.error("Failed to create import dictionary: " + ApplicationConfig.OFFLINE_IMPORT_PATH);
}
}
File[] allFiles = dir.listFiles();
if (allFiles == null) {
LOG.info("Dir is empty: " + ApplicationConfig.OFFLINE_IMPORT_PATH + System.currentTimeMillis());
} else {
for (File file : allFiles) {
if (file.isFile() && file.getName().endsWith(ApplicationConfig.OFFLINE_IMPORT_FILENAME_SUFFIX)) {
newFiles.add(file.toString());
}
}
}
return newFiles;
}
private Boolean checkFileName(String fileName) {
boolean isStandard = true;
if (!fileName.endsWith(ApplicationConfig.OFFLINE_IMPORT_FILENAME_SUFFIX)) {
isStandard = false;
}
return isStandard;
}
}

119
src/main/java/cn/ac/iie/service/SingleTermTask.java Normal file
View File

@@ -0,0 +1,119 @@
package cn.ac.iie.service;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.FqdnFile;
import org.apache.commons.lang3.StringUtils;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 2:40 下午
*/
public class SingleTermTask {
/**
* 提取三级域名
* @return 返回域名或空字符""
*/
public static String getStandardFqdn(String url){
if(url==null|| "".equals(url.trim())){
return "";
}
if (url.contains("://")){
url = url.split("://")[1];
}
String host = "";
Pattern p = Pattern.compile("^((http://)|(https://)|())?([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61})?\\.)+[a-zA-Z]{2,6}");
Matcher matcher = p.matcher(url);
if(matcher.find()){
String tmp = matcher.group();
String regex=".*[a-zA-Z]+.*";
Matcher m = Pattern.compile(regex).matcher(tmp);
if (m.matches()){
host = tmp;
}
}
if (host.split("\\.").length > ApplicationConfig.QUERY_STANDARD_FQDN_LEVEL){
List<String> domains = Arrays.asList(host.split("\\."));
host = StringUtils.join(domains.subList(domains.size()-ApplicationConfig.QUERY_STANDARD_FQDN_LEVEL,
domains.size()), ".");
}
return host;
}
private static boolean isValidDomain(String str)
{
String regex = "^((?!-)[A-Za-z0-9-]"
+ "{1,63}(?<!-)\\.)"
+ "+[A-Za-z]{2,6}";
Pattern p = Pattern.compile(regex);
if (str == null) {
return false;
}
Matcher m = p.matcher(str);
return m.matches();
}
public static List<String> getCheckedFqdn(List<String> fqdns){
List<String> res = new ArrayList<>();
for (String fqdn:fqdns){
// 去重 & 校验
if (isValidDomain(fqdn) && !res.contains(fqdn)){
res.add(fqdn.toLowerCase());
}
}
return res;
}
public static FqdnFile ResSet2FqdnFile(ResultSet resultSet){
FqdnFile fqdnFile = null;
try {
fqdnFile = new FqdnFile(
resultSet.getString("fqdn"),
resultSet.getBoolean("query_success"),
resultSet.getInt("reputation_score"),
resultSet.getString("reputation_level"),
resultSet.getInt("category_id"),
resultSet.getString("category_Name"),
resultSet.getString("category_group"),
resultSet.getInt("category_conf"),
resultSet.getBoolean("is_a1_cat"),
resultSet.getInt("popularity"),
resultSet.getInt("observ_age"),
resultSet.getString("country"),
resultSet.getInt("threat_history"),
resultSet.getString("whois_domain"),
resultSet.getDate("whois_update_date"),
resultSet.getDate("whois_create_date"),
resultSet.getDate("whois_expire_date"),
resultSet.getString("whois_email"),
resultSet.getString("whois_ns"),
resultSet.getString("whois_registrar_name"),
resultSet.getString("whois_registrant_org"),
resultSet.getString("whois_regisrante_name"),
resultSet.getString("whois_regisrante_street"),
resultSet.getString("whois_regisrante_city"),
resultSet.getString("whois_regisrante_state"),
resultSet.getString("whois_regisrante_postcode"),
resultSet.getString("whois_regisrante_country"),
resultSet.getString("whois_regisrante_phone")
);
} catch (SQLException e) {
e.printStackTrace();
}
return fqdnFile;
}
}

91
src/main/java/cn/ac/iie/service/UpdateTask.java Normal file
View File

@@ -0,0 +1,91 @@
package cn.ac.iie.service;
import cn.ac.iie.config.ApplicationConfig;
import cn.ac.iie.dao.BaseMariaDB;
import cn.ac.iie.dao.FqdnFile;
import cn.ac.iie.utils.BrightCloudUtils;
import cn.ac.iie.utils.FileUtils;
import cn.ac.iie.utils.LogUtils;
import cn.ac.iie.utils.MariaDBConnect;
import com.alibaba.fastjson.JSONObject;
import org.apache.log4j.Logger;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.TimerTask;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 11:29 上午
*/
public class UpdateTask extends TimerTask {
private static final Logger LOG = Logger.getLogger(UpdateTask.class);
private static final LogUtils logutils = new LogUtils();
@Override
public void run() {
try {
runTask();
} catch (SQLException | IOException exception) {
exception.printStackTrace();
}
}
private void runTask() throws SQLException, IOException {
LOG.info("Start update task ...");
List<String> updateFqdns = new ArrayList<>();
Connection mariaConn = MariaDBConnect.getConnection();
Statement mariaStat = mariaConn.createStatement();
BaseMariaDB mariaDB = new BaseMariaDB(mariaConn, mariaStat);
BrightCloudUtils brightCloudUtils = new BrightCloudUtils();
List<String> queryTypes = BrightCloudUtils.getQueryTypes();
// expired records
ResultSet expiredSet = mariaDB.getExpiredRecord();
while (expiredSet.next()) {
updateFqdns.add(expiredSet.getString("fqdn"));
}
long expiredNum = updateFqdns.size();
// unlabelled records
ResultSet unlabeledSet = mariaDB.getUnlabelRecord();
while (unlabeledSet.next()) {
updateFqdns.add(unlabeledSet.getString("fqdn"));
}
long unlabeledNum = updateFqdns.size() - expiredNum;
//TODO 分批查询
if (updateFqdns.size()>0){
JSONObject jsonObj = brightCloudUtils.getQueryResults(updateFqdns);
List<FqdnFile> updateFiles = brightCloudUtils.responseSparse(jsonObj);
mariaDB.updateRecords(updateFiles);
MariaDBConnect.close(mariaStat, mariaConn);
// 打印处理结果至日志
LOG.info("[UpdateTask]-update records: " + expiredNum +" expired records, " + unlabeledNum + " unlabeled or failed-query records");
// 打印查询操作记录日志
OutputStream bcQueryLogStream = new FileOutputStream(ApplicationConfig.LOG_BC_QUERY_REPORT_FILE, true);
OutputStreamWriter bcQueryLogWriter = new OutputStreamWriter(bcQueryLogStream, StandardCharsets.UTF_8);
for (String type : queryTypes) {
java.sql.Date d = new java.sql.Date(System.currentTimeMillis());
bcQueryLogWriter.write(d + "," + "UpdateTask," + null + "," + type + "," + updateFqdns.size() + "\n");
}
FileUtils.writerClose(bcQueryLogWriter, bcQueryLogStream);
}
}
}

253
src/main/java/cn/ac/iie/utils/BrightCloudUtils.java Normal file
View File

@@ -0,0 +1,253 @@
package cn.ac.iie.utils;
import cn.ac.iie.dao.FqdnFile;
import cn.ac.iie.config.ApplicationConfig;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import org.apache.log4j.Logger;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author yjy
* @version 1.0
* @date 2021/2/22 2:37 下午
*/
public class BrightCloudUtils {
private static final Logger LOG = Logger.getLogger(BrightCloudUtils.class);
private static final Properties props = new Properties();
private HttpURLConnection con;
private static List<String> queryTypes = new ArrayList<>();
static {
try {
props.load(BrightCloudUtils.class.getClassLoader().getResourceAsStream("brightcloud.properties"));
} catch (IOException e) {
e.printStackTrace();
}
if (ApplicationConfig.QUERY_URL_INFO_SWITCH){
queryTypes.add(props.getProperty("bc.api.queries.urlcat"));
}
if (ApplicationConfig.QUERY_URL_REP_SWITCH){
queryTypes.add(props.getProperty("bc.api.queries.urlrep"));
}
if (ApplicationConfig.QUERY_URL_WHOIS_SWITCH){
queryTypes.add(props.getProperty("bc.api.queries.urlwhois"));
}
assert queryTypes.size()>0: "Switch of all query type has been turned off, please edit the application.properties";
}
public HashMap<Integer, List<String>> getCatId2Info() {
return catId2Info;
}
private final HashMap<Integer, List<String>> catId2Info = new HashMap<>();
public JSONObject getQueryResults (List<String> urls) {
if (urls.size()>ApplicationConfig.MAXIMUM_URL_ONCE){
LOG.warn("Too many urls in a http post request!");
}
JSONObject jsonRes = null;
try {
URL url = new URL(props.getProperty("bc.api.url"));
// 打开和URL之间的连接
con = (HttpURLConnection) url.openConnection();
con.setRequestMethod(props.getProperty("bc.api.method"));
con.setDoOutput(true);
con.setDoInput(true);
con.setRequestProperty("Content-Type", "application/json");
JSONObject param = new JSONObject();
param.put("oemid", props.getProperty("bc.oemid"));
param.put("deviceid", props.getProperty("bc.deviceid"));
param.put("uid", props.getProperty("bc.uid"));
param.put("queries", queryTypes);
param.put("a1cat", props.getProperty("bc.api.a1cat"));
param.put("reputation", props.getProperty("bc.api.reputation"));
param.put("xml", props.getProperty("bc.api.xml"));
param.put("urls", urls);
// 建立实际的连接
con.connect();
OutputStreamWriter writer = new OutputStreamWriter(this.con.getOutputStream(), StandardCharsets.UTF_8);
writer.write(param.toString());
writer.flush();
} catch (IOException e) {
e.printStackTrace();
}
try {
// 获取服务端响应，通过输入流来读取URL的响应
InputStream is = con.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
StringBuffer sbf = new StringBuffer();
String strRead = null;
while ((strRead = reader.readLine()) != null) {
sbf.append(strRead);
sbf.append("\r\n");
}
reader.close();
jsonRes = JSONObject.parseObject(sbf.toString());
con.disconnect();
} catch (IOException e) {
e.printStackTrace();
}
return jsonRes;
}
public HttpURLConnection getCon() {
return con;
}
public List<FqdnFile> responseSparse(JSONObject records){
List<FqdnFile> fqdnFiles = new ArrayList<>();
Boolean querySucess = records.get("status").equals(200);
if (!querySucess) {
System.out.print(records.toString());
LOG.error("Wrong query. Query type: " + records.get("type"));
} else {
JSONArray array = records.getJSONArray("results");
for (int i = 0; i < array.size(); i++) {
JSONObject jo = array.getJSONObject(i);
// json处理
JSONObject queries = jo.getJSONObject("queries");
JSONObject getInfo = ApplicationConfig.QUERY_URL_INFO_SWITCH ?
queries.getJSONObject(props.getProperty("bc.api.queries.urlcat")): new JSONObject();
JSONObject getRepInfo = ApplicationConfig.QUERY_URL_REP_SWITCH ?
queries.getJSONObject(props.getProperty("bc.api.queries.urlrep")): new JSONObject();
JSONObject getWhoisInfo = ApplicationConfig.QUERY_URL_WHOIS_SWITCH ?
queries.getJSONObject(props.getProperty("bc.api.queries.urlwhois")): new JSONObject();
JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0);
Integer catId = cat.getInteger("catid");
String whoisEmail = "";
if (isEmail(getWhoisInfo.getString("contactemail"))){
whoisEmail = getWhoisInfo.getString("contactemail");
}
fqdnFiles.add(new FqdnFile(
jo.getString("url"),
querySucess,
getInfo.getInteger("reputation"),
getRepLevel(getInfo.getInteger("reputation")),
catId,
getCatInfo(catId).get(0),
getCatInfo(catId).get(1),
cat.getInteger("conf"),
getInfo.getBoolean("a1cat"),
getRepInfo.getInteger("popularity"),
getRepInfo.getInteger("age"),
getRepInfo.getString("country"),
getRepInfo.getInteger("threathistory"),
getWhoisInfo.getString("domainname"),
getWhoisInfo.getDate("audit_auditupdateddate"),
getWhoisInfo.getDate("createddate"),
getWhoisInfo.getDate("expiresdate"),
whoisEmail,
getWhoisInfo.getString("nameservers"),
getWhoisInfo.getString("registrarname"),
getWhoisInfo.getString("registrant_organization"),
getWhoisInfo.getString("registrant_name"),
getWhoisInfo.getString("registrant_street1"),
getWhoisInfo.getString("registrant_city"),
getWhoisInfo.getString("registrant_state"),
getWhoisInfo.getString("registrant_postalcode"),
getWhoisInfo.getString("registrant_country"),
getWhoisInfo.getString("registrant_telephone")));
}
}
return fqdnFiles;
}
private String getRepLevel(Integer repScore){
String level = null; //用str存放数据
if (repScore > 80) level="Trustworthy";
else if (repScore > 60) level="Low Risk";
else if (repScore > 40) level="Moderate Risk";
else if (repScore > 20) level="Suspicious";
else if (repScore > 0) level="High Risk";
return level;
}
public static boolean isEmail(String string) {
if (string == null)
return false;
String regEx1 = "^([a-z0-9A-Z]+[-|\\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\\.)+[a-zA-Z]{2,}$";
Pattern p;
Matcher m;
p = Pattern.compile(regEx1);
m = p.matcher(string);
return m.matches();
}
private void geneCatInfo(){
if (catId2Info.size()==0){
JSONObject jsonObject = null;
String s = FileUtils.readJsonFile(props.getProperty("bc.cateinfo.filepath"));
jsonObject = JSON.parseObject(s);
if (!(jsonObject==null)){
JSONObject tmp = (JSONObject) jsonObject.getJSONArray("results").get(0);
JSONArray catInfoArray = tmp.getJSONObject("queries").getJSONObject("getcatlist").getJSONArray("cats");
for (int i = 0; i < catInfoArray.size(); i++){
JSONObject keyObject = catInfoArray.getJSONObject(i);
List<String> value = new ArrayList<>(Arrays.asList(
keyObject.getString("catname"),
keyObject.getString("catgroup")));
catId2Info.put(i+1, value);
}
}
}
}
public List<String> getCatInfo(Integer catId){
List<String> info = Arrays.asList("", "");
if (0 < catId && catId <= 83) {
if (catId2Info.size()==0){
geneCatInfo();
}
info = catId2Info.get(catId);
if (info == null){
LOG.error("Failed at geneCatInfo function");
System.out.print("Failed at geneCatInfo function");
}
}
return info;
}
public static List<String> getQueryTypes() {
return queryTypes;
}
}

41
src/main/java/cn/ac/iie/utils/ConfigUtils.java Normal file
View File

@@ -0,0 +1,41 @@
package cn.ac.iie.utils;
import org.apache.log4j.Logger;
import org.mariadb.jdbc.MariaDbConnection;
import java.util.Properties;
public class ConfigUtils {
private static final Logger LOG = Logger.getLogger(ConfigUtils.class);
private static Properties propCommon = new Properties();
public static String getStringProperty(String key) {
return propCommon.getProperty(key);
}
public static Integer getIntProperty(String key) {
return Integer.parseInt(propCommon.getProperty(key));
}
public static Long getLongProperty(String key) {
return Long.parseLong(propCommon.getProperty(key));
}
public static Boolean getBooleanProperty(String key) {
return "true".equals(propCommon.getProperty(key).toLowerCase().trim());
}
static {
try {
propCommon.load(ConfigUtils.class.getClassLoader().getResourceAsStream("application.properties"));
propCommon.load(MariaDbConnection.class.getClassLoader().getResourceAsStream("druid.properties"));
} catch (Exception e) {
propCommon = null;
LOG.error("配置加载失败");
}
}
}

122
src/main/java/cn/ac/iie/utils/FileUtils.java Normal file
View File

@@ -0,0 +1,122 @@
package cn.ac.iie.utils;
import org.apache.log4j.Logger;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 6:11 下午
*/
public class FileUtils {
private static final Logger LOG = Logger.getLogger(FileUtils.class);
public static List<String> readTxtFileIntoStringArrList(String filePath)
{
List<String> list = new ArrayList<>();
try
{
String encoding = "GBK";
File file = new File(filePath);
if (file.isFile() && file.exists())
{ // 判断文件是否存在
InputStreamReader read = new InputStreamReader(
new FileInputStream(file), encoding);
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null)
{
if (!lineTxt.equals("")) {
list.add(lineTxt.trim());
}
}
bufferedReader.close();
read.close();
}
else
{
System.out.println("Can not find file: " + filePath);
}
}
catch (Exception e)
{
System.out.println("Error occurred in Function 'readTxtFileIntoStringArrList'");
e.printStackTrace();
}
return list;
}
public static void createFile(File filePath, String fileName){
try {
File file = new File(filePath.toString() + "/" + fileName);
if (!filePath.exists()){
filePath.mkdirs();
}
boolean isCreate = file.createNewFile();
if (isCreate){
LOG.info("File " + fileName + " is created.");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static void createFile(File file){
try {
boolean isCreate = file.createNewFile();
if (isCreate){
LOG.info("File " + file + " is created.");
}
} catch (IOException e) {
e.printStackTrace();
}
}
public static String readJsonFile(String fileName) {
String jsonStr = "";
try {
File jsonFile = new File(fileName);
FileReader fileReader = new FileReader(jsonFile);
Reader reader = new InputStreamReader(new FileInputStream(jsonFile), "utf-8");
int ch = 0;
StringBuffer sb = new StringBuffer();
while ((ch = reader.read()) != -1) {
sb.append((char) ch);
}
fileReader.close();
reader.close();
jsonStr = sb.toString();
return jsonStr;
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
public static String getFileName(File file){
String[] tmp = file.toString().split("/");
String fileName = tmp[tmp.length-1];
return fileName;
}
public static void writerClose(OutputStreamWriter outWriter, OutputStream outStream) throws IOException {
assert outWriter != null;
outWriter.close();
outStream.close();
}
}

18
src/main/java/cn/ac/iie/utils/LogUtils.java Normal file
View File

@@ -0,0 +1,18 @@
package cn.ac.iie.utils;
import cn.ac.iie.config.ApplicationConfig;
import java.io.*;
/**
* @author yjy
* @version 1.0
* @date 2021/3/10 3:31 下午
*/
public class LogUtils {
// 确认BrightCloud查询记录日志文件存在
static {
assert new File(ApplicationConfig.LOG_BC_QUERY_REPORT_FILE).exists():
"Cannot find Bright Cloud query log file: " + ApplicationConfig.LOG_BC_QUERY_REPORT_FILE;
}
}

92
src/main/java/cn/ac/iie/utils/MariaDBConnect.java Normal file
View File

@@ -0,0 +1,92 @@
package cn.ac.iie.utils; /**
* @author yjy
* @version 1.0
* @date 2021/2/24 4:17 下午
*/
import com.alibaba.druid.pool.DruidDataSourceFactory;
import me.geso.jdbcutils.JDBCUtils;
import javax.sql.DataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
/**
* Druid连接池的工具类
*/
public class MariaDBConnect {
//1.定义成员变量 DataSource
private static DataSource ds ;
static{
try {
//1.加载配置文件
Properties pro = new Properties();
pro.load(JDBCUtils.class.getClassLoader().getResourceAsStream("druid.properties"));
//2.获取DataSource
ds = DruidDataSourceFactory.createDataSource(pro);
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 获取连接
*/
public static Connection getConnection() throws SQLException {
return ds.getConnection();
}
/**
* 释放资源
*/
public static void close(Statement stmt,Connection conn){
close(null,stmt,conn);
}
public static void close(ResultSet rs , Statement stmt, Connection conn){
if(rs != null){
try {
rs.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if(stmt != null){
try {
stmt.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if(conn != null){
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
/**
* 获取连接池方法
*/
public static DataSource getDataSource(){
return ds;
}
}

65
src/main/java/cn/ac/iie/utils/TimeUtils.java Normal file
View File

@@ -0,0 +1,65 @@
package cn.ac.iie.utils;
import cn.ac.iie.config.ApplicationConfig;
import java.util.Calendar;
import java.util.Date;
/**
* @author yjy
* @version 1.0
* @date 2021/2/25 11:26 上午
*/
public class TimeUtils {
public static final Long HOUR_TO_MILLISECONDS = 3600000L;
public static final Long DAY_TO_MILLSEDONDS = 86400000L;
public static final Integer SECOND_TO_MILLSEDONDS = 1000;
public static Date getStartOfHour() {
return getStartOfHour(0);
}
public static Date getStartOfHour(Integer offset) {
Calendar ca = Calendar.getInstance();
ca.add(Calendar.HOUR, offset);
ca.set(Calendar.MINUTE, 0);
ca.set(Calendar.SECOND, 0);
ca.set(Calendar.MILLISECOND, 0);
return ca.getTime();
}
public static Date getStartOfDay() {
return getStartOfDay(0);
}
public static Date getStartOfDay(Integer bias) {
Calendar ca = Calendar.getInstance();
ca.add(Calendar.DATE, bias);
ca.set(Calendar.HOUR, -12);
ca.set(Calendar.MINUTE, 0);
ca.set(Calendar.SECOND, 0);
ca.set(Calendar.MILLISECOND, 0);
return ca.getTime();
}
public static Date getStartOfMonth() {
Calendar ca = Calendar.getInstance();
ca.set(Calendar.DATE, 1);
ca.set(Calendar.HOUR, -12);
ca.set(Calendar.MINUTE, 0);
ca.set(Calendar.SECOND, 0);
ca.set(Calendar.MILLISECOND, 0);
return ca.getTime();
}
public static Date getExpiredTime(){
return new java.sql.Timestamp(TimeUtils.getStartOfDay(-ApplicationConfig.UPDATE_SCHEDULE_DAY).getTime());
}
public static Date getExpiredTime(String test){
Date date = Calendar.getInstance().getTime();
date.setHours(11);
return date;
}
}

44
src/main/resources/application.properties Normal file
View File

@@ -0,0 +1,44 @@
# fqdn域名保留层级数
query.standard.fqdn.level = 3
# 离线导入指定目录
offline.import.path = /home/WebSKT/Data/request_file
# 离线指定查询结果保存目录
offline.output.path = /home/WebSKT/Data/output_file
# switch
offline.switch = true
update.switch = false
query.url_info.switch = true
query.url_rep.switch = false
query.url_whois.switch = false
# 离线导入文件后缀名
offline.import.filename.suffix = .compl
offline.import.filedone.suffix = .done
offline.output.filename.suffix = .result
# 离线导入定时任务时间间隔10s
offline.schedule.second = 5
# 数据库更新定时任务时间间隔1d 1000 * 3600 * 24 = 86400000
update.schedule.day = 7
# mariadb
database = web_sketch
table = fqdn_profile_via_brightcloud
# bc api单次查询url长度限制 API最高限制
maximum.url.once = 100
# log
# 打印进度日志的查询条数
log.offline.number = 10000
log.bc.query.report.file = /home/WebSKT/Data/bright_cloud_query_count.csv
## 离线导入指定目录
#offline.import.path = /Users/joy/work/iie/project/cyber_narrator/APP/WebSketch/QueryAgentV1/files/import_file
## 离线指定查询结果保存目录
#offline.output.path = /Users/joy/work/iie/project/cyber_narrator/APP/WebSketch/QueryAgentV1/files/output_file
## 服务调用统计结果
#log.bc.query.report.file = /Users/joy/work/iie/project/cyber_narrator/App/WebSketch/LOG_IMPORTANT/bright_cloud_query_count.csv

23
src/main/resources/brightcloud.properties Normal file
View File

@@ -0,0 +1,23 @@
bc.oemid = GeedgeNet
bc.deviceid = TSG-Dev
bc.uid = GN0001
bc.api.url = https://api.bcti.brightcloud.com/1.0/url/getinfo
bc.api.method = POST
bc.api.type = url
bc.api.queries.urlcat = getinfo
bc.api.queries.urlrep = getrepinfo
bc.api.queries.urlwhois = getwhoisinfo
bc.api.a1cat = 1
# 是否默认返回信誉评分
bc.api.reputation = 1
# 返回json格式
bc.api.xml = 0
bc.cateinfo.filepath = /home/WebSKT/Data/categoryinfo.json
# for test
#bc.cateinfo.filepath = /Users/joy/work/iie/project/cyber_narrator/App/WebSketch/QueryAgentV1/src/main/resources/categoryinfo.json

431
src/main/resources/categoryinfo.json Normal file
View File

@@ -0,0 +1,431 @@
{
"status": 200,
"requestid": "12345",
"type": "url",
"results": [
{
"url": "getcatlist",
"queries": {
"getcatlist": {
"cats": [
{
"catid": 1,
"catname": "Real Estate",
"catgroup": "Productivity"
},
{
"catid": 2,
"catname": "Computer and Internet Security",
"catgroup": "Productivity"
},
{
"catid": 3,
"catname": "Financial Services",
"catgroup": "Privacy"
},
{
"catid": 4,
"catname": "Business and Economy",
"catgroup": "Productivity"
},
{
"catid": 5,
"catname": "Computer and Internet Info",
"catgroup": "Productivity"
},
{
"catid": 6,
"catname": "Auctions",
"catgroup": "Productivity"
},
{
"catid": 7,
"catname": "Shopping",
"catgroup": "Productivity"
},
{
"catid": 8,
"catname": "Cult and Occult",
"catgroup": "Sensitive"
},
{
"catid": 9,
"catname": "Travel",
"catgroup": "Productivity"
},
{
"catid": 10,
"catname": "Abused Drugs",
"catgroup": "Sensitive"
},
{
"catid": 11,
"catname": "Adult and Pornography",
"catgroup": "Sensitive"
},
{
"catid": 12,
"catname": "Home and Garden",
"catgroup": "Productivity"
},
{
"catid": 13,
"catname": "Military",
"catgroup": "Productivity"
},
{
"catid": 14,
"catname": "Social Networking",
"catgroup": "Productivity"
},
{
"catid": 15,
"catname": "Dead Sites",
"catgroup": "Misc"
},
{
"catid": 16,
"catname": "Individual Stock Advice and Tools",
"catgroup": "Productivity"
},
{
"catid": 17,
"catname": "Training and Tools",
"catgroup": "Productivity"
},
{
"catid": 18,
"catname": "Dating",
"catgroup": "Sensitive"
},
{
"catid": 19,
"catname": "Sex Education",
"catgroup": "Sensitive"
},
{
"catid": 20,
"catname": "Religion",
"catgroup": "Sensitive"
},
{
"catid": 21,
"catname": "Entertainment and Arts",
"catgroup": "Productivity"
},
{
"catid": 22,
"catname": "Personal sites and Blogs",
"catgroup": "Productivity"
},
{
"catid": 23,
"catname": "Legal",
"catgroup": "Privacy"
},
{
"catid": 24,
"catname": "Local Information",
"catgroup": "Productivity"
},
{
"catid": 25,
"catname": "Streaming Media",
"catgroup": "IT Resources"
},
{
"catid": 26,
"catname": "Job Search",
"catgroup": "Productivity"
},
{
"catid": 27,
"catname": "Gambling",
"catgroup": "Sensitive"
},
{
"catid": 28,
"catname": "Translation",
"catgroup": "Sensitive"
},
{
"catid": 29,
"catname": "Reference and Research",
"catgroup": "Productivity"
},
{
"catid": 30,
"catname": "Shareware and Freeware",
"catgroup": "IT Resources"
},
{
"catid": 31,
"catname": "Peer to Peer",
"catgroup": "IT Resources"
},
{
"catid": 32,
"catname": "Marijuana",
"catgroup": "Sensitive"
},
{
"catid": 33,
"catname": "Hacking",
"catgroup": "Sensitive"
},
{
"catid": 34,
"catname": "Games",
"catgroup": "Productivity"
},
{
"catid": 35,
"catname": "Philosophy and Political Advocacy",
"catgroup": "Productivity"
},
{
"catid": 36,
"catname": "Weapons",
"catgroup": "Sensitive"
},
{
"catid": 37,
"catname": "Pay to Surf",
"catgroup": "Productivity"
},
{
"catid": 38,
"catname": "Hunting and Fishing",
"catgroup": "Productivity"
},
{
"catid": 39,
"catname": "Society",
"catgroup": "Productivity"
},
{
"catid": 40,
"catname": "Educational Institutions",
"catgroup": "Productivity"
},
{
"catid": 41,
"catname": "Online Greeting Cards",
"catgroup": "IT Resources"
},
{
"catid": 42,
"catname": "Sports",
"catgroup": "Productivity"
},
{
"catid": 43,
"catname": "Swimsuits and Intimate Apparel",
"catgroup": "Sensitive"
},
{
"catid": 44,
"catname": "Questionable",
"catgroup": "Sensitive"
},
{
"catid": 45,
"catname": "Kids",
"catgroup": "Productivity"
},
{
"catid": 46,
"catname": "Hate and Racism",
"catgroup": "Sensitive"
},
{
"catid": 47,
"catname": "Personal Storage",
"catgroup": "IT Resources"
},
{
"catid": 48,
"catname": "Violence",
"catgroup": "Sensitive"
},
{
"catid": 49,
"catname": "Keyloggers and Monitoring",
"catgroup": "Security"
},
{
"catid": 50,
"catname": "Search Engines",
"catgroup": "Productivity"
},
{
"catid": 51,
"catname": "Internet Portals",
"catgroup": "Productivity"
},
{
"catid": 52,
"catname": "Web Advertisements",
"catgroup": "IT Resources"
},
{
"catid": 53,
"catname": "Cheating",
"catgroup": "Sensitive"
},
{
"catid": 54,
"catname": "Gross",
"catgroup": "Sensitive"
},
{
"catid": 55,
"catname": "Web-based Email",
"catgroup": "Privacy"
},
{
"catid": 56,
"catname": "Malware Sites",
"catgroup": "Security"
},
{
"catid": 57,
"catname": "Phishing and Other Frauds",
"catgroup": "Security"
},
{
"catid": 58,
"catname": "Proxy Avoidance and Anonymizers",
"catgroup": "Security"
},
{
"catid": 59,
"catname": "Spyware and Adware",
"catgroup": "Security"
},
{
"catid": 60,
"catname": "Music",
"catgroup": "Productivity"
},
{
"catid": 61,
"catname": "Government",
"catgroup": "Privacy"
},
{
"catid": 62,
"catname": "Nudity",
"catgroup": "Sensitive"
},
{
"catid": 63,
"catname": "News and Media",
"catgroup": "Productivity"
},
{
"catid": 64,
"catname": "Illegal",
"catgroup": "Sensitive"
},
{
"catid": 65,
"catname": "Content Delivery Networks",
"catgroup": "IT Resources"
},
{
"catid": 66,
"catname": "Internet Communications",
"catgroup": "IT Resources"
},
{
"catid": 67,
"catname": "Bot Nets",
"catgroup": "Security"
},
{
"catid": 68,
"catname": "Abortion",
"catgroup": "Sensitive"
},
{
"catid": 69,
"catname": "Health and Medicine",
"catgroup": "Privacy"
},
{
"catid": 70,
"catname": "Confirmed SPAM Sources",
"catgroup": "Unmaintained"
},
{
"catid": 71,
"catname": "SPAM URLs",
"catgroup": "Security"
},
{
"catid": 72,
"catname": "Unconfirmed SPAM Sources",
"catgroup": "Unmaintained"
},
{
"catid": 73,
"catname": "Open HTTP Proxies",
"catgroup": "Unmaintained"
},
{
"catid": 74,
"catname": "Dynamically Generated Content",
"catgroup": "Productivity"
},
{
"catid": 75,
"catname": "Parked Domains",
"catgroup": "Sensitive"
},
{
"catid": 76,
"catname": "Alcohol and Tobacco",
"catgroup": "Sensitive"
},
{
"catid": 77,
"catname": "Private IP Addresses",
"catgroup": "Unmaintained"
},
{
"catid": 78,
"catname": "Image and Video Search",
"catgroup": "Sensitive"
},
{
"catid": 79,
"catname": "Fashion and Beauty",
"catgroup": "Productivity"
},
{
"catid": 80,
"catname": "Recreation and Hobbies",
"catgroup": "Productivity"
},
{
"catid": 81,
"catname": "Motor Vehicles",
"catgroup": "Productivity"
},
{
"catid": 82,
"catname": "Web Hosting",
"catgroup": "IT Resources"
},
{
"catid": 83,
"catname": "Food and Dining",
"catgroup": "Unmaintained"
}
]
}
}
}
]
}

9
src/main/resources/druid.properties Normal file
View File

@@ -0,0 +1,9 @@
driverClassName=org.mariadb.jdbc.Driver
url=jdbc:mariadb://api.geedge.net:3306/web_sketch
username=root
password=webskt2021
initialSize=5
maxActive=10
maxWait=1000
validationQuery=SELECT 1

23
src/main/resources/log4j.properties Normal file
View File

@@ -0,0 +1,23 @@
########################## logger ##############################
### 设置###
log4j.rootLogger = debug,E,stdout
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n
log4j.appender.E = org.apache.log4j.DailyRollingFileAppender
log4j.appender.E.File =/home/WebSKT/Source/query_agent/logs/run.log
log4j.appender.E.Append = true
log4j.appender.E.Threshold = INFO
log4j.appender.E.layout = org.apache.log4j.PatternLayout
log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n
#########################
#### for test ###########
#########################
#log4j.appender.E.File =/Users/joy/work/iie/project/cyber_narrator/App/WebSketch/QueryAgentV1/logs/run.log