分类/信誉/whois结果分表存储，分开查询；修改查询逻辑，二级域名截断
This commit is contained in:
yinjiangyi
@@ -1,6 +1,6 @@
|
||||
package cn.ac.iie.utils;
|
||||
|
||||
import cn.ac.iie.config.ApplicationConfig;
|
||||
import cn.ac.iie.config.CommonConfig;
|
||||
import cn.ac.iie.dao.FqdnFile;
|
||||
import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.JSONArray;
|
||||
@@ -11,6 +11,7 @@ import java.io.*;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Connection;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -35,14 +36,14 @@ public class BrightCloudUtils {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
if (ApplicationConfig.QUERY_URL_INFO_SWITCH){
|
||||
queryTypes.add(props.getProperty("bc.api.queries.urlcat"));
|
||||
if (CommonConfig.QUERY_URL_INFO_SWITCH){
|
||||
queryTypes.add(props.getProperty("bc.api.name.category"));
|
||||
}
|
||||
if (ApplicationConfig.QUERY_URL_REP_SWITCH){
|
||||
queryTypes.add(props.getProperty("bc.api.queries.urlrep"));
|
||||
if (CommonConfig.QUERY_URL_REP_SWITCH){
|
||||
queryTypes.add(props.getProperty("bc.api.name.reputation"));
|
||||
}
|
||||
if (ApplicationConfig.QUERY_URL_WHOIS_SWITCH){
|
||||
queryTypes.add(props.getProperty("bc.api.queries.urlwhois"));
|
||||
if (CommonConfig.QUERY_URL_WHOIS_SWITCH){
|
||||
queryTypes.add(props.getProperty("bc.api.name.whois"));
|
||||
}
|
||||
assert queryTypes.size()>0: "Switch of all query type has been turned off, please edit the application.properties";
|
||||
}
|
||||
@@ -54,7 +55,11 @@ public class BrightCloudUtils {
|
||||
private final HashMap<Integer, List<String>> catId2Info = new HashMap<>();
|
||||
|
||||
public JSONObject getQueryResults (List<String> urls) {
|
||||
if (urls.size()>ApplicationConfig.MAXIMUM_URL_ONCE_BC_QUERY){
|
||||
return getQueryResults(urls, CommonConfig.BC_API_NAME_CATEGORY);
|
||||
}
|
||||
|
||||
public JSONObject getQueryResults (List<String> urls, String queryType) {
|
||||
if (urls.size()> CommonConfig.MAXIMUM_URL_ONCE_BC_QUERY){
|
||||
LOG.warn("Too many urls in a http post request!");
|
||||
}
|
||||
JSONObject jsonRes = null;
|
||||
@@ -74,7 +79,7 @@ public class BrightCloudUtils {
|
||||
param.put("deviceid", props.getProperty("bc.deviceid"));
|
||||
param.put("uid", props.getProperty("bc.uid"));
|
||||
|
||||
param.put("queries", queryTypes);
|
||||
param.put("queries", new ArrayList<>(Collections.singletonList(queryType)));
|
||||
param.put("a1cat", props.getProperty("bc.api.a1cat"));
|
||||
param.put("reputation", props.getProperty("bc.api.reputation"));
|
||||
param.put("xml", props.getProperty("bc.api.xml"));
|
||||
@@ -115,6 +120,10 @@ public class BrightCloudUtils {
|
||||
}
|
||||
|
||||
public List<FqdnFile> responseSparse(JSONObject records){
|
||||
return responseSparse(records, CommonConfig.BC_API_NAME_CATEGORY);
|
||||
}
|
||||
|
||||
public List<FqdnFile> responseSparse(JSONObject records, String queryType){
|
||||
List<FqdnFile> fqdnFiles = new ArrayList<>();
|
||||
Boolean querySucess = records.get("status").equals(200);
|
||||
|
||||
@@ -128,54 +137,56 @@ public class BrightCloudUtils {
|
||||
|
||||
// json处理
|
||||
JSONObject queries = jo.getJSONObject("queries");
|
||||
JSONObject getInfo = ApplicationConfig.QUERY_URL_INFO_SWITCH ?
|
||||
queries.getJSONObject(props.getProperty("bc.api.queries.urlcat")): new JSONObject();
|
||||
JSONObject getRepInfo = ApplicationConfig.QUERY_URL_REP_SWITCH ?
|
||||
queries.getJSONObject(props.getProperty("bc.api.queries.urlrep")): new JSONObject();
|
||||
JSONObject getWhoisInfo = ApplicationConfig.QUERY_URL_WHOIS_SWITCH ?
|
||||
queries.getJSONObject(props.getProperty("bc.api.queries.urlwhois")): new JSONObject();
|
||||
|
||||
JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0);
|
||||
Integer catId = cat.getInteger("catid");
|
||||
JSONObject getInfo = queries.getJSONObject(queryType);
|
||||
|
||||
String whoisEmail = "";
|
||||
if (isEmail(getWhoisInfo.getString("contactemail"))){
|
||||
whoisEmail = getWhoisInfo.getString("contactemail");
|
||||
if (queryType.equals(CommonConfig.BC_API_NAME_CATEGORY)){
|
||||
JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0);
|
||||
Integer catId = cat.getInteger("catid");
|
||||
fqdnFiles.add(new FqdnFile(
|
||||
jo.getString("url"),
|
||||
querySucess,
|
||||
getInfo.getInteger("reputation"),
|
||||
getRepLevel(getInfo.getInteger("reputation")),
|
||||
catId,
|
||||
getCatInfo(catId).get(0),
|
||||
getCatInfo(catId).get(1),
|
||||
cat.getInteger("conf"),
|
||||
getInfo.getBoolean("a1cat")));
|
||||
} else if (queryType.equals(CommonConfig.BC_API_NAME_REPUTATION)){
|
||||
fqdnFiles.add(new FqdnFile(
|
||||
jo.getString("url"),
|
||||
querySucess,
|
||||
getInfo.getInteger("reputation"),
|
||||
getRepLevel(getInfo.getInteger("reputation")),
|
||||
getInfo.getInteger("popularity"),
|
||||
getInfo.getInteger("age"),
|
||||
getInfo.getString("country"),
|
||||
getInfo.getInteger("threathistory")));
|
||||
} else if (queryType.equals(CommonConfig.BC_API_NAME_WHOIS)){
|
||||
String whoisEmail = "";
|
||||
if (isEmail(getInfo.getString("contactemail"))){
|
||||
whoisEmail = getInfo.getString("contactemail");
|
||||
}
|
||||
fqdnFiles.add(new FqdnFile(
|
||||
jo.getString("url"),
|
||||
querySucess,
|
||||
getInfo.getString("domainname"),
|
||||
getInfo.getDate("audit_auditupdateddate"),
|
||||
getInfo.getDate("createddate"),
|
||||
getInfo.getDate("expiresdate"),
|
||||
whoisEmail,
|
||||
getInfo.getString("nameservers"),
|
||||
getInfo.getString("registrarname"),
|
||||
getInfo.getString("registrant_organization"),
|
||||
getInfo.getString("registrant_name"),
|
||||
getInfo.getString("registrant_street1"),
|
||||
getInfo.getString("registrant_city"),
|
||||
getInfo.getString("registrant_state"),
|
||||
getInfo.getString("registrant_postalcode"),
|
||||
getInfo.getString("registrant_country"),
|
||||
getInfo.getString("registrant_telephone")));
|
||||
}
|
||||
|
||||
fqdnFiles.add(new FqdnFile(
|
||||
jo.getString("url"),
|
||||
querySucess,
|
||||
|
||||
getInfo.getInteger("reputation"),
|
||||
getRepLevel(getInfo.getInteger("reputation")),
|
||||
catId,
|
||||
getCatInfo(catId).get(0),
|
||||
getCatInfo(catId).get(1),
|
||||
cat.getInteger("conf"),
|
||||
getInfo.getBoolean("a1cat"),
|
||||
|
||||
getRepInfo.getInteger("popularity"),
|
||||
getRepInfo.getInteger("age"),
|
||||
getRepInfo.getString("country"),
|
||||
getRepInfo.getInteger("threathistory"),
|
||||
|
||||
getWhoisInfo.getString("domainname"),
|
||||
getWhoisInfo.getDate("audit_auditupdateddate"),
|
||||
getWhoisInfo.getDate("createddate"),
|
||||
getWhoisInfo.getDate("expiresdate"),
|
||||
whoisEmail,
|
||||
getWhoisInfo.getString("nameservers"),
|
||||
getWhoisInfo.getString("registrarname"),
|
||||
getWhoisInfo.getString("registrant_organization"),
|
||||
getWhoisInfo.getString("registrant_name"),
|
||||
getWhoisInfo.getString("registrant_street1"),
|
||||
getWhoisInfo.getString("registrant_city"),
|
||||
getWhoisInfo.getString("registrant_state"),
|
||||
getWhoisInfo.getString("registrant_postalcode"),
|
||||
getWhoisInfo.getString("registrant_country"),
|
||||
getWhoisInfo.getString("registrant_telephone")));
|
||||
|
||||
}
|
||||
}
|
||||
return fqdnFiles;
|
||||
@@ -183,17 +194,18 @@ public class BrightCloudUtils {
|
||||
|
||||
private String getRepLevel(Integer repScore){
|
||||
String level = null; //用str存放数据
|
||||
if (repScore > 80) level="Trustworthy";
|
||||
else if (repScore > 60) level="Low Risk";
|
||||
else if (repScore > 40) level="Moderate Risk";
|
||||
else if (repScore > 20) level="Suspicious";
|
||||
else if (repScore > 0) level="High Risk";
|
||||
if (repScore > 80){ level="Trustworthy";}
|
||||
else if (repScore > 60){ level="Low Risk";}
|
||||
else if (repScore > 40){ level="Moderate Risk";}
|
||||
else if (repScore > 20){ level="Suspicious";}
|
||||
else if (repScore > 0){ level="High Risk";}
|
||||
return level;
|
||||
}
|
||||
|
||||
public static boolean isEmail(String string) {
|
||||
if (string == null)
|
||||
if (string == null){
|
||||
return false;
|
||||
}
|
||||
String regEx1 = "^([a-z0-9A-Z]+[-|\\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\\.)+[a-zA-Z]{2,}$";
|
||||
Pattern p;
|
||||
Matcher m;
|
||||
|
||||
Reference in New Issue
Block a user