254 lines
9.3 KiB
Java
254 lines
9.3 KiB
Java
|
package cn.ac.iie.utils;
|
|||
|
|
|
|||
|
|
import cn.ac.iie.dao.FqdnFile;
|
|||
|
|
import cn.ac.iie.config.ApplicationConfig;
|
|||
|
|
import com.alibaba.fastjson.JSON;
|
|||
|
|
import com.alibaba.fastjson.JSONArray;
|
|||
|
|
import com.alibaba.fastjson.JSONObject;
|
|||
|
|
import com.google.common.collect.Lists;
|
|||
|
|
import org.apache.log4j.Logger;
|
|||
|
|
|
|||
|
|
import java.io.*;
|
|||
|
|
import java.net.HttpURLConnection;
|
|||
|
|
import java.net.URL;
|
|||
|
|
import java.nio.charset.StandardCharsets;
|
|||
|
|
import java.util.*;
|
|||
|
|
import java.util.regex.Matcher;
|
|||
|
|
import java.util.regex.Pattern;
|
|||
|
|
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* @author yjy
|
|||
|
|
* @version 1.0
|
|||
|
|
* @date 2021/2/22 2:37 下午
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
public class BrightCloudUtils {
|
|||
|
|
private static final Logger LOG = Logger.getLogger(BrightCloudUtils.class);
|
|||
|
|
private static final Properties props = new Properties();
|
|||
|
|
private HttpURLConnection con;
|
|||
|
|
private static List<String> queryTypes = new ArrayList<>();
|
|||
|
|
|
|||
|
|
static {
|
|||
|
|
try {
|
|||
|
|
props.load(BrightCloudUtils.class.getClassLoader().getResourceAsStream("brightcloud.properties"));
|
|||
|
|
} catch (IOException e) {
|
|||
|
|
e.printStackTrace();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (ApplicationConfig.QUERY_URL_INFO_SWITCH){
|
|||
|
|
queryTypes.add(props.getProperty("bc.api.queries.urlcat"));
|
|||
|
|
}
|
|||
|
|
if (ApplicationConfig.QUERY_URL_REP_SWITCH){
|
|||
|
|
queryTypes.add(props.getProperty("bc.api.queries.urlrep"));
|
|||
|
|
}
|
|||
|
|
if (ApplicationConfig.QUERY_URL_WHOIS_SWITCH){
|
|||
|
|
queryTypes.add(props.getProperty("bc.api.queries.urlwhois"));
|
|||
|
|
}
|
|||
|
|
assert queryTypes.size()>0: "Switch of all query type has been turned off, please edit the application.properties";
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public HashMap<Integer, List<String>> getCatId2Info() {
|
|||
|
|
return catId2Info;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private final HashMap<Integer, List<String>> catId2Info = new HashMap<>();
|
|||
|
|
|
|||
|
|
public JSONObject getQueryResults (List<String> urls) {
|
|||
|
|
if (urls.size()>ApplicationConfig.MAXIMUM_URL_ONCE){
|
|||
|
|
LOG.warn("Too many urls in a http post request!");
|
|||
|
|
}
|
|||
|
|
JSONObject jsonRes = null;
|
|||
|
|
try {
|
|||
|
|
|
|||
|
|
URL url = new URL(props.getProperty("bc.api.url"));
|
|||
|
|
// 打开和URL之间的连接
|
|||
|
|
con = (HttpURLConnection) url.openConnection();
|
|||
|
|
con.setRequestMethod(props.getProperty("bc.api.method"));
|
|||
|
|
con.setDoOutput(true);
|
|||
|
|
con.setDoInput(true);
|
|||
|
|
|
|||
|
|
con.setRequestProperty("Content-Type", "application/json");
|
|||
|
|
|
|||
|
|
JSONObject param = new JSONObject();
|
|||
|
|
param.put("oemid", props.getProperty("bc.oemid"));
|
|||
|
|
param.put("deviceid", props.getProperty("bc.deviceid"));
|
|||
|
|
param.put("uid", props.getProperty("bc.uid"));
|
|||
|
|
|
|||
|
|
param.put("queries", queryTypes);
|
|||
|
|
param.put("a1cat", props.getProperty("bc.api.a1cat"));
|
|||
|
|
param.put("reputation", props.getProperty("bc.api.reputation"));
|
|||
|
|
param.put("xml", props.getProperty("bc.api.xml"));
|
|||
|
|
|
|||
|
|
param.put("urls", urls);
|
|||
|
|
|
|||
|
|
// 建立实际的连接
|
|||
|
|
con.connect();
|
|||
|
|
OutputStreamWriter writer = new OutputStreamWriter(this.con.getOutputStream(), StandardCharsets.UTF_8);
|
|||
|
|
writer.write(param.toString());
|
|||
|
|
writer.flush();
|
|||
|
|
} catch (IOException e) {
|
|||
|
|
e.printStackTrace();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
// 获取服务端响应，通过输入流来读取URL的响应
|
|||
|
|
InputStream is = con.getInputStream();
|
|||
|
|
BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
|
|||
|
|
StringBuffer sbf = new StringBuffer();
|
|||
|
|
String strRead = null;
|
|||
|
|
while ((strRead = reader.readLine()) != null) {
|
|||
|
|
sbf.append(strRead);
|
|||
|
|
sbf.append("\r\n");
|
|||
|
|
}
|
|||
|
|
reader.close();
|
|||
|
|
|
|||
|
|
jsonRes = JSONObject.parseObject(sbf.toString());
|
|||
|
|
con.disconnect();
|
|||
|
|
} catch (IOException e) {
|
|||
|
|
e.printStackTrace();
|
|||
|
|
}
|
|||
|
|
return jsonRes;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public HttpURLConnection getCon() {
|
|||
|
|
return con;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public List<FqdnFile> responseSparse(JSONObject records){
|
|||
|
|
List<FqdnFile> fqdnFiles = new ArrayList<>();
|
|||
|
|
Boolean querySucess = records.get("status").equals(200);
|
|||
|
|
|
|||
|
|
if (!querySucess) {
|
|||
|
|
System.out.print(records.toString());
|
|||
|
|
LOG.error("Wrong query. Query type: " + records.get("type"));
|
|||
|
|
} else {
|
|||
|
|
JSONArray array = records.getJSONArray("results");
|
|||
|
|
for (int i = 0; i < array.size(); i++) {
|
|||
|
|
JSONObject jo = array.getJSONObject(i);
|
|||
|
|
|
|||
|
|
// json处理
|
|||
|
|
JSONObject queries = jo.getJSONObject("queries");
|
|||
|
|
JSONObject getInfo = ApplicationConfig.QUERY_URL_INFO_SWITCH ?
|
|||
|
|
queries.getJSONObject(props.getProperty("bc.api.queries.urlcat")): new JSONObject();
|
|||
|
|
JSONObject getRepInfo = ApplicationConfig.QUERY_URL_REP_SWITCH ?
|
|||
|
|
queries.getJSONObject(props.getProperty("bc.api.queries.urlrep")): new JSONObject();
|
|||
|
|
JSONObject getWhoisInfo = ApplicationConfig.QUERY_URL_WHOIS_SWITCH ?
|
|||
|
|
queries.getJSONObject(props.getProperty("bc.api.queries.urlwhois")): new JSONObject();
|
|||
|
|
|
|||
|
|
JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0);
|
|||
|
|
Integer catId = cat.getInteger("catid");
|
|||
|
|
|
|||
|
|
String whoisEmail = "";
|
|||
|
|
if (isEmail(getWhoisInfo.getString("contactemail"))){
|
|||
|
|
whoisEmail = getWhoisInfo.getString("contactemail");
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
fqdnFiles.add(new FqdnFile(
|
|||
|
|
jo.getString("url"),
|
|||
|
|
querySucess,
|
|||
|
|
|
|||
|
|
getInfo.getInteger("reputation"),
|
|||
|
|
getRepLevel(getInfo.getInteger("reputation")),
|
|||
|
|
catId,
|
|||
|
|
getCatInfo(catId).get(0),
|
|||
|
|
getCatInfo(catId).get(1),
|
|||
|
|
cat.getInteger("conf"),
|
|||
|
|
getInfo.getBoolean("a1cat"),
|
|||
|
|
|
|||
|
|
getRepInfo.getInteger("popularity"),
|
|||
|
|
getRepInfo.getInteger("age"),
|
|||
|
|
getRepInfo.getString("country"),
|
|||
|
|
getRepInfo.getInteger("threathistory"),
|
|||
|
|
|
|||
|
|
getWhoisInfo.getString("domainname"),
|
|||
|
|
getWhoisInfo.getDate("audit_auditupdateddate"),
|
|||
|
|
getWhoisInfo.getDate("createddate"),
|
|||
|
|
getWhoisInfo.getDate("expiresdate"),
|
|||
|
|
whoisEmail,
|
|||
|
|
getWhoisInfo.getString("nameservers"),
|
|||
|
|
getWhoisInfo.getString("registrarname"),
|
|||
|
|
getWhoisInfo.getString("registrant_organization"),
|
|||
|
|
getWhoisInfo.getString("registrant_name"),
|
|||
|
|
getWhoisInfo.getString("registrant_street1"),
|
|||
|
|
getWhoisInfo.getString("registrant_city"),
|
|||
|
|
getWhoisInfo.getString("registrant_state"),
|
|||
|
|
getWhoisInfo.getString("registrant_postalcode"),
|
|||
|
|
getWhoisInfo.getString("registrant_country"),
|
|||
|
|
getWhoisInfo.getString("registrant_telephone")));
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return fqdnFiles;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
private String getRepLevel(Integer repScore){
|
|||
|
|
String level = null; //用str存放数据
|
|||
|
|
if (repScore > 80) level="Trustworthy";
|
|||
|
|
else if (repScore > 60) level="Low Risk";
|
|||
|
|
else if (repScore > 40) level="Moderate Risk";
|
|||
|
|
else if (repScore > 20) level="Suspicious";
|
|||
|
|
else if (repScore > 0) level="High Risk";
|
|||
|
|
return level;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public static boolean isEmail(String string) {
|
|||
|
|
if (string == null)
|
|||
|
|
return false;
|
|||
|
|
String regEx1 = "^([a-z0-9A-Z]+[-|\\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\\.)+[a-zA-Z]{2,}$";
|
|||
|
|
Pattern p;
|
|||
|
|
Matcher m;
|
|||
|
|
p = Pattern.compile(regEx1);
|
|||
|
|
m = p.matcher(string);
|
|||
|
|
return m.matches();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
private void geneCatInfo(){
|
|||
|
|
if (catId2Info.size()==0){
|
|||
|
|
|
|||
|
|
JSONObject jsonObject = null;
|
|||
|
|
|
|||
|
|
String s = FileUtils.readJsonFile(props.getProperty("bc.cateinfo.filepath"));
|
|||
|
|
jsonObject = JSON.parseObject(s);
|
|||
|
|
|
|||
|
|
if (!(jsonObject==null)){
|
|||
|
|
JSONObject tmp = (JSONObject) jsonObject.getJSONArray("results").get(0);
|
|||
|
|
JSONArray catInfoArray = tmp.getJSONObject("queries").getJSONObject("getcatlist").getJSONArray("cats");
|
|||
|
|
|
|||
|
|
for (int i = 0; i < catInfoArray.size(); i++){
|
|||
|
|
JSONObject keyObject = catInfoArray.getJSONObject(i);
|
|||
|
|
List<String> value = new ArrayList<>(Arrays.asList(
|
|||
|
|
keyObject.getString("catname"),
|
|||
|
|
keyObject.getString("catgroup")));
|
|||
|
|
catId2Info.put(i+1, value);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public List<String> getCatInfo(Integer catId){
|
|||
|
|
List<String> info = Arrays.asList("", "");
|
|||
|
|
|
|||
|
|
if (0 < catId && catId <= 83) {
|
|||
|
|
if (catId2Info.size()==0){
|
|||
|
|
geneCatInfo();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
info = catId2Info.get(catId);
|
|||
|
|
|
|||
|
|
if (info == null){
|
|||
|
|
LOG.error("Failed at geneCatInfo function");
|
|||
|
|
System.out.print("Failed at geneCatInfo function");
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return info;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
public static List<String> getQueryTypes() {
|
|||
|
|
return queryTypes;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|