package cn.ac.iie.utils; import cn.ac.iie.config.ApplicationConfig; import cn.ac.iie.dao.FqdnFile; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.apache.log4j.Logger; import java.io.*; import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * @author yjy * @version 1.0 * @date 2021/2/22 2:37 下午 */ public class BrightCloudUtils { private static final Logger LOG = Logger.getLogger(BrightCloudUtils.class); private static final Properties props = new Properties(); private HttpURLConnection con; private static List queryTypes = new ArrayList<>(); static { try { props.load(BrightCloudUtils.class.getClassLoader().getResourceAsStream("brightcloud.properties")); } catch (IOException e) { e.printStackTrace(); } if (ApplicationConfig.QUERY_URL_INFO_SWITCH){ queryTypes.add(props.getProperty("bc.api.queries.urlcat")); } if (ApplicationConfig.QUERY_URL_REP_SWITCH){ queryTypes.add(props.getProperty("bc.api.queries.urlrep")); } if (ApplicationConfig.QUERY_URL_WHOIS_SWITCH){ queryTypes.add(props.getProperty("bc.api.queries.urlwhois")); } assert queryTypes.size()>0: "Switch of all query type has been turned off, please edit the application.properties"; } public HashMap> getCatId2Info() { return catId2Info; } private final HashMap> catId2Info = new HashMap<>(); public JSONObject getQueryResults (List urls) { if (urls.size()>ApplicationConfig.MAXIMUM_URL_ONCE_BC_QUERY){ LOG.warn("Too many urls in a http post request!"); } JSONObject jsonRes = null; try { URL url = new URL(props.getProperty("bc.api.url")); // 打开和URL之间的连接 con = (HttpURLConnection) url.openConnection(); con.setRequestMethod(props.getProperty("bc.api.method")); con.setDoOutput(true); con.setDoInput(true); con.setRequestProperty("Content-Type", "application/json"); JSONObject param = new JSONObject(); param.put("oemid", props.getProperty("bc.oemid")); param.put("deviceid", props.getProperty("bc.deviceid")); param.put("uid", props.getProperty("bc.uid")); param.put("queries", queryTypes); param.put("a1cat", props.getProperty("bc.api.a1cat")); param.put("reputation", props.getProperty("bc.api.reputation")); param.put("xml", props.getProperty("bc.api.xml")); param.put("urls", urls); // 建立实际的连接 con.connect(); OutputStreamWriter writer = new OutputStreamWriter(this.con.getOutputStream(), StandardCharsets.UTF_8); writer.write(param.toString()); writer.flush(); } catch (IOException e) { e.printStackTrace(); } try { // 获取服务端响应,通过输入流来读取URL的响应 InputStream is = con.getInputStream(); BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); StringBuffer sbf = new StringBuffer(); String strRead = null; while ((strRead = reader.readLine()) != null) { sbf.append(strRead); sbf.append("\r\n"); } reader.close(); jsonRes = JSONObject.parseObject(sbf.toString()); con.disconnect(); } catch (IOException e) { e.printStackTrace(); } return jsonRes; } public HttpURLConnection getCon() { return con; } public List responseSparse(JSONObject records){ List fqdnFiles = new ArrayList<>(); Boolean querySucess = records.get("status").equals(200); if (!querySucess) { System.out.print(records.toString()); LOG.error("Wrong query. Query type: " + records.get("type")); } else { JSONArray array = records.getJSONArray("results"); for (int i = 0; i < array.size(); i++) { JSONObject jo = array.getJSONObject(i); // json处理 JSONObject queries = jo.getJSONObject("queries"); JSONObject getInfo = ApplicationConfig.QUERY_URL_INFO_SWITCH ? queries.getJSONObject(props.getProperty("bc.api.queries.urlcat")): new JSONObject(); JSONObject getRepInfo = ApplicationConfig.QUERY_URL_REP_SWITCH ? queries.getJSONObject(props.getProperty("bc.api.queries.urlrep")): new JSONObject(); JSONObject getWhoisInfo = ApplicationConfig.QUERY_URL_WHOIS_SWITCH ? queries.getJSONObject(props.getProperty("bc.api.queries.urlwhois")): new JSONObject(); JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0); Integer catId = cat.getInteger("catid"); String whoisEmail = ""; if (isEmail(getWhoisInfo.getString("contactemail"))){ whoisEmail = getWhoisInfo.getString("contactemail"); } fqdnFiles.add(new FqdnFile( jo.getString("url"), querySucess, getInfo.getInteger("reputation"), getRepLevel(getInfo.getInteger("reputation")), catId, getCatInfo(catId).get(0), getCatInfo(catId).get(1), cat.getInteger("conf"), getInfo.getBoolean("a1cat"), getRepInfo.getInteger("popularity"), getRepInfo.getInteger("age"), getRepInfo.getString("country"), getRepInfo.getInteger("threathistory"), getWhoisInfo.getString("domainname"), getWhoisInfo.getDate("audit_auditupdateddate"), getWhoisInfo.getDate("createddate"), getWhoisInfo.getDate("expiresdate"), whoisEmail, getWhoisInfo.getString("nameservers"), getWhoisInfo.getString("registrarname"), getWhoisInfo.getString("registrant_organization"), getWhoisInfo.getString("registrant_name"), getWhoisInfo.getString("registrant_street1"), getWhoisInfo.getString("registrant_city"), getWhoisInfo.getString("registrant_state"), getWhoisInfo.getString("registrant_postalcode"), getWhoisInfo.getString("registrant_country"), getWhoisInfo.getString("registrant_telephone"))); } } return fqdnFiles; } private String getRepLevel(Integer repScore){ String level = null; //用str存放数据 if (repScore > 80) level="Trustworthy"; else if (repScore > 60) level="Low Risk"; else if (repScore > 40) level="Moderate Risk"; else if (repScore > 20) level="Suspicious"; else if (repScore > 0) level="High Risk"; return level; } public static boolean isEmail(String string) { if (string == null) return false; String regEx1 = "^([a-z0-9A-Z]+[-|\\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\\.)+[a-zA-Z]{2,}$"; Pattern p; Matcher m; p = Pattern.compile(regEx1); m = p.matcher(string); return m.matches(); } private void geneCatInfo(){ if (catId2Info.size()==0){ JSONObject jsonObject = null; String s = FileUtils.readJsonFile(props.getProperty("bc.cateinfo.filepath")); jsonObject = JSON.parseObject(s); if (!(jsonObject==null)){ JSONObject tmp = (JSONObject) jsonObject.getJSONArray("results").get(0); JSONArray catInfoArray = tmp.getJSONObject("queries").getJSONObject("getcatlist").getJSONArray("cats"); for (int i = 0; i < catInfoArray.size(); i++){ JSONObject keyObject = catInfoArray.getJSONObject(i); List value = new ArrayList<>(Arrays.asList( keyObject.getString("catname"), keyObject.getString("catgroup"))); catId2Info.put(i+1, value); } } } } public List getCatInfo(Integer catId){ List info = Arrays.asList("", ""); if (0 < catId && catId <= 83) { if (catId2Info.size()==0){ geneCatInfo(); } info = catId2Info.get(catId); if (info == null){ LOG.error("Failed at geneCatInfo function"); System.out.print("Failed at geneCatInfo function"); } } return info; } public static List getQueryTypes() { return queryTypes; } }