yinjiangyi-webskt-query-agent/src/main/java/cn/ac/iie/utils/BrightCloudUtils.java

package cn.ac.iie.utils;

import cn.ac.iie.config.CommonConfig;
import cn.ac.iie.dao.FqdnFile;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.log4j.Logger;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;


/**
 * @author yjy
 * @version 1.0
 * @date 2021/2/22 2:37 下午
 */

public class BrightCloudUtils {
    private static final Logger LOG = Logger.getLogger(BrightCloudUtils.class);
    private static final Properties props = new Properties();
    private HttpURLConnection con;
    private static List<String> queryTypes = new ArrayList<>();

    static {
        try {
            props.load(BrightCloudUtils.class.getClassLoader().getResourceAsStream("brightcloud.properties"));
        } catch (IOException e) {
            e.printStackTrace();
        }

        if (CommonConfig.QUERY_URL_INFO_SWITCH){
            queryTypes.add(props.getProperty("bc.api.name.category"));
        }
        if (CommonConfig.QUERY_URL_REP_SWITCH){
            queryTypes.add(props.getProperty("bc.api.name.reputation"));
        }
        if (CommonConfig.QUERY_URL_WHOIS_SWITCH){
            queryTypes.add(props.getProperty("bc.api.name.whois"));
        }
        assert queryTypes.size()>0: "Switch of all query type has been turned off, please edit the application.properties";
    }

    public HashMap<Integer, List<String>> getCatId2Info() {
        return catId2Info;
    }

    private final HashMap<Integer, List<String>> catId2Info = new HashMap<>();

    public JSONObject getQueryResults (List<String> urls) {
        return getQueryResults(urls, CommonConfig.BC_API_NAME_CATEGORY);
    }

    public JSONObject getQueryResults (List<String> urls, String queryType) {
        if (urls.size()> CommonConfig.MAXIMUM_URL_ONCE_BC_QUERY){
            LOG.warn("Too many urls in a http post request!");
        }
        JSONObject jsonRes = null;
        try {

            URL url = new URL(props.getProperty("bc.api.url"));
            // 打开和URL之间的连接
            con = (HttpURLConnection) url.openConnection();
            con.setRequestMethod(props.getProperty("bc.api.method"));
            con.setDoOutput(true);
            con.setDoInput(true);

            con.setRequestProperty("Content-Type", "application/json");

            JSONObject param = new JSONObject();
            param.put("oemid", props.getProperty("bc.oemid"));
            param.put("deviceid", props.getProperty("bc.deviceid"));
            param.put("uid", props.getProperty("bc.uid"));

            param.put("queries", new ArrayList<>(Collections.singletonList(queryType)));
            param.put("a1cat", props.getProperty("bc.api.a1cat"));
            param.put("reputation", props.getProperty("bc.api.reputation"));
            param.put("xml", props.getProperty("bc.api.xml"));

            param.put("urls", urls);

            // 建立实际的连接
            con.connect();
            OutputStreamWriter writer = new OutputStreamWriter(this.con.getOutputStream(), StandardCharsets.UTF_8);
            writer.write(param.toString());
            writer.flush();
        } catch (IOException e) {
            e.printStackTrace();
        }

        try {
            // 获取服务端响应，通过输入流来读取URL的响应
            InputStream is = con.getInputStream();
            BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
            StringBuffer sbf = new StringBuffer();
            String strRead = null;
            while ((strRead = reader.readLine()) != null) {
                sbf.append(strRead);
                sbf.append("\r\n");
            }
            reader.close();

            jsonRes = JSONObject.parseObject(sbf.toString());
            con.disconnect();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return jsonRes;
    }

    public HttpURLConnection getCon() {
        return con;
    }

    public List<FqdnFile> responseSparse(JSONObject records){
        return responseSparse(records, CommonConfig.BC_API_NAME_CATEGORY);
    }

    public List<FqdnFile> responseSparse(JSONObject records, String queryType){
        List<FqdnFile> fqdnFiles = new ArrayList<>();
        Boolean querySucess = records.get("status").equals(200);

        if (!querySucess) {
            System.out.print(records.toString());
            LOG.error("Wrong query. Query type: " + records.get("type"));
        } else {
            JSONArray array = records.getJSONArray("results");
            for (int i = 0; i < array.size(); i++) {
                JSONObject jo = array.getJSONObject(i);

                // json处理
                JSONObject queries = jo.getJSONObject("queries");

                JSONObject getInfo = queries.getJSONObject(queryType);

                if (queryType.equals(CommonConfig.BC_API_NAME_CATEGORY)){
                    JSONObject cat = getInfo.getJSONArray("cats").getJSONObject(0);
                    Integer catId = cat.getInteger("catid");
                    fqdnFiles.add(new FqdnFile(
                            jo.getString("url"),
                            querySucess,
                            getInfo.getInteger("reputation"),
                            getRepLevel(getInfo.getInteger("reputation")),
                            catId,
                            getCatInfo(catId).get(0),
                            getCatInfo(catId).get(1),
                            cat.getInteger("conf"),
                            getInfo.getBoolean("a1cat")));
                } else if (queryType.equals(CommonConfig.BC_API_NAME_REPUTATION)){
                    fqdnFiles.add(new FqdnFile(
                            jo.getString("url"),
                            querySucess,
                            getInfo.getInteger("reputation"),
                            getRepLevel(getInfo.getInteger("reputation")),
                            getInfo.getInteger("popularity"),
                            getInfo.getInteger("age"),
                            getInfo.getString("country"),
                            getInfo.getInteger("threathistory")));
                } else if (queryType.equals(CommonConfig.BC_API_NAME_WHOIS)){
                    String whoisEmail = "";
                    if (isEmail(getInfo.getString("contactemail"))){
                        whoisEmail = getInfo.getString("contactemail");
                    }
                    fqdnFiles.add(new FqdnFile(
                            jo.getString("url"),
                            querySucess,
                            getInfo.getString("domainname"),
                            getInfo.getDate("audit_auditupdateddate"),
                            getInfo.getDate("createddate"),
                            getInfo.getDate("expiresdate"),
                            whoisEmail,
                            getInfo.getString("nameservers"),
                            getInfo.getString("registrarname"),
                            getInfo.getString("registrant_organization"),
                            getInfo.getString("registrant_name"),
                            getInfo.getString("registrant_street1"),
                            getInfo.getString("registrant_city"),
                            getInfo.getString("registrant_state"),
                            getInfo.getString("registrant_postalcode"),
                            getInfo.getString("registrant_country"),
                            getInfo.getString("registrant_telephone")));
                }
            }
        }
        return fqdnFiles;
    }

    private String getRepLevel(Integer repScore){
        String level = null;   //用str存放数据
        if (repScore > 80){ level="Trustworthy";}
        else if (repScore > 60){ level="Low Risk";}
        else if (repScore > 40){ level="Moderate Risk";}
        else if (repScore > 20){ level="Suspicious";}
        else if (repScore > 0){ level="High Risk";}
        return level;
    }

    public static boolean isEmail(String string) {
        if (string == null){
            return false;
        }
        String regEx1 = "^([a-z0-9A-Z]+[-|\\.]?)+[a-z0-9A-Z]@([a-z0-9A-Z]+(-[a-z0-9A-Z]+)?\\.)+[a-zA-Z]{2,}$";
        Pattern p;
        Matcher m;
        p = Pattern.compile(regEx1);
        m = p.matcher(string);
        return m.matches();
    }


    private void geneCatInfo(){
        if (catId2Info.size()==0){

            JSONObject jsonObject = null;

            String s = FileUtils.readJsonFile(props.getProperty("bc.cateinfo.filepath"));
            jsonObject = JSON.parseObject(s);

            if (!(jsonObject==null)){
                JSONObject tmp = (JSONObject) jsonObject.getJSONArray("results").get(0);
                JSONArray catInfoArray = tmp.getJSONObject("queries").getJSONObject("getcatlist").getJSONArray("cats");

                for (int i = 0; i < catInfoArray.size(); i++){
                    JSONObject keyObject = catInfoArray.getJSONObject(i);
                    List<String> value = new ArrayList<>(Arrays.asList(
                            keyObject.getString("catname"),
                            keyObject.getString("catgroup")));
                    catId2Info.put(i+1, value);
                }
            }
        }
    }

    public List<String> getCatInfo(Integer catId){
        List<String> info = Arrays.asList("", "");

        if (0 < catId && catId <= 83) {
            if (catId2Info.size()==0){
                geneCatInfo();
            }

            info = catId2Info.get(catId);

            if (info == null){
                LOG.error("Failed at geneCatInfo function");
                System.out.print("Failed at geneCatInfo function");
            }
        }

        return info;
    }

    public static List<String> getQueryTypes() {
        return queryTypes;
    }
}