This repository has been archived on 2025-09-14. You can view files and clone it, but cannot push or open issues or pull requests.
Files
web-sketch-webskt-query-agent/src/main/java/com/mesasoft/cn/sketch/service/SketchService.java
2023-01-10 15:51:02 +08:00

233 lines
21 KiB
Java
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*
package cn.ac.iie.efo.sketch.service;
import cn.ac.iie.api.ChinaZ;
import cn.ac.iie.dao.MariaDbBase;
import cn.ac.iie.util.MariaDBUtils;
import cn.ac.iie.util.ValidationUtils;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import cn.ac.iie.efo.sketch.api.BrightCloud;
import cn.ac.iie.efo.sketch.api.ChinaZ;
import cn.ac.iie.efo.sketch.config.ApplicationConfig;
import cn.ac.iie.efo.sketch.entity.domain.DomainCategory;
import cn.ac.iie.efo.sketch.entity.domain.DomainWhois;
import cn.ac.iie.efo.util.FileUtils;
import org.apache.log4j.Logger;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.Date;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
*/
/**
* Created with IntelliJ IDEA.
* User: joy
* Date: 2021/12/31
* Time: 11:28 AM
* Description: No Description
*//*
public class SketchService {
private static final Logger LOG = Logger.getLogger(SketchService.class);
private static final int MAX_DB_BATCH_SIZE = ApplicationConfig.DB_QUERY_BATCH_SIZE;
private List<String> queryObjects;
private String curQueryType;
private MariaDbBase mariaDB;
long queryNum;
long dbResultNum;
long apiResultNum;
long failedQueryNum;
public JSONArray getQueryResults(String objectType, String queryType, List<String> queryObjects, String username, Boolean isLocal) throws SQLException, IOException {
queryNum = 0;
dbResultNum = 0;
apiResultNum = 0;
failedQueryNum = 0;
this.queryObjects = queryObjects;
this.curQueryType = queryType;
JSONArray json = null;
Connection mariaConn = MariaDBUtils.getConnection();
Statement mariaStat = mariaConn.createStatement();
mariaDB = new MariaDbBase(mariaConn, mariaStat);
//去重
queryObjects = queryObjects.stream().distinct().collect(Collectors.toList());
// 校验
queryObjects = ValidationUtils.getChecked(queryObjects, objectType);
this.queryNum = queryObjects.size();
// 执行查询
if (queryType.equals("domain_category")) {
json = getDomainCategory(queryObjects, username, isLocal);
} else if (queryType.equals("domain_whois")) {
json = getDomainWhois(queryObjects, isLocal);
} else {
// TODO get dns server info
LOG.error("Wrong query type: " + queryType);
}
LOG.info("Query Statistic - number of query objects: " + queryNum
+ "\nresults from local db: " + dbResultNum
+ " results from external api: " + apiResultNum);
MariaDBUtils.close(mariaStat, mariaConn);
return json;
}
public JSONArray getDomainCategory(List<String> domains, String username, boolean isLocal) throws SQLException, IOException {
JSONArray results = new JSONArray();
// 查询本地数据库
ArrayList<String> objectsFromDB = new ArrayList<>();
List<List<String>> partitions = Lists.partition(queryObjects, MAX_DB_BATCH_SIZE);
for (List<String> partition : partitions) { // 批量查询
List<DomainCategory> dbRecords = DomainCategory.getDbRecord(partition, mariaDB, "brightcloud");
for (DomainCategory record : dbRecords) {
objectsFromDB.add(record.getFqdn()); // 保存查询记录
JSONObject jsonObject = DomainCategory.schema2json(record);
results.add(jsonObject); // 保存查询结果
}
}
dbResultNum = results.size();
// 调用api
List<DomainCategory> bcResults = new ArrayList<>();
List<String> objectsFromApi = new ArrayList<>(queryObjects);
objectsFromApi.removeAll(objectsFromDB);
if (!isLocal && objectsFromApi.size() > 0) {
BrightCloud brightCloud = new BrightCloud();
List<List<String>> apiPartitions = Lists.partition(objectsFromApi, ApplicationConfig.API_BC_MAXIMUM_QUERYNUM);
for (List<String> partition : apiPartitions) { // 批量查询
List<DomainCategory> recordsFromBcApi = brightCloud.responseSparse(brightCloud.getQueryResults(partition));
for (DomainCategory record : recordsFromBcApi) {
if (record.getQuery_success().equals(true)) { //查询成功的结果
record.setSubmit_user(username);
bcResults.add(record);
if (bcResults.size() > MAX_DB_BATCH_SIZE) { //超过一定量时写入数据库
DomainCategory.insertRecords(bcResults, mariaDB);
bcResults.clear();
}
results.add(DomainCategory.schema2json(record));
} else { // 查询失败的结果
failedQueryNum += 1;
}
}
}
}
apiResultNum = results.size() - dbResultNum - failedQueryNum;
DomainCategory.insertRecords(bcResults, mariaDB);
if (apiResultNum > 0) { // 记录api调用次数
OutputStream bcQueryLogStream = new FileOutputStream(ApplicationConfig.API_BC_USE_REPORT_FILE, true);
OutputStreamWriter bcQueryLogWriter = new OutputStreamWriter(bcQueryLogStream, StandardCharsets.UTF_8);
Date d = new Date(System.currentTimeMillis());
bcQueryLogWriter.write(d + "," + "List Query," + "-" + "," + curQueryType + "," + apiResultNum + "\n");
FileUtils.writerClose(bcQueryLogWriter, bcQueryLogStream);
}
return results;
}
public JSONArray getDomainWhois(List<String> domains, boolean isLocal) throws SQLException, IOException {
JSONArray results = new JSONArray();
// 查询本地数据库
ArrayList<String> objectsFromDB = new ArrayList<>();
List<List<String>> partitions = Lists.partition(queryObjects, MAX_DB_BATCH_SIZE);
for (List<String> partition : partitions) { // 批量查询
List<DomainWhois> dbRecords =
DomainWhois.getDbRecord(partition, mariaDB, "chinaz");
for (DomainWhois record : dbRecords) {
objectsFromDB.add(record.getFqdn()); // 保存查询记录
JSONObject jsonObject = DomainWhois.schema2json(record);
results.add(jsonObject); // 保存查询结果
}
}
dbResultNum = results.size();
// 调用api
List<DomainWhois> chinazResults = new ArrayList<>();
if (!isLocal) {
ChinaZ chinaz = new ChinaZ();
List<String> objectsFromApi = new ArrayList<>(queryObjects);
objectsFromApi.removeAll(objectsFromDB); // 需要调用api查询的部分对象
List<List<String>> apiPartitions = Lists.partition(objectsFromApi, ApplicationConfig.API_CHINAZ_MAXIMUM_QUERYNUM);
for (List<String> partition : apiPartitions) { // 批量查询
List<DomainWhois> recordsFromApi = chinaz.responseSparse(chinaz.getQueryResults(partition));
for (DomainWhois record : recordsFromApi) {
if (record.getQuery_success().equals(true)) { // 查询成功的结果
chinazResults.add(record);
if (chinazResults.size() > MAX_DB_BATCH_SIZE) { // 超过一定量时写入数据库
DomainWhois.insertRecords(chinazResults, mariaDB);
chinazResults.clear();
}
results.add(DomainWhois.schema2json(record));
} else { // 查询失败的结果
failedQueryNum += 1;
}
}
}
}
DomainWhois.insertRecords(chinazResults, mariaDB);
apiResultNum = results.size() - dbResultNum - failedQueryNum;
// todo 不直接写入,维护变量,导出一次
if (apiResultNum > 0) { // 记录api调用次数
OutputStream bcQueryLogStream = new FileOutputStream(ApplicationConfig.API_CHINAZ_USE_REPORT_FILE, true);
OutputStreamWriter bcQueryLogWriter = new OutputStreamWriter(bcQueryLogStream, StandardCharsets.UTF_8);
Date d = new Date(System.currentTimeMillis());
bcQueryLogWriter.write(d + "," + "List Query," + "-" + "," + curQueryType + "," + apiResultNum + "\n");
FileUtils.writerClose(bcQueryLogWriter, bcQueryLogStream);
}
return results;
}
public static void main(String[] args) throws Exception {
// String objectType = args[0];
// String queryType = args[1];
// List<String> queryObjects = Arrays.asList(args[2].split(","));
// String username = args[4];
// boolean isLocal = true;
// if (args.length >= 3) {
// isLocal = Boolean.parseBoolean(args[3]);
// }
// System.out.println(new SketchService().getQueryResults(objectType, queryType, queryObjects, username, isLocal).toString());
JSONArray queryResults = new SketchService().getQueryResults("domain",
"domain_category",
Arrays.asList("baidu.com", "cctv.com"), "", true);
System.err.println(queryResults.toJSONString());
String json = "{\"namespace\":\"log.session\",\"type\":\"record\",\"name\":\"session\",\"fields\":[{\"name\":\"common_log_id\",\"type\":\"long\"},{\"name\":\"common_service\",\"type\":\"long\"},{\"name\":\"common_recv_time\",\"type\":\"long\"},{\"name\":\"common_direction\",\"type\":\"long\"},{\"name\":\"common_l4_protocol\",\"type\":\"string\"},{\"name\":\"common_address_type\",\"type\":\"long\"},{\"name\":\"common_schema_type\",\"type\":\"string\"},{\"name\":\"common_policy_id\",\"type\":\"long\"},{\"name\":\"common_user_tags\",\"type\":\"string\"},{\"name\":\"common_action\",\"type\":\"long\"},{\"name\":\"common_sub_action\",\"type\":\"string\"},{\"name\":\"common_user_region\",\"type\":\"string\"},{\"name\":\"common_client_ip\",\"type\":\"string\"},{\"name\":\"common_client_port\",\"type\":\"long\"},{\"name\":\"common_internal_ip\",\"type\":\"string\"},{\"name\":\"common_entrance_id\",\"type\":\"long\"},{\"name\":\"common_device_id\",\"type\":\"string\"},{\"name\":\"common_egress_link_id\",\"type\":\"long\"},{\"name\":\"common_ingress_link_id\",\"type\":\"long\"},{\"name\":\"common_isp\",\"type\":\"string\"},{\"name\":\"common_device_tag\",\"type\":\"string\"},{\"name\":\"common_data_center\",\"type\":\"string\"},{\"name\":\"common_encapsulation\",\"type\":\"long\"},{\"name\":\"common_tunnels\",\"type\":\"string\"},{\"name\":\"common_sled_ip\",\"type\":\"string\"},{\"name\":\"common_device_group\",\"type\":\"string\"},{\"name\":\"common_app_behavior\",\"type\":\"string\"},{\"name\":\"common_client_location\",\"type\":\"string\"},{\"name\":\"common_client_asn\",\"type\":\"string\"},{\"name\":\"common_subscriber_id\",\"type\":\"string\"},{\"name\":\"common_imei\",\"type\":\"string\"},{\"name\":\"common_imsi\",\"type\":\"string\"},{\"name\":\"common_phone_number\",\"type\":\"string\"},{\"name\":\"common_server_ip\",\"type\":\"string\"},{\"name\":\"common_server_port\",\"type\":\"long\"},{\"name\":\"common_external_ip\",\"type\":\"string\"},{\"name\":\"common_server_location\",\"type\":\"string\"},{\"name\":\"common_server_asn\",\"type\":\"string\"},{\"name\":\"common_protocol_label\",\"type\":\"string\"},{\"name\":\"common_service_category\",\"type\":{\"type\":\"array\",\"items\":\"long\"}},{\"name\":\"common_app_label\",\"type\":\"string\"},{\"name\":\"common_app_id\",\"type\":\"string\"},{\"name\":\"common_userdefine_app_name\",\"type\":\"string\"},{\"name\":\"common_app_surrogate_id\",\"type\":\"string\"},{\"name\":\"common_l7_protocol\",\"type\":\"string\"},{\"name\":\"common_sessions\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_num\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_num\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_diff\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_diff\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_diff\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_diff\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_num\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_num\",\"type\":\"long\"},{\"name\":\"common_start_time\",\"type\":\"long\"},{\"name\":\"common_end_time\",\"type\":\"long\"},{\"name\":\"common_establish_latency_ms\",\"type\":\"long\"},{\"name\":\"common_con_duration_ms\",\"type\":\"long\"},{\"name\":\"common_stream_dir\",\"type\":\"long\"},{\"name\":\"common_address_list\",\"type\":\"string\"},{\"name\":\"common_has_dup_traffic\",\"type\":\"long\"},{\"name\":\"common_stream_error\",\"type\":\"string\"},{\"name\":\"common_stream_trace_id\",\"type\":\"long\"},{\"name\":\"common_link_info_c2s\",\"type\":\"string\"},{\"name\":\"common_link_info_s2c\",\"type\":\"string\"},{\"name\":\"common_packet_capture_file\",\"type\":\"string\"},{\"name\":\"common_c2s_ipfrag_num\",\"type\":\"long\"},{\"name\":\"common_s2c_ipfrag_num\",\"type\":\"long\"},{\"name\":\"common_c2s_tcp_lostlen\",\"type\":\"long\"},{\"name\":\"common_s2c_tcp_lostlen\",\"type\":\"long\"},{\"name\":\"common_c2s_tcp_unorder_num\",\"type\":\"long\"},{\"name\":\"common_s2c_tcp_unorder_num\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_retrans\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_retrans\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_retrans\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_retrans\",\"type\":\"long\"},{\"name\":\"common_tcp_client_isn\",\"type\":\"long\"},{\"name\":\"common_tcp_server_isn\",\"type\":\"long\"},{\"name\":\"common_mirrored_pkts\",\"type\":\"long\"},{\"name\":\"common_mirrored_bytes\",\"type\":\"long\"},{\"name\":\"common_first_ttl\",\"type\":\"long\"},{\"name\":\"common_processing_time\",\"type\":\"long\"},{\"name\":\"http_url\",\"type\":\"string\"},{\"name\":\"http_host\",\"type\":\"string\"},{\"name\":\"http_domain\",\"type\":\"string\"},{\"name\":\"http_request_line\",\"type\":\"string\"},{\"name\":\"http_response_line\",\"type\":\"string\"},{\"name\":\"http_request_header\",\"type\":\"string\"},{\"name\":\"http_response_header\",\"type\":\"string\"},{\"name\":\"http_request_content\",\"type\":\"string\"},{\"name\":\"http_response_content\",\"type\":\"string\"},{\"name\":\"http_request_body\",\"type\":\"string\"},{\"name\":\"http_response_body\",\"type\":\"string\"},{\"name\":\"http_request_body_key\",\"type\":\"string\"},{\"name\":\"http_response_body_key\",\"type\":\"string\"},{\"name\":\"http_proxy_flag\",\"type\":\"long\"},{\"name\":\"http_sequence\",\"type\":\"long\"},{\"name\":\"http_snapshot\",\"type\":\"string\"},{\"name\":\"http_cookie\",\"type\":\"string\"},{\"name\":\"http_referer\",\"type\":\"string\"},{\"name\":\"http_user_agent\",\"type\":\"string\"},{\"name\":\"http_request_content_length\",\"type\":\"string\"},{\"name\":\"http_request_content_type\",\"type\":\"string\"},{\"name\":\"http_response_content_length\",\"type\":\"string\"},{\"name\":\"http_response_content_type\",\"type\":\"string\"},{\"name\":\"http_content_length\",\"type\":\"string\"},{\"name\":\"http_content_type\",\"type\":\"string\"},{\"name\":\"http_set_cookie\",\"type\":\"string\"},{\"name\":\"http_version\",\"type\":\"string\"},{\"name\":\"http_response_latency_ms\",\"type\":\"long\"},{\"name\":\"http_session_duration_ms\",\"type\":\"long\"},{\"name\":\"http_action_file_size\",\"type\":\"long\"},{\"name\":\"mail_protocol_type\",\"type\":\"string\"},{\"name\":\"mail_account\",\"type\":\"string\"},{\"name\":\"mail_to_cmd\",\"type\":\"string\"},{\"name\":\"mail_from_cmd\",\"type\":\"string\"},{\"name\":\"mail_from\",\"type\":\"string\"},{\"name\":\"mail_to\",\"type\":\"string\"},{\"name\":\"mail_cc\",\"type\":\"string\"},{\"name\":\"mail_bcc\",\"type\":\"string\"},{\"name\":\"mail_subject\",\"type\":\"string\"},{\"name\":\"mail_subject_charset\",\"type\":\"string\"},{\"name\":\"mail_content\",\"type\":\"string\"},{\"name\":\"mail_content_charset\",\"type\":\"string\"},{\"name\":\"mail_attachment_name\",\"type\":\"string\"},{\"name\":\"mail_attachment_name_charset\",\"type\":\"string\"},{\"name\":\"mail_attachment_content\",\"type\":\"string\"},{\"name\":\"mail_eml_file\",\"type\":\"string\"},{\"name\":\"mail_snapshot\",\"type\":\"string\"},{\"name\":\"dns_message_id\",\"type\":\"long\"},{\"name\":\"dns_qr\",\"type\":\"long\"},{\"name\":\"dns_opcode\",\"type\":\"long\"},{\"name\":\"dns_aa\",\"type\":\"long\"},{\"name\":\"dns_tc\",\"type\":\"long\"},{\"name\":\"dns_rd\",\"type\":\"long\"},{\"name\":\"dns_ra\",\"type\":\"long\"},{\"name\":\"dns_rcode\",\"type\":\"long\"},{\"name\":\"dns_qdcount\",\"type\":\"long\"},{\"name\":\"dns_ancount\",\"type\":\"long\"},{\"name\":\"dns_nscount\",\"type\":\"long\"},{\"name\":\"dns_arcount\",\"type\":\"long\"},{\"name\":\"dns_qname\",\"type\":\"string\"},{\"name\":\"dns_qtype\",\"type\":\"long\"},{\"name\":\"dns_qclass\",\"type\":\"long\"},{\"name\":\"dns_cname\",\"type\":\"string\"},{\"name\":\"dns_sub\",\"type\":\"long\"},{\"name\":\"dns_rr\",\"type\":\"string\"},{\"name\":\"ssl_version\",\"type\":\"string\"},{\"name\":\"ssl_sni\",\"type\":\"string\"},{\"name\":\"ssl_san\",\"type\":\"string\"},{\"name\":\"ssl_cn\",\"type\":\"string\"},{\"name\":\"ssl_pinningst\",\"type\":\"long\"},{\"name\":\"ssl_intercept_state\",\"type\":\"long\"},{\"name\":\"ssl_passthrough_reason\",\"type\":\"string\"},{\"name\":\"ssl_server_side_latency\",\"type\":\"long\"},{\"name\":\"ssl_client_side_latency\",\"type\":\"long\"},{\"name\":\"ssl_server_side_version\",\"type\":\"string\"},{\"name\":\"ssl_client_side_version\",\"type\":\"string\"},{\"name\":\"ssl_cert_verify\",\"type\":\"long\"},{\"name\":\"ssl_error\",\"type\":\"string\"},{\"name\":\"ssl_con_latency_ms\",\"type\":\"long\"},{\"name\":\"ssl_ja3_fingerprint\",\"type\":\"string\"},{\"name\":\"ssl_ja3_hash\",\"type\":\"string\"},{\"name\":\"ssl_cert_issuer\",\"type\":\"string\"},{\"name\":\"ssl_cert_subject\",\"type\":\"string\"},{\"name\":\"quic_version\",\"type\":\"string\"},{\"name\":\"quic_sni\",\"type\":\"string\"},{\"name\":\"quic_user_agent\",\"type\":\"string\"},{\"name\":\"ftp_account\",\"type\":\"string\"},{\"name\":\"ftp_url\",\"type\":\"string\"},{\"name\":\"ftp_content\",\"type\":\"string\"},{\"name\":\"ftp_link_type\",\"type\":\"string\"},{\"name\":\"bgp_type\",\"type\":\"long\"},{\"name\":\"bgp_as_num\",\"type\":\"string\"},{\"name\":\"bgp_route\",\"type\":\"string\"},{\"name\":\"voip_calling_account\",\"type\":\"string\"},{\"name\":\"voip_called_account\",\"type\":\"string\"},{\"name\":\"voip_calling_number\",\"type\":\"string\"},{\"name\":\"voip_called_number\",\"type\":\"string\"},{\"name\":\"sip_call_id\",\"type\":\"string\"},{\"name\":\"sip_originator_description\",\"type\":\"string\"},{\"name\":\"sip_responder_description\",\"type\":\"string\"},{\"name\":\"sip_user_agent\",\"type\":\"string\"},{\"name\":\"sip_server\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_connect_ip\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_media_port\",\"type\":\"long\"},{\"name\":\"sip_originator_sdp_media_type\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_content\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_connect_ip\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_media_port\",\"type\":\"long\"},{\"name\":\"sip_responder_sdp_media_type\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_content\",\"type\":\"string\"},{\"name\":\"sip_duration_s\",\"type\":\"long\"},{\"name\":\"sip_bye\",\"type\":\"string\"},{\"name\":\"rtp_payload_type_c2s\",\"type\":\"long\"},{\"name\":\"rtp_payload_type_s2c\",\"type\":\"long\"},{\"name\":\"rtp_pcap_path\",\"type\":\"string\"},{\"name\":\"rtp_originator_dir\",\"type\":\"long\"},{\"name\":\"ssh_version\",\"type\":\"string\"},{\"name\":\"ssh_auth_success\",\"type\":\"string\"},{\"name\":\"ssh_client_version\",\"type\":\"string\"},{\"name\":\"ssh_server_version\",\"type\":\"string\"},{\"name\":\"ssh_cipher_alg\",\"type\":\"string\"},{\"name\":\"ssh_mac_alg\",\"type\":\"string\"},{\"name\":\"ssh_compression_alg\",\"type\":\"string\"},{\"name\":\"ssh_kex_alg\",\"type\":\"string\"},{\"name\":\"ssh_host_key_alg\",\"type\":\"string\"},{\"name\":\"ssh_host_key\",\"type\":\"string\"},{\"name\":\"ssh_hassh\",\"type\":\"string\"},{\"name\":\"stratum_cryptocurrency\",\"type\":\"string\"},{\"name\":\"stratum_mining_pools\",\"type\":\"string\"},{\"name\":\"stratum_mining_program\",\"type\":\"string\"},{\"name\":\"streaming_media_url\",\"type\":\"string\"},{\"name\":\"streaming_media_protocol\",\"type\":\"string\"},{\"name\":\"app_extra_info\",\"type\":\"string\"}]}";
Object parse = JSONObject.parse(json);
System.err.println(parse);
}
}
*/