diff --git a/pom.xml b/pom.xml
index 44528db..df7d7ab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -263,5 +263,12 @@
+
+
+ com.google.guava
+ guava
+ 25.1-jre
+
+
diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties
index 66f5690..c307cc7 100644
--- a/properties/service_flow_config.properties
+++ b/properties/service_flow_config.properties
@@ -74,4 +74,4 @@ influx.ip=http://192.168.40.151:8086
influx.username=admin
#influx密码
-influx.password=admin
+influx.password=admin
\ No newline at end of file
diff --git a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java
index 72003c0..ffed96b 100644
--- a/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java
+++ b/src/main/java/cn/ac/iie/utils/general/TransFormUtils.java
@@ -7,7 +7,9 @@ import cn.ac.iie.bean.security.SecurityPolicyLog;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.hbase.HBaseUtils;
import cn.ac.iie.utils.system.SnowflakeId;
+import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
+import com.google.common.net.InternetDomainName;
import com.zdjizhi.utils.IpLookup;
import com.zdjizhi.utils.StringUtil;
import org.apache.log4j.Logger;
@@ -28,10 +30,9 @@ import java.util.regex.Pattern;
public class TransFormUtils {
private static Logger logger = Logger.getLogger(TransFormUtils.class);
- private final static Set PUBLIC_SUFFIX_SET = new HashSet(
- Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se"
- .split("\\|")));
- private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})");
+ // private final static Set PUBLIC_SUFFIX_SET = new HashSet(
+// Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se"
+// .split("\\|")));
private static IpLookup ipLookup = new IpLookup.Builder(false)
.loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb")
.loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb")
@@ -58,9 +59,10 @@ public class TransFormUtils {
radiusSessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp));
radiusSessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true));
radiusSessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true));
+ radiusSessionRecordLog.setCommon_subscriber_id(radiusSessionRecordLog.getRadius_account());
return JSONObject.toJSONString(radiusSessionRecordLog);
} catch (Exception e) {
- logger.error("{} 日志解析过程出现异常" + FlowWriteConfig.KAFKA_TOPIC);
+ logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常");
e.printStackTrace();
return "";
}
@@ -149,7 +151,8 @@ public class TransFormUtils {
collectProtocolRecordLog.setHttp_domain(getTopDomain(collectProtocolRecordLog.getSsl_sni(), collectProtocolRecordLog.getHttp_host()));
return JSONObject.toJSONString(collectProtocolRecordLog);
} catch (Exception e) {
- logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e);
+ logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常");
+ e.printStackTrace();
return "";
}
}
@@ -179,23 +182,13 @@ public class TransFormUtils {
* @return 顶级域名
*/
private static String getDomainName(String host) {
- if (host.endsWith(FlowWriteConfig.DOMAIN_SPLITTER)) {
- host = host.substring(0, host.length() - 1);
+ String domain = "";
+ try {
+ domain = InternetDomainName.from(host).topPrivateDomain().toString();
+ } catch (Exception e) {
+ logger.error("host解析顶级域名异常: " + e.getMessage());
}
- if (IP_PATTERN.matcher(host).matches()) {
- return host;
- }
- int index = 0;
- String candidate = host;
- for (; index >= 0; ) {
- index = candidate.indexOf(FlowWriteConfig.DOMAIN_SPLITTER);
- String subCandidate = candidate.substring(index + 1);
- if (PUBLIC_SUFFIX_SET.contains(subCandidate)) {
- return candidate;
- }
- candidate = subCandidate;
- }
- return candidate;
+ return domain;
}
diff --git a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java
index 2b60e43..dfcf3b8 100644
--- a/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java
+++ b/src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java
@@ -26,7 +26,7 @@ import java.util.concurrent.ConcurrentSkipListMap;
public class HBaseUtils {
private final static Logger logger = Logger.getLogger(HBaseUtils.class);
private static Map subIdMap = new HashMap<>(333334);
-// private static Map subIdMap = new ConcurrentSkipListMap<>();
+ // private static Map subIdMap = new ConcurrentSkipListMap<>();
private static Connection connection;
private static Long time;
@@ -137,4 +137,5 @@ public class HBaseUtils {
public static String getAccount(String clientIp) {
return subIdMap.get(clientIp);
}
+
}
diff --git a/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java b/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java
index 1bf76eb..78deae0 100644
--- a/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java
+++ b/src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java
@@ -72,7 +72,7 @@ public class KafkaLogNtc {
properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
properties.put("acks", "1");
properties.put("linger.ms", "2");
- properties.put("request.timeout.ms", 60000);
+ properties.put("request.timeout.ms", 30000);
properties.put("batch.size", 262144);
properties.put("buffer.memory", 33554432);
// properties.put("compression.type", "snappy");
diff --git a/src/test/java/cn/ac/iie/test/DomainUtils.java b/src/test/java/cn/ac/iie/test/DomainUtils.java
index ec2cce4..3a0a427 100644
--- a/src/test/java/cn/ac/iie/test/DomainUtils.java
+++ b/src/test/java/cn/ac/iie/test/DomainUtils.java
@@ -1,67 +1,50 @@
package cn.ac.iie.test;
+import com.google.common.net.InternetDomainName;
import com.zdjizhi.utils.StringUtil;
import javax.xml.bind.SchemaOutputResolver;
import java.net.MalformedURLException;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class DomainUtils {
-
- private static Pattern pattern = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.me|\\.cc|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)");
-
private final static Set PublicSuffixSet = new HashSet(
- Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se"
+ Arrays.asList("com|edu|gov|int|mil|net|org|biz|info|pro|name|museum|coop|aero|xxx|idv|top|xyz|xin|vip|win|red|wang|co|mobi|travel|club|post|rec|asia"
.split("\\|")));
- private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})");
public static void main(String[] args) {
- System.out.println(getTopDomain("http://www.ccb.com"));
- System.out.println(getDomainName("www.comaa.com"));
+// InternetDomainName.from("foo.co.uk").topPrivateDomain().toString();
+ String host = "www.aaa.co.uk";
+// if (host.contains(":")){
+// String s = host.split(":")[0];
+// System.out.println(InternetDomainName.from(s));
+// System.out.println(InternetDomainName.from(s).topPrivateDomain());
+// }else {
+ System.out.println(InternetDomainName.from(host).topDomainUnderRegistrySuffix());
+ System.out.println(InternetDomainName.from(host).topPrivateDomain());
+// }
+// System.out.println(InternetDomainName.from("shence.hupu.com").topPrivateDomain());
}
- private static String getTopDomain(String url) {
- try {
-// 获取值转换为小写
- String host = new URL(url).getHost().toLowerCase();//news.hexun.com
-// Pattern pattern = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)");
- Matcher matcher = IP_PATTERN.matcher(host);
- if (matcher.find()){
- return matcher.group();
- }
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- return null;
- }
-
-
-// private final static Set PublicSuffixSet = new HashSet(
-// Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se"
-// .split("\\|")));
-// private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})");
-
/**
* 获取url的顶级域名
-// * @param url
+ * // * @param url
+ *
* @return
*/
public static String getDomainName(String host) {
-// String host = url.getHost();
- if (host.endsWith(".")){
+ if (host.endsWith(".") || host.contains("/")) {
host = host.substring(0, host.length() - 1);
}
- if (IP_PATTERN.matcher(host).matches()){
- return host;
- }
int index = 0;
String candidate = host;
- for (; index >= 0;) {
+ for (; index >= 0; ) {
index = candidate.indexOf('.');
String subCandidate = candidate.substring(index + 1);
if (PublicSuffixSet.contains(subCandidate)) {
@@ -73,4 +56,38 @@ public class DomainUtils {
}
+ public static String getSourceDomain(String host) {
+ if (host.endsWith(".")) {
+ host = host.substring(0, host.length() - 1);
+ }
+ String[] hostStr = host.split("\\.");
+ int length = hostStr.length;
+ if (hostStr.length >= 2) {
+ if (PublicSuffixSet.contains(hostStr[length - 2])) {
+ return hostStr[length - 3] + "." + hostStr[length - 2] + "." + hostStr[length - 1];
+ } else {
+ return hostStr[length - 2] + "." + hostStr[length - 1];
+ }
+ } else {
+ return host;
+ }
+ }
+
+
+ // 定义正则表达式，域名的根需要自定义，这里不全
+ private static final String RE_TOP = "[\\w-]+\\.(com.cn|net.cn|gov.cn|org\\.nz|org.cn|com|net|org|gov|cc|biz|info|cn|co)\\b()*";
+ private static Pattern pattern = Pattern.compile(RE_TOP, Pattern.CASE_INSENSITIVE);
+
+ public static String getTopDomain(String url) {
+ String result = url;
+ try {
+ Matcher matcher = pattern.matcher(url);
+ matcher.find();
+ result = matcher.group();
+ } catch (Exception e) {
+ System.out.println("[getTopDomain ERROR]====>");
+ e.printStackTrace();
+ }
+ return result;
+ }
}
diff --git a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java
index a6fb619..26d9d09 100644
--- a/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java
+++ b/src/test/java/cn/ac/iie/test/hbase/HBaseTest.java
@@ -32,7 +32,7 @@ public class HBaseTest {
// 设置zookeeper节点
// configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS);
// configuration.set("hbase.zookeeper.quorum", "192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181");
- configuration.set("hbase.zookeeper.quorum", "192.168.40.224:2182");
+ configuration.set("hbase.zookeeper.quorum", "192.168.40.186:2182");
configuration.set("hbase.client.retries.number", "3");
configuration.set("hbase.bulkload.retries.number", "3");
configuration.set("zookeeper.recovery.retry", "3");
diff --git a/src/test/java/cn/ac/iie/test/test.java b/src/test/java/cn/ac/iie/test/test.java
index aa5b7db..e361e31 100644
--- a/src/test/java/cn/ac/iie/test/test.java
+++ b/src/test/java/cn/ac/iie/test/test.java
@@ -1,16 +1,22 @@
package cn.ac.iie.test;
-import cn.ac.iie.test.bean.Student;
+import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
+import com.google.common.net.InternetDomainName;
import org.apache.log4j.Logger;
public class test {
private static Logger logger = Logger.getLogger(test.class);
public static void main(String[] args) {
- String message = "{\"name\":\"aaa\"}";
- Student student = JSONObject.parseObject(message, Student.class);
- System.out.println(JSONObject.toJSONString(student));
- }
+ String message = "{\"common_schema_type\":\"SSL\",\"ssl_sni\":\"www.myssl.cn\",\"common_server_ip\":\"101.37.81.250\",\"common_client_ip\":\"192.168.50.12\",\"common_server_port\":443,\"common_client_port\":51398,\"common_stream_dir\":3,\"common_address_type\":4,\"common_s2c_pkt_num\":1,\"common_s2c_byte_num\":0,\"common_c2s_pkt_num\":3,\"common_c2s_byte_num\":517,\"common_stream_trace_id\":7629423224647,\"common_l4_protocol\":\"IPv4_TCP\",\"common_address_list\":\"51398-443-192.168.50.12-101.37.81.250\",\"common_sled_ip\":\"192.168.40.161\",\"common_start_time\":1575008799,\"common_end_time\":1575008799,\"common_con_duration_ms\":0,\"common_policy_id\":282,\"common_service\":0,\"common_action\":128,\"common_user_region\":\"{\\\"protocol\\\":\\\"SSL\\\"}\"}";
+ JSONObject jsonObject = JSON.parseObject(message);
+ String common_server_ip = jsonObject.getString("common_server_ip");
+ String common_client_ip = jsonObject.getString("common_client_ip");
+ jsonObject.put("common__recv_time",(System.currentTimeMillis() / 1000));
+
+ System.out.println(jsonObject.toString());
+
+ }
}