gfwleak/galaxy-tsg-olap-storm-log-stream-completion
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改 domain获取方式为 使用guava获取

Browse Source
This commit is contained in:
qidaijie
2019-12-17 10:42:06 +08:00
parent fb22d60bfe
commit a7e25af875
8 changed files with 90 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
pom.xml
View File

@@ -263,5 +263,12 @@
</exclusions>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.guava/guava -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>25.1-jre</version>
</dependency>
</dependencies>
</project>

2
properties/service_flow_config.properties
View File

@@ -74,4 +74,4 @@ influx.ip=http://192.168.40.151:8086
influx.username=admin
#influx密码
influx.password=admin
influx.password=admin

37
src/main/java/cn/ac/iie/utils/general/TransFormUtils.java
View File

@@ -7,7 +7,9 @@ import cn.ac.iie.bean.security.SecurityPolicyLog;
import cn.ac.iie.common.FlowWriteConfig;
import cn.ac.iie.utils.hbase.HBaseUtils;
import cn.ac.iie.utils.system.SnowflakeId;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.net.InternetDomainName;
import com.zdjizhi.utils.IpLookup;
import com.zdjizhi.utils.StringUtil;
import org.apache.log4j.Logger;
@@ -28,10 +30,9 @@ import java.util.regex.Pattern;
public class TransFormUtils {
private static Logger logger = Logger.getLogger(TransFormUtils.class);
private final static Set<String> PUBLIC_SUFFIX_SET = new HashSet<String>(
Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se"
.split("\\|")));
private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})");
// private final static Set<String> PUBLIC_SUFFIX_SET = new HashSet<String>(
// Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se"
// .split("\\|")));
private static IpLookup ipLookup = new IpLookup.Builder(false)
.loadDataFileV4(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb")
.loadDataFileV6(FlowWriteConfig.IP_LIBRARY + "Kazakhstan.mmdb")
@@ -58,9 +59,10 @@ public class TransFormUtils {
radiusSessionRecordLog.setCommon_client_location(ipLookup.cityLookupDetail(clientIp));
radiusSessionRecordLog.setCommon_client_asn(ipLookup.asnLookup(clientIp, true));
radiusSessionRecordLog.setCommon_server_asn(ipLookup.asnLookup(serverIp, true));
radiusSessionRecordLog.setCommon_subscriber_id(radiusSessionRecordLog.getRadius_account());
return JSONObject.toJSONString(radiusSessionRecordLog);
} catch (Exception e) {
logger.error("{} 日志解析过程出现异常" + FlowWriteConfig.KAFKA_TOPIC);
logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常");
e.printStackTrace();
return "";
}
@@ -149,7 +151,8 @@ public class TransFormUtils {
collectProtocolRecordLog.setHttp_domain(getTopDomain(collectProtocolRecordLog.getSsl_sni(), collectProtocolRecordLog.getHttp_host()));
return JSONObject.toJSONString(collectProtocolRecordLog);
} catch (Exception e) {
logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常", e);
logger.error(FlowWriteConfig.KAFKA_TOPIC + "日志解析过程出现异常");
e.printStackTrace();
return "";
}
}
@@ -179,23 +182,13 @@ public class TransFormUtils {
* @return 顶级域名
*/
private static String getDomainName(String host) {
if (host.endsWith(FlowWriteConfig.DOMAIN_SPLITTER)) {
host = host.substring(0, host.length() - 1);
String domain = "";
try {
domain = InternetDomainName.from(host).topPrivateDomain().toString();
} catch (Exception e) {
logger.error("host解析顶级域名异常: " + e.getMessage());
}
if (IP_PATTERN.matcher(host).matches()) {
return host;
}
int index = 0;
String candidate = host;
for (; index >= 0; ) {
index = candidate.indexOf(FlowWriteConfig.DOMAIN_SPLITTER);
String subCandidate = candidate.substring(index + 1);
if (PUBLIC_SUFFIX_SET.contains(subCandidate)) {
return candidate;
}
candidate = subCandidate;
}
return candidate;
return domain;
}

3
src/main/java/cn/ac/iie/utils/hbase/HBaseUtils.java
View File

@@ -26,7 +26,7 @@ import java.util.concurrent.ConcurrentSkipListMap;
public class HBaseUtils {
private final static Logger logger = Logger.getLogger(HBaseUtils.class);
private static Map<String, String> subIdMap = new HashMap<>(333334);
// private static Map<String, String> subIdMap = new ConcurrentSkipListMap<>();
// private static Map<String, String> subIdMap = new ConcurrentSkipListMap<>();
private static Connection connection;
private static Long time;
@@ -137,4 +137,5 @@ public class HBaseUtils {
public static String getAccount(String clientIp) {
return subIdMap.get(clientIp);
}
}

2
src/main/java/cn/ac/iie/utils/kafka/KafkaLogNtc.java
View File

@@ -72,7 +72,7 @@ public class KafkaLogNtc {
properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
properties.put("acks", "1");
properties.put("linger.ms", "2");
properties.put("request.timeout.ms", 60000);
properties.put("request.timeout.ms", 30000);
properties.put("batch.size", 262144);
properties.put("buffer.memory", 33554432);
// properties.put("compression.type", "snappy");

87
src/test/java/cn/ac/iie/test/DomainUtils.java
View File

@@ -1,67 +1,50 @@
package cn.ac.iie.test;
import com.google.common.net.InternetDomainName;
import com.zdjizhi.utils.StringUtil;
import javax.xml.bind.SchemaOutputResolver;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class DomainUtils {
private static Pattern pattern = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.me|\\.cc|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)");
private final static Set<String> PublicSuffixSet = new HashSet<String>(
Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se"
Arrays.asList("com|edu|gov|int|mil|net|org|biz|info|pro|name|museum|coop|aero|xxx|idv|top|xyz|xin|vip|win|red|wang|co|mobi|travel|club|post|rec|asia"
.split("\\|")));
private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})");
public static void main(String[] args) {
System.out.println(getTopDomain("http://www.ccb.com"));
System.out.println(getDomainName("www.comaa.com"));
// InternetDomainName.from("foo.co.uk").topPrivateDomain().toString();
String host = "www.aaa.co.uk";
// if (host.contains(":")){
// String s = host.split(":")[0];
// System.out.println(InternetDomainName.from(s));
// System.out.println(InternetDomainName.from(s).topPrivateDomain());
// }else {
System.out.println(InternetDomainName.from(host).topDomainUnderRegistrySuffix());
System.out.println(InternetDomainName.from(host).topPrivateDomain());
// }
// System.out.println(InternetDomainName.from("shence.hupu.com").topPrivateDomain());
}
private static String getTopDomain(String url) {
try {
// 获取值转换为小写
String host = new URL(url).getHost().toLowerCase();//news.hexun.com
// Pattern pattern = Pattern.compile("[^\\\\.]+(\\.com\\.cn|\\.net\\.cn|\\.org\\.cn|\\.gov\\.cn|\\.com|\\.net|\\.cn|\\.org|\\.cc|\\.me|\\.tel|\\.mobi|\\.asia|\\.biz|\\.info|\\.name|\\.tv|\\.hk|\\.公司|\\.中国|\\.网络)");
Matcher matcher = IP_PATTERN.matcher(host);
if (matcher.find()){
return matcher.group();
}
} catch (MalformedURLException e) {
e.printStackTrace();
}
return null;
}
// private final static Set<String> PublicSuffixSet = new HashSet<String>(
// Arrays.asList("com|org|net|gov|edu|co|tv|mobi|info|asia|xxx|onion|cc|cn|com.cn|edu.cn|gov.cn|net.cn|org.cn|jp|kr|tw|com.hk|hk|com.hk|org.hk|se|com.se|org.se"
// .split("\\|")));
// private static Pattern IP_PATTERN = Pattern.compile("(\\d{1,3}\\.){3}(\\d{1,3})");
/**
* 获取url的顶级域名
// * @param url
* // * @param url
*
* @return
*/
public static String getDomainName(String host) {
// String host = url.getHost();
if (host.endsWith(".")){
if (host.endsWith(".") || host.contains("/")) {
host = host.substring(0, host.length() - 1);
}
if (IP_PATTERN.matcher(host).matches()){
return host;
}
int index = 0;
String candidate = host;
for (; index >= 0;) {
for (; index >= 0; ) {
index = candidate.indexOf('.');
String subCandidate = candidate.substring(index + 1);
if (PublicSuffixSet.contains(subCandidate)) {
@@ -73,4 +56,38 @@ public class DomainUtils {
}
public static String getSourceDomain(String host) {
if (host.endsWith(".")) {
host = host.substring(0, host.length() - 1);
}
String[] hostStr = host.split("\\.");
int length = hostStr.length;
if (hostStr.length >= 2) {
if (PublicSuffixSet.contains(hostStr[length - 2])) {
return hostStr[length - 3] + "." + hostStr[length - 2] + "." + hostStr[length - 1];
} else {
return hostStr[length - 2] + "." + hostStr[length - 1];
}
} else {
return host;
}
}
// 定义正则表达式，域名的根需要自定义，这里不全
private static final String RE_TOP = "[\\w-]+\\.(com.cn|net.cn|gov.cn|org\\.nz|org.cn|com|net|org|gov|cc|biz|info|cn|co)\\b()*";
private static Pattern pattern = Pattern.compile(RE_TOP, Pattern.CASE_INSENSITIVE);
public static String getTopDomain(String url) {
String result = url;
try {
Matcher matcher = pattern.matcher(url);
matcher.find();
result = matcher.group();
} catch (Exception e) {
System.out.println("[getTopDomain ERROR]====>");
e.printStackTrace();
}
return result;
}
}

2
src/test/java/cn/ac/iie/test/hbase/HBaseTest.java
View File

@@ -32,7 +32,7 @@ public class HBaseTest {
// 设置zookeeper节点
// configuration.set("hbase.zookeeper.quorum", FlowWriteConfig.HBASE_ZOOKEEPER_SERVERS);
// configuration.set("hbase.zookeeper.quorum", "192.168.40.119:2181,192.168.40.122:2181,192.168.40.123:2181");
configuration.set("hbase.zookeeper.quorum", "192.168.40.224:2182");
configuration.set("hbase.zookeeper.quorum", "192.168.40.186:2182");
configuration.set("hbase.client.retries.number", "3");
configuration.set("hbase.bulkload.retries.number", "3");
configuration.set("zookeeper.recovery.retry", "3");

16
src/test/java/cn/ac/iie/test/test.java
View File

@@ -1,16 +1,22 @@
package cn.ac.iie.test;
import cn.ac.iie.test.bean.Student;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.net.InternetDomainName;
import org.apache.log4j.Logger;
public class test {
private static Logger logger = Logger.getLogger(test.class);
public static void main(String[] args) {
String message = "{\"name\":\"aaa\"}";
Student student = JSONObject.parseObject(message, Student.class);
System.out.println(JSONObject.toJSONString(student));
}
String message = "{\"common_schema_type\":\"SSL\",\"ssl_sni\":\"www.myssl.cn\",\"common_server_ip\":\"101.37.81.250\",\"common_client_ip\":\"192.168.50.12\",\"common_server_port\":443,\"common_client_port\":51398,\"common_stream_dir\":3,\"common_address_type\":4,\"common_s2c_pkt_num\":1,\"common_s2c_byte_num\":0,\"common_c2s_pkt_num\":3,\"common_c2s_byte_num\":517,\"common_stream_trace_id\":7629423224647,\"common_l4_protocol\":\"IPv4_TCP\",\"common_address_list\":\"51398-443-192.168.50.12-101.37.81.250\",\"common_sled_ip\":\"192.168.40.161\",\"common_start_time\":1575008799,\"common_end_time\":1575008799,\"common_con_duration_ms\":0,\"common_policy_id\":282,\"common_service\":0,\"common_action\":128,\"common_user_region\":\"{\\\"protocol\\\":\\\"SSL\\\"}\"}";
JSONObject jsonObject = JSON.parseObject(message);
String common_server_ip = jsonObject.getString("common_server_ip");
String common_client_ip = jsonObject.getString("common_client_ip");
jsonObject.put("common__recv_time",(System.currentTimeMillis() / 1000));
System.out.println(jsonObject.toString());
}
}