gfwleak/galaxy-tsg-olap-dos-detection-job
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增读取bifang静态阈值配置接口

Browse Source 
修改galaxy工具类库版本
This commit is contained in:
wanglihui
2021-08-18 19:15:49 +08:00
parent c957f3ec1c
commit f744677021
10 changed files with 303 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@@ -218,7 +218,7 @@
<dependency> <dependency>
<groupId>com.zdjizhi</groupId> <groupId>com.zdjizhi</groupId>
<artifactId>galaxy</artifactId> <artifactId>galaxy</artifactId>
<version>1.0.4</version> <version>1.0.6</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<artifactId>slf4j-log4j12</artifactId> <artifactId>slf4j-log4j12</artifactId>

93
src/main/java/com/zdjizhi/common/DosDetectionThreshold.java Normal file
View File

@@ -0,0 +1,93 @@
package com.zdjizhi.common;
import java.io.Serializable;
import java.util.ArrayList;
public class DosDetectionThreshold implements Serializable {
private String profileId;
private String attackType;
private ArrayList<String> serverIpList;
private String serverIpAddr;
private long packetsPerSec;
private long bitsPerSec;
private long sessionsPerSec;
private int isValid;
@Override
public String toString() {
return "DosDetectionThreshold{" +
"profileId='" + profileId + '\'' +
", attackType='" + attackType + '\'' +
", serverIpList=" + serverIpList +
", serverIpAddr='" + serverIpAddr + '\'' +
", packetsPerSec=" + packetsPerSec +
", bitsPerSec=" + bitsPerSec +
", sessionsPerSec=" + sessionsPerSec +
", isValid=" + isValid +
'}';
}
public String getProfileId() {
return profileId;
}
public void setProfileId(String profileId) {
this.profileId = profileId;
}
public String getAttackType() {
return attackType;
}
public void setAttackType(String attackType) {
this.attackType = attackType;
}
public ArrayList<String> getServerIpList() {
return serverIpList;
}
public void setServerIpList(ArrayList<String> serverIpList) {
this.serverIpList = serverIpList;
}
public String getServerIpAddr() {
return serverIpAddr;
}
public void setServerIpAddr(String serverIpAddr) {
this.serverIpAddr = serverIpAddr;
}
public long getPacketsPerSec() {
return packetsPerSec;
}
public void setPacketsPerSec(long packetsPerSec) {
this.packetsPerSec = packetsPerSec;
}
public long getBitsPerSec() {
return bitsPerSec;
}
public void setBitsPerSec(long bitsPerSec) {
this.bitsPerSec = bitsPerSec;
}
public long getSessionsPerSec() {
return sessionsPerSec;
}
public void setSessionsPerSec(long sessionsPerSec) {
this.sessionsPerSec = sessionsPerSec;
}
public int getIsValid() {
return isValid;
}
public void setIsValid(int isValid) {
this.isValid = isValid;
}
}

101
src/main/java/com/zdjizhi/etl/DosDetection.java
View File

@@ -28,50 +28,47 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
private final static NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance(); private final static NumberFormat PERCENT_INSTANCE = NumberFormat.getPercentInstance();
@Override @Override
public void open(Configuration parameters){ public void open(Configuration parameters) {
baselineMap = HbaseUtils.baselineMap; baselineMap = HbaseUtils.baselineMap;
PERCENT_INSTANCE.setMinimumFractionDigits(2); PERCENT_INSTANCE.setMinimumFractionDigits(2);
} }
@Override @Override
public DosEventLog map(DosSketchLog value){ public DosEventLog map(DosSketchLog value) {
try { try {
String destinationIp = value.getDestination_ip(); String destinationIp = value.getDestination_ip();
String attackType = value.getAttack_type(); String attackType = value.getAttack_type();
logger.debug("当前判断IP：{}, 类型: {}",destinationIp,attackType); logger.debug("当前判断IP：{}, 类型: {}", destinationIp, attackType);
if (baselineMap.containsKey(destinationIp)){ if (baselineMap.containsKey(destinationIp)) {
Tuple2<ArrayList<Integer>, Integer> floodTypeTup = baselineMap.get(destinationIp).get(attackType); Tuple2<ArrayList<Integer>, Integer> floodTypeTup = baselineMap.get(destinationIp).get(attackType);
List<Integer> baselines = floodTypeTup.f0; Integer base = getBaseValue(floodTypeTup, value);
if (baselines != null && baselines.size() == BASELINE_SIZE){ long diff = value.getSketch_sessions() - base;
Integer base = getBaseValue(baselines,value,floodTypeTup.f1); if (diff > 0 && base != 0) {
long diff = value.getSketch_sessions() - base; String percent = getDiffPercent(diff, base);
if (diff > 0 && base != 0){ double diffPercentDouble = getDiffPercentDouble(percent);
String percent = getDiffPercent(diff, base); Severity severity = judgeSeverity(diffPercentDouble);
double diffPercentDouble = getDiffPercentDouble(percent); if (severity != Severity.NORMAL) {
Severity severity = judgeSeverity(diffPercentDouble); DosEventLog result = getResult(value, severity, percent);
if (severity != Severity.NORMAL){ logger.info("检测到当前server IP {} 存在 {} 异常,日志详情\n {}", destinationIp, attackType, result.toString());
DosEventLog result = getResult(value, severity, percent); return result;
logger.info("检测到当前server IP {} 存在 {} 异常,日志详情\n {}",destinationIp,attackType,result.toString()); } else {
return result; logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}", destinationIp, attackType, value.toString());
}else {
logger.debug("当前server IP：{} 未出现 {} 异常,日志详情 {}",destinationIp,attackType,value.toString());
}
} }
} }
}else { } else {
logger.debug("未获取到当前server IP：{} 类型 {} baseline数据",destinationIp,attackType); logger.debug("未获取到当前server IP：{} 类型 {} baseline数据", destinationIp, attackType);
} }
}catch (Exception e){ } catch (Exception e) {
logger.error("判定失败\n {} \n{}",value,e); logger.error("判定失败\n {} \n{}", value, e);
} }
return null; return null;
} }
private DosEventLog getResult(DosSketchLog value, Severity severity, String percent){ private DosEventLog getResult(DosSketchLog value, Severity severity, String percent) {
DosEventLog dosEventLog = new DosEventLog(); DosEventLog dosEventLog = new DosEventLog();
dosEventLog.setLog_id(SnowflakeId.generateId()); dosEventLog.setLog_id(SnowflakeId.generateId());
dosEventLog.setStart_time(value.getSketch_start_time()); dosEventLog.setStart_time(value.getSketch_start_time());
dosEventLog.setEnd_time(value.getSketch_start_time()+CommonConfig.FLINK_WINDOW_MAX_TIME); dosEventLog.setEnd_time(value.getSketch_start_time() + CommonConfig.FLINK_WINDOW_MAX_TIME);
dosEventLog.setAttack_type(value.getAttack_type()); dosEventLog.setAttack_type(value.getAttack_type());
dosEventLog.setSeverity(severity.toString()); dosEventLog.setSeverity(severity.toString());
dosEventLog.setConditions(getConditions(percent)); dosEventLog.setConditions(getConditions(percent));
@@ -86,41 +83,47 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return dosEventLog; return dosEventLog;
} }
private Integer getBaseValue(List<Integer> baselines,DosSketchLog value,int defauleVaule){ private Integer getBaseValue(Tuple2<ArrayList<Integer>, Integer> floodTypeTup, DosSketchLog value) {
Integer base = 0;
try { try {
int timeIndex = getCurrentTimeIndex(value.getSketch_start_time()); if (floodTypeTup != null){
Integer base = baselines.get(timeIndex); ArrayList<Integer> baselines = floodTypeTup.f0;
if (base == 0){ Integer defaultVaule = floodTypeTup.f1;
logger.debug("获取到当前IP: {},类型: {} baseline值为0,替换为P95观测值{}",value.getDestination_ip(),value.getAttack_type(),defauleVaule); if (baselines != null && baselines.size() == BASELINE_SIZE) {
base = defauleVaule; int timeIndex = getCurrentTimeIndex(value.getSketch_start_time());
base = baselines.get(timeIndex);
if (base == 0) {
logger.debug("获取到当前IP: {},类型: {} baseline值为0,替换为P95观测值{}", value.getDestination_ip(), value.getAttack_type(), defaultVaule);
base = defaultVaule;
}
}
} }
return base; } catch (Exception e) {
}catch (Exception e){ logger.error("解析baseline数据失败,返回默认值0", e);
logger.error("解析baseline数据失败,返回默认值0",e);
return 0;
} }
return base;
} }
private String getConditions(String percent){ private String getConditions(String percent) {
return "sessions > "+percent+" of baseline"; return "sessions > " + percent + " of baseline";
} }
private String getSourceCountryList(String sourceIpList){ private String getSourceCountryList(String sourceIpList) {
String[] ipArr = sourceIpList.split(","); String[] ipArr = sourceIpList.split(",");
HashSet<String> countrySet = new HashSet<>(); HashSet<String> countrySet = new HashSet<>();
for (String ip:ipArr){ for (String ip : ipArr) {
countrySet.add(IpUtils.ipLookup.countryLookup(ip)); countrySet.add(IpUtils.ipLookup.countryLookup(ip));
} }
return StringUtils.join(countrySet,","); return StringUtils.join(countrySet, ",");
} }
private int getCurrentTimeIndex(long sketchStartTime){ private int getCurrentTimeIndex(long sketchStartTime) {
long currentDayTime = sketchStartTime / (60 * 60 * 24) * 60 * 60 * 24; long currentDayTime = sketchStartTime / (60 * 60 * 24) * 60 * 60 * 24;
long indexLong = (sketchStartTime - currentDayTime) / 600; long indexLong = (sketchStartTime - currentDayTime) / 600;
return Integer.parseInt(Long.toString(indexLong)); return Integer.parseInt(Long.toString(indexLong));
} }
private String getDiffPercent(long diff,long base){ private String getDiffPercent(long diff, long base) {
double diffDou = Double.parseDouble(Long.toString(diff)); double diffDou = Double.parseDouble(Long.toString(diff));
double baseDou = Double.parseDouble(Long.toString(base)); double baseDou = Double.parseDouble(Long.toString(base));
return PERCENT_INSTANCE.format(diffDou / baseDou); return PERCENT_INSTANCE.format(diffDou / baseDou);
@@ -135,18 +138,18 @@ public class DosDetection extends RichMapFunction<DosSketchLog, DosEventLog> {
return PERCENT_INSTANCE.parse(diffPercent).doubleValue(); return PERCENT_INSTANCE.parse(diffPercent).doubleValue();
} }
private Severity judgeSeverity(double diffPercent){ private Severity judgeSeverity(double diffPercent) {
if (diffPercent >= CommonConfig.BASELINE_SESSIONS_MINOR_THRESHOLD && diffPercent < CommonConfig.BASELINE_SESSIONS_WARNING_THRESHOLD){ if (diffPercent >= CommonConfig.BASELINE_SESSIONS_MINOR_THRESHOLD && diffPercent < CommonConfig.BASELINE_SESSIONS_WARNING_THRESHOLD) {
return Severity.MINOR; return Severity.MINOR;
}else if (diffPercent >= CommonConfig.BASELINE_SESSIONS_WARNING_THRESHOLD && diffPercent < CommonConfig.BASELINE_SESSIONS_MAJOR_THRESHOLD){ } else if (diffPercent >= CommonConfig.BASELINE_SESSIONS_WARNING_THRESHOLD && diffPercent < CommonConfig.BASELINE_SESSIONS_MAJOR_THRESHOLD) {
return Severity.WARNING; return Severity.WARNING;
}else if (diffPercent >= CommonConfig.BASELINE_SESSIONS_MAJOR_THRESHOLD && diffPercent < CommonConfig.BASELINE_SESSIONS_SEVERE_THRESHOLD){ } else if (diffPercent >= CommonConfig.BASELINE_SESSIONS_MAJOR_THRESHOLD && diffPercent < CommonConfig.BASELINE_SESSIONS_SEVERE_THRESHOLD) {
return Severity.MAJOR; return Severity.MAJOR;
}else if (diffPercent >= CommonConfig.BASELINE_SESSIONS_SEVERE_THRESHOLD && diffPercent < CommonConfig.BASELINE_SESSIONS_CRITICAL_THRESHOLD){ } else if (diffPercent >= CommonConfig.BASELINE_SESSIONS_SEVERE_THRESHOLD && diffPercent < CommonConfig.BASELINE_SESSIONS_CRITICAL_THRESHOLD) {
return Severity.SEVERE; return Severity.SEVERE;
}else if (diffPercent >= CommonConfig.BASELINE_SESSIONS_CRITICAL_THRESHOLD){ } else if (diffPercent >= CommonConfig.BASELINE_SESSIONS_CRITICAL_THRESHOLD) {
return Severity.CRITICAL; return Severity.CRITICAL;
}else { } else {
return Severity.NORMAL; return Severity.NORMAL;
} }
} }

126
src/main/java/com/zdjizhi/etl/ParseStaticThreshold.java Normal file
View File

@@ -0,0 +1,126 @@
package com.zdjizhi.etl;
import com.fasterxml.jackson.databind.JavaType;
import com.zdjizhi.common.CommonConfig;
import com.zdjizhi.common.DosDetectionThreshold;
import com.zdjizhi.utils.HttpClientUtils;
import com.zdjizhi.utils.JsonMapper;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.message.BasicHeader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
/**
* @author wlh
*/
public class ParseStaticThreshold {
private static Logger logger = LoggerFactory.getLogger(ParseStaticThreshold.class);
private static String encryptpwd;
private static JsonMapper jsonMapperInstance = JsonMapper.getInstance();
private static JavaType hashmapJsonType = jsonMapperInstance.createCollectionType(HashMap.class, String.class, Object.class);
private static JavaType thresholdType = jsonMapperInstance.createCollectionType(ArrayList.class, DosDetectionThreshold.class);
static {
//加载加密登录密码
encryptpwd = getEncryptpwd();
}
/**
* 获取加密密码
*/
private static String getEncryptpwd(){
String psw = HttpClientUtils.ERROR_MESSAGE;
try {
URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
HashMap<String, String> parms = new HashMap<>();
parms.put("password",CommonConfig.BIFANG_SERVER_PASSWORD);
HttpClientUtils.setUrlWithParams(uriBuilder,CommonConfig.BIFANG_SERVER_ENCRYPTPWD_PATH,parms);
String resposeJsonStr = HttpClientUtils.httpGet(uriBuilder.build());
if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)){
HashMap<String, Object> resposeMap = jsonMapperInstance.fromJson(resposeJsonStr, hashmapJsonType);
boolean success = (boolean)resposeMap.get("success");
if (success){
HashMap<String, Object> data = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(resposeMap.get("data")), hashmapJsonType);
psw = data.get("encryptpwd").toString();
}
}
}catch (URISyntaxException e){
logger.error("构造URI异常",e);
}catch (Exception e){
logger.error("获取encryptpwd失败",e);
}
return psw;
}
/**
* 登录bifang服务，获取token
* @return token
*/
private static String loginBifangServer(){
String token = HttpClientUtils.ERROR_MESSAGE;
try {
if (!HttpClientUtils.ERROR_MESSAGE.equals(encryptpwd)){
URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
HashMap<String, String> parms = new HashMap<>();
parms.put("username",CommonConfig.BIFANG_SERVER_USER);
parms.put("password",encryptpwd);
HttpClientUtils.setUrlWithParams(uriBuilder,CommonConfig.BIFANG_SERVER_LOGIN_PATH,parms);
String resposeJsonStr = HttpClientUtils.httpPost(uriBuilder.build(), null);
if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)){
HashMap<String, Object> resposeMap = jsonMapperInstance.fromJson(resposeJsonStr, hashmapJsonType);
boolean success = (boolean)resposeMap.get("success");
if (success){
HashMap<String, Object> data = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(resposeMap.get("data")), hashmapJsonType);
token = data.get("token").toString();
}
}
}
}catch (Exception e){
logger.error("登录失败,未获取到token",e);
}
return token;
}
/**
* 获取静态阈值配置列表
* @return thresholds
*/
private static ArrayList<DosDetectionThreshold> getDosDetectionThreshold(){
ArrayList<DosDetectionThreshold> thresholds = null;
try {
URIBuilder uriBuilder = new URIBuilder(CommonConfig.BIFANG_SERVER_URI);
HttpClientUtils.setUrlWithParams(uriBuilder,CommonConfig.BIFANG_SERVER_POLICY_THRESHOLD_PATH,null);
String token = loginBifangServer();
if (!HttpClientUtils.ERROR_MESSAGE.equals(token)){
BasicHeader authorization = new BasicHeader("Authorization", token);
String resposeJsonStr = HttpClientUtils.httpGet(uriBuilder.build(), authorization);
if (!HttpClientUtils.ERROR_MESSAGE.equals(resposeJsonStr)){
HashMap<String, Object> resposeMap = jsonMapperInstance.fromJson(resposeJsonStr, hashmapJsonType);
boolean success = (boolean)resposeMap.get("success");
if (success){
HashMap<String, Object> data = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(resposeMap.get("data")), hashmapJsonType);
thresholds = jsonMapperInstance.fromJson(jsonMapperInstance.toJson(data.get("list")), thresholdType);
logger.info("获取到静态阈值配置{}条",thresholds.size());
}
}
}
}catch (Exception e){
logger.error("获取静态阈值配置失败",e);
}
return thresholds;
}
public static void main(String[] args) {
ArrayList<DosDetectionThreshold> dosDetectionThreshold = getDosDetectionThreshold();
if (dosDetectionThreshold != null && !dosDetectionThreshold.isEmpty()){
dosDetectionThreshold.forEach(s -> System.out.println(s.toString()));
}
}
}

9
src/main/java/com/zdjizhi/sink/OutputStreamSink.java
View File

@@ -28,8 +28,7 @@ public class OutputStreamSink {
public static void finalOutputSink(){ public static void finalOutputSink(){
try { try {
SingleOutputStreamOperator<DosSketchLog> middleStream = getMiddleStream(); SingleOutputStreamOperator<DosSketchLog> middleStream = getMiddleStream();
SingleOutputStreamOperator<DosEventLog> dosEventLogOutputStream = getOutputSinkStream(middleStream); DosEventSink.dosEventOutputSink(getEventSinkStream(middleStream));
DosEventSink.dosEventOutputSink(dosEventLogOutputStream);
TrafficServerIpMetricsSink.sideOutputMetricsSink(middleStream); TrafficServerIpMetricsSink.sideOutputMetricsSink(middleStream);
FlinkEnvironmentUtils.streamExeEnv.execute(CommonConfig.STREAM_EXECUTION_JOB_NAME); FlinkEnvironmentUtils.streamExeEnv.execute(CommonConfig.STREAM_EXECUTION_JOB_NAME);
} catch (Exception e) { } catch (Exception e) {
@@ -37,19 +36,19 @@ public class OutputStreamSink {
} }
} }
private static SingleOutputStreamOperator<DosEventLog> getOutputSinkStream(SingleOutputStreamOperator<DosSketchLog> middleStream){ private static SingleOutputStreamOperator<DosEventLog> getEventSinkStream(SingleOutputStreamOperator<DosSketchLog> middleStream){
return middleStream.map(new DosDetection()).setParallelism(CommonConfig.FLINK_DETECTION_MAP_PARALLELISM); return middleStream.map(new DosDetection()).setParallelism(CommonConfig.FLINK_DETECTION_MAP_PARALLELISM);
} }
private static SingleOutputStreamOperator<DosSketchLog> getMiddleStream(){ private static SingleOutputStreamOperator<DosSketchLog> getMiddleStream(){
return ParseSketchLog.getSketchSource() return ParseSketchLog.getSketchSource()
.keyBy(new FirstKeySelector()) .keyBy(new KeysSelector())
.window(TumblingEventTimeWindows.of(Time.seconds(CommonConfig.FLINK_WINDOW_MAX_TIME))) .window(TumblingEventTimeWindows.of(Time.seconds(CommonConfig.FLINK_WINDOW_MAX_TIME)))
.process(new EtlProcessFunction()) .process(new EtlProcessFunction())
.setParallelism(CommonConfig.FLINK_FIRST_AGG_PARALLELISM); .setParallelism(CommonConfig.FLINK_FIRST_AGG_PARALLELISM);
} }
private static class FirstKeySelector implements KeySelector<DosSketchLog, Tuple2<String, String>>{ private static class KeysSelector implements KeySelector<DosSketchLog, Tuple2<String, String>>{
@Override @Override
public Tuple2<String, String> getKey(DosSketchLog dosSketchLog){ public Tuple2<String, String> getKey(DosSketchLog dosSketchLog){
return Tuple2.of( return Tuple2.of(

13
src/main/java/com/zdjizhi/utils/HbaseUtils.java
View File

@@ -56,7 +56,12 @@ public class HbaseUtils {
public static void main(String[] args) { public static void main(String[] args) {
Set<String> keySet = baselineMap.keySet(); Set<String> keySet = baselineMap.keySet();
for (String key : keySet) { for (String key : keySet) {
Map<String, Tuple2<ArrayList<Integer>, Integer>> stringTuple2Map = baselineMap.get(key);
Set<String> strings = stringTuple2Map.keySet();
for (String s:strings){
Tuple2<ArrayList<Integer>, Integer> arrayListIntegerTuple2 = stringTuple2Map.get(s);
System.out.println(key+"---"+s+"---"+arrayListIntegerTuple2.f0+"---"+arrayListIntegerTuple2.f1);
}
} }
System.out.println(baselineMap.size()); System.out.println(baselineMap.size());
} }
@@ -71,8 +76,10 @@ public class HbaseUtils {
String rowkey = Bytes.toString(result.getRow()); String rowkey = Bytes.toString(result.getRow());
for (String type:floodTypeList){ for (String type:floodTypeList){
ArrayList<Integer> sessionRate = getArraylist(result, type, "session_rate"); ArrayList<Integer> sessionRate = getArraylist(result, type, "session_rate");
Integer defaultValue = getDefaultValue(result, type, "session_rate_default_value"); if (sessionRate != null && !sessionRate.isEmpty()){
floodTypeMap.put(type,Tuple2.of(sessionRate, defaultValue)); Integer defaultValue = getDefaultValue(result, type, "session_rate_default_value");
floodTypeMap.put(type,Tuple2.of(sessionRate, defaultValue));
}
} }
baselineMap.put(rowkey, floodTypeMap); baselineMap.put(rowkey, floodTypeMap);
} }

42
src/main/java/com/zdjizhi/utils/HttpClientUtils.java
View File

@@ -26,14 +26,11 @@ import javax.net.ssl.SSLHandshakeException;
import java.io.IOException; import java.io.IOException;
import java.io.InterruptedIOException; import java.io.InterruptedIOException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Map; import java.util.Map;
/** /**
* @author wlh
* http client工具类 * http client工具类
*/ */
public class HttpClientUtils { public class HttpClientUtils {
@@ -41,6 +38,7 @@ public class HttpClientUtils {
private static final PoolingHttpClientConnectionManager CONN_MANAGER = new PoolingHttpClientConnectionManager(); private static final PoolingHttpClientConnectionManager CONN_MANAGER = new PoolingHttpClientConnectionManager();
private static Logger logger = LoggerFactory.getLogger(HttpClientUtils.class); private static Logger logger = LoggerFactory.getLogger(HttpClientUtils.class);
public static final String ERROR_MESSAGE = "-1";
/* /*
* 静态代码块配置连接池信息 * 静态代码块配置连接池信息
@@ -136,21 +134,17 @@ public class HttpClientUtils {
/** /**
* GET请求 * GET请求
* *
* @param url 请求地 * @param uri 请求地
* @return message * @return message
*/ */
public static String httpGet(String url, Header... headers) { public static String httpGet(URI uri, Header... headers) {
String msg = "-1"; String msg = ERROR_MESSAGE;
// 获取客户端连接对象 // 获取客户端连接对象
CloseableHttpClient httpClient = getHttpClient(); CloseableHttpClient httpClient = getHttpClient();
CloseableHttpResponse response = null; CloseableHttpResponse response = null;
try { try {
URL ul = new URL(url);
URI uri = new URI(ul.getProtocol(),null, ul.getHost(), ul.getPort(), ul.getPath(), ul.getQuery(), null);
logger.info("http get uri {}",uri); logger.info("http get uri {}",uri);
// 创建GET请求对象 // 创建GET请求对象
HttpGet httpGet = new HttpGet(uri); HttpGet httpGet = new HttpGet(uri);
@@ -173,8 +167,6 @@ public class HttpClientUtils {
logger.error("Http get content is :{}" , msg); logger.error("Http get content is :{}" , msg);
} }
} catch (URISyntaxException e) {
logger.error("URI 转换错误: {}", e.getMessage());
} catch (ClientProtocolException e) { } catch (ClientProtocolException e) {
logger.error("协议错误: {}", e.getMessage()); logger.error("协议错误: {}", e.getMessage());
} catch (ParseException e) { } catch (ParseException e) {
@@ -197,27 +189,23 @@ public class HttpClientUtils {
} }
/** /**
* POST 请求 * POST 请求
* @param url url参数 * @param uri uri参数
* @param requestBody 请求体 * @param requestBody 请求体
* @return post请求返回结果 * @return post请求返回结果
*/ */
public static String httpPost(String url, String requestBody, Header... headers) { public static String httpPost(URI uri, String requestBody, Header... headers) {
String msg = "-1"; String msg = ERROR_MESSAGE;
// 获取客户端连接对象 // 获取客户端连接对象
CloseableHttpClient httpClient = getHttpClient(); CloseableHttpClient httpClient = getHttpClient();
// 创建POST请求对象 // 创建POST请求对象
CloseableHttpResponse response = null; CloseableHttpResponse response = null;
try { try {
URL ul = new URL(url);
URI uri = new URI(ul.getProtocol(),null, ul.getHost(), ul.getPort(), ul.getPath(), ul.getQuery(), null);
logger.info("http post uri:{}， http post body:{}", uri, requestBody); logger.info("http post uri:{}， http post body:{}", uri, requestBody);
HttpPost httpPost = new HttpPost(uri); HttpPost httpPost = new HttpPost(uri);
httpPost.setHeader("Content-Type", "application/json"); httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded");
if (StringUtil.isNotEmpty(headers)) { if (StringUtil.isNotEmpty(headers)) {
for (Header h : headers) { for (Header h : headers) {
httpPost.addHeader(h); httpPost.addHeader(h);
@@ -240,9 +228,7 @@ public class HttpClientUtils {
if (statusCode != HttpStatus.SC_OK) { if (statusCode != HttpStatus.SC_OK) {
logger.error("Http post content is :{}" , msg); logger.error("Http post content is :{}" , msg);
} }
} catch (URISyntaxException e) { } catch (ClientProtocolException e) {
logger.error("URI 转换错误: {}", e.getMessage());
} catch (ClientProtocolException e) {
logger.error("协议错误: {}", e.getMessage()); logger.error("协议错误: {}", e.getMessage());
} catch (ParseException e) { } catch (ParseException e) {
logger.error("解析错误: {}", e.getMessage()); logger.error("解析错误: {}", e.getMessage());
@@ -269,16 +255,14 @@ public class HttpClientUtils {
public static void setUrlWithParams(URIBuilder uriBuilder,String path, Map<String, String> params) { public static void setUrlWithParams(URIBuilder uriBuilder,String path, Map<String, String> params) {
try { try {
uriBuilder.setPath(path); uriBuilder.setPath(path);
for (Map.Entry<String, String> kv : params.entrySet()) { if (params != null && !params.isEmpty()){
uriBuilder.setParameter(kv.getKey(),kv.getValue()); for (Map.Entry<String, String> kv : params.entrySet()) {
uriBuilder.setParameter(kv.getKey(),kv.getValue());
}
} }
} catch (Exception e) { } catch (Exception e) {
logger.error("拼接url出错,uri : {}, path : {},参数: {}",uriBuilder.toString(),path,params); logger.error("拼接url出错,uri : {}, path : {},参数: {}",uriBuilder.toString(),path,params);
} }
} }
public static void getEncryptpwd(){
}
} }

6
src/main/java/com/zdjizhi/utils/IpUtils.java
View File

@@ -8,14 +8,14 @@ public class IpUtils {
* IP定位库工具类 * IP定位库工具类
*/ */
public static IpLookup ipLookup = new IpLookup.Builder(false) public static IpLookup ipLookup = new IpLookup.Builder(false)
// .loadDataFileV4(CommonConfig.IP_MMDB_PATH + "ip_v4.mmdb") .loadDataFileV4(CommonConfig.IP_MMDB_PATH + "ip_v4.mmdb")
// .loadDataFileV6(CommonConfig.IP_MMDB_PATH + "ip_v6.mmdb") // .loadDataFileV6(CommonConfig.IP_MMDB_PATH + "ip_v6.mmdb")
.loadDataFilePrivateV4(CommonConfig.IP_MMDB_PATH + "ip_private_v4.mmdb") // .loadDataFilePrivateV4(CommonConfig.IP_MMDB_PATH + "ip_private_v4.mmdb")
// .loadDataFilePrivateV6(CommonConfig.IP_MMDB_PATH + "ip_private_v6.mmdb") // .loadDataFilePrivateV6(CommonConfig.IP_MMDB_PATH + "ip_private_v6.mmdb")
.build(); .build();
public static void main(String[] args) { public static void main(String[] args) {
System.out.println(ipLookup.countryLookup("94.23.23.52")); System.out.println(ipLookup.countryLookup("49.7.115.37"));
// String ips = "192.168.50.23,192.168.50.45,192.168.56.9,192.168.56.8,192.168.50.58,192.168.56.7,192.168.56.6,192.168.50.40,192.168.50.19,192.168.50.6,192.168.50.4,192.168.56.17,192.168.50.27,192.168.50.26,192.168.50.18,192.168.56.3,192.168.56.10"; // String ips = "192.168.50.23,192.168.50.45,192.168.56.9,192.168.56.8,192.168.50.58,192.168.56.7,192.168.56.6,192.168.50.40,192.168.50.19,192.168.50.6,192.168.50.4,192.168.56.17,192.168.50.27,192.168.50.26,192.168.50.18,192.168.56.3,192.168.56.10";
// for (String ip:ips.split(",")){ // for (String ip:ips.split(",")){

3
src/main/resources/common.properties
View File

@@ -71,8 +71,7 @@ destination.ip.partition.num=10000
data.center.id.num=15 data.center.id.num=15
#IP mmdb库路径 #IP mmdb库路径
#ip.mmdb.path=D:\\data\\dat_test\\ ip.mmdb.path=D:\\data\\dat\\
ip.mmdb.path=D:\\data\\
#ip.mmdb.path=/home/bigdata/topology/dat/ #ip.mmdb.path=/home/bigdata/topology/dat/
#ip.mmdb.path=/home/bigdata/wlh/topology/dos-detection/dat/ #ip.mmdb.path=/home/bigdata/wlh/topology/dos-detection/dat/

48
src/test/java/com/zdjizhi/common/HttpTest.java
View File

@@ -1,55 +1,7 @@
package com.zdjizhi.common; package com.zdjizhi.common;
import com.zdjizhi.utils.HttpClientUtils;
import com.zdjizhi.utils.JsonMapper;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
public class HttpTest { public class HttpTest {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
CloseableHttpClient httpclient = HttpClients.custom().build();
URIBuilder uriBuilder = new URIBuilder("http://192.168.44.3:80");
HashMap<String, String> parms = new HashMap<>();
parms.put("password",CommonConfig.BIFANG_SERVER_PASSWORD);
HttpClientUtils.setUrlWithParams(uriBuilder,CommonConfig.BIFANG_SERVER_ENCRYPTPWD_PATH,parms);
System.out.println(uriBuilder.toString());
URI uri = uriBuilder.build();
System.out.println(HttpClientUtils.httpGet(uri.toString()));
/*
URI uri = uriBuilder
.setPath("/v1/user/encryptpwd")
.setParameter("password", "admin").build();
System.out.println(uri.toString());
HttpGet httpGet = new HttpGet(uri);
CloseableHttpResponse response = null;
try {
// 执行http get请求
response = httpclient.execute(httpGet);
// 判断返回状态是否为200
if (response.getStatusLine().getStatusCode() == 200) {
String content = EntityUtils.toString(response.getEntity(), "UTF-8");
System.out.println(content);
HashMap<String, Object> map = (HashMap<String, Object>) JsonMapper.fromJsonString(content, Object.class);
}
} finally {
if (response != null) {
response.close();
}
httpclient.close();
}
*/
} }
} }