diff --git a/pom.xml b/pom.xml
index 7d84db1..d894412 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
com.zdjizhi
p19-file-sync-service
- 21.12.01
+ 23.03.09
p19-file-sync-service
@@ -20,14 +20,14 @@
nexus
Team Nexus Repository
- http://192.168.40.125:8099/content/groups/public/
+ http://192.168.40.153:8099/content/groups/public/
nexus
Team Nexus Repository
- http://192.168.40.125:8099/content/groups/public/
+ http://192.168.40.153:8099/content/groups/public/
@@ -238,7 +238,7 @@
docker
- 192.168.40.153:9080/common/jdk:1.8.0_73-jre
+ 192.168.40.153:9080/common/jdk:1.8.0_202
192.168.40.153:9080/common/golang:1.15.6
${project.build.finalName}.xjar
diff --git a/src/main/java/com/zdjizhi/syncfile/config/HttpClientPool.java b/src/main/java/com/zdjizhi/syncfile/config/HttpClientPool.java
index e9f47c5..2903d63 100644
--- a/src/main/java/com/zdjizhi/syncfile/config/HttpClientPool.java
+++ b/src/main/java/com/zdjizhi/syncfile/config/HttpClientPool.java
@@ -1,20 +1,35 @@
package com.zdjizhi.syncfile.config;
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
import com.alibaba.nacos.api.config.ConfigType;
import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties;
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpRequest;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.protocol.HttpContext;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
+import javax.net.ssl.*;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.ConnectException;
+import java.net.UnknownHostException;
@Component
//@ConfigurationProperties(prefix = "http")
@NacosConfigurationProperties(prefix = "http", dataId = "${nacos.config.data-id}", groupId = "${nacos.config.group}", type = ConfigType.YAML, autoRefreshed = true)
public class HttpClientPool {
+ private Log log = LogFactory.get();
private Integer maxTotal;
@@ -28,6 +43,7 @@ public class HttpClientPool {
private boolean staleConnectionCheckEnabled;
+ private Integer retryNum;
public void setMaxTotal(Integer maxTotal) {
this.maxTotal = maxTotal;
@@ -53,6 +69,14 @@ public class HttpClientPool {
this.staleConnectionCheckEnabled = staleConnectionCheckEnabled;
}
+ public Integer getRetryNum() {
+ return retryNum;
+ }
+
+ public void setRetryNum(Integer retryNum) {
+ this.retryNum = retryNum;
+ }
+
/**
* 首先实例化一个连接池管理器,设置最大连接数、并发连接数
*
@@ -90,8 +114,10 @@ public class HttpClientPool {
* @return
*/
@Bean
- public CloseableHttpClient getCloseableHttpClient(@Qualifier("httpClientBuilder") HttpClientBuilder httpClientBuilder) {
- return httpClientBuilder.build();
+ public CloseableHttpClient getCloseableHttpClient(@Qualifier("httpClientBuilder") HttpClientBuilder httpClientBuilder,@Qualifier("httpRetryHandler") HttpRequestRetryHandler httpRetryHandler){
+ return httpClientBuilder
+ .setRetryHandler(httpRetryHandler)
+ .build();
}
/**
@@ -109,7 +135,6 @@ public class HttpClientPool {
.setConnectionRequestTimeout(connectionRequestTimeout)
.setSocketTimeout(socketTimeout)
.setStaleConnectionCheckEnabled(staleConnectionCheckEnabled);
-
}
/**
@@ -120,6 +145,41 @@ public class HttpClientPool {
return builder.build();
}
-
-}
-
+ @Bean(name = "httpRetryHandler")
+ public HttpRequestRetryHandler getHttpRetryHandler() {
+ return new HttpRequestRetryHandler() {
+ @Override
+ public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
+ if (executionCount >= retryNum) {// 如果已经重试了3次,就放弃
+ log.error("已完成重试次数");
+ return false;
+ }
+ if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试
+ return true;
+ }
+ if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
+ return false;
+ }
+ if (exception instanceof ConnectException) {// 连接被拒绝
+ return false;
+ }
+ if (exception instanceof InterruptedIOException) {// 超时
+ return true;
+ }
+ if (exception instanceof UnknownHostException) {// 目标服务器不可达
+ return false;
+ }
+ if (exception instanceof SSLException) {// ssl握手异常
+ return false;
+ }
+ HttpClientContext clientContext = HttpClientContext.adapt(context);
+ HttpRequest request = clientContext.getRequest();
+ // 如果请求是幂等的,就再次尝试
+ if (!(request instanceof HttpEntityEnclosingRequest)) {
+ return true;
+ }
+ return false;
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/zdjizhi/syncfile/core/SyncFiles.java b/src/main/java/com/zdjizhi/syncfile/core/SyncFiles.java
index 6fc49df..b83ee5c 100644
--- a/src/main/java/com/zdjizhi/syncfile/core/SyncFiles.java
+++ b/src/main/java/com/zdjizhi/syncfile/core/SyncFiles.java
@@ -1,6 +1,5 @@
package com.zdjizhi.syncfile.core;
-import cn.hutool.core.io.IoUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.zdjizhi.syncfile.config.ThreadPoolFactory;
@@ -10,7 +9,6 @@ import com.zdjizhi.syncfile.utils.HttpUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
@@ -35,16 +33,15 @@ public class SyncFiles {
for (List sourceList : fileList) {
callableList.add(() -> {
boolean status = false;
- InputStream content = null;
try {
for (Source source : sourceList) {
String source_oss_path = source.getSource_oss_path();
String destination_oss_path = source.getDestination_oss_path();
if (source_oss_path != null && !"".equals(source_oss_path)
&& destination_oss_path != null && !"".equals(destination_oss_path)) {
- content = httpUtil.httpGetInputStream(source_oss_path);
- if (content != null) {
- boolean isSuccess = httpUtil.httpPostInputStream(destination_oss_path, content);
+ byte[] file = httpUtil.httpGetFile(source_oss_path);
+ if (file != null) {
+ boolean isSuccess = httpUtil.httpPostFile(destination_oss_path, file);
if (!isSuccess) {
log.error("Sync file failed, post oss file error. destination_oss_path: {}", destination_oss_path);
monitorProperties.addPostFileErrorCount();
@@ -67,8 +64,6 @@ public class SyncFiles {
log.error("Sync file failed.", e);
monitorProperties.addFileSyncError();
status = false;
- } finally {
- IoUtil.close(content);
}
return status;
});
diff --git a/src/main/java/com/zdjizhi/syncfile/utils/HttpUtil.java b/src/main/java/com/zdjizhi/syncfile/utils/HttpUtil.java
index 98c9a07..4398129 100644
--- a/src/main/java/com/zdjizhi/syncfile/utils/HttpUtil.java
+++ b/src/main/java/com/zdjizhi/syncfile/utils/HttpUtil.java
@@ -1,25 +1,22 @@
package com.zdjizhi.syncfile.utils;
import cn.hutool.core.io.IoUtil;
+import cn.hutool.json.JSONObject;
+import cn.hutool.json.JSONUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONObject;
import com.zdjizhi.syncfile.entity.PostFileResponse;
import com.zdjizhi.syncfile.monitor.MonitorProperties;
-import org.apache.commons.io.IOUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import java.io.InputStream;
-
@Component
public class HttpUtil {
private static Log log = LogFactory.get();
@@ -31,19 +28,19 @@ public class HttpUtil {
@Autowired
MonitorProperties monitorProperties;
- public InputStream httpGetInputStream(String url) {
- InputStream result = null;
+ public byte[] httpGetFile(String url) {
+ byte[] data = null;
CloseableHttpResponse response = null;
try {
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(requestConfig);
response = httpClient.execute(httpGet);
if (response.getStatusLine().getStatusCode() == 200) {
- result = IOUtils.toBufferedInputStream(response.getEntity().getContent());
+ data = IoUtil.readBytes(response.getEntity().getContent());
log.info("get file success. current url: {}", url);
monitorProperties.addDownloadFileSuccessCount();
- monitorProperties.addDownloadFileSize(Integer.parseInt(response.getFirstHeader("Content-Length").getValue()));
- }else if (response.getStatusLine().getStatusCode() == 500){
+ monitorProperties.addDownloadFileSize(data.length);
+ } else if (response.getStatusLine().getStatusCode() == 500) {
log.error("get file error. Hos service error, please check hos. current url: {}", url);
monitorProperties.addHosError();
} else {
@@ -56,40 +53,40 @@ public class HttpUtil {
} finally {
IoUtil.close(response);
}
- return result;
+ return data;
}
- public boolean httpPostInputStream(String url, InputStream data) {
+ public boolean httpPostFile(String url, byte[] file) {
boolean isSuccess = false;
CloseableHttpResponse response = null;
try {
HttpPost httpPost = new HttpPost(url);
httpPost.setConfig(requestConfig);
- httpPost.setEntity(new InputStreamEntity(data));
+ httpPost.setEntity(new ByteArrayEntity(file));
response = httpClient.execute(httpPost);
+ String responseEntity = EntityUtils.toString(response.getEntity(), "UTF-8");
+ log.info("post file response: " + responseEntity);
if (response.getStatusLine().getStatusCode() == 200) {
- String responseEntity = EntityUtils.toString(response.getEntity(), "UTF-8");
- JSONObject jsonObj = (JSONObject) JSON.parse(responseEntity);
- if(jsonObj!=null){
- if (responseEntity.contains("\"code\":200")) {
- PostFileResponse postFileResponse = JSON.toJavaObject(jsonObj, PostFileResponse.class);
+ if (responseEntity.contains("\"code\":200")) {
+ if (JSONUtil.isJsonObj(responseEntity)) {
+ JSONObject jsonObj = JSONUtil.parseObj(responseEntity);
+ PostFileResponse postFileResponse = JSONUtil.toBean(jsonObj, PostFileResponse.class);
isSuccess = true;
log.info("post file success. current url: {}, msg: {}", url, responseEntity);
monitorProperties.addPostFileSuccessCount();
monitorProperties.addPostFileSize(postFileResponse.getData().getFileSize());
} else {
- log.error("post file error. current url: {}, msg: {}", url,responseEntity);
- monitorProperties.addFileSyncError();
+ log.error("post file successfully response is not json data. current url: {}, msg: {}", url, responseEntity);
}
- }else {
- log.error("post file error, response body error. current url: {}", url);
- monitorProperties.addOssError();
+ } else {
+ log.error("post file error. current url: {}, msg: {}", url, responseEntity);
+ monitorProperties.addFileSyncError();
}
- } else if(response.getStatusLine().getStatusCode() == 500){
- log.error("post file error. Oss service error.current url: {}, code: 500, msg: {}", url, response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), "UTF-8"));
+ } else if (response.getStatusLine().getStatusCode() == 500) {
+ log.error("post file error. Oss service error.current url: {}, code: 500, msg: {}", url, response.getStatusLine().getStatusCode(), responseEntity);
monitorProperties.addOssError();
- }else {
- log.error("post file error. current url: {}, code: {}, msg: {}", url, response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), "UTF-8"));
+ } else {
+ log.error("post file error. current url: {}, code: {}, msg: {}", url, response.getStatusLine().getStatusCode(), responseEntity);
monitorProperties.addFileSyncError();
}
} catch (Exception e) {