diff --git a/pom.xml b/pom.xml index 7d84db1..d894412 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.zdjizhi p19-file-sync-service - 21.12.01 + 23.03.09 p19-file-sync-service @@ -20,14 +20,14 @@ nexus Team Nexus Repository - http://192.168.40.125:8099/content/groups/public/ + http://192.168.40.153:8099/content/groups/public/ nexus Team Nexus Repository - http://192.168.40.125:8099/content/groups/public/ + http://192.168.40.153:8099/content/groups/public/ @@ -238,7 +238,7 @@ docker - 192.168.40.153:9080/common/jdk:1.8.0_73-jre + 192.168.40.153:9080/common/jdk:1.8.0_202 192.168.40.153:9080/common/golang:1.15.6 ${project.build.finalName}.xjar diff --git a/src/main/java/com/zdjizhi/syncfile/config/HttpClientPool.java b/src/main/java/com/zdjizhi/syncfile/config/HttpClientPool.java index e9f47c5..2903d63 100644 --- a/src/main/java/com/zdjizhi/syncfile/config/HttpClientPool.java +++ b/src/main/java/com/zdjizhi/syncfile/config/HttpClientPool.java @@ -1,20 +1,35 @@ package com.zdjizhi.syncfile.config; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; import com.alibaba.nacos.api.config.ConfigType; import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties; +import org.apache.http.HttpEntityEnclosingRequest; +import org.apache.http.HttpRequest; +import org.apache.http.NoHttpResponseException; +import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.protocol.HttpContext; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import javax.net.ssl.*; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.ConnectException; +import java.net.UnknownHostException; @Component //@ConfigurationProperties(prefix = "http") @NacosConfigurationProperties(prefix = "http", dataId = "${nacos.config.data-id}", groupId = "${nacos.config.group}", type = ConfigType.YAML, autoRefreshed = true) public class HttpClientPool { + private Log log = LogFactory.get(); private Integer maxTotal; @@ -28,6 +43,7 @@ public class HttpClientPool { private boolean staleConnectionCheckEnabled; + private Integer retryNum; public void setMaxTotal(Integer maxTotal) { this.maxTotal = maxTotal; @@ -53,6 +69,14 @@ public class HttpClientPool { this.staleConnectionCheckEnabled = staleConnectionCheckEnabled; } + public Integer getRetryNum() { + return retryNum; + } + + public void setRetryNum(Integer retryNum) { + this.retryNum = retryNum; + } + /** * 首先实例化一个连接池管理器,设置最大连接数、并发连接数 * @@ -90,8 +114,10 @@ public class HttpClientPool { * @return */ @Bean - public CloseableHttpClient getCloseableHttpClient(@Qualifier("httpClientBuilder") HttpClientBuilder httpClientBuilder) { - return httpClientBuilder.build(); + public CloseableHttpClient getCloseableHttpClient(@Qualifier("httpClientBuilder") HttpClientBuilder httpClientBuilder,@Qualifier("httpRetryHandler") HttpRequestRetryHandler httpRetryHandler){ + return httpClientBuilder + .setRetryHandler(httpRetryHandler) + .build(); } /** @@ -109,7 +135,6 @@ public class HttpClientPool { .setConnectionRequestTimeout(connectionRequestTimeout) .setSocketTimeout(socketTimeout) .setStaleConnectionCheckEnabled(staleConnectionCheckEnabled); - } /** @@ -120,6 +145,41 @@ public class HttpClientPool { return builder.build(); } - -} - + @Bean(name = "httpRetryHandler") + public HttpRequestRetryHandler getHttpRetryHandler() { + return new HttpRequestRetryHandler() { + @Override + public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { + if (executionCount >= retryNum) {// 如果已经重试了3次,就放弃 + log.error("已完成重试次数"); + return false; + } + if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试 + return true; + } + if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常 + return false; + } + if (exception instanceof ConnectException) {// 连接被拒绝 + return false; + } + if (exception instanceof InterruptedIOException) {// 超时 + return true; + } + if (exception instanceof UnknownHostException) {// 目标服务器不可达 + return false; + } + if (exception instanceof SSLException) {// ssl握手异常 + return false; + } + HttpClientContext clientContext = HttpClientContext.adapt(context); + HttpRequest request = clientContext.getRequest(); + // 如果请求是幂等的,就再次尝试 + if (!(request instanceof HttpEntityEnclosingRequest)) { + return true; + } + return false; + } + }; + } +} \ No newline at end of file diff --git a/src/main/java/com/zdjizhi/syncfile/core/SyncFiles.java b/src/main/java/com/zdjizhi/syncfile/core/SyncFiles.java index 6fc49df..b83ee5c 100644 --- a/src/main/java/com/zdjizhi/syncfile/core/SyncFiles.java +++ b/src/main/java/com/zdjizhi/syncfile/core/SyncFiles.java @@ -1,6 +1,5 @@ package com.zdjizhi.syncfile.core; -import cn.hutool.core.io.IoUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.zdjizhi.syncfile.config.ThreadPoolFactory; @@ -10,7 +9,6 @@ import com.zdjizhi.syncfile.utils.HttpUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; @@ -35,16 +33,15 @@ public class SyncFiles { for (List sourceList : fileList) { callableList.add(() -> { boolean status = false; - InputStream content = null; try { for (Source source : sourceList) { String source_oss_path = source.getSource_oss_path(); String destination_oss_path = source.getDestination_oss_path(); if (source_oss_path != null && !"".equals(source_oss_path) && destination_oss_path != null && !"".equals(destination_oss_path)) { - content = httpUtil.httpGetInputStream(source_oss_path); - if (content != null) { - boolean isSuccess = httpUtil.httpPostInputStream(destination_oss_path, content); + byte[] file = httpUtil.httpGetFile(source_oss_path); + if (file != null) { + boolean isSuccess = httpUtil.httpPostFile(destination_oss_path, file); if (!isSuccess) { log.error("Sync file failed, post oss file error. destination_oss_path: {}", destination_oss_path); monitorProperties.addPostFileErrorCount(); @@ -67,8 +64,6 @@ public class SyncFiles { log.error("Sync file failed.", e); monitorProperties.addFileSyncError(); status = false; - } finally { - IoUtil.close(content); } return status; }); diff --git a/src/main/java/com/zdjizhi/syncfile/utils/HttpUtil.java b/src/main/java/com/zdjizhi/syncfile/utils/HttpUtil.java index 98c9a07..4398129 100644 --- a/src/main/java/com/zdjizhi/syncfile/utils/HttpUtil.java +++ b/src/main/java/com/zdjizhi/syncfile/utils/HttpUtil.java @@ -1,25 +1,22 @@ package com.zdjizhi.syncfile.utils; import cn.hutool.core.io.IoUtil; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; import com.zdjizhi.syncfile.entity.PostFileResponse; import com.zdjizhi.syncfile.monitor.MonitorProperties; -import org.apache.commons.io.IOUtils; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.InputStreamEntity; +import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.InputStream; - @Component public class HttpUtil { private static Log log = LogFactory.get(); @@ -31,19 +28,19 @@ public class HttpUtil { @Autowired MonitorProperties monitorProperties; - public InputStream httpGetInputStream(String url) { - InputStream result = null; + public byte[] httpGetFile(String url) { + byte[] data = null; CloseableHttpResponse response = null; try { HttpGet httpGet = new HttpGet(url); httpGet.setConfig(requestConfig); response = httpClient.execute(httpGet); if (response.getStatusLine().getStatusCode() == 200) { - result = IOUtils.toBufferedInputStream(response.getEntity().getContent()); + data = IoUtil.readBytes(response.getEntity().getContent()); log.info("get file success. current url: {}", url); monitorProperties.addDownloadFileSuccessCount(); - monitorProperties.addDownloadFileSize(Integer.parseInt(response.getFirstHeader("Content-Length").getValue())); - }else if (response.getStatusLine().getStatusCode() == 500){ + monitorProperties.addDownloadFileSize(data.length); + } else if (response.getStatusLine().getStatusCode() == 500) { log.error("get file error. Hos service error, please check hos. current url: {}", url); monitorProperties.addHosError(); } else { @@ -56,40 +53,40 @@ public class HttpUtil { } finally { IoUtil.close(response); } - return result; + return data; } - public boolean httpPostInputStream(String url, InputStream data) { + public boolean httpPostFile(String url, byte[] file) { boolean isSuccess = false; CloseableHttpResponse response = null; try { HttpPost httpPost = new HttpPost(url); httpPost.setConfig(requestConfig); - httpPost.setEntity(new InputStreamEntity(data)); + httpPost.setEntity(new ByteArrayEntity(file)); response = httpClient.execute(httpPost); + String responseEntity = EntityUtils.toString(response.getEntity(), "UTF-8"); + log.info("post file response: " + responseEntity); if (response.getStatusLine().getStatusCode() == 200) { - String responseEntity = EntityUtils.toString(response.getEntity(), "UTF-8"); - JSONObject jsonObj = (JSONObject) JSON.parse(responseEntity); - if(jsonObj!=null){ - if (responseEntity.contains("\"code\":200")) { - PostFileResponse postFileResponse = JSON.toJavaObject(jsonObj, PostFileResponse.class); + if (responseEntity.contains("\"code\":200")) { + if (JSONUtil.isJsonObj(responseEntity)) { + JSONObject jsonObj = JSONUtil.parseObj(responseEntity); + PostFileResponse postFileResponse = JSONUtil.toBean(jsonObj, PostFileResponse.class); isSuccess = true; log.info("post file success. current url: {}, msg: {}", url, responseEntity); monitorProperties.addPostFileSuccessCount(); monitorProperties.addPostFileSize(postFileResponse.getData().getFileSize()); } else { - log.error("post file error. current url: {}, msg: {}", url,responseEntity); - monitorProperties.addFileSyncError(); + log.error("post file successfully response is not json data. current url: {}, msg: {}", url, responseEntity); } - }else { - log.error("post file error, response body error. current url: {}", url); - monitorProperties.addOssError(); + } else { + log.error("post file error. current url: {}, msg: {}", url, responseEntity); + monitorProperties.addFileSyncError(); } - } else if(response.getStatusLine().getStatusCode() == 500){ - log.error("post file error. Oss service error.current url: {}, code: 500, msg: {}", url, response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), "UTF-8")); + } else if (response.getStatusLine().getStatusCode() == 500) { + log.error("post file error. Oss service error.current url: {}, code: 500, msg: {}", url, response.getStatusLine().getStatusCode(), responseEntity); monitorProperties.addOssError(); - }else { - log.error("post file error. current url: {}, code: {}, msg: {}", url, response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), "UTF-8")); + } else { + log.error("post file error. current url: {}, code: {}, msg: {}", url, response.getStatusLine().getStatusCode(), responseEntity); monitorProperties.addFileSyncError(); } } catch (Exception e) {