gfwleak/galaxy-tsg-olap-p19-file-sync-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加重试，修改JDK版本

Browse Source
This commit is contained in:
houjinchuan
2023-08-01 10:02:34 +08:00
parent 4517d2744f
commit bea54cff55
4 changed files with 98 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pom.xml
View File

@@ -6,7 +6,7 @@
<groupId>com.zdjizhi</groupId>
<artifactId>p19-file-sync-service</artifactId>
<version>21.12.01</version>
<version>23.03.09</version>
<name>p19-file-sync-service</name>
<parent>
@@ -20,14 +20,14 @@
<repository>
<id>nexus</id>
<name>Team Nexus Repository</name>
<url>http://192.168.40.125:8099/content/groups/public/</url>
<url>http://192.168.40.153:8099/content/groups/public/</url>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>nexus</id>
<name>Team Nexus Repository</name>
<url>http://192.168.40.125:8099/content/groups/public/</url>
<url>http://192.168.40.153:8099/content/groups/public/</url>
</pluginRepository>
</pluginRepositories>
@@ -238,7 +238,7 @@
<!--Dockerfile文件所在目录-->
<dockerDirectory>docker</dockerDirectory>
<buildArgs>
<JDK_IMAGE>192.168.40.153:9080/common/jdk:1.8.0_73-jre</JDK_IMAGE>
<JDK_IMAGE>192.168.40.153:9080/common/jdk:1.8.0_202</JDK_IMAGE>
<GO_IMAGE>192.168.40.153:9080/common/golang:1.15.6</GO_IMAGE>
<JAR_FILE>${project.build.finalName}.xjar</JAR_FILE>
</buildArgs>

72
src/main/java/com/zdjizhi/syncfile/config/HttpClientPool.java
View File

@@ -1,20 +1,35 @@
package com.zdjizhi.syncfile.config;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.alibaba.nacos.api.config.ConfigType;
import com.alibaba.nacos.api.config.annotation.NacosConfigurationProperties;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequest;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
import javax.net.ssl.*;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.ConnectException;
import java.net.UnknownHostException;
@Component
//@ConfigurationProperties(prefix = "http")
@NacosConfigurationProperties(prefix = "http", dataId = "${nacos.config.data-id}", groupId = "${nacos.config.group}", type = ConfigType.YAML, autoRefreshed = true)
public class HttpClientPool {
private Log log = LogFactory.get();
private Integer maxTotal;
@@ -28,6 +43,7 @@ public class HttpClientPool {
private boolean staleConnectionCheckEnabled;
private Integer retryNum;
public void setMaxTotal(Integer maxTotal) {
this.maxTotal = maxTotal;
@@ -53,6 +69,14 @@ public class HttpClientPool {
this.staleConnectionCheckEnabled = staleConnectionCheckEnabled;
}
public Integer getRetryNum() {
return retryNum;
}
public void setRetryNum(Integer retryNum) {
this.retryNum = retryNum;
}
/**
* 首先实例化一个连接池管理器，设置最大连接数、并发连接数
*
@@ -90,8 +114,10 @@ public class HttpClientPool {
* @return
*/
@Bean
public CloseableHttpClient getCloseableHttpClient(@Qualifier("httpClientBuilder") HttpClientBuilder httpClientBuilder) {
return httpClientBuilder.build();
public CloseableHttpClient getCloseableHttpClient(@Qualifier("httpClientBuilder") HttpClientBuilder httpClientBuilder,@Qualifier("httpRetryHandler") HttpRequestRetryHandler httpRetryHandler){
return httpClientBuilder
.setRetryHandler(httpRetryHandler)
.build();
}
/**
@@ -109,7 +135,6 @@ public class HttpClientPool {
.setConnectionRequestTimeout(connectionRequestTimeout)
.setSocketTimeout(socketTimeout)
.setStaleConnectionCheckEnabled(staleConnectionCheckEnabled);
}
/**
@@ -120,6 +145,41 @@ public class HttpClientPool {
return builder.build();
}
}
@Bean(name = "httpRetryHandler")
public HttpRequestRetryHandler getHttpRetryHandler() {
return new HttpRequestRetryHandler() {
@Override
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
if (executionCount >= retryNum) {// 如果已经重试了3次，就放弃
log.error("已完成重试次数");
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof ConnectException) {// 连接被拒绝
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return true;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的，就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
};
}
}

11
src/main/java/com/zdjizhi/syncfile/core/SyncFiles.java
View File

@@ -1,6 +1,5 @@
package com.zdjizhi.syncfile.core;
import cn.hutool.core.io.IoUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.zdjizhi.syncfile.config.ThreadPoolFactory;
@@ -10,7 +9,6 @@ import com.zdjizhi.syncfile.utils.HttpUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
@@ -35,16 +33,15 @@ public class SyncFiles {
for (List<Source> sourceList : fileList) {
callableList.add(() -> {
boolean status = false;
InputStream content = null;
try {
for (Source source : sourceList) {
String source_oss_path = source.getSource_oss_path();
String destination_oss_path = source.getDestination_oss_path();
if (source_oss_path != null && !"".equals(source_oss_path)
&& destination_oss_path != null && !"".equals(destination_oss_path)) {
content = httpUtil.httpGetInputStream(source_oss_path);
if (content != null) {
boolean isSuccess = httpUtil.httpPostInputStream(destination_oss_path, content);
byte[] file = httpUtil.httpGetFile(source_oss_path);
if (file != null) {
boolean isSuccess = httpUtil.httpPostFile(destination_oss_path, file);
if (!isSuccess) {
log.error("Sync file failed, post oss file error. destination_oss_path: {}", destination_oss_path);
monitorProperties.addPostFileErrorCount();
@@ -67,8 +64,6 @@ public class SyncFiles {
log.error("Sync file failed.", e);
monitorProperties.addFileSyncError();
status = false;
} finally {
IoUtil.close(content);
}
return status;
});

53
src/main/java/com/zdjizhi/syncfile/utils/HttpUtil.java
View File

@@ -1,25 +1,22 @@
package com.zdjizhi.syncfile.utils;
import cn.hutool.core.io.IoUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zdjizhi.syncfile.entity.PostFileResponse;
import com.zdjizhi.syncfile.monitor.MonitorProperties;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.InputStreamEntity;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.InputStream;
@Component
public class HttpUtil {
private static Log log = LogFactory.get();
@@ -31,19 +28,19 @@ public class HttpUtil {
@Autowired
MonitorProperties monitorProperties;
public InputStream httpGetInputStream(String url) {
InputStream result = null;
public byte[] httpGetFile(String url) {
byte[] data = null;
CloseableHttpResponse response = null;
try {
HttpGet httpGet = new HttpGet(url);
httpGet.setConfig(requestConfig);
response = httpClient.execute(httpGet);
if (response.getStatusLine().getStatusCode() == 200) {
result = IOUtils.toBufferedInputStream(response.getEntity().getContent());
data = IoUtil.readBytes(response.getEntity().getContent());
log.info("get file success. current url: {}", url);
monitorProperties.addDownloadFileSuccessCount();
monitorProperties.addDownloadFileSize(Integer.parseInt(response.getFirstHeader("Content-Length").getValue()));
}else if (response.getStatusLine().getStatusCode() == 500){
monitorProperties.addDownloadFileSize(data.length);
} else if (response.getStatusLine().getStatusCode() == 500) {
log.error("get file error. Hos service error, please check hos. current url: {}", url);
monitorProperties.addHosError();
} else {
@@ -56,40 +53,40 @@ public class HttpUtil {
} finally {
IoUtil.close(response);
}
return result;
return data;
}
public boolean httpPostInputStream(String url, InputStream data) {
public boolean httpPostFile(String url, byte[] file) {
boolean isSuccess = false;
CloseableHttpResponse response = null;
try {
HttpPost httpPost = new HttpPost(url);
httpPost.setConfig(requestConfig);
httpPost.setEntity(new InputStreamEntity(data));
httpPost.setEntity(new ByteArrayEntity(file));
response = httpClient.execute(httpPost);
String responseEntity = EntityUtils.toString(response.getEntity(), "UTF-8");
log.info("post file response: " + responseEntity);
if (response.getStatusLine().getStatusCode() == 200) {
String responseEntity = EntityUtils.toString(response.getEntity(), "UTF-8");
JSONObject jsonObj = (JSONObject) JSON.parse(responseEntity);
if(jsonObj!=null){
if (responseEntity.contains("\"code\":200")) {
PostFileResponse postFileResponse = JSON.toJavaObject(jsonObj, PostFileResponse.class);
if (responseEntity.contains("\"code\":200")) {
if (JSONUtil.isJsonObj(responseEntity)) {
JSONObject jsonObj = JSONUtil.parseObj(responseEntity);
PostFileResponse postFileResponse = JSONUtil.toBean(jsonObj, PostFileResponse.class);
isSuccess = true;
log.info("post file success. current url: {}, msg: {}", url, responseEntity);
monitorProperties.addPostFileSuccessCount();
monitorProperties.addPostFileSize(postFileResponse.getData().getFileSize());
} else {
log.error("post file error. current url: {}, msg: {}", url,responseEntity);
monitorProperties.addFileSyncError();
log.error("post file successfully response is not json data. current url: {}, msg: {}", url, responseEntity);
}
}else {
log.error("post file error, response body error. current url: {}", url);
monitorProperties.addOssError();
} else {
log.error("post file error. current url: {}, msg: {}", url, responseEntity);
monitorProperties.addFileSyncError();
}
} else if(response.getStatusLine().getStatusCode() == 500){
log.error("post file error. Oss service error.current url: {}, code: 500, msg: {}", url, response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), "UTF-8"));
} else if (response.getStatusLine().getStatusCode() == 500) {
log.error("post file error. Oss service error.current url: {}, code: 500, msg: {}", url, response.getStatusLine().getStatusCode(), responseEntity);
monitorProperties.addOssError();
}else {
log.error("post file error. current url: {}, code: {}, msg: {}", url, response.getStatusLine().getStatusCode(), EntityUtils.toString(response.getEntity(), "UTF-8"));
} else {
log.error("post file error. current url: {}, code: {}, msg: {}", url, response.getStatusLine().getStatusCode(), responseEntity);
monitorProperties.addFileSyncError();
}
} catch (Exception e) {