gfwleak/galaxy-tsg-olap-dll-multipoint-aggregation
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

临时文件缺省以.tmp 结尾

Browse Source
This commit is contained in:
zhanghongqing
2020-12-28 17:32:01 +08:00
parent 7ef4eb8da8
commit cc155b0f78
3 changed files with 57 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
safe-file-roll-sink/pom.xml
View File

@@ -4,9 +4,9 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>cn.huyanping.flume</groupId>
<groupId>cn.flume</groupId>
<artifactId>safe-roll-file-sink</artifactId>
<version>0.2</version>
<version>1.0</version>
<dependencies>
<dependency>

3
safe-file-roll-sink/src/main/java/cn/flume/sinks/SafePathManager.java
View File

@@ -14,6 +14,7 @@ public class SafePathManager {
private File baseDirectory;
private AtomicInteger fileIndex;
private String filePrefix = "";
private static final String IN_USE_SUFFIX = ".tmp";
private File currentFile;
@@ -24,7 +25,7 @@ public class SafePathManager {
//重新生成微秒级文件名
seriesTimestamp = System.currentTimeMillis();
fileIndex = new AtomicInteger();
currentFile = new File(baseDirectory, filePrefix + fileIndex.incrementAndGet() + "-" + seriesTimestamp );
currentFile = new File(baseDirectory, filePrefix + fileIndex.incrementAndGet() + "-" + seriesTimestamp + IN_USE_SUFFIX );
return currentFile;
}

29
safe-file-roll-sink/src/main/java/cn/flume/sinks/SafeRollingFileSink.java
View File

@@ -18,8 +18,7 @@ import java.util.concurrent.TimeUnit;
public class SafeRollingFileSink extends AbstractSink implements Configurable {
private static final Logger logger = LoggerFactory
.getLogger(SafeRollingFileSink.class);
private static final Logger logger = LoggerFactory.getLogger(SafeRollingFileSink.class);
private static final long defaultRollInterval = 30;
private static final int defaultBatchSize = 100;
@@ -339,23 +338,27 @@ public class SafeRollingFileSink extends AbstractSink implements Configurable {
return current.delete();
}
if (useFileSuffix && moveFile) {
return current.renameTo(new File(this.targetDirectory, current.getName() + fileSuffix));
return current.renameTo(new File(this.targetDirectory, removeFileSuffix(current) + fileSuffix));
} else if (useFileSuffix) {
return current.renameTo(new File(this.directory, current.getName() + fileSuffix));
return current.renameTo(new File(this.directory, removeFileSuffix(current) + fileSuffix));
} else if (moveFile) {
return current.renameTo(new File(this.targetDirectory, current.getName()));
return current.renameTo(new File(this.targetDirectory, removeFileSuffix(current)));
} else {
return true;
}
}
private String removeFileSuffix(File current) {
return current.getName().substring(0, current.getName().lastIndexOf("."));
}
private boolean copyLogFile(File current) throws IOException {
if (current.length() == 0L) {
logger.info("Delete empty file{}", current.getName());
return current.delete();
}
for (File targetDir : this.copyDirectory) {
File targetFile = new File(targetDir.getAbsolutePath(), current.getName() + fileSuffix);
File targetFile = new File(targetDir.getAbsolutePath(), removeFileSuffix(current) + fileSuffix);
boolean copyResult = this.copyFile(current, targetFile, false);
if (!copyResult) return false;
}
@@ -366,18 +369,12 @@ public class SafeRollingFileSink extends AbstractSink implements Configurable {
/**
* 复制单个文件
*
* @param srcFile
* 待复制的文件名
* @param destFile
* 目标文件名
* @param overlay
* 如果目标文件存在，是否覆盖
* @param srcFile 待复制的文件名
* @param destFile 目标文件名
* @param overlay 如果目标文件存在，是否覆盖
* @return 如果复制成功返回true，否则返回false
*/
public boolean copyFile(File srcFile, File destFile,
boolean overlay) throws IOException {
public boolean copyFile(File srcFile, File destFile, boolean overlay) throws IOException {
// 判断源文件是否存在
if (!srcFile.exists()) {
throw new IOException("Copy file failed, source file does not exists");