gfwleak/galaxy-tsg-olap-file-chunk-combiner
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化代码，完善单元测试

Browse Source
This commit is contained in:
houjinchuan
2024-03-05 17:26:52 +08:00
parent 7795ebb318
commit 5be9f84f96
10 changed files with 228 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/com/zdjizhi/FileChunkCombiner.java
View File

@@ -42,7 +42,7 @@ public class FileChunkCombiner {
.name("Kafka Source")
.map(new ParseMessagePackMapFunction())
.name("Map: Parse Message Pack")
.filter(new FileChunkFilter(configuration.getLong(Configs.FILE_MAX_SIZE), configuration.getString(Configs.FILTER_EXPRESSION)))
.filter(new FileChunkFilterFunction(configuration.getLong(Configs.FILE_MAX_SIZE), configuration.getString(Configs.FILTER_EXPRESSION)))
.assignTimestampsAndWatermarks(watermarkStrategy);
OutputTag<FileChunk> delayedChunkOutputTag = new OutputTag<FileChunk>("delayed-chunk") {
@@ -82,5 +82,4 @@ public class FileChunkCombiner {
environment.execute(configuration.get(Configs.FLINK_JOB_NAME));
}
}
}

8
src/main/java/com/zdjizhi/function/CombineChunkProcessWindowFunction.java
View File

@@ -15,9 +15,9 @@ import java.util.*;
public class CombineChunkProcessWindowFunction extends ProcessWindowFunction<FileChunk, FileChunk, String, TimeWindow> {
private transient Counter duplicateChunkCounter;
private transient Counter combineErrorCounter;
private transient Counter seekChunkCounter;
private transient Counter appendChunkCounter;
public transient Counter combineErrorCounter;
public transient Counter seekChunkCounter;
public transient Counter appendChunkCounter;
private final Configuration configuration;
public CombineChunkProcessWindowFunction(Configuration configuration) {
@@ -41,4 +41,4 @@ public class CombineChunkProcessWindowFunction extends ProcessWindowFunction<Fil
out.collect(fileChunk);
}
}
}
}

4
src/main/java/com/zdjizhi/function/FileChunkFilter.java → src/main/java/com/zdjizhi/function/FileChunkFilterFunction.java
View File

@@ -8,14 +8,14 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.MetricGroup;
public class FileChunkFilter extends RichFilterFunction<FileChunk> {
public class FileChunkFilterFunction extends RichFilterFunction<FileChunk> {
private final long maxFileSize;
private final String filterExpression;
private transient Counter filterChunkCounter;
private JexlExpression jexlExpression;
private JexlContext jexlContext;
public FileChunkFilter(long maxFileSize, String filterExpression) {
public FileChunkFilterFunction(long maxFileSize, String filterExpression) {
this.maxFileSize = maxFileSize;
this.filterExpression = filterExpression;
}

24
src/main/java/com/zdjizhi/pojo/FileChunk.java
View File

@@ -22,6 +22,30 @@ public class FileChunk implements Serializable {
public FileChunk() {
}
public FileChunk(String uuid, String fileType, long length, byte[] chunk, String combineMode, int chunkCount, long timestamp, Map<String, Object> meta, String chunkNumbers) {
this.uuid = uuid;
this.fileType = fileType;
this.length = length;
this.chunk = chunk;
this.combineMode = combineMode;
this.chunkCount = chunkCount;
this.timestamp = timestamp;
this.meta = meta;
this.chunkNumbers = chunkNumbers;
}
public FileChunk(String uuid, String fileType, long offset, long length, byte[] chunk, String combineMode, int lastChunkFlag, int chunkCount, long timestamp) {
this.uuid = uuid;
this.fileType = fileType;
this.offset = offset;
this.length = length;
this.chunk = chunk;
this.combineMode = combineMode;
this.lastChunkFlag = lastChunkFlag;
this.chunkCount = chunkCount;
this.timestamp = timestamp;
}
public String getChunkNumbers() {
return chunkNumbers;
}

14
src/main/java/com/zdjizhi/sink/HBaseSink.java
View File

@@ -28,10 +28,10 @@ public class HBaseSink extends RichSinkFunction<FileChunk> {
private static final Log LOG = LogFactory.get();
private final Configuration configuration;
private transient Counter sendHBaseCounter;
private transient Counter sendHBaseErrorCounter;
private transient Counter sendHBaseFileCounter;
private transient Counter sendHBaseChunkCounter;
public transient Counter sendHBaseCounter;
public transient Counter sendHBaseErrorCounter;
public transient Counter sendHBaseFileCounter;
public transient Counter sendHBaseChunkCounter;
private boolean isAsync;
private Connection syncHBaseConnection;
private AsyncConnection AsyncHBaseConnection;
@@ -165,7 +165,7 @@ public class HBaseSink extends RichSinkFunction<FileChunk> {
} catch (IOException | InterruptedException e) {
LOG.error("put chunk to hbase data table error. ", e.getMessage());
sendHBaseErrorCounter.inc();
}finally {
} finally {
dataPutList.clear();
}
}
@@ -176,7 +176,7 @@ public class HBaseSink extends RichSinkFunction<FileChunk> {
} catch (IOException | InterruptedException e) {
LOG.error("put chunk to hbase index time table error. ", e.getMessage());
sendHBaseErrorCounter.inc();
}finally {
} finally {
indexTimePutList.clear();
}
}
@@ -187,7 +187,7 @@ public class HBaseSink extends RichSinkFunction<FileChunk> {
} catch (IOException | InterruptedException e) {
LOG.error("put chunk to hbase index filename table error. ", e.getMessage());
sendHBaseErrorCounter.inc();
}finally {
} finally {
indexFilenamePutList.clear();
}
}

8
src/main/java/com/zdjizhi/sink/HosSink.java
View File

@@ -35,10 +35,10 @@ public class HosSink extends RichSinkFunction<FileChunk> {
private static final Log LOG = LogFactory.get();
private final Configuration configuration;
private transient Counter sendHosCounter;
private transient Counter sendHosErrorCounter;
private transient Counter sendHosFileCounter;
private transient Counter sendHosChunkCounter;
public transient Counter sendHosCounter;
public transient Counter sendHosErrorCounter;
public transient Counter sendHosFileCounter;
public transient Counter sendHosChunkCounter;
private boolean isAsync;
private CloseableHttpClient syncHttpClient;
private CloseableHttpAsyncClient asyncHttpClient;

2
src/main/java/com/zdjizhi/trigger/LastChunkOrNoDataInTimeTrigger.java
View File

@@ -69,4 +69,4 @@ public class LastChunkOrNoDataInTimeTrigger<W extends TimeWindow> extends Trigge
return Math.max(value1, value2);
}
}
}
}

2
src/main/java/com/zdjizhi/trigger/MultipleTrigger.java
View File

@@ -64,4 +64,4 @@ public class MultipleTrigger<T, W extends Window> extends Trigger<T, W> {
trigger.clear(window, ctx);
}
}
}
}

2
src/main/java/com/zdjizhi/utils/PublicUtil.java
View File

@@ -1,7 +1,6 @@
package com.zdjizhi.utils;
import cn.hutool.core.util.ArrayUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.crypto.digest.DigestUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
@@ -21,7 +20,6 @@ public class PublicUtil {
List<FileChunk> combinedFileChunkList = new ArrayList<>();
try {
List<FileChunk> originalFileChunkList = StreamSupport.stream(input.spliterator(), false).collect(Collectors.toList());
System.out.println(originalFileChunkList);
List<byte[]> waitingToCombineChunkList = new ArrayList<>();
if (COMBINE_MODE_SEEK.equals(originalFileChunkList.get(0).getCombineMode())) {
seekChunkCounter.inc();