57 lines
2.3 KiB
Java
57 lines
2.3 KiB
Java
|
package com.zdjizhi.topology;
|
|||
|
|
|
|||
|
|
import cn.hutool.log.Log;
|
|||
|
|
import cn.hutool.log.LogFactory;
|
|||
|
|
import com.zdjizhi.common.FlowWriteConfig;
|
|||
|
|
import com.zdjizhi.utils.functions.FilterNullFunction;
|
|||
|
|
import com.zdjizhi.utils.functions.MapCompletedFunction;
|
|||
|
|
import com.zdjizhi.utils.kafka.Consumer;
|
|||
|
|
import com.zdjizhi.utils.kafka.Producer;
|
|||
|
|
import org.apache.flink.streaming.api.TimeCharacteristic;
|
|||
|
|
import org.apache.flink.streaming.api.datastream.DataStream;
|
|||
|
|
import org.apache.flink.streaming.api.datastream.DataStreamSource;
|
|||
|
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* @author qidaijie
|
|||
|
|
* @Package com.zdjizhi.topology
|
|||
|
|
* @Description:
|
|||
|
|
* @date 2021/5/2016:42
|
|||
|
|
*/
|
|||
|
|
public class LogFlowWriteTopology {
|
|||
|
|
private static final Log logger = LogFactory.get();
|
|||
|
|
|
|||
|
|
public static void main(String[] args) {
|
|||
|
|
final StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
|
|||
|
|
|
|||
|
|
//开启Checkpoint，interval用于指定checkpoint的触发间隔(单位milliseconds)
|
|||
|
|
// environment.enableCheckpointing(5000);
|
|||
|
|
|
|||
|
|
DataStreamSource<String> streamSource = environment.addSource(Consumer.getKafkaConsumer())
|
|||
|
|
.setParallelism(FlowWriteConfig.CONSUMER_PARALLELISM);
|
|||
|
|
|
|||
|
|
if (FlowWriteConfig.LOG_NEED_COMPLETE == 1) {
|
|||
|
|
//对原始日志进行处理补全转换等
|
|||
|
|
DataStream<String> cleaningLog = streamSource.map(new MapCompletedFunction())
|
|||
|
|
.name("TransFormLogs").setParallelism(FlowWriteConfig.MAP_PARALLELISM);
|
|||
|
|
//过滤空数据不发送到Kafka内
|
|||
|
|
DataStream<String> result = cleaningLog.filter(new FilterNullFunction()).name("FilterAbnormalData");
|
|||
|
|
//发送数据到Kafka
|
|||
|
|
result.addSink(Producer.getKafkaProducer()).name("LogSinkKafka")
|
|||
|
|
.setParallelism(FlowWriteConfig.PRODUCER_PARALLELISM);
|
|||
|
|
} else {
|
|||
|
|
DataStream<String> result = streamSource.filter(new FilterNullFunction()).name("FilterOriginalData");
|
|||
|
|
result.addSink(Producer.getKafkaProducer()).name("LogSinkKafka").setParallelism(FlowWriteConfig.PRODUCER_PARALLELISM);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
environment.execute(args[0]);
|
|||
|
|
} catch (Exception e) {
|
|||
|
|
logger.error("This Flink task start ERROR! Exception information is :" + e);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
}
|