package com.zdjizhi.etl; import cn.hutool.core.convert.Convert; import cn.hutool.core.date.DateUtil; import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.LinkedHashMap; import java.util.Map; import static com.zdjizhi.common.FlowWriteConfig.LOG_AGGREGATE_DURATION; /** * @author 94976 */ public class DnsRelationProcessFunction extends ProcessWindowFunction, Map, Tuple3, TimeWindow> { private static final Logger logger = LoggerFactory.getLogger(DnsRelationProcessFunction.class); /** * 拆分dns_record * 聚合统计 * 五种:a/aaaa/cname/mx/ns * * @param elements * @return */ @Override public void process(Tuple3 keys, Context context, Iterable> elements, Collector> out) { try { long sessions = 0L; long startTime = DateUtil.currentSeconds(); long endTime = DateUtil.currentSeconds(); for (Map log : elements) { sessions++; long logStartTime = Convert.toLong(log.get("start_time")); startTime = logStartTime < startTime ? logStartTime : startTime; endTime = logStartTime > endTime ? logStartTime : endTime; } Map newDns = new LinkedHashMap<>(); newDns.put("start_time", startTime); newDns.put("end_time", endTime + LOG_AGGREGATE_DURATION); newDns.put("record_type", keys.f0); newDns.put("qname", keys.f1); newDns.put("record", keys.f2); newDns.put("sessions", sessions); out.collect(newDns); } catch (Exception e) { logger.error("dns relation 日志聚合失败: {}", e); } } }