package com.zdjizhi.etl; import cn.hutool.core.convert.Convert; import cn.hutool.core.util.StrUtil; import com.zdjizhi.enums.DnsType; import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; import org.apache.flink.util.Collector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashMap; import java.util.List; import java.util.Map; import static com.zdjizhi.common.FlowWriteConfig.LOG_AGGREGATE_DURATION; /** * @author 94976 */ @Deprecated public class DnsProcessFunction extends ProcessWindowFunction, Map, String, TimeWindow> { private static final Logger logger = LoggerFactory.getLogger(DnsProcessFunction.class); /** * 拆分dns_record * 五种:a/aaaa/cname/mx/ns * * @param elements * @return */ @Override public void process(String keys, Context context, Iterable> elements, Collector> out) { try { long startTime = System.currentTimeMillis() / 1000; long endTime = System.currentTimeMillis() / 1000; try { Map distinctA = new HashMap<>(); Map distinctAAAA = new HashMap<>(); Map distinctCname = new HashMap<>(); Map distinctNs = new HashMap<>(); Map distinctMx = new HashMap<>(); for (Map log : elements) { List dnsA = splitDns(log, "dns_a"); List dnsAAAA = splitDns(log, "dns_aaaa"); List dnsCname = splitDns(log, "dns_cname"); List dnsNs = splitDns(log, "dns_ns"); List dnsMx = splitDns(log, "dns_mx"); dnsA.forEach(x -> distinctA.merge(x, 1L, Long::sum)); dnsAAAA.forEach(x -> distinctAAAA.merge(x, 1L, Long::sum)); dnsCname.forEach(x -> distinctCname.merge(x, 1L, Long::sum)); dnsNs.forEach(x -> distinctNs.merge(x, 1L, Long::sum)); dnsMx.forEach(x -> distinctMx.merge(x, 1L, Long::sum)); long connStartTimetime = Convert.toLong(log.get("capture_time")); startTime = connStartTimetime < startTime ? connStartTimetime : startTime; endTime = connStartTimetime > endTime ? connStartTimetime : endTime; } getNewDns(startTime, endTime, keys, distinctA, DnsType.A.getType(), out); getNewDns(startTime, endTime, keys, distinctAAAA, DnsType.AAAA.getType(), out); getNewDns(startTime, endTime, keys, distinctCname, DnsType.CNAME.getType(), out); getNewDns(startTime, endTime, keys, distinctNs, DnsType.NS.getType(), out); getNewDns(startTime, endTime, keys, distinctMx, DnsType.MX.getType(), out); } catch (Exception e) { logger.error("聚合中间结果集失败 {}", e); } } catch (Exception e) { logger.error("获取中间聚合结果失败,middleResult: {}", e); } } private static List splitDns(Map log, String key) { return StrUtil.split(StrUtil.toString(log.get(key)), StrUtil.COMMA); } private void getNewDns(long startTime, long endTime, String dnsQname, Map distinctMap, String type, Collector> out) { for (Map.Entry dns : distinctMap.entrySet()) { Map newDns = new HashMap<>(); newDns.put("start_time", startTime); newDns.put("end_time", endTime + LOG_AGGREGATE_DURATION); newDns.put("record_type", type); newDns.put("qname", dnsQname); newDns.put("record", dns.getKey()); newDns.put("sessions", dns.getValue()); out.collect(newDns); } } }