59 lines
2.1 KiB
Java
59 lines
2.1 KiB
Java
package com.zdjizhi.etl;
|
||
|
||
import cn.hutool.core.convert.Convert;
|
||
import cn.hutool.core.date.DateUtil;
|
||
import org.apache.flink.api.java.tuple.Tuple3;
|
||
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
|
||
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
|
||
import org.apache.flink.util.Collector;
|
||
import org.slf4j.Logger;
|
||
import org.slf4j.LoggerFactory;
|
||
|
||
import java.util.LinkedHashMap;
|
||
import java.util.Map;
|
||
|
||
import static com.zdjizhi.common.FlowWriteConfig.LOG_AGGREGATE_DURATION;
|
||
|
||
|
||
/**
|
||
* @author 94976
|
||
*/
|
||
public class DnsRelationProcessFunction extends ProcessWindowFunction<Map<String, Object>, Map<String, Object>, Tuple3<String, String, String>, TimeWindow> {
|
||
|
||
private static final Logger logger = LoggerFactory.getLogger(DnsRelationProcessFunction.class);
|
||
|
||
/**
|
||
* 拆分dns_record
|
||
* 聚合统计
|
||
* 五种：a/aaaa/cname/mx/ns
|
||
*
|
||
* @param elements
|
||
* @return
|
||
*/
|
||
@Override
|
||
public void process(Tuple3<String, String, String> keys, Context context, Iterable<Map<String, Object>> elements, Collector<Map<String, Object>> out) {
|
||
|
||
try {
|
||
long sessions = 0L;
|
||
long startTime = DateUtil.currentSeconds();
|
||
long endTime = DateUtil.currentSeconds();
|
||
for (Map<String, Object> log : elements) {
|
||
sessions++;
|
||
long logStartTime = Convert.toLong(log.get("start_time"));
|
||
startTime = logStartTime < startTime ? logStartTime : startTime;
|
||
endTime = logStartTime > endTime ? logStartTime : endTime;
|
||
}
|
||
Map<String, Object> newDns = new LinkedHashMap<>();
|
||
newDns.put("start_time", startTime);
|
||
newDns.put("end_time", endTime + LOG_AGGREGATE_DURATION);
|
||
newDns.put("record_type", keys.f0);
|
||
newDns.put("qname", keys.f1);
|
||
newDns.put("record", keys.f2);
|
||
newDns.put("sessions", sessions);
|
||
out.collect(newDns);
|
||
} catch (Exception e) {
|
||
logger.error("dns relation 日志聚合失败: {}", e);
|
||
}
|
||
}
|
||
}
|