59 lines
2.1 KiB
Java
59 lines
2.1 KiB
Java
|
package com.zdjizhi.etl;
|
|||
|
|
|
|||
|
|
import cn.hutool.core.convert.Convert;
|
|||
|
|
import cn.hutool.core.date.DateUtil;
|
|||
|
|
import org.apache.flink.api.java.tuple.Tuple3;
|
|||
|
|
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
|
|||
|
|
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
|
|||
|
|
import org.apache.flink.util.Collector;
|
|||
|
|
import org.slf4j.Logger;
|
|||
|
|
import org.slf4j.LoggerFactory;
|
|||
|
|
|
|||
|
|
import java.util.LinkedHashMap;
|
|||
|
|
import java.util.Map;
|
|||
|
|
|
|||
|
|
import static com.zdjizhi.common.FlowWriteConfig.LOG_AGGREGATE_DURATION;
|
|||
|
|
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* @author 94976
|
|||
|
|
*/
|
|||
|
|
public class DnsRelationProcessFunction extends ProcessWindowFunction<Map<String, Object>, Map<String, Object>, Tuple3<String, String, String>, TimeWindow> {
|
|||
|
|
|
|||
|
|
private static final Logger logger = LoggerFactory.getLogger(DnsRelationProcessFunction.class);
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 拆分dns_record
|
|||
|
|
* 聚合统计
|
|||
|
|
* 五种：a/aaaa/cname/mx/ns
|
|||
|
|
*
|
|||
|
|
* @param elements
|
|||
|
|
* @return
|
|||
|
|
*/
|
|||
|
|
@Override
|
|||
|
|
public void process(Tuple3<String, String, String> keys, Context context, Iterable<Map<String, Object>> elements, Collector<Map<String, Object>> out) {
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
long sessions = 0L;
|
|||
|
|
long startTime = DateUtil.currentSeconds();
|
|||
|
|
long endTime = DateUtil.currentSeconds();
|
|||
|
|
for (Map<String, Object> log : elements) {
|
|||
|
|
sessions++;
|
|||
|
|
long logStartTime = Convert.toLong(log.get("start_time"));
|
|||
|
|
startTime = logStartTime < startTime ? logStartTime : startTime;
|
|||
|
|
endTime = logStartTime > endTime ? logStartTime : endTime;
|
|||
|
|
}
|
|||
|
|
Map<String, Object> newDns = new LinkedHashMap<>();
|
|||
|
|
newDns.put("start_time", startTime);
|
|||
|
|
newDns.put("end_time", endTime + LOG_AGGREGATE_DURATION);
|
|||
|
|
newDns.put("record_type", keys.f0);
|
|||
|
|
newDns.put("qname", keys.f1);
|
|||
|
|
newDns.put("record", keys.f2);
|
|||
|
|
newDns.put("sessions", sessions);
|
|||
|
|
out.collect(newDns);
|
|||
|
|
} catch (Exception e) {
|
|||
|
|
logger.error("dns relation 日志聚合失败: {}", e);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|