代码优化，使用定时线程池刷新写入clickhouse

2022-09-01 15:39:14 +08:00
parent 812fb82c95
commit 25e5b51766
18 changed files with 736 additions and 85 deletions
--- a/src/main/java/com/zdjizhi/etl/connection/ConnReduceFunction.java
+++ b/src/main/java/com/zdjizhi/etl/connection/ConnReduceFunction.java
@@ -0,0 +1,80 @@
+package com.zdjizhi.etl.connection;
+
+import cn.hutool.core.convert.Convert;
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.log.Log;
+import cn.hutool.log.LogFactory;
+import com.alibaba.fastjson.util.TypeUtils;
+import org.apache.flink.api.common.functions.ReduceFunction;
+import org.apache.flink.api.java.tuple.Tuple5;
+
+import java.util.Map;
+
+
+/**
+ * @author 94976
+ */
+public class ConnReduceFunction implements ReduceFunction<Map<String, Object>> {
+
+    private static final Log logger = LogFactory.get();
+
+    //
+//    public void process(Tuple2<String, String> keys, Context context, Iterable<Map<String, Object>> elements, Collector<Map<String, Object>> out) {
+//        try {
+//            Tuple5<Long, Long, Long, Long, Long> values = sum(elements);
+//            if (values != null) {
+//                Map<String, Object> result = new HashMap<>();
+//                result.put("start_time", values.f0);
+//                result.put("end_time", values.f1);
+//                result.put("src_ip", keys.f0);
+//                result.put("dst_ip", keys.f1);
+//                result.put("sessions", values.f2);
+//                result.put("packets", values.f3);
+//                result.put("bytes", values.f4);
+//                out.collect(result);
+//                logger.debug("获取中间聚合结果：{}", result.toString());
+//            }
+//        } catch (Exception e) {
+//            logger.error("获取中间聚合结果失败,middleResult: {}", e);
+//        }
+//    }
+//
+    private Tuple5<Long, Long, Long, Long, Long> sum(Map<String, Object> map1, Map<String, Object> map2) {
+
+        try {
+            long sessions = 0L;
+            long packets = 0L;
+            long bytes = 0L;
+            long startTime = DateUtil.currentSeconds();
+            long endTime = DateUtil.currentSeconds();
+
+            long connStartTime1 = Convert.toLong(map1.get("conn_start_time"));
+            long connStartTime2 = Convert.toLong(map2.get("conn_start_time"));
+            if (connStartTime1 > 0 && connStartTime2 > 0) {
+                sessions++;
+                packets = TypeUtils.castToLong(map1.get("total_cs_pkts")) + TypeUtils.castToLong(map1.get("total_sc_pkts")) +
+                        TypeUtils.castToLong(map2.get("total_cs_pkts")) + TypeUtils.castToLong(map2.get("total_sc_pkts"));
+
+                bytes = bytes + TypeUtils.castToLong(map1.get("total_cs_bytes")) + TypeUtils.castToLong(map1.get("total_sc_bytes")) +
+                        TypeUtils.castToLong(map2.get("total_cs_bytes")) + TypeUtils.castToLong(map2.get("total_sc_bytes"));
+
+                startTime = connStartTime1 < connStartTime2 ? connStartTime1 : connStartTime2;
+                endTime = connStartTime2 < connStartTime1 ? connStartTime1 : connStartTime2;
+
+                packets = packets > Long.MAX_VALUE ? 0 : packets;
+                bytes = bytes > Long.MAX_VALUE ? 0 : bytes;
+
+            }
+
+        } catch (Exception e) {
+            logger.error("聚合中间结果集失败 {}", e);
+        }
+        return null;
+    }
+
+    @Override
+    public Map<String, Object> reduce(Map<String, Object> map1, Map<String, Object> map2) throws Exception {
+
+        return null;
+    }
+}