Initial commit 单线程成功，并行报错

src/main/java/cn/mesalab/utils/ConcurrentUtils.java

@@ -0,0 +1,154 @@
+package cn.mesalab.utils;
+
+import com.google.common.collect.Lists;
+import org.springframework.core.task.AsyncTaskExecutor;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+
+/**
+ * 〈一句话功能简述〉:
+ * 〈并发工具〉
+ *
+ * @create 2020-05-28
+ * @since 1.0.0
+ */
+public class ConcurrentUtils {
+    /**
+     * @Description
+     * 并发执行工具类
+     *
+     * @param batchProcessNum 并发执行的个数
+     * @param executor 执行需要的线程池
+     * @param keys 批量执行的key
+     * @param kvConvert 从key生成value
+     * @param valueProcessor 对value的后置处理
+     * @param exceptionHook 通过key获取value时发生异常自定义处理
+     * @return void
+     **/
+    public static <K, V> void concurrentProcess(int batchProcessNum, AsyncTaskExecutor executor, K[] keys,
+                                                Function<K, V> kvConvert,
+                                                ValueProcessor<K, V> valueProcessor,
+                                                ExceptionHook<K> exceptionHook ){
+        AtomicInteger index = new AtomicInteger(1);
+        List<Future> futureList = Lists.newArrayListWithExpectedSize(batchProcessNum);
+        final int length = keys.length;
+        System.out.println("total jobs size:{}" + length);
+        for (int i = 0; i < batchProcessNum; i++) {
+            int finalI = i;
+            futureList.add(executor.submit(() -> {
+                System.out.println("batch process start thread num:{}"+ finalI);
+                int currentIndex;
+                while (length >= (currentIndex = index.getAndIncrement())) {
+                    System.out.println("current job index:"+currentIndex+" of "+ length);
+                    K key = keys[currentIndex-1];
+                    try {
+                        valueProcessor.process(key, kvConvert.apply(key));
+                    } catch (Exception e) {
+                        exceptionHook.process(key, e);
+                    }
+                }
+                System.out.println("batch process end thread num:{}"+finalI);
+            }));
+        }
+        waitFutureFinished(futureList);
+    }
+
+    public static <K, V> void concurrentProcess(int batchProcessNum, AsyncTaskExecutor executor, K[] keys,
+                                                Function<K, V> kvConvert,
+                                                ValueProcessor<K, V> valueProcessor) {
+        concurrentProcess(batchProcessNum, executor, keys, kvConvert, valueProcessor,
+                (key,e) -> System.out.println("通过key:"+ key +"  获取value异常, e:"+e));
+    }
+
+    /**
+     * @Description
+     * 并发执行工具类
+     *
+     * @param batchProcessNum 并发执行的个数
+     * @param executor 执行需要的线程池
+     * @param keys 批量执行的key
+     * @param kvConvert 从key生成value
+     * @return Collection<V> 返回value的集合
+     **/
+    public static <K, V> Collection<V> concurrentGet(int batchProcessNum, AsyncTaskExecutor executor, K[] keys,
+                                                     Function<K, V> kvConvert) {
+        List<V> rt = Lists.newArrayListWithCapacity(keys.length);
+        concurrentProcess(batchProcessNum, executor, keys, kvConvert, (k, v) -> {
+            if (v == null) {
+                System.out.println("key:{} apply value is null");
+                return;
+            }
+            rt.add(v);
+        });
+        return rt;
+    }
+
+    public static void waitFutureFinished(List<Future> unfinishedFuture, boolean ignoreExcetion) {
+        boolean interrupt = false;
+        while (!unfinishedFuture.isEmpty()) {
+            Iterator<Future> iterator = unfinishedFuture.iterator();
+            while (iterator.hasNext()) {
+                Future next = iterator.next();
+                if (next.isDone()) {
+                    try {
+                        next.get();
+                    } catch (InterruptedException | ExecutionException e) {
+                        System.out.println("执行异常， e:"+e);
+                        if (!ignoreExcetion) {
+                            interrupt = true;
+                            break;
+                        }
+                    }
+                    iterator.remove();
+                }
+            }
+            if (interrupt) {
+                break;
+            }
+            sleep();
+        }
+        if (interrupt) {
+            Iterator<Future> iterator = unfinishedFuture.iterator();
+            while (iterator.hasNext()) {
+                Future next = iterator.next();
+                if (next.isDone()) {
+                    next.cancel(true);
+                }
+            }
+            throw new RuntimeException("任务异常终止");
+        }
+    }
+
+    public static void waitFutureFinished(List<Future> unfinishedFuture) {
+        waitFutureFinished(unfinishedFuture, false);
+    }
+
+    public static void sleep() {
+        sleep(5000);
+    }
+
+    public static void sleep(long millis) {
+        try {
+            Thread.sleep(millis);
+        } catch (InterruptedException e) {
+            System.out.println("sleep error, e:" + e);
+        }
+    }
+
+    @FunctionalInterface
+    public interface ValueProcessor<K, V> {
+        void process(K key, V value);
+    }
+
+    @FunctionalInterface
+    public interface ExceptionHook<K>{
+        void process(K key, Exception e);
+    }
+}
+

src/main/java/cn/mesalab/utils/ConfigUtils.java

@@ -0,0 +1,45 @@
+package cn.mesalab.utils;
+
+
+import org.apache.log4j.Logger;
+
+import java.util.Properties;
+
+public class ConfigUtils {
+    private static final Logger LOG = Logger.getLogger(ConfigUtils.class);
+    private static Properties propCommon = new Properties();
+
+    public static String getStringProperty(String key) {
+            return propCommon.getProperty(key);
+    }
+    public static Float getFloatProperty(String key) {
+        return Float.parseFloat(propCommon.getProperty(key));
+    }
+
+
+    public static Integer getIntProperty(String key) {
+            return Integer.parseInt(propCommon.getProperty(key));
+    }
+
+    public static Long getLongProperty(String key) {
+        return Long.parseLong(propCommon.getProperty(key));
+    }
+
+    public static Double getDoubleProperty(String key) {
+        return Double.parseDouble(propCommon.getProperty(key));
+    }
+
+    public static Boolean getBooleanProperty(String key) {
+        return "true".equals(propCommon.getProperty(key).toLowerCase().trim());
+    }
+
+    static {
+        try {
+            propCommon.load(ConfigUtils.class.getClassLoader().getResourceAsStream("application.properties"));
+
+        } catch (Exception e) {
+            propCommon = null;
+            LOG.error("配置加载失败");
+        }
+    }
+}

src/main/java/cn/mesalab/utils/DruidUtils.java

@@ -0,0 +1,77 @@
+package cn.mesalab.utils;
+
+import cn.mesalab.config.ApplicationConfig;
+import cn.mesalab.service.BaselineGeneration;
+import org.apache.calcite.avatica.AvaticaConnection;
+import org.apache.calcite.avatica.AvaticaStatement;
+import org.jfree.util.Log;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Properties;
+
+/**
+ * @author yjy
+ * @version 1.0
+ * @date 2021/7/23 4:50 下午
+ */
+public class DruidUtils {
+    private static final Logger LOG = LoggerFactory.getLogger(DruidUtils.class);
+
+    private static DruidUtils druidUtils;
+    private static final String DRUID_URL = ApplicationConfig.DRUID_URL;
+    private static AvaticaConnection connection;
+    private static AvaticaStatement statement;
+
+    public DruidUtils() throws SQLException {
+        Properties properties = new Properties();
+        connection = (AvaticaConnection) DriverManager.getConnection(DRUID_URL, properties);
+        statement = connection.createStatement();
+    }
+
+    public static DruidUtils getInstance() {
+        if (connection==null){
+            try{
+                druidUtils = new DruidUtils();
+            } catch (SQLException e){
+                LOG.error("Druid 建立连接失败！");
+                e.printStackTrace();
+            }
+        }
+        return druidUtils;
+    }
+
+    private static AvaticaConnection getConn() throws SQLException {
+        Properties properties = new Properties();
+        AvaticaConnection connection = (AvaticaConnection) DriverManager.getConnection(DRUID_URL, properties);
+        return connection;
+    }
+
+    public void closeConnection() {
+        if(connection != null){
+            try{
+                connection.close();
+            } catch (SQLException e){
+                LOG.error("Druid 关闭连接失败！");
+                e.printStackTrace();
+            }
+        }
+    }
+
+    public ResultSet executeQuery (String sql) throws SQLException{
+        System.out.println("executeQuery:"+sql);
+        ResultSet resultSet = statement.executeQuery(sql);
+        return resultSet;
+    }
+
+    public AvaticaConnection getConnection() {
+        return connection;
+    }
+
+    public AvaticaStatement getStatement() {
+        return statement;
+    }
+}

src/main/java/cn/mesalab/utils/HbaseUtils.java

@@ -0,0 +1 @@
+package cn.mesalab.utils;

src/main/java/cn/mesalab/utils/SeriesUtils.java

@@ -0,0 +1,214 @@
+package cn.mesalab.utils;
+
+import cn.mesalab.config.ApplicationConfig;
+import cn.mesalab.dao.DruidData;
+import cn.mesalab.service.BaselineGeneration;
+import com.google.common.collect.Lists;
+import org.jfree.util.Log;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.lang.reflect.Array;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+import java.util.stream.Stream;
+
+
+/**
+ * @author joy
+ */
+public class SeriesUtils {
+    private static final Logger LOG = LoggerFactory.getLogger(SeriesUtils.class);
+
+    private static DruidData druidData = new DruidData();
+
+    public static List<Map<String, Object>> readCsvToList(String filePath) {
+        List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
+
+        String line;
+        try (BufferedReader br = new BufferedReader(new FileReader(filePath))) {
+            br.readLine();
+            while ((line = br.readLine()) != null) {
+                List<String> column = Arrays.asList(line.split(","));
+                // 保存记录中的每个<字段名-字段值>
+                Map<String, Object> rowData = new HashMap<String, Object>();
+                rowData.put("__time", column.get(0));
+                rowData.put(ApplicationConfig.BASELINE_METRIC_TYPE, Integer.valueOf(column.get(1)));
+
+                list.add(rowData);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return list;
+    }
+
+
+    /**
+     * 时序数据补齐
+     */
+    public static List<Map<String, Object>> complementSeries(List<Map<String, Object>> originSeries){
+        LocalDateTime startTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(druidData.getTimeLimit()._2), TimeZone
+                .getDefault().toZoneId());
+        LocalDateTime endTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(druidData.getTimeLimit()._1), TimeZone
+                .getDefault().toZoneId());
+        List<String> dateList = completionDate(startTime, endTime);
+
+        // 补全后的结果
+        List<Map<String, Object>> result = new ArrayList<>();
+        boolean dbDateExist = false;
+        for (String date : dateList) {
+            //table为数据库查询出来的对象列表，结构为List<Map<String, Object>>
+            for (Map<String, Object> row : originSeries) {
+                if (row.get(ApplicationConfig.DRUID_RECVTIME_COLUMN_NAME).toString().substring(0,19).equals(date)) {
+                    //集合已包含该日期
+                    dbDateExist = true;
+                    result.add(row);
+                    break;
+                }
+            }
+            //添加补全的数据到最后结果列表
+            if (!dbDateExist) {
+                Map<String, Object> temp = new HashMap<>(2);
+                temp.put(ApplicationConfig.DRUID_RECVTIME_COLUMN_NAME, date);
+                temp.put(ApplicationConfig.BASELINE_METRIC_TYPE, 0);
+                result.add(temp);
+            }
+            dbDateExist = false;
+        }
+
+        return result;
+    }
+
+    private static List<String> completionDate(LocalDateTime startTime, LocalDateTime endTime) {
+        //日期格式化
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern(ApplicationConfig.TIME_FORMAT);
+        List<String> timeList = new ArrayList<>();
+        //遍历给定的日期期间的每一天
+        for (int i = 0; !Duration.between(startTime.plusMinutes(i+1), endTime).isNegative(); i+= ApplicationConfig.HISTORICAL_GRAD) {
+            //添加日期
+            timeList.add(startTime.plusMinutes(i).format(formatter));
+        }
+        return timeList;
+    }
+
+    /**
+     * 判断是否存在以天为单位的周期特征
+     * @param historicalSeries
+     * @return
+     */
+    public static Boolean isPeriod(List<Integer> historicalSeries){
+        Boolean result = true;
+        List<List<Integer>> partitions = Lists.partition(historicalSeries, 24*60/ApplicationConfig.HISTORICAL_GRAD);
+        List<Integer> aggregatedPart = Arrays.asList();
+        try{
+            aggregatedPart = columnAverage(partitions.subList(0, ApplicationConfig.READ_HISTORICAL_DAYS-1));
+        } catch (IndexOutOfBoundsException e){
+            Log.error("历史");
+        }
+
+        // Pearson corrcoef
+        double pearsonCorrelationScore = getPearsonCorrelationScore(aggregatedPart.stream().mapToInt(Integer::valueOf).toArray(),
+                partitions.get(partitions.size() - 1).stream().mapToInt(Integer::valueOf).toArray());
+
+        if (pearsonCorrelationScore < ApplicationConfig.BASELINE_PERIOD_CORR_THRE){
+            result=false;
+        }
+        return result;
+    }
+
+    public static double getPearsonCorrelationScore(int[] xData, int[] yData) {
+        if (xData.length != yData.length) {
+            Log.error("Pearson CorrelationScore 数组长度不相等!");
+        }
+        int xMeans;
+        int yMeans;
+        double numerator = 0;
+        double denominator = 0;
+
+        double result = 0;
+        // 拿到两个数据的平均值
+        xMeans = (int) getMeans(xData);
+        yMeans = (int) getMeans(yData);
+        // 计算皮尔逊系数的分子
+        numerator = generateNumerator(xData, xMeans, yData, yMeans);
+        // 计算皮尔逊系数的分母
+        denominator = generateDenomiator(xData, xMeans, yData, yMeans);
+        // 计算皮尔逊系数
+        if(denominator>0) {
+            result = numerator / denominator;
+        }
+        //FOR TEST
+        System.out.println(result);
+        return result;
+    }
+
+    private static int generateNumerator(int[] xData, int xMeans, int[] yData, int yMeans) {
+        int numerator = 0;
+        for (int i = 0; i < xData.length; i++) {
+            numerator += (xData[i] - xMeans) * (yData[i] - yMeans);
+        }
+        return numerator;
+    }
+
+    private static double generateDenomiator(int[] xData, int xMeans, int[] yData, int yMeans) {
+        double xSum = 0.0;
+        for (int i = 0; i < xData.length; i++) {
+            xSum += (xData[i] - xMeans) * (xData[i] - xMeans);
+        }
+        double ySum = 0.0;
+        for (int i = 0; i < yData.length; i++) {
+            ySum += (yData[i] - yMeans) * (yData[i] - yMeans);
+        }
+        return Math.sqrt(xSum) * Math.sqrt(ySum);
+    }
+
+    private static double getMeans(int[] datas) {
+        double sum = 0.0;
+        for (int i = 0; i < datas.length; i++) {
+            sum += datas[i];
+        }
+        return sum / datas.length;
+    }
+
+    public static List<Integer> columnAverage(List<List<Integer>> list){
+        ArrayList<Integer> averages = new ArrayList<>();
+        for(int i=0; i<list.get(0).size(); i++){
+            int columnSum = 0;
+            for(int j = 0; j< list.size(); j++){
+                columnSum += list.get(j).get(i);
+            }
+            averages.add(columnSum / list.size());
+        }
+        return averages;
+    }
+
+    public static int percentile(List<Integer> latencies, double percentile) {
+        Collections.sort(latencies);
+        int index = (int) Math.ceil(percentile * latencies.size());
+        return latencies.get(index-1);
+    }
+
+    public static void main(String[] args) {
+        List<Integer> test = Arrays.asList(
+                1,2,3,4,5,
+                1,2,3,4,5,
+                1,2,3,4,5,
+                1,2,3,4,5,
+                1,2,3,4,5,
+                1,2,3,4,5,
+                1,2,3,4,5);
+        System.out.println(columnAverage(Lists.partition(test, 5)));
+
+
+
+    }
+
+
+}
+