多线程批量进行baseline生成和写入
This commit is contained in:
yinjiangyi
@@ -1,154 +0,0 @@
|
||||
package cn.mesalab.utils;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import org.springframework.core.task.AsyncTaskExecutor;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Function;
|
||||
|
||||
/**
|
||||
* 〈一句话功能简述〉:
|
||||
* 〈并发工具〉
|
||||
*
|
||||
* @create 2020-05-28
|
||||
* @since 1.0.0
|
||||
*/
|
||||
public class ConcurrentUtils {
|
||||
/**
|
||||
* @Description
|
||||
* 并发执行工具类
|
||||
*
|
||||
* @param batchProcessNum 并发执行的个数
|
||||
* @param executor 执行需要的线程池
|
||||
* @param keys 批量执行的key
|
||||
* @param kvConvert 从key生成value
|
||||
* @param valueProcessor 对value的后置处理
|
||||
* @param exceptionHook 通过key获取value时发生异常自定义处理
|
||||
* @return void
|
||||
**/
|
||||
public static <K, V> void concurrentProcess(int batchProcessNum, AsyncTaskExecutor executor, K[] keys,
|
||||
Function<K, V> kvConvert,
|
||||
ValueProcessor<K, V> valueProcessor,
|
||||
ExceptionHook<K> exceptionHook ){
|
||||
AtomicInteger index = new AtomicInteger(1);
|
||||
List<Future> futureList = Lists.newArrayListWithExpectedSize(batchProcessNum);
|
||||
final int length = keys.length;
|
||||
System.out.println("total jobs size:{}" + length);
|
||||
for (int i = 0; i < batchProcessNum; i++) {
|
||||
int finalI = i;
|
||||
futureList.add(executor.submit(() -> {
|
||||
System.out.println("batch process start thread num:{}"+ finalI);
|
||||
int currentIndex;
|
||||
while (length >= (currentIndex = index.getAndIncrement())) {
|
||||
System.out.println("current job index:"+currentIndex+" of "+ length);
|
||||
K key = keys[currentIndex-1];
|
||||
try {
|
||||
valueProcessor.process(key, kvConvert.apply(key));
|
||||
} catch (Exception e) {
|
||||
exceptionHook.process(key, e);
|
||||
}
|
||||
}
|
||||
System.out.println("batch process end thread num:{}"+finalI);
|
||||
}));
|
||||
}
|
||||
waitFutureFinished(futureList);
|
||||
}
|
||||
|
||||
public static <K, V> void concurrentProcess(int batchProcessNum, AsyncTaskExecutor executor, K[] keys,
|
||||
Function<K, V> kvConvert,
|
||||
ValueProcessor<K, V> valueProcessor) {
|
||||
concurrentProcess(batchProcessNum, executor, keys, kvConvert, valueProcessor,
|
||||
(key,e) -> System.out.println("通过key:"+ key +" 获取value异常, e:"+e));
|
||||
}
|
||||
|
||||
/**
|
||||
* @Description
|
||||
* 并发执行工具类
|
||||
*
|
||||
* @param batchProcessNum 并发执行的个数
|
||||
* @param executor 执行需要的线程池
|
||||
* @param keys 批量执行的key
|
||||
* @param kvConvert 从key生成value
|
||||
* @return Collection<V> 返回value的集合
|
||||
**/
|
||||
public static <K, V> Collection<V> concurrentGet(int batchProcessNum, AsyncTaskExecutor executor, K[] keys,
|
||||
Function<K, V> kvConvert) {
|
||||
List<V> rt = Lists.newArrayListWithCapacity(keys.length);
|
||||
concurrentProcess(batchProcessNum, executor, keys, kvConvert, (k, v) -> {
|
||||
if (v == null) {
|
||||
System.out.println("key:{} apply value is null");
|
||||
return;
|
||||
}
|
||||
rt.add(v);
|
||||
});
|
||||
return rt;
|
||||
}
|
||||
|
||||
public static void waitFutureFinished(List<Future> unfinishedFuture, boolean ignoreExcetion) {
|
||||
boolean interrupt = false;
|
||||
while (!unfinishedFuture.isEmpty()) {
|
||||
Iterator<Future> iterator = unfinishedFuture.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Future next = iterator.next();
|
||||
if (next.isDone()) {
|
||||
try {
|
||||
next.get();
|
||||
} catch (InterruptedException | ExecutionException e) {
|
||||
System.out.println("执行异常， e:"+e);
|
||||
if (!ignoreExcetion) {
|
||||
interrupt = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
if (interrupt) {
|
||||
break;
|
||||
}
|
||||
sleep();
|
||||
}
|
||||
if (interrupt) {
|
||||
Iterator<Future> iterator = unfinishedFuture.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Future next = iterator.next();
|
||||
if (next.isDone()) {
|
||||
next.cancel(true);
|
||||
}
|
||||
}
|
||||
throw new RuntimeException("任务异常终止");
|
||||
}
|
||||
}
|
||||
|
||||
public static void waitFutureFinished(List<Future> unfinishedFuture) {
|
||||
waitFutureFinished(unfinishedFuture, false);
|
||||
}
|
||||
|
||||
public static void sleep() {
|
||||
sleep(5000);
|
||||
}
|
||||
|
||||
public static void sleep(long millis) {
|
||||
try {
|
||||
Thread.sleep(millis);
|
||||
} catch (InterruptedException e) {
|
||||
System.out.println("sleep error, e:" + e);
|
||||
}
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ValueProcessor<K, V> {
|
||||
void process(K key, V value);
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
public interface ExceptionHook<K>{
|
||||
void process(K key, Exception e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
package cn.mesalab.utils;
|
||||
|
||||
import cn.mesalab.config.ApplicationConfig;
|
||||
import cn.mesalab.service.BaselineGeneration;
|
||||
import org.apache.calcite.avatica.AvaticaConnection;
|
||||
import org.apache.calcite.avatica.AvaticaStatement;
|
||||
import org.jfree.util.Log;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.hadoop.hbase.client.Table;
|
||||
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
@@ -19,59 +16,39 @@ import java.util.Properties;
|
||||
* @date 2021/7/23 4:50 下午
|
||||
*/
|
||||
public class DruidUtils {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(DruidUtils.class);
|
||||
private static ThreadLocal<AvaticaConnection> threadLocal = new ThreadLocal<AvaticaConnection>();
|
||||
|
||||
private static DruidUtils druidUtils;
|
||||
private static final String DRUID_URL = ApplicationConfig.DRUID_URL;
|
||||
private static AvaticaConnection connection;
|
||||
private static AvaticaStatement statement;
|
||||
|
||||
public DruidUtils() throws SQLException {
|
||||
Properties properties = new Properties();
|
||||
connection = (AvaticaConnection) DriverManager.getConnection(DRUID_URL, properties);
|
||||
statement = connection.createStatement();
|
||||
}
|
||||
|
||||
public static DruidUtils getInstance() {
|
||||
if (connection==null){
|
||||
try{
|
||||
druidUtils = new DruidUtils();
|
||||
} catch (SQLException e){
|
||||
LOG.error("Druid 建立连接失败！");
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
return druidUtils;
|
||||
}
|
||||
|
||||
private static AvaticaConnection getConn() throws SQLException {
|
||||
/**
|
||||
* 打开连接
|
||||
* @throws SQLException
|
||||
*/
|
||||
public static AvaticaConnection getConn() throws SQLException {
|
||||
Properties properties = new Properties();
|
||||
AvaticaConnection connection = (AvaticaConnection) DriverManager.getConnection(DRUID_URL, properties);
|
||||
threadLocal.set(connection);
|
||||
return connection;
|
||||
}
|
||||
|
||||
public void closeConnection() {
|
||||
if(connection != null){
|
||||
try{
|
||||
connection.close();
|
||||
} catch (SQLException e){
|
||||
LOG.error("Druid 关闭连接失败！");
|
||||
e.printStackTrace();
|
||||
}
|
||||
/**
|
||||
* 关闭连接
|
||||
*/
|
||||
public static void closeConnection() throws SQLException{
|
||||
AvaticaConnection conn = threadLocal.get();
|
||||
if(conn != null){
|
||||
conn.close();
|
||||
threadLocal.remove();
|
||||
}
|
||||
}
|
||||
|
||||
public ResultSet executeQuery (String sql) throws SQLException{
|
||||
System.out.println("executeQuery:"+sql);
|
||||
/**
|
||||
* 根据sql查询结果
|
||||
*/
|
||||
public static ResultSet executeQuery (AvaticaConnection connection, String sql) throws SQLException{
|
||||
AvaticaStatement statement = connection.createStatement();
|
||||
ResultSet resultSet = statement.executeQuery(sql);
|
||||
return resultSet;
|
||||
}
|
||||
|
||||
public AvaticaConnection getConnection() {
|
||||
return connection;
|
||||
}
|
||||
|
||||
public AvaticaStatement getStatement() {
|
||||
return statement;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,8 +143,6 @@ public class SeriesUtils {
|
||||
if(denominator>0) {
|
||||
result = numerator / denominator;
|
||||
}
|
||||
//FOR TEST
|
||||
System.out.println(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user