gfwleak/zhanghongqing-knowledge-log
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增入库批量操作，clickhouse负载均衡调用

Browse Source
This commit is contained in:
zhanghongqing
2022-07-11 10:05:14 +08:00
parent e9c92fb286
commit c1b70a6da0
32 changed files with 882 additions and 826 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
src/main/java/com/zdjizhi/utils/ck/ClickhouseSink.java
View File

@@ -1,5 +1,6 @@
package com.zdjizhi.utils.ck;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
@@ -18,7 +19,7 @@ import java.util.stream.Collectors;
import static com.zdjizhi.common.FlowWriteConfig.*;
public class ClickhouseSink extends RichSinkFunction<Map<String,Object>> {
public class ClickhouseSink extends RichSinkFunction<List<Map<String, Object>>> {
private static final Log log = LogFactory.get();
@@ -27,7 +28,15 @@ public class ClickhouseSink extends RichSinkFunction<Map<String,Object>> {
public String sink;
static {
try {
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
connection = DriverManager.getConnection("jdbc:clickhouse://" + CK_HOSTS + "/" + CK_DATABASE, CK_USERNAME, CK_PIN);
// BalancedClickhouseDataSource dataSource = new BalancedClickhouseDataSource("jdbc:clickhouse://node01:8123,node02:8123,node03:8123/default", props);
// connection = dataSource.getConnection();
log.info("get clickhouse connection success");
} catch (ClassNotFoundException | SQLException e) {
log.error("clickhouse connection error ,{}", e);
}
}
public ClickhouseSink(String sink) {
@@ -43,21 +52,13 @@ public class ClickhouseSink extends RichSinkFunction<Map<String,Object>> {
}
@Override
public void invoke(Map<String,Object> log, Context context) throws Exception {
executeInsert(log, getSink());
public void invoke(List<Map<String, Object>> logs, Context context) throws Exception {
executeInsert(logs, getSink());
}
@Override
public void open(Configuration parameters) throws Exception {
try {
Class.forName("ru.yandex.clickhouse.ClickHouseDriver");
connection = DriverManager.getConnection("jdbc:clickhouse://" + CK_HOSTS + "/" + CK_DATABASE, CK_USERNAME, CK_PIN);
// BalancedClickhouseDataSource dataSource = new BalancedClickhouseDataSource("jdbc:clickhouse://node01:8123,node02:8123,node03:8123/default", props);
// connection = dataSource.getConnection();
log.info("get clickhouse connection success");
} catch (ClassNotFoundException | SQLException e) {
log.error("clickhouse connection error ,{}", e);
}
}
@Override
@@ -70,50 +71,52 @@ public class ClickhouseSink extends RichSinkFunction<Map<String,Object>> {
}
}
public void executeInsert(Map<String, Object> data, String tableName) {
public void executeInsert(List<Map<String, Object>> data, String tableName) {
try {
int count = 1;
List<String> keys = new LinkedList<>(data.keySet());
List<String> keys = new LinkedList<>(data.get(0).keySet());
connection.setAutoCommit(false);
preparedStatement = connection.prepareStatement(preparedSql(keys, tableName));
List<Object> values = new LinkedList<>(data.values());
for (int i = 1; i <= values.size(); i++) {
Object val = values.get(i - 1);
if (val instanceof Long) {
preparedStatement.setLong((i), Long.valueOf(StrUtil.toString(val)));
} else if (val instanceof Integer) {
preparedStatement.setInt((i), Integer.valueOf(StrUtil.toString(val)));
} else if (val instanceof Boolean) {
preparedStatement.setBoolean((i), Boolean.valueOf(StrUtil.toString(val)));
} else {
preparedStatement.setString((i), StrUtil.toString(val));
int count = 0;
for (Map<String, Object> map : data) {
List<Object> values = new LinkedList<>(map.values());
for (int i = 1; i <= values.size(); i++) {
Object val = values.get(i - 1);
if (val instanceof Long) {
preparedStatement.setLong((i), Long.valueOf(StrUtil.toString(val)));
} else if (val instanceof Integer) {
preparedStatement.setInt((i), Integer.valueOf(StrUtil.toString(val)));
} else if (val instanceof Boolean) {
preparedStatement.setBoolean((i), Boolean.valueOf(StrUtil.toString(val)));
} else {
preparedStatement.setString((i), StrUtil.toString(val));
}
}
}
preparedStatement.addBatch();
count = count + 1;
try {
preparedStatement.addBatch();
count++;
//1w提交一次
if (count % 10000 == 0) {
if (count % SINK_BATCH == 0) {
preparedStatement.executeBatch();
connection.commit();
preparedStatement.clearBatch();
count = 1;
count = 0;
}
}
if (count > 0) {
preparedStatement.executeBatch();
connection.commit();
} catch (Exception ee) {
log.error("数据插入clickhouse 报错：", ee);
}
} catch (Exception ex) {
log.error("ClickhouseSink插入报错", ex);
} finally {
IoUtil.close(preparedStatement);
IoUtil.close(connection);
}
}
public String preparedSql(List<String> fields, String tableName) {
public static String preparedSql(List<String> fields, String tableName) {
String placeholders = fields.stream()
.filter(Objects::nonNull)
@@ -124,7 +127,7 @@ public class ClickhouseSink extends RichSinkFunction<Map<String,Object>> {
.collect(Collectors.joining(", "));
String sql = StrUtil.concat(true, "INSERT INTO ", CK_DATABASE, ".", tableName,
"(", columns, ") VALUES (", placeholders, ")");
log.info(sql);
log.debug(sql);
return sql;
}