gfwleak/nezha-nz-transfer
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: NEZ-2203 21.07 -> 22.02 版本升级数据处理

Browse Source
This commit is contained in:
shizhendong
2022-09-28 18:16:06 +08:00
parent 02be6ef27b
commit 21e629ec54
11 changed files with 393 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/net/geedge/dao/TopoDao.java Normal file
View File

@@ -0,0 +1,10 @@
package net.geedge.dao;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import net.geedge.entity.Topo;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface TopoDao extends BaseMapper<Topo> {
}

10
src/main/java/net/geedge/dao/VisualChartDao.java Normal file
View File

@@ -0,0 +1,10 @@
package net.geedge.dao;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import net.geedge.entity.VisualChart;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface VisualChartDao extends BaseMapper<VisualChart> {
}

23
src/main/java/net/geedge/entity/Topo.java Normal file
View File

@@ -0,0 +1,23 @@
package net.geedge.entity;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("project_topo")
public class Topo {
@TableId
private Integer id;
private Integer projectId;
private String topo;
private Date updateAt;
private Integer updateBy;
}

51
src/main/java/net/geedge/entity/VisualChart.java Normal file
View File

@@ -0,0 +1,51 @@
package net.geedge.entity;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import java.util.Date;
@Data
@TableName("visual_chart")
public class VisualChart {
private Integer id;
private String name;
private Integer panelId;
private Integer groupId;
private Double span;
private Double height;
private Integer updateBy;
private Date updateAt;
private String type;
private Integer unit;
private Integer weight;
private String param;
private Integer pid;
private Integer buildIn;
private String remark;
private String seq;
private Double x;
private Double y;
private Integer varType;
private Integer varId;
}

22
src/main/java/net/geedge/handler/AfterHandler.java
View File

@@ -1,12 +1,13 @@
package net.geedge.handler;
import cn.hutool.log.Log;
import net.geedge.dao.SqlDao;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.context.annotation.DependsOn;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import cn.hutool.log.Log;
import java.util.List;
/**
* 此类用于数据同步完成后 清除无用表
@@ -20,14 +21,25 @@ public class AfterHandler implements CommandLineRunner {
private Log log = Log.get();
@Autowired
private SqlHandler sqlHandler;
private SqlDao sqlDao;
@Override
public void run(String... args) throws Exception {
sqlHandler.removeColumn();
String dropColumnSql = "ALTER TABLE `monitor_module` DROP COLUMN `type`;"
+ "ALTER TABLE `monitor_endpoint` DROP COLUMN `configs`;"
+ "ALTER TABLE `monitor_endpoint` DROP COLUMN `hash`;"
+ "ALTER TABLE `monitor_endpoint` DROP COLUMN `enabled`;";
sqlDao.execute(dropColumnSql);
log.info("remove column successful");
sqlHandler.removeCopy();
StringBuilder sb = new StringBuilder();
List<String> sqls = sqlDao.allRemoveCopyTables();
for (String sql : sqls) {
sb.append(sql);
}
sqlDao.execute(sb.toString());
log.info("remove table successful");
log.info("-----------------transfer data success-----------------------");

268
src/main/java/net/geedge/handler/SqlHandler.java
View File

@@ -1,17 +1,18 @@
package net.geedge.handler;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.date.StopWatch;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.log.Log;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONPath;
import net.geedge.dao.SqlDao;
import net.geedge.entity.*;
import net.geedge.service.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
@@ -19,26 +20,13 @@ import org.springframework.jdbc.datasource.init.ScriptException;
import org.springframework.jdbc.datasource.init.ScriptUtils;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSONObject;
import javax.sql.DataSource;
import java.io.IOException;
import java.sql.SQLException;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.log.Log;
import net.geedge.dao.SqlDao;
import net.geedge.entity.AlertRule;
import net.geedge.entity.MonitorEndpoint;
import net.geedge.entity.MonitorEndpointConfig;
import net.geedge.entity.MonitorModule;
import net.geedge.service.AlertRuleService;
import net.geedge.service.MonitorEndpointConfigService;
import net.geedge.service.MonitorEndpointService;
import net.geedge.service.MonitorModuleService;
/**
* 此类主要为执行各类sql语句
* @author admin
*
*/
@Service
public class SqlHandler {
@@ -50,9 +38,15 @@ public class SqlHandler {
@Autowired
private DataSource dataSource;
@Autowired
private TopoService topoService;
@Autowired
private AlertRuleService alertRuleService;
@Autowired
private VisualChartService visualChartService;
@Autowired
private MonitorModuleService monitorModuleService;
@@ -62,14 +56,6 @@ public class SqlHandler {
@Autowired
private MonitorEndpointConfigService monitorEndpointConfigService;
public void removeColumn() {
String dropColumnSql = "ALTER TABLE `monitor_module` DROP COLUMN `type`;"
+ "ALTER TABLE `monitor_endpoint` DROP COLUMN `configs`;"
+ "ALTER TABLE `monitor_endpoint` DROP COLUMN `hash`;"
+ "ALTER TABLE `monitor_endpoint` DROP COLUMN `enabled`;";
sqlDao.execute(dropColumnSql);
}
/**
* @throws IOException
* @throws SQLException
@@ -94,23 +80,40 @@ public class SqlHandler {
sqlDao.execute(sb.toString());
}
public void removeCopy() {
StringBuilder sb = new StringBuilder();
List<String> sqls = sqlDao.allRemoveCopyTables();
for (String sql : sqls) {
sb.append(sql);
}
sqlDao.execute(sb.toString());
}
/**
* 处理数据变化
*/
public void transferData() {
// 1. 处理 Module 数据变化
// 获取 之前数据
List<MonitorModule> moduleBackupList = monitorModuleService.getModuleBackupData();
Map<Integer, MonitorModule> moduleIdAndTypeMap = moduleBackupList.stream().collect(Collectors.toMap(MonitorModule::getId, Function.identity()));
Map<Integer, MonitorModule> backupModuleIdAndTypeMap = moduleBackupList.stream().collect(Collectors.toMap(MonitorModule::getId, Function.identity()));
// 1. Module
this.transferModuleData(moduleBackupList);
// 2. endpoint
this.transferEndpointData(backupModuleIdAndTypeMap);
// 3. alert rule
this.transferAlertRuleData();
// 4. chart valueMapping format
this.transferChartData();
// 5. project topo
this.transferProjectTopoData();
// 6. alert message data
this.transferAlertMessageData();
}
/**
* module 支持 logs config
*/
private void transferModuleData(List<MonitorModule> moduleBackupList) {
StopWatch sw = new StopWatch();
sw.start();
log.info("transfer module data begin...");
// 清除现有表数据
monitorModuleService.truncateData();
@@ -123,16 +126,39 @@ public class SqlHandler {
Map metricsConfigMap = configMapList.stream().filter(map -> {
String configType = (String) map.get("type");
return StrUtil.equals("metrics", configType);
}).findFirst().orElse(new HashMap());
}).findFirst().orElse(new HashMap(2));
metricsConfigMap.put("enable", 1);
Map configMap = (Map) metricsConfigMap.get("config");
configMap.put("protocol", module.getType());
Map basicAuth = (Map) metricsConfigMap.get("basic_auth");
if (MapUtil.isEmpty(basicAuth)) {
Map<Object, Object> m = MapUtil.builder().put("username", "").put("pin", "").build();
configMap.put("basic_auth", m);
}
String bearerToken = (String) metricsConfigMap.get("bearer_token");
if (StrUtil.isNullOrUndefined(bearerToken)) {
configMap.put("bearer_token", StrUtil.EMPTY);
}
module.setConfigs(JSONObject.toJSONString(configMapList));
saveModule.add(module);
}
monitorModuleService.saveBatch(saveModule);
// 2. 处理 endpoint
sw.stop();
log.info("transfer module data Run Time:{}", sw.prettyPrint());
}
/**
* endpoint 支持 logs config
*/
private void transferEndpointData(Map<Integer, MonitorModule> backupModuleIdAndTypeMap) {
StopWatch sw = new StopWatch();
sw.start();
log.info("transfer endpoint data begin...");
List<MonitorEndpoint> endpointBackupList = monitorEndpointService.getEndpointBackupData();
List<MonitorEndpointConfig> saveEndpointConfigList = new ArrayList<>();
@@ -140,7 +166,7 @@ public class SqlHandler {
// metrics
Map metricsConfigMap = JSONObject.parseObject(endpoint.getConfigs(), Map.class);
// protocol 为关联 module type 类型
MonitorModule monitorModule = moduleIdAndTypeMap.get(endpoint.getModuleId());
MonitorModule monitorModule = backupModuleIdAndTypeMap.get(endpoint.getModuleId());
metricsConfigMap.put("protocol", monitorModule.getType());
if (StrUtil.equals("snmp", monitorModule.getType())) {
// 补充 snmpCredentialsId
@@ -148,12 +174,23 @@ public class SqlHandler {
Map moduleMetricsMap = moduleConfigMapList.stream().filter(map -> {
String configType = (String) map.get("type");
return StrUtil.equals("metrics", configType);
}).findFirst().orElse(new HashMap());
}).findFirst().orElse(new HashMap(2));
Map moduleMetricsConfigMap = (Map) moduleMetricsMap.get("config");
metricsConfigMap.put("snmpCredentialsId", moduleMetricsConfigMap.get("snmpCredentialsId"));
metricsConfigMap.put("walk", moduleMetricsConfigMap.get("walk"));
}
Map basicAuth = (Map) metricsConfigMap.get("basic_auth");
if (MapUtil.isEmpty(basicAuth)) {
Map<Object, Object> m = MapUtil.builder().put("username", "").put("pin", "").build();
metricsConfigMap.put("basic_auth", m);
}
String bearerToken = (String) metricsConfigMap.get("bearer_token");
if (StrUtil.isNullOrUndefined(bearerToken)) {
metricsConfigMap.put("bearer_token", StrUtil.EMPTY);
}
MonitorEndpointConfig metricseEndpointConfig = new MonitorEndpointConfig();
metricseEndpointConfig.setEndpointId(endpoint.getId());
metricseEndpointConfig.setType("metrics");
@@ -176,7 +213,18 @@ public class SqlHandler {
}
monitorEndpointConfigService.saveBatch(saveEndpointConfigList);
// 3. 处理 alert rule 数据
sw.stop();
log.info("transfer endpoint data Run Time:{}", sw.prettyPrint());
}
/**
* rule 增加 notifyActive notifyExpired 属性
*/
private void transferAlertRuleData() {
StopWatch sw = new StopWatch();
sw.start();
log.info("transfer alert rule data begin...");
List<AlertRule> ruleList = alertRuleService.list();
for (AlertRule alertRule : ruleList) {
String receiver = alertRule.getReceiver();
@@ -189,5 +237,115 @@ public class SqlHandler {
}
}
alertRuleService.saveOrUpdateBatch(ruleList);
sw.stop();
log.info("transfer alert rule data Run Time:{}", sw.prettyPrint());
}
/**
* chart 重新定义 param valuemapping
*/
private void transferChartData() {
StopWatch sw = new StopWatch();
sw.start();
log.info("transfer chart data begin...");
List<VisualChart> chartList = visualChartService.list();
List<VisualChart> updateChartList = ListUtil.list(true);
for (VisualChart chart : chartList) {
String paramJsonStr = chart.getParam();
paramJsonStr = StrUtil.emptyToDefault(paramJsonStr, StrUtil.EMPTY_JSON);
// 配置不为空
List<Map> mappingList = (List<Map>) JSONPath.read(paramJsonStr, "valueMapping.mapping");
if (CollUtil.isNotEmpty(mappingList)) {
String type = (String) JSONPath.read(paramJsonStr, "valueMapping.type");
Iterator<Map> iterator = mappingList.iterator();
while (iterator.hasNext()) {
Map map = iterator.next();
Object value = map.get("value");
if (ObjectUtil.isEmpty(value)) {
iterator.remove();
} else {
// 补充前端所需必填属性
// text 转换成 value
map.put("type", StrUtil.equals("text", type) ? "value" : type);
Map colorMap = (Map) map.get("color");
if (MapUtil.isEmpty(colorMap)) {
Map<Object, Object> cMap = MapUtil.builder()
.put("bac", "#fff")
.put("text", "#000")
.build();
map.put("color", cMap);
}
}
}
// 不存在 valueMapping 配置则删掉 param 中配置项
JSONObject jsonObject = JSONObject.parseObject(paramJsonStr);
if (CollUtil.isEmpty(mappingList)) {
jsonObject.remove("valueMapping");
} else {
// 替换处理的内容
jsonObject.put("valueMapping", mappingList);
}
// 重新 set param
chart.setParam(JSONObject.toJSONString(jsonObject));
updateChartList.add(chart);
}
}
if (CollUtil.isNotEmpty(updateChartList)) {
visualChartService.updateBatchById(updateChartList);
}
sw.stop();
log.info("transfer chart data Run Time:{}", sw.prettyPrint());
}
/**
* remove the topo config of the node to empty
*/
private void transferProjectTopoData() {
StopWatch sw = new StopWatch();
sw.start();
log.info("transfer topo data begin...");
List<Topo> topoList = topoService.list();
for (Topo pojo : topoList) {
String topoStr = pojo.getTopo();
topoStr = StrUtil.emptyToDefault(topoStr, StrUtil.EMPTY_JSON);
List pens = (List) JSONPath.read(topoStr, "pens");
List nodes = (List) JSONPath.read(topoStr, "nodes");
if (CollUtil.isEmpty(pens) && CollUtil.isEmpty(nodes)) {
topoService.removeById(pojo.getId());
}
}
sw.stop();
log.info("transfer chart data Run Time:{}", sw.prettyPrint());
}
/**
* alert message last_at add data when it is empty
*/
private void transferAlertMessageData() {
StopWatch sw = new StopWatch();
sw.start();
log.info("transfer alert message data begin...");
// 对 alert_message_active alert_message_active history silence last_at 为空的数据进行赋值，防止 flyway 中的 SQL 报错
String removeColumnSql = "UPDATE alert_message_active SET end_at = start_at WHERE end_at IS NULL;"
+ "UPDATE alert_message_history SET end_at = start_at WHERE end_at IS NULL;"
+ "UPDATE alert_message_silence SET end_at = start_at WHERE end_at IS NULL;";
sqlDao.execute(removeColumnSql);
sw.stop();
log.info("transfer alert message data Run Time:{}", sw.prettyPrint());
}
}

8
src/main/java/net/geedge/service/TopoService.java Normal file
View File

@@ -0,0 +1,8 @@
package net.geedge.service;
import com.baomidou.mybatisplus.extension.service.IService;
import net.geedge.entity.Topo;
public interface TopoService extends IService<Topo> {
}

8
src/main/java/net/geedge/service/VisualChartService.java Normal file
View File

@@ -0,0 +1,8 @@
package net.geedge.service;
import com.baomidou.mybatisplus.extension.service.IService;
import net.geedge.entity.VisualChart;
public interface VisualChartService extends IService<VisualChart> {
}

12
src/main/java/net/geedge/service/impl/TopoServiceImpl.java Normal file
View File

@@ -0,0 +1,12 @@
package net.geedge.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import net.geedge.dao.TopoDao;
import net.geedge.entity.Topo;
import net.geedge.service.TopoService;
import org.springframework.stereotype.Service;
@Service
public class TopoServiceImpl extends ServiceImpl<TopoDao, Topo> implements TopoService {
}

12
src/main/java/net/geedge/service/impl/VisualChartServiceImpl.java Normal file
View File

@@ -0,0 +1,12 @@
package net.geedge.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import net.geedge.dao.VisualChartDao;
import net.geedge.entity.VisualChart;
import net.geedge.service.VisualChartService;
import org.springframework.stereotype.Service;
@Service
public class VisualChartServiceImpl extends ServiceImpl<VisualChartDao, VisualChart> implements VisualChartService {
}

1
src/main/resources/db/init.sql
View File

@@ -293,6 +293,7 @@ UPDATE `alert_notify_method` SET `name` = 'email', `file_path` = '', `account` =
update sys_user set pin_time = now();
UPDATE flyway_schema_history SET `checksum` = 963731206 WHERE version = "2021.05.14";
INSERT INTO `flyway_schema_history` VALUES (6, '2021.07.26', 'insert asset client config', 'SQL', 'V2021.07.26__insert asset client config.sql', 254186640, 'root', now(), 21, 1);
INSERT INTO `flyway_schema_history` VALUES (7, '2021.07.29', 'insert sys config loki query configuration', 'SQL', 'V2021.07.29__insert sys_config loki query configuration.sql', -1603067350, 'root', now(), 2, 1);
INSERT INTO `flyway_schema_history` VALUES (8, '2021.07.30', 'modifiy monitor module table column', 'SQL', 'V2021.07.30__modifiy monitor_module table column.sql', 1996489389, 'root', now(), 8, 1);