gfwleak/galaxy-deployment-schema-updater-tool
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

原schema-upgrade项目更名，发布初版

Browse Source
This commit is contained in:
qidaijie
2023-09-26 14:48:35 +08:00
parent 28f935a8fc
commit ae9ea847dc
70 changed files with 30477 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/main/java/com/zdjizhi/common/SchemaConfig.java Normal file
View File

@@ -0,0 +1,19 @@
package com.zdjizhi.common;
import com.zdjizhi.utils.system.SchemaConfigurations;
/**
* @author Administrator
*/
public class SchemaConfig {
/**
* Nacos
*/
public static final String NACOS_PIN = SchemaConfigurations.getStringProperty(0, "nacos.pin");
public static final String NACOS_GROUP = SchemaConfigurations.getStringProperty(0, "nacos.group");
public static final String NACOS_USERNAME = SchemaConfigurations.getStringProperty(0, "nacos.username");
public static final String NON_SCHEMA_TABLES = SchemaConfigurations.getStringProperty(0, "non.schema.tables");
}

233
src/main/java/com/zdjizhi/topology/UpgradeSchema.java Normal file
View File

@@ -0,0 +1,233 @@
package com.zdjizhi.topology;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.JsonPath;
import com.zdjizhi.common.SchemaConfig;
import com.zdjizhi.utils.JsonUtil;
import com.zdjizhi.utils.StringUtil;
import com.zdjizhi.utils.nacos.ApiUtil;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
public class UpgradeSchema {
private static final Log logger = LogFactory.get();
@SuppressWarnings("unchecked")
public static void main(String[] args) {
try {
String path = args[0];
String nacos_server = args[1];
String namespace = args[2];
File file = new File(path);
File[] fileNames = file.listFiles();
if (fileNames != null) {
for (File fileName : fileNames) {
if (!fileName.isDirectory()) {
String dataId = fileName.getName();
if (!SchemaConfig.NON_SCHEMA_TABLES.contains(dataId)) {
String oldSchema = ApiUtil.getConfiguration(nacos_server, dataId, namespace);
if (StringUtil.isNotBlank(oldSchema)) {
String tmpSchema = FileUtils.readFileToString(fileName, "UTF-8");
if ("json".equals(JsonUtil.lastName(dataId)) && documentTypeJudgment(dataId, tmpSchema) && StringUtil.isNotBlank(tmpSchema)) {
logger.info("================开始对{}表进行更新操作================", dataId);
Map<String, Integer> ttlMap = new HashMap<>(16);
ttlMap.putAll(getOldTTL(oldSchema));
Map<String, String> visibilityMap = new HashMap<>(16);
visibilityMap.putAll(getOldVisibility(oldSchema));
JSONObject json = new JSONObject(tmpSchema, false, true);
changeTableTTL(oldSchema, json);
String newSchema = upgradeSchema(json, ttlMap, visibilityMap);
boolean status = ApiUtil.pushConfiguration(nacos_server, dataId, newSchema, namespace);
outMessage(status, dataId);
} else {
boolean status = ApiUtil.pushConfiguration(nacos_server, dataId, tmpSchema, namespace);
outMessage(status, dataId);
}
} else {
logger.warn("Nacos未查询到{}，判断为新文件，直接推送至Nacos", dataId);
boolean status = ApiUtil.pushConfiguration(nacos_server, dataId, FileUtils.readFileToString(fileName, "UTF-8"), namespace);
outMessage(status, dataId);
}
} else {
logger.warn("{}非Schema表，直接推送至Nacos", dataId);
boolean status = ApiUtil.pushConfiguration(nacos_server, dataId, FileUtils.readFileToString(fileName, "UTF-8"), namespace);
outMessage(status, dataId);
}
}
}
}
} catch (ArrayIndexOutOfBoundsException ae) {
logger.error("Please enter parameters!");
logger.error("example: java -jar schema-updater-tool-{version}.jar {schema folder} {nacos address} {nacos namespcae}");
logger.error("example: java -jar schema-updater-tool-3.1.jar schema/ 192.168.44.12 test");
ae.printStackTrace();
} catch (RuntimeException | IOException e) {
e.printStackTrace();
}
}
/**
* 获取nacos内已有schema的Visibility信息
*
* @param schema schema内容
* @return Visibility合集{name:Visibility}
*/
private static Map<String, String> getOldVisibility(String schema) {
Map<String, String> tmpMap = new HashMap<>(16);
Object document = Configuration.defaultConfiguration().jsonProvider().parse(schema);
ArrayList<Object> oldFields = JsonPath.read(document, "$.fields");
for (Object oldField : oldFields) {
try {
if (oldField.toString().contains("visibility")) {
String fieldName = JsonPath.read(oldField, "$.name");
String fieldVisbility = JsonPath.read(oldField, "$.doc.visibility");
tmpMap.put(fieldName, fieldVisbility);
}
} catch (Exception e) {
logger.error("解析visibility信息异常" + e.getMessage());
}
}
return tmpMap;
}
/**
* 获取nacos内已有schema的TTL信息
*
* @param schema schema内容
* @return TTL合集{name:ttl}
*/
private static Map<String, Integer> getOldTTL(String schema) {
Map<String, Integer> tmpMap = new HashMap<>(16);
try {
JSONObject json = new JSONObject(schema, false, true);
if (json.containsKey("fields")) {
JSONArray oldFields = json.getJSONArray("fields");
for (Object oldField : oldFields) {
JSONObject jsonTmp = new JSONObject(oldField.toString(), false, true);
String fieldName = jsonTmp.getStr("name");
JSONObject doc = new JSONObject(jsonTmp.getStr("doc"), false, true);
if (doc.containsKey("ttl")) {
tmpMap.put(fieldName, doc.getInt("ttl"));
}
}
}
} catch (Exception e) {
logger.error("解析ttl信息异常" + e.getMessage());
}
return tmpMap;
}
/**
* 获取nacos内已有schema表TTL，修改新Schema表TTL
*
* @param schema 旧schema内容
* @param json 新schema-json
*/
private static void changeTableTTL(String schema, JSONObject json) {
Object document = Configuration.defaultConfiguration().jsonProvider().parse(schema);
try {
Integer oldTTL = JsonPath.read(document, "$.doc.ttl");
if (oldTTL != null) {
json.getJSONObject("doc").set("ttl", oldTTL);
}
} catch (RuntimeException e) {
logger.warn("该表不包含表TTL信息!");
}
}
/**
* 更新schema
*
* @param json schema
* @param ttlMap ttl数据
* @return 同步ttl后的schema
*/
private static String upgradeSchema(JSONObject json, Map<String, Integer> ttlMap, Map<String, String> visibilityMap) {
JSONArray newFields = json.getJSONArray("fields");
JSONArray tmpFields = new JSONArray();
for (Object fields : newFields) {
JSONObject jsonTmp = new JSONObject(fields.toString(), false, true);
String name = jsonTmp.getStr("name");
JSONObject doc = new JSONObject(jsonTmp.getStr("doc"), false, true);
if (ttlMap.containsKey(name)) {
Integer ttlValue = ttlMap.get(name);
if (ttlValue != null) {
doc.set("ttl", ttlMap.get(name));
logger.info("字段:{}的TTL值变更为：{}", name, ttlMap.get(name));
}
}
if (visibilityMap.containsKey(name)) {
String nowType = doc.get("visibility").toString();
String oldType = visibilityMap.get(name);
if (!oldType.equals(nowType)) {
if (!"hidden".equals(oldType)) {
if ("enabled".equals(nowType) || "disabled".equals(nowType)) {
doc.set("visibility", visibilityMap.get(name));
logger.info("字段:{}的visibility值变更为：{}", name, visibilityMap.get(name));
}
}
}
}
jsonTmp.set("doc", doc);
tmpFields.add(jsonTmp);
}
json.set("fields", tmpFields);
return json.toStringPretty();
}
/**
* 读取nacos内schema中内容，判断是否有需要处理的ttl字段和visibility字段
*
* @param schema schema内容
* @return 是否需要处理当前表
*/
private static boolean documentTypeJudgment(String filesName, String schema) {
try {
JSONObject json = new JSONObject(schema, false, true);
if (json.containsKey("fields")) {
JSONArray fields = json.getJSONArray("fields");
for (Object field : fields) {
JSONObject jsonTmp = new JSONObject(field.toString(), false, true);
if (jsonTmp.containsKey("doc")) {
JSONObject doc = new JSONObject(jsonTmp.getStr("doc"), false, true);
if (doc.containsKey("ttl") || doc.containsKey("visibility")) {
return true;
}
}
}
}
} catch (Exception e) {
logger.error(filesName + "解析JSON异常！异常信息为：" + e.getMessage());
}
return false;
}
private static void outMessage(boolean status, String dataId) {
if (status) {
logger.info("配置{}已成功推送至Nacos！", dataId);
} else {
logger.error("配置{}推送Nacos失败！", dataId);
}
}
}

29
src/main/java/com/zdjizhi/utils/JsonUtil.java Normal file
View File

@@ -0,0 +1,29 @@
package com.zdjizhi.utils;
/**
* @author qidaijie
* @Package com.zdjizhi.utils
* @Description:
* @date 2022/7/2615:32
*/
public class JsonUtil {
/**
* 截取文件的后缀名，判断文件类型是否处理
*
* @param fileName 要截取的文件
* @return 文件拓展名
*/
public static String lastName(String fileName) {
if (StringUtil.isNotBlank(fileName)) {
String[] split = fileName.split("\\.");
if (split.length > 1) {
return split[split.length - 1];
} else {
return "";
}
}
return "";
}
}

68
src/main/java/com/zdjizhi/utils/nacos/ApiUtil.java Normal file
View File

@@ -0,0 +1,68 @@
package com.zdjizhi.utils.nacos;
import cn.hutool.log.Log;
import cn.hutool.log.LogFactory;
import com.alibaba.nacos.api.NacosFactory;
import com.alibaba.nacos.api.PropertyKeyConst;
import com.alibaba.nacos.api.config.ConfigService;
import com.alibaba.nacos.api.exception.NacosException;
import com.zdjizhi.common.SchemaConfig;
import java.util.Properties;
public class ApiUtil {
private static final Log logger = LogFactory.get();
/**
* 获取nacos上的配置文件
*
* @param dataId 配置文件名称
* @return 配置内容
*/
public static String getConfiguration(String nacos_server, String dataId, String namespace) {
String content = null;
Properties properties = new Properties();
properties.setProperty(PropertyKeyConst.SERVER_ADDR, nacos_server);
properties.setProperty(PropertyKeyConst.NAMESPACE, namespace);
properties.setProperty(PropertyKeyConst.USERNAME, SchemaConfig.NACOS_USERNAME);
properties.setProperty(PropertyKeyConst.PASSWORD, SchemaConfig.NACOS_PIN);
try {
ConfigService configService = NacosFactory.createConfigService(properties);
content = configService.getConfig(dataId, SchemaConfig.NACOS_GROUP, 5000);
} catch (NacosException e) {
logger.error("Failed to get configuration content from NACOS! The exception message is:" + e.getMessage());
e.printStackTrace();
}
return content;
}
/**
* 推送配置文件到nacos
*
* @param dataId 配置文件名称
* @param schema 配置内容
* @return 推送状态
*/
public static boolean pushConfiguration(String nacos_server, String dataId, String schema, String namespace) {
boolean pushStatus = false;
Properties properties = new Properties();
properties.setProperty(PropertyKeyConst.SERVER_ADDR, nacos_server);
properties.setProperty(PropertyKeyConst.NAMESPACE, namespace);
properties.setProperty(PropertyKeyConst.USERNAME, SchemaConfig.NACOS_USERNAME);
properties.setProperty(PropertyKeyConst.PASSWORD, SchemaConfig.NACOS_PIN);
try {
String[] split = dataId.split("\\.");
String type = split[split.length - 1];
ConfigService configService = NacosFactory.createConfigService(properties);
pushStatus = configService.publishConfig(dataId, SchemaConfig.NACOS_GROUP, schema, type);
} catch (NacosException e) {
logger.error("Failed to push configuration to NACOS! The exception message is:" + e.getMessage());
e.printStackTrace();
}
return pushStatus;
}
}

58
src/main/java/com/zdjizhi/utils/system/SchemaConfigurations.java Normal file
View File

@@ -0,0 +1,58 @@
package com.zdjizhi.utils.system;
import com.zdjizhi.utils.StringUtil;
import java.io.IOException;
import java.util.Locale;
import java.util.Properties;
/**
* @author Administrator
*/
public final class SchemaConfigurations {
private static Properties propService = new Properties();
public static String getStringProperty(Integer type, String key) {
if (type == 0) {
return propService.getProperty(key);
} else {
return null;
}
}
public static Integer getIntProperty(Integer type, String key) {
if (type == 0) {
return Integer.parseInt(propService.getProperty(key));
} else {
return null;
}
}
public static Long getLongProperty(Integer type, String key) {
if (type == 0) {
return Long.parseLong(propService.getProperty(key));
} else {
return null;
}
}
public static Boolean getBooleanProperty(Integer type, String key) {
if (type == 0) {
return StringUtil.equals(propService.getProperty(key).toLowerCase().trim().toUpperCase(Locale.ENGLISH), "true");
} else {
return null;
}
}
static {
try {
propService.load(SchemaConfigurations.class.getClassLoader().getResourceAsStream("common_config.properties"));
} catch (IOException | RuntimeException e) {
propService = null;
}
}
}

25
src/main/log4j.properties Normal file
View File

@@ -0,0 +1,25 @@
#Log4j
log4j.rootLogger=console,file
# 控制台日志设置
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.Threshold=INFO
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] [Thread\:%t] %l %x - <%m>%n
# 文件日志设置
log4j.appender.file=org.apache.log4j.DailyRollingFileAppender
log4j.appender.file.Threshold=info
log4j.appender.file.encoding=UTF-8
log4j.appender.file.Append=true
#路径请用相对路径，做好相关测试输出到应用目下
log4j.appender.file.file=${nis.root}/log/galaxy-name.log
log4j.appender.file.DatePattern='.'yyyy-MM-dd
log4j.appender.file.layout=org.apache.log4j.PatternLayout
#log4j.appender.file.layout.ConversionPattern=%d{HH:mm:ss} %X{ip} [%t] %5p %c{1} %m%n
log4j.appender.file.layout.ConversionPattern=[%d{yyyy-MM-dd HH\:mm\:ss}] [%-5p] %X{ip} [Thread\:%t] %l %x - %m%n
##MyBatis 配置，com.nis.web.dao是mybatis接口所在包
#log4j.logger.com.nis.web.dao=debug
##bonecp数据源配置
#log4j.category.com.jolbox=debug,console