根据04版补全程序更新P19双写程序。
This commit is contained in:
wangchengcheng
134
src/main/java/com/zdjizhi/common/FlowWriteConfig.java
Normal file
134
src/main/java/com/zdjizhi/common/FlowWriteConfig.java
Normal file
@@ -0,0 +1,134 @@
|
||||
package com.zdjizhi.common;
|
||||
|
||||
|
||||
import com.zdjizhi.utils.system.FlowWriteConfigurations;
|
||||
import org.jasypt.encryption.pbe.StandardPBEStringEncryptor;
|
||||
|
||||
/**
|
||||
* @author Administrator
|
||||
*/
|
||||
public class FlowWriteConfig {
|
||||
|
||||
private static StandardPBEStringEncryptor encryptor = new StandardPBEStringEncryptor();
|
||||
|
||||
static {
|
||||
encryptor.setPassword("galaxy");
|
||||
}
|
||||
|
||||
public static final int IF_PARAM_LENGTH = 3;
|
||||
/**
|
||||
* 有此标识的字段为失效字段，不计入最终日志字段
|
||||
*/
|
||||
public static final String VISIBILITY = "disabled";
|
||||
/**
|
||||
* 默认的切分符号
|
||||
*/
|
||||
public static final String FORMAT_SPLITTER = ",";
|
||||
/**
|
||||
* 标识字段为日志字段还是schema指定字段
|
||||
*/
|
||||
public static final String IS_JSON_KEY_TAG = "$.";
|
||||
/**
|
||||
* if函数连接分隔符
|
||||
*/
|
||||
public static final String IF_CONDITION_SPLITTER = "=";
|
||||
/**
|
||||
* 默认的字符串解析编码
|
||||
*/
|
||||
public static final String ENCODING = "UTF8";
|
||||
|
||||
/**
|
||||
* Nacos
|
||||
*/
|
||||
public static final String NACOS_SERVER = FlowWriteConfigurations.getStringProperty(0, "nacos.server");
|
||||
public static final String NACOS_SCHEMA_NAMESPACE = FlowWriteConfigurations.getStringProperty(0, "nacos.schema.namespace");
|
||||
public static final String NACOS_COMMON_NAMESPACE = FlowWriteConfigurations.getStringProperty(0, "nacos.common.namespace");
|
||||
public static final String NACOS_DATA_ID = FlowWriteConfigurations.getStringProperty(0, "nacos.data.id");
|
||||
public static final String NACOS_PIN = FlowWriteConfigurations.getStringProperty(1, "nacos.pin");
|
||||
public static final String NACOS_GROUP = FlowWriteConfigurations.getStringProperty(1, "nacos.group");
|
||||
public static final String NACOS_USERNAME = FlowWriteConfigurations.getStringProperty(1, "nacos.username");
|
||||
|
||||
/**
|
||||
* System config
|
||||
*/
|
||||
public static final Integer SOURCE_PARALLELISM = FlowWriteConfigurations.getIntProperty(0, "source.parallelism");
|
||||
public static final Integer PERCENT_SINK_PARALLELISM = FlowWriteConfigurations.getIntProperty(0, "percent.sink.parallelism");
|
||||
public static final Integer FILE_DATA_SINK_PARALLELISM = FlowWriteConfigurations.getIntProperty(0, "file.data.sink.parallelism");
|
||||
public static final Integer TRANSFORM_PARALLELISM = FlowWriteConfigurations.getIntProperty(0, "transform.parallelism");
|
||||
public static final Integer DATA_CENTER_ID_NUM = FlowWriteConfigurations.getIntProperty(0, "data.center.id.num");
|
||||
public static final Integer LOG_NEED_COMPLETE = FlowWriteConfigurations.getIntProperty(0, "log.need.complete");
|
||||
public static final String MAIL_DEFAULT_CHARSET = FlowWriteConfigurations.getStringProperty(1, "mail.default.charset");
|
||||
public static final Integer LOG_TRANSFORM_TYPE = FlowWriteConfigurations.getIntProperty(1, "log.transform.type");
|
||||
public static final Integer BUFFER_TIMEOUT = FlowWriteConfigurations.getIntProperty(1, "buffer.timeout");
|
||||
|
||||
/**
|
||||
* HBase
|
||||
*/
|
||||
public static final Integer HBASE_TICK_TUPLE_FREQ_SECS = FlowWriteConfigurations.getIntProperty(0, "hbase.tick.tuple.freq.secs");
|
||||
public static final String HBASE_TABLE_NAME = FlowWriteConfigurations.getStringProperty(1, "hbase.table.name");
|
||||
|
||||
/**
|
||||
* kafka common
|
||||
*/
|
||||
public static final String KAFKA_SASL_JAAS_USER = encryptor.decrypt(FlowWriteConfigurations.getStringProperty(1, "kafka.user"));
|
||||
public static final String KAFKA_SASL_JAAS_PIN = encryptor.decrypt(FlowWriteConfigurations.getStringProperty(1, "kafka.pin"));
|
||||
|
||||
public static final String PERCENT_KAFKA_TOPIC = FlowWriteConfigurations.getStringProperty(0, "percent.kafka.topic");
|
||||
/**
|
||||
* kafka source config
|
||||
*/
|
||||
public static final String SOURCE_KAFKA_TOPIC = FlowWriteConfigurations.getStringProperty(0, "source.kafka.topic");
|
||||
public static final String GROUP_ID = FlowWriteConfigurations.getStringProperty(0, "group.id");
|
||||
public static final String SESSION_TIMEOUT_MS = FlowWriteConfigurations.getStringProperty(1, "session.timeout.ms");
|
||||
public static final String MAX_POLL_RECORDS = FlowWriteConfigurations.getStringProperty(1, "max.poll.records");
|
||||
public static final String MAX_PARTITION_FETCH_BYTES = FlowWriteConfigurations.getStringProperty(1, "max.partition.fetch.bytes");
|
||||
|
||||
/**
|
||||
* kafka sink config
|
||||
*/
|
||||
|
||||
|
||||
public static final String FILE_DATA_SINK_KAFKA_TOPIC = FlowWriteConfigurations.getStringProperty(0, "file.data.kafka.topic");
|
||||
public static final String PRODUCER_ACK = FlowWriteConfigurations.getStringProperty(1, "producer.ack");
|
||||
public static final String PRODUCER_KAFKA_COMPRESSION_TYPE = FlowWriteConfigurations.getStringProperty(0, "producer.kafka.compression.type");
|
||||
public static final String OUTPUT_KAFKA_TOPIC = FlowWriteConfigurations.getStringProperty(0, "output.kafka.topic");
|
||||
|
||||
|
||||
/**
|
||||
* connection kafka
|
||||
*/
|
||||
public static final String RETRIES = FlowWriteConfigurations.getStringProperty(1, "retries");
|
||||
public static final String LINGER_MS = FlowWriteConfigurations.getStringProperty(1, "linger.ms");
|
||||
public static final Integer REQUEST_TIMEOUT_MS = FlowWriteConfigurations.getIntProperty(1, "request.timeout.ms");
|
||||
public static final Integer BATCH_SIZE = FlowWriteConfigurations.getIntProperty(1, "batch.size");
|
||||
public static final Integer BUFFER_MEMORY = FlowWriteConfigurations.getIntProperty(1, "buffer.memory");
|
||||
public static final Integer MAX_REQUEST_SIZE = FlowWriteConfigurations.getIntProperty(1, "max.request.size");
|
||||
|
||||
/**
|
||||
* http
|
||||
*/
|
||||
public static final String APP_ID_HTTP = FlowWriteConfigurations.getStringProperty(1, "app.id.http");
|
||||
public static final Integer APP_TICK_TUPLE_FREQ_SECS = FlowWriteConfigurations.getIntProperty(1, "app.tick.tuple.freq.secs");
|
||||
|
||||
/**
|
||||
* common config
|
||||
*/
|
||||
/**
|
||||
* public static final String SOURCE_KAFKA_SERVERS = NacosConfig.getStringProperty("source.kafka.servers");
|
||||
* public static final String SINK_KAFKA_SERVERS = NacosConfig.getStringProperty("sink.kafka.servers");
|
||||
* public static final String ZOOKEEPER_SERVERS = NacosConfig.getStringProperty("zookeeper.servers");
|
||||
* public static final String TOOLS_LIBRARY = NacosConfig.getStringProperty("tools.library");
|
||||
* public static final String HBASE_ZOOKEEPER_SERVERS = NacosConfig.getStringProperty("hbase.zookeeper.servers");
|
||||
*/
|
||||
public static final String SOURCE_KAFKA_SERVERS = FlowWriteConfigurations.getStringProperty(0,"source.kafka.servers");
|
||||
public static final String PERCENT_SINK_KAFKA_SERVERS = FlowWriteConfigurations.getStringProperty(0,"percent.sink.kafka.servers");
|
||||
public static final String ZOOKEEPER_SERVERS = FlowWriteConfigurations.getStringProperty(0,"zookeeper.servers");
|
||||
public static final String TOOLS_LIBRARY = FlowWriteConfigurations.getStringProperty(0,"tools.library");
|
||||
public static final String HBASE_ZOOKEEPER_SERVERS = FlowWriteConfigurations.getStringProperty(0,"hbase.zookeeper.servers");
|
||||
|
||||
public static final String FILE_DATA_SINK_KAFKA_SERVERS = FlowWriteConfigurations.getStringProperty(0,"file.data.sink.kafka.servers");
|
||||
/**
|
||||
* oos
|
||||
*/
|
||||
public static final String OOS_SERVERS = FlowWriteConfigurations.getStringProperty(0, "oos.servers");
|
||||
}
|
||||
107
src/main/java/com/zdjizhi/common/NacosConfig.java
Normal file
107
src/main/java/com/zdjizhi/common/NacosConfig.java
Normal file
@@ -0,0 +1,107 @@
|
||||
package com.zdjizhi.common;
|
||||
|
||||
import cn.hutool.log.Log;
|
||||
import cn.hutool.log.LogFactory;
|
||||
import com.alibaba.nacos.api.NacosFactory;
|
||||
import com.alibaba.nacos.api.PropertyKeyConst;
|
||||
import com.alibaba.nacos.api.config.ConfigService;
|
||||
import com.alibaba.nacos.api.exception.NacosException;
|
||||
import com.zdjizhi.utils.StringUtil;
|
||||
import com.zdjizhi.utils.system.FlowWriteConfigurations;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.StringReader;
|
||||
import java.util.Properties;
|
||||
|
||||
/**
|
||||
* @author qidaijie
|
||||
* @Package com.zdjizhi.common
|
||||
* @Description:
|
||||
* @date 2022/3/189:36
|
||||
*/
|
||||
@Deprecated
|
||||
public class NacosConfig {
|
||||
private static final Log logger = LogFactory.get();
|
||||
private static Properties propCommon = new Properties();
|
||||
private static Properties propNacos = new Properties();
|
||||
|
||||
private static NacosConfig nacosConfig;
|
||||
|
||||
private static void getInstance() {
|
||||
nacosConfig = new NacosConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* 构造函数-新
|
||||
*/
|
||||
private NacosConfig() {
|
||||
//获取连接
|
||||
getConnection();
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化Nacos配置列表
|
||||
*/
|
||||
private static void getConnection() {
|
||||
try {
|
||||
propNacos.setProperty(PropertyKeyConst.SERVER_ADDR, FlowWriteConfig.NACOS_SERVER);
|
||||
propNacos.setProperty(PropertyKeyConst.NAMESPACE, FlowWriteConfig.NACOS_COMMON_NAMESPACE);
|
||||
propNacos.setProperty(PropertyKeyConst.USERNAME, FlowWriteConfig.NACOS_USERNAME);
|
||||
propNacos.setProperty(PropertyKeyConst.PASSWORD, FlowWriteConfig.NACOS_PIN);
|
||||
ConfigService configService = NacosFactory.createConfigService(propNacos);
|
||||
String commonConfig = configService.getConfig("etl_connection_config.properties", FlowWriteConfig.NACOS_GROUP, 5000);
|
||||
if (StringUtil.isNotBlank(commonConfig)) {
|
||||
propCommon.load(new StringReader(commonConfig));
|
||||
}
|
||||
} catch (NacosException | IOException e) {
|
||||
logger.error("Get topology run configuration error,The exception message is ：" + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取String类型配置
|
||||
*
|
||||
* @param key config key
|
||||
* @return value
|
||||
*/
|
||||
public static String getStringProperty(String key) {
|
||||
|
||||
if (nacosConfig == null) {
|
||||
getInstance();
|
||||
}
|
||||
return propCommon.getProperty(key);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取Integer类型配置
|
||||
*
|
||||
* @param key config key
|
||||
* @return value
|
||||
*/
|
||||
public static Integer getIntegerProperty(String key) {
|
||||
if (nacosConfig == null) {
|
||||
getInstance();
|
||||
}
|
||||
|
||||
return Integer.parseInt(propCommon.getProperty(key));
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 获取Long类型配置
|
||||
*
|
||||
* @param key config key
|
||||
* @return value
|
||||
*/
|
||||
public static Long getLongProperty(String key) {
|
||||
if (nacosConfig == null) {
|
||||
getInstance();
|
||||
}
|
||||
|
||||
return Long.parseLong(propCommon.getProperty(key));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user