gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

为日志添加clickhouse数据源及添加开关切换

Browse Source
This commit is contained in:
RenKaiGe-Office
2018-09-04 09:24:22 +08:00
parent 4ec5282ebd
commit 7c8e32de8b
12 changed files with 197 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/main/java/com/nis/util/Configurations.java
View File

@@ -1,11 +1,9 @@
package com.nis.util;
import java.io.File;
import java.util.Map;
import java.util.Properties;
public final class Configurations {
private static Properties prop = new Properties();
@@ -13,10 +11,11 @@ public final class Configurations {
try {
prop.load(Configurations.class.getResourceAsStream("/nis.properties"));
prop.load(Configurations.class.getResourceAsStream("/applicationLog-hive.properties"));
prop.load(Configurations.class.getResourceAsStream("/applicationLog-clickhouse.properties"));
prop.load(Configurations.class.getResourceAsStream("/applicationConfig-maatOracle.properties"));
prop.load(Configurations.class.getResourceAsStream("/applicationConfig-rule.properties"));
prop.load(Configurations.class.getResourceAsStream("/fastdfs-client.properties"));
} catch (Exception e) {
prop = null;
System.err.println("未知nis.properties,请确定文件是否存在!");
@@ -24,36 +23,35 @@ public final class Configurations {
}
public static String getStringProperty(String key, String defaultValue) {
if (prop==null||StringUtil.isBlank(prop.getProperty(key))) {
if (prop == null || StringUtil.isBlank(prop.getProperty(key))) {
return defaultValue;
}
return prop.getProperty(key).trim();
}
public static int getIntProperty(String key, int defaultValue) {
if (prop==null||StringUtil.isBlank(prop.getProperty(key))) {
if (prop == null || StringUtil.isBlank(prop.getProperty(key))) {
return defaultValue;
}
return Integer.parseInt(prop.getProperty(key).trim());
}
public static long getLongProperty(String key, long defaultValue) {
if (prop==null||StringUtil.isBlank(prop.getProperty(key))) {
if (prop == null || StringUtil.isBlank(prop.getProperty(key))) {
return defaultValue;
}
return Long.parseLong(prop.getProperty(key).trim());
}
public static boolean getBooleanProperty(String key, boolean defaultValue) {
if (prop==null||StringUtil.isBlank(prop.getProperty(key))) {
if (prop == null || StringUtil.isBlank(prop.getProperty(key))) {
return defaultValue;
}
return prop.getProperty(key).toLowerCase().trim().equals("true");
}
public static String getFileDirPathProperty(String key,
String defaultValue) {
if (prop==null||StringUtil.isBlank(prop.getProperty(key))) {
public static String getFileDirPathProperty(String key, String defaultValue) {
if (prop == null || StringUtil.isBlank(prop.getProperty(key))) {
return defaultValue;
}
String path = prop.getProperty(key).trim();
@@ -70,16 +68,9 @@ public final class Configurations {
}
return true;
}
public static Map getProp() {
return prop;
}
}

11
src/main/java/com/nis/util/Constants.java
View File

@@ -1,6 +1,5 @@
package com.nis.util;
import java.text.SimpleDateFormat;
import java.util.Date;
public final class Constants {
@@ -187,7 +186,15 @@ public final class Constants {
/**
* 数据中心数据库名称,程序中每次查询时使用的数据库名称 use HIVEDBNAME
*/
public static final String HIVEDBNAME = Configurations.getStringProperty("jdbc.hive.Name", "xa_dfbhit_hive");
public static final String HIVEDBNAME = Configurations.getStringProperty("jdbc.hive.DBName", "maat");
/**
* clickhouse数据库名称
*/
public static final String CLICKHOUSEDBNAME = Configurations.getStringProperty("jdbc.clickHouse.DBName", "k18_ods");
/**
* 日志查询是否使用clickhouse,否则使用hive
*/
public static final Boolean ISUSECLICKHOUSE = Configurations.getBooleanProperty("isUseClickHouse", true);
public static final String DIGEST_GEN_TOOL_PATH = Configurations.getStringProperty("digest.gen.tool.path", "maat-redis/digest_gen");

72
src/main/java/com/nis/util/LogJDBCByDruid.java
View File

@@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSource;
import com.nis.domain.Page;
import com.nis.web.service.SpringContextHolder;
/**
*
* <p>Title: HiveJDBCByDruid</p>
@@ -42,8 +43,11 @@ public class LogJDBCByDruid {
public static Connection getConnection() throws SQLException {
if (datasource == null) {
datasource = (DruidDataSource) SpringContextHolder.getBean("HiveDataSourceByDruid");
// datasource = (DruidDataSource) SpringContextHolder.getBean("ClickHouseDataSourceByDruid");
if (Constants.ISUSECLICKHOUSE) {
datasource = (DruidDataSource) SpringContextHolder.getBean("ClickHouseDataSourceByDruid");
} else {
datasource = (DruidDataSource) SpringContextHolder.getBean("HiveDataSourceByDruid");
}
}
return datasource.getConnection();
}
@@ -61,8 +65,8 @@ public class LogJDBCByDruid {
* @return
* @throws Exception
*/
public Map<String, List<Object>> tableMapping(Page<?> page, String redisKey, String sql, Class<?> entityClass, Object... obj)
throws Exception {
public Map<String, List<Object>> tableMapping(Page<?> page, String redisKey, String sql, Class<?> entityClass,
Object... obj) throws Exception {
Map<String, List<Object>> mapList = new HashMap<String, List<Object>>();
try {
Map<String, String> filedAndColumnMap = getColumn2FiledMap(entityClass);
@@ -73,11 +77,11 @@ public class LogJDBCByDruid {
conn = getConnection();
logger.info("连接数据中心日志库成功--------------------------");
st = conn.createStatement();
logger.info("开始执行日志查询语句sql={}", sql);
rs = st.executeQuery(sql);
logger.info("执行查询语句成功sql={}", sql);
ResultSetMetaData metaData = rs.getMetaData();
while (rs.next()) {
Map<String,Object> map = new HashMap<String,Object>();
Map<String, Object> map = new HashMap<String, Object>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
Object value = rs.getObject(i);
String filedName = filedAndColumnMap.get(metaData.getColumnName(i).toString().toLowerCase());
@@ -104,34 +108,38 @@ public class LogJDBCByDruid {
if (null == listString || listString.size() == 0 || null == listObject || listObject.size() == 0) {
return null;
} else {
//暂时没有往缓存写的操作
// if (Constants.IS_OPEN_REDIS && Constants.DATACENTER_OPEN_REDIS) {
// new SaveRedisListThread(redisKey, listString, Constants.HIVE_EXPIRE).start();
// }
// 暂时没有往缓存写的操作
// if (Constants.IS_OPEN_REDIS && Constants.DATACENTER_OPEN_REDIS) {
// new SaveRedisListThread(redisKey, listString, Constants.HIVE_EXPIRE).start();
// }
}
// sublist包前不包后,0-30实际获取的是0-29的数据
Integer startNum = (page.getPageNo() - 1) * page.getPageSize();
Integer endNum = startNum - 1 + page.getPageSize() + 1;
if (listString.size() >= startNum) {
if (listString.size() >= endNum) {
mapList.put("str", listString.subList(startNum, endNum));
} else {
mapList.put("str", listString.subList(startNum, listString.size()));
}
if (Constants.ISUSECLICKHOUSE) {// sql查询时已经分页了
mapList.put("str", listString);
mapList.put("obj", listObject);
} else {
mapList.put("str", new ArrayList<Object>());
}
if (listObject.size() >= startNum) {
if (listObject.size() >= endNum) {
mapList.put("obj", listObject.subList(startNum, endNum));
} else {
mapList.put("obj", listObject.subList(startNum, listObject.size()));
}
} else {
mapList.put("obj", new ArrayList<Object>());
}
// sublist包前不包后,0-30实际获取的是0-29的数据
Integer startNum = (page.getPageNo() - 1) * page.getPageSize();
Integer endNum = startNum - 1 + page.getPageSize() + 1;
if (listString.size() >= startNum) {
if (listString.size() >= endNum) {
mapList.put("str", listString.subList(startNum, endNum));
} else {
mapList.put("str", listString.subList(startNum, listString.size()));
}
} else {
mapList.put("str", new ArrayList<Object>());
}
if (listObject.size() >= startNum) {
if (listObject.size() >= endNum) {
mapList.put("obj", listObject.subList(startNum, endNum));
} else {
mapList.put("obj", listObject.subList(startNum, listObject.size()));
}
} else {
mapList.put("obj", new ArrayList<Object>());
}
}
} finally {
closeConn();
}
@@ -221,8 +229,6 @@ public class LogJDBCByDruid {
return obj;
}
public static Map<String, String> getColumn2FiledMap(Class<?> clazz) {
Map<String, String> map = new HashMap<String, String>();
SqlSessionFactory sqlSessionFactory = SpringContextHolder.getBean(SqlSessionFactory.class);