gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1:为日志查询类代码各个方法添加注释

Browse Source 
2:修改orderby的内容
3:修改applicationLog-hive.properties中MmSampleVideoLog多写了个Log导致无法读取表名的问题
4:添加是否开启日志查询count和last功能
This commit is contained in:
renkaige
2018-11-02 16:21:56 +08:00
parent 88360eb30a
commit bbf34d95c6
5 changed files with 163 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/nis/util/Constants.java
View File

@@ -196,6 +196,10 @@ public final class Constants {
* 日志查询是否使用clickhouse,否则使用hive
*/
public static final Boolean ISUSECLICKHOUSE = Configurations.getBooleanProperty("isUseClickHouse", true);
/**
* 是否开启日志查询count和last功能
*/
public static final Boolean ISOPENLOGCOUNTANDLAST = Configurations.getBooleanProperty("isOpenLogCountAndLast", true);
public static final String DIGEST_GEN_TOOL_PATH = Configurations.getStringProperty("digest.gen.tool.path", "maat-redis/digest_gen");

44
src/main/java/com/nis/util/LogJDBCByDruid.java
View File

@@ -29,10 +29,10 @@ import com.zdjizhi.utils.StringUtil;
/**
*
* <p>
* Title: HiveJDBCByDruid
* Title: LogJDBCByDruid
* </p>
* <p>
* Description: 使用druid连接池对hive进行查询并解析结果
* Description: 使用druid连接池对hive或clickhouse进行查询并解析结果
* </p>
* <p>
* Company: IIE
@@ -49,7 +49,7 @@ public class LogJDBCByDruid {
ResultSet rs = null;
Statement st = null;
public static Connection getConnection() throws SQLException {
private static Connection getConnection() throws SQLException {
if (datasource == null) {
if (Constants.ISUSECLICKHOUSE) {
datasource = (DruidDataSource) SpringContextHolder.getBean("ClickHouseDataSourceByDruid");
@@ -105,12 +105,16 @@ public class LogJDBCByDruid {
} else {
page.setList(listObject);
}
logger.info("执行日志查询语句成功,sql={}", sql);
} finally {
closeConn();
}
}
public void closeConn() {
/**
* 关闭数据库连接
*/
private void closeConn() {
try {
if (rs != null) {
rs.close();
@@ -138,7 +142,7 @@ public class LogJDBCByDruid {
* @return
* @throws Exception
*/
public static List<String> getDateColumn(Class<?> type) throws Exception {
private static List<String> getDateColumn(Class<?> type) throws Exception {
List<String> columnList = new ArrayList<String>();
BeanInfo beanInfo = Introspector.getBeanInfo(type);
PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors();
@@ -153,7 +157,15 @@ public class LogJDBCByDruid {
return columnList;
}
public static Object map2Obj(Map<String, Object> map, Class<?> beanClass) throws Exception {
/**
* 将map中的数据利用反射set到Class中,并返回设置好值的对象
*
* @param map
* @param beanClass
* @return
* @throws Exception
*/
private static Object map2Obj(Map<String, Object> map, Class<?> beanClass) throws Exception {
BeanInfo beanInfo = Introspector.getBeanInfo(beanClass);
Object obj = beanClass.newInstance();
PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors();
@@ -194,7 +206,13 @@ public class LogJDBCByDruid {
return obj;
}
public static Map<String, String> getColumn2FiledMap(Class<?> clazz) {
/**
* 根据class从DfLogSearchDao.xml中获取对应的resultMap里column与property的关系,key是column
*
* @param clazz
* @return
*/
private static Map<String, String> getColumn2FiledMap(Class<?> clazz) {
Map<String, String> map = new HashMap<String, String>();
SqlSessionFactory sqlSessionFactory = SpringContextHolder.getBean(SqlSessionFactory.class);
ResultMap resultMap = sqlSessionFactory.getConfiguration().getResultMap(clazz.getSimpleName() + "Map");
@@ -206,10 +224,19 @@ public class LogJDBCByDruid {
}
/**
* 执行count的sql并将结果和计算的last的值set到page对象中
*
* @param page
* @param sql
* @throws Exception
*/
public <T> void getCount(Page<T> page, String sql) throws Exception {
try {
conn = getConnection();
logger.info("连接数据中心日志库成功--------------------------");
st = conn.createStatement();
logger.info("开始执行查询日志总条数sql={}", sql);
rs = st.executeQuery(sql);
String countStr = null;
while (rs.next()) {
@@ -217,11 +244,12 @@ public class LogJDBCByDruid {
break;
}
if (countStr == null || countStr.trim().equals("")) {
logger.info("获取数据中心日志总条数成功总共===================0条配置");
logger.info("获取数据中心日志总条数成功,总共0条日志,sql={}", sql);
page.setCount(0l);
page.setLast(1);
} else {
Long count = Long.valueOf(countStr);
logger.info("获取数据中心日志总条数成功,总共{}条日志,sql={}", count, sql);
page.setCount(count);
page.setLast(getLastPageNum(count.intValue(), page.getPageSize()));
}

222
src/main/java/com/nis/web/service/LogDataService.java
View File

@@ -5,10 +5,8 @@ import java.lang.reflect.Method;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -16,8 +14,6 @@ import java.util.Map;
import org.apache.ibatis.mapping.ResultMap;
import org.apache.ibatis.mapping.ResultMapping;
import org.apache.ibatis.session.SqlSessionFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import com.alibaba.druid.pool.DruidDataSource;
@@ -27,16 +23,22 @@ import com.nis.util.Constants;
import com.nis.util.LogJDBCByDruid;
import com.zdjizhi.utils.StringUtil;
/**
* 从clickhouse或者hive中查询数据并set到page.list返回给界面
*
* @author rkg
*
*/
@Service
public class LogDataService {
private final static Logger logger = LoggerFactory.getLogger(LogDataService.class);
// private final static Logger logger =
// LoggerFactory.getLogger(LogDataService.class);
static DruidDataSource datasource = null;
Connection conn = null;
ResultSet rs = null;
Statement st = null;
private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private static SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMdd");
private static Map<String, Map<String, String>> col2col = new HashMap<String, Map<String, String>>();
static {
Map<String, String> startMap = new HashMap<String, String>();
@@ -47,6 +49,13 @@ public class LogDataService {
col2col.put("searchFoundEndTime", endMap);
}
/**
* 根据类名加对应的标识获取hive或者clickhouse中对应的表名
*
* @param key
* @param defaultTableName
* @return
*/
private String getTableName(String key, String defaultTableName) {
if (Constants.ISUSECLICKHOUSE) {
key = key.replace("HiveTable", "ClickHouseTable");
@@ -54,28 +63,51 @@ public class LogDataService {
return Configurations.getStringProperty(key, defaultTableName);
}
/**
* 根据page及obj对象中的属性值,利用反射拼接对应的查询sql(支持分页,排序) 和计算总量的sql并去对应的数据库中查询,
* 并将结果set到page对象中,返回给界面展示
*
* @param page
* @param obj
* @throws Exception
*/
public <T> void getData(Page<T> page, Object obj) throws Exception {
String className = obj.getClass().getSimpleName();
String tableName = getTableName(className + "HiveTable", "");
if (tableName == null || tableName.trim().equals("")) {
throw new RuntimeException("日志类" + className + "对应的表名为空,请检查配置文件");
}
String orderBy = "";
String orderBy = " order by ";
if (null != page.getOrderBy() && !page.getOrderBy().equals("")) {
orderBy = Page.getOrderBySql(className, page.getOrderBy());
orderBy = orderBy + Page.getOrderBySql(className, page.getOrderBy());
} else {
orderBy = " found_Time desc";
orderBy = orderBy + "found_Time desc ";
}
if (Constants.ISUSECLICKHOUSE) {
getDataFromClickHouse(page, obj, tableName, className, orderBy);
getDataFromClickHouse(page, obj, tableName, className, orderBy.toLowerCase());
} else {
getDataFromHive(page, obj, tableName, className, orderBy);
getDataFromHive(page, obj, tableName, className, orderBy.toLowerCase());
}
}
public <T> void getDataFromClickHouse(Page<T> page, Object bean, String tableName, String className, String orderBy)
throws Exception {
/**
* 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写
*
* @param page
* 里面含有pagesize和pageno,order by
* @param bean
* 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名
* @param tableName
* 表名
* @param className
* 类名
* @param orderBy
* orderby条件
* @throws Exception
*/
private <T> void getDataFromClickHouse(Page<T> page, Object bean, String tableName, String className,
String orderBy) throws Exception {
tableName = tableName.toLowerCase();
String showColmun = getFiledsSql(className, page.getFields());
StringBuffer sql = new StringBuffer();
@@ -163,46 +195,42 @@ public class LogDataService {
int indexOf = whereFoundTime.indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(indexOf));
foundTimeSql.append(whereFoundTime.substring(indexOf));
if (orderBy.toLowerCase().contains("asc") || orderBy.toLowerCase().contains("desc")) {
foundTimeSql.append(" order by " + orderBy.toLowerCase());
} else {
foundTimeSql.append(" order by " + orderBy.toLowerCase() + " desc");
}
foundTimeSql.append(" limit " + startNum + "," + page.getPageSize());
foundTimeSql.append(whereFoundTime.substring(indexOf) + orderBy.toLowerCase() + " limit " + startNum
+ "," + page.getPageSize());
sql.append(" found_time in(" + foundTimeSql + ") ");
}else {
throw new RuntimeException("从clickhouse的"+tableName+"表查询时,必须要有一个where条件");
} else {
throw new RuntimeException("从clickhouse的" + tableName + "表查询时,必须要有一个where条件");
}
} else {
int foundIndexOf = whereFoundTime.append(whereSB).indexOf("and") + "and".length();
countSql.append(whereFoundTime.substring(foundIndexOf));
foundTimeSql.append(whereFoundTime.substring(foundIndexOf));
if (orderBy.toLowerCase().contains("asc") || orderBy.toLowerCase().contains("desc")) {
foundTimeSql.append(" order by " + orderBy.toLowerCase());
} else {
foundTimeSql.append(" order by " + orderBy.toLowerCase() + " desc");
}
foundTimeSql.append(" limit " + startNum + "," + page.getPageSize());
foundTimeSql.append(whereFoundTime.substring(foundIndexOf) + orderBy.toLowerCase() + " limit " + startNum
+ "," + page.getPageSize());
int indexOf = whereSB.indexOf("and") + "and".length();
sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") ");
}
if (orderBy.toLowerCase().contains("asc") || orderBy.toLowerCase().contains("desc")) {
sql.append(" order by " + orderBy.toLowerCase());
} else {
sql.append(" order by " + orderBy.toLowerCase() + " desc");
}
sql.append(" limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
searchFromDataCenter(page, bean, sql, countSql);
}
public <T> void getDataFromHive(Page<T> page, Object bean, String tableName, String className, String orderBy)
/**
* 从hive中查询数据
*
* @param page
* 里面含有pagesize和pageno,order by
* @param bean
* 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名
* @param tableName
* 表名
* @param className
* 类名
* @param orderBy
* orderby条件
* @throws Exception
*/
private <T> void getDataFromHive(Page<T> page, Object bean, String tableName, String className, String orderBy)
throws Exception {
tableName = tableName.toLowerCase();
String showColmun = getFiledsSql(className, page.getFields());
@@ -221,11 +249,6 @@ public class LogDataService {
if (sqlTrim.endsWith(",")) {
sqlTrim = sqlTrim.substring(0, sqlTrim.length() - 1);
}
if (orderBy.toLowerCase().contains("asc") || orderBy.toLowerCase().contains("desc")) {
orderBy = " order by " + orderBy;
} else {
orderBy = " order by " + orderBy + " desc ";
}
sql.setLength(0);
sql.append(" select " + sqlTrim.toLowerCase() + " from (select " + sqlTrim.toLowerCase()
+ ",row_number() over(partition by found_time_partition " + orderBy + ") as row_num from "
@@ -331,18 +354,35 @@ public class LogDataService {
searchFromDataCenter(page, bean, sql, countSql);
}
/**
* 执行sql
*
* @param page
* @param bean
* @param selSql
* @param countSql
* @throws Exception
*/
private <T> void searchFromDataCenter(Page<T> page, Object bean, StringBuffer selSql, StringBuffer countSql)
throws Exception {
new LogJDBCByDruid().getTableData(page, selSql.toString(), bean.getClass());
new LogJDBCByDruid().getTableData(page, selSql.toString().toLowerCase(), bean.getClass());
if (Constants.ISOPENLOGCOUNTANDLAST) {
if (page.getList() != null && page.getList().size() > 0) {
new LogJDBCByDruid().getCount(page, countSql.toString());
new LogJDBCByDruid().getCount(page, countSql.toString().toLowerCase());
}
}
}
private static Map<String, String> getFiledsType(Object o) {
Field[] fields = o.getClass().getSuperclass().getDeclaredFields();
Field[] superfields = o.getClass().getDeclaredFields();
/**
* 利用反射获取class中各个属性的数据类型,key是属性名称,value是数据类型
*
* @param obj
* @return
*/
private static Map<String, String> getFiledsType(Object obj) {
Field[] fields = obj.getClass().getSuperclass().getDeclaredFields();
Field[] superfields = obj.getClass().getDeclaredFields();
Map<String, String> infoMap = new HashMap<String, String>();
for (int i = 0; i < fields.length; i++) {
infoMap.put(fields[i].getName(), fields[i].getType().toString().replace("class", ""));
@@ -353,7 +393,16 @@ public class LogDataService {
return infoMap;
}
public static String getFiledsSql(String mapName, String fileds) throws Exception {
/**
* 将fileds中的字段根据DfLogSearchDao.xml中对应的resultMap转换为数据库中的字段
*
* @param mapName
* @param fileds与界面商定好的是传日志类中的对象名(界面没有表结构不知道对象属性对应的数据库字段名称是什么),不是数据库中的字段名
* @return
* @throws Exception
*/
private static String getFiledsSql(String mapName, String fileds) throws Exception {
if (!StringUtil.isBlank(fileds)) {
String[] fieldsColoumn = null;
// 所有字段名
List<String> columnList = new ArrayList<String>();
@@ -361,8 +410,6 @@ public class LogDataService {
List<String> propertyList = new ArrayList<String>();
// 属性名称为key，字段名称为value
Map<String, String> columnMap = new HashMap<String, String>();
if (!StringUtil.isBlank(fileds)) {
// 解析Fileds的字段/属性名称
fieldsColoumn = fileds.split(",");
@@ -399,7 +446,13 @@ public class LogDataService {
return fileds;
}
public static Map<String, String> getFiledAndColumnMap(Class<?> clazz) {
/**
* 根据class从DfLogSearchDao.xml中获取对应的resultMap里column与property的关系,key是property
*
* @param clazz
* @return
*/
private static Map<String, String> getFiledAndColumnMap(Class<?> clazz) {
Map<String, String> map = new HashMap<String, String>();
SqlSessionFactory sqlSessionFactory = SpringContextHolder.getBean(SqlSessionFactory.class);
ResultMap resultMap = sqlSessionFactory.getConfiguration().getResultMap(clazz.getSimpleName() + "Map");
@@ -436,7 +489,7 @@ public class LogDataService {
}
/**
* 利用发射调用bean.set方法将value设置到字段
* 利用反射调用bean.set方法将value设置到字段
*
* @param bean
* @param fieldName
@@ -456,63 +509,4 @@ public class LogDataService {
method.invoke(bean, value);
}
/**
* 比较开始时间或者结束时间是否比当前系统时间早48小时
*
* @param startTime
* 开始时间
* @param endTime
* 结束时间
* @return
* @throws ParseException
*/
public static boolean ifTimeGreaterThan48(String startTime, String endTime) throws Exception {
logger.info("ifTimeGreaterThan48方法开始" + System.currentTimeMillis());
if (null != startTime && !startTime.equals("") && null != endTime && !endTime.equals("")) {// 开始和结束时间都不为空
Date startDate = sdf.parse(startTime);
Date endDate = sdf.parse(endTime);
if (startDate.getTime() < endDate.getTime()) {// 开始时间比结束时间早
logger.info("ifTimeGreaterThan48方法结束" + System.currentTimeMillis());
return gt48(endTime);
} else {// 开始时间比结束时间晚,不符合一般情况
logger.info("ifTimeGreaterThan48方法结束" + System.currentTimeMillis());
return false;
}
} else if (null != endTime && !endTime.equals("")) {// 开始时间为空,结束时间不为空
logger.info("ifTimeGreaterThan48方法结束" + System.currentTimeMillis());
return gt48(endTime);
} else if (null != startTime && !startTime.equals("")) {// 结束时间为空,开始时间不为空
logger.info("ifTimeGreaterThan48方法结束" + System.currentTimeMillis());
return gt48(startTime);
} else {// 开始和结束时间都为空
logger.info("ifTimeGreaterThan48方法结束" + System.currentTimeMillis());
return false;
}
}
public static boolean gt48(String eqTime) throws ParseException {
logger.info("gt48方法开始" + System.currentTimeMillis());
Date eqDate = sdf.parse(eqTime);
Long dateNum = eqDate.getTime();
Long currentDate = new Date().getTime();
Long time = 0l;
if (dateNum < currentDate) {
time = currentDate - dateNum;// 获取结束时间与当前系统时间的时间差毫秒数
} else {
logger.info("gt48方法结束" + System.currentTimeMillis());
return false;// 结束时间比当前系统时间晚,不符合从数据中心查询数据要求(当前日期48小时前数据从数据中心查询)
}
double hours = time.doubleValue() / (1000 * 60 * 60);
Long datacenterTime = Constants.DATACENTER_TIME;
double doubleValue = datacenterTime.doubleValue();
if (hours > doubleValue) {
logger.info("gt48方法结束" + System.currentTimeMillis());
return true;// 符合要求
} else {
logger.info("gt48方法结束" + System.currentTimeMillis());
return false;
}
}
}

2
src/main/resources/applicationLog-hive.properties
View File

@@ -22,7 +22,7 @@ MmPicUrlLogHiveTable=MM_PIC_URL_LOG
MmVoipIpLogHiveTable=MM_VOIP_IP_LOG
MmVoipAccountLogHiveTable=MM_VOIP_ACCOUNT_LOG
MmSampleAudioLogHiveTable=MM_SAMPLE_AUDIO_LOG
MmSampleVideoLogLogHiveTable=MM_SAMPLE_VIDEO_LOG
MmSampleVideoLogHiveTable=MM_SAMPLE_VIDEO_LOG
MmPornAudioLevelLogHiveTable=MM_PORN_AUDIO_LEVEL_LOG
MmPornVideoLevelLogHiveTable=MM_PORN_VIDEO_LEVEL_LOG
MmSamplePicLogHiveTable=MM_SAMPLE_PIC_LOG

4
src/main/resources/nis.properties
View File

@@ -209,3 +209,7 @@ isUseMinio=true
##存储到redis中的文件类型(拼接的前缀)
fileProtocol=redis://
#是否开启日志查询count和last功能
isOpenLogCountAndLast=true