diff --git a/src/main/java/com/nis/domain/restful/NtcConnRecordLog.java b/src/main/java/com/nis/domain/restful/NtcConnRecordLog.java new file mode 100644 index 0000000..eadac05 --- /dev/null +++ b/src/main/java/com/nis/domain/restful/NtcConnRecordLog.java @@ -0,0 +1,72 @@ +package com.nis.domain.restful; + +import com.nis.domain.LogEntity; +import com.wordnik.swagger.annotations.ApiModelProperty; + +/** + * + * @ClassName:NtcConnRecordLog + * @Description:TODO(这里用一句话描述这个类的作用) + * @author rkg + * @date 2018年12月12日 下午2:00:16 + * @version V1.0 + */ +public class NtcConnRecordLog extends LogEntity { + + + /** + * + */ + private static final long serialVersionUID = 1L; + @ApiModelProperty(value = "", required = true) + protected Long appLabel; + @ApiModelProperty(value = "", required = true) + protected String c2sPktNum; + @ApiModelProperty(value = "", required = true) + protected String s2cPktNum; + @ApiModelProperty(value = "", required = true) + protected String c2sByteNum; + @ApiModelProperty(value = "", required = true) + protected String s2cByteNum; + + public Long getAppLabel() { + return appLabel; + } + + public void setAppLabel(Long appLabel) { + this.appLabel = appLabel; + } + + public String getC2sPktNum() { + return c2sPktNum; + } + + public void setC2sPktNum(String c2sPktNum) { + this.c2sPktNum = c2sPktNum; + } + + public String getS2cPktNum() { + return s2cPktNum; + } + + public void setS2cPktNum(String s2cPktNum) { + this.s2cPktNum = s2cPktNum; + } + + public String getC2sByteNum() { + return c2sByteNum; + } + + public void setC2sByteNum(String c2sByteNum) { + this.c2sByteNum = c2sByteNum; + } + + public String getS2cByteNum() { + return s2cByteNum; + } + + public void setS2cByteNum(String s2cByteNum) { + this.s2cByteNum = s2cByteNum; + } + +} diff --git a/src/main/java/com/nis/util/PropertyPlaceholderConfigurerCrypt.java b/src/main/java/com/nis/util/PropertyPlaceholderConfigurerCrypt.java index f616bf3..3469fe5 100644 --- a/src/main/java/com/nis/util/PropertyPlaceholderConfigurerCrypt.java +++ b/src/main/java/com/nis/util/PropertyPlaceholderConfigurerCrypt.java @@ -30,6 +30,20 @@ public class PropertyPlaceholderConfigurerCrypt extends PropertyPlaceholderConfi props.setProperty("jdbc.clickhouse.password", new String(AESUtil.decrypt(Base64.decodeBase64(clickHousePassword), clickHouseScretKey))); } + + + + + + String ckLocalPassword = props.getProperty("jdbc.ckLocal.password"); + String ckLocalScretKey = props.getProperty("jdbc.ckLocal.key"); + if (null != ckLocalPassword) { + props.setProperty("jdbc.ckLocal.password", + new String(AESUtil.decrypt(Base64.decodeBase64(ckLocalPassword), ckLocalScretKey))); + } + + + // hive String hivePassword = props.getProperty("jdbc.hive.password"); String hiveScretKey = props.getProperty("jdbc.hive.key"); diff --git a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java index 635ec4e..5182d85 100644 --- a/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java +++ b/src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java @@ -16,6 +16,7 @@ import com.nis.domain.restful.NtcAppLog; import com.nis.domain.restful.NtcBgpLog; import com.nis.domain.restful.NtcCollectRadiusLog; import com.nis.domain.restful.NtcCollectVoipLog; +import com.nis.domain.restful.NtcConnRecordLog; import com.nis.domain.restful.NtcDdosLog; import com.nis.domain.restful.NtcDnsLog; import com.nis.domain.restful.NtcFtpLog; @@ -676,4 +677,40 @@ public class NtcLogSearchController extends BaseRestController { return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "RADIUS泛收日志检索成功", page, 0); } + + + @RequestMapping(value = "/ntcConnRecordLogs", method = RequestMethod.GET) + @ApiOperation(value = "通联关系日志查询", httpMethod = "GET", notes = "对日志功能“通联关系日志查询”提供数据基础查询服务") + public Map ntcConnRecordLogs(Page page, NtcConnRecordLog ntcConnRecordLog, Model model, + HttpServletRequest request, HttpServletResponse response) { + long start = System.currentTimeMillis(); + AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request, + null); + try { + if (StringUtil.isEmpty(ntcConnRecordLog.getSearchFoundStartTime()) + && StringUtil.isEmpty(ntcConnRecordLog.getSearchFoundEndTime())) { + Map map = DateUtils.getLocalTime(null, + null, Constants.LOG_LOCAL_TIME, "minute"); + ntcConnRecordLog.setSearchFoundStartTime(map.get("startTime")); + ntcConnRecordLog.setSearchFoundEndTime(map.get("endTime")); + } + ntcLogService.queryConditionCheck(auditLogThread, start,ntcConnRecordLog, NtcConnRecordLog.class, page); + logDataService.getData(page, ntcConnRecordLog); + } catch (Exception e) { + auditLogThread.setExceptionInfo("通联关系日志检索失败:"+e.getMessage()); + logger.error("通联关系日志检索失败："+ExceptionUtil.getExceptionMsg(e)); + if (e instanceof RestServiceException) { + throw new RestServiceException(auditLogThread, System.currentTimeMillis() - start, + "通联关系日志检索失败：" + e.getMessage(), ((RestServiceException) e).getErrorCode()); + } else if (e instanceof ServiceRuntimeException) { + throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start, + "通联关系日志检索失败：" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode()); + } else { + throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start, + "通联关系日志检索失败：" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue()); + } + } + return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "通联关系日志检索成功", page, + 0); + } } diff --git a/src/main/java/com/nis/web/dao/DfLogSearchDao.xml b/src/main/java/com/nis/web/dao/DfLogSearchDao.xml index 4faca4f..ac37fdc 100644 --- a/src/main/java/com/nis/web/dao/DfLogSearchDao.xml +++ b/src/main/java/com/nis/web/dao/DfLogSearchDao.xml @@ -1470,4 +1470,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/main/java/com/nis/web/dao/NtcReportDao.xml b/src/main/java/com/nis/web/dao/NtcReportDao.xml index 575ddcb..9107190 100644 --- a/src/main/java/com/nis/web/dao/NtcReportDao.xml +++ b/src/main/java/com/nis/web/dao/NtcReportDao.xml @@ -623,7 +623,7 @@ - group by NAS_IP order by num desc; + group by NAS_IP order by num desc limit 10; \ No newline at end of file diff --git a/src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java b/src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java new file mode 100644 index 0000000..3f199a8 --- /dev/null +++ b/src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java @@ -0,0 +1,268 @@ +package com.nis.web.dao.impl; + +import java.beans.BeanInfo; +import java.beans.Introspector; +import java.beans.PropertyDescriptor; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.ibatis.mapping.ResultMap; +import org.apache.ibatis.mapping.ResultMapping; +import org.apache.ibatis.session.SqlSessionFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Repository; + +import com.alibaba.druid.pool.DruidDataSource; +import com.nis.domain.Page; +import com.nis.util.Constants; +import com.nis.web.service.SpringContextHolder; +import com.zdjizhi.utils.StringUtil; + +/** + * + *

+ * Title: LogJDBCByDruid + *

+ *

+ * Description: 使用druid连接池对hive或clickhouse进行查询并解析结果 + *

+ *

+ * Company: IIE + *

+ * + * @author rkg + * @date 2018年8月20日 + * + */ +@Repository +public class LocalLogJDBCByDruid { + private final static Logger logger = LoggerFactory.getLogger(LocalLogJDBCByDruid.class); + static DruidDataSource datasource = null; + Connection conn = null; + ResultSet rs = null; + Statement st = null; + + private static Connection getConnection() throws SQLException { + if (datasource == null) { + datasource = (DruidDataSource) SpringContextHolder.getBean("LocalClickHouseDataSourceByDruid"); + } + return datasource.getConnection(); + } + + /** + * 根据sql从数据中心中获取日志并set到list中 + * + * @param page + * @param sql + * @param entityClass + * @throws Exception + */ + public void getTableData(Page page, String sql, Class entityClass) throws Exception { + List listObject = new ArrayList(); + try { + Map filedAndColumnMap = getColumn2FiledMap(entityClass); + // 不从Object... obj中获取需要date类型的字段了,调用的时候容易漏写,改为反射获取date类型的字段 + List columnList = getDateColumn(entityClass); + conn = getConnection(); + logger.info("连接数据中心日志库成功--------------------------"); + st = conn.createStatement(); + logger.info("开始执行日志查询语句sql={}", sql); + rs = st.executeQuery(sql); + ResultSetMetaData metaData = rs.getMetaData(); + while (rs.next()) { + Map map = new HashMap(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + Object value = rs.getObject(i); + String filedName = filedAndColumnMap.get(metaData.getColumnName(i).toString().toLowerCase()); + if (!StringUtil.isEmpty(value)) { + // 如果是日期类型的属性需要把时间戳转换成日期，如果时间戳为0直接把值设置为null + if (null != columnList && columnList.contains(filedName.toLowerCase())) { + long time = 0L; + time = Long.parseLong(value.toString()); + map.put(filedName, time == 0L ? null : new Date(time * 1000)); + } else { + map.put(filedName, value); + } + } else { + map.put(filedName, null); + } + } + listObject.add((T) map2Obj(map, entityClass)); + } + if (null == listObject || listObject.size() == 0) { + page.setList(new ArrayList()); + } else { + page.setList(listObject); + } + logger.info("执行日志查询语句成功,sql={}", sql); + } finally { + closeConn(); + } + } + + /** + * 关闭数据库连接 + */ + private void closeConn() { + try { + if (rs != null) { + rs.close(); + rs = null; + } + if (st != null) { + st.close(); + st = null; + } + if (conn != null) { + conn.close(); + conn = null; + } + logger.info("关闭数据中心连接成功"); + } catch (Exception e) { + e.printStackTrace(); + logger.error("关闭数据中心连接失败,失败原因" + e); + } + } + + /** + * 反射获取类中date类型的字段名称 + * + * @param type + * @return + * @throws Exception + */ + private static List getDateColumn(Class type) throws Exception { + List columnList = new ArrayList(); + BeanInfo beanInfo = Introspector.getBeanInfo(type); + PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors(); + for (int i = 0; i < propertyDescriptors.length; i++) { + PropertyDescriptor propertyDescriptor = propertyDescriptors[i]; + String name = propertyDescriptor.getName(); + String fieldTypeName = propertyDescriptor.getPropertyType().getName(); + if (fieldTypeName.equals("java.util.Date")) { + columnList.add(name.toLowerCase()); + } + } + return columnList; + } + + /** + * 将map中的数据利用反射set到Class中,并返回设置好值的对象 + * + * @param map + * @param beanClass + * @return + * @throws Exception + */ + private static Object map2Obj(Map map, Class beanClass) throws Exception { + BeanInfo beanInfo = Introspector.getBeanInfo(beanClass); + Object obj = beanClass.newInstance(); + PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors(); + for (int i = 0; i < propertyDescriptors.length; i++) { + PropertyDescriptor propertyDescriptor = propertyDescriptors[i]; + String name = propertyDescriptor.getName(); + String fieldTypeName = propertyDescriptor.getPropertyType().getName(); + if (map.containsKey(name)) { + Object value = map.get(name); + if (!StringUtil.isEmpty(value)) { + if (fieldTypeName.equals("java.lang.String")) { + propertyDescriptor.getWriteMethod().invoke(obj, value.toString()); + } else if (fieldTypeName.equals("java.lang.Integer")) { + propertyDescriptor.getWriteMethod().invoke(obj, Integer.parseInt(value.toString())); + } else if (fieldTypeName.equals("java.lang.Long")) { + propertyDescriptor.getWriteMethod().invoke(obj, Long.parseLong(value.toString())); + } else if (fieldTypeName.equals("java.lang.Boolean")) { + propertyDescriptor.getWriteMethod().invoke(obj, Boolean.parseBoolean(value.toString())); + } else if (fieldTypeName.equals("java.lang.Character")) { + propertyDescriptor.getWriteMethod().invoke(obj, value.toString().toCharArray()); + } else if (fieldTypeName.equals("java.lang.Byte")) { + propertyDescriptor.getWriteMethod().invoke(obj, value.toString().getBytes()); + } else if (fieldTypeName.equals("java.lang.Short")) { + propertyDescriptor.getWriteMethod().invoke(obj, Short.parseShort(value.toString())); + } else if (fieldTypeName.equals("java.lang.Float")) { + propertyDescriptor.getWriteMethod().invoke(obj, Float.parseFloat(value.toString())); + } else if (fieldTypeName.equals("java.lang.Double")) { + propertyDescriptor.getWriteMethod().invoke(obj, Double.parseDouble(value.toString())); + } else if (fieldTypeName.equals("java.math.BigDecimal")) { + propertyDescriptor.getWriteMethod().invoke(obj, + BigDecimal.valueOf(Long.parseLong(value.toString()))); + } else if (fieldTypeName.equals("java.util.Date")) { + propertyDescriptor.getWriteMethod().invoke(obj, (Date) value); + } + } + } + } + return obj; + } + + /** + * 根据class从DfLogSearchDao.xml中获取对应的resultMap里column与property的关系,key是column + * + * @param clazz + * @return + */ + private static Map getColumn2FiledMap(Class clazz) { + Map map = new HashMap(); + SqlSessionFactory sqlSessionFactory = SpringContextHolder.getBean(SqlSessionFactory.class); + ResultMap resultMap = sqlSessionFactory.getConfiguration().getResultMap(clazz.getSimpleName() + "Map"); + List mapping = resultMap.getResultMappings(); + for (ResultMapping mapp : mapping) { + map.put(mapp.getColumn().toLowerCase(), mapp.getProperty()); + } + return map; + + } + + /** + * 执行count的sql并将结果和计算的last的值set到page对象中 + * + * @param page + * @param sql + * @throws Exception + */ + public void getCount(Page page, String sql) throws Exception { + try { + conn = getConnection(); + logger.info("连接数据中心日志库成功--------------------------"); + st = conn.createStatement(); + logger.info("开始执行查询日志总条数sql={}", sql); + rs = st.executeQuery(sql); + String countStr = null; + while (rs.next()) { + countStr = rs.getObject(1).toString(); + break; + } + if (countStr == null || countStr.trim().equals("")) { + logger.info("获取数据中心日志总条数成功,总共0条日志,sql={}", sql); + page.setCount(0l); + page.setLast(1); + } else { + Long count = Long.valueOf(countStr); + logger.info("获取数据中心日志总条数成功,总共{}条日志,sql={}", count, sql); + page.setCount(count); + page.setLast(getLastPageNum(count.intValue(), page.getPageSize())); + } + } finally { + closeConn(); + } + } + + private int getLastPageNum(int totalCount, int pageSize) { + int pageNum = totalCount / pageSize; + if (totalCount % pageSize > 0) { + pageNum++; + } + return pageNum; + } + +} diff --git a/src/main/java/com/nis/web/service/LogDataService.java b/src/main/java/com/nis/web/service/LogDataService.java index d585450..06867a9 100644 --- a/src/main/java/com/nis/web/service/LogDataService.java +++ b/src/main/java/com/nis/web/service/LogDataService.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import com.nis.domain.Page; import com.nis.util.Configurations; import com.nis.util.Constants; +import com.nis.web.dao.impl.LocalLogJDBCByDruid; import com.nis.web.dao.impl.LogJDBCByDruid; import com.zdjizhi.utils.StringUtil; @@ -35,6 +36,8 @@ public class LogDataService { @Autowired private LogJDBCByDruid logJDBCByDruid; + @Autowired + private LocalLogJDBCByDruid localLogJDBCByDruid; private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private static SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMdd"); private static Map> col2col = new HashMap>(); @@ -210,7 +213,15 @@ public class LogDataService { sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") "); } sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同 - searchFromDataCenter(page, bean, sql, countSql); + + if(tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG_LOCAL")) { + searchFromLocalCK(page, bean, sql, countSql); + }else { + searchFromDataCenter(page, bean, sql, countSql); + } + + + } /** @@ -375,6 +386,21 @@ public class LogDataService { logger.info("没有查询到数据,sql={}",countSql.toString()); } + } + private void searchFromLocalCK(Page page, Object bean, StringBuffer selSql, StringBuffer countSql) + throws Exception { +// if (Constants.ISOPENLOGCOUNTANDLAST) { + localLogJDBCByDruid.getCount(page, countSql.toString()); +// } + if (page.getCount() > 0) { + localLogJDBCByDruid.getTableData(page, selSql.toString(), bean.getClass()); + if(page.getLast()>100) { + page.setLast(100); + } + } else { + logger.info("没有查询到数据,sql={}",countSql.toString()); + } + } /** diff --git a/src/main/java/com/nis/web/service/restful/NtcReportService.java b/src/main/java/com/nis/web/service/restful/NtcReportService.java index 7cb2976..3408289 100644 --- a/src/main/java/com/nis/web/service/restful/NtcReportService.java +++ b/src/main/java/com/nis/web/service/restful/NtcReportService.java @@ -147,5 +147,6 @@ public class NtcReportService extends BaseLogService { } } return resultList; + } } diff --git a/src/main/resources/applicationContext-mybatis.xml b/src/main/resources/applicationContext-mybatis.xml index eb7a2e8..9c93955 100644 --- a/src/main/resources/applicationContext-mybatis.xml +++ b/src/main/resources/applicationContext-mybatis.xml @@ -202,6 +202,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/applicationLog-clickhouse.properties b/src/main/resources/applicationLog-clickhouse.properties index 795b9d6..72aa286 100644 --- a/src/main/resources/applicationLog-clickhouse.properties +++ b/src/main/resources/applicationLog-clickhouse.properties @@ -37,6 +37,7 @@ MmFileDigestLogClickHouseTable=TBS_ODS_MM_FILE_DIGEST_LOG NtcKeywordsUrlLogClickHouseTable=TBS_ODS_NTC_KEYWORDS_URL_LOG NtcCollectVoipLogClickHouseTable=TBS_ODS_NTC_COLLECT_VOIP_LOG NtcCollectRadiusLogClickHouseTable=TBS_ODS_NTC_COLLECT_RADIUS_LOG +NtcConnRecordLogClickHouseTable=TBS_ODS_NTC_CONN_RECORD_LOG_LOCAL diff --git a/src/main/resources/jdbc.properties b/src/main/resources/jdbc.properties index 4c72855..3fe267a 100644 --- a/src/main/resources/jdbc.properties +++ b/src/main/resources/jdbc.properties @@ -104,8 +104,12 @@ jdbc.clickhouse.key=aUkjs+fcwf6p4rDqHiC+ng== jdbc.clickhouse.password=jOA3tbfJiJlPci6XUHIbVg== - - +#本地的clickhouse地址,新增的TBS_ODS_NTC_CONN_RECORD_LOG_LOCAL在本地 +jdbc.ckLocal.url=jdbc:clickhouse://10.0.8.14:8123/k18_ods?socket_timeout=90000 +jdbc.ckLocal.username=default +#实际密码ceiec2018 +jdbc.ckLocal.key=p8yBsnjQ2S4qT0XeSTi7lQ== +jdbc.ckLocal.password=obYXo/qhb8fDaQbTpX9slA== diff --git a/src/main/resources/jdbc.properties.astana b/src/main/resources/jdbc.properties.astana index d6678ee..da6fd93 100644 --- a/src/main/resources/jdbc.properties.astana +++ b/src/main/resources/jdbc.properties.astana @@ -39,6 +39,14 @@ jdbc.clickhouse.key=jT4yKPwwg2J4JE/KUZknnA== jdbc.clickhouse.password=XmaxnU2yLPQ5dpWjY3RDjQ== +jdbc.ckLocal.url=jdbc:clickhouse://10.0.8.14:8123/k18_ods?socket_timeout=90000 +jdbc.ckLocal.username=default +#实际密码k18 +jdbc.ckLocal.key=p8yBsnjQ2S4qT0XeSTi7lQ== +jdbc.ckLocal.password=obYXo/qhb8fDaQbTpX9slA== + + + #########################配置日志查询使用druid连接池######################################### #配置初始化连接池数量 druid.log.initialSize=5