gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1:新增通联关系日志查询接口

Browse Source 
2:新增从本地clickhouse查询的连接信息
This commit is contained in:
renkaige
2018-12-15 15:59:59 +06:00
parent 36229034a8
commit 4a436cdb74
12 changed files with 534 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
src/main/java/com/nis/domain/restful/NtcConnRecordLog.java Normal file
View File

@@ -0,0 +1,72 @@
package com.nis.domain.restful;
import com.nis.domain.LogEntity;
import com.wordnik.swagger.annotations.ApiModelProperty;
/**
*
* @ClassName:NtcConnRecordLog
* @Description:TODO(这里用一句话描述这个类的作用)
* @author rkg
* @date 2018年12月12日 下午2:00:16
* @version V1.0
*/
public class NtcConnRecordLog extends LogEntity<NtcConnRecordLog> {
/**
*
*/
private static final long serialVersionUID = 1L;
@ApiModelProperty(value = "", required = true)
protected Long appLabel;
@ApiModelProperty(value = "", required = true)
protected String c2sPktNum;
@ApiModelProperty(value = "", required = true)
protected String s2cPktNum;
@ApiModelProperty(value = "", required = true)
protected String c2sByteNum;
@ApiModelProperty(value = "", required = true)
protected String s2cByteNum;
public Long getAppLabel() {
return appLabel;
}
public void setAppLabel(Long appLabel) {
this.appLabel = appLabel;
}
public String getC2sPktNum() {
return c2sPktNum;
}
public void setC2sPktNum(String c2sPktNum) {
this.c2sPktNum = c2sPktNum;
}
public String getS2cPktNum() {
return s2cPktNum;
}
public void setS2cPktNum(String s2cPktNum) {
this.s2cPktNum = s2cPktNum;
}
public String getC2sByteNum() {
return c2sByteNum;
}
public void setC2sByteNum(String c2sByteNum) {
this.c2sByteNum = c2sByteNum;
}
public String getS2cByteNum() {
return s2cByteNum;
}
public void setS2cByteNum(String s2cByteNum) {
this.s2cByteNum = s2cByteNum;
}
}

14
src/main/java/com/nis/util/PropertyPlaceholderConfigurerCrypt.java
View File

@@ -30,6 +30,20 @@ public class PropertyPlaceholderConfigurerCrypt extends PropertyPlaceholderConfi
props.setProperty("jdbc.clickhouse.password",
new String(AESUtil.decrypt(Base64.decodeBase64(clickHousePassword), clickHouseScretKey)));
}
String ckLocalPassword = props.getProperty("jdbc.ckLocal.password");
String ckLocalScretKey = props.getProperty("jdbc.ckLocal.key");
if (null != ckLocalPassword) {
props.setProperty("jdbc.ckLocal.password",
new String(AESUtil.decrypt(Base64.decodeBase64(ckLocalPassword), ckLocalScretKey)));
}
// hive
String hivePassword = props.getProperty("jdbc.hive.password");
String hiveScretKey = props.getProperty("jdbc.hive.key");

37
src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java
View File

@@ -16,6 +16,7 @@ import com.nis.domain.restful.NtcAppLog;
import com.nis.domain.restful.NtcBgpLog;
import com.nis.domain.restful.NtcCollectRadiusLog;
import com.nis.domain.restful.NtcCollectVoipLog;
import com.nis.domain.restful.NtcConnRecordLog;
import com.nis.domain.restful.NtcDdosLog;
import com.nis.domain.restful.NtcDnsLog;
import com.nis.domain.restful.NtcFtpLog;
@@ -676,4 +677,40 @@ public class NtcLogSearchController extends BaseRestController {
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "RADIUS泛收日志检索成功", page,
0);
}
@RequestMapping(value = "/ntcConnRecordLogs", method = RequestMethod.GET)
@ApiOperation(value = "通联关系日志查询", httpMethod = "GET", notes = "对日志功能“通联关系日志查询”提供数据基础查询服务")
public Map<String, ?> ntcConnRecordLogs(Page page, NtcConnRecordLog ntcConnRecordLog, Model model,
HttpServletRequest request, HttpServletResponse response) {
long start = System.currentTimeMillis();
AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,
null);
try {
if (StringUtil.isEmpty(ntcConnRecordLog.getSearchFoundStartTime())
&& StringUtil.isEmpty(ntcConnRecordLog.getSearchFoundEndTime())) {
Map<String, String> map = DateUtils.getLocalTime(null,
null, Constants.LOG_LOCAL_TIME, "minute");
ntcConnRecordLog.setSearchFoundStartTime(map.get("startTime"));
ntcConnRecordLog.setSearchFoundEndTime(map.get("endTime"));
}
ntcLogService.queryConditionCheck(auditLogThread, start,ntcConnRecordLog, NtcConnRecordLog.class, page);
logDataService.getData(page, ntcConnRecordLog);
} catch (Exception e) {
auditLogThread.setExceptionInfo("通联关系日志检索失败:"+e.getMessage());
logger.error("通联关系日志检索失败："+ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(auditLogThread, System.currentTimeMillis() - start,
"通联关系日志检索失败：" + e.getMessage(), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
"通联关系日志检索失败：" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
"通联关系日志检索失败：" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
}
}
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "通联关系日志检索成功", page,
0);
}
}

55
src/main/java/com/nis/web/dao/DfLogSearchDao.xml
View File

@@ -1470,4 +1470,59 @@
<result column="framed_ip" jdbcType="VARCHAR" property="framedIp" />
<result column="account" jdbcType="VARCHAR" property="account" />
</resultMap>
<resultMap id="NtcConnRecordLogMap"
type="com.nis.domain.restful.NtcConnRecordLog">
<result column="cfg_id" jdbcType="BIGINT" property="cfgId" />
<result column="found_Time" jdbcType="TIMESTAMP" property="foundTime" />
<result column="recv_Time" jdbcType="TIMESTAMP" property="recvTime" />
<result column="trans_proto" jdbcType="VARCHAR" property="transProto" />
<result column="addr_type" jdbcType="INTEGER" property="addrType" />
<result column="d_ip" jdbcType="VARCHAR" property="dIp" />
<result column="s_ip" jdbcType="VARCHAR" property="sIp" />
<result column="d_port" jdbcType="VARCHAR" property="dPort" />
<result column="s_port" jdbcType="VARCHAR" property="sPort" />
<result column="service" jdbcType="INTEGER" property="service" />
<result column="entrance_id" jdbcType="BIGINT" property="entranceId" />
<result column="device_id" jdbcType="INTEGER" property="deviceId" />
<result column="direction" jdbcType="INTEGER" property="direction" />
<result column="stream_dir" jdbcType="INTEGER" property="streamDir" />
<result column="cap_ip" jdbcType="VARCHAR" property="capIp" />
<result column="addr_list" jdbcType="VARCHAR" property="addrList" />
<result column="user_region" jdbcType="VARCHAR" property="userRegion" />
<result column="server_locate" jdbcType="VARCHAR" property="serverLocate" />
<result column="client_locate" jdbcType="VARCHAR" property="clientLocate" />
<result column="s_asn" jdbcType="VARCHAR" property="sAsn" />
<result column="d_asn" jdbcType="VARCHAR" property="dAsn" />
<result column="s_subscribe_id" jdbcType="VARCHAR" property="sSubscribeId" />
<result column="d_subscribe_id" jdbcType="VARCHAR" property="dSubscribeId" />
<result column="scene_file" jdbcType="VARCHAR" property="sceneFile" />
<result column="link_id" jdbcType="BIGINT" property="linkId" />
<result column="encap_type" jdbcType="INTEGER" property="encapType" />
<result column="inner_smac" jdbcType="VARCHAR" property="innerSmac" />
<result column="inner_dmac" jdbcType="VARCHAR" property="innerDmac" />
<result column="app_label" jdbcType="BIGINT" property="appLabel" />
<result column="c2s_pkt_num" jdbcType="VARCHAR" property="c2sPktNum" />
<result column="s2c_pkt_num" jdbcType="VARCHAR" property="s2cPktNum" />
<result column="c2s_byte_num" jdbcType="VARCHAR" property="c2sByteNum" />
<result column="s2c_byte_num" jdbcType="VARCHAR" property="s2cByteNum" />
</resultMap>
</mapper>

6
src/main/java/com/nis/web/dao/NtcReportDao.xml
View File

@@ -623,7 +623,7 @@
</choose>
</where>
group by NAS_IP order by num desc;
group by NAS_IP order by num desc limit 10;
</select>
<select id="findAccounList" parameterType="com.nis.domain.restful.NtcRadiusReport"
resultMap="NtcRadiusReportMap">
@@ -639,7 +639,7 @@
</choose>
</where>
group by account order by num desc;
group by account order by num desc limit 10;
</select>
<select id="findNtcRadiusReport" parameterType="com.nis.domain.restful.NtcRadiusReport"
@@ -664,6 +664,6 @@
</choose>
</where>
ORDER BY
REPORT_TIME ASC
REPORT_TIME ASC limit 10
</select>
</mapper>

268
src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java Normal file
View File

@@ -0,0 +1,268 @@
package com.nis.web.dao.impl;
import java.beans.BeanInfo;
import java.beans.Introspector;
import java.beans.PropertyDescriptor;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.mapping.ResultMap;
import org.apache.ibatis.mapping.ResultMapping;
import org.apache.ibatis.session.SqlSessionFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Repository;
import com.alibaba.druid.pool.DruidDataSource;
import com.nis.domain.Page;
import com.nis.util.Constants;
import com.nis.web.service.SpringContextHolder;
import com.zdjizhi.utils.StringUtil;
/**
*
* <p>
* Title: LogJDBCByDruid
* </p>
* <p>
* Description: 使用druid连接池对hive或clickhouse进行查询并解析结果
* </p>
* <p>
* Company: IIE
* </p>
*
* @author rkg
* @date 2018年8月20日
*
*/
@Repository
public class LocalLogJDBCByDruid {
private final static Logger logger = LoggerFactory.getLogger(LocalLogJDBCByDruid.class);
static DruidDataSource datasource = null;
Connection conn = null;
ResultSet rs = null;
Statement st = null;
private static Connection getConnection() throws SQLException {
if (datasource == null) {
datasource = (DruidDataSource) SpringContextHolder.getBean("LocalClickHouseDataSourceByDruid");
}
return datasource.getConnection();
}
/**
* 根据sql从数据中心中获取日志并set到list中
*
* @param page
* @param sql
* @param entityClass
* @throws Exception
*/
public <T> void getTableData(Page<T> page, String sql, Class<?> entityClass) throws Exception {
List<T> listObject = new ArrayList<T>();
try {
Map<String, String> filedAndColumnMap = getColumn2FiledMap(entityClass);
// 不从Object... obj中获取需要date类型的字段了,调用的时候容易漏写,改为反射获取date类型的字段
List<String> columnList = getDateColumn(entityClass);
conn = getConnection();
logger.info("连接数据中心日志库成功--------------------------");
st = conn.createStatement();
logger.info("开始执行日志查询语句sql={}", sql);
rs = st.executeQuery(sql);
ResultSetMetaData metaData = rs.getMetaData();
while (rs.next()) {
Map<String, Object> map = new HashMap<String, Object>();
for (int i = 1; i <= metaData.getColumnCount(); i++) {
Object value = rs.getObject(i);
String filedName = filedAndColumnMap.get(metaData.getColumnName(i).toString().toLowerCase());
if (!StringUtil.isEmpty(value)) {
// 如果是日期类型的属性需要把时间戳转换成日期，如果时间戳为0直接把值设置为null
if (null != columnList && columnList.contains(filedName.toLowerCase())) {
long time = 0L;
time = Long.parseLong(value.toString());
map.put(filedName, time == 0L ? null : new Date(time * 1000));
} else {
map.put(filedName, value);
}
} else {
map.put(filedName, null);
}
}
listObject.add((T) map2Obj(map, entityClass));
}
if (null == listObject || listObject.size() == 0) {
page.setList(new ArrayList());
} else {
page.setList(listObject);
}
logger.info("执行日志查询语句成功,sql={}", sql);
} finally {
closeConn();
}
}
/**
* 关闭数据库连接
*/
private void closeConn() {
try {
if (rs != null) {
rs.close();
rs = null;
}
if (st != null) {
st.close();
st = null;
}
if (conn != null) {
conn.close();
conn = null;
}
logger.info("关闭数据中心连接成功");
} catch (Exception e) {
e.printStackTrace();
logger.error("关闭数据中心连接失败,失败原因" + e);
}
}
/**
* 反射获取类中date类型的字段名称
*
* @param type
* @return
* @throws Exception
*/
private static List<String> getDateColumn(Class<?> type) throws Exception {
List<String> columnList = new ArrayList<String>();
BeanInfo beanInfo = Introspector.getBeanInfo(type);
PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors();
for (int i = 0; i < propertyDescriptors.length; i++) {
PropertyDescriptor propertyDescriptor = propertyDescriptors[i];
String name = propertyDescriptor.getName();
String fieldTypeName = propertyDescriptor.getPropertyType().getName();
if (fieldTypeName.equals("java.util.Date")) {
columnList.add(name.toLowerCase());
}
}
return columnList;
}
/**
* 将map中的数据利用反射set到Class中,并返回设置好值的对象
*
* @param map
* @param beanClass
* @return
* @throws Exception
*/
private static Object map2Obj(Map<String, Object> map, Class<?> beanClass) throws Exception {
BeanInfo beanInfo = Introspector.getBeanInfo(beanClass);
Object obj = beanClass.newInstance();
PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors();
for (int i = 0; i < propertyDescriptors.length; i++) {
PropertyDescriptor propertyDescriptor = propertyDescriptors[i];
String name = propertyDescriptor.getName();
String fieldTypeName = propertyDescriptor.getPropertyType().getName();
if (map.containsKey(name)) {
Object value = map.get(name);
if (!StringUtil.isEmpty(value)) {
if (fieldTypeName.equals("java.lang.String")) {
propertyDescriptor.getWriteMethod().invoke(obj, value.toString());
} else if (fieldTypeName.equals("java.lang.Integer")) {
propertyDescriptor.getWriteMethod().invoke(obj, Integer.parseInt(value.toString()));
} else if (fieldTypeName.equals("java.lang.Long")) {
propertyDescriptor.getWriteMethod().invoke(obj, Long.parseLong(value.toString()));
} else if (fieldTypeName.equals("java.lang.Boolean")) {
propertyDescriptor.getWriteMethod().invoke(obj, Boolean.parseBoolean(value.toString()));
} else if (fieldTypeName.equals("java.lang.Character")) {
propertyDescriptor.getWriteMethod().invoke(obj, value.toString().toCharArray());
} else if (fieldTypeName.equals("java.lang.Byte")) {
propertyDescriptor.getWriteMethod().invoke(obj, value.toString().getBytes());
} else if (fieldTypeName.equals("java.lang.Short")) {
propertyDescriptor.getWriteMethod().invoke(obj, Short.parseShort(value.toString()));
} else if (fieldTypeName.equals("java.lang.Float")) {
propertyDescriptor.getWriteMethod().invoke(obj, Float.parseFloat(value.toString()));
} else if (fieldTypeName.equals("java.lang.Double")) {
propertyDescriptor.getWriteMethod().invoke(obj, Double.parseDouble(value.toString()));
} else if (fieldTypeName.equals("java.math.BigDecimal")) {
propertyDescriptor.getWriteMethod().invoke(obj,
BigDecimal.valueOf(Long.parseLong(value.toString())));
} else if (fieldTypeName.equals("java.util.Date")) {
propertyDescriptor.getWriteMethod().invoke(obj, (Date) value);
}
}
}
}
return obj;
}
/**
* 根据class从DfLogSearchDao.xml中获取对应的resultMap里column与property的关系,key是column
*
* @param clazz
* @return
*/
private static Map<String, String> getColumn2FiledMap(Class<?> clazz) {
Map<String, String> map = new HashMap<String, String>();
SqlSessionFactory sqlSessionFactory = SpringContextHolder.getBean(SqlSessionFactory.class);
ResultMap resultMap = sqlSessionFactory.getConfiguration().getResultMap(clazz.getSimpleName() + "Map");
List<ResultMapping> mapping = resultMap.getResultMappings();
for (ResultMapping mapp : mapping) {
map.put(mapp.getColumn().toLowerCase(), mapp.getProperty());
}
return map;
}
/**
* 执行count的sql并将结果和计算的last的值set到page对象中
*
* @param page
* @param sql
* @throws Exception
*/
public <T> void getCount(Page<T> page, String sql) throws Exception {
try {
conn = getConnection();
logger.info("连接数据中心日志库成功--------------------------");
st = conn.createStatement();
logger.info("开始执行查询日志总条数sql={}", sql);
rs = st.executeQuery(sql);
String countStr = null;
while (rs.next()) {
countStr = rs.getObject(1).toString();
break;
}
if (countStr == null || countStr.trim().equals("")) {
logger.info("获取数据中心日志总条数成功,总共0条日志,sql={}", sql);
page.setCount(0l);
page.setLast(1);
} else {
Long count = Long.valueOf(countStr);
logger.info("获取数据中心日志总条数成功,总共{}条日志,sql={}", count, sql);
page.setCount(count);
page.setLast(getLastPageNum(count.intValue(), page.getPageSize()));
}
} finally {
closeConn();
}
}
private int getLastPageNum(int totalCount, int pageSize) {
int pageNum = totalCount / pageSize;
if (totalCount % pageSize > 0) {
pageNum++;
}
return pageNum;
}
}

28
src/main/java/com/nis/web/service/LogDataService.java
View File

@@ -20,6 +20,7 @@ import org.springframework.stereotype.Service;
import com.nis.domain.Page;
import com.nis.util.Configurations;
import com.nis.util.Constants;
import com.nis.web.dao.impl.LocalLogJDBCByDruid;
import com.nis.web.dao.impl.LogJDBCByDruid;
import com.zdjizhi.utils.StringUtil;
@@ -35,6 +36,8 @@ public class LogDataService {
@Autowired
private LogJDBCByDruid logJDBCByDruid;
@Autowired
private LocalLogJDBCByDruid localLogJDBCByDruid;
private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private static SimpleDateFormat sdf2 = new SimpleDateFormat("yyyyMMdd");
private static Map<String, Map<String, String>> col2col = new HashMap<String, Map<String, String>>();
@@ -210,7 +213,15 @@ public class LogDataService {
sql.append(whereSB.substring(indexOf) + " and found_time in(" + foundTimeSql + ") ");
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
searchFromDataCenter(page, bean, sql, countSql);
if(tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG_LOCAL")) {
searchFromLocalCK(page, bean, sql, countSql);
}else {
searchFromDataCenter(page, bean, sql, countSql);
}
}
/**
@@ -375,6 +386,21 @@ public class LogDataService {
logger.info("没有查询到数据,sql={}",countSql.toString());
}
}
private <T> void searchFromLocalCK(Page<T> page, Object bean, StringBuffer selSql, StringBuffer countSql)
throws Exception {
// if (Constants.ISOPENLOGCOUNTANDLAST) {
localLogJDBCByDruid.getCount(page, countSql.toString());
// }
if (page.getCount() > 0) {
localLogJDBCByDruid.getTableData(page, selSql.toString(), bean.getClass());
if(page.getLast()>100) {
page.setLast(100);
}
} else {
logger.info("没有查询到数据,sql={}",countSql.toString());
}
}
/**

1
src/main/java/com/nis/web/service/restful/NtcReportService.java
View File

@@ -147,5 +147,6 @@ public class NtcReportService extends BaseLogService {
}
}
return resultList;
}
}