gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1:添加通联关系日志百分比查询接口

Browse Source 
2:用户行为统计删除只取前10个
This commit is contained in:
renkaige
2018-12-15 20:36:05 +06:00
parent 68e37c8f56
commit e9f838998b
6 changed files with 313 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
src/main/java/com/nis/domain/restful/NtcConnRecordPercent.java Normal file
View File

@@ -0,0 +1,83 @@
package com.nis.domain.restful;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonIgnore;
/**
*
* @ClassName:NtcConnRecordLog
* @Description:TODO(这里用一句话描述这个类的作用)
* @author rkg
* @date 2018年12月12日 下午2:00:16
* @version V1.0
*/
public class NtcConnRecordPercent implements Serializable {
private String bps;
private String bpsPercent;
private String pps;
private String ppsPercent;
private String searchFoundStartTime;// 开始发现时间
private String searchFoundEndTime;// 结束发现时间
private String searchStreamDir;// 流类型
public String getBps() {
return bps;
}
public void setBps(String bps) {
this.bps = bps;
}
public String getBpsPercent() {
return bpsPercent;
}
public void setBpsPercent(String bpsPercent) {
this.bpsPercent = bpsPercent;
}
public String getPps() {
return pps;
}
public void setPps(String pps) {
this.pps = pps;
}
public String getPpsPercent() {
return ppsPercent;
}
public void setPpsPercent(String ppsPercent) {
this.ppsPercent = ppsPercent;
}
@JsonIgnore
public String getSearchFoundStartTime() {
return searchFoundStartTime;
}
public void setSearchFoundStartTime(String searchFoundStartTime) {
this.searchFoundStartTime = searchFoundStartTime;
}
@JsonIgnore
public String getSearchFoundEndTime() {
return searchFoundEndTime;
}
public void setSearchFoundEndTime(String searchFoundEndTime) {
this.searchFoundEndTime = searchFoundEndTime;
}
@JsonIgnore
public String getSearchStreamDir() {
return searchStreamDir;
}
public void setSearchStreamDir(String searchStreamDir) {
this.searchStreamDir = searchStreamDir;
}
}

92
src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java
View File

@@ -1,5 +1,7 @@
package com.nis.web.controller.restful;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
@@ -17,6 +19,7 @@ import com.nis.domain.restful.NtcBgpLog;
import com.nis.domain.restful.NtcCollectRadiusLog;
import com.nis.domain.restful.NtcCollectVoipLog;
import com.nis.domain.restful.NtcConnRecordLog;
import com.nis.domain.restful.NtcConnRecordPercent;
import com.nis.domain.restful.NtcDdosLog;
import com.nis.domain.restful.NtcDnsLog;
import com.nis.domain.restful.NtcFtpLog;
@@ -615,13 +618,13 @@ public class NtcLogSearchController extends BaseRestController {
try {
if (StringUtil.isEmpty(ntcCollectVoipLog.getSearchFoundStartTime())
&& StringUtil.isEmpty(ntcCollectVoipLog.getSearchFoundEndTime())) {
Map<String, String> map = DateUtils.getLocalTime(null,
null, Constants.LOG_LOCAL_TIME, "minute");
Map<String, String> map = DateUtils.getLocalTime(null, null, Constants.LOG_LOCAL_TIME, "minute");
ntcCollectVoipLog.setSearchFoundStartTime(map.get("startTime"));
ntcCollectVoipLog.setSearchFoundEndTime(map.get("endTime"));
}
ntcLogService.collectConditionCheck(auditLogThread, start,ntcCollectVoipLog, NtcCollectVoipLog.class, page);
ntcLogService.collectConditionCheck(auditLogThread, start, ntcCollectVoipLog, NtcCollectVoipLog.class,
page);
logDataService.getData(page, ntcCollectVoipLog);
} catch (Exception e) {
auditLogThread.setExceptionInfo("VoIp泛收日志检索失败:" + e.getMessage());
@@ -637,8 +640,7 @@ public class NtcLogSearchController extends BaseRestController {
"VoIp泛收日志检索失败：" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
}
}
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "VoIp泛收日志检索成功", page,
0);
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "VoIp泛收日志检索成功", page, 0);
}
@RequestMapping(value = "/ntcCollectRadiusLogs", method = RequestMethod.GET)
@@ -652,13 +654,13 @@ public class NtcLogSearchController extends BaseRestController {
try {
if (StringUtil.isEmpty(ntcCollectRadiusLog.getSearchFoundStartTime())
&& StringUtil.isEmpty(ntcCollectRadiusLog.getSearchFoundEndTime())) {
Map<String, String> map = DateUtils.getLocalTime(null,
null, Constants.LOG_LOCAL_TIME, "minute");
Map<String, String> map = DateUtils.getLocalTime(null, null, Constants.LOG_LOCAL_TIME, "minute");
ntcCollectRadiusLog.setSearchFoundStartTime(map.get("startTime"));
ntcCollectRadiusLog.setSearchFoundEndTime(map.get("endTime"));
}
ntcLogService.queryConditionCheck(auditLogThread, start,ntcCollectRadiusLog, NtcCollectRadiusLog.class, page);
ntcLogService.queryConditionCheck(auditLogThread, start, ntcCollectRadiusLog, NtcCollectRadiusLog.class,
page);
logDataService.getData(page, ntcCollectRadiusLog);
} catch (Exception e) {
auditLogThread.setExceptionInfo("RADIUS泛收日志检索失败:" + e.getMessage());
@@ -678,7 +680,6 @@ public class NtcLogSearchController extends BaseRestController {
0);
}
@RequestMapping(value = "/ntcConnRecordLogs", method = RequestMethod.GET)
@ApiOperation(value = "通联关系日志查询", httpMethod = "GET", notes = "对日志功能“通联关系日志查询”提供数据基础查询服务")
public Map<String, ?> ntcConnRecordLogs(Page page, NtcConnRecordLog ntcConnRecordLog, Model model,
@@ -689,8 +690,7 @@ public class NtcLogSearchController extends BaseRestController {
try {
if (StringUtil.isEmpty(ntcConnRecordLog.getSearchFoundStartTime())
&& StringUtil.isEmpty(ntcConnRecordLog.getSearchFoundEndTime())) {
Map<String, String> map = DateUtils.getLocalTime(null,
null, Constants.LOG_LOCAL_TIME, "minute");
Map<String, String> map = DateUtils.getLocalTime(null, null, Constants.LOG_LOCAL_TIME, "minute");
ntcConnRecordLog.setSearchFoundStartTime(map.get("startTime"));
ntcConnRecordLog.setSearchFoundEndTime(map.get("endTime"));
}
@@ -710,7 +710,73 @@ public class NtcLogSearchController extends BaseRestController {
"通联关系日志检索失败：" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
}
}
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "通联关系日志检索成功", page,
0);
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "通联关系日志检索成功", page, 0);
}
@RequestMapping(value = "/ntcConnRecordPercent", method = RequestMethod.GET)
@ApiOperation(value = "通联关系日志百分比查询", httpMethod = "GET", notes = "对日志功能“通联关系日志百分比查询”提供数据基础查询服务")
public Map<String, ?> ntcConnRecordPercent(NtcConnRecordPercent ntcConnRecordPercent, Model model,
HttpServletRequest request, HttpServletResponse response) {
long start = System.currentTimeMillis();
AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,
null);
try {
if (StringUtil.isEmpty(ntcConnRecordPercent.getSearchFoundStartTime())
&& StringUtil.isEmpty(ntcConnRecordPercent.getSearchFoundEndTime())) {
Map<String, String> map = DateUtils.getLocalTime(null, null, Constants.LOG_LOCAL_TIME, "minute");
ntcConnRecordPercent.setSearchFoundStartTime(map.get("startTime"));
ntcConnRecordPercent.setSearchFoundEndTime(map.get("endTime"));
}
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
try {
if (!StringUtil.isBlank(ntcConnRecordPercent.getSearchFoundEndTime())) {
sdf.setLenient(false);
sdf.parse(ntcConnRecordPercent.getSearchFoundEndTime());
}else {
throw new RestServiceException("searchFoundEndTime参数不能为空",
RestBusinessCode.param_formate_error.getValue());
}
} catch (ParseException e) {
throw new RestServiceException("searchFoundEndTime参数格式错误",
RestBusinessCode.param_formate_error.getValue());
} catch (Exception e) {
throw new RestServiceException("searchFoundEndTime参数错误",
RestBusinessCode.param_formate_error.getValue());
}
try {
if (!StringUtil.isBlank(ntcConnRecordPercent.getSearchFoundStartTime())) {
sdf.setLenient(false);
sdf.parse(ntcConnRecordPercent.getSearchFoundStartTime());
}else {
throw new RestServiceException("searchFoundStartTime参数不能为空",
RestBusinessCode.param_formate_error.getValue());
}
} catch (ParseException e) {
throw new RestServiceException("searchFoundStartTime参数格式错误",
RestBusinessCode.param_formate_error.getValue());
} catch (Exception e) {
throw new RestServiceException("searchFoundStartTime参数错误",
RestBusinessCode.param_formate_error.getValue());
}
logDataService.getNtcConnRecordPercent(ntcConnRecordPercent);
} catch (Exception e) {
auditLogThread.setExceptionInfo("通联关系日志检索失败:" + e.getMessage());
logger.error("通联关系日志百分比检索失败：" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(auditLogThread, System.currentTimeMillis() - start,
"通联关系日志百分比检索失败：" + e.getMessage(), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
"通联关系日志百分比检索失败：" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
"通联关系日志百分比检索失败：" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
}
}
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "通联关系日志百分比检索成功",
ntcConnRecordPercent, 0);
}
}

6
src/main/java/com/nis/web/dao/NtcReportDao.xml
View File

@@ -623,7 +623,7 @@
</choose>
</where>
group by NAS_IP order by num desc limit 10;
group by NAS_IP order by num desc
</select>
<select id="findAccounList" parameterType="com.nis.domain.restful.NtcRadiusReport"
resultMap="NtcRadiusReportMap">
@@ -639,7 +639,7 @@
</choose>
</where>
group by account order by num desc limit 10;
group by account order by num desc
</select>
<select id="findNtcRadiusReport" parameterType="com.nis.domain.restful.NtcRadiusReport"
@@ -664,6 +664,6 @@
</choose>
</where>
ORDER BY
REPORT_TIME ASC limit 10
REPORT_TIME ASC
</select>
</mapper>

45
src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java
View File

@@ -9,6 +9,7 @@ import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@@ -24,7 +25,7 @@ import org.springframework.stereotype.Repository;
import com.alibaba.druid.pool.DruidDataSource;
import com.nis.domain.Page;
import com.nis.util.Constants;
import com.nis.domain.restful.NtcConnRecordPercent;
import com.nis.web.service.SpringContextHolder;
import com.zdjizhi.utils.StringUtil;
@@ -110,6 +111,48 @@ public class LocalLogJDBCByDruid {
}
}
public NtcConnRecordPercent getNtcConnRecordPercentData(NtcConnRecordPercent ntcConnRecordPercent, String sql,
boolean isPersent) throws Exception {
try {
conn = getConnection();
logger.info("连接数据中心日志库成功--------------------------");
st = conn.createStatement();
logger.info("开始执行日志查询语句sql={}", sql);
rs = st.executeQuery(sql);
NumberFormat num = NumberFormat.getPercentInstance();
num.setMinimumFractionDigits(2);// 保留两位小数
while (rs.next()) {
if (isPersent) {
String allBps = rs.getString("bps");
if (!allBps.equals("0")) {
ntcConnRecordPercent.setBpsPercent(
num.format(Double.valueOf(ntcConnRecordPercent.getBps()) / Double.valueOf(allBps)));
ntcConnRecordPercent.setBps(num.format(Double.valueOf(ntcConnRecordPercent.getBps())));
} else {
ntcConnRecordPercent.setBpsPercent("0%");
}
String allPps = rs.getString("pps");
if (!allPps.equals("0")) {
ntcConnRecordPercent.setPpsPercent(
num.format(Double.valueOf(ntcConnRecordPercent.getPps()) / Double.valueOf(allPps)));
ntcConnRecordPercent.setPps(num.format(Double.valueOf(ntcConnRecordPercent.getPps())));
} else {
ntcConnRecordPercent.setPpsPercent("0%");
}
} else {
String bps = rs.getString("bps");
String pps = rs.getString("pps");
ntcConnRecordPercent.setBps(bps);
ntcConnRecordPercent.setPps(pps);
}
}
logger.info("执行日志查询语句成功,sql={}", sql);
} finally {
closeConn();
}
return ntcConnRecordPercent;
}
/**
* 关闭数据库连接
*/

68
src/main/java/com/nis/web/service/LogDataService.java
View File

@@ -2,6 +2,7 @@ package com.nis.web.service;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
@@ -18,6 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.nis.domain.Page;
import com.nis.domain.restful.NtcConnRecordPercent;
import com.nis.restful.RestBusinessCode;
import com.nis.restful.RestServiceException;
import com.nis.util.Configurations;
import com.nis.util.Constants;
import com.nis.web.dao.impl.LocalLogJDBCByDruid;
@@ -50,6 +54,34 @@ public class LogDataService {
col2col.put("searchFoundEndTime", endMap);
}
public NtcConnRecordPercent getNtcConnRecordPercent(NtcConnRecordPercent ntcConnRecordPercent) throws Exception {
long startTime = sdf.parse(ntcConnRecordPercent.getSearchFoundStartTime().toString().trim()).getTime() / 1000;
long endTime = sdf.parse(ntcConnRecordPercent.getSearchFoundEndTime().toString().trim()).getTime() / 1000;
if (endTime - startTime < 0) {
throw new RestServiceException("searchFoundStartTime() can not exceed searchFoundEndTime",
RestBusinessCode.param_formate_error.getValue());
}
long second = endTime - startTime;
StringBuffer sql = new StringBuffer();
sql.append("SELECT SUM(s2c_pkt_num + s2c_pkt_num)*8/");
sql.append(second);
sql.append(" AS pps , SUM(c2s_byte_num + s2c_byte_num)*8/");
sql.append(second);
sql.append(
" AS bps FROM tbs_ods_ntc_conn_record_log_local_1 t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local_1 WHERE found_time >= ");
sql.append(startTime);
sql.append(" and found_time< ");
sql.append(endTime);
StringBuffer totalSql = new StringBuffer();
totalSql.append(sql);
totalSql.append(" and stream_dir in(1,2,3)) and stream_dir in(1,2,3)");
sql.append(" and stream_dir in(1,2) ) and stream_dir in(1,2)");
localLogJDBCByDruid.getNtcConnRecordPercentData(ntcConnRecordPercent, sql.toString(), false);
localLogJDBCByDruid.getNtcConnRecordPercentData(ntcConnRecordPercent, totalSql.toString(), true);
return ntcConnRecordPercent;
}
/**
* 根据类名加对应的标识获取hive或者clickhouse中对应的表名
*
@@ -95,16 +127,11 @@ public class LogDataService {
/**
* 从clickhouse中查询数据,注意clickhouse区分大小写,目前和百分点商定都是用小写
*
* @param page
* 里面含有pagesize和pageno,order by
* @param bean
* 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名
* @param tableName
* 表名
* @param className
* 类名
* @param orderBy
* orderby条件
* @param page 里面含有pagesize和pageno,order by
* @param bean 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名
* @param tableName 表名
* @param className 类名
* @param orderBy orderby条件
* @throws Exception
*/
private <T> void getDataFromClickHouse(Page<T> page, Object bean, String tableName, String className,
@@ -172,7 +199,8 @@ public class LogDataService {
if (field.equals("url")) {
whereSB.append(" and " + field + " like '" + value.toString().trim() + "%'");
} else {
whereSB.append(" and " + field + "='" + StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'");
whereSB.append(" and " + field + "='"
+ StringEscapeUtils.unescapeHtml4(value.toString().trim()) + "'");
}
} else if (type.equals("java.lang.Integer") || type.equals("int")
|| type.equals("java.lang.Long") || type.equals("long")) {
@@ -220,23 +248,16 @@ public class LogDataService {
searchFromDataCenter(page, bean, sql, countSql);
}
}
/**
* 从hive中查询数据
*
* @param page
* 里面含有pagesize和pageno,order by
* @param bean
* 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名
* @param tableName
* 表名
* @param className
* 类名
* @param orderBy
* orderby条件
* @param page 里面含有pagesize和pageno,order by
* @param bean 日志类对象(从DfLogSearchDao.xml中获取对应的map,类名+Map),用来获取各个属性对应的数据库字段名
* @param tableName 表名
* @param className 类名
* @param orderBy orderby条件
* @throws Exception
*/
private <T> void getDataFromHive(Page<T> page, Object bean, String tableName, String className, String orderBy)
@@ -387,6 +408,7 @@ public class LogDataService {
}
}
private <T> void searchFromLocalCK(Page<T> page, Object bean, StringBuffer selSql, StringBuffer countSql)
throws Exception {
// if (Constants.ISOPENLOGCOUNTANDLAST) {

2
src/main/resources/jdbc.properties.astana
View File

@@ -41,7 +41,7 @@ jdbc.clickhouse.password=XmaxnU2yLPQ5dpWjY3RDjQ==
jdbc.ckLocal.url=jdbc:clickhouse://10.0.8.14:8123/k18_ods?socket_timeout=90000
jdbc.ckLocal.username=default
#实际密码k18
#实际密码ceiec2018
jdbc.ckLocal.key=p8yBsnjQ2S4qT0XeSTi7lQ==
jdbc.ckLocal.password=obYXo/qhb8fDaQbTpX9slA==