gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1:修改线上配置文件applicationContext-redis.xml.astana中哨兵的名称masterName,及jdbc中mysql的地址

Browse Source 
2:修改ip范围统计的sql,及解决ip范围统计使用linkedlist速度很慢的原因
3:新增asn统计接口(ntcAsnRecord)从mysql查询,老的接口(从clickhouse中查询)改为ntcAsnRecordold
This commit is contained in:
renkaige
2019-01-10 11:21:29 +06:00
parent b38a3652dd
commit 99d3b87e1a
11 changed files with 198 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/com/nis/domain/restful/NtcAsnRecord.java → src/main/java/com/nis/domain/restful/dashboard/TrafficAsnStatistic.java
View File

@@ -1,8 +1,13 @@
package com.nis.domain.restful;
package com.nis.domain.restful.dashboard;
import java.io.Serializable;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.poi.ss.formula.functions.T;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.nis.domain.Page;
/**
*
@@ -12,7 +17,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
* @date 2018年12月12日 下午2:00:16
* @version V1.0
*/
public class NtcAsnRecord implements Serializable {
public class TrafficAsnStatistic<T> implements Serializable {
/**
*
@@ -29,6 +34,10 @@ public class NtcAsnRecord implements Serializable {
private String searchAsnType;// 1是d_asn,2是s_asn
protected Page<T> page;
public String getBps() {
return bps;
}
@@ -88,4 +97,19 @@ public class NtcAsnRecord implements Serializable {
public void setSearchFoundEndTime(String searchFoundEndTime) {
this.searchFoundEndTime = searchFoundEndTime;
}
@JsonIgnore
@XmlTransient
public Page<T> getPage() {
if (page == null) {
page = new Page<T>();
}
return page;
}
public Page<T> setPage(Page<T> page) {
this.page = page;
return page;
}
}

100
src/main/java/com/nis/web/controller/restful/DashboardServiceController.java
View File

@@ -21,11 +21,13 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.nis.datasource.CustomerContextHolder;
import com.nis.domain.Page;
import com.nis.domain.restful.NtcConnNumReport;
import com.nis.domain.restful.NtcRadiusReport;
import com.nis.domain.restful.dashboard.AppConnRecordStatistic;
import com.nis.domain.restful.dashboard.AppTrendEntity;
import com.nis.domain.restful.dashboard.TrafficAppFocusStatistic;
import com.nis.domain.restful.dashboard.TrafficAsnStatistic;
import com.nis.restful.RestBusinessCode;
import com.nis.restful.RestServiceException;
import com.nis.restful.ServiceRuntimeException;
@@ -1088,6 +1090,104 @@ public class DashboardServiceController extends BaseRestController {
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "App通联关系Top100数据检索成功", list,
0);
}
@RequestMapping(value = "/ntcAsnRecord", method = RequestMethod.GET)
@ApiOperation(value = "ASN通联关系(源,目的)查询", httpMethod = "GET", notes = "对日志功能“ASN通联关系(源,目的)查询”提供数据基础查询服务")
public Map<String, ?> ntcAsnRecord(Page page, TrafficAsnStatistic ntcAsnRecord, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis();
AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,
null);
try {
if (StringUtil.isEmpty(ntcAsnRecord.getSearchFoundStartTime())
&& StringUtil.isEmpty(ntcAsnRecord.getSearchFoundEndTime())) {
Map<String, String> map = DateUtils.getLocalTime(null, null, Constants.LOG_LOCAL_TIME, "minute");
ntcAsnRecord.setSearchFoundStartTime(map.get("startTime"));
ntcAsnRecord.setSearchFoundEndTime(map.get("endTime"));
}
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
try {
if (!StringUtil.isBlank(ntcAsnRecord.getSearchFoundEndTime())) {
sdf.setLenient(false);
sdf.parse(ntcAsnRecord.getSearchFoundEndTime());
}
} catch (ParseException e) {
throw new RestServiceException("searchFoundEndTime参数格式错误",
RestBusinessCode.param_formate_error.getValue());
} catch (Exception e) {
throw new RestServiceException("searchFoundEndTime参数错误",
RestBusinessCode.param_formate_error.getValue());
}
try {
if (!StringUtil.isBlank(ntcAsnRecord.getSearchFoundStartTime())) {
sdf.setLenient(false);
sdf.parse(ntcAsnRecord.getSearchFoundStartTime());
}
} catch (ParseException e) {
throw new RestServiceException("searchFoundStartTime参数格式错误",
RestBusinessCode.param_formate_error.getValue());
} catch (Exception e) {
throw new RestServiceException("searchFoundStartTime参数错误",
RestBusinessCode.param_formate_error.getValue());
}
try {
if (!StringUtil.isBlank(ntcAsnRecord.getSearchAsnType())) {
Integer.parseInt(ntcAsnRecord.getSearchAsnType());
} else {
throw new RestServiceException("searchAsnType参数不能为空",
RestBusinessCode.param_formate_error.getValue());
}
} catch (NumberFormatException e) {
throw new RestServiceException("searchAsnType参数格式错误" + e.getMessage(),
RestBusinessCode.param_formate_error.getValue());
} catch (Exception e) {
throw new RestServiceException("searchAsnType参数错误" + e.getMessage(),
RestBusinessCode.param_formate_error.getValue());
}
dashboardService.getTrafficAsnStatisticList(page, ntcAsnRecord);
} catch (Exception e) {
auditLogThread.setExceptionInfo("ASN通联关系(源,目的)日志检索失败:" + e.getMessage());
logger.error("ASN通联关系(源,目的)检索失败：" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(auditLogThread, System.currentTimeMillis() - start,
"ASN通联关系(源,目的)检索失败：" + e.getMessage(), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
"ASN通联关系(源,目的)检索失败：" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(auditLogThread, System.currentTimeMillis() - start,
"ASN通联关系(源,目的)检索失败：" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
}
}
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "ASN通联关系(源,目的)检索成功",
page, 0);
}
/**
*/
public void checkAppTrendCondition(AppTrendEntity entity) {

6
src/main/java/com/nis/web/controller/restful/NtcLogSearchController.java
View File

@@ -15,7 +15,6 @@ import org.springframework.web.bind.annotation.RestController;
import com.nis.domain.Page;
import com.nis.domain.restful.NtcAppLog;
import com.nis.domain.restful.NtcAsnRecord;
import com.nis.domain.restful.NtcBgpLog;
import com.nis.domain.restful.NtcCollectRadiusLog;
import com.nis.domain.restful.NtcCollectVoipLog;
@@ -37,6 +36,7 @@ import com.nis.domain.restful.NtcSshLog;
import com.nis.domain.restful.NtcSslLog;
import com.nis.domain.restful.NtcStreamingMediaLog;
import com.nis.domain.restful.NtcVoipLog;
import com.nis.domain.restful.dashboard.TrafficAsnStatistic;
import com.nis.restful.RestBusinessCode;
import com.nis.restful.RestServiceException;
import com.nis.restful.ServiceRuntimeException;
@@ -788,9 +788,9 @@ public class NtcLogSearchController extends BaseRestController {
ntcConnRecordPercent, 0);
}
@RequestMapping(value = "/ntcAsnRecord", method = RequestMethod.GET)
@RequestMapping(value = "/ntcAsnRecordold", method = RequestMethod.GET)
@ApiOperation(value = "ASN通联关系(源,目的)查询", httpMethod = "GET", notes = "对日志功能“ASN通联关系(源,目的)查询”提供数据基础查询服务")
public Map<String, ?> ntcAsnRecord(Page page, NtcAsnRecord ntcAsnRecord, Model model, HttpServletRequest request,
public Map<String, ?> ntcAsnRecord(Page page, TrafficAsnStatistic ntcAsnRecord, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis();
AuditLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,

13
src/main/java/com/nis/web/dao/dashboard/TrafficAsnStatisticDao.java Normal file
View File

@@ -0,0 +1,13 @@
package com.nis.web.dao.dashboard;
import java.util.List;
import com.nis.domain.restful.dashboard.TrafficAsnStatistic;
import com.nis.web.dao.MyBatisDao;
@MyBatisDao
public interface TrafficAsnStatisticDao {
List<TrafficAsnStatistic> getTrafficAsnStatisticList(TrafficAsnStatistic trafficAsnStatistic);
}

34
src/main/java/com/nis/web/dao/dashboard/TrafficAsnStatisticDao.xml Normal file
View File

@@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper
namespace="com.nis.web.dao.dashboard.TrafficAsnStatisticDao">
<resultMap id="BaseResultMap"
type="com.nis.domain.restful.dashboard.TrafficAsnStatistic">
<id column="bps" jdbcType="VARCHAR" property="bps" />
<result column="pps" jdbcType="VARCHAR" property="pps" />
<result column="asn" jdbcType="VARCHAR" property="asn" />
</resultMap>
<select id="getTrafficAsnStatisticList"
parameterType="com.nis.domain.restful.dashboard.TrafficAsnStatistic"
resultMap="BaseResultMap">
SELECT
TRUNCATE(packets / 3600 ,0)AS pps,
TRUNCATE(bytes * 8 / 1024 / 1024 / 1024 / 3600,2) AS bps,asn
FROM
traffic_asn_statistic t
WHERE
<![CDATA[
stat_time>=#{searchFoundStartTime} and stat_time<#{searchFoundEndTime} ]]>
<if test="searchAsnType !=null and searchAsnType != ''">
<![CDATA[and asn_type=#{searchAsnType}]]>
</if>
ORDER BY
bps DESC,
pps DESC
</select>
</mapper>

13
src/main/java/com/nis/web/dao/impl/LocalLogJDBCByDruid.java
View File

@@ -26,10 +26,9 @@ import org.springframework.stereotype.Repository;
import com.alibaba.druid.pool.DruidDataSource;
import com.nis.domain.Page;
import com.nis.domain.restful.NtcAsnRecord;
import com.nis.domain.restful.NtcConnRecordPercent;
import com.nis.domain.restful.dashboard.TrafficAsnStatistic;
import com.nis.web.service.SpringContextHolder;
import com.zdjizhi.utils.CommonUtil;
import com.zdjizhi.utils.StringUtil;
/**
@@ -181,8 +180,8 @@ public class LocalLogJDBCByDruid {
* @return
* @throws Exception
*/
public List<NtcAsnRecord> getNtcAsnRecordData(String sql) throws Exception {
List<NtcAsnRecord> list = new LinkedList<>();
public List<TrafficAsnStatistic> getNtcAsnRecordData(String sql) throws Exception {
List<TrafficAsnStatistic> list = new LinkedList<>();
try {
conn = getConnection();
logger.info("连接数据中心日志库成功--------------------------");
@@ -191,7 +190,7 @@ public class LocalLogJDBCByDruid {
rs = st.executeQuery(sql);
num.setMinimumFractionDigits(2);// 保留两位小数
while (rs.next()) {
NtcAsnRecord ntcAsnRecord = new NtcAsnRecord();
TrafficAsnStatistic ntcAsnRecord = new TrafficAsnStatistic();
String bps = rs.getString("bps");
String pps = rs.getString("pps");
String ppsNum =pps;
@@ -212,10 +211,10 @@ public class LocalLogJDBCByDruid {
}
public List<String> getAllIp(Integer frontier) throws Exception {
List<String> list = new LinkedList<>();
List<String> list = new ArrayList<>();
try {
// String sql = "select distinct ip from ip_location_database_local where ip like '145%' and frontier='"+frontier+"'";
String sql = "select distinct ip from ip_location_database_local where frontier='"+frontier+"'";
String sql = "select distinct ip from (select ip,frontier from ip_location_database ) where frontier='"+frontier+"'";
conn = getConnection();
logger.info("连接数据中心日志库成功--------------------------");
st = conn.createStatement();

4
src/main/java/com/nis/web/service/LogDataService.java
View File

@@ -18,8 +18,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.nis.domain.Page;
import com.nis.domain.restful.NtcAsnRecord;
import com.nis.domain.restful.NtcConnRecordPercent;
import com.nis.domain.restful.dashboard.TrafficAsnStatistic;
import com.nis.restful.RestBusinessCode;
import com.nis.restful.RestServiceException;
import com.nis.util.Configurations;
@@ -90,7 +90,7 @@ public class LogDataService {
return ntcConnRecordPercent;
}
public void getNtcAsnRecord(Page page, NtcAsnRecord ntcAsnRecord) throws Exception {
public void getNtcAsnRecord(Page page, TrafficAsnStatistic ntcAsnRecord) throws Exception {
long startTime = sdf.parse(ntcAsnRecord.getSearchFoundStartTime().toString().trim()).getTime() / 1000;
long endTime = sdf.parse(ntcAsnRecord.getSearchFoundEndTime().toString().trim()).getTime() / 1000;
if (endTime - startTime < 0) {

9
src/main/java/com/nis/web/service/restful/DashboardService.java
View File

@@ -22,12 +22,14 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.beust.jcommander.internal.Maps;
import com.nis.domain.Page;
import com.nis.domain.restful.NtcConnNumReport;
import com.nis.domain.restful.NtcEntranceReport;
import com.nis.domain.restful.dashboard.AppConnRecordStatistic;
import com.nis.domain.restful.dashboard.NtcTotalReport;
import com.nis.domain.restful.dashboard.TrafficAppFocusStatistic;
import com.nis.domain.restful.dashboard.TrafficAppStatistic;
import com.nis.domain.restful.dashboard.TrafficAsnStatistic;
import com.nis.domain.restful.dashboard.TrafficHttpFocusStatistic;
import com.nis.domain.restful.dashboard.TrafficIpActiveStatistic;
import com.nis.domain.restful.dashboard.TrafficPortActiveStatistic;
@@ -41,6 +43,7 @@ import com.nis.util.DateUtils;
import com.nis.util.StringUtils;
import com.nis.web.dao.dashboard.NtcTotalReportDao;
import com.nis.web.dao.dashboard.TrafficAppStatisticDao;
import com.nis.web.dao.dashboard.TrafficAsnStatisticDao;
import com.nis.web.dao.dashboard.TrafficHttpStatisticDao;
import com.nis.web.dao.dashboard.TrafficIpActiveStatisticDao;
import com.nis.web.dao.dashboard.TrafficPortActiveStatisticDao;
@@ -65,6 +68,8 @@ public class DashboardService extends BaseService {
@Autowired
public NtcTotalReportDao ntcTotalReportDao;
@Autowired
public TrafficAsnStatisticDao trafficAsnStatisticDao;
@Autowired
public TrafficPortActiveStatisticDao trafficPortActiveStatisticDao;
//ip地址类型
private final String[] addrTypes = {"4","6"} ;
@@ -1715,4 +1720,8 @@ public class DashboardService extends BaseService {
}
return list;
}
public void getTrafficAsnStatisticList(Page page, TrafficAsnStatistic ntcAsnRecord){
ntcAsnRecord.setPage(page);
page.setList(trafficAsnStatisticDao.getTrafficAsnStatisticList(ntcAsnRecord));
}
}

5
src/main/java/com/nis/web/task/GetIpRangeTask.java
View File

@@ -3,7 +3,7 @@ package com.nis.web.task;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -65,7 +65,7 @@ public class GetIpRangeTask {
*/
public Map<Long, String> getNumAndIpReal(List<String> allIp) throws InterruptedException, ExecutionException {
logger.info("开始将ip转换为数字");
Map<Long, String> map = new LinkedHashMap<>();
Map<Long, String> map = new HashMap<>();
int ever = 10000;// 每个线程处理10000条ip转数字
int count = allIp.size() / ever;
if (allIp.size() % ever != 0) {
@@ -231,7 +231,6 @@ public class GetIpRangeTask {
} else {
logger.info("获取境外ip成功,共{}条", allIp.size());
}
logger.info("获取境外ip成功,共{}条", allIp.size());
Map<Long, String> map = getNumAndIpReal(allIp);
Set<Long> keySet = map.keySet();
long[] array = new long[keySet.size()];