gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' of git.mesalab.cn:galaxy/galaxy-service into develop

Browse Source
This commit is contained in:
wangwei
2018-12-29 16:33:21 +08:00
parent 813cc69218 486b3fb63c
commit 8a4d831615
9 changed files with 187 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/com/nis/web/dao/NtcReportDao.xml
View File

@@ -876,14 +876,16 @@
<insert id="insertNtcIpRangeBatch"
parameterType="com.nis.domain.restful.NtcIpRangeReport">
INSERT INTO traffic_ip_identify
(ip_start,ip_end,ip_start_num,ip_end_num,ip_sub,area_type,country)
(ip_start,ip_end,ip_start_num,ip_end_num,ip_sub,area_type,country,desc_detail)
VALUES
<foreach collection="list" item="ipRange" separator=",">
(#{ipRange.ipStart}, #{ipRange.ipEnd},
#{ipRange.ipStartNum},
#{ipRange.ipEndNum},
#{ipRange.ipSub}, #{ipRange.areaType},
#{ipRange.country})
#{ipRange.country},
#{ipRange.desc}
)
</foreach>
</insert>

19
src/main/java/com/nis/web/dao/TrafficReportDao.xml
View File

@@ -82,7 +82,7 @@
<resultMap id="NtcRadiusReportMap"
type="com.nis.domain.restful.NtcRadiusReport">
<result column="account" jdbcType="VARCHAR" property="account" />
<result column="nas_ip" jdbcType="VARCHAR" property="nasIp" />
<result column="frame_ip" jdbcType="VARCHAR" property="nasIp" />
<result column="sum" jdbcType="BIGINT" property="sum" />
<result column="report_time" jdbcType="TIMESTAMP"
property="reportTime" />
@@ -786,7 +786,7 @@
<select id="findNasIpList"
parameterType="com.nis.domain.restful.NtcRadiusReport"
resultMap="NtcRadiusReportMap">
select nas_ip,sum(num) num from ntc_radius_report
select frame_ip,sum(num) num from ntc_radius_report
<where>
<choose>
<when
@@ -800,7 +800,7 @@
</choose>
</where>
group by nas_ip order by num desc
group by frame_ip order by num desc
</select>
<select id="findAccounList"
parameterType="com.nis.domain.restful.NtcRadiusReport"
@@ -826,7 +826,7 @@
parameterType="com.nis.domain.restful.NtcRadiusReport"
resultMap="NtcRadiusReportMap">
SELECT
*
frame_ip,num,account,report_time
FROM ntc_radius_report
<where>
<if
@@ -835,20 +835,17 @@
</if>
<if
test="searchBusinessType !=null and searchBusinessType != '' and searchBusinessType == 3 and searchNasIp != null and searchNasIp !=''">
<![CDATA[AND nas_ip =#{searchNasIp}]]>
<![CDATA[AND frame_ip =#{searchNasIp}]]>
</if>
<choose>
<when
test="searchReportStartTime != null and searchReportStartTime !=''">
<![CDATA[AND REPORT_TIME >= STR_TO_DATE(#{searchReportStartTime},'%Y-%m-%d %H:%i:%s') AND REPORT_TIME < STR_TO_DATE(#{searchReportEndTime},'%Y-%m-%d %H:%i:%s')]]>
<![CDATA[AND report_time >= #{searchReportStartTime} AND report_time < #{searchReportEndTime}]]>
</when>
<otherwise>
<![CDATA[AND REPORT_TIME>DATE_SUB(now(), INTERVAL 1 HOUR) ]]>
</otherwise>
</choose>
</where>
ORDER BY
REPORT_TIME ASC
report_time ASC
</select>
<select id="findNtcIpRangeReport"
@@ -934,7 +931,7 @@
<!-- 根据最近的时间获取阻断，监测等流量信息 -->
<select id="getTotalReportList" resultType="java.util.HashMap">
SELECT SUM(reject_num) rejectNum,SUM(monitor_num) monitorNum,SUM(c2s_pkt_num) c2sPktNum,SUM(s2c_pkt_num) s2cPktNum,SUM(c2s_byte_len) c2sByteLen,SUM(s2c_byte_len) s2cByteLen,SUM(new_uni_conn_num) newUniConnNum,SUM(live_conn_num) liveConnNum,
SELECT SUM(reject_num) rejectNum,SUM(monitor_num) monitorNum,SUM(c2s_pkt_num) c2sPktNum,SUM(s2c_pkt_num) s2cPktNum,SUM(c2s_byte_len) c2sByteLen,SUM(s2c_byte_len) s2cByteLen,SUM(new_uni_conn_num) newUniConnNum,SUM(close_conn_num) closeConnNum,
SUM(drop_conn_num) dropConnNum,SUM(loop_conn_num) loopConnNum FROM ntc_total_report
where report_time > DATE_SUB(#{reportTime},INTERVAL 1 hour)
</select>

4
src/main/java/com/nis/web/dao/dashboard/NtcTotalReportDao.xml
View File

@@ -21,7 +21,7 @@
<!-- 根据最近的时间获取阻断，监测等流量信息 -->
<select id="getTotalReportList" resultType="java.util.HashMap">
SELECT SUM(reject_num) rejectNum,SUM(monitor_num) monitorNum,SUM(c2s_pkt_num) c2sPktNum,SUM(s2c_pkt_num) s2cPktNum,SUM(c2s_byte_len) c2sByteLen,SUM(s2c_byte_len) s2cByteLen,SUM(new_uni_conn_num) newUniConnNum,SUM(live_conn_num) liveConnNum,
SELECT SUM(reject_num) rejectNum,SUM(monitor_num) monitorNum,SUM(c2s_pkt_num) c2sPktNum,SUM(s2c_pkt_num) s2cPktNum,SUM(c2s_byte_len) c2sByteLen,SUM(s2c_byte_len) s2cByteLen,SUM(new_uni_conn_num) newUniConnNum,SUM(close_conn_num) closeConnNum,
SUM(drop_conn_num) dropConnNum,SUM(loop_conn_num) loopConnNum FROM ntc_total_report
where report_time > DATE_SUB(#{reportTime},INTERVAL 1 hour)
</select>
@@ -145,7 +145,7 @@
new_uni_conn_num
</when>
<when test="searchBusinessType !=null and searchBusinessType != '' and searchBusinessType == 4 ">
live_conn_num
close_conn_num
</when>
<when test="searchBusinessType !=null and searchBusinessType != '' and searchBusinessType == 5 ">
(new_uni_conn_num-reject_num)

6
src/main/java/com/nis/web/service/LogDataService.java
View File

@@ -68,7 +68,7 @@ public class LogDataService {
sql.append(" AS pps , SUM(c2s_byte_num + s2c_byte_num)*8/1024/1024/1024/");
sql.append(second);
sql.append(
" AS bps FROM tbs_ods_ntc_conn_record_log_local t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local WHERE found_time >= ");
" AS bps FROM tbs_ods_ntc_conn_record_log t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log WHERE found_time >= ");
sql.append(startTime);
sql.append(" and found_time< ");
sql.append(endTime);
@@ -104,7 +104,7 @@ public class LogDataService {
sql.append(" AS bps,");
sql.append(asn);
sql.append(
" asn FROM tbs_ods_ntc_conn_record_log_local t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local WHERE found_time >= ");
" asn FROM tbs_ods_ntc_conn_record_log t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log WHERE found_time >= ");
sql.append(startTime);
sql.append(" and found_time< ");
sql.append(endTime);
@@ -302,7 +302,7 @@ public class LogDataService {
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG_LOCAL")) {
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")) {
searchFromLocalCK(page, bean, sql, countSql);
} else {
searchFromDataCenter(page, bean, sql, countSql);

5
src/main/java/com/nis/web/service/restful/DashboardService.java
View File

@@ -1092,10 +1092,7 @@ public class DashboardService extends BaseService {
// String linkNumStr = String.valueOf(webIdAndCountMap.get("linkNum"));
// String packetsStr = String.valueOf(webIdAndCountMap.get("pktCount"));
if (countStr != null) {
Long pl = (Long.parseLong(countStr)) / (1024 * 1024 * 1024);
if (pl > 0) {
count += Long.parseLong(countStr);// 将count累加 转换单位小于0的不加
}
count += Long.parseLong(countStr);// 将count累加 转换单位小于0的不加
}
// if (linkNumStr != null) {
// linkNum += Long.parseLong(linkNumStr);// 将count累加

192
src/main/java/com/nis/web/task/GetIpRangeTask.java
View File

@@ -1,10 +1,13 @@
package com.nis.web.task;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -41,7 +44,7 @@ public class GetIpRangeTask {
.loadDataFileV6(Constants.IPLOCATIONLIBRARYPATH).build();
// @Scheduled(cron = "${getIpRangeTaskCron}")
public void test() {
public void getIpRangeTask() {
try {
// 删除数据,重新全量导入
dao.truncateNtcIpRange();
@@ -52,16 +55,24 @@ public class GetIpRangeTask {
}
}
/**
* 将ip转换为数字与ip的对应关系,key是ip的数字形式
*
* @param allIp
* @return
* @throws InterruptedException
* @throws ExecutionException
*/
public Map<Long, String> getNumAndIpReal(List<String> allIp) throws InterruptedException, ExecutionException {
logger.info("开始将ip转换为数字");
Map<Long, String> map = new HashMap<>();
Map<Long, String> map = new LinkedHashMap<>();
int ever = 10000;// 每个线程处理10000条ip转数字
int count = allIp.size() / ever;
if (allIp.size() % ever != 0) {
count++;
}
List<Future<Map<Long, String>>> futures = new ArrayList<>();
ExecutorService executor = Executors.newFixedThreadPool(10);// 10个线程
ExecutorService executor = Executors.newFixedThreadPool(35);// 10个线程
for (int i = 0; i < count; i++) {
int start = i * ever;
int end = i * ever + ever;
@@ -86,20 +97,107 @@ public class GetIpRangeTask {
return map;
}
public static void main(String[] args) {
// a();
String ip = "2.72.0.0";
long currentTimeMillis = System.currentTimeMillis();
System.out.println(ipLookup.countryLookup(ip));
// System.out.println(ipLookup.cityLookup(ip));
// System.out.println(ipLookup.cityLatLngLookup(ip));
// System.out.println(ipLookup.cityLookupDetail(ip));
long end = System.currentTimeMillis();
System.out.println(end-currentTimeMillis);
}
public static void b() {
long[] arr = { 1l, 2l, 3l, 5l, 6l, 7l };
// String groupByNumer = test(arr);
String groupByNumer = groupByNumer(arr);
String[] split = org.apache.commons.lang.StringUtils.split(groupByNumer, ",");
for (String str : split) {
System.out.println(str);
}
}
public static void a() {
int number = 2000000;
int range = number + 1;
long start = System.currentTimeMillis();
Set<Long> hashSet = new LinkedHashSet<>();
Random random = new Random();
for (int i = 0; i < number; i++) {
hashSet.add(Long.parseLong(random.nextInt(range) + ""));
}
long end1 = System.currentTimeMillis();
System.out.println("生成数组需要" + (end1 - start) / 1000 + "秒");
System.out.println(hashSet.size());
long[] arr = new long[hashSet.size()];
int i = 0;
for (long l : hashSet) {
arr[i] = l;
i++;
}
long end2 = System.currentTimeMillis();
System.out.println("set转数组需要" + (end2 - end1) / 1000 + "秒");
Arrays.sort(arr);
long end3 = System.currentTimeMillis();
System.out.println("排序需要" + (end3 - end2) / 1000 + "秒");
String groupByNumer = groupByNumer(arr);
long end4 = System.currentTimeMillis();
System.out.println("获取连续数字需要" + (end4 - end3) / 1000 + "秒");
String[] split = org.apache.commons.lang.StringUtils.split(groupByNumer, ",");
long end5 = System.currentTimeMillis();
System.out.println("切割需要" + (end5 - end4) / 1000 + "秒");
for (String str : split) {
System.out.println(str);
}
}
/**
* 将数组中连续的数据合并在一起输出,例如传入1,2,3,5,6,7,9,11输出1-3,4-7,9,11 .注意传入的数组必须是有序的,从小到大排序
*
* @param noNum
* @return
*/
private static String groupByNumer(Long[] sortArr) {
private static String groupByNumer(long[] sortArr) {
int state = 0;
StringBuffer result = new StringBuffer();
for (int i = 0; i < sortArr.length; i++) {
if (i == sortArr.length - 1) {
state = 2;
}
if (state == 0) {
if (sortArr[i + 1] == sortArr[i] + 1) {
result.append(sortArr[i]);
result.append("-");
state = 1;
} else {
result.append(sortArr[i]);
result.append(",");
}
} else if (state == 1) {
if (sortArr[i + 1] != sortArr[i] + 1) {
result.append(sortArr[i]);
result.append(",");
state = 0;
}
} else {
result.append(sortArr[i]);
}
}
return result.toString();
}
public static String test(long[] sortArr) {
int state = 0;
String result = "";
for (int i = 0; i < sortArr.length; i++) {
if (i == sortArr.length - 1)
state = 2;
if (state == 0) {
if (sortArr[i + 1].longValue() == sortArr[i].longValue() + 1) {
if (sortArr[i + 1] == sortArr[i] + 1) {
result += String.valueOf(sortArr[i]);
result += "-";
state = 1;
@@ -118,17 +216,36 @@ public class GetIpRangeTask {
}
}
return result;
}
public void getAllIp(Integer frontier) throws Exception {
if (frontier == 1) {
logger.info("开始获取境内ip");
} else {
logger.info("开始获取境外ip");
}
List<String> allIp = localLogJDBCByDruid.getAllIp(frontier);
if (frontier == 1) {
logger.info("获取境内ip成功,共{}条", allIp.size());
} else {
logger.info("获取境外ip成功,共{}条", allIp.size());
}
logger.info("获取境外ip成功,共{}条", allIp.size());
Map<Long, String> map = getNumAndIpReal(allIp);
Set<Long> keySet = map.keySet();
List<Long> ipList = new ArrayList<>(keySet);
Long[] array = new Long[ipList.size()];
Collections.sort(ipList);
ipList.toArray(array);
long[] array = new long[keySet.size()];
int index = 0;
for (long l : keySet) {
array[index] = l;
index++;
}
Arrays.sort(array);
long startTime = System.currentTimeMillis();
String result = groupByNumer(array);
long endTime = System.currentTimeMillis();
logger.info("解析连续数字用时{}秒", (endTime - startTime) / 1000);
// String result = null;
if (result != null && !result.trim().equals("")) {
String substring = result;
if (result.endsWith(",")) {
@@ -142,17 +259,23 @@ public class GetIpRangeTask {
if (ipAddr.length > 1) {
String startIp = ipAddr[0];
String endIp = ipAddr[1];
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(Long.parseLong(startIp));
String endIpStr = map.get(Long.parseLong(endIp));
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(Long.parseLong(endIp));
ntcIpRangeReport.setIpStartNum(Long.parseLong(startIp));
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReportList.add(ntcIpRangeReport);
try {
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(Long.parseLong(startIp));
String endIpStr = map.get(Long.parseLong(endIp));
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(Long.parseLong(endIp));
ntcIpRangeReport.setIpStartNum(Long.parseLong(startIp));
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setDesc(ipLookup.cityLookup(startIpStr));
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReportList.add(ntcIpRangeReport);
} catch (Exception e) {
logger.error("数据转NtcIpRangeReport,失败原因:{}", ExceptionUtil.getExceptionMsg(e));
}
} else {
list.add(Long.parseLong(str));
}
@@ -163,17 +286,22 @@ public class GetIpRangeTask {
if (i < list.size() - 1) {
Long startIp = list.get(i);
Long endIp = list.get(i + 1);
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(startIp);
String endIpStr = map.get(endIp);
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(endIp);
ntcIpRangeReport.setIpStartNum(startIp);
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReportList.add(ntcIpRangeReport);
try {
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(startIp);
String endIpStr = map.get(endIp);
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(endIp);
ntcIpRangeReport.setIpStartNum(startIp);
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setDesc(ipLookup.cityLookup(startIpStr));
ntcIpRangeReportList.add(ntcIpRangeReport);
} catch (Exception e) {
logger.error("数据转NtcIpRangeReport,失败原因:{}", ExceptionUtil.getExceptionMsg(e));
}
}
}
int size = ntcIpRangeReportList.size();

1
src/main/java/com/nis/web/task/IpToLongThread.java
View File

@@ -25,6 +25,7 @@ public class IpToLongThread implements Callable<Map<Long, String>> {
@Override
public Map<Long, String> call() throws Exception {
Map<Long, String> map=new HashMap<>();
// logger.info("线程{}开始执行,list.size={}", Thread.currentThread().getName(),list.size());
logger.info("线程{}开始执行", Thread.currentThread().getName());
for (int i = 0; i < list.size(); i++) {
map.put(ipToLong(list.get(i)), list.get(i));