gfwleak/k18-ntcs-web-argus-service
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1:合并0x200,0x201为0x200.

Browse Source 
2:合并0x202,0x203为0x202.
3:修改NtcConnRecordLogClickHouseTable对应的clickhouse名称
4:修改TrafficReportDao.xml.findNtcRadiusReport中字段为小写
5:修改保存traffic_ip_identify时添加desc_detail字段
6:修改ip范围段的计算方法
This commit is contained in:
renkaige
2018-12-29 11:28:13 +06:00
parent 0cff980477
commit 4102afee9d
7 changed files with 178 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/com/nis/web/dao/NtcReportDao.xml
View File

@@ -876,14 +876,16 @@
<insert id="insertNtcIpRangeBatch"
parameterType="com.nis.domain.restful.NtcIpRangeReport">
INSERT INTO traffic_ip_identify
(ip_start,ip_end,ip_start_num,ip_end_num,ip_sub,area_type,country)
(ip_start,ip_end,ip_start_num,ip_end_num,ip_sub,area_type,country,desc_detail)
VALUES
<foreach collection="list" item="ipRange" separator=",">
(#{ipRange.ipStart}, #{ipRange.ipEnd},
#{ipRange.ipStartNum},
#{ipRange.ipEndNum},
#{ipRange.ipSub}, #{ipRange.areaType},
#{ipRange.country})
#{ipRange.country},
#{ipRange.desc}
)
</foreach>
</insert>

7
src/main/java/com/nis/web/dao/TrafficReportDao.xml
View File

@@ -840,15 +840,12 @@
<choose>
<when
test="searchReportStartTime != null and searchReportStartTime !=''">
<![CDATA[AND REPORT_TIME >= STR_TO_DATE(#{searchReportStartTime},'%Y-%m-%d %H:%i:%s') AND REPORT_TIME < STR_TO_DATE(#{searchReportEndTime},'%Y-%m-%d %H:%i:%s')]]>
<![CDATA[AND report_time >= #{searchReportStartTime} AND report_time < #{searchReportEndTime}]]>
</when>
<otherwise>
<![CDATA[AND REPORT_TIME>DATE_SUB(now(), INTERVAL 1 HOUR) ]]>
</otherwise>
</choose>
</where>
ORDER BY
REPORT_TIME ASC
report_time ASC
</select>
<select id="findNtcIpRangeReport"

6
src/main/java/com/nis/web/service/LogDataService.java
View File

@@ -68,7 +68,7 @@ public class LogDataService {
sql.append(" AS pps , SUM(c2s_byte_num + s2c_byte_num)*8/1024/1024/1024/");
sql.append(second);
sql.append(
" AS bps FROM tbs_ods_ntc_conn_record_log_local t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local WHERE found_time >= ");
" AS bps FROM tbs_ods_ntc_conn_record_log t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log WHERE found_time >= ");
sql.append(startTime);
sql.append(" and found_time< ");
sql.append(endTime);
@@ -104,7 +104,7 @@ public class LogDataService {
sql.append(" AS bps,");
sql.append(asn);
sql.append(
" asn FROM tbs_ods_ntc_conn_record_log_local t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local WHERE found_time >= ");
" asn FROM tbs_ods_ntc_conn_record_log t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log WHERE found_time >= ");
sql.append(startTime);
sql.append(" and found_time< ");
sql.append(endTime);
@@ -302,7 +302,7 @@ public class LogDataService {
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG_LOCAL")) {
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")) {
searchFromLocalCK(page, bean, sql, countSql);
} else {
searchFromDataCenter(page, bean, sql, countSql);

192
src/main/java/com/nis/web/task/GetIpRangeTask.java
View File

@@ -1,10 +1,13 @@
package com.nis.web.task;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -41,7 +44,7 @@ public class GetIpRangeTask {
.loadDataFileV6(Constants.IPLOCATIONLIBRARYPATH).build();
// @Scheduled(cron = "${getIpRangeTaskCron}")
public void test() {
public void getIpRangeTask() {
try {
// 删除数据,重新全量导入
dao.truncateNtcIpRange();
@@ -52,16 +55,24 @@ public class GetIpRangeTask {
}
}
/**
* 将ip转换为数字与ip的对应关系,key是ip的数字形式
*
* @param allIp
* @return
* @throws InterruptedException
* @throws ExecutionException
*/
public Map<Long, String> getNumAndIpReal(List<String> allIp) throws InterruptedException, ExecutionException {
logger.info("开始将ip转换为数字");
Map<Long, String> map = new HashMap<>();
Map<Long, String> map = new LinkedHashMap<>();
int ever = 10000;// 每个线程处理10000条ip转数字
int count = allIp.size() / ever;
if (allIp.size() % ever != 0) {
count++;
}
List<Future<Map<Long, String>>> futures = new ArrayList<>();
ExecutorService executor = Executors.newFixedThreadPool(10);// 10个线程
ExecutorService executor = Executors.newFixedThreadPool(35);// 10个线程
for (int i = 0; i < count; i++) {
int start = i * ever;
int end = i * ever + ever;
@@ -86,20 +97,107 @@ public class GetIpRangeTask {
return map;
}
public static void main(String[] args) {
// a();
String ip = "2.72.0.0";
long currentTimeMillis = System.currentTimeMillis();
System.out.println(ipLookup.countryLookup(ip));
// System.out.println(ipLookup.cityLookup(ip));
// System.out.println(ipLookup.cityLatLngLookup(ip));
// System.out.println(ipLookup.cityLookupDetail(ip));
long end = System.currentTimeMillis();
System.out.println(end-currentTimeMillis);
}
public static void b() {
long[] arr = { 1l, 2l, 3l, 5l, 6l, 7l };
// String groupByNumer = test(arr);
String groupByNumer = groupByNumer(arr);
String[] split = org.apache.commons.lang.StringUtils.split(groupByNumer, ",");
for (String str : split) {
System.out.println(str);
}
}
public static void a() {
int number = 2000000;
int range = number + 1;
long start = System.currentTimeMillis();
Set<Long> hashSet = new LinkedHashSet<>();
Random random = new Random();
for (int i = 0; i < number; i++) {
hashSet.add(Long.parseLong(random.nextInt(range) + ""));
}
long end1 = System.currentTimeMillis();
System.out.println("生成数组需要" + (end1 - start) / 1000 + "秒");
System.out.println(hashSet.size());
long[] arr = new long[hashSet.size()];
int i = 0;
for (long l : hashSet) {
arr[i] = l;
i++;
}
long end2 = System.currentTimeMillis();
System.out.println("set转数组需要" + (end2 - end1) / 1000 + "秒");
Arrays.sort(arr);
long end3 = System.currentTimeMillis();
System.out.println("排序需要" + (end3 - end2) / 1000 + "秒");
String groupByNumer = groupByNumer(arr);
long end4 = System.currentTimeMillis();
System.out.println("获取连续数字需要" + (end4 - end3) / 1000 + "秒");
String[] split = org.apache.commons.lang.StringUtils.split(groupByNumer, ",");
long end5 = System.currentTimeMillis();
System.out.println("切割需要" + (end5 - end4) / 1000 + "秒");
for (String str : split) {
System.out.println(str);
}
}
/**
* 将数组中连续的数据合并在一起输出,例如传入1,2,3,5,6,7,9,11输出1-3,4-7,9,11 .注意传入的数组必须是有序的,从小到大排序
*
* @param noNum
* @return
*/
private static String groupByNumer(Long[] sortArr) {
private static String groupByNumer(long[] sortArr) {
int state = 0;
StringBuffer result = new StringBuffer();
for (int i = 0; i < sortArr.length; i++) {
if (i == sortArr.length - 1) {
state = 2;
}
if (state == 0) {
if (sortArr[i + 1] == sortArr[i] + 1) {
result.append(sortArr[i]);
result.append("-");
state = 1;
} else {
result.append(sortArr[i]);
result.append(",");
}
} else if (state == 1) {
if (sortArr[i + 1] != sortArr[i] + 1) {
result.append(sortArr[i]);
result.append(",");
state = 0;
}
} else {
result.append(sortArr[i]);
}
}
return result.toString();
}
public static String test(long[] sortArr) {
int state = 0;
String result = "";
for (int i = 0; i < sortArr.length; i++) {
if (i == sortArr.length - 1)
state = 2;
if (state == 0) {
if (sortArr[i + 1].longValue() == sortArr[i].longValue() + 1) {
if (sortArr[i + 1] == sortArr[i] + 1) {
result += String.valueOf(sortArr[i]);
result += "-";
state = 1;
@@ -118,17 +216,36 @@ public class GetIpRangeTask {
}
}
return result;
}
public void getAllIp(Integer frontier) throws Exception {
if (frontier == 1) {
logger.info("开始获取境内ip");
} else {
logger.info("开始获取境外ip");
}
List<String> allIp = localLogJDBCByDruid.getAllIp(frontier);
if (frontier == 1) {
logger.info("获取境内ip成功,共{}条", allIp.size());
} else {
logger.info("获取境外ip成功,共{}条", allIp.size());
}
logger.info("获取境外ip成功,共{}条", allIp.size());
Map<Long, String> map = getNumAndIpReal(allIp);
Set<Long> keySet = map.keySet();
List<Long> ipList = new ArrayList<>(keySet);
Long[] array = new Long[ipList.size()];
Collections.sort(ipList);
ipList.toArray(array);
long[] array = new long[keySet.size()];
int index = 0;
for (long l : keySet) {
array[index] = l;
index++;
}
Arrays.sort(array);
long startTime = System.currentTimeMillis();
String result = groupByNumer(array);
long endTime = System.currentTimeMillis();
logger.info("解析连续数字用时{}秒", (endTime - startTime) / 1000);
// String result = null;
if (result != null && !result.trim().equals("")) {
String substring = result;
if (result.endsWith(",")) {
@@ -142,17 +259,23 @@ public class GetIpRangeTask {
if (ipAddr.length > 1) {
String startIp = ipAddr[0];
String endIp = ipAddr[1];
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(Long.parseLong(startIp));
String endIpStr = map.get(Long.parseLong(endIp));
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(Long.parseLong(endIp));
ntcIpRangeReport.setIpStartNum(Long.parseLong(startIp));
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReportList.add(ntcIpRangeReport);
try {
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(Long.parseLong(startIp));
String endIpStr = map.get(Long.parseLong(endIp));
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(Long.parseLong(endIp));
ntcIpRangeReport.setIpStartNum(Long.parseLong(startIp));
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setDesc(ipLookup.cityLookup(startIpStr));
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReportList.add(ntcIpRangeReport);
} catch (Exception e) {
logger.error("数据转NtcIpRangeReport,失败原因:{}", ExceptionUtil.getExceptionMsg(e));
}
} else {
list.add(Long.parseLong(str));
}
@@ -163,17 +286,22 @@ public class GetIpRangeTask {
if (i < list.size() - 1) {
Long startIp = list.get(i);
Long endIp = list.get(i + 1);
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(startIp);
String endIpStr = map.get(endIp);
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(endIp);
ntcIpRangeReport.setIpStartNum(startIp);
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReportList.add(ntcIpRangeReport);
try {
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(startIp);
String endIpStr = map.get(endIp);
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(endIp);
ntcIpRangeReport.setIpStartNum(startIp);
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setDesc(ipLookup.cityLookup(startIpStr));
ntcIpRangeReportList.add(ntcIpRangeReport);
} catch (Exception e) {
logger.error("数据转NtcIpRangeReport,失败原因:{}", ExceptionUtil.getExceptionMsg(e));
}
}
}
int size = ntcIpRangeReportList.size();

1
src/main/java/com/nis/web/task/IpToLongThread.java
View File

@@ -25,6 +25,7 @@ public class IpToLongThread implements Callable<Map<Long, String>> {
@Override
public Map<Long, String> call() throws Exception {
Map<Long, String> map=new HashMap<>();
// logger.info("线程{}开始执行,list.size={}", Thread.currentThread().getName(),list.size());
logger.info("线程{}开始执行", Thread.currentThread().getName());
for (int i = 0; i < list.size(); i++) {
map.put(ipToLong(list.get(i)), list.get(i));