1:合并0x200,0x201为0x200.

2:合并0x202,0x203为0x202.
3:修改NtcConnRecordLogClickHouseTable对应的clickhouse名称
4:修改TrafficReportDao.xml.findNtcRadiusReport中字段为小写
5:修改保存traffic_ip_identify时添加desc_detail字段
6:修改ip范围段的计算方法
This commit is contained in:
renkaige
2018-12-29 11:28:13 +06:00
parent 0cff980477
commit 4102afee9d
7 changed files with 178 additions and 50 deletions

View File

@@ -876,14 +876,16 @@
<insert id="insertNtcIpRangeBatch"
parameterType="com.nis.domain.restful.NtcIpRangeReport">
INSERT INTO traffic_ip_identify
(ip_start,ip_end,ip_start_num,ip_end_num,ip_sub,area_type,country)
(ip_start,ip_end,ip_start_num,ip_end_num,ip_sub,area_type,country,desc_detail)
VALUES
<foreach collection="list" item="ipRange" separator=",">
(#{ipRange.ipStart}, #{ipRange.ipEnd},
#{ipRange.ipStartNum},
#{ipRange.ipEndNum},
#{ipRange.ipSub}, #{ipRange.areaType},
#{ipRange.country})
#{ipRange.country},
#{ipRange.desc}
)
</foreach>
</insert>

View File

@@ -840,15 +840,12 @@
<choose>
<when
test="searchReportStartTime != null and searchReportStartTime !=''">
<![CDATA[AND REPORT_TIME >= STR_TO_DATE(#{searchReportStartTime},'%Y-%m-%d %H:%i:%s') AND REPORT_TIME < STR_TO_DATE(#{searchReportEndTime},'%Y-%m-%d %H:%i:%s')]]>
<![CDATA[AND report_time >= #{searchReportStartTime} AND report_time < #{searchReportEndTime}]]>
</when>
<otherwise>
<![CDATA[AND REPORT_TIME>DATE_SUB(now(), INTERVAL 1 HOUR) ]]>
</otherwise>
</choose>
</where>
ORDER BY
REPORT_TIME ASC
report_time ASC
</select>
<select id="findNtcIpRangeReport"

View File

@@ -68,7 +68,7 @@ public class LogDataService {
sql.append(" AS pps , SUM(c2s_byte_num + s2c_byte_num)*8/1024/1024/1024/");
sql.append(second);
sql.append(
" AS bps FROM tbs_ods_ntc_conn_record_log_local t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local WHERE found_time >= ");
" AS bps FROM tbs_ods_ntc_conn_record_log t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log WHERE found_time >= ");
sql.append(startTime);
sql.append(" and found_time< ");
sql.append(endTime);
@@ -104,7 +104,7 @@ public class LogDataService {
sql.append(" AS bps,");
sql.append(asn);
sql.append(
" asn FROM tbs_ods_ntc_conn_record_log_local t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local WHERE found_time >= ");
" asn FROM tbs_ods_ntc_conn_record_log t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log WHERE found_time >= ");
sql.append(startTime);
sql.append(" and found_time< ");
sql.append(endTime);
@@ -302,7 +302,7 @@ public class LogDataService {
}
sql.append(orderBy.toLowerCase() + " limit " + startNum + "," + page.getPageSize());// clickhouse的分页与mysql相同
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG_LOCAL")) {
if (tableName.toUpperCase().equals("TBS_ODS_NTC_CONN_RECORD_LOG")) {
searchFromLocalCK(page, bean, sql, countSql);
} else {
searchFromDataCenter(page, bean, sql, countSql);

View File

@@ -1,10 +1,13 @@
package com.nis.web.task;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -41,7 +44,7 @@ public class GetIpRangeTask {
.loadDataFileV6(Constants.IPLOCATIONLIBRARYPATH).build();
// @Scheduled(cron = "${getIpRangeTaskCron}")
public void test() {
public void getIpRangeTask() {
try {
// 删除数据,重新全量导入
dao.truncateNtcIpRange();
@@ -52,16 +55,24 @@ public class GetIpRangeTask {
}
}
/**
* 将ip转换为数字与ip的对应关系,key是ip的数字形式
*
* @param allIp
* @return
* @throws InterruptedException
* @throws ExecutionException
*/
public Map<Long, String> getNumAndIpReal(List<String> allIp) throws InterruptedException, ExecutionException {
logger.info("开始将ip转换为数字");
Map<Long, String> map = new HashMap<>();
Map<Long, String> map = new LinkedHashMap<>();
int ever = 10000;// 每个线程处理10000条ip转数字
int count = allIp.size() / ever;
if (allIp.size() % ever != 0) {
count++;
}
List<Future<Map<Long, String>>> futures = new ArrayList<>();
ExecutorService executor = Executors.newFixedThreadPool(10);// 10个线程
ExecutorService executor = Executors.newFixedThreadPool(35);// 10个线程
for (int i = 0; i < count; i++) {
int start = i * ever;
int end = i * ever + ever;
@@ -86,20 +97,107 @@ public class GetIpRangeTask {
return map;
}
public static void main(String[] args) {
// a();
String ip = "2.72.0.0";
long currentTimeMillis = System.currentTimeMillis();
System.out.println(ipLookup.countryLookup(ip));
// System.out.println(ipLookup.cityLookup(ip));
// System.out.println(ipLookup.cityLatLngLookup(ip));
// System.out.println(ipLookup.cityLookupDetail(ip));
long end = System.currentTimeMillis();
System.out.println(end-currentTimeMillis);
}
public static void b() {
long[] arr = { 1l, 2l, 3l, 5l, 6l, 7l };
// String groupByNumer = test(arr);
String groupByNumer = groupByNumer(arr);
String[] split = org.apache.commons.lang.StringUtils.split(groupByNumer, ",");
for (String str : split) {
System.out.println(str);
}
}
public static void a() {
int number = 2000000;
int range = number + 1;
long start = System.currentTimeMillis();
Set<Long> hashSet = new LinkedHashSet<>();
Random random = new Random();
for (int i = 0; i < number; i++) {
hashSet.add(Long.parseLong(random.nextInt(range) + ""));
}
long end1 = System.currentTimeMillis();
System.out.println("生成数组需要" + (end1 - start) / 1000 + "");
System.out.println(hashSet.size());
long[] arr = new long[hashSet.size()];
int i = 0;
for (long l : hashSet) {
arr[i] = l;
i++;
}
long end2 = System.currentTimeMillis();
System.out.println("set转数组需要" + (end2 - end1) / 1000 + "");
Arrays.sort(arr);
long end3 = System.currentTimeMillis();
System.out.println("排序需要" + (end3 - end2) / 1000 + "");
String groupByNumer = groupByNumer(arr);
long end4 = System.currentTimeMillis();
System.out.println("获取连续数字需要" + (end4 - end3) / 1000 + "");
String[] split = org.apache.commons.lang.StringUtils.split(groupByNumer, ",");
long end5 = System.currentTimeMillis();
System.out.println("切割需要" + (end5 - end4) / 1000 + "");
for (String str : split) {
System.out.println(str);
}
}
/**
* 将数组中连续的数据合并在一起输出,例如传入1,2,3,5,6,7,9,11输出1-3,4-7,9,11 .注意传入的数组必须是有序的,从小到大排序
*
* @param noNum
* @return
*/
private static String groupByNumer(Long[] sortArr) {
private static String groupByNumer(long[] sortArr) {
int state = 0;
StringBuffer result = new StringBuffer();
for (int i = 0; i < sortArr.length; i++) {
if (i == sortArr.length - 1) {
state = 2;
}
if (state == 0) {
if (sortArr[i + 1] == sortArr[i] + 1) {
result.append(sortArr[i]);
result.append("-");
state = 1;
} else {
result.append(sortArr[i]);
result.append(",");
}
} else if (state == 1) {
if (sortArr[i + 1] != sortArr[i] + 1) {
result.append(sortArr[i]);
result.append(",");
state = 0;
}
} else {
result.append(sortArr[i]);
}
}
return result.toString();
}
public static String test(long[] sortArr) {
int state = 0;
String result = "";
for (int i = 0; i < sortArr.length; i++) {
if (i == sortArr.length - 1)
state = 2;
if (state == 0) {
if (sortArr[i + 1].longValue() == sortArr[i].longValue() + 1) {
if (sortArr[i + 1] == sortArr[i] + 1) {
result += String.valueOf(sortArr[i]);
result += "-";
state = 1;
@@ -118,17 +216,36 @@ public class GetIpRangeTask {
}
}
return result;
}
public void getAllIp(Integer frontier) throws Exception {
if (frontier == 1) {
logger.info("开始获取境内ip");
} else {
logger.info("开始获取境外ip");
}
List<String> allIp = localLogJDBCByDruid.getAllIp(frontier);
if (frontier == 1) {
logger.info("获取境内ip成功,共{}条", allIp.size());
} else {
logger.info("获取境外ip成功,共{}条", allIp.size());
}
logger.info("获取境外ip成功,共{}条", allIp.size());
Map<Long, String> map = getNumAndIpReal(allIp);
Set<Long> keySet = map.keySet();
List<Long> ipList = new ArrayList<>(keySet);
Long[] array = new Long[ipList.size()];
Collections.sort(ipList);
ipList.toArray(array);
long[] array = new long[keySet.size()];
int index = 0;
for (long l : keySet) {
array[index] = l;
index++;
}
Arrays.sort(array);
long startTime = System.currentTimeMillis();
String result = groupByNumer(array);
long endTime = System.currentTimeMillis();
logger.info("解析连续数字用时{}秒", (endTime - startTime) / 1000);
// String result = null;
if (result != null && !result.trim().equals("")) {
String substring = result;
if (result.endsWith(",")) {
@@ -142,17 +259,23 @@ public class GetIpRangeTask {
if (ipAddr.length > 1) {
String startIp = ipAddr[0];
String endIp = ipAddr[1];
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(Long.parseLong(startIp));
String endIpStr = map.get(Long.parseLong(endIp));
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(Long.parseLong(endIp));
ntcIpRangeReport.setIpStartNum(Long.parseLong(startIp));
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReportList.add(ntcIpRangeReport);
try {
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(Long.parseLong(startIp));
String endIpStr = map.get(Long.parseLong(endIp));
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(Long.parseLong(endIp));
ntcIpRangeReport.setIpStartNum(Long.parseLong(startIp));
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setDesc(ipLookup.cityLookup(startIpStr));
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReportList.add(ntcIpRangeReport);
} catch (Exception e) {
logger.error("数据转NtcIpRangeReport,失败原因:{}", ExceptionUtil.getExceptionMsg(e));
}
} else {
list.add(Long.parseLong(str));
}
@@ -163,17 +286,22 @@ public class GetIpRangeTask {
if (i < list.size() - 1) {
Long startIp = list.get(i);
Long endIp = list.get(i + 1);
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(startIp);
String endIpStr = map.get(endIp);
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(endIp);
ntcIpRangeReport.setIpStartNum(startIp);
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReportList.add(ntcIpRangeReport);
try {
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(startIp);
String endIpStr = map.get(endIp);
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(endIp);
ntcIpRangeReport.setIpStartNum(startIp);
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setDesc(ipLookup.cityLookup(startIpStr));
ntcIpRangeReportList.add(ntcIpRangeReport);
} catch (Exception e) {
logger.error("数据转NtcIpRangeReport,失败原因:{}", ExceptionUtil.getExceptionMsg(e));
}
}
}
int size = ntcIpRangeReportList.size();

View File

@@ -25,6 +25,7 @@ public class IpToLongThread implements Callable<Map<Long, String>> {
@Override
public Map<Long, String> call() throws Exception {
Map<Long, String> map=new HashMap<>();
// logger.info("线程{}开始执行,list.size={}", Thread.currentThread().getName(),list.size());
logger.info("线程{}开始执行", Thread.currentThread().getName());
for (int i = 0; i < list.size(); i++) {
map.put(ipToLong(list.get(i)), list.get(i));

View File

@@ -170,12 +170,12 @@ service=1:128;2:128;5:96;6:1;16:16;17:16;18:16;19:16;20:16;21:16;22:16;23:16;24:
399=10:MM_COMPILE;11:MM_GROUP;14:MM_SUBSCRIBE_ID;16:MM_FILE_DIGEST;18:NTC_IP_RANGE
#0x200 PXY拦截IP
512=10:PXY_INTERCEPT_COMPILE;11:PXY_INTERCEPT_GROUP;12:PXY_INTERCEPT_IP;14:PXY_CTRL_SUBSCRIBE_ID;18:NTC_IP_RANGE
#0x201 PXY拦截域名
512=10:PXY_INTERCEPT_COMPILE;11:PXY_INTERCEPT_GROUP;12:PXY_INTERCEPT_IP;14:PXY_INTERCEPT_DOMAIN,PXY_CTRL_SUBSCRIBE_ID;18:NTC_IP_RANGE
#0x201 PXY拦截域名(与0x200合并)
513=10:PXY_INTERCEPT_COMPILE;11:PXY_INTERCEPT_GROUP;14:PXY_INTERCEPT_DOMAIN,PXY_CTRL_SUBSCRIBE_ID;18:NTC_IP_RANGE
#0x202 PXY限速IP
514=10:PXY_INTERCEPT_COMPILE;11:PXY_INTERCEPT_GROUP;12:PXY_INTERCEPT_IP;14:PXY_CTRL_SUBSCRIBE_ID;18:NTC_IP_RANGE
#0x203 PXY限速域名
514=10:PXY_INTERCEPT_COMPILE;11:PXY_INTERCEPT_GROUP;12:PXY_INTERCEPT_IP;14:PXY_INTERCEPT_DOMAIN,PXY_CTRL_SUBSCRIBE_ID;18:NTC_IP_RANGE
#0x203 PXY限速域名(与0x202合并)
515=10:PXY_INTERCEPT_COMPILE;11:PXY_INTERCEPT_GROUP;14:PXY_INTERCEPT_DOMAIN,PXY_CTRL_SUBSCRIBE_ID;18:NTC_IP_RANGE
#0x205 基于PXY IP替换
517=10:PXY_INTERCEPT_COMPILE;11:PXY_INTERCEPT_GROUP;12:PXY_INTERCEPT_IP;14:PXY_INTERCEPT_PKT_BIN,PXY_CTRL_SUBSCRIBE_ID;18:NTC_IP_RANGE
@@ -342,7 +342,7 @@ unMaatService=3:32;4:96;25:32;28:32;64;65;261:16;262:16;263:16;264:16;265:16;266
#0x402 APP DOMAIN特征发现 1026:2,5;
#0x404 APP IP特征 1028:2,5;
##########
serviceDBIndex=1:2,4,5;2:2,4,5;5:2,5;6:2,5;66:2,4,5;16:2;17:2;18:2;19:2;20:2;21:2;22:2;23:2;24:2;26:2;27:2;30:2;31:2;32:2;33:2,5;34:2;35:2;36:2;37:2;38:2;39:2;40:2;128:2;129:2;130:2;131:2;132:2;133:2;134:2;135:2;136:2;137:2;138:2;139:2;140:2;142:2;143:2;144:2;145:2;146:2;147:2;148:2;149:2;150:2;151:2;152:2;256:2;257:2;258:2;259:2;260:2;267:2;271:2;272:2;273:3;274:5;384:2;385:2;386:2;387:2;388:2;395:2;399:3;512:4,5;513:4,2,5,7;514:4,5;515:4,2,5,7;517:4,5;518:4,5;520:4;521:4;528:4;544:4;560:4;576:4;592:4;608:4;624:4;625:4;640:4;641:4;642:4;750:4;1024:2;1025:2;1026:2,5;1027:2;1028:2,5;1029:2;1030:2;1040:2,5;1041:2;1042:2;1056:2,5;1152:2;3:5;4:5;25:5;28:5;64:2;65:2;261:3;262:3;263:3;264:3;265:3;266:3;268:3;269:3;270:3;389:3;390:3;391:3;392:3;393:3;394:3;396:3;397:3;398:3;834:6;835:6;836:6
serviceDBIndex=1:2,4,5;2:2,4,5;5:2,5;6:2,5;66:2,4,5;16:2;17:2;18:2;19:2;20:2;21:2;22:2;23:2;24:2;26:2;27:2;30:2;31:2;32:2;33:2,5;34:2;35:2;36:2;37:2;38:2;39:2;40:2;128:2;129:2;130:2;131:2;132:2;133:2;134:2;135:2;136:2;137:2;138:2;139:2;140:2;142:2;143:2;144:2;145:2;146:2;147:2;148:2;149:2;150:2;151:2;152:2;256:2;257:2;258:2;259:2;260:2;267:2;271:2;272:2;273:3;274:5;384:2;385:2;386:2;387:2;388:2;395:2;399:3;512:4,5,7;513:4,2,5,7;514:4,5,7;515:4,2,5,7;517:4,5;518:4,5;520:4;521:4;528:4;544:4;560:4;576:4;592:4;608:4;624:4;625:4;640:4;641:4;642:4;750:4;1024:2;1025:2;1026:2,5;1027:2;1028:2,5;1029:2;1030:2;1040:2,5;1041:2;1042:2;1056:2,5;1152:2;3:5;4:5;25:5;28:5;64:2;65:2;261:3;262:3;263:3;264:3;265:3;266:3;268:3;269:3;270:3;389:3;390:3;391:3;392:3;393:3;394:3;396:3;397:3;398:3;834:6;835:6;836:6
##阀门配置在redisdb的序号
tapRedisDb=5
@@ -358,13 +358,13 @@ tapRedisDb=5
#0x205 基于PXY IP替换 517
#0x402 APP DOMAIN特征发现 1026
#0x404 APP IP特征 1028
maat2Valve=1:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;2:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;33:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;512:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;513:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;514:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;515:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;517:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;518:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;1026:strRegion|strStrRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;1028:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;1040:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;1056:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;5:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE&COMPILE_ID;6:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE&COMPILE_ID
maat2Valve=1:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;2:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;33:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;512:ipRegion|strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;514:ipRegion|strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;517:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;518:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;1026:strRegion|strStrRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;1028:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;1040:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;1056:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;5:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE&COMPILE_ID;6:ipRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE&COMPILE_ID
##webfocus配置在redisdb的序号
webFocusDb=7
maat2WebFocus=513:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;515:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;
maat2WebFocus=512:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;514:strRegion@ACTION&SERVICE&USER_REGION&EFFECTIVE_RANGE;
##记录哪些service可以被分组复用(只有maat类配置可以被分组复用)

View File

@@ -37,7 +37,7 @@ MmFileDigestLogClickHouseTable=TBS_ODS_MM_FILE_DIGEST_LOG
NtcKeywordsUrlLogClickHouseTable=TBS_ODS_NTC_KEYWORDS_URL_LOG
NtcCollectVoipLogClickHouseTable=TBS_ODS_NTC_COLLECT_VOIP_LOG
NtcCollectRadiusLogClickHouseTable=TBS_ODS_NTC_COLLECT_RADIUS_LOG
NtcConnRecordLogClickHouseTable=TBS_ODS_NTC_CONN_RECORD_LOG_LOCAL
NtcConnRecordLogClickHouseTable=TBS_ODS_NTC_CONN_RECORD_LOG