1:更新642的表达式结构,及添加opTime字段

2:为0x401  APP载荷特征表添加数值类表APP_TCP_SESSION_BYTE
3:更新urlreport接口
4:更新asnreport接口
5:添加iprange统计入库
6:修改配置入库时,先打印错误信息在回滚事务
7:修改配置入库打印的日志信息
8:更改通联关系app_label的数据类型为varchar
9:提交洪庆发来的trafficIpActiveStatisticDao.xml
This commit is contained in:
renkaige
2018-12-17 22:48:15 +06:00
parent 60a85266de
commit 9f8fb67584
23 changed files with 392 additions and 694 deletions

View File

@@ -20,11 +20,12 @@ public class NtcAsnRecord implements Serializable {
private static final long serialVersionUID = 1L;
private String bps;
private String pps;
private String sAsn;
private String dAsn;
private String asn;
private String searchFoundStartTime;// 开始发现时间
private String searchFoundEndTime;// 结束发现时间
private String searchAsnType;// 1是d_asn,2是s_asn
public String getBps() {
return bps;
}
@@ -41,20 +42,21 @@ public class NtcAsnRecord implements Serializable {
this.pps = pps;
}
public String getsAsn() {
return sAsn;
public String getAsn() {
return asn;
}
public void setsAsn(String sAsn) {
this.sAsn = sAsn;
public void setAsn(String asn) {
this.asn = asn;
}
public String getdAsn() {
return dAsn;
@JsonIgnore
public String getSearchAsnType() {
return searchAsnType;
}
public void setdAsn(String dAsn) {
this.dAsn = dAsn;
public void setSearchAsnType(String searchAsnType) {
this.searchAsnType = searchAsnType;
}
@JsonIgnore

View File

@@ -17,9 +17,9 @@ public class NtcConnRecordLog extends LogEntity<NtcConnRecordLog> {
/**
*
*/
private static final long serialVersionUID = 1L;
private static final Long serialVersionUID = 1L;
@ApiModelProperty(value = "", required = true)
protected Long appLabel;
protected String appLabel;
@ApiModelProperty(value = "", required = true)
protected String c2sPktNum;
@ApiModelProperty(value = "", required = true)
@@ -29,11 +29,11 @@ public class NtcConnRecordLog extends LogEntity<NtcConnRecordLog> {
@ApiModelProperty(value = "", required = true)
protected String s2cByteNum;
public Long getAppLabel() {
public String getAppLabel() {
return appLabel;
}
public void setAppLabel(Long appLabel) {
public void setAppLabel(String appLabel) {
this.appLabel = appLabel;
}

View File

@@ -24,11 +24,11 @@ public class NtcIpRangeReport<T> implements Serializable {
private Long ipEndNum;
private String ipSub;
private String country;
private Integer areaType;
private String searchType;
private String searchCountry;
public String getIpStart() {
return ipStart;
}
@@ -77,6 +77,14 @@ public class NtcIpRangeReport<T> implements Serializable {
this.country = country;
}
public Integer getAreaType() {
return areaType;
}
public void setAreaType(Integer areaType) {
this.areaType = areaType;
}
@JsonIgnore
public String getSearchType() {
return searchType;
@@ -95,5 +103,4 @@ public class NtcIpRangeReport<T> implements Serializable {
this.searchCountry = searchCountry;
}
}

View File

@@ -1,46 +0,0 @@
package com.nis.domain.restful;
import java.io.Serializable;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.wordnik.swagger.annotations.ApiModelProperty;
public class NtcIpURLReport implements Serializable {
/**
*
*/
private static final long serialVersionUID = 1L;
@ApiModelProperty(value = "日志数量", required = true)
private Long sum;
@ApiModelProperty(value = "ip地址", required = true)
private String ipAddr;
@ApiModelProperty(value = "url", required = true)
private String url;
public Long getSum() {
return sum;
}
public void setSum(Long sum) {
this.sum = sum;
}
public String getIpAddr() {
return ipAddr;
}
public void setIpAddr(String ipAddr) {
this.ipAddr = ipAddr;
}
@JsonIgnore
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
}

View File

@@ -1,11 +1,9 @@
package com.nis.domain.restful;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.wordnik.swagger.annotations.ApiModelProperty;
public class NtcURLReport extends NtcReportEntity<NtcURLReport> {
public class NtcURLIpReport extends NtcReportEntity<NtcURLIpReport> {
/**
*
@@ -13,11 +11,10 @@ public class NtcURLReport extends NtcReportEntity<NtcURLReport> {
private static final long serialVersionUID = 1L;
@ApiModelProperty(value = "url", required = true)
private String url;
@ApiModelProperty(value = "日志数量", required = true)
private Long urlCount;
@ApiModelProperty(value = "ip数量列表", required = true)
private List<NtcIpURLReport> dataList;
@ApiModelProperty(value = "唯一ip的数量", required = true)
private Long ipCount;
@ApiModelProperty(value = "连接数量", required = true)
private Long connCount;
private String searchUrl;
@@ -29,20 +26,20 @@ public class NtcURLReport extends NtcReportEntity<NtcURLReport> {
this.url = url;
}
public Long getUrlCount() {
return urlCount;
public Long getIpCount() {
return ipCount;
}
public void setUrlCount(Long urlCount) {
this.urlCount = urlCount;
public void setIpCount(Long ipCount) {
this.ipCount = ipCount;
}
public List<NtcIpURLReport> getDataList() {
return dataList;
public Long getConnCount() {
return connCount;
}
public void setDataList(List<NtcIpURLReport> dataList) {
this.dataList = dataList;
public void setConnCount(Long connCount) {
this.connCount = connCount;
}
@JsonIgnore
public String getSearchUrl() {
@@ -53,6 +50,4 @@ public class NtcURLReport extends NtcReportEntity<NtcURLReport> {
this.searchUrl = searchUrl;
}
}

View File

@@ -207,6 +207,10 @@ public final class Constants {
* 样例文件存放目录
*/
public static final String MM_SAMPLE_DST_PATH = Configurations.getStringProperty("mmSampleDstPath", "/home/mesasoft/{tableType}/full/{fileName}");
/**
* ip定位库
*/
public static final String IPLOCATIONLIBRARYPATH = Configurations.getStringProperty("ipLocationLibraryPath", "/usr/local/ipLocalLibraay/Kazakhstan.mmdb");
/**
*是否使用Minio
*/

View File

@@ -98,17 +98,19 @@ public class ConfigSourcesController extends BaseRestController {
}
} catch (Exception e) {
// TODO: handle exception
thread.setExceptionInfo("Maat 规则存储异常:" + e.getMessage());
logger.error("Maat 规则存储异常:" + e.getMessage());
thread.setExceptionInfo("Maat 规则存储异常:" + ExceptionUtil.getExceptionMsg(e));
logger.error("Maat 规则存储异常:" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"Maat 规则存储异常:" + e.getMessage(), ((RestServiceException) e).getErrorCode());
"Maat 规则存储异常:" + ExceptionUtil.getExceptionMsg(e), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"Maat 规则存储异常:" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
"Maat 规则存储异常:" + ExceptionUtil.getExceptionMsg(e),
((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"Maat 规则存储异常:" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
"Maat 规则存储异常:" + ExceptionUtil.getExceptionMsg(e),
RestBusinessCode.service_runtime_error.getValue());
}
} finally {
@@ -152,17 +154,21 @@ public class ConfigSourcesController extends BaseRestController {
}
} catch (Exception e) {
// TODO: handle exception
thread.setExceptionInfo("MAAT规则状态更新时出现异常:" + e.getMessage());
logger.error("MAAT规则状态更新时出现异常:" + e.getMessage());
thread.setExceptionInfo("MAAT规则状态更新时出现异常:" + ExceptionUtil.getExceptionMsg(e));
logger.error("MAAT规则状态更新时出现异常:" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"MAAT规则状态更新时出现异常:" + e.getMessage(), ((RestServiceException) e).getErrorCode());
"MAAT规则状态更新时出现异常:" + ExceptionUtil.getExceptionMsg(e),
((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"MAAT规则状态更新时出现异常:" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
"MAAT规则状态更新时出现异常:" + ExceptionUtil.getExceptionMsg(e),
((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"MAAT规则状态更新时出现异常:" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
"MAAT规则状态更新时出现异常:" + ExceptionUtil.getExceptionMsg(e),
RestBusinessCode.service_runtime_error.getValue());
}
} finally {
deblocking(requestId);
@@ -213,17 +219,18 @@ public class ConfigSourcesController extends BaseRestController {
}
} catch (Exception e) {
// TODO: handle exception
thread.setExceptionInfo("回调规则存储异常:" + e.getMessage());
logger.error("回调规则存储异常:" + e.getMessage());
thread.setExceptionInfo("回调规则存储异常:" + ExceptionUtil.getExceptionMsg(e));
logger.error("回调规则存储异常:" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start, "回调规则存储异常:" + e.getMessage(),
((RestServiceException) e).getErrorCode());
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"回调规则存储异常:" + ExceptionUtil.getExceptionMsg(e), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start, "回调规则存储异常:" + e.getMessage(),
((ServiceRuntimeException) e).getErrorCode());
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"回调规则存储异常:" + ExceptionUtil.getExceptionMsg(e), ((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"回调规则存储异常:" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
"回调规则存储异常:" + ExceptionUtil.getExceptionMsg(e),
RestBusinessCode.service_runtime_error.getValue());
}
} finally {
deblocking(requestId);
@@ -247,17 +254,18 @@ public class ConfigSourcesController extends BaseRestController {
}
} catch (Exception e) {
// TODO: handle exception
thread.setExceptionInfo("回调规则内容更新异常:" + e.getMessage());
logger.error("回调规则内容更新异常:" + e.getMessage());
thread.setExceptionInfo("回调规则内容更新异常:" + ExceptionUtil.getExceptionMsg(e));
logger.error("回调规则内容更新异常:" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"回调规则内容更新异常:" + e.getMessage(), ((RestServiceException) e).getErrorCode());
"回调规则内容更新异常:" + ExceptionUtil.getExceptionMsg(e), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"回调规则内容更新异常:" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
"回调规则内容更新异常:" + ExceptionUtil.getExceptionMsg(e), ((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"回调规则内容更新异常:" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
"回调规则内容更新异常:" + ExceptionUtil.getExceptionMsg(e),
RestBusinessCode.service_runtime_error.getValue());
}
} finally {
deblocking(requestId);
@@ -282,17 +290,18 @@ public class ConfigSourcesController extends BaseRestController {
}
} catch (Exception e) {
// TODO: handle exception
thread.setExceptionInfo("回调规则状态更新异常:" + e.getMessage());
logger.error("回调规则状态更新异常:" + e.getMessage());
thread.setExceptionInfo("回调规则状态更新异常:" + ExceptionUtil.getExceptionMsg(e));
logger.error("回调规则状态更新异常:" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"回调规则状态更新异常:" + e.getMessage(), ((RestServiceException) e).getErrorCode());
"回调规则状态更新异常:" + ExceptionUtil.getExceptionMsg(e), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"回调规则状态更新异常:" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
"回调规则状态更新异常:" + ExceptionUtil.getExceptionMsg(e), ((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"回调规则状态更新异常:" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
"回调规则状态更新异常:" + ExceptionUtil.getExceptionMsg(e),
RestBusinessCode.service_runtime_error.getValue());
}
} finally {
deblocking(requestId);
@@ -344,24 +353,25 @@ public class ConfigSourcesController extends BaseRestController {
}
} catch (IOException e) {
// TODO Auto-generated catch block
logger.error(RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + e.getMessage());
thread.setExceptionInfo(RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + e.getMessage());
logger.error(RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + ExceptionUtil.getExceptionMsg(e));
thread.setExceptionInfo(
RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + ExceptionUtil.getExceptionMsg(e));
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + e.getMessage(),
RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + ExceptionUtil.getExceptionMsg(e),
RestBusinessCode.FileUploadFailure.getValue());
} catch (Exception e) {
// TODO: handle exception
logger.error("文件上传异常:" + e.getMessage());
thread.setExceptionInfo("文件上传异常:" + e.getMessage());
logger.error("文件上传异常:" + ExceptionUtil.getExceptionMsg(e));
thread.setExceptionInfo("文件上传异常:" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start, e.getMessage(),
((RestServiceException) e).getErrorCode());
throw new RestServiceException(thread, System.currentTimeMillis() - start,
ExceptionUtil.getExceptionMsg(e), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start, e.getMessage(),
((ServiceRuntimeException) e).getErrorCode());
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
ExceptionUtil.getExceptionMsg(e), ((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start, e.getMessage(),
RestBusinessCode.FileUploadFailure.getValue());
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
ExceptionUtil.getExceptionMsg(e), RestBusinessCode.FileUploadFailure.getValue());
}
}
@@ -415,24 +425,26 @@ public class ConfigSourcesController extends BaseRestController {
}
} catch (IOException e) {
// TODO Auto-generated catch block
logger.error(RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + e.getMessage());
thread.setExceptionInfo(RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + e.getMessage());
logger.error(RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + ExceptionUtil.getExceptionMsg(e));
thread.setExceptionInfo(
RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + ExceptionUtil.getExceptionMsg(e));
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + e.getMessage(),
RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + ExceptionUtil.getExceptionMsg(e),
RestBusinessCode.FileUploadFailure.getValue());
} catch (Exception e) {
// TODO: handle exception
logger.error(RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + e.getMessage());
thread.setExceptionInfo(RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + e.getMessage());
logger.error(RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + ExceptionUtil.getExceptionMsg(e));
thread.setExceptionInfo(
RestBusinessCode.FileUploadFailure.getErrorReason() + ":" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start, e.getMessage(),
((RestServiceException) e).getErrorCode());
throw new RestServiceException(thread, System.currentTimeMillis() - start,
ExceptionUtil.getExceptionMsg(e), ((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start, e.getMessage(),
((ServiceRuntimeException) e).getErrorCode());
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
ExceptionUtil.getExceptionMsg(e), ((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start, e.getMessage(),
RestBusinessCode.FileUploadFailure.getValue());
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
ExceptionUtil.getExceptionMsg(e), RestBusinessCode.FileUploadFailure.getValue());
}
}
try {
@@ -449,10 +461,12 @@ public class ConfigSourcesController extends BaseRestController {
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
logger.error(RestBusinessCode.GetFileDigestFailure.getValue() + ":" + e.getMessage() + ",请检查摘要获取工具是否安装成功");
thread.setExceptionInfo("摘要获取过程中出现异常:" + e.getMessage() + ",请检查摘要获取工具是否安装成功");
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
RestBusinessCode.GetFileDigestFailure.getErrorReason() + ":" + e.getMessage() + ",请检查摘要获取工具是否安装成功",
logger.error(RestBusinessCode.GetFileDigestFailure.getValue() + ":" + ExceptionUtil.getExceptionMsg(e)
+ ",请检查摘要获取工具是否安装成功");
thread.setExceptionInfo("摘要获取过程中出现异常:" + ExceptionUtil.getExceptionMsg(e) + ",请检查摘要获取工具是否安装成功");
throw new ServiceRuntimeException(
thread, System.currentTimeMillis() - start, RestBusinessCode.GetFileDigestFailure.getErrorReason()
+ ":" + ExceptionUtil.getExceptionMsg(e) + ",请检查摘要获取工具是否安装成功",
RestBusinessCode.GetFileDigestFailure.getValue());
}
return serviceResponse(thread, System.currentTimeMillis() - start, request, response, "摘要获取成功", resultObject);
@@ -485,17 +499,20 @@ public class ConfigSourcesController extends BaseRestController {
}
} catch (Exception e) {
// TODO: handle exception
thread.setExceptionInfo("Maat 分组复用规则存储异常:" + e.getMessage());
logger.error("Maat 分组复用规则存储异常:" + e.getMessage());
thread.setExceptionInfo("Maat 分组复用规则存储异常:" + ExceptionUtil.getExceptionMsg(e));
logger.error("Maat 分组复用规则存储异常:" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"Maat 分组复用规则存储异常:" + e.getMessage(), ((RestServiceException) e).getErrorCode());
"Maat 分组复用规则存储异常:" + ExceptionUtil.getExceptionMsg(e),
((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"Maat 分组复用规则存储异常:" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
"Maat 分组复用规则存储异常:" + ExceptionUtil.getExceptionMsg(e),
((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"Maat 分组复用规则存储异常:" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
"Maat 分组复用规则存储异常:" + ExceptionUtil.getExceptionMsg(e),
RestBusinessCode.service_runtime_error.getValue());
}
} finally {
@@ -535,17 +552,20 @@ public class ConfigSourcesController extends BaseRestController {
}
} catch (Exception e) {
// TODO: handle exception
thread.setExceptionInfo("MAAT规则状态更新时出现异常:" + e.getMessage());
logger.error("删除MAAT规则分组复用域配置时出现异常:" + e.getMessage());
thread.setExceptionInfo("MAAT规则状态更新时出现异常:" + ExceptionUtil.getExceptionMsg(e));
logger.error("删除MAAT规则分组复用域配置时出现异常:" + ExceptionUtil.getExceptionMsg(e));
if (e instanceof RestServiceException) {
throw new RestServiceException(thread, System.currentTimeMillis() - start,
"删除MAAT规则分组复用域配置时出现异常:" + e.getMessage(), ((RestServiceException) e).getErrorCode());
"删除MAAT规则分组复用域配置时出现异常:" + ExceptionUtil.getExceptionMsg(e),
((RestServiceException) e).getErrorCode());
} else if (e instanceof ServiceRuntimeException) {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"删除MAAT规则分组复用域配置时出现异常:" + e.getMessage(), ((ServiceRuntimeException) e).getErrorCode());
"删除MAAT规则分组复用域配置时出现异常:" + ExceptionUtil.getExceptionMsg(e),
((ServiceRuntimeException) e).getErrorCode());
} else {
throw new ServiceRuntimeException(thread, System.currentTimeMillis() - start,
"删除MAAT规则分组复用域配置时出现异常:" + e.getMessage(), RestBusinessCode.service_runtime_error.getValue());
"删除MAAT规则分组复用域配置时出现异常:" + ExceptionUtil.getExceptionMsg(e),
RestBusinessCode.service_runtime_error.getValue());
}
} finally {
deblocking(requestId);
@@ -567,10 +587,9 @@ public class ConfigSourcesController extends BaseRestController {
try {
map = configRedisService.getAllKVByCompileId(compileId);
} catch (Exception e) {
thread.setExceptionInfo("根据配置id获取对应的编译,组,域等信息时出现异常:" + e.getMessage());
thread.setExceptionInfo("根据配置id获取对应的编译,组,域等信息时出现异常:" + ExceptionUtil.getExceptionMsg(e));
logger.error("根据配置id获取对应的编译,组,域等信息时出现异常:" + ExceptionUtil.getExceptionMsg(e));
throw new RestServiceException("根据配置id获取对应的编译,组,域等信息失败",
RestBusinessCode.ConfigSourceIsNull.getValue());
throw new RestServiceException("根据配置id获取对应的编译,组,域等信息失败", RestBusinessCode.ConfigSourceIsNull.getValue());
}
return compileServiceResponse(thread, System.currentTimeMillis() - start, request, response,

View File

@@ -734,10 +734,8 @@ public class NtcLogSearchController extends BaseRestController {
if (!StringUtil.isBlank(ntcConnRecordPercent.getSearchFoundEndTime())) {
sdf.setLenient(false);
sdf.parse(ntcConnRecordPercent.getSearchFoundEndTime());
} else {
throw new RestServiceException("searchFoundEndTime参数不能为空",
RestBusinessCode.param_formate_error.getValue());
}
} catch (ParseException e) {
throw new RestServiceException("searchFoundEndTime参数格式错误",
RestBusinessCode.param_formate_error.getValue());
@@ -750,9 +748,6 @@ public class NtcLogSearchController extends BaseRestController {
if (!StringUtil.isBlank(ntcConnRecordPercent.getSearchFoundStartTime())) {
sdf.setLenient(false);
sdf.parse(ntcConnRecordPercent.getSearchFoundStartTime());
} else {
throw new RestServiceException("searchFoundStartTime参数不能为空",
RestBusinessCode.param_formate_error.getValue());
}
} catch (ParseException e) {
throw new RestServiceException("searchFoundStartTime参数格式错误",
@@ -801,10 +796,8 @@ public class NtcLogSearchController extends BaseRestController {
if (!StringUtil.isBlank(ntcAsnRecord.getSearchFoundEndTime())) {
sdf.setLenient(false);
sdf.parse(ntcAsnRecord.getSearchFoundEndTime());
} else {
throw new RestServiceException("searchFoundEndTime参数不能为空",
RestBusinessCode.param_formate_error.getValue());
}
} catch (ParseException e) {
throw new RestServiceException("searchFoundEndTime参数格式错误",
RestBusinessCode.param_formate_error.getValue());
@@ -817,10 +810,8 @@ public class NtcLogSearchController extends BaseRestController {
if (!StringUtil.isBlank(ntcAsnRecord.getSearchFoundStartTime())) {
sdf.setLenient(false);
sdf.parse(ntcAsnRecord.getSearchFoundStartTime());
} else {
throw new RestServiceException("searchFoundStartTime参数不能为空",
RestBusinessCode.param_formate_error.getValue());
}
} catch (ParseException e) {
throw new RestServiceException("searchFoundStartTime参数格式错误",
RestBusinessCode.param_formate_error.getValue());
@@ -828,6 +819,19 @@ public class NtcLogSearchController extends BaseRestController {
throw new RestServiceException("searchFoundStartTime参数错误",
RestBusinessCode.param_formate_error.getValue());
}
try {
if (!StringUtil.isBlank(ntcAsnRecord.getSearchAsnType())) {
Integer.parseInt(ntcAsnRecord.getSearchAsnType());
}else {
throw new RestServiceException("searchAsnType参数不能为空",
RestBusinessCode.param_formate_error.getValue());
}
} catch (NumberFormatException e) {
throw new RestServiceException("searchAsnType参数格式错误"+e.getMessage(), RestBusinessCode.param_formate_error.getValue());
} catch (Exception e) {
throw new RestServiceException("searchAsnType参数错误"+e.getMessage(), RestBusinessCode.param_formate_error.getValue());
}
logDataService.getNtcAsnRecord(page, ntcAsnRecord);
} catch (Exception e) {
auditLogThread.setExceptionInfo("ASN通联关系(源,目的)日志检索失败:" + e.getMessage());

View File

@@ -25,7 +25,7 @@ import com.nis.domain.restful.NtcReportEntity;
import com.nis.domain.restful.NtcServiceReport;
import com.nis.domain.restful.NtcSrcipDomesticReport;
import com.nis.domain.restful.NtcTagReport;
import com.nis.domain.restful.NtcURLReport;
import com.nis.domain.restful.NtcURLIpReport;
import com.nis.restful.RestBusinessCode;
import com.nis.restful.RestServiceException;
import com.nis.restful.ServiceRuntimeException;
@@ -413,7 +413,7 @@ public class SingleDimensionReport extends BaseRestController {
@RequestMapping(value = "/ntcURLReport", method = RequestMethod.GET)
@ApiOperation(value = "URL统计查询服务", httpMethod = "GET", notes = "URL统计查询服务")
public Map<String, ?> ntcURLReport(Page page, NtcURLReport urlReport, Model model, HttpServletRequest request,
public Map<String, ?> ntcURLReport(Page page, NtcURLIpReport urlReport, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis();
AuditLogThread saveLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET, request,
@@ -422,16 +422,16 @@ public class SingleDimensionReport extends BaseRestController {
Page<?> reportPage = null;
try {
resetReportTime(urlReport, false);
ntcReportService.queryReportConditionCheck(saveLogThread, start, urlReport, NtcURLReport.class, page);
ntcReportService.queryReportConditionCheck(saveLogThread, start, urlReport, NtcURLIpReport.class, page);
String orderBy = "";
if (null != page.getOrderBy() && !page.getOrderBy().equals("")) {
orderBy = Page.getOrderBySql(NtcURLReport.class.getSimpleName(), page.getOrderBy());
orderBy = Page.getOrderBySql(NtcURLIpReport.class.getSimpleName(), page.getOrderBy());
} else {
orderBy = "report_time";
}
page.setOrderBy(orderBy);
reportPage = ntcReportService
.findNtcURLReport(new Page<NtcURLReport>(request, response, NtcURLReport.class), urlReport);
.findNtcURLReport(new Page<NtcURLIpReport>(request, response, NtcURLIpReport.class), urlReport);
} catch (Exception e) {
saveLogThread.setExceptionInfo("URL统计查询失败:" + e.getMessage());
logger.error("URL统计查询失败:" + ExceptionUtil.getExceptionMsg(e));

View File

@@ -1517,7 +1517,7 @@
<result column="inner_dmac" jdbcType="VARCHAR" property="innerDmac" />
<result column="app_label" jdbcType="BIGINT" property="appLabel" />
<result column="app_label" jdbcType="VARCHAR" property="appLabel" />
<result column="c2s_pkt_num" jdbcType="VARCHAR" property="c2sPktNum" />
<result column="s2c_pkt_num" jdbcType="VARCHAR" property="s2cPktNum" />
<result column="c2s_byte_num" jdbcType="VARCHAR" property="c2sByteNum" />

View File

@@ -6,14 +6,13 @@ import com.nis.domain.restful.NtcAttrTypeReport;
import com.nis.domain.restful.NtcDestipCountryReport;
import com.nis.domain.restful.NtcEntranceReport;
import com.nis.domain.restful.NtcIpRangeReport;
import com.nis.domain.restful.NtcIpURLReport;
import com.nis.domain.restful.NtcLwhhReport;
import com.nis.domain.restful.NtcPzReport;
import com.nis.domain.restful.NtcRadiusReport;
import com.nis.domain.restful.NtcServiceReport;
import com.nis.domain.restful.NtcSrcipDomesticReport;
import com.nis.domain.restful.NtcTagReport;
import com.nis.domain.restful.NtcURLReport;
import com.nis.domain.restful.NtcURLIpReport;
/**
*
@@ -26,18 +25,32 @@ import com.nis.domain.restful.NtcURLReport;
@MyBatisDao
public interface NtcReportDao extends CrudDao {
List<NtcPzReport> findNtcPzReport(NtcPzReport pz);
List<NtcServiceReport> findNtcServiceReport(NtcServiceReport pz);
List<NtcTagReport> findNtcTagReport(NtcTagReport pz);
List<NtcAttrTypeReport> findNtcAttrTypeReport(NtcAttrTypeReport pz);
List<NtcLwhhReport> findNtcLwhhReport(NtcLwhhReport pz);
List<NtcSrcipDomesticReport> findNtcSrcipDomesticReport(NtcSrcipDomesticReport pz);
List<NtcDestipCountryReport> findNtcDestipCountryReport(NtcDestipCountryReport pz);
List<NtcEntranceReport> findNtcEntranceReport(NtcEntranceReport pz);
List<NtcIpURLReport> findNtcIpURLReport(NtcURLReport pz);
List<NtcURLIpReport> findNtcIpURLReport(NtcURLIpReport pz);
List<NtcRadiusReport> findAccounList(NtcRadiusReport pz);
List<NtcRadiusReport> findNasIpList(NtcRadiusReport pz);
List<NtcRadiusReport> findNtcRadiusReport(NtcRadiusReport pz);
List<NtcIpRangeReport> findNtcIpRangeReport(NtcIpRangeReport pz);
void insertNtcIpRangeBatch(List<NtcIpRangeReport> list);
void truncateNtcIpRange();
}

View File

@@ -71,11 +71,12 @@
property="reportTime" />
</resultMap>
<resultMap id="NtcIpURLReportMap"
type="com.nis.domain.restful.NtcIpURLReport">
<resultMap id="NtcURLIpReportMap"
type="com.nis.domain.restful.NtcURLIpReport">
<result column="url" jdbcType="VARCHAR" property="url" />
<result column="SUM" jdbcType="BIGINT" property="sum" />
<result column="ip_addr" jdbcType="VARCHAR" property="ipAddr" />
<result column="ipCount" jdbcType="BIGINT" property="ipCount" />
<result column="connCount" jdbcType="BIGINT" property="connCount" />
<result column="reportTime" jdbcType="TIMESTAMP" property="reportTime" />
</resultMap>
<resultMap id="NtcRadiusReportMap"
@@ -744,8 +745,8 @@
</select>
<select id="findNtcIpURLReport"
parameterType="com.nis.domain.restful.NtcURLReport"
resultMap="NtcIpURLReportMap">
parameterType="com.nis.domain.restful.NtcURLIpReport"
resultMap="NtcURLIpReportMap">
SELECT
<choose>
<when
@@ -753,10 +754,13 @@
${page.fields}
</when>
<otherwise>
url, ip_addr,SUM
url, ipCount,connCount,reportTime
</otherwise>
</choose>
FROM (SELECT url, SUM(SUM) SUM ,ip_addr
FROM (SELECT url,
count(distinct ip_addr) ipCount,
sum(sum) connCount,
stat_time reportTime
FROM ntc_reject_url_statistic
<where>
@@ -772,7 +776,7 @@
AND url = #{searchUrl}
</if>
</where>
GROUP BY url,ip_addr
GROUP BY url,stat_time
) ntc_reject_url_statistic
<choose>
<when
@@ -853,7 +857,8 @@
parameterType="com.nis.domain.restful.NtcIpRangeReport"
resultMap="NtcIpRangeReportMap">
SELECT ip_start,ip_end,ip_start_num,ip_end_num,ip_sub
FROM traffic_ip_identify
FROM
traffic_ip_identify
<where>
<if test="searchType != null and searchType !=''">
<![CDATA[AND areaType = #{searchType}]]>
@@ -864,4 +869,21 @@
</where>
</select>
<insert id="insertNtcIpRangeBatch"
parameterType="com.nis.domain.restful.NtcIpRangeReport">
INSERT INTO traffic_ip_identify
(ip_start,ip_end,ip_start_num,ip_end_num,ip_sub,area_type,country)
VALUES
<foreach collection="list" item="ipRange" separator=",">
(#{ipRange.ipStart}, #{ipRange.ipEnd},
#{ipRange.ipStartNum}, #{ipRange.ipEndNum},
#{ipRange.ipSub}, #{ipRange.areaType},
#{ipRange.country})
</foreach>
</insert>
<update id="truncateNtcIpRange">
truncate table traffic_ip_identify
</update>
</mapper>

View File

@@ -50,7 +50,7 @@
<select id="ipActiveFiveMinute" resultMap="BaseResultMap">
select SUM(c2s_byte_len+s2c_byte_len) count ,stat_time time from traffic_ip_active_statistic
WHERE ip_addr=#{ipAddr} and
<![CDATA[ stat_time> #{beginDate} and stat_time<=#{endDate}]]>
<![CDATA[ stat_time> #{beginDate} and stat_time<=#{endDate} GROUP BY stat_time]]>
</select>
<!-- IP流量统计一小时-->
<select id="ipActiveOneHour" resultType="java.util.HashMap">

View File

@@ -191,8 +191,7 @@ public class LocalLogJDBCByDruid {
String pps = rs.getString("pps");
ntcAsnRecord.setBps(subStr(bps));
ntcAsnRecord.setPps(subStr(pps));
ntcAsnRecord.setdAsn(rs.getString("d_asn"));
ntcAsnRecord.setsAsn(rs.getString("s_asn"));
ntcAsnRecord.setAsn(rs.getString("asn"));
list.add(ntcAsnRecord);
}
logger.info("执行日志查询语句成功,sql={}", sql);
@@ -202,10 +201,11 @@ public class LocalLogJDBCByDruid {
return list;
}
public List<String> getAllIp(String frontier) throws Exception {
public List<String> getAllIp(Integer frontier) throws Exception {
List<String> list = new LinkedList<>();
try {
String sql = "select distinct ip from ip_location_database_local where ip like '145%' and frontier='"+frontier+"'";
// String sql = "select distinct ip from ip_location_database_local where ip like '145%' and frontier='"+frontier+"'";
String sql = "select distinct ip from ip_location_database_local where frontier='"+frontier+"'";
conn = getConnection();
logger.info("连接数据中心日志库成功--------------------------");
st = conn.createStatement();

View File

@@ -2,6 +2,7 @@ package com.nis.web.service;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
@@ -81,6 +82,58 @@ public class LogDataService {
return ntcConnRecordPercent;
}
public static void main(String[] args) throws ParseException {
String asn = "s_asn ";
long startTime = sdf.parse("2018-12-17 21:30:00").getTime() / 1000;
long endTime = sdf.parse("2018-12-17 21:50:00").getTime() / 1000;
long second = 200l;
StringBuffer sql = new StringBuffer();
sql.append("SELECT SUM(s2c_pkt_num + s2c_pkt_num)*8/");
sql.append(second);
sql.append(" AS pps , SUM(c2s_byte_num + s2c_byte_num)*8/");
sql.append(second);
sql.append(" AS bps,");
sql.append(asn);
sql.append(
" asn FROM tbs_ods_ntc_conn_record_log_local t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local WHERE found_time >= ");
sql.append(startTime);
sql.append(" and found_time< ");
sql.append(endTime);
StringBuffer countSql = new StringBuffer();
countSql.append("select count(1) from (");
countSql.append(sql);
countSql.append(" ) group by ");
countSql.append(asn+")");
Integer startNum =0;
Integer limitCount = startNum + 20;
sql.append(" limit " + limitCount + " ) group by "+asn+" limit " + startNum + "," + 20);
System.out.println(sql);
System.out.println(countSql);
}
public void getNtcAsnRecord(Page page, NtcAsnRecord ntcAsnRecord) throws Exception {
long startTime = sdf.parse(ntcAsnRecord.getSearchFoundStartTime().toString().trim()).getTime() / 1000;
long endTime = sdf.parse(ntcAsnRecord.getSearchFoundEndTime().toString().trim()).getTime() / 1000;
@@ -88,26 +141,36 @@ public class LogDataService {
throw new RestServiceException("searchFoundStartTime() can not exceed searchFoundEndTime",
RestBusinessCode.param_formate_error.getValue());
}
String asn = "s_asn ";
if (ntcAsnRecord.getSearchAsnType().trim().equals("1")) {
asn = "d_asn ";
}
long second = endTime - startTime;
StringBuffer sql = new StringBuffer();
sql.append("SELECT SUM(s2c_pkt_num + s2c_pkt_num)*8/");
sql.append(second);
sql.append(" AS pps , SUM(c2s_byte_num + s2c_byte_num)*8/");
sql.append(second);
sql.append(" AS bps,");
sql.append(asn);
sql.append(
" AS bps,d_asn,s_asn FROM tbs_ods_ntc_conn_record_log_local t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local WHERE found_time >= ");
" asn FROM tbs_ods_ntc_conn_record_log_local t WHERE found_time IN ( SELECT DISTINCT found_time FROM tbs_ods_ntc_conn_record_log_local WHERE found_time >= ");
sql.append(startTime);
sql.append(" and found_time< ");
sql.append(endTime);
StringBuffer countSql = new StringBuffer();
countSql.append("select count(1) from (");
countSql.append(sql);
countSql.append(" ) group by s_asn,d_asn)");
countSql.append(" ) group by ");
countSql.append(asn+")");
Integer startNum = (page.getPageNo() - 1) * page.getPageSize();
Integer limitCount = startNum + page.getPageSize();
sql.append(" limit " + limitCount + " ) group by s_asn,d_asn limit " + startNum + "," + page.getPageSize());
sql.append(" limit " + limitCount + " ) group by "+asn+" limit " + startNum + "," + page.getPageSize());
localLogJDBCByDruid.getCount(page, countSql.toString());

View File

@@ -160,9 +160,9 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
throw new ServiceRuntimeException(error, RestBusinessCode.CannotConnectionRedis.getValue());
} catch (Exception e) {
transaction.discard();
String error = "保存非maat类配置发生了异常," + e.getMessage();
logger.error(error + " " + ExceptionUtil.getExceptionMsg(e));
transaction.discard();
throw new ServiceRuntimeException(error, RestBusinessCode.SaveDataInError.getValue());
} finally {
// 释放连接到连接池
@@ -500,9 +500,9 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
logger.error(error + " " + ExceptionUtil.getExceptionMsg(e));
throw new ServiceRuntimeException(error, RestBusinessCode.CannotConnectionRedis.getValue());
} catch (Exception e) {
transaction.discard();
String error = "保存maat类配置发生了异常" + e.getMessage();
logger.error(error + " " + ExceptionUtil.getExceptionMsg(e));
transaction.discard();
throw new ServiceRuntimeException(error, RestBusinessCode.SaveDataInError.getValue());
} finally {
// 释放连接到连接池
@@ -879,9 +879,9 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
logger.error(error + " " + ExceptionUtil.getExceptionMsg(e));
throw new ServiceRuntimeException(error, RestBusinessCode.CannotConnectionRedis.getValue());
} catch (Exception e) {
transaction.discard();
String error = "删除非maat类配置发生了异常," + e.getMessage();
logger.error(error + " " + ExceptionUtil.getExceptionMsg(e));
transaction.discard();
throw new ServiceRuntimeException(error, RestBusinessCode.DeleteDataInError.getValue());
} finally {
// 释放连接到连接池
@@ -993,12 +993,13 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
throw new ServiceRuntimeException(error, RestBusinessCode.CannotConnectionRedis.getValue());
} catch (Exception e) {
transaction.discard();
int businessCode = RestBusinessCode.service_runtime_error.getValue();
if (e instanceof ServiceRuntimeException) {
businessCode = ((ServiceRuntimeException) e).getErrorCode();
}
logger.error("删除maat配置发生了异常" + ExceptionUtil.getExceptionMsg(e));
transaction.discard();
throw new ServiceRuntimeException("删除maat配置发生了异常," + e.getMessage(), businessCode);
} finally {
// 释放连接到连接池
@@ -1643,9 +1644,10 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
logger.error(error + " " + ExceptionUtil.getExceptionMsg(e));
throw new ServiceRuntimeException(error, RestBusinessCode.CannotConnectionRedis.getValue());
} catch (Exception e) {
transaction.discard();
String error = "保存分组复用maat类域配置发生了异常" + e.getMessage();
logger.error(error + " " + ExceptionUtil.getExceptionMsg(e));
transaction.discard();
throw new ServiceRuntimeException(error, RestBusinessCode.SaveDataInError.getValue());
} finally {
// 释放连接到连接池
@@ -1936,10 +1938,10 @@ public class ConfigJedisServiceimpl implements ConfigRedisService {
logger.error(error + " " + ExceptionUtil.getExceptionMsg(e));
throw new ServiceRuntimeException(error, RestBusinessCode.CannotConnectionRedis.getValue());
} catch (Exception e) {
e.printStackTrace();
transaction.discard();
String error = "删除分组复用maat类域配置发生了异常" + e.getMessage();
logger.error(error + " " + ExceptionUtil.getExceptionMsg(e));
transaction.discard();
throw new ServiceRuntimeException(error, RestBusinessCode.SaveDataInError.getValue());
} finally {
// 释放连接到连接池

View File

@@ -15,14 +15,13 @@ import com.nis.domain.restful.NtcAttrTypeReport;
import com.nis.domain.restful.NtcDestipCountryReport;
import com.nis.domain.restful.NtcEntranceReport;
import com.nis.domain.restful.NtcIpRangeReport;
import com.nis.domain.restful.NtcIpURLReport;
import com.nis.domain.restful.NtcLwhhReport;
import com.nis.domain.restful.NtcPzReport;
import com.nis.domain.restful.NtcRadiusReport;
import com.nis.domain.restful.NtcServiceReport;
import com.nis.domain.restful.NtcSrcipDomesticReport;
import com.nis.domain.restful.NtcTagReport;
import com.nis.domain.restful.NtcURLReport;
import com.nis.domain.restful.NtcURLIpReport;
import com.nis.web.dao.NtcReportDao;
import com.nis.web.service.BaseLogService;
@@ -109,37 +108,9 @@ public class NtcReportService extends BaseLogService {
return page;
}
public Page<NtcURLReport> findNtcURLReport(Page<NtcURLReport> page, NtcURLReport entity) throws Exception {
public Page<NtcURLIpReport> findNtcURLReport(Page<NtcURLIpReport> page, NtcURLIpReport entity) throws Exception {
entity.setPage(page);
Map<String, List<NtcIpURLReport>> map = new HashMap<>();
List<NtcURLReport> list = new ArrayList<>();
List<NtcIpURLReport> findNtcIpURLReport = dao.findNtcIpURLReport(entity);
for (NtcIpURLReport ntcIpURLReport : findNtcIpURLReport) {
String url = ntcIpURLReport.getUrl();
if (map.containsKey(url)) {
map.get(url).add(ntcIpURLReport);
} else {
List<NtcIpURLReport> ipUrlList = new ArrayList<>();
ipUrlList.add(ntcIpURLReport);
map.put(url, ipUrlList);
}
}
for (String url : map.keySet()) {
NtcURLReport ntcURLReport = new NtcURLReport();
ntcURLReport.setUrl(url);
long count = 0l;
List<NtcIpURLReport> list2 = map.get(url);
for (NtcIpURLReport ntcIpURLReport : list2) {
count += ntcIpURLReport.getSum();
}
ntcURLReport.setUrlCount(count);
ntcURLReport.setDataList(list2);
list.add(ntcURLReport);
}
page.setList(list);
page.setList(dao.findNtcIpURLReport(entity));
return page;
}

View File

@@ -1,24 +1,16 @@
package com.nis.web.task;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.avro.file.SyncableFileOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -26,7 +18,13 @@ import org.springframework.context.annotation.PropertySource;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.nis.domain.restful.NtcIpRangeReport;
import com.nis.util.Constants;
import com.nis.util.ExceptionUtil;
import com.nis.web.dao.NtcReportDao;
import com.nis.web.dao.impl.LocalLogJDBCByDruid;
import com.zdjizhi.utils.IPUtil;
import com.zdjizhi.utils.IpLookup;
@Component
@PropertySource(value = { "classpath:nis.properties", "classpath:jdbc.properties" })
@@ -36,25 +34,34 @@ public class GetIpRangeTask {
@Autowired
private LocalLogJDBCByDruid localLogJDBCByDruid;
// @Scheduled(cron = "0/5 * * * * ?")
@Autowired
protected NtcReportDao dao;
private static IpLookup ipLookup = new IpLookup.Builder(false).loadDataFileV4(Constants.IPLOCATIONLIBRARYPATH)
.loadDataFileV6(Constants.IPLOCATIONLIBRARYPATH).build();
// @Scheduled(cron = "${getIpRangeTaskCron}")
public void test() {
try {
getAllIp4();
// 删除数据,重新全量导入
dao.truncateNtcIpRange();
getAllIp(1);
getAllIp(2);
} catch (Exception e) {
e.printStackTrace();
logger.error("处理ip范围数据失败,失败原因{}", ExceptionUtil.getExceptionMsg(e));
}
}
public Map<Long, String> getNumAndIpReal(List<String> allIp) throws InterruptedException, ExecutionException {
logger.info("开始将ip转换为数字");
Map<Long, String> map = new HashMap<>();
int ever = 10000;
int ever = 10000;// 每个线程处理10000条ip转数字
int count = allIp.size() / ever;
if (allIp.size() % ever != 0) {
count++;
}
List<Future<Map<Long, String>>> futures = new ArrayList<>();
ExecutorService executor = Executors.newFixedThreadPool(10);
ExecutorService executor = Executors.newFixedThreadPool(10);// 10个线程
for (int i = 0; i < count; i++) {
int start = i * ever;
int end = i * ever + ever;
@@ -79,43 +86,49 @@ public class GetIpRangeTask {
return map;
}
private static String testbb(Long[] NoNum) {
/**
* 将数组中连续的数据合并在一起输出,例如传入1,2,3,5,6,7,9,11输出1-3,4-7,9,11 .注意传入的数组必须是有序的,从小到大排序
*
* @param noNum
* @return
*/
private static String groupByNumer(Long[] sortArr) {
int state = 0;
String result = "";
for (int i = 0; i < NoNum.length; i++) {
if (i == NoNum.length - 1)
for (int i = 0; i < sortArr.length; i++) {
if (i == sortArr.length - 1)
state = 2;
if (state == 0) {
if (NoNum[i + 1].longValue() == NoNum[i].longValue() + 1) {
result += String.valueOf(NoNum[i]);
if (sortArr[i + 1].longValue() == sortArr[i].longValue() + 1) {
result += String.valueOf(sortArr[i]);
result += "-";
state = 1;
} else {
result += String.valueOf(NoNum[i]);
result += String.valueOf(sortArr[i]);
result += ",";
}
} else if (state == 1) {
if (NoNum[i + 1] != NoNum[i] + 1) {
result += String.valueOf(NoNum[i]);
if (sortArr[i + 1] != sortArr[i] + 1) {
result += String.valueOf(sortArr[i]);
result += ",";
state = 0;
}
} else {
result += String.valueOf(NoNum[i]);
result += String.valueOf(sortArr[i]);
}
}
return result;
}
public void getAllIp4() throws Exception {
List<String> allIp = localLogJDBCByDruid.getAllIp("1");
public void getAllIp(Integer frontier) throws Exception {
List<String> allIp = localLogJDBCByDruid.getAllIp(frontier);
Map<Long, String> map = getNumAndIpReal(allIp);
Set<Long> keySet = map.keySet();
List<Long> ipList = new ArrayList<>(keySet);
Long[] array = new Long[ipList.size()];
Collections.sort(ipList);
ipList.toArray(array);
String result = testbb(array);
String result = groupByNumer(array);
if (result != null && !result.trim().equals("")) {
String substring = result;
if (result.endsWith(",")) {
@@ -123,13 +136,26 @@ public class GetIpRangeTask {
}
String[] split = substring.split(",");
List<Long> list = new ArrayList<>();
List<NtcIpRangeReport> ntcIpRangeReportList = new ArrayList<>();
for (String str : split) {
String[] ipAddr = str.split("-");
if (ipAddr.length > 1) {
for (String ipStr : ipAddr) {
System.out.print(ipStr + "[" + map.get(Long.parseLong(ipStr)) + "]");
String startIp = ipAddr[0];
if(startIp.toString().length()>11) {
System.out.println("1");
}
System.out.println();
String endIp = ipAddr[1];
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(Long.parseLong(startIp));
String endIpStr = map.get(Long.parseLong(endIp));
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(Long.parseLong(endIp));
ntcIpRangeReport.setIpStartNum(Long.parseLong(startIp));
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReportList.add(ntcIpRangeReport);
} else {
list.add(Long.parseLong(str));
}
@@ -138,9 +164,41 @@ public class GetIpRangeTask {
for (int i = 0; i < list.size(); i = i + 2) {
System.out.print(list.get(i) + "[" + map.get(list.get(i)) + "]");
if (i < list.size() - 1) {
System.out.print(list.get(i + 1) + "[" + map.get(list.get(i + 1)) + "]");
Long startIp = list.get(i);
if(startIp.toString().length()>11) {
System.out.println("1");
}
Long endIp = list.get(i + 1);
NtcIpRangeReport ntcIpRangeReport = new NtcIpRangeReport();
String startIpStr = map.get(startIp);
String endIpStr = map.get(endIp);
ntcIpRangeReport.setIpStart(startIpStr);
ntcIpRangeReport.setIpEnd(endIpStr);
ntcIpRangeReport.setIpEndNum(endIp);
ntcIpRangeReport.setIpStartNum(startIp);
ntcIpRangeReport.setAreaType(frontier);
ntcIpRangeReport.setIpSub(IPUtil.getMask(startIpStr, endIpStr));
ntcIpRangeReport.setCountry(ipLookup.countryLookup(startIpStr));
ntcIpRangeReportList.add(ntcIpRangeReport);
}
}
int size = ntcIpRangeReportList.size();
if (size > 0) {
int ever = 1000;// 每个线程处理10000条ip转数字
int count = size / ever;
if (size % ever != 0) {
count++;
}
for (int i = 0; i < count; i++) {
int start = i * ever;
int end = i * ever + ever;
if (end > size) {
end = size;
}
dao.insertNtcIpRangeBatch(ntcIpRangeReportList.subList(start, end));
}
System.out.println();
}
}

View File

@@ -1,423 +0,0 @@
package com.nis.web.task;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.avro.file.SyncableFileOutputStream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.PropertySource;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import com.nis.web.dao.impl.LocalLogJDBCByDruid;
@Component
@PropertySource(value = { "classpath:nis.properties", "classpath:jdbc.properties" })
public class GetIpRangeTask2 {
@Autowired
private LocalLogJDBCByDruid localLogJDBCByDruid;
// @Scheduled(cron = "0/5 * * * * ?")
public void test() {
try {
getAllIp4();
} catch (Exception e) {
e.printStackTrace();
}
}
public Map<Long, String> threadTest(List<String> allIp) throws InterruptedException, ExecutionException {
Map<Long, String> map = new HashMap<>();
int ever = 10000;
int count = allIp.size() / ever;
if (allIp.size() % ever != 0) {
count++;
}
List<Future<Map<Long, String>>> futures = new ArrayList<>();
ExecutorService executor = Executors.newFixedThreadPool(10);
for (int i = 0; i < count; i++) {
int start = i * ever;
int end = i * ever + ever;
if (end > allIp.size()) {
end = allIp.size();
}
IpToLongThread ipToLongThread = new IpToLongThread(allIp.subList(start, end));
Future<Map<Long, String>> res = executor.submit(ipToLongThread);// 异步提交, non blocking.
futures.add(res);
}
executor.shutdown();
System.out.println("开始关闭线程");
for (Future<Map<Long, String>> future : futures) {
try {
map.putAll(future.get());
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
System.out.println("list=" + map.size() + ",allIp=" + allIp.size());
System.out.println("线程执行完毕");
return map;
}
private static String testbb(Long[] NoNum) {
int state = 0;
String result = "";
for (int i = 0; i < NoNum.length; i++) {
if (i == NoNum.length - 1)
state = 2;
if (state == 0) {
if (NoNum[i + 1].longValue() == NoNum[i].longValue() + 1) {
result += String.valueOf(NoNum[i]);
result += "-";
state = 1;
} else {
result += String.valueOf(NoNum[i]);
result += ",";
}
} else if (state == 1) {
if (NoNum[i + 1] != NoNum[i] + 1) {
result += String.valueOf(NoNum[i]);
result += ",";
state = 0;
}
} else {
result += String.valueOf(NoNum[i]);
}
}
return result;
}
public void getAllIp4() throws Exception {
List<String> allIp = localLogJDBCByDruid.getAllIp("1");
Map<Long, String> map = threadTest(allIp);
Set<Long> keySet = map.keySet();
List<Long> ipList = new ArrayList<>(keySet);
Long[] array = new Long[ipList.size()];
Collections.sort(ipList);
ipList.toArray(array);
String result = testbb(array);
if (result != null && !result.trim().equals("")) {
String substring = result;
if(result.endsWith(",")) {
substring=result.substring(0, result.length() - 1);
}
String[] split = substring.split(",");
List<Long> list = new ArrayList<>();
for (String str : split) {
String[] ipAddr = str.split("-");
if (ipAddr.length > 1) {
for (String ipStr : ipAddr) {
System.out.print(ipStr + "[" + map.get(Long.parseLong(ipStr)) + "]");
}
System.out.println();
} else {
list.add(Long.parseLong(str));
}
}
Collections.sort(list);
for (int i = 0; i < list.size(); i = i + 2) {
System.out.print(list.get(i) + "[" + map.get(list.get(i)) + "]");
if (i < list.size() - 1) {
System.out.print(list.get(i + 1) + "[" + map.get(list.get(i + 1)) + "]");
}
System.out.println();
}
}
}
public String longToIp(long ip) {
return ((ip >> 24) & 0xFF) + "." + ((ip >> 16) & 0xFF) + "." + ((ip >> 8) & 0xFF) + "." + (ip & 0xFF);
}
public static void main(String[] args) {
// Long [] NoNum = { 2449052146l,
// 2449052148l,
// 2449052136l,
// 2449052139l,
// 2449052138l,
// 2449052141l,
// 2449052128l,
// 2449052134l,
// 2449052120l,
// 2449052124l,
// 2449052127l};
Set<Long> keySet = new HashSet<>();
keySet.add(2449052146l);
keySet.add(2449052136l);
keySet.add(2449052139l);
keySet.add(2449052120l);
List<Long> ipList = new ArrayList<>(keySet);
Long[] array = new Long[ipList.size()];
Collections.sort(ipList);
ipList.toArray(array);
System.out.println(Arrays.toString(array));
Long[] NoNum = { 3l, 2l, 1l };
String testbb = testbb(NoNum);
System.out.println(testbb);
}
public void a() {
int[] NoNum = { 1, 2, 3, 5, 7, 8, 9, 10, 13 };
int state = 0;
String result = "";
for (int i = 0; i < NoNum.length; i++) {
if (i == NoNum.length - 1)
state = 2;
if (state == 0) {
if (NoNum[i + 1] == NoNum[i] + 1) {
result += Integer.toString(NoNum[i]);
result += "-";
state = 1;
} else {
result += Integer.toString(NoNum[i]);
result += ",";
}
} else if (state == 1) {
if (NoNum[i + 1] != NoNum[i] + 1) {
result += Integer.toString(NoNum[i]);
result += ",";
state = 0;
}
} else {
result += Integer.toString(NoNum[i]);
}
}
System.out.println(result);
}
public static String convert(Long[] ints, int index) {
int end = index;
if (ints.length == index) {// 结束条件,遍历完数组
return "";
} else {
for (int i = index; i < ints.length; i++) {
if (i < ints.length - 1) {
if (ints[i] + 1 == ints[i + 1]) {
end = i;
} else {
if (i > index)
end = end + 1;
break;
}
} else {
if (end == ints.length - 2) {
end = ints.length - 1;
break;
}
}
}
if (index == end)// 相等说明不连续
return ints[index] + "," + convert(ints, end + 1);
else// 连续
return ints[index] + "-" + ints[end] + "," + convert(ints, end + 1);
}
}
public void getAllIp() throws Exception {
List<String> allIp = localLogJDBCByDruid.getAllIp("1");
Map<String, Map<Integer, List<Integer>>> map = new HashMap<>();
for (String ip : allIp) {
String topTwo = ip.substring(0, ip.indexOf(".", ip.indexOf(".") + 1));
String lastTwo = ip.substring(ip.indexOf(".", ip.indexOf(".") + 1) + 1);
String[] endIpAddr = lastTwo.split("\\.");
int third = Integer.parseInt(endIpAddr[0]);
int fourth = Integer.parseInt(endIpAddr[1]);
if (map.containsKey(topTwo)) {
Map<Integer, List<Integer>> thridMap = map.get(topTwo);
if (thridMap.containsKey(third)) {
thridMap.get(third).add(fourth);
} else {
List<Integer> list = new ArrayList<>();
list.add(fourth);
thridMap.put(third, list);
}
} else {
Map<Integer, List<Integer>> thridMap = new HashMap<>();
List<Integer> list = new ArrayList<>();
list.add(fourth);
thridMap.put(third, list);
map.put(topTwo, thridMap);
}
}
Map<String, String> rangMap = new HashMap<>();
for (Entry<String, Map<Integer, List<Integer>>> entry : map.entrySet()) {
String topTwo = entry.getKey();
Map<Integer, List<Integer>> endIpMap = entry.getValue();
Set<Integer> keySet = endIpMap.keySet();
ArrayList<Integer> arrayList = new ArrayList<Integer>(keySet);
Collections.sort(arrayList);
Integer minThrid = arrayList.get(0);
Integer maxThrid = arrayList.get(arrayList.size() - 1);
List<Integer> minThridList = endIpMap.get(minThrid);
List<Integer> maxThridList = endIpMap.get(maxThrid);
Collections.sort(minThridList);
Collections.sort(maxThridList);
if (minThridList.size() > 0 && maxThridList.size() > 0) {
rangMap.put(topTwo + "." + minThrid + "." + minThridList.get(0),
topTwo + "." + maxThrid + "." + maxThridList.get(maxThridList.size() - 1));
} else {
System.out.println("1");
}
}
for (String minIp : rangMap.keySet()) {
System.out.println(minIp + "--" + rangMap.get(minIp));
}
}
public void getAllIp2() throws Exception {
Map<String, List<Integer>> map = new HashMap<>();
List<String> allIp = localLogJDBCByDruid.getAllIp("1");
for (String ip : allIp) {
String[] split = ip.split(",");
for (int i = 0; i < split.length; i++) {
String prev = ip.substring(0, ip.lastIndexOf("."));
int forth = Integer.parseInt(ip.substring(ip.lastIndexOf(".") + 1));
if (map.containsKey(prev)) {
map.get(prev).add(forth);
} else {
List<Integer> list = new ArrayList<>();
list.add(forth);
map.put(prev, list);
}
}
}
if (map.size() > 0) {
Map<String, String> ipMap = new HashMap<>();
for (Entry<String, List<Integer>> entry : map.entrySet()) {
String prev = entry.getKey();
List<Integer> endList = entry.getValue();
a(prev, endList, ipMap);
}
Map<String, List<Integer>> map2 = new HashMap<>();
for (String minIp : ipMap.keySet()) {
String endIp = ipMap.get(minIp);
String prev = minIp.substring(0, minIp.lastIndexOf("."));
int minEnd = Integer.parseInt(minIp.substring(minIp.lastIndexOf(".") + 1));
int maxEnd = Integer.parseInt(endIp.substring(endIp.lastIndexOf(".") + 1));
for (int i = minEnd; i <= maxEnd; i++) {
if (map.containsKey(prev)) {
map.get(prev).add(i);
} else {
List<Integer> list = new ArrayList<>();
list.add(i);
map.put(prev, list);
}
}
}
Map<String, String> ipMap2 = new HashMap<>();
for (Entry<String, List<Integer>> entry : map2.entrySet()) {
String prev = entry.getKey();
List<Integer> endList = entry.getValue();
a(prev, endList, ipMap2);
}
for (String minIp : ipMap2.keySet()) {
System.out.println(minIp + "--" + ipMap2.get(minIp));
}
}
}
private void a(String prev, List<Integer> endList, Map<String, String> ipMap) {
Collections.sort(endList);
Integer start = endList.get(0);
if (endList.size() == 256) {// 0-255
ipMap.put(prev + "." + start, prev + ".255");
} else {
for (int i = 0; i < endList.size(); i++) {
Integer curr = endList.get(i);
if (i < endList.size() - 1) {
Integer next = endList.get(i + 1);
if (curr + 1 == next) {
ipMap.put(prev + "." + start, prev + "." + next);
} else {
start = curr;
}
}
}
}
}
public void getAllIp1() throws Exception {
Map<Integer, Map<Integer, Map<Integer, List<Integer>>>> map = new HashMap<>();
List<String> allIp = localLogJDBCByDruid.getAllIp("1");
for (String ip : allIp) {
String[] split = ip.split(",");
for (int i = 0; i < split.length; i++) {
int first = Integer.parseInt(split[0]);
int second = Integer.parseInt(split[1]);
int third = Integer.parseInt(split[2]);
int fourth = Integer.parseInt(split[3]);
if (map.containsKey(first)) {
Map<Integer, Map<Integer, List<Integer>>> secondMap = map.get(first);
if (secondMap.containsKey(second)) {
Map<Integer, List<Integer>> thridMap = secondMap.get(second);
if (thridMap.containsKey(third)) {
thridMap.get(third).add(fourth);
} else {
List<Integer> list = new ArrayList<>();
list.add(fourth);
thridMap.put(third, list);
}
} else {
Map<Integer, List<Integer>> thridMap = new HashMap<>();
List<Integer> list = new ArrayList<>();
list.add(fourth);
thridMap.put(third, list);
secondMap.put(second, thridMap);
}
} else {
Map<Integer, Map<Integer, List<Integer>>> secondMap = new HashMap<>();
Map<Integer, List<Integer>> thridMap = new HashMap<>();
List<Integer> list = new ArrayList<>();
list.add(fourth);
thridMap.put(third, list);
secondMap.put(second, thridMap);
map.put(first, secondMap);
}
}
}
if (map.size() > 0) {
for (Entry<Integer, Map<Integer, Map<Integer, List<Integer>>>> entry : map.entrySet()) {
Integer key = entry.getKey();
}
}
}
}

View File

@@ -208,7 +208,7 @@ service=1:128;2:128;5:96;6:1;16:16;17:16;18:16;19:16;20:16;21:16;22:16;23:16;24:
#0x400 APP HTTP特征发现
1024=10:APP_COMPILE;11:APP_GROUP;15:APP_HTTP;12:APP_IP_RANGE
#0x401 APP载荷特征表
1025=10:APP_COMPILE;11:APP_GROUP;12:APP_IP_RANGE;15:APP_PAYLOAD
1025=10:APP_COMPILE;11:APP_GROUP;12:APP_IP_RANGE;13:APP_TCP_SESSION_BYTE;15:APP_PAYLOAD
#0x402 APP DOMAIN特征发现
1026=10:APP_COMPILE;11:APP_GROUP;14:APP_DOMAIN
#0x403 APP DNS特征匹配

View File

@@ -260,6 +260,7 @@
<t:field fieldType="Number" srcName="service" dstName="service" isRequired="true"/>
<t:field fieldType="Number" srcName="policyGroup" dstName="policy_group" isRequired="true"/>
<t:field fieldType="String" srcName="effectiveRange" dstName="effective_range" isRequired="true"/>
<t:field fieldType="Date" srcName="opTime" dstName="op_time" isRequired="true"/>
</t:commonSource>
</t:commonSources>

View File

@@ -289,7 +289,7 @@
<p:maatType service="642">
<p:expressions>
<p:keyExpression>EFFECTIVE_RULE;:;{un_maat_table_name};,;[cfg_id]</p:keyExpression>
<p:valueExpression>[cfg_id];\t;0;\t;[addr_type];\t;[protocol];\t;[ip];\t;[port];\t;[direction];\t;[user_region];\t;[location];\t;[is_valid];\t;[service];\t;[policy_group];\t;[effective_range];\t;[op_time];&amp;nbsp;0;\n</p:valueExpression>
<p:valueExpression>[cfg_id];\t;[addr_type];\t;[protocol];\t;[ip];\t;[port];\t;[direction];\t;[user_region];\t;[location];\t;[is_valid];\t;[service];\t;[policy_group];\t;[effective_range];\t;[op_time];&amp;nbsp;0</p:valueExpression>
</p:expressions>
<p:sequences>
<p:operation>1</p:operation>

View File

@@ -204,6 +204,9 @@ mmSampleDstPath=/home/mesasoft/{tableType}/full/{fileName}
##定时将redis主从库的实时统计数据同步到redis集群中
syncRedisToClusterCron=1 * * * * ?
#定时同步ip范围段到mysql
getIpRangeTaskCron=0 0 19 * * ?
#文件服务器是否使用Minio
isUseMinio=true
@@ -217,3 +220,6 @@ isOpenLogCountAndLast=true
redisLockTime=3000
#获取redis分布式锁失败后的尝试获取锁的次数,每次失败暂停一秒钟后再次尝试
redisRetryNum=5
#ip定位库的地址
#ipLocationLibraryPath=/usr/local/ipLocalLibraay/Kazakhstan.mmdb
ipLocationLibraryPath=C:\\ipLocation\\Kazakhstan.mmdb