1、IP限流下发阀门只添加action和service;2、APP阻断、APP监测添加两个域表;IPSec监测IP类域表改为NTC_UNIVERSAL_IP;

2、添加service和action关系不存在的验证;
3、添加行为识别原始日志查询接口;
This commit is contained in:
zhangdongxu
2018-07-13 12:06:34 +08:00
parent d8e736d048
commit db4bdd7722
10 changed files with 175 additions and 19 deletions

View File

@@ -0,0 +1,58 @@
package com.nis.domain.restful;
import java.util.Date;
import com.nis.domain.LogEntity;
import com.wordnik.swagger.annotations.ApiModelProperty;
/**
* @ClassName:DkBehaviorLog
* @Description:TODO(这里用一句话描述这个类的作用)
* @author (zdx)
* @date 2018年7月12日 下午5:09:21
* @version V1.0
*/
public class DkBehaviorLog extends LogEntity<DkBehaviorLog> {
private static final long serialVersionUID = -589390375816690510L;
@ApiModelProperty(value = "协议标签", required = true)
protected Integer appLabel;
@ApiModelProperty(value = "c2s包数", required = true)
protected String c2sPktNum;
@ApiModelProperty(value = "s2c包数", required = true)
protected String s2cPktNum;
@ApiModelProperty(value = "c2s字节数", required = true)
protected String c2sByteNum;
@ApiModelProperty(value = "s2c字节数", required = true)
protected String s2cByteNum;
public Integer getAppLabel() {
return appLabel;
}
public void setAppLabel(Integer appLabel) {
this.appLabel = appLabel;
}
public String getC2sPktNum() {
return c2sPktNum;
}
public void setC2sPktNum(String c2sPktNum) {
this.c2sPktNum = c2sPktNum;
}
public String getS2cPktNum() {
return s2cPktNum;
}
public void setS2cPktNum(String s2cPktNum) {
this.s2cPktNum = s2cPktNum;
}
public String getC2sByteNum() {
return c2sByteNum;
}
public void setC2sByteNum(String c2sByteNum) {
this.c2sByteNum = c2sByteNum;
}
public String getS2cByteNum() {
return s2cByteNum;
}
public void setS2cByteNum(String s2cByteNum) {
this.s2cByteNum = s2cByteNum;
}
}

View File

@@ -346,6 +346,11 @@ public enum RestBusinessCode {
*/
CompileIdIsNeq(4002009,"Maat配置分组与编译配置的compileId不一致"),
/**
* 配置的service与action关系不存在
*/
Service2ActionIsNull(4002010, "配置的service与action关系不存在"),
/**
* 配置分组中同一groupId出现多次
*/

View File

@@ -879,6 +879,11 @@ public class CompileVal {
public static String serviceConfigCompileVal(ConfigCompile configCompile) {
Long compileId = configCompile.getCompileId();
Integer action = ServiceAndRDBIndexReal.getActionByService(configCompile.getService());
if (StringUtil.isEmpty(action)) {
businessCode = RestBusinessCode.Service2ActionIsNull.getValue();
return "编译配置id为" + compileId + "的配置中service与action的关系不存在";
}
if (action.compareTo(configCompile.getAction()) != 0) {
businessCode = RestBusinessCode.ServiceUnmatchAction.getValue();
return "编译配置id为" + compileId + "的配置中service的值为" + configCompile.getService() + "时action只能为" + action;

View File

@@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.RestController;
import com.nis.domain.LogEntity;
import com.nis.domain.Page;
import com.nis.domain.restful.DkBehaviorLog;
import com.nis.domain.restful.NtcAppLog;
import com.nis.domain.restful.NtcDdosLog;
import com.nis.domain.restful.NtcDnsLog;
@@ -847,7 +848,64 @@ public class LogController extends BaseRestController {
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "DDos日志检索成功",
ntcDdosLogPage, 0);
}
@RequestMapping(value = "/dkBehaviorLogs", method = RequestMethod.GET)
@ApiOperation(value = "行为识别日志查询", httpMethod = "GET", notes = "对日志功能“行为识别”提供数据基础查询服务")
public Map<String, ?> dkBehaviorLogs(Page page, DkBehaviorLog dkBehaviorLog, Model model, HttpServletRequest request,
HttpServletResponse response) {
long start = System.currentTimeMillis();
SaveRequestLogThread auditLogThread = super.saveRequestLog(servicesRequestLogService, Constants.OPACTION_GET,
request, null);
Page<DkBehaviorLog> logPage = null;
try {
resetTime(dkBehaviorLog);
testService.queryConditionCheck(auditLogThread, start, dkBehaviorLog, DkBehaviorLog.class, page);
logPage = new Page<DkBehaviorLog>();
logPage.setPageNo(page.getPageNo());
logPage.setPageSize(page.getPageSize());
String orderBy = "";
if (null != page.getOrderBy() && !page.getOrderBy().equals("")) {
orderBy = Page.getOrderBySql(DkBehaviorLog.class.getSimpleName(), page.getOrderBy());
} else {
orderBy = "found_Time";
}
ResultSet rs = HiveSqlService.getResultSet(page, dkBehaviorLog,
Configurations.getStringProperty(DkBehaviorLog.class.getSimpleName() + "HiveTable", "DK_BEHAVIOR_LOG"),
getCol2Col(), orderBy, null);
Map<String, List> tableMapping = HiveJDBC.tableMapping(page, null, rs, DkBehaviorLog.class, "foundTime",
"recvTime");
if (tableMapping == null) {
logPage.setList(new ArrayList());
} else {
List list = tableMapping.get("obj");
// if (tableMapping.get("obj").size() > page.getPageSize()) {
// list = tableMapping.get("obj").subList(0, page.getPageSize());
// } else {
// list = tableMapping.get("obj").subList(0, tableMapping.get("obj").size());
// }
if (list.size() > 0) {
String jsonString = JsonMapper.toJsonString(list);
List<DkBehaviorLog> List = (java.util.List<DkBehaviorLog>) JsonMapper.fromJsonList(jsonString,
NtcDdosLog.class);
logPage.setList(List);
logPage.setCount(List.size());
} else {
logPage.setList(new ArrayList());
}
}
} catch (Exception e) {
e.printStackTrace();
auditLogThread.setExceptionInfo(e.getMessage() + " " + e.getCause());
logger.error(e);
if (!(e instanceof RestServiceException)) {
e = new RestServiceException(auditLogThread, System.currentTimeMillis() - start, "行为识别日志检索失败");
}
throw ((RestServiceException) e);
}
return serviceLogResponse(auditLogThread, System.currentTimeMillis() - start, request, "行为识别日志检索成功",
logPage, 0);
}
@RequestMapping(value = "/ntcPzReport", method = RequestMethod.GET)
@ApiOperation(value = "配置日志总量统计", httpMethod = "GET", notes = "配置命中日志数量实时统计报表,对外提供多种数据表现形式,具体可应用于界面配置命中总量业务、配置报表业务等")
public Map<String, ?> ntcPzReport(Page page, NtcPzReport ntcPzReport, Model model, HttpServletRequest request,

View File

@@ -354,7 +354,30 @@
<result column="attack_total_byte" jdbcType="VARCHAR" property="attackTotalByte" />
<result column="is_blcok" jdbcType="INTEGER" property="isBlcok" />
</resultMap>
<resultMap id="DkBehaviorLogMap" type="com.nis.domain.restful.DkBehaviorLog">
<result column="cfg_id" jdbcType="BIGINT" property="cfgId" />
<result column="found_Time" jdbcType="TIMESTAMP" property="foundTime" />
<result column="recv_Time" jdbcType="TIMESTAMP" property="recvTime" />
<result column="trans_proto" jdbcType="VARCHAR" property="transProto" />
<result column="addr_type" jdbcType="INTEGER" property="addrType" />
<result column="d_ip" jdbcType="VARCHAR" property="dIp" />
<result column="s_ip" jdbcType="VARCHAR" property="sIp" />
<result column="d_port" jdbcType="VARCHAR" property="dPort" />
<result column="s_port" jdbcType="VARCHAR" property="sPort" />
<result column="service" jdbcType="INTEGER" property="service" />
<result column="entrance_id" jdbcType="BIGINT" property="entranceId" />
<result column="device_id" jdbcType="INTEGER" property="deviceId" />
<result column="direction" jdbcType="INTEGER" property="direction" />
<result column="stream_dir" jdbcType="INTEGER" property="streamDir" />
<result column="cap_ip" jdbcType="VARCHAR" property="capIp" />
<result column="addr_list" jdbcType="VARCHAR" property="addrList" />
<result column="user_region" jdbcType="VARCHAR" property="userRegion" />
<result column="c2s_pkt_num" jdbcType="VARCHAR" property="c2sPktNum" />
<result column="s2c_pkt_num" jdbcType="VARCHAR" property="s2cPktNum" />
<result column="c2s_byte_num" jdbcType="VARCHAR" property="c2sByteNum" />
<result column="s2c_byte_num" jdbcType="VARCHAR" property="s2cByteNum" />
</resultMap>
</mapper>

View File

@@ -1246,6 +1246,7 @@ public class ConfigSourcesService extends BaseService {
if (!"ok".equals(msg)) {
logger1.error(msg);
thread.setExceptionInfo(msg + sb.toString());
thread.setBusinessCode(CompileVal.getBusinessCode());
throw new RestServiceException(thread,
System.currentTimeMillis() - start, msg
+ sb.toString(),
@@ -1550,7 +1551,7 @@ public class ConfigSourcesService extends BaseService {
} catch (Exception e) {
// TODO: handle exception
msgList.add(e);
CompileVal.setBusinessCode(RestBusinessCode.unknow_error.getValue());
//CompileVal.setBusinessCode(RestBusinessCode.unknow_error.getValue());
return "error";
}
try {
@@ -2950,7 +2951,7 @@ public class ConfigSourcesService extends BaseService {
Integer action = null;
for (CommonSourceFieldCfg commonSourceFieldCfg : commonSourceFieldCfgList) {
if (commonSourceFieldCfg.getDstName().equals("ip_type")) {
if (commonSourceFieldCfg.getDstName().equals("addr_type")) {
String dstVal = srcMap.get(
commonSourceFieldCfg.getSrcName()).toString();
ipTypeName = commonSourceFieldCfg.getSrcName();
@@ -2996,6 +2997,9 @@ public class ConfigSourcesService extends BaseService {
}
}
}
if (ipType==null) {
ipType = 4;
}
for (CommonSourceFieldCfg commonSourceFieldCfg : commonSourceFieldCfgList) {
// 是否必填
if (commonSourceFieldCfg.getIsRequired()
@@ -3055,7 +3059,7 @@ public class ConfigSourcesService extends BaseService {
RestBusinessCode.MastDate.getValue());
}
break;
case "Ip":
case "IP":
if (!BasicProvingUtil.isIpOrIpMask(dstStr, ipType)) {
logger1.error(commonSourceFieldCfg.getSrcName()
+ "参数格式不正确或与" + ipTypeName + "不一致");

View File

@@ -7,10 +7,9 @@
#0x04 IP地址丢弃 4=0:INLINE_IP_CB
#0x19 IPSec丢弃 25=0:INLINE_IP_CB
#0x1C GRE 丢弃 28=0:INLINE_IP_CB
#0x1D IPSEC丢弃 29=0:INLINE_IP_CB
#0x05 DDOS流量丢弃 5=0:DDOS_PROTECT_TARGET_IP_CB
-->
<t:commonSource serviceIds="3,4,25,28,29,5">
<t:commonSource serviceIds="3,4,25,28,5">
<t:field fieldType="String" srcName="service" dstName="service" isRequired="true"/>
<t:field fieldType="Number" srcName="action" dstName="action" isRequired="true"/>
<t:field fieldType="Number" srcName="cfgId" dstName="cfg_id" isRequired="true" />
@@ -72,9 +71,11 @@
<t:field fieldType="Number" srcName="cfgId" dstName="cfg_id" isRequired="true" />
<t:field fieldType="Number" srcName="policyGroup" dstName="policy_group" isRequired="true"/>
<t:field fieldType="Number" srcName="addrType" dstName="addr_type" isRequired="true" regexp="[4|6]"/>
<t:field fieldType="Number" srcName="protocol" dstName="protocol" isRequired="true" defaultVal="0"/>
<t:field fieldType="IP" srcName="ip" dstName="ip" isRequired="true"/>
<t:field fieldType="Port" srcName="port" dstName="port" isRequired="true"/>
<t:field fieldType="String" srcName="userRegion" dstName="user_region" isRequired="true"/>
<t:field fieldType="Number" srcName="direction" dstName="direction" isRequired="true" regexp="[0|1|2]"/>
<t:field fieldType="String" srcName="userRegion" dstName="user_region"/>
<t:field fieldType="Number" srcName="location" dstName="location" isRequired="true" regexp="[0|1]"/>
<t:field fieldType="Number" srcName="isValid" dstName="is_valid" isRequired="true" range="0-1"/>
<t:field fieldType="Date" srcName="opTime" dstName="op_time" isRequired="true"/>

View File

@@ -63,10 +63,9 @@
#0x04 IP地址丢弃 4=0:INLINE_IP_CB
#0x19 IPSec丢弃 25=0:INLINE_IP_CB
#0x1C GRE 丢弃 28=0:INLINE_IP_CB
#0x1D IPSEC丢弃 29=0:INLINE_IP_CB
#0x05 DDOS流量丢弃 5=0:DDOS_PROTECT_TARGET_IP_CB
-->
<p:maatType service="3;4;25;28;29;5">
<p:maatType service="3;4;25;28;5">
<p:expressions>
<p:keyExpression>EFFECTIVE_RULE;:;{un_maat_table_name};,;[cfg_id]</p:keyExpression>
<p:valueExpression>[cfg_id];\t;[addr_type];\t;[src_ip];\t;[mask_src_ip];\t;[src_port];\t;[mask_src_port];\t;[dst_ip];\t;[mask_dst_ip];\t;[dst_port];\t;[mask_dst_port];\t;[protocol];\t;[direction];\t;[is_valid];\t;[action];\t;[service];\t;[op_time];&amp;nbsp;\n</p:valueExpression>
@@ -138,7 +137,7 @@
<p:maatType service="832">
<p:expressions>
<p:keyExpression>EFFECTIVE_RULE;:;{un_maat_table_name};,;[cfg_id]</p:keyExpression>
<p:valueExpression>[cfg_id];\t;[policy_group];\t;[addr_type];\t;[ip];\t;[port];\t;[user_region];\t;[location];\t;[is_valid];\t;[op_time];&amp;nbsp;[service];\t;[action];\n
<p:valueExpression>[cfg_id];\t;[addr_type];\t;[protocol];\t;[ip];\t;[port];\t;[direction];\t;[user_region];\t;[location];\t;[is_valid];\t;[action];\t;[service];\t;[policy_group];\t;[op_time];&amp;nbsp;\n
</p:valueExpression>
</p:expressions>
<p:sequences>

View File

@@ -1,6 +1,6 @@
#################################################################################
####maat格式配置各业务类型对应的编译,分组,域配置等的表名
####1:固定需要写service=业务类型1:动作;业务类型2:动作
####1:固定需要写service=业务类型1:动作;业务类型2:动作 用于验证maat配置service与action对应关系
####2:然后每个业务类型:业务类型1=type1:表名1;type2:表名2;type3:表名3
####3:例如:80=10:MM_COMPILE;11:MM_GROUP;12:AV_CONT_IP_PORT
####4:type中10代表是编译配置,11代表是分组配置,12代表是ip类域配置,13代表是数值类配置,14代表是字符串类域配置,15代表是增强字符串类域配置,16代表是文件摘要类域配置,17代表是文本相似性域配置,18代表是生效范围IP段配置
@@ -56,7 +56,7 @@ service=1:128;2:128;16:16;17:16;18:16;19:16;20:16;21:16;22:16;23:16;24:16;26:16;
#0X20 文件摘要配置
32=10:NTC_COMPILE;11:NTC_GROUP;16:NTC_FILE_DIGEST;18:NTC_IP_RANGE
#0x21 APP阻断
33=10:NTC_COMPILE;11:NTC_GROUP;14:APP_POLICY;18:NTC_IP_RANGE
33=10:NTC_COMPILE;11:NTC_GROUP;12:NTC_UNIVERSAL_IP;13:NTC_UNIVERSAL_PROTO_TYPE;14:APP_POLICY;18:NTC_IP_RANGE
#0x22 P2P阻断
34=10:NTC_COMPILE;11:NTC_GROUP;12:NTC_P2P_IP;14:NTC_P2P_HASH_BIN,NTC_P2P_KEYWORDS;18:NTC_IP_RANGE
@@ -79,7 +79,7 @@ service=1:128;2:128;16:16;17:16;18:16;19:16;20:16;21:16;22:16;23:16;24:16;26:16;
#0x88 OpenVPN监测
136=10:NTC_COMPILE;11:NTC_GROUP;12:NTC_UNIVERSAL_IP;13:NTC_UNIVERSAL_PROTO_TYPE;18:NTC_IP_RANGE
#0x89 IPSec监测
137=10:NTC_COMPILE;11:NTC_GROUP;12:NTC_IP;13:NTC_UNIVERSAL_PROTO_TYPE;18:NTC_IP_RANGE
137=10:NTC_COMPILE;11:NTC_GROUP;12:NTC_UNIVERSAL_IP;13:NTC_UNIVERSAL_PROTO_TYPE;18:NTC_IP_RANGE
#0x8A SSH监测
138=10:NTC_COMPILE;11:NTC_GROUP;12:NTC_UNIVERSAL_IP;13:NTC_UNIVERSAL_PROTO_TYPE;18:NTC_IP_RANGE
#0x8B SOCKS监测
@@ -95,7 +95,7 @@ service=1:128;2:128;16:16;17:16;18:16;19:16;20:16;21:16;22:16;23:16;24:16;26:16;
#0X90 文件摘要监测
144=10:NTC_COMPILE;11:NTC_GROUP;16:NTC_FILE_DIGEST;18:NTC_IP_RANGE
#0x91 APP监测
145=10:NTC_COMPILE;11:NTC_GROUP;14:APP_POLICY;18:NTC_IP_RANGE
145=10:NTC_COMPILE;11:NTC_GROUP;12:NTC_UNIVERSAL_IP;13:NTC_UNIVERSAL_PROTO_TYPE;14:APP_POLICY;18:NTC_IP_RANGE
#0xA0 通联关系监测 monit 无配置
#0xA1 邮件泛收 monit 无配置
#0xA2 RADIUS泛收 monit 无配置
@@ -167,7 +167,8 @@ service=1:128;2:128;16:16;17:16;18:16;19:16;20:16;21:16;22:16;23:16;24:16;26:16;
#0x422 域名限流
1058=10:LIMIT_COMPILE;11:LIMIT_GROUP;14:LIMIT_DOMAIN;18:NTC_IP_RANGE
unMaatService=3:32;4:96;5:32;25:32;28:32;29:32;64:16;65:16;261:16;262:16;263:16;264:16;265:16;266:16;389:1;390:1;391:1;392:1;393:1;394:1;832:96;
#格式业务类型1:动作;业务类型2:动作 用于验证回调类service与action对应关系
unMaatService=3:32;4:96;5:32;25:32;28:32;64:16;65:16;261:16;262:16;263:16;264:16;265:16;266:16;389:1;390:1;391:1;392:1;393:1;394:1;832:96;
#0x03 IP地址丢弃
3=0:INLINE_IP_CB
@@ -180,7 +181,7 @@ unMaatService=3:32;4:96;5:32;25:32;28:32;29:32;64:16;65:16;261:16;262:16;263:16;
#0x1C GRE 丢弃
28=0:INLINE_IP_CB
#0x1D IPSEC丢弃
29=0:INLINE_IP_CB
#29=0:INLINE_IP_CB
#0x40 DNS欺骗配置(回调)
64=0:NTC_DNS_FAKE_IP_CB
@@ -244,7 +245,7 @@ unMaatService=3:32;4:96;5:32;25:32;28:32;29:32;64:16;65:16;261:16;262:16;263:16;
#0x19 IPSec丢弃 25:2,7;
#0x340 IP复用地址池配置回调832:2,4,7
##########
serviceDBIndex=1:2,3,4,6,7;2:2,3,4,6,7;16:2;17:2;18:2;19:2;20:2;21:2;22:2;23:2;24:2;26:2;27:2;30:2;31:2;32:2;33:2,6,7;34:2;128:2;129:2;130:2;131:2;132:2;133:2;134:2;135:2;136:2;137:2;138:2;139:2;140:2;141:2;143:2;144:2;145:2,6,7;256:3;257:3;258:3;259:3;260:3;384:3;385:3;386:3;387:3;388:3;512:4,7;513:4,2,7;528:4;529:4;544:4;545:4;560:4;561:4;768:4;848:2;1024:2,6,7;1025:2;1026:2,6,7;1027:2;1028:2,6,7;1040:2,6,7;1056:2,6,7;1057:6,7;1058:6,2,7;3:2,7;4:2,7;5:2;25:2,7;28:2,7;29:2;64:2;65:2;261:3;262:3;263:3;264:3;265:3;266:3;389:3;390:3;391:3;392:3;393:3;394:3;832:2,4,7
serviceDBIndex=1:2,3,4,6,7;2:2,3,4,6,7;16:2;17:2;18:2;19:2;20:2;21:2;22:2;23:2;24:2;26:2;27:2;30:2;31:2;32:2;33:2,6,7;34:2;128:2;129:2;130:2;131:2;132:2;133:2;134:2;135:2;136:2;137:2;138:2;139:2;140:2;141:2;143:2;144:2;145:2,6,7;256:3;257:3;258:3;259:3;260:3;384:3;385:3;386:3;387:3;388:3;512:4,7;513:4,2,7;528:4;529:4;544:4;545:4;560:4;561:4;768:4;848:2;1024:2,6,7;1025:2;1026:2,6,7;1027:2;1028:2,6,7;1040:2,6,7;1056:2,6,7;1057:6,7;1058:6,2,7;3:2,7;4:2,7;5:2;25:2,7;28:2,7;64:2;65:2;261:3;262:3;263:3;264:3;265:3;266:3;389:3;390:3;391:3;392:3;393:3;394:3;832:2,4,7
##maat配置入阀门需要将编译中的部分参数写到域配置中目前最多包含ACTIONSERVICEUSER_REGION三个属性
@@ -262,5 +263,5 @@ serviceDBIndex=1:2,3,4,6,7;2:2,3,4,6,7;16:2;17:2;18:2;19:2;20:2;21:2;22:2;23:2;2
#0x420 APP限流 1056
#0x421 IP限流 1057
#0x422 域名限流 1058
maat2Valve=33:strRegion@ACTION&SERVICE&USER_REGION;145:strRegion@ACTION&SERVICE&USER_REGION;512:ipRegion@ACTION&SERVICE;513:strRegion@ACTION&SERVICE&USER_REGION;1024:strRegion|strStrRegion@USER_REGION;1026:strRegion|strStrRegion@USER_REGION;1028:ipRegion@ACTION&SERVICE&USER_REGION;1040:strRegion@ACTION&SERVICE&USER_REGION;1056:strRegion@ACTION&SERVICE&USER_REGION;1057:ipRegion@ACTION&SERVICE&USER_REGION;1058:strRegion@ACTION&SERVICE&USER_REGION
maat2Valve=33:strRegion@ACTION&SERVICE&USER_REGION;145:strRegion@ACTION&SERVICE&USER_REGION;512:ipRegion@ACTION&SERVICE;513:strRegion@ACTION&SERVICE&USER_REGION;1024:strRegion|strStrRegion@USER_REGION;1026:strRegion|strStrRegion@USER_REGION;1028:ipRegion@USER_REGION;1040:strRegion@ACTION&SERVICE&USER_REGION;1056:strRegion@ACTION&SERVICE&USER_REGION;1057:ipRegion@ACTION&SERVICE;1058:strRegion@ACTION&SERVICE&USER_REGION

View File

@@ -12,6 +12,8 @@ NtcSshLogHiveTable=NTC_SSH_LOG
NtcFtpLogHiveTable=NTC_FTP_LOG
NtcAppLogHiveTable=NTC_APP_LOG
NtcDdosLogHiveTable=NTC_DDOS_LOG
DkBehaviorLogHiveTable=DK_BEHAVIOR_LOG