gfwleak/enderbyendera-realtime-protection
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into haskafka

Browse Source
This commit is contained in:
PushM
2024-04-25 22:30:08 +08:00
parent d4aca11aee 7086af3832
commit 3936c63457
7 changed files with 101 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/main/java/com/realtime/protection/configuration/entity/task/DynamicTaskInfo.java
View File

@@ -1,5 +1,6 @@
package com.realtime.protection.configuration.entity.task; package com.realtime.protection.configuration.entity.task;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data; import lombok.Data;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@@ -10,23 +11,44 @@ public class DynamicTaskInfo {
@Data @Data
private static class SimpleProtectObject { private static class SimpleProtectObject {
@JsonProperty("ip")
private String IP; private String IP;
@JsonProperty("port")
private Integer port; private Integer port;
@JsonProperty("url")
private String URL; private String URL;
@JsonProperty("protocol")
private String protocol; private String protocol;
} }
// 从任务中获取 // 从任务中获取
@JsonProperty("task_id")
private Long taskId; private Long taskId;
@JsonProperty("start_time")
private LocalDateTime startTime; private LocalDateTime startTime;
@JsonProperty("end_time")
private LocalDateTime endTime; private LocalDateTime endTime;
// 从规则中获取 // 从规则中获取
@JsonProperty("rule_id")
private Integer ruleId; private Integer ruleId;
@JsonProperty("source_system")
private String sourceSystem; private String sourceSystem;
@JsonProperty("event_type")
private String eventType; private String eventType;
@JsonProperty("log_rule_id")
private Long logRuleId; private Long logRuleId;
// 从防护对象列表中获取 // 从防护对象列表中获取
@JsonProperty("protect_objects")
private List<SimpleProtectObject> protectObjects; private List<SimpleProtectObject> protectObjects;
} }

8
src/main/java/com/realtime/protection/server/task/TaskController.java
View File

@@ -80,7 +80,7 @@ public class TaskController implements TaskControllerApi {
List<Task> tasks = taskService.queryTasks(taskStatus, taskType, taskName, taskCreator, auditStatus, page, pageSize); List<Task> tasks = taskService.queryTasks(taskStatus, taskType, taskName, taskCreator, auditStatus, page, pageSize);
return ResponseResult.ok() return ResponseResult.ok()
.setData("task_list", tasks) .setData("task_list", tasks)
.setData("total_num", taskService.queryTaskTotalNum(taskStatus, taskType, taskName, taskCreator)); .setData("total_num", taskService.queryTaskTotalNum(taskStatus, taskType, taskName, taskCreator, auditStatus));
} }
@Override @Override
@@ -192,9 +192,9 @@ public class TaskController implements TaskControllerApi {
@GetMapping("/statistics") @GetMapping("/statistics")
public ResponseResult statistics() { public ResponseResult statistics() {
return ResponseResult.ok() return ResponseResult.ok()
.setData("total_num", taskService.queryTaskTotalNum(null, null, null, null)) .setData("total_num", taskService.queryTaskTotalNum(null, null, null, null, null))
.setData("running_num", taskService.queryTaskTotalNum(StateEnum.RUNNING.getStateNum(), null, null, null)) .setData("running_num", taskService.queryTaskTotalNum(StateEnum.RUNNING.getStateNum(), null, null, null, null))
.setData("finished_num", taskService.queryTaskTotalNum(StateEnum.FINISHED.getStateNum(), null, null, null)) .setData("finished_num", taskService.queryTaskTotalNum(StateEnum.FINISHED.getStateNum(), null, null, null, null))
.setData("unaudit_num", taskService.queryAuditTaskTotalNum( .setData("unaudit_num", taskService.queryAuditTaskTotalNum(
AuditStatusEnum.PENDING.getNum() AuditStatusEnum.PENDING.getNum()
)); ));

9
src/main/java/com/realtime/protection/server/task/TaskMapper.java
View File

@@ -49,16 +49,19 @@ public interface TaskMapper {
Integer queryTaskStatus(@Param("task_id") Long taskId); Integer queryTaskStatus(@Param("task_id") Long taskId);
List<Integer> queryDynamicRuleIdsFromTaskId(@Param("task_id") Long taskId); List<Integer> queryDynamicRuleIdsFromTaskId(@Param("task_id") Long taskId,
@Param("audit_status_list") List<Integer> auditStatusList);
List<Integer> queryStaticRuleIdsFromTaskId(@Param("task_id") Long taskId); List<Integer> queryStaticRuleIdsFromTaskId(@Param("task_id") Long taskId,
@Param("audit_status_list") List<Integer> auditStatusList);
@Select("SELECT task_id FROM t_task WHERE task_end_time < NOW() AND task_status != #{task_status}") @Select("SELECT task_id FROM t_task WHERE task_end_time < NOW() AND task_status != #{task_status}")
List<Long> queryTasksByStatus(@Param("task_status") Integer taskStatus); List<Long> queryTasksByStatus(@Param("task_status") Integer taskStatus);
Integer queryTaskTotalNum(@Param("task_status") Integer taskStatus, @Param("task_type") Integer task_type, Integer queryTaskTotalNum(@Param("task_status") Integer taskStatus, @Param("task_type") Integer task_type,
@Param("task_name") String taskName, @Param("task_creator") String taskCreator); @Param("task_name") String taskName, @Param("task_creator") String taskCreator,
@Param("audit_status") Integer auditStatus);
void updateAuditStatusByIdBatch(@Param("idWithAuditStatusBatch") Map<Integer, Integer> idWithAuditStatusBatch); void updateAuditStatusByIdBatch(@Param("idWithAuditStatusBatch") Map<Integer, Integer> idWithAuditStatusBatch);

34
src/main/java/com/realtime/protection/server/task/TaskService.java
View File

@@ -92,7 +92,13 @@ public class TaskService {
return; return;
} }
List<StaticRuleObject> staticRuleObjects = staticRuleMapper.queryStaticRuleByIds(taskMapper.queryDynamicRuleIdsFromTaskId(taskId)); List<Integer> staticRuleIds = taskMapper.queryStaticRuleIdsFromTaskId(taskId,
List.of(AuditStatusEnum.AUDITED.getNum(), AuditStatusEnum.USING.getNum()));
if (staticRuleIds == null || staticRuleIds.isEmpty()) {
return;
}
List<StaticRuleObject> staticRuleObjects = staticRuleMapper.queryStaticRuleByIds(staticRuleIds);
if (staticRuleObjects == null || staticRuleObjects.isEmpty()) { if (staticRuleObjects == null || staticRuleObjects.isEmpty()) {
throw new IllegalArgumentException("静态规则列表中的ID不存在，请检查静态规则是否真实存在"); throw new IllegalArgumentException("静态规则列表中的ID不存在，请检查静态规则是否真实存在");
} }
@@ -147,9 +153,15 @@ public class TaskService {
return; return;
} }
List<DynamicRuleObject> dynamicRuleObjects = dynamicRuleMapper.queryDynamicRuleByIds(taskMapper.queryDynamicRuleIdsFromTaskId(taskId)); List<Integer> dynamicRuleIds = taskMapper.queryDynamicRuleIdsFromTaskId(taskId,
List.of(AuditStatusEnum.AUDITED.getNum(), AuditStatusEnum.USING.getNum()));
if (dynamicRuleIds == null || dynamicRuleIds.isEmpty()) {
return;
}
List<DynamicRuleObject> dynamicRuleObjects = dynamicRuleMapper.queryDynamicRuleByIds(dynamicRuleIds);
if (dynamicRuleObjects == null || dynamicRuleObjects.isEmpty()) { if (dynamicRuleObjects == null || dynamicRuleObjects.isEmpty()) {
throw new IllegalArgumentException("静态规则列表中的ID不存在，请检查静态规则是否真实存在"); throw new IllegalArgumentException("动态规则列表中的ID不存在，请检查动态规则是否真实存在");
} }
// 检查所有的动态规则列表的审批状态是否正确，如不正确则报错 // 检查所有的动态规则列表的审批状态是否正确，如不正确则报错
@@ -197,8 +209,10 @@ public class TaskService {
if (task == null) { if (task == null) {
continue; continue;
} }
task.setStaticRuleIds(taskMapper.queryStaticRuleIdsFromTaskId(task.getTaskId())); task.setStaticRuleIds(taskMapper.queryStaticRuleIdsFromTaskId(task.getTaskId(),
task.setDynamicRuleIds(taskMapper.queryDynamicRuleIdsFromTaskId(task.getTaskId())); List.of(AuditStatusEnum.AUDITED.getNum(), AuditStatusEnum.USING.getNum())));
task.setDynamicRuleIds(taskMapper.queryDynamicRuleIdsFromTaskId(task.getTaskId(),
List.of(AuditStatusEnum.AUDITED.getNum(), AuditStatusEnum.USING.getNum())));
} }
return tasks; return tasks;
@@ -211,8 +225,10 @@ public class TaskService {
return null; return null;
} }
task.setStaticRuleIds(taskMapper.queryStaticRuleIdsFromTaskId(task.getTaskId())); task.setStaticRuleIds(taskMapper.queryStaticRuleIdsFromTaskId(task.getTaskId(),
task.setDynamicRuleIds(taskMapper.queryDynamicRuleIdsFromTaskId(task.getTaskId())); List.of(AuditStatusEnum.AUDITED.getNum(), AuditStatusEnum.USING.getNum())));
task.setDynamicRuleIds(taskMapper.queryDynamicRuleIdsFromTaskId(task.getTaskId(),
List.of(AuditStatusEnum.AUDITED.getNum(), AuditStatusEnum.USING.getNum())));
return task; return task;
} }
@@ -303,8 +319,8 @@ public class TaskService {
return taskMapper.queryTasksByStatus(StateEnum.FINISHED.getStateNum()); return taskMapper.queryTasksByStatus(StateEnum.FINISHED.getStateNum());
} }
public Integer queryTaskTotalNum(Integer taskStatus, Integer taskType, String taskName, String taskCreator) { public Integer queryTaskTotalNum(Integer taskStatus, Integer taskType, String taskName, String taskCreator, Integer auditStatus) {
return taskMapper.queryTaskTotalNum(taskStatus, taskType, taskName, taskCreator); return taskMapper.queryTaskTotalNum(taskStatus, taskType, taskName, taskCreator, auditStatus);
} }
public Object updateAuditStatusBatch(Map<Integer, Integer> idsWithAuditStatusMap) { public Object updateAuditStatusBatch(Map<Integer, Integer> idsWithAuditStatusMap) {

18
src/main/java/com/realtime/protection/server/task/status/StateHandler.java
View File

@@ -13,13 +13,14 @@ import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClientResponseException; import org.springframework.web.reactive.function.client.WebClientResponseException;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import java.time.Duration;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReference;
public class StateHandler { public class StateHandler {
private final WebClient client = WebClient.builder() private final WebClient client = WebClient.builder()
.baseUrl("") // todo: unfinished .baseUrl("http://192.168.107.89:9081")
.build(); .build();
protected Boolean handleStart(TaskService taskService, CommandService commandService, Long taskId) { protected Boolean handleStart(TaskService taskService, CommandService commandService, Long taskId) {
@@ -79,8 +80,8 @@ public class StateHandler {
return true; return true;
} }
// todo: 如果是实时任务或者研判后处置任务，那么就需要在任务启动之后，立刻向动态规则中指定的系统发送日志筛选请求。 // 如果是实时任务或者研判后处置任务，那么就需要在任务启动之后，立刻向动态规则中指定的系统发送日志筛选请求。
// 筛选完成后，系统返回日志，需要由接收端点提取字段，并且合成一条静态规则，再按照任务开始时间、结束时间和任务类型进行指令创建 // 筛选完成后，系统返回日志，需要由接收端点提取字段，并且合成一条静态规则，再按照任务开始时间、结束时间和任务类型进行指令创建
private Boolean handleJudgedTaskStart(TaskService taskService, Task task) { private Boolean handleJudgedTaskStart(TaskService taskService, Task task) {
return sendFilters(taskService, task); return sendFilters(taskService, task);
} }
@@ -98,6 +99,7 @@ public class StateHandler {
// 将所有关联的静态规则全部设置为已使用状态 // 将所有关联的静态规则全部设置为已使用状态
taskService.updateStaticRuleAuditStatusInTask(task.getTaskId(), AuditStatusEnum.USING); taskService.updateStaticRuleAuditStatusInTask(task.getTaskId(), AuditStatusEnum.USING);
// taskService.changeTaskAuditStatus(task.getTaskId(), AuditStatusEnum.USING.getNum());
commandService.createCommands(staticTaskCommandInfos); commandService.createCommands(staticTaskCommandInfos);
return true; return true;
@@ -110,12 +112,13 @@ public class StateHandler {
throw new IllegalArgumentException("动态规则列表为空，请至少选择一个动态规则以启动动态/研判后类型任务"); throw new IllegalArgumentException("动态规则列表为空，请至少选择一个动态规则以启动动态/研判后类型任务");
} }
taskService.updateDynamicRuleAuditStatusInTask(task.getTaskId(), AuditStatusEnum.AUDITED); // 将所有关联的动态规则审批状态修改为“已使用”
taskService.updateDynamicRuleAuditStatusInTask(task.getTaskId(), AuditStatusEnum.USING);
AtomicReference<Boolean> success = new AtomicReference<>(false); AtomicReference<Boolean> success = new AtomicReference<>(false);
Mono<SimpleResponse> mono = client.post() Mono<SimpleResponse> mono = client.post()
.uri("http://192.168.107.89:9081/api/v1/kafkasend") // todo: untested .uri("/api/v1/kafkasend")
.bodyValue(dynamicTaskInfos) .bodyValue(dynamicTaskInfos)
.exchangeToMono(res -> { .exchangeToMono(res -> {
if (res.statusCode().equals(HttpStatus.OK)) { if (res.statusCode().equals(HttpStatus.OK)) {
@@ -126,9 +129,10 @@ public class StateHandler {
}) })
.doOnError(WebClientResponseException.class, res -> success.set(false)); .doOnError(WebClientResponseException.class, res -> success.set(false));
SimpleResponse response = mono.block();
if (response == null) { SimpleResponse response = mono.block(Duration.ofSeconds(5));
if (response == null || response.getSuccess() == null) {
return false; return false;
} }

1
src/main/resources/mappers/DynamicRuleMapper.xml
View File

@@ -150,6 +150,7 @@
<result column="dynamic_rule_priority" property="dynamicRulePriority"/> <result column="dynamic_rule_priority" property="dynamicRulePriority"/>
<result column="dynamic_rule_range" property="dynamicRuleRange"/> <result column="dynamic_rule_range" property="dynamicRuleRange"/>
<result column="dynamic_rule_frequency" property="dynamicRuleFrequency"/> <result column="dynamic_rule_frequency" property="dynamicRuleFrequency"/>
<result column="audit_status" property="auditStatus"/>
<!-- <result column="protect_object_id" property="protectObjectIds"/>--> <!-- <result column="protect_object_id" property="protectObjectIds"/>-->
<result column="strategy_template_name" property="dynamicRuleEventType"/> <result column="strategy_template_name" property="dynamicRuleEventType"/>
<result column="strategy_template_source_system" property="dynamicRuleSourceSystem"/> <result column="strategy_template_source_system" property="dynamicRuleSourceSystem"/>

40
src/main/resources/mappers/TaskMapper.xml
View File

@@ -81,13 +81,33 @@
<select id="queryStaticRuleIdsFromTaskId" resultType="java.lang.Integer"> <select id="queryStaticRuleIdsFromTaskId" resultType="java.lang.Integer">
SELECT static_rule_id SELECT static_rule_id
FROM t_static_rule FROM t_static_rule
WHERE static_rule_used_task_id = #{task_id} AND static_rule_audit_status = 2 <where>
<if test="task_id != null">
AND static_rule_used_task_id = #{task_id}
</if>
<if test="audit_status_list != null">
AND static_rule_audit_status IN
<foreach collection="audit_status_list" item="audit_status" open="(" close=")" separator=",">
#{audit_status}
</foreach>
</if>
</where>
</select> </select>
<select id="queryDynamicRuleIdsFromTaskId" resultType="java.lang.Integer"> <select id="queryDynamicRuleIdsFromTaskId" resultType="java.lang.Integer">
SELECT dynamic_rule_id SELECT dynamic_rule_id
FROM t_dynamic_rule FROM t_dynamic_rule
WHERE dynamic_rule_used_task_id = #{task_id} AND audit_status = 2 <where>
<if test="task_id != null">
AND dynamic_rule_used_task_id = #{task_id}
</if>
<if test="audit_status_list != null">
AND audit_status IN
<foreach collection="audit_status_list" item="audit_status" open="(" close=")" separator=",">
#{audit_status}
</foreach>
</if>
</where>
</select> </select>
<select id="queryTask" resultMap="taskMap"> <select id="queryTask" resultMap="taskMap">
@@ -257,7 +277,8 @@
<result column="log_rule_id" property="logRuleId"/> <result column="log_rule_id" property="logRuleId"/>
<result column="source_system" property="sourceSystem"/> <result column="source_system" property="sourceSystem"/>
<result column="event_type" property="eventType"/> <result column="event_type" property="eventType"/>
<collection property="protectObjects" ofType="com.realtime.protection.configuration.entity.task.DynamicTaskInfo$SimpleProtectObject"> <collection property="protectObjects"
ofType="com.realtime.protection.configuration.entity.task.DynamicTaskInfo$SimpleProtectObject">
<result column="protect_object_ip" property="IP"/> <result column="protect_object_ip" property="IP"/>
<result column="protect_object_port" property="port"/> <result column="protect_object_port" property="port"/>
<result column="protect_object_url" property="URL"/> <result column="protect_object_url" property="URL"/>
@@ -274,16 +295,16 @@
strategy_template_source_system as source_system, strategy_template_source_system as source_system,
strategy_template_name as event_type, strategy_template_name as event_type,
tdr.log_rule_id, tdr.log_rule_id,
INET_NTOA(protect_object_ip) as protect_object_ip, INET_NTOA(protect_object_ip) as protect_object_ip,
protect_object_port, protect_object_port,
protect_object_url, protect_object_url,
protect_object_protocol protect_object_protocol
FROM t_task AS tt FROM t_task AS tt
INNER JOIN realtime_protection.t_dynamic_rule tdr on tt.task_id = tdr.dynamic_rule_used_task_id LEFT JOIN realtime_protection.t_dynamic_rule tdr on tt.task_id = tdr.dynamic_rule_used_task_id
INNER JOIN realtime_protection.t_protect_object_dynamic_rule_conn tpodrc LEFT JOIN realtime_protection.t_protect_object_dynamic_rule_conn tpodrc
on tdr.dynamic_rule_id = tpodrc.dynamic_rule_id on tdr.dynamic_rule_id = tpodrc.dynamic_rule_id
INNER JOIN realtime_protection.t_protect_object tpo on tpo.protect_object_id = tpodrc.protect_object_id LEFT JOIN realtime_protection.t_protect_object tpo on tpo.protect_object_id = tpodrc.protect_object_id
INNER JOIN realtime_protection.t_strategy_template tst on tdr.template_id = tst.strategy_template_id LEFT JOIN realtime_protection.t_strategy_template tst on tdr.template_id = tst.strategy_template_id
WHERE task_id = #{task_id} WHERE task_id = #{task_id}
</select> </select>
<select id="queryTaskTotalNum" resultType="java.lang.Integer"> <select id="queryTaskTotalNum" resultType="java.lang.Integer">
@@ -301,6 +322,9 @@
<if test="task_creator != null"> <if test="task_creator != null">
AND task_create_username LIKE CONCAT('%', #{task_creator}, '%') AND task_create_username LIKE CONCAT('%', #{task_creator}, '%')
</if> </if>
<if test="audit_status != null">
AND task_audit_status = #{audit_status}
</if>
</where> </where>
</select> </select>