gfwleak/appsketch-works-asw-controller
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: job 接口调整

Browse Source
This commit is contained in:
zhangshuai
2024-10-10 17:15:15 +08:00
parent a10e37bbd1
commit d9964d52ba
15 changed files with 363 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/net/geedge/asw/common/util/RCode.java
View File

@@ -87,6 +87,7 @@ public enum RCode {
ENVIRONMENT_NOT_EXIST(601002, "environment does not exist"),
ENVIRONMENT_USED(601003, "The environment is already in use"),
ENVIRONMENT_STATUS_ERROR(601004, "The environment status is unavailable"),
ENVIRONMENT_ID_CANNOT_EMPTY(601005, "environment id cannot be empty"),

40
src/main/java/net/geedge/asw/module/runner/controller/JobController.java
View File

@@ -10,55 +10,67 @@ import net.geedge.asw.module.runner.service.IJobService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@RestController
@RequestMapping("/api/v1/job")
@RequestMapping("/api/v1/workspace")
public class JobController {
@Autowired
private IJobService jobService;
@GetMapping("/{id}")
public R detail(@PathVariable("id") String id) {
@GetMapping("/{workspaceId}/job/{id}")
public R detail(@PathVariable("workspaceId") String workspaceId,
@PathVariable("id") String id) {
JobEntity jobEntity = jobService.queryInfo(id);
return R.ok().putData("record", jobEntity);
}
@GetMapping
public R list(@RequestParam Map<String, Object> params) {
@GetMapping("/{workspaceId}/job")
public R list(@PathVariable("workspaceId") String workspaceId,
@RequestParam Map<String, Object> params) {
T.VerifyUtil.is(params).notNull()
.and(T.MapUtil.getStr(params, "workspaceId")).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY);
Page page = jobService.queryList(params);
return R.ok(page);
}
@PostMapping
public R add(@RequestBody JobEntity entity) {
@PostMapping("/{workspaceId}/job")
public R add(@PathVariable("workspaceId") String workspaceId,
@RequestBody JobEntity entity) {
T.VerifyUtil.is(entity).notNull()
.and(entity.getRunnerId()).notEmpty(RCode.RUNNER_ID_CANNOT_EMPTY)
.and(entity.getEnvironmentId()).notEmpty(RCode.ENVIRONMENT_ID_CANNOT_EMPTY)
.and(entity.getPackageId()).notEmpty(RCode.PACKAGE_ID_CANNOT_EMPTY)
.and(entity.getPlaybookId()).notEmpty(RCode.PLAYBOOK_ID_CANNOT_EMPTY)
.and(entity.getWorkspaceId()).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY);
entity.setEnvId(entity.getEnvironmentId());
JobEntity jobEntity = jobService.saveJob(entity);
return R.ok().putData("id", jobEntity.getId());
}
@DeleteMapping
public R delete(String[] ids) {
@DeleteMapping("/{workspaceId}/job")
public R delete(@PathVariable("workspaceId") String workspaceId,
@RequestParam String ids) {
T.VerifyUtil.is(ids).notEmpty();
jobService.removeJob(T.ListUtil.of(ids));
List<String> idList = Arrays.asList(ids.split(","));
jobService.removeJob(idList);
return R.ok();
}
@PutMapping("/cancel")
public R cancel(String[] ids) {
@PutMapping("/{workspaceId}/job/cancel")
public R cancel(@PathVariable("workspaceId") String workspaceId,
@RequestParam String ids) {
T.VerifyUtil.is(ids).notEmpty();
List<String> idList = Arrays.asList(ids.split(","));
// TODO 其他处理
// update state
jobService.update(new LambdaUpdateWrapper<JobEntity>()
.in(JobEntity::getId, ids)
.in(JobEntity::getId, idList)
.set(JobEntity::getStatus, "cancel")
);
return R.ok();

348
src/main/java/net/geedge/asw/module/runner/controller/RunnerController.java
View File

@@ -1,174 +1,174 @@
package net.geedge.asw.module.runner.controller;
import cn.dev33.satoken.annotation.SaIgnore;
import cn.hutool.core.lang.Opt;
import cn.hutool.log.Log;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import jakarta.servlet.http.HttpServletResponse;
import net.geedge.asw.common.util.R;
import net.geedge.asw.common.util.RCode;
import net.geedge.asw.common.util.T;
import net.geedge.asw.module.app.entity.PackageEntity;
import net.geedge.asw.module.runner.entity.JobEntity;
import net.geedge.asw.module.runner.entity.PlaybookEntity;
import net.geedge.asw.module.runner.entity.RunnerEntity;
import net.geedge.asw.module.runner.service.IJobService;
import net.geedge.asw.module.runner.service.IRunnerService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.Map;
@RestController
@RequestMapping("/api/v1/runner")
public class RunnerController {
private static final Log log = Log.get();
@Autowired
private IJobService jobService;
@Autowired
private IRunnerService runnerService;
@GetMapping("/{id}")
public R detail(@PathVariable("id") String id) {
RunnerEntity runnerEntity = runnerService.getById(id);
return R.ok().putData("record", runnerEntity);
}
@GetMapping
public R list(@RequestParam Map<String, Object> params) {
T.VerifyUtil.is(params).notNull()
.and(T.MapUtil.getStr(params, "workspaceId")).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY);
Page page = runnerService.queryList(params);
return R.ok(page);
}
@PostMapping
public R add(@RequestBody RunnerEntity entity) {
T.VerifyUtil.is(entity).notNull()
.and(entity.getWorkspaceId()).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY);
RunnerEntity runner = runnerService.saveRunner(entity);
return R.ok().putData("record", runner);
}
@PutMapping
public R update(@RequestBody RunnerEntity entity) {
T.VerifyUtil.is(entity).notNull()
.and(entity.getId()).notEmpty(RCode.ID_CANNOT_EMPTY)
.and(entity.getWorkspaceId()).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY);
RunnerEntity runner = runnerService.updateRunner(entity);
return R.ok().putData("record", runner);
}
@DeleteMapping("/{id}")
public R delete(@PathVariable("id") String id) {
runnerService.removeById(id);
return R.ok();
}
@SaIgnore
@PostMapping("/register")
public void register(@RequestHeader("Authorization") String token, HttpServletResponse response) throws IOException {
RunnerEntity runner = runnerService.getOne(new LambdaUpdateWrapper<RunnerEntity>().eq(RunnerEntity::getToken, token));
String status = Opt.ofNullable(runner).map(RunnerEntity::getStatus).orElseGet(() -> null);
if (!T.StrUtil.equals("online", status)) {
log.warn("[register] [runner is offline] [token: {}]", token);
response.sendError(HttpServletResponse.SC_FORBIDDEN, "Runner is offline");
}
}
@SaIgnore
@PostMapping("/heartbeat")
public void heartbeat(@RequestHeader("Authorization") String token, @RequestBody Map<String, Integer> platformMap,
HttpServletResponse response) throws IOException {
RunnerEntity runner = runnerService.getOne(new LambdaUpdateWrapper<RunnerEntity>().eq(RunnerEntity::getToken, token));
String status = Opt.ofNullable(runner).map(RunnerEntity::getStatus).orElseGet(() -> null);
if (!T.StrUtil.equals("online", status)) {
log.warn("[heartbeat] [runner is offline] [token: {}]", token);
response.sendError(HttpServletResponse.SC_FORBIDDEN, "Runner is offline");
return;
}
// update last_heartbeat_timestamp
runnerService.update(new LambdaUpdateWrapper<RunnerEntity>()
.set(RunnerEntity::getLastHeartbeatTimestamp, System.currentTimeMillis())
.eq(RunnerEntity::getId, runner.getId()));
// findjob by platform
String platform = platformMap.entrySet().stream().filter(entry -> entry.getValue() > 0).findFirst().map(entry -> entry.getKey()).orElseGet(null);
JobEntity job = jobService.assignPendingJob(runner.getId(), platform);
if (T.ObjectUtil.isNotNull(job)) {
// package
PackageEntity pkg = job.getPkg();
Map<String, String> pkgInfo = T.MapUtil.builder("id", pkg.getId())
.put("platform", pkg.getPlatform())
.put("identifier", pkg.getIdentifier())
.put("version", pkg.getVersion())
.build();
// playbook
PlaybookEntity playbook = job.getPlaybook();
Map<String, String> pbInfo = T.MapUtil.builder("id", playbook.getId())
.put("name", playbook.getName())
.build();
// response job info
Map<Object, Object> responseData = T.MapUtil.builder()
.put("id", job.getId())
.put("pkg", pkgInfo)
.put("playbook", pbInfo)
.build();
response.setCharacterEncoding("UTF-8");
response.setContentType("text/html; charset=UTF-8");
response.getWriter().write(T.JSONUtil.toJsonStr(responseData));
}
}
@SaIgnore
@PutMapping("/trace/{jobId}")
public void trace(@RequestHeader("Authorization") String token, @PathVariable String jobId, @RequestBody byte[] bytes,
HttpServletResponse response) throws IOException {
RunnerEntity runner = runnerService.getOne(new LambdaUpdateWrapper<RunnerEntity>().eq(RunnerEntity::getToken, token));
String status = Opt.ofNullable(runner).map(RunnerEntity::getStatus).orElseGet(() -> null);
if (!T.StrUtil.equals("online", status)) {
log.warn("[trace] [runner is offline] [token: {}]", token);
response.sendError(HttpServletResponse.SC_FORBIDDEN, "Runner is offline");
return;
}
try {
// 追加到文件中
String content = T.StrUtil.str(bytes, T.CharsetUtil.CHARSET_UTF_8);
jobService.appendTraceLogStrToFile(jobId, content);
} catch (Exception e) {
log.error("[trace] [error] [job: {}]", jobId);
response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
}
}
@SaIgnore
@PutMapping("/jobResult/{jobId}")
public void jobResult(@RequestHeader("Authorization") String token, @PathVariable String jobId, @RequestParam String state,
@RequestParam(value = "file", required = false) MultipartFile pcapFile,
HttpServletResponse response) throws IOException {
RunnerEntity runner = runnerService.getOne(new LambdaUpdateWrapper<RunnerEntity>().eq(RunnerEntity::getToken, token));
String status = Opt.ofNullable(runner).map(RunnerEntity::getStatus).orElseGet(() -> null);
if (!T.StrUtil.equals("online", status)) {
log.warn("[trace] [runner is offline] [token: {}]", token);
response.sendError(HttpServletResponse.SC_FORBIDDEN, "Runner is offline");
return;
}
// 更新任务状态
jobService.updateJobResult(jobId, state, pcapFile);
}
}
//package net.geedge.asw.module.runner.controller;
//
//import cn.dev33.satoken.annotation.SaIgnore;
//import cn.hutool.core.lang.Opt;
//import cn.hutool.log.Log;
//import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
//import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
//import jakarta.servlet.http.HttpServletResponse;
//import net.geedge.asw.common.util.R;
//import net.geedge.asw.common.util.RCode;
//import net.geedge.asw.common.util.T;
//import net.geedge.asw.module.app.entity.PackageEntity;
//import net.geedge.asw.module.runner.entity.JobEntity;
//import net.geedge.asw.module.runner.entity.PlaybookEntity;
//import net.geedge.asw.module.runner.entity.RunnerEntity;
//import net.geedge.asw.module.runner.service.IJobService;
//import net.geedge.asw.module.runner.service.IRunnerService;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.web.bind.annotation.*;
//import org.springframework.web.multipart.MultipartFile;
//
//import java.io.IOException;
//import java.util.Map;
//
//@RestController
//@RequestMapping("/api/v1/runner")
//public class RunnerController {
//
// private static final Log log = Log.get();
//
// @Autowired
// private IJobService jobService;
//
// @Autowired
// private IRunnerService runnerService;
//
// @GetMapping("/{id}")
// public R detail(@PathVariable("id") String id) {
// RunnerEntity runnerEntity = runnerService.getById(id);
// return R.ok().putData("record", runnerEntity);
// }
//
// @GetMapping
// public R list(@RequestParam Map<String, Object> params) {
// T.VerifyUtil.is(params).notNull()
// .and(T.MapUtil.getStr(params, "workspaceId")).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY);
//
// Page page = runnerService.queryList(params);
// return R.ok(page);
// }
//
// @PostMapping
// public R add(@RequestBody RunnerEntity entity) {
// T.VerifyUtil.is(entity).notNull()
// .and(entity.getWorkspaceId()).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY);
//
// RunnerEntity runner = runnerService.saveRunner(entity);
// return R.ok().putData("record", runner);
// }
//
// @PutMapping
// public R update(@RequestBody RunnerEntity entity) {
// T.VerifyUtil.is(entity).notNull()
// .and(entity.getId()).notEmpty(RCode.ID_CANNOT_EMPTY)
// .and(entity.getWorkspaceId()).notEmpty(RCode.WORKSPACE_ID_CANNOT_EMPTY);
//
// RunnerEntity runner = runnerService.updateRunner(entity);
// return R.ok().putData("record", runner);
// }
//
// @DeleteMapping("/{id}")
// public R delete(@PathVariable("id") String id) {
// runnerService.removeById(id);
// return R.ok();
// }
//
// @SaIgnore
// @PostMapping("/register")
// public void register(@RequestHeader("Authorization") String token, HttpServletResponse response) throws IOException {
// RunnerEntity runner = runnerService.getOne(new LambdaUpdateWrapper<RunnerEntity>().eq(RunnerEntity::getToken, token));
// String status = Opt.ofNullable(runner).map(RunnerEntity::getStatus).orElseGet(() -> null);
// if (!T.StrUtil.equals("online", status)) {
// log.warn("[register] [runner is offline] [token: {}]", token);
// response.sendError(HttpServletResponse.SC_FORBIDDEN, "Runner is offline");
// }
// }
//
// @SaIgnore
// @PostMapping("/heartbeat")
// public void heartbeat(@RequestHeader("Authorization") String token, @RequestBody Map<String, Integer> platformMap,
// HttpServletResponse response) throws IOException {
// RunnerEntity runner = runnerService.getOne(new LambdaUpdateWrapper<RunnerEntity>().eq(RunnerEntity::getToken, token));
// String status = Opt.ofNullable(runner).map(RunnerEntity::getStatus).orElseGet(() -> null);
// if (!T.StrUtil.equals("online", status)) {
// log.warn("[heartbeat] [runner is offline] [token: {}]", token);
// response.sendError(HttpServletResponse.SC_FORBIDDEN, "Runner is offline");
// return;
// }
//
// // update last_heartbeat_timestamp
// runnerService.update(new LambdaUpdateWrapper<RunnerEntity>()
// .set(RunnerEntity::getLastHeartbeatTimestamp, System.currentTimeMillis())
// .eq(RunnerEntity::getId, runner.getId()));
//
// // findjob by platform
// String platform = platformMap.entrySet().stream().filter(entry -> entry.getValue() > 0).findFirst().map(entry -> entry.getKey()).orElseGet(null);
// JobEntity job = jobService.assignPendingJob(runner.getId(), platform);
// if (T.ObjectUtil.isNotNull(job)) {
// // package
// PackageEntity pkg = job.getPkg();
// Map<String, String> pkgInfo = T.MapUtil.builder("id", pkg.getId())
// .put("platform", pkg.getPlatform())
// .put("identifier", pkg.getIdentifier())
// .put("version", pkg.getVersion())
// .build();
//
// // playbook
// PlaybookEntity playbook = job.getPlaybook();
// Map<String, String> pbInfo = T.MapUtil.builder("id", playbook.getId())
// .put("name", playbook.getName())
// .build();
//
// // response job info
// Map<Object, Object> responseData = T.MapUtil.builder()
// .put("id", job.getId())
// .put("pkg", pkgInfo)
// .put("playbook", pbInfo)
// .build();
// response.setCharacterEncoding("UTF-8");
// response.setContentType("text/html; charset=UTF-8");
// response.getWriter().write(T.JSONUtil.toJsonStr(responseData));
// }
// }
//
// @SaIgnore
// @PutMapping("/trace/{jobId}")
// public void trace(@RequestHeader("Authorization") String token, @PathVariable String jobId, @RequestBody byte[] bytes,
// HttpServletResponse response) throws IOException {
// RunnerEntity runner = runnerService.getOne(new LambdaUpdateWrapper<RunnerEntity>().eq(RunnerEntity::getToken, token));
// String status = Opt.ofNullable(runner).map(RunnerEntity::getStatus).orElseGet(() -> null);
// if (!T.StrUtil.equals("online", status)) {
// log.warn("[trace] [runner is offline] [token: {}]", token);
// response.sendError(HttpServletResponse.SC_FORBIDDEN, "Runner is offline");
// return;
// }
//
// try {
// // 追加到文件中
// String content = T.StrUtil.str(bytes, T.CharsetUtil.CHARSET_UTF_8);
// jobService.appendTraceLogStrToFile(jobId, content);
// } catch (Exception e) {
// log.error("[trace] [error] [job: {}]", jobId);
// response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
// }
// }
//
// @SaIgnore
// @PutMapping("/jobResult/{jobId}")
// public void jobResult(@RequestHeader("Authorization") String token, @PathVariable String jobId, @RequestParam String state,
// @RequestParam(value = "file", required = false) MultipartFile pcapFile,
// HttpServletResponse response) throws IOException {
// RunnerEntity runner = runnerService.getOne(new LambdaUpdateWrapper<RunnerEntity>().eq(RunnerEntity::getToken, token));
// String status = Opt.ofNullable(runner).map(RunnerEntity::getStatus).orElseGet(() -> null);
// if (!T.StrUtil.equals("online", status)) {
// log.warn("[trace] [runner is offline] [token: {}]", token);
// response.sendError(HttpServletResponse.SC_FORBIDDEN, "Runner is offline");
// return;
// }
//
// // 更新任务状态
// jobService.updateJobResult(jobId, state, pcapFile);
// }
//
//}

2
src/main/java/net/geedge/asw/module/runner/dao/JobDao.java
View File

@@ -14,6 +14,4 @@ public interface JobDao extends BaseMapper<JobEntity>{
List<JobEntity> queryList(IPage page, Map<String, Object> params);
JobEntity getPendingJobByPlatform(@Param("platform") String platform);
}

17
src/main/java/net/geedge/asw/module/runner/entity/JobEntity.java
View File

@@ -6,8 +6,8 @@ import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
import net.geedge.asw.module.app.entity.ApplicationEntity;
import net.geedge.asw.module.app.entity.PackageEntity;
import net.geedge.asw.module.environment.entity.EnvironmentEntity;
@Data
@TableName("job")
@@ -15,12 +15,10 @@ public class JobEntity {
@TableId(type = IdType.ASSIGN_UUID)
private String id;
private String playbookId;
private String packageId;
private String runnerId;
private String scheduleId;
private String signatureIds;
private String tags;
private String envId;
private String playbookId;
private String playbookParam;
private Long startTimestamp;
private Long endTimestamp;
private String status;
@@ -35,17 +33,14 @@ public class JobEntity {
private String workspaceId;
@TableField(exist = false)
private String workbookId;
@TableField(exist = false)
private ApplicationEntity application;
private String environmentId;
@TableField(exist = false)
@JsonProperty(value = "package")
private PackageEntity pkg;
@TableField(exist = false)
private RunnerEntity runner;
private EnvironmentEntity environment;
@TableField(exist = false)
private PlaybookEntity playbook;

3
src/main/java/net/geedge/asw/module/runner/entity/PcapEntity.java
View File

@@ -10,6 +10,7 @@ import lombok.Data;
import net.geedge.asw.common.util.T;
import net.geedge.asw.module.app.entity.ApplicationEntity;
import net.geedge.asw.module.app.entity.PackageEntity;
import net.geedge.asw.module.environment.entity.EnvironmentEntity;
import net.geedge.asw.module.workspace.entity.WorkspaceEntity;
import java.nio.file.Path;
@@ -45,7 +46,7 @@ public class PcapEntity {
private PackageEntity pkg;
@TableField(exist = false)
private RunnerEntity runner;
private EnvironmentEntity environment;
@TableField(exist = false)
private PlaybookEntity playbook;

11
src/main/java/net/geedge/asw/module/runner/service/IJobService.java
View File

@@ -3,7 +3,6 @@ package net.geedge.asw.module.runner.service;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.IService;
import net.geedge.asw.module.runner.entity.JobEntity;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
import java.util.Map;
@@ -18,10 +17,10 @@ public interface IJobService extends IService<JobEntity>{
void removeJob(List<String> ids);
JobEntity assignPendingJob(String id, String platform);
void appendTraceLogStrToFile(String jobId, String content) throws RuntimeException;
void updateJobResult(String jobId, String state, MultipartFile pcapFile);
// JobEntity assignPendingJob(String id, String platform);
//
// void appendTraceLogStrToFile(String jobId, String content) throws RuntimeException;
//
// void updateJobResult(String jobId, String state, MultipartFile pcapFile);
}

3
src/main/java/net/geedge/asw/module/runner/service/IPcapService.java
View File

@@ -4,7 +4,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.IService;
import net.geedge.asw.module.runner.entity.PcapEntity;
import org.springframework.core.io.Resource;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.Map;
@@ -14,8 +13,6 @@ public interface IPcapService extends IService<PcapEntity>{
Page queryList(Map<String, Object> params);
PcapEntity savePcap(String jobId, Resource fileResource);
PcapEntity savePcap(Resource fileResource,String... params);
void deletePcap(String... ids);

34
src/main/java/net/geedge/asw/module/runner/service/IRunnerService.java
View File

@@ -1,17 +1,17 @@
package net.geedge.asw.module.runner.service;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.IService;
import net.geedge.asw.module.runner.entity.RunnerEntity;
import java.util.Map;
public interface IRunnerService extends IService<RunnerEntity>{
Page queryList(Map<String, Object> params);
RunnerEntity saveRunner(RunnerEntity entity);
RunnerEntity updateRunner(RunnerEntity entity);
}
//package net.geedge.asw.module.runner.service;
//
//import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
//import com.baomidou.mybatisplus.extension.service.IService;
//import net.geedge.asw.module.runner.entity.RunnerEntity;
//
//import java.util.Map;
//
//public interface IRunnerService extends IService<RunnerEntity>{
//
// Page queryList(Map<String, Object> params);
//
// RunnerEntity saveRunner(RunnerEntity entity);
//
// RunnerEntity updateRunner(RunnerEntity entity);
//
//}

140
src/main/java/net/geedge/asw/module/runner/service/impl/JobServiceImpl.java
View File

@@ -1,33 +1,25 @@
package net.geedge.asw.module.runner.service.impl;
import cn.dev33.satoken.stp.StpUtil;
import cn.hutool.core.io.IORuntimeException;
import cn.hutool.log.Log;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import net.geedge.asw.common.util.RCode;
import net.geedge.asw.common.util.T;
import net.geedge.asw.module.app.entity.ApplicationEntity;
import net.geedge.asw.module.app.entity.PackageEntity;
import net.geedge.asw.module.app.service.IApplicationService;
import net.geedge.asw.module.app.service.IPackageService;
import net.geedge.asw.module.environment.entity.EnvironmentEntity;
import net.geedge.asw.module.environment.service.IEnvironmentService;
import net.geedge.asw.module.runner.dao.JobDao;
import net.geedge.asw.module.runner.entity.JobEntity;
import net.geedge.asw.module.runner.entity.PcapEntity;
import net.geedge.asw.module.runner.entity.PlaybookEntity;
import net.geedge.asw.module.runner.entity.RunnerEntity;
import net.geedge.asw.module.runner.service.IJobService;
import net.geedge.asw.module.runner.service.IPcapService;
import net.geedge.asw.module.runner.service.IPlaybookService;
import net.geedge.asw.module.runner.service.IRunnerService;
import net.geedge.asw.module.runner.util.RunnerConstant;
import net.geedge.asw.module.workbook.service.IWorkbookResourceService;
import net.geedge.asw.module.workbook.util.WorkbookConstant;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.util.List;
@@ -39,10 +31,7 @@ public class JobServiceImpl extends ServiceImpl<JobDao, JobEntity> implements IJ
private static final Log log = Log.get();
@Autowired
private IPcapService pcapService;
@Autowired
private IRunnerService runnerService;
private IEnvironmentService environmentService;
@Autowired
private IPlaybookService playbookService;
@@ -50,11 +39,6 @@ public class JobServiceImpl extends ServiceImpl<JobDao, JobEntity> implements IJ
@Autowired
private IPackageService packageService;
@Autowired
private IApplicationService applicationService;
@Autowired
private IWorkbookResourceService workbookResourceService;
/**
* rootPath/result/{jobId}
@@ -71,8 +55,8 @@ public class JobServiceImpl extends ServiceImpl<JobDao, JobEntity> implements IJ
JobEntity job = this.getById(id);
T.VerifyUtil.is(job).notNull(RCode.SYS_RECORD_NOT_FOUND);
RunnerEntity runner = runnerService.getById(job.getRunnerId());
job.setRunner(runner);
EnvironmentEntity env = environmentService.getById(job.getEnvId());
job.setEnvironment(env);
PlaybookEntity playbook = playbookService.getById(job.getPlaybookId());
job.setPlaybook(playbook);
@@ -97,13 +81,11 @@ public class JobServiceImpl extends ServiceImpl<JobDao, JobEntity> implements IJ
entity.setUpdateTimestamp(System.currentTimeMillis());
entity.setCreateUserId(StpUtil.getLoginIdAsString());
entity.setUpdateUserId(StpUtil.getLoginIdAsString());
entity.setStatus(RunnerConstant.JobStatus.CREATED.getValue());
// save
this.save(entity);
// workbook resource
workbookResourceService.saveResource(entity.getWorkbookId(), entity.getId(), WorkbookConstant.ResourceType.JOB.getValue());
// trace log file path
File traceLogFile = T.FileUtil.file(this.getJobResultPath(entity.getId()), "trace.log");
this.update(new LambdaUpdateWrapper<JobEntity>()
@@ -117,63 +99,61 @@ public class JobServiceImpl extends ServiceImpl<JobDao, JobEntity> implements IJ
public void removeJob(List<String> ids) {
// remove
this.removeBatchByIds(ids);
// workbook resource
workbookResourceService.removeResource(ids, WorkbookConstant.ResourceType.JOB.getValue());
}
@Override
public synchronized JobEntity assignPendingJob(String runnerId, String platform) {
if (T.StrUtil.hasEmpty(runnerId, platform)) {
return null;
}
// query
JobEntity job = this.getBaseMapper().getPendingJobByPlatform(platform);
if (T.ObjectUtil.isNotNull(job)) {
// update
this.update(new LambdaUpdateWrapper<JobEntity>()
.set(JobEntity::getRunnerId, runnerId)
.set(JobEntity::getStatus, RunnerConstant.JobStatus.RUNNING.getValue())
.set(JobEntity::getStartTimestamp, System.currentTimeMillis())
.eq(JobEntity::getId, job.getId())
);
}
return job;
}
@Override
public void appendTraceLogStrToFile(String jobId, String content) throws RuntimeException {
try {
JobEntity job = this.getById(jobId);
if (T.StrUtil.isEmpty(job.getLogPath())) {
File traceLogFile = T.FileUtil.file(this.getJobResultPath(jobId), "trace.log");
job.setLogPath(traceLogFile.getPath());
}
// append content
T.FileUtil.appendString(content, T.FileUtil.file(job.getLogPath()), T.CharsetUtil.CHARSET_UTF_8);
} catch (IORuntimeException e) {
log.error(e, "[appendTraceLogStrToFile] [error] [job: {}] [content: {}]", jobId, content);
throw new RuntimeException(e.getMessage());
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public void updateJobResult(String jobId, String state, MultipartFile pcapFile) {
String pcapId = T.StrUtil.EMPTY;
// save pcap file
if (T.ObjectUtil.isNotNull(pcapFile)) {
PcapEntity pcapEntity = pcapService.savePcap(jobId, pcapFile.getResource());
pcapId = pcapEntity.getId();
}
// update job status&pcap_id
state = T.StrUtil.equals("success", state) ? RunnerConstant.JobStatus.PASSED.getValue() : state;
this.update(new LambdaUpdateWrapper<JobEntity>()
.set(JobEntity::getStatus, state)
.set(T.StrUtil.isNotEmpty(pcapId), JobEntity::getPcapId, pcapId)
.set(JobEntity::getEndTimestamp, System.currentTimeMillis())
.eq(JobEntity::getId, jobId)
);
}
// @Override
// public synchronized JobEntity assignPendingJob(String runnerId, String platform) {
// if (T.StrUtil.hasEmpty(runnerId, platform)) {
// return null;
// }
// // query
// JobEntity job = this.getBaseMapper().getPendingJobByPlatform(platform);
// if (T.ObjectUtil.isNotNull(job)) {
// // update
// this.update(new LambdaUpdateWrapper<JobEntity>()
// .set(JobEntity::getRunnerId, runnerId)
// .set(JobEntity::getStatus, RunnerConstant.JobStatus.RUNNING.getValue())
// .set(JobEntity::getStartTimestamp, System.currentTimeMillis())
// .eq(JobEntity::getId, job.getId())
// );
// }
// return job;
// }
//
// @Override
// public void appendTraceLogStrToFile(String jobId, String content) throws RuntimeException {
// try {
// JobEntity job = this.getById(jobId);
// if (T.StrUtil.isEmpty(job.getLogPath())) {
// File traceLogFile = T.FileUtil.file(this.getJobResultPath(jobId), "trace.log");
// job.setLogPath(traceLogFile.getPath());
// }
// // append content
// T.FileUtil.appendString(content, T.FileUtil.file(job.getLogPath()), T.CharsetUtil.CHARSET_UTF_8);
// } catch (IORuntimeException e) {
// log.error(e, "[appendTraceLogStrToFile] [error] [job: {}] [content: {}]", jobId, content);
// throw new RuntimeException(e.getMessage());
// }
// }
//
// @Override
// @Transactional(rollbackFor = Exception.class)
// public void updateJobResult(String jobId, String state, MultipartFile pcapFile) {
// String pcapId = T.StrUtil.EMPTY;
// // save pcap file
// if (T.ObjectUtil.isNotNull(pcapFile)) {
// PcapEntity pcapEntity = pcapService.savePcap(jobId, pcapFile.getResource());
// pcapId = pcapEntity.getId();
// }
//
// // update job status&pcap_id
// state = T.StrUtil.equals("success", state) ? RunnerConstant.JobStatus.PASSED.getValue() : state;
// this.update(new LambdaUpdateWrapper<JobEntity>()
// .set(JobEntity::getStatus, state)
// .set(T.StrUtil.isNotEmpty(pcapId), JobEntity::getPcapId, pcapId)
// .set(JobEntity::getEndTimestamp, System.currentTimeMillis())
// .eq(JobEntity::getId, jobId)
// );
// }
}

19
src/main/java/net/geedge/asw/module/runner/service/impl/PcapServiceImpl.java
View File

@@ -9,27 +9,25 @@ import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import net.geedge.asw.common.config.SpringContextUtils;
import net.geedge.asw.common.util.ASWException;
import net.geedge.asw.common.util.RCode;
import net.geedge.asw.common.util.T;
import net.geedge.asw.module.app.entity.ApplicationEntity;
import net.geedge.asw.module.app.entity.PackageEntity;
import net.geedge.asw.module.app.service.IApplicationService;
import net.geedge.asw.module.app.service.IPackageService;
import net.geedge.asw.module.environment.entity.EnvironmentEntity;
import net.geedge.asw.module.environment.service.IEnvironmentService;
import net.geedge.asw.module.feign.client.KibanaClient;
import net.geedge.asw.module.runner.dao.PcapDao;
import net.geedge.asw.module.runner.entity.JobEntity;
import net.geedge.asw.module.runner.entity.PcapEntity;
import net.geedge.asw.module.runner.entity.PlaybookEntity;
import net.geedge.asw.module.runner.entity.RunnerEntity;
import net.geedge.asw.module.runner.service.IJobService;
import net.geedge.asw.module.runner.service.IPcapService;
import net.geedge.asw.module.runner.service.IPlaybookService;
import net.geedge.asw.module.runner.service.IRunnerService;
import net.geedge.asw.module.runner.util.PcapParserThread;
import net.geedge.asw.module.runner.util.RunnerConstant;
import net.geedge.asw.module.workbook.service.IWorkbookResourceService;
@@ -68,7 +66,7 @@ public class PcapServiceImpl extends ServiceImpl<PcapDao, PcapEntity> implements
private IJobService jobService;
@Autowired
private IRunnerService runnerService;
private IEnvironmentService environmentService;
@Autowired
private IPlaybookService playbookService;
@@ -103,8 +101,8 @@ public class PcapServiceImpl extends ServiceImpl<PcapDao, PcapEntity> implements
if (T.ObjectUtil.isNotNull(job)) {
pcap.setJobId(job.getId());
RunnerEntity runner = runnerService.getById(job.getRunnerId());
pcap.setRunner(runner);
EnvironmentEntity environment = environmentService.getById(job.getEnvId());
pcap.setEnvironment(environment);
PackageEntity pkg = packageService.getById(job.getPackageId());
pcap.setPkg(pkg);
@@ -123,13 +121,6 @@ public class PcapServiceImpl extends ServiceImpl<PcapDao, PcapEntity> implements
return page;
}
@Override
public PcapEntity savePcap(String jobId, Resource fileResource) {
JobEntity job = jobService.getById(jobId);
return this.savePcap(fileResource, job.getTags(), job.getWorkbookId(), job.getWorkspaceId(), job.getCreateUserId());
}
@Override
public PcapEntity savePcap(Resource fileResource, String... params) {
String description = T.ArrayUtil.get(params, 0);

102
src/main/java/net/geedge/asw/module/runner/service/impl/RunnerServiceImpl.java
View File

@@ -1,51 +1,51 @@
package net.geedge.asw.module.runner.service.impl;
import cn.dev33.satoken.stp.StpUtil;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import net.geedge.asw.common.util.T;
import net.geedge.asw.module.runner.dao.RunnerDao;
import net.geedge.asw.module.runner.entity.RunnerEntity;
import net.geedge.asw.module.runner.service.IRunnerService;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
@Service
public class RunnerServiceImpl extends ServiceImpl<RunnerDao, RunnerEntity> implements IRunnerService {
@Override
public Page queryList(Map<String, Object> params) {
Page page = T.PageUtil.getPage(params);
List<RunnerEntity> jobList = this.getBaseMapper().queryList(page, params);
page.setRecords(jobList);
return page;
}
@Override
public RunnerEntity saveRunner(RunnerEntity entity) {
entity.setCreateTimestamp(System.currentTimeMillis());
entity.setUpdateTimestamp(System.currentTimeMillis());
entity.setCreateUserId(StpUtil.getLoginIdAsString());
entity.setUpdateUserId(StpUtil.getLoginIdAsString());
// token
entity.setToken(T.IdUtil.fastSimpleUUID());
// save
this.save(entity);
return entity;
}
@Override
public RunnerEntity updateRunner(RunnerEntity entity) {
entity.setUpdateTimestamp(System.currentTimeMillis());
entity.setUpdateUserId(StpUtil.getLoginIdAsString());
// update
this.updateById(entity);
return entity;
}
}
//package net.geedge.asw.module.runner.service.impl;
//
//import cn.dev33.satoken.stp.StpUtil;
//import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
//import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
//import net.geedge.asw.common.util.T;
//import net.geedge.asw.module.runner.dao.RunnerDao;
//import net.geedge.asw.module.runner.entity.RunnerEntity;
//import net.geedge.asw.module.runner.service.IRunnerService;
//import org.springframework.stereotype.Service;
//
//import java.util.List;
//import java.util.Map;
//
//@Service
//public class RunnerServiceImpl extends ServiceImpl<RunnerDao, RunnerEntity> implements IRunnerService {
//
// @Override
// public Page queryList(Map<String, Object> params) {
// Page page = T.PageUtil.getPage(params);
// List<RunnerEntity> jobList = this.getBaseMapper().queryList(page, params);
// page.setRecords(jobList);
// return page;
// }
//
// @Override
// public RunnerEntity saveRunner(RunnerEntity entity) {
// entity.setCreateTimestamp(System.currentTimeMillis());
// entity.setUpdateTimestamp(System.currentTimeMillis());
// entity.setCreateUserId(StpUtil.getLoginIdAsString());
// entity.setUpdateUserId(StpUtil.getLoginIdAsString());
//
// // token
// entity.setToken(T.IdUtil.fastSimpleUUID());
//
// // save
// this.save(entity);
// return entity;
// }
//
// @Override
// public RunnerEntity updateRunner(RunnerEntity entity) {
// entity.setUpdateTimestamp(System.currentTimeMillis());
// entity.setUpdateUserId(StpUtil.getLoginIdAsString());
//
// // update
// this.updateById(entity);
// return entity;
// }
//
//}

72
src/main/resources/db/mapper/runner/JobMapper.xml
View File

@@ -7,10 +7,7 @@
<id property="id" column="id"/>
<result property="playbookId" column="playbook_id"/>
<result property="packageId" column="package_id"/>
<result property="runnerId" column="runner_id"/>
<result property="scheduleId" column="schedule_id"/>
<result property="signatureIds" column="signature_ids"/>
<result property="tags" column="tags"/>
<result property="envId" column="env_id"/>
<result property="startTimestamp" column="start_timestamp"/>
<result property="endTimestamp" column="end_timestamp"/>
<result property="status" column="status"/>
@@ -25,18 +22,11 @@
<association property="pkg" columnPrefix="pkg_" javaType="net.geedge.asw.module.app.entity.PackageEntity">
<id property="id" column="id"/>
<result property="platform" column="platform"/>
<result property="identifier" column="identifier"/>
<result property="version" column="version"/>
<result property="logo" column="logo"/>
</association>
<association property="application" columnPrefix="app_"
javaType="net.geedge.asw.module.app.entity.ApplicationEntity">
<id property="id" column="id"/>
<result property="name" column="name"/>
</association>
<association property="runner" columnPrefix="run_" javaType="net.geedge.asw.module.runner.entity.RunnerEntity">
<association property="environment" columnPrefix="em_" javaType="net.geedge.asw.module.environment.entity.EnvironmentEntity">
<id property="id" column="id"/>
<result property="name" column="name"/>
</association>
@@ -55,60 +45,38 @@
pkg.id AS pkg_id,
pkg.platform AS pkg_platform,
pkg.version AS pkg_version,
pkg.logo AS pkg_logo,
pkg.identifier AS pkg_identifier,
pkg.name AS pkg_name,
app.id AS app_id,
app.name AS app_name,
run.id AS run_id,
run.name AS run_name,
env.id AS em_id,
env.name AS em_name,
pb.id AS pb_id,
pb.name AS pb_name
FROM
job job
LEFT JOIN runner run ON job.runner_id = run.id
LEFT JOIN environment env ON job.env_id = env.id
LEFT JOIN package pkg ON job.package_id = pkg.id
LEFT JOIN playbook pb ON job.playbook_id = pb.id
LEFT JOIN application app ON pb.app_id = app.id
LEFT JOIN workbook_resource wr ON job.id = wr.resource_id AND wr.resource_type = 'job'
<where>
<if test="params.ids != null and params.ids != ''">
job.id in
<foreach item="id" collection="params.ids.split(',')" separator="," open="(" close=")">#{id}</foreach>
</if>
<if test="params.appIds != null and params.appIds != ''">
AND app.id in
<foreach item="id" collection="params.appIds.split(',')" separator="," open="(" close=")">#{id}</foreach>
</if>
<if test="params.packageIds != null and params.packageIds != ''">
AND pkg.id in
<foreach item="id" collection="params.packageIds.split(',')" separator="," open="(" close=")">#{id}</foreach>
</if>
<if test="params.runnerIds != null and params.runnerIds != ''">
AND run.id in
<foreach item="id" collection="params.runnerIds.split(',')" separator="," open="(" close=")">#{id}</foreach>
<if test="params.environmentIds != null and params.environmentIds != ''">
AND env.id in
<foreach item="id" collection="params.environmentIds.split(',')" separator="," open="(" close=")">#{id}</foreach>
</if>
<if test="params.playbooks != null and params.playbooks != ''">
AND pb.id in
<foreach item="id" collection="params.playbooks.split(',')" separator="," open="(" close=")">#{id}</foreach>
</if>
<if test="params.signatureIds != null and params.signatureIds != ''">
AND <foreach item="item" collection="params.signatureIds.split(',')" separator="OR" index="" open="(" close=")">
locate(#{item}, job.signature_ids)
</foreach>
</if>
<if test="params.workbookId != null and params.workbookId != ''">
AND wr.workbook_id = #{params.workbookId}
</if>
<if test="params.workspaceId != null and params.workspaceId != ''">
AND job.workspace_id = #{params.workspaceId}
</if>
@@ -122,26 +90,4 @@
</if>
</select>
<select id="getPendingJobByPlatform" resultMap="jobResultMap">
SELECT
job.*,
pkg.id AS pkg_id,
pkg.platform AS pkg_platform,
pkg.identifier AS pkg_identifier,
pkg.version AS pkg_version,
pb.id AS pb_id,
pb.name AS pb_name
FROM
job job
LEFT JOIN package pkg ON job.package_id = pkg.id
LEFT JOIN playbook pb ON job.playbook_id = pb.id
WHERE
job.status = 'pending' and pkg.platform = #{platform}
ORDER BY job.create_timestamp ASC
LIMIT 1
</select>
</mapper>

5
src/main/resources/db/migration/R__AZ_sys_i18n.sql
View File

@@ -135,6 +135,9 @@ INSERT INTO `sys_i18n`(`id`, `name`, `code`, `value`, `lang`, `remark`, `update_
INSERT INTO `sys_i18n`(`id`, `name`, `code`, `value`, `lang`, `remark`, `update_user_id`, `update_timestamp`) VALUES (227, '601004', 'ENVIRONMENT_STATUS_ERROR', '环境状态不可用', 'zh', '', 'admin', 1724030366000);
INSERT INTO `sys_i18n`(`id`, `name`, `code`, `value`, `lang`, `remark`, `update_user_id`, `update_timestamp`) VALUES (228, '100019', 'SYS_USER_OLDPWD_INCORRECT', 'Incorrect old password. Please try again.', 'en', '', 'admin', 1724030366000);
INSERT INTO `sys_i18n`(`id`, `name`, `code`, `value`, `lang`, `remark`, `update_user_id`, `update_timestamp`) VALUES (229, '100019', 'SYS_USER_OLDPWD_INCORRECT', '旧密码不正确，请重新输入', 'zh', '', 'admin', 1724030366000);
INSERT INTO `sys_i18n`(`id`, `name`, `code`, `value`, `lang`, `remark`, `update_user_id`, `update_timestamp`) VALUES (230, '302002', 'PLAYBOOK_NAME_DUPLICATE', 'playbook name duplicate', 'en', '', 'admin', 1724030366000);
INSERT INTO `sys_i18n`(`id`, `name`, `code`, `value`, `lang`, `remark`, `update_user_id`, `update_timestamp`) VALUES (231, '302002', 'PLAYBOOK_NAME_DUPLICATE', '剧本名称重复', 'zh', '', 'admin', 1724030366000);
INSERT INTO `sys_i18n`(`id`, `name`, `code`, `value`, `lang`, `remark`, `update_user_id`, `update_timestamp`) VALUES (232, '601005', 'ENVIRONMENT_ID_CANNOT_EMPTY', 'environment id cannot be empty', 'en', '', 'admin', 1724030366000);
INSERT INTO `sys_i18n`(`id`, `name`, `code`, `value`, `lang`, `remark`, `update_user_id`, `update_timestamp`) VALUES (233, '601005', 'ENVIRONMENT_ID_CANNOT_EMPTY', '环境 id 不能为空', 'zh', '', 'admin', 1724030366000);
SET FOREIGN_KEY_CHECKS = 1;

8
src/main/resources/db/migration/V1.0.01__INIT_TABLES.sql
View File

@@ -200,10 +200,8 @@ CREATE TABLE `job` (
`id` varchar(64) NOT NULL COMMENT '主键',
`playbook_id` varchar(64) NOT NULL DEFAULT '' COMMENT 'Playbook ID',
`package_id` varchar(64) NOT NULL DEFAULT '' COMMENT 'Package ID',
`runner_id` varchar(64) NOT NULL DEFAULT '' COMMENT 'Runner ID',
`schedule_id` varchar(64) NOT NULL DEFAULT '' COMMENT '定时器ID',
`signature_ids` text NOT NULL DEFAULT '' COMMENT '特征ID,多个逗号分隔',
`tags` varchar(256) NOT NULL DEFAULT '' COMMENT '标签; 默认："";多个用逗号分隔;例：kz,vpn,android',
`env_id` varchar(64) NOT NULL DEFAULT '' COMMENT 'env ID',
`playbook_param` varchar(1024) NOT NULL DEFAULT '' COMMENT 'playbook运行参数',
`start_timestamp` bigint(20) NOT NULL DEFAULT -1 COMMENT '开始时间戳',
`end_timestamp` bigint(20) NOT NULL DEFAULT -1 COMMENT '结束时间戳',
`status` varchar(64) NOT NULL DEFAULT '' COMMENT '状态; 可选值: created,pending,running,passed,failed,cancel',
@@ -217,7 +215,7 @@ CREATE TABLE `job` (
PRIMARY KEY (`id`) USING BTREE,
KEY `idx_playbook_id` (`playbook_id`) USING BTREE,
KEY `idx_package_id` (`package_id`) USING BTREE,
KEY `idx_runner_id` (`runner_id`) USING BTREE,
KEY `idx_env_id` (`env_id`) USING BTREE,
KEY `idx_workspace_id` (`workspace_id`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;