gfwleak/appsketch-works-device-api
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: playbook 支持 --job_id --job_path --packate_name 参数，支持下载 artifacts 文件

Browse Source
This commit is contained in:
zhangshuai
2024-11-25 15:01:29 +08:00
parent 63f4e6ae4a
commit b3a790db74
5 changed files with 111 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

257
src/main/java/net/geedge/api/controller/APIController.java
View File

@@ -5,11 +5,13 @@ import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.core.util.ArrayUtil;
import cn.hutool.log.Log;
import jakarta.servlet.http.HttpServletResponse;
import net.geedge.api.entity.EnvApiYml;
import net.geedge.api.util.AdbUtil;
import net.geedge.api.util.CommandExec;
import net.geedge.api.util.PlaybookRunnable;
import net.geedge.common.*;
import net.lingala.zip4j.ZipFile;
import net.lingala.zip4j.model.ZipParameters;
@@ -20,7 +22,10 @@ import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.regex.Pattern;
@RestController
@RequestMapping("/api/v1/env")
@@ -30,8 +35,6 @@ public class APIController {
private final AdbUtil adbUtil;
static final List<Thread> ACTIVE_TASKS = Collections.synchronizedList(new ArrayList<>());
@Autowired
public APIController(EnvApiYml envApiYml) {
this.adbUtil = AdbUtil.getInstance(envApiYml.getAdb(), new CommandExec(null));
@@ -312,7 +315,6 @@ public class APIController {
}
PlaybookRunnable playbookRunnable = new PlaybookRunnable(apiYml, apkFile, playbookAirDir, id, packageName, reInstall, clearCache, unInstall);
playbookRunnable.setName(T.StrUtil.concat(true, id, "-", apkFile.getName()));
ThreadUtil.execAsync(playbookRunnable);
return R.ok();
}
@@ -332,9 +334,9 @@ public class APIController {
if (T.StrUtil.isEmpty(id)) {
throw new APIException(RCode.BAD_REQUEST);
}
if (CollUtil.isNotEmpty(ACTIVE_TASKS)) {
ACTIVE_TASKS.stream().filter(thread -> T.StrUtil.startWith(thread.getName(), id)).forEach(thread -> {
log.info(String.format("playbook thread: %s has been canceled", thread.getName()));
if (CollUtil.isNotEmpty(Constant.ACTIVE_TASKS)) {
Constant.ACTIVE_TASKS.stream().forEach(thread -> {
log.info(String.format("playbook thread: %s has been canceled", id));
thread.interrupt();
});
}
@@ -371,21 +373,58 @@ public class APIController {
@GetMapping("/playbook/{id}/artifact")
public void getJobResultArtifact(@PathVariable("id") String id, HttpServletResponse response) throws IOException {
public void getJobResultArtifact(@PathVariable("id") String id, @RequestParam(value = "artifacts",required = false) String[] artifacts , HttpServletResponse response) throws IOException {
if (T.StrUtil.isEmpty(id)) {
throw new APIException(RCode.BAD_REQUEST);
}
// job dir
File jobResult = T.FileUtil.file(Constant.TEMP_PATH, id);
File zipFile = T.FileUtil.file(Constant.TEMP_PATH, id, T.StrUtil.concat(true, id, ".zip"));
File[] files = jobResult.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".log") || name.endsWith(".pcap");
}
});
// artifact
List<File> artifactFiles = ListUtil.list(false);
File playbookDir = Arrays.stream(jobResult.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".air");
}
})).toList().getFirst();
if (ArrayUtil.isNotEmpty(artifacts)) {
for (String artifact : artifacts) {
if (containsRegex(artifact)) {
int lastSeparator = artifact.lastIndexOf(FileSystems.getDefault().getSeparator());
String regex = (lastSeparator >= 0) ? artifact.substring(lastSeparator + 1) : artifact;
String parent = (lastSeparator >= 0) ? artifact.substring(0, lastSeparator) : "";
// Resolve parent directory
Path parentPath = parent.isEmpty() ? Paths.get(playbookDir.getPath()) : Paths.get(playbookDir.getPath(), parent).normalize();
// Compile regex pattern
Pattern pattern = Pattern.compile(regex);
// Find matching files
artifactFiles = findMatchingFiles(parentPath, pattern);
} else {
Path resolvedPath = Paths.get(artifact);
if (!resolvedPath.isAbsolute()) {
resolvedPath = Paths.get(playbookDir.getPath(), artifact).normalize();
}
if (Files.exists(resolvedPath)) {
artifactFiles.add(resolvedPath.toFile());
}
}
}
}
ZipFile zip = null;
try {
File zipFile = T.FileUtil.file(Constant.TEMP_PATH, id, T.StrUtil.concat(true, id, ".zip"));
zip = new ZipFile(zipFile);
ZipParameters parameters = new ZipParameters();
@@ -396,196 +435,38 @@ public class APIController {
for (File file : files) {
zip.addFile(file, parameters);
}
for (File artifactFile : artifactFiles) {
zip.addFile(artifactFile, parameters);
}
T.ResponseUtil.downloadFile(response, zipFile.getName(), T.FileUtil.readBytes(zipFile.getPath()));
} finally {
zip.close();
}
}
public class PlaybookRunnable extends Thread {
private static boolean containsRegex(String path) {
return path.contains("*") || path.contains("?") || path.contains("[") || path.contains("]");
}
private AdbUtil adbUtil;
private EnvApiYml envApiYml;
private String tid;
private File apkFile;
private String packageName;
private File playbookDir;
private boolean reInstall;
private boolean clearCache;
private boolean unInstall;
private boolean interrupt;
public PlaybookRunnable(EnvApiYml envApiYml, File apkFile, File playbookDir, String tid, String packageName, Boolean reInstall, Boolean clearCache, Boolean unInstall) {
this.envApiYml = envApiYml;
this.tid = tid;
this.apkFile = apkFile;
this.packageName = packageName;
this.playbookDir = playbookDir;
this.reInstall = reInstall;
this.clearCache = clearCache;
this.unInstall = unInstall;
this.interrupt = false;
}
@Override
public void run() {
ACTIVE_TASKS.add(this);
File logFile = FileUtil.file(Constant.TEMP_PATH, tid, "result.log");
File statusFile = FileUtil.file(Constant.TEMP_PATH, tid, "result.json");
AdbUtil.CommandResult tcpdumpPackage = null;
AdbUtil.CommandResult tcpdumpAll = null;
try {
Map resultMap = T.MapUtil.builder()
.put("status", "running")
.build();
T.FileUtil.writeString(T.JSONUtil.toJsonStr(resultMap), statusFile, "UTF-8");
if (interrupt) return;
T.FileUtil.appendString(String.format("Running with %s:%s Android Simulator \n", envApiYml.getAdb().getHost(), envApiYml.getAdb().getPort()), logFile, "UTF-8");
adbUtil = new AdbUtil(envApiYml.getAdb(), new CommandExec(logFile));
// Check if the package is installed
if (interrupt) return;
boolean packageIsInstall = adbUtil.findPackageInstall(packageName);
if (packageIsInstall){
if (!reInstall){
// install apk
if (interrupt) return;
AdbUtil.CommandResult install = adbUtil.install(apkFile.getAbsolutePath(), true, true);
if (0 != install.exitCode()) {
T.FileUtil.appendString(String.format("ERROR: Install apk failed: exit code %s \n", install.exitCode()), logFile, "UTF-8");
throw new APIException(install.output());
}
}
}else {
// install apk
if (interrupt) return;
AdbUtil.CommandResult install = adbUtil.install(apkFile.getAbsolutePath(), true, true);
if (0 != install.exitCode()) {
T.FileUtil.appendString(String.format("ERROR: Install apk failed: exit code %s \n", install.exitCode()), logFile, "UTF-8");
throw new APIException(install.output());
}
private static List<File> findMatchingFiles(Path directory, Pattern pattern) throws IOException {
List<File> artifactFiles = new ArrayList<>();
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (pattern.matcher(file.getFileName().toString()).matches()) {
artifactFiles.add(file.toFile());
}
//Close other apps
if (interrupt) return;
List<String> packageNameList = adbUtil.findPackageNameList();
this.closeApp(packageNameList, packageName);
// clear app data
if (interrupt) return;
if (clearCache){
AdbUtil.CommandResult clearData = adbUtil.clearAppData(packageName);
if (0 != clearData.exitCode()) {
T.FileUtil.appendString(String.format("ERROR: Clear %s data error: exit code %s \n", packageName, clearData.exitCode()), logFile, "UTF-8");
throw new APIException(clearData.output());
}
}
// Launch the app
if (interrupt) return;
adbUtil.startApp(packageName);
// star tcpdump: package name
if (interrupt) return;
tcpdumpPackage = adbUtil.startTcpdump(packageName);
if (0 != tcpdumpPackage.exitCode()) {
T.FileUtil.appendString(String.format("ERROR: Start tcpdump %s failed: exit code %s \n", packageName, tcpdumpPackage.exitCode()), logFile, "UTF-8");
throw new APIException(String.format("tcpdump %s error", packageName));
}
// star tcpdump: all
if (interrupt) return;
tcpdumpAll = adbUtil.startTcpdump(T.StrUtil.EMPTY);
if (0 != tcpdumpAll.exitCode()) {
T.FileUtil.appendString(String.format("ERROR: Start tcpdump all failed: exit code %s \n", tcpdumpAll.exitCode()), logFile, "UTF-8");
throw new APIException("tcpdump all error");
}
// exec playbook
if (interrupt) return;
AdbUtil.CommandResult airtestResult = adbUtil.execPlaybook(playbookDir.getPath(), logFile);
if (0 != airtestResult.exitCode()) {
T.FileUtil.appendString(String.format("ERROR: Exec playbook failed: exit code %s \n", airtestResult.exitCode()), logFile, "UTF-8");
throw new APIException("playbook exec error");
}
// stop package tcpdump
if (interrupt) return;
stopTcpdump(tcpdumpPackage, logFile, packageName);
// stop all tcpdump
if (interrupt) return;
stopTcpdump(tcpdumpAll, logFile, T.StrUtil.EMPTY);
resultMap = T.MapUtil.builder()
.put("status", "done")
.build();
T.FileUtil.writeString(T.JSONUtil.toJsonStr(resultMap), statusFile, "UTF-8");
} catch (Exception e) {
log.error(e);
Map resultMap = T.MapUtil.builder()
.put("status", "error")
.build();
T.FileUtil.writeString(T.JSONUtil.toJsonStr(resultMap), statusFile, "UTF-8");
} finally {
if (T.StrUtil.isNotEmpty(tcpdumpPackage.output())){
AdbUtil.CommandResult packageTcpdump = adbUtil.stopTcpdump(tcpdumpPackage.output());
adbUtil.execShellCommand(String.format("shell rm -rf %s", packageTcpdump.output()));
}
if (T.StrUtil.isNotEmpty(tcpdumpAll.output())){
AdbUtil.CommandResult allTcpdump = adbUtil.stopTcpdump(tcpdumpAll.output());
adbUtil.execShellCommand(String.format("shell rm -rf %s", allTcpdump.output()));
}
this.closeApp(ListUtil.empty(), packageName);
if (unInstall) {
adbUtil.uninstall(packageName);
}
T.FileUtil.appendString(String.format("Job execution ends"), logFile, "UTF-8");
ACTIVE_TASKS.remove(this);
}
}
private void closeApp(List<String> packageNameList, String packageName) {
if (CollUtil.isNotEmpty(packageNameList)){
for (String name : packageNameList) {
adbUtil.stopApp(name);
}
}
adbUtil.stopApp(packageName);
}
@Override
public void interrupt() {
super.interrupt();
this.interrupt = true;
adbUtil.setInterrupt(true);
}
private void stopTcpdump(AdbUtil.CommandResult tcpdump, File logFile, String packageName) {
// stop tcpdump
AdbUtil.CommandResult stopTcpdump = adbUtil.stopTcpdump(tcpdump.output());
if (0 != stopTcpdump.exitCode()) {
T.FileUtil.appendString(String.format("ERROR: Stop tcpdump failed: exit code %s \n", stopTcpdump.exitCode()), logFile, "UTF-8");
throw new APIException(stopTcpdump.output());
return FileVisitResult.CONTINUE;
}
// pull pcap file
String filePath = stopTcpdump.output();
packageName = T.StrUtil.isEmpty(packageName) ? "all" : packageName;
File localPcapFile = T.FileUtil.file(Constant.TEMP_PATH, tid, String.format("%s-%s%s", tcpdump.output(), packageName, ".pcap"));
if (T.StrUtil.isEmpty(filePath)) {
throw new APIException(RCode.NOT_EXISTS);
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
if (pattern.matcher(dir.getFileName().toString()).matches()) {
artifactFiles.add(dir.toFile());
}
return FileVisitResult.CONTINUE;
}
AdbUtil.CommandResult pull = adbUtil.pull(filePath, localPcapFile.getAbsolutePath());
if (0 != pull.exitCode()) {
T.FileUtil.appendString(String.format("ERROR: Pull pcap file failed: exit code %s \n", pull.exitCode()), logFile, "UTF-8");
throw new APIException(pull.output());
}
// delete android pcap
adbUtil.execShellCommand(String.format("shell rm -rf %s", filePath));
}
});
return artifactFiles;
}
}

10
src/main/java/net/geedge/api/util/AdbCommandBuilder.java
View File

@@ -229,11 +229,17 @@ public class AdbCommandBuilder {
return this;
}
public AdbCommandBuilder buildRunPlaybook(String path, String serial) {
this.command.add("run");
public AdbCommandBuilder buildRunPlaybook(String launcher, String path, String jobId, String packageName, String serial) {
this.command.add(launcher);
this.command.add(path);
this.command.add("--device");
this.command.add(T.StrUtil.concat(true,"Android://127.0.0.1:5037/", serial));
this.command.add("--job_id");
this.command.add(jobId);
this.command.add("--job_path");
this.command.add(path);
this.command.add("--package_name");
this.command.add(packageName);
return this;
}

7
src/main/java/net/geedge/api/util/AdbUtil.java
View File

@@ -26,6 +26,7 @@ public class AdbUtil {
private static String DEFAULT_DROIDVNC_NG_PKG_NAME = "net.christianbeier.droidvnc_ng";
private static String DEFAULT_DROIDVNC_NG_APK_PATH = "./lib/droidvnc-np-2.6.0.apk";
private static String DEFAULT_DROIDVNC_NG_DEFAULTS_JSON_PATH = "./lib/droidvnc-np-defaults.json";
private static String ANDROID_LAUNCHER = "./lib/android-launcher.py";
private String serial;
private String host;
@@ -900,10 +901,10 @@ public class AdbUtil {
}
public CommandResult execPlaybook(String playbookPath, File logFile) {
public CommandResult execPlaybook(String playbookPath, String tid, String packageName, File logFile) {
log.info("[execPlaybook] [begin!] [serial:{}]", this.getSerial());
List<String> command = new AdbCommandBuilder("airtest")
.buildRunPlaybook(playbookPath, this.getSerial())
List<String> command = new AdbCommandBuilder("python")
.buildRunPlaybook(ANDROID_LAUNCHER, playbookPath, tid, packageName, this.getSerial())
.build();
Process process = commandExec.execForProcess(command);

7
src/main/java/net/geedge/common/Constant.java
View File

@@ -1,6 +1,11 @@
package net.geedge.common;
import net.geedge.api.util.PlaybookRunnable;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class Constant {
@@ -13,6 +18,8 @@ public class Constant {
public static final Map<String, Map> PLAYBOOK_RUN_RESULT = T.MapUtil.newConcurrentHashMap();
public static final List<PlaybookRunnable> ACTIVE_TASKS = Collections.synchronizedList(new ArrayList<>());
static {
File tempPath = T.FileUtil.file(TEMP_PATH);
// 程序启动清空临时目录